01HVWFBBCZMSBR8D467KEFJTP8: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HVWFBQ0B622KR3JYH3F7TCW6

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-04-20T01:01:23.187Zjob dependencies complete; ready to run (waiting for 17 m 17 s)
22024-04-20T01:03:34.827Zjob assigned to worker 01HVWGBKT3EF1RFQT1PYDDZP22 (queued for 2 m 11 s)
32024-04-20T01:03:42.224Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-04-20T01:03:54.997Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-04-20T01:03:54.998Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-04-20T01:04:09.919Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-04-20T01:04:09.919Zdownloading input: /input/build/work/bins/crutest.gz
82024-04-20T01:04:27.208Zdownloaded input: /input/build/work/bins/crutest.gz
92024-04-20T01:04:27.208Zdownloading input: /input/build/work/bins/dsc.gz
102024-04-20T01:04:32.651Zdownloaded input: /input/build/work/bins/dsc.gz
112024-04-20T01:04:32.656Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-04-20T01:04:32.673Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-04-20T01:04:32.673Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-04-20T01:04:32.688Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-04-20T01:04:32.699Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-04-20T01:04:32.704Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-04-20T01:04:32.709Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-04-20T01:04:32.723Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-04-20T01:04:32.723Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-04-20T01:04:32.742Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-04-20T01:04:32.743Zdownloading input: /input/build/work/scripts/test_up.sh
222024-04-20T01:04:32.754Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-04-20T01:04:32.754Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-04-20T01:04:32.778Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-04-20T01:04:32.786Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-04-20T01:04:44.510Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-04-20T01:04:44.511Zstarting task 0: "setup"
282024-04-20T01:04:44.553Z++ uname -s
292024-04-20T01:04:44.557Z+ kern=SunOS
302024-04-20T01:04:44.557Z+ case "$kern" in
312024-04-20T01:04:44.557Z+ groupadd -g 12345 build
322024-04-20T01:04:44.609Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-04-20T01:04:47.033Z+ zfs create -o mountpoint=/work rpool/work
342024-04-20T01:04:47.046Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-04-20T01:04:47.046Z+ home_fs=zfs
362024-04-20T01:04:47.046Z+ [[ zfs == autofs ]]
372024-04-20T01:04:47.046Z+ mkdir -p /home/build
382024-04-20T01:04:47.046Z+ chown build:build /home/build /work
392024-04-20T01:04:48.731Z+ chmod 0700 /home/build /work
402024-04-20T01:04:48.735Zprocess exited: duration 4221 ms, exit code 0
 
412024-04-20T01:04:48.742Zstarting task 1: "authentication"
422024-04-20T01:04:48.759Zprocess exited: duration 17 ms, exit code 0
 
432024-04-20T01:04:48.767Zstarting task 2: "build"
442024-04-20T01:04:48.774Z+ banner cores
452024-04-20T01:04:48.777Z
462024-04-20T01:04:48.777Z #### #### ##### ###### ####
472024-04-20T01:04:48.778Z # # # # # # # #
482024-04-20T01:04:48.778Z # # # # # ##### ####
492024-04-20T01:04:48.778Z # # # ##### # #
502024-04-20T01:04:48.778Z # # # # # # # # #
512024-04-20T01:04:48.778Z #### #### # # ###### ####
522024-04-20T01:04:48.778Z
532024-04-20T01:04:48.778Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-04-20T01:04:48.785Z+ echo 'input bins dir contains:'
552024-04-20T01:04:48.788Zinput bins dir contains:
562024-04-20T01:04:48.788Z+ ls -ltr /input/build/work/bins
572024-04-20T01:04:48.788Ztotal 658470
582024-04-20T01:04:48.788Z-rw-r--r-- 1 root root 98276289 Apr 20 01:03 crucible-downstairs.gz
592024-04-20T01:04:48.788Z-rw-r--r-- 1 root root 79513064 Apr 20 01:04 crucible-hammer.gz
602024-04-20T01:04:48.788Z-rw-r--r-- 1 root root 95398132 Apr 20 01:04 crutest.gz
612024-04-20T01:04:48.788Z-rw-r--r-- 1 root root 63730486 Apr 20 01:04 dsc.gz
622024-04-20T01:04:48.788Z+ echo 'input script dir contains:'
632024-04-20T01:04:48.788Zinput script dir contains:
642024-04-20T01:04:48.788Z+ ls -ltr /input/build/work/scripts
652024-04-20T01:04:48.792Ztotal 40
662024-04-20T01:04:48.792Z-rw-r--r-- 1 root root 3056 Apr 20 01:04 perf-downstairs-tick.d
672024-04-20T01:04:48.792Z-rw-r--r-- 1 root root 2102 Apr 20 01:04 test_ds.sh
682024-04-20T01:04:48.793Z-rw-r--r-- 1 root root 3746 Apr 20 01:04 test_live_repair.sh
692024-04-20T01:04:48.793Z-rw-r--r-- 1 root root 6416 Apr 20 01:04 test_repair.sh
702024-04-20T01:04:48.793Z-rw-r--r-- 1 root root 2969 Apr 20 01:04 test_replay.sh
712024-04-20T01:04:48.793Z-rw-r--r-- 1 root root 9465 Apr 20 01:04 test_up.sh
722024-04-20T01:04:48.793Z-rw-r--r-- 1 root root 2942 Apr 20 01:04 upstairs_info.d
732024-04-20T01:04:48.793Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-04-20T01:04:48.793Z+ echo ' chmod input script dir contains:'
752024-04-20T01:04:48.793Z chmod input script dir contains:
762024-04-20T01:04:48.793Z+ ls -ltr /input/build/work/scripts
772024-04-20T01:04:48.797Ztotal 40
782024-04-20T01:04:48.798Z-rwxr-xr-x 1 root root 3056 Apr 20 01:04 perf-downstairs-tick.d
792024-04-20T01:04:48.798Z-rwxr-xr-x 1 root root 2102 Apr 20 01:04 test_ds.sh
802024-04-20T01:04:48.798Z-rwxr-xr-x 1 root root 3746 Apr 20 01:04 test_live_repair.sh
812024-04-20T01:04:48.798Z-rwxr-xr-x 1 root root 6416 Apr 20 01:04 test_repair.sh
822024-04-20T01:04:48.798Z-rwxr-xr-x 1 root root 2969 Apr 20 01:04 test_replay.sh
832024-04-20T01:04:48.798Z-rwxr-xr-x 1 root root 9465 Apr 20 01:04 test_up.sh
842024-04-20T01:04:48.798Z-rwxr-xr-x 1 root root 2942 Apr 20 01:04 upstairs_info.d
852024-04-20T01:04:48.798Z+ banner unpack
862024-04-20T01:04:48.798Z
872024-04-20T01:04:48.798Z # # # # ##### ## #### # #
882024-04-20T01:04:48.798Z # # ## # # # # # # # # #
892024-04-20T01:04:48.798Z # # # # # # # # # # ####
902024-04-20T01:04:48.798Z # # # # # ##### ###### # # #
912024-04-20T01:04:48.798Z # # # ## # # # # # # #
922024-04-20T01:04:48.798Z #### # # # # # #### # #
932024-04-20T01:04:48.798Z
942024-04-20T01:04:48.798Z+ mkdir -p /var/tmp/bins
952024-04-20T01:04:48.802Z+ for t in "$input/bins/"*.gz
962024-04-20T01:04:48.802Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-04-20T01:04:48.802Z+ b=crucible-downstairs.gz
982024-04-20T01:04:48.802Z+ b=crucible-downstairs
992024-04-20T01:04:48.802Z+ gunzip
1002024-04-20T01:04:50.857Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-04-20T01:04:50.861Z+ for t in "$input/bins/"*.gz
1022024-04-20T01:04:50.861Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-04-20T01:04:50.861Z+ b=crucible-hammer.gz
1042024-04-20T01:04:50.861Z+ b=crucible-hammer
1052024-04-20T01:04:50.861Z+ gunzip
1062024-04-20T01:04:52.536Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-04-20T01:04:52.540Z+ for t in "$input/bins/"*.gz
1082024-04-20T01:04:52.540Z++ basename /input/build/work/bins/crutest.gz
1092024-04-20T01:04:52.540Z+ b=crutest.gz
1102024-04-20T01:04:52.540Z+ b=crutest
1112024-04-20T01:04:52.540Z+ gunzip
1122024-04-20T01:04:54.522Z+ chmod +x /var/tmp/bins/crutest
1132024-04-20T01:04:54.526Z+ for t in "$input/bins/"*.gz
1142024-04-20T01:04:54.526Z++ basename /input/build/work/bins/dsc.gz
1152024-04-20T01:04:54.529Z+ b=dsc.gz
1162024-04-20T01:04:54.529Z+ b=dsc
1172024-04-20T01:04:54.529Z+ gunzip
1182024-04-20T01:04:55.932Z+ chmod +x /var/tmp/bins/dsc
1192024-04-20T01:04:55.936Z+ export BINDIR=/var/tmp/bins
1202024-04-20T01:04:55.936Z+ BINDIR=/var/tmp/bins
1212024-04-20T01:04:55.936Z+ banner repair
1222024-04-20T01:04:55.936Z
1232024-04-20T01:04:55.936Z ##### ###### ##### ## # #####
1242024-04-20T01:04:55.936Z # # # # # # # # # #
1252024-04-20T01:04:55.936Z # # ##### # # # # # # #
1262024-04-20T01:04:55.936Z ##### # ##### ###### # #####
1272024-04-20T01:04:55.937Z # # # # # # # # #
1282024-04-20T01:04:55.937Z # # ###### # # # # # #
1292024-04-20T01:04:55.937Z
1302024-04-20T01:04:55.940Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-04-20T01:04:56.012ZTurn off color for downstairs dump
1322024-04-20T01:04:56.026ZCreating region directory at: /var/tmp/test_repair
1332024-04-20T01:04:56.032ZCreating dsc directory at: /tmp/dsc
1342024-04-20T01:04:56.134ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.1038798
1352024-04-20T01:04:56.171ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.03906255
1362024-04-20T01:04:56.212ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.038943227
1372024-04-20T01:04:56.212ZRegion set with 3 regions was created
1382024-04-20T01:04:56.221Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1170 1171 1172
1392024-04-20T01:04:56.221ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-04-20T01:04:56.250Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.761506107Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174}
1412024-04-20T01:04:56.250Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.762092958Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174}
1422024-04-20T01:04:56.250Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.762111798Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174}
1432024-04-20T01:04:56.250Z{"msg":"Crucible 6c515a32-f908-4c63-b810-ec0901c5ec47 has session id: 87b53a5b-4257-458e-92e5-5ce33bc58e27","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.762125658Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1442024-04-20T01:04:56.254Z{"msg":"Upstairs opts: Upstairs UUID: 6c515a32-f908-4c63-b810-ec0901c5ec47, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.762148488Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1452024-04-20T01:04:56.255Z{"msg":"Crucible stats registered with UUID: 6c515a32-f908-4c63-b810-ec0901c5ec47","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.762163208Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1462024-04-20T01:04:56.255ZCrucible runtime is spawned
1472024-04-20T01:04:56.255Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76314207Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174}
1482024-04-20T01:04:56.255Z{"msg":"6c515a32-f908-4c63-b810-ec0901c5ec47 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76318092Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1492024-04-20T01:04:56.255Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.763211771Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1502024-04-20T01:04:56.255Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.763246561Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1512024-04-20T01:04:56.255Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.763270671Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"1","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1522024-04-20T01:04:56.255Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.763306671Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"1","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1532024-04-20T01:04:56.255Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.763333471Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"2","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1542024-04-20T01:04:56.255Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.763365531Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"2","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1552024-04-20T01:04:56.255Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.763399481Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"io task","client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1562024-04-20T01:04:56.255Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.763435881Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"io task","client":"1","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1572024-04-20T01:04:56.255Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.763459991Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"io task","client":"2","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1582024-04-20T01:04:56.255Z{"msg":"ds_connection connected from Ok(127.0.0.1:64538)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.763682061Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"io task","client":"1","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1592024-04-20T01:04:56.255Z{"msg":"ds_connection connected from Ok(127.0.0.1:53615)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.763729622Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"io task","client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1602024-04-20T01:04:56.255Z{"msg":"ds_connection connected from Ok(127.0.0.1:39614)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.763754492Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"io task","client":"2","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1612024-04-20T01:04:56.255Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.765723116Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1622024-04-20T01:04:56.255Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.765763306Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1632024-04-20T01:04:56.255Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.765801556Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"1","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1642024-04-20T01:04:56.256Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.765846116Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"1","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1652024-04-20T01:04:56.256Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.765876217Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"2","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1662024-04-20T01:04:56.256Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.765918137Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"2","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1672024-04-20T01:04:56.260Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.766768499Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"2","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1682024-04-20T01:04:56.260Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.766808539Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1692024-04-20T01:04:56.260Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.766852829Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"1","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1702024-04-20T01:04:56.260Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76742675Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"2","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1712024-04-20T01:04:56.260Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76747015Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"2","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1722024-04-20T01:04:56.260Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76748682Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1732024-04-20T01:04:56.260Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76749972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"1","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1742024-04-20T01:04:56.261Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76751378Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"1","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1752024-04-20T01:04:56.261Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76753436Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1762024-04-20T01:04:56.261Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76754662Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1772024-04-20T01:04:56.261Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76756649Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1782024-04-20T01:04:56.261Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76758039Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1792024-04-20T01:04:56.261Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76760134Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1802024-04-20T01:04:56.261Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76767611Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1812024-04-20T01:04:56.261Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.76771625Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1822024-04-20T01:04:56.261Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767731371Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1832024-04-20T01:04:56.261Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767746311Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1842024-04-20T01:04:56.261Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767759851Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1852024-04-20T01:04:56.261Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767780151Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1862024-04-20T01:04:56.261Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767794501Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1872024-04-20T01:04:56.261Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767814061Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1882024-04-20T01:04:56.261Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767829841Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1892024-04-20T01:04:56.261Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767864071Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1902024-04-20T01:04:56.261Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767882481Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"downstairs","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1912024-04-20T01:04:56.261Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767902661Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1922024-04-20T01:04:56.261Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767914901Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1932024-04-20T01:04:56.261Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767934931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"1","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1942024-04-20T01:04:56.261ZWait for a query_work_queue command to finish before sending IO
1952024-04-20T01:04:56.261Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767950931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"client":"2","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1962024-04-20T01:04:56.261Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767972051Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1972024-04-20T01:04:56.262Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767984841Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1982024-04-20T01:04:56.262Z{"msg":"6c515a32-f908-4c63-b810-ec0901c5ec47 is now active with session: 87b53a5b-4257-458e-92e5-5ce33bc58e27","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.767998041Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
1992024-04-20T01:04:56.262Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.768010311Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174}
2002024-04-20T01:04:56.262Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:54.768021411Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
2012024-04-20T01:04:56.262ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-04-20T01:04:56.262ZFill test
2032024-04-20T01:04:56.317ZRead and Verify all blocks (0..600 range:false)
2042024-04-20T01:04:56.339ZWrote out file "/tmp/test_repair_verify.data"
2052024-04-20T01:04:56.339ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-04-20T01:04:56.343Z----------------------------------------------------------------
2072024-04-20T01:04:56.343Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3
2082024-04-20T01:04:56.343ZStates: Active Active Active
2092024-04-20T01:04:56.343ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-04-20T01:04:56.343Z 5 Acked 1004 Read 88 Done Done Done false
2112024-04-20T01:04:56.343Z 6 Acked 1005 Read 256 Done Done Done false
2122024-04-20T01:04:56.344Z 7 Acked 1006 Read 256 Done Done Done false
2132024-04-20T01:04:56.344Z STATES DS:0 DS:1 DS:2 TOTAL
2142024-04-20T01:04:56.344Z New 0 0 0 0
2152024-04-20T01:04:56.344Z Sent 0 0 0 0
2162024-04-20T01:04:56.344Z Done 3 3 3 9
2172024-04-20T01:04:56.344Z Skipped 0 0 0 0
2182024-04-20T01:04:56.344Z Error 0 0 0 0
2192024-04-20T01:04:56.344ZLast Flush: 1003 1003 1003
2202024-04-20T01:04:56.344ZDownstairs last five completed: 1003 1002 1001 1000
2212024-04-20T01:04:56.344ZUpstairs last five completed: 7 6 5 4 3
2222024-04-20T01:04:56.344ZCLIENT: Up:0 ds:3 act:3
2232024-04-20T01:05:00.340Z----------------------------------------------------------------
2242024-04-20T01:05:00.341Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2252024-04-20T01:05:00.341ZDownstairs last five completed: 1007 1006 1005 1004 1003
2262024-04-20T01:05:00.341ZUpstairs last five completed: 8 7 6 5 4
2272024-04-20T01:05:00.341ZCLIENT: Up:0 ds:0 act:3
2282024-04-20T01:05:00.341ZCLIENT: All crucible jobs finished, exiting program
2292024-04-20T01:05:00.344Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:04:58.852372513Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174}
2302024-04-20T01:05:00.345Z{"msg":"client_stop_rx closed unexpectedly: RecvError(())","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:04:58.852787324Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"io task","client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
2312024-04-20T01:05:00.345Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:04:58.852832334Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"io task","client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
2322024-04-20T01:05:00.345Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:04:58.852858984Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"io task","client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
2332024-04-20T01:05:00.345Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.852896424Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1174,"":"io task","client":"0","":"downstairs","session_id":"87b53a5b-4257-458e-92e5-5ce33bc58e27"}
2342024-04-20T01:05:00.348Z
2352024-04-20T01:05:00.348ZBegin loop 1
2362024-04-20T01:05:00.348ZDownstairs to restart: 1
2372024-04-20T01:05:00.364Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876236476Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176}
2382024-04-20T01:05:00.364Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876462337Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176}
2392024-04-20T01:05:00.364Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876479197Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176}
2402024-04-20T01:05:00.364Z{"msg":"Crucible 8190e53b-f300-46e5-92d8-23c510aa502b has session id: a836816d-1f38-4839-9841-dbf2ad2eb4bf","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876501247Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2412024-04-20T01:05:00.371Z{"msg":"Upstairs opts: Upstairs UUID: 8190e53b-f300-46e5-92d8-23c510aa502b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876532417Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2422024-04-20T01:05:00.371Z{"msg":"Crucible stats registered with UUID: 8190e53b-f300-46e5-92d8-23c510aa502b","v":0,"name":"crucible","level":30Crucible runtime is spawned
2432024-04-20T01:05:00.371Z,"time":"2024-04-20T01:04:58.876566687Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2442024-04-20T01:05:00.371Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876654247Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176}
2452024-04-20T01:05:00.371Z{"msg":"8190e53b-f300-46e5-92d8-23c510aa502b active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876683817Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2462024-04-20T01:05:00.371Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876697967Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"0","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2472024-04-20T01:05:00.371Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876717647Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"0","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2482024-04-20T01:05:00.371Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876757777Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"1","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2492024-04-20T01:05:00.371Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876783307Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"1","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2502024-04-20T01:05:00.371Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876808297Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"2","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2512024-04-20T01:05:00.371Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876849708Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"2","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2522024-04-20T01:05:00.371Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876873597Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"io task","client":"0","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2532024-04-20T01:05:00.371Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876907328Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"io task","client":"1","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2542024-04-20T01:05:00.371Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876931588Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"io task","client":"2","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2552024-04-20T01:05:00.371Z{"msg":"ds_connection connected from Ok(127.0.0.1:37487)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.876982648Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"io task","client":"1","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2562024-04-20T01:05:00.371Z{"msg":"ds_connection connected from Ok(127.0.0.1:55989)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.877030738Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"io task","client":"0","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2572024-04-20T01:05:00.371Z{"msg":"ds_connection connected from Ok(127.0.0.1:46500)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.877076318Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"io task","client":"2","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2582024-04-20T01:05:00.371Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.877678179Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"2","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2592024-04-20T01:05:00.372Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.877713799Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"2","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2602024-04-20T01:05:00.372Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.87773001Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"0","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2612024-04-20T01:05:00.372Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.877748699Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"0","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2622024-04-20T01:05:00.372Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.878201431Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"0","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2632024-04-20T01:05:00.372Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.878288301Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"2","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2642024-04-20T01:05:00.372Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.878481321Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"1","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2652024-04-20T01:05:00.372Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.878544681Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"1","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2662024-04-20T01:05:00.372Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.878645601Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"0","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2672024-04-20T01:05:00.372Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.878685742Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"0","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2682024-04-20T01:05:00.372Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.878707132Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2692024-04-20T01:05:00.372Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.878731182Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"2","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2702024-04-20T01:05:00.372Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.878760732Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"2","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2712024-04-20T01:05:00.372Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.878774922Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2722024-04-20T01:05:00.373Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879019702Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"1","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2732024-04-20T01:05:00.373Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879406053Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"1","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2742024-04-20T01:05:00.373Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879439943Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"1","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2752024-04-20T01:05:00.373Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879455393Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2762024-04-20T01:05:00.373Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879470373Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2772024-04-20T01:05:00.373Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879492854Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2782024-04-20T01:05:00.373Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879507934Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2792024-04-20T01:05:00.373Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879529754Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2802024-04-20T01:05:00.373Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879545163Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2812024-04-20T01:05:00.373Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2822024-04-20T01:05:00.373Z,"time":"2024-04-20T01:04:58.879559834Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2832024-04-20T01:05:00.373Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879575754Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2842024-04-20T01:05:00.373Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879590404Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2852024-04-20T01:05:00.373Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879603894Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2862024-04-20T01:05:00.374Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879617034Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2872024-04-20T01:05:00.374Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879640694Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2882024-04-20T01:05:00.374Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879657374Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"":"downstairs","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2892024-04-20T01:05:00.374Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879670504Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2902024-04-20T01:05:00.374Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879683794Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"0","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2912024-04-20T01:05:00.374Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879697254Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"1","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2922024-04-20T01:05:00.374Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879710794Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"client":"2","":"downstairs","session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2932024-04-20T01:05:00.374Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879732644Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2942024-04-20T01:05:00.374Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879745284Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2952024-04-20T01:05:00.374Z{"msg":"8190e53b-f300-46e5-92d8-23c510aa502b is now active with session: a836816d-1f38-4839-9841-dbf2ad2eb4bf","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879758744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2962024-04-20T01:05:00.375Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879778054Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176,"session_id":"a836816d-1f38-4839-9841-dbf2ad2eb4bf"}
2972024-04-20T01:05:00.375Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:04:58.879790724Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176}
2982024-04-20T01:05:00.375ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2992024-04-20T01:05:00.389ZLoading write count information from file "/tmp/test_repair_verify.data"
3002024-04-20T01:05:00.389ZRun Repair workload
3012024-04-20T01:05:00.389Z01/30 Write block 416 len 4096 data: 2
3022024-04-20T01:05:00.389Z02/30 Write block 366 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3032024-04-20T01:05:00.390Z03/30 Read block 56 len 24576
3042024-04-20T01:05:00.390Z04/30 Write block 443 len 32768 data: 2 2 2 2 2 2 2 2
3052024-04-20T01:05:00.390Z05/30 Read block 274 len 28672
3062024-04-20T01:05:00.390Z06/30 Read block 360 len 36864
3072024-04-20T01:05:00.390Z07/30 Write block 512 len 36864 data: 2 2 2 2 2 2 2 2 2
3082024-04-20T01:05:00.390Z08/30 Write block 361 len 12288 data: 2 2 2
3092024-04-20T01:05:00.390Z09/30 Write block 441 len 12288 data: 2 2 3
3102024-04-20T01:05:00.390Z10/30 Flush
3112024-04-20T01:05:00.390Z11/30 Write block 436 len 8192 data: 2 2
3122024-04-20T01:05:00.390Z12/30 Read block 73 len 36864
3132024-04-20T01:05:00.390Z13/30 Write block 569 len 8192 data: 2 2
3142024-04-20T01:05:00.390Z14/30 Write block 501 len 36864 data: 2 2 2 2 2 2 2 2 2
3152024-04-20T01:05:00.802Z15/30 Read block 312 len 40960
3162024-04-20T01:05:00.803Z16/30 Write block 160 len 16384 data: 2 2 2 2
3172024-04-20T01:05:00.803Z17/30 Write block 370 len 20480 data: 3 3 3 3 3
3182024-04-20T01:05:00.803Z18/30 Read block 177 len 40960
3192024-04-20T01:05:00.803Z19/30 Read block 223 len 4096
3202024-04-20T01:05:00.803Z20/30 Read block 8 len 40960
3212024-04-20T01:05:00.803Z21/30 Read block 574 len 12288
3222024-04-20T01:05:00.803Z22/30 Write block 566 len 24576 data: 2 2 2 3 3 2
3232024-04-20T01:05:00.803Z23/30 Read block 21 len 28672
3242024-04-20T01:05:00.803Z24/30 Write block 50 len 28672 data: 2 2 2 2 2 2 2
3252024-04-20T01:05:00.803Z25/30 Read block 295 len 24576
3262024-04-20T01:05:00.803Z26/30 Flush
3272024-04-20T01:05:00.803Z27/30 Write block 18 len 24576 data: 2 2 2 2 2 2
3282024-04-20T01:05:00.808Z28/30 Read block 551 len 12288
3292024-04-20T01:05:00.808Z29/30 Read block 496 len 20480
3302024-04-20T01:05:00.808Z30/30 Read block 81 len 4096
3312024-04-20T01:05:00.808Z----------------------------------------------------------------
3322024-04-20T01:05:00.808Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3332024-04-20T01:05:00.808ZStates: Active Active Active
3342024-04-20T01:05:00.809ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3352024-04-20T01:05:00.809Z 1 Acked 1000 Write 1 Done Done Done false
3362024-04-20T01:05:00.809Z 2 Acked 1001 Write 10 Done Done Done false
3372024-04-20T01:05:00.809Z 3 Acked 1002 Read 6 Done Done Done false
3382024-04-20T01:05:00.809Z 4 Acked 1003 Write 8 Done Done Done false
3392024-04-20T01:05:00.809Z 5 Acked 1004 Read 7 Done Sent Done false
3402024-04-20T01:05:00.809Z 6 Acked 1005 Read 9 Done Sent Done false
3412024-04-20T01:05:00.809Z 7 Acked 1006 Write 9 Done Sent Done false
3422024-04-20T01:05:00.809Z 8 Acked 1007 Write 3 Done Sent Done false
3432024-04-20T01:05:00.809Z 9 Acked 1008 Write 3 Done Sent Done false
3442024-04-20T01:05:00.809Z 10 Acked 1009 Flush 0 Done Sent Done false
3452024-04-20T01:05:00.809Z 11 Acked 1010 Write 2 Done Sent Done false
3462024-04-20T01:05:00.809Z 12 Acked 1011 Read 9 Done Sent Done false
3472024-04-20T01:05:00.809Z 13 Acked 1012 Write 2 Done Sent Done false
3482024-04-20T01:05:00.809Z 14 Acked 1013 Write 9 Done Sent Done false
3492024-04-20T01:05:00.809Z 15 Acked 1014 Read 10 Done Sent Done false
3502024-04-20T01:05:00.809Z 16 Acked 1015 Write 4 Done Sent Done false
3512024-04-20T01:05:00.809Z 17 Acked 1016 Write 5 Done Sent Done false
3522024-04-20T01:05:00.809Z 18 Acked 1017 Read 10 Done Sent Done false
3532024-04-20T01:05:00.809Z 19 Acked 1018 Read 1 Done Sent Done false
3542024-04-20T01:05:00.809Z 20 Acked 1019 Read 10 Done Sent Done false
3552024-04-20T01:05:00.809Z 21 Acked 1020 Read 3 Done Sent Done false
3562024-04-20T01:05:00.809Z 22 Acked 1021 Write 6 Done Sent Done false
3572024-04-20T01:05:00.809Z 23 Acked 1022 Read 7 Done Sent Done false
3582024-04-20T01:05:00.810Z 24 Acked 1023 Write 7 Done Sent Done false
3592024-04-20T01:05:00.810Z 25 Acked 1024 Read 6 Done Sent Done false
3602024-04-20T01:05:00.810Z 26 Acked 1025 Flush 0 Done Sent Done false
3612024-04-20T01:05:00.810Z 27 Acked 1026 Write 6 Done Sent Done false
3622024-04-20T01:05:00.810Z 28 Acked 1027 Read 3 Done Sent Done false
3632024-04-20T01:05:00.810Z 29 Acked 1028 Read 5 Done Sent Done false
3642024-04-20T01:05:00.810Z 30 Acked 1029 Read 1 Done Sent Sent false
3652024-04-20T01:05:00.810Z STATES DS:0 DS:1 DS:2 TOTAL
3662024-04-20T01:05:00.810Z New 0 0 0 0
3672024-04-20T01:05:00.810Z Sent 0 26 1 27
3682024-04-20T01:05:00.810Z Done 30 4 29 63
3692024-04-20T01:05:00.810Z Skipped 0 0 0 0
3702024-04-20T01:05:00.810Z Error 0 0 0 0
3712024-04-20T01:05:00.810ZLast Flush: 1025 0 1025
3722024-04-20T01:05:00.810ZDownstairs last five completed:
3732024-04-20T01:05:00.810ZUpstairs last five completed: 30 29 28 27 26
3742024-04-20T01:05:00.810Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:04:58.913121768Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1176}
3752024-04-20T01:05:00.810ZWrote out file "/tmp/test_repair_verify.data"
3762024-04-20T01:05:00.810Z
3772024-04-20T01:05:00.810ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3782024-04-20T01:05:00.810ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3792024-04-20T01:05:00.810Z 0 000-019 1 1 1 1 1 1 T F T <---
3802024-04-20T01:05:00.810Z 1 020-039 1 1 1 1 1 1 T F T <---
3812024-04-20T01:05:00.811Z 2 040-059 2 1 2 3 1 3 F F F <---
3822024-04-20T01:05:00.811Z 3 060-079 1 1 1 1 1 1 F F F
3832024-04-20T01:05:00.811Z 4 080-099 1 1 1 1 1 1 F F F
3842024-04-20T01:05:00.811Z 5 100-119 1 1 1 1 1 1 F F F
3852024-04-20T01:05:00.811Z 6 120-139 1 1 1 1 1 1 F F F
3862024-04-20T01:05:00.811Z 7 140-159 1 1 1 1 1 1 F F F
3872024-04-20T01:05:00.811Z 8 160-179 2 1 2 3 1 3 F F F <---
3882024-04-20T01:05:00.811Z 9 180-199 1 1 1 1 1 1 F F F
3892024-04-20T01:05:00.811Z 10 200-219 1 1 1 1 1 1 F F F
3902024-04-20T01:05:00.811Z 11 220-239 1 1 1 1 1 1 F F F
3912024-04-20T01:05:00.811Z 12 240-259 1 1 1 1 1 1 F F F
3922024-04-20T01:05:00.811Z 13 260-279 1 1 1 1 1 1 F F F
3932024-04-20T01:05:00.811Z 14 280-299 1 1 1 1 1 1 F F F
3942024-04-20T01:05:00.811Z 15 300-319 1 1 1 1 1 1 F F F
3952024-04-20T01:05:00.811Z 16 320-339 1 1 1 1 1 1 F F F
3962024-04-20T01:05:00.811Z 17 340-359 1 1 1 1 1 1 F F F
3972024-04-20T01:05:00.811Z 18 360-379 2 1 2 3 1 3 F T F <---
3982024-04-20T01:05:00.811Z 19 380-399 1 1 1 1 1 1 F F F
3992024-04-20T01:05:00.811Z 20 400-419 2 1 2 2 1 2 F T F <---
4002024-04-20T01:05:00.811Z 21 420-439 2 1 2 3 1 3 F F F <---
4012024-04-20T01:05:00.811Z 22 440-459 2 1 2 2 1 2 F T F <---
4022024-04-20T01:05:00.811Z 23 460-479 1 1 1 1 1 1 F F F
4032024-04-20T01:05:00.812Z 24 480-499 1 1 1 1 1 1 F F F
4042024-04-20T01:05:00.812Z 25 500-519 2 1 2 3 1 3 F F F <---
4052024-04-20T01:05:00.812Z 26 520-539 2 1 2 2 1 2 F F F <---
4062024-04-20T01:05:00.812Z 27 540-559 1 1 1 1 1 1 F F F
4072024-04-20T01:05:00.812Z 28 560-579 2 1 2 3 1 3 F F F <---
4082024-04-20T01:05:00.812Z 29 580-599 1 1 1 1 1 1 F F F
4092024-04-20T01:05:00.812ZMax gen: 2, Max flush: 3
4102024-04-20T01:05:00.812ZError: Difference in extent metadata found!
4112024-04-20T01:05:00.812ZA Difference in extent metadata is expected here
4122024-04-20T01:05:00.812ZOn loop 1
4132024-04-20T01:05:00.812Z
4142024-04-20T01:05:00.812ZVerifying data now
4152024-04-20T01:05:05.068ZNo change after verify
4162024-04-20T01:05:05.068ZLoop: 1 Downstairs dump after verify (and repair):
4172024-04-20T01:05:05.083ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4182024-04-20T01:05:05.083Z 0 000-019 3 3 3 4 4 4 F F F
4192024-04-20T01:05:05.084Z 1 020-039 3 3 3 4 4 4 F F F
4202024-04-20T01:05:05.084Z 2 040-059 2 2 2 3 3 3 F F F
4212024-04-20T01:05:05.084Z 3 060-079 1 1 1 1 1 1 F F F
4222024-04-20T01:05:05.084Z 4 080-099 1 1 1 1 1 1 F F F
4232024-04-20T01:05:05.084Z 5 100-119 1 1 1 1 1 1 F F F
4242024-04-20T01:05:05.084Z 6 120-139 1 1 1 1 1 1 F F F
4252024-04-20T01:05:05.084Z 7 140-159 1 1 1 1 1 1 F F F
4262024-04-20T01:05:05.084Z 8 160-179 2 2 2 3 3 3 F F F
4272024-04-20T01:05:05.084Z 9 180-199 1 1 1 1 1 1 F F F
4282024-04-20T01:05:05.095Z 10 200-219 1 1 1 1 1 1 F F F
4292024-04-20T01:05:05.095Z 11 220-239 1 1 1 1 1 1 F F F
4302024-04-20T01:05:05.095Z 12 240-259 1 1 1 1 1 1 F F F
4312024-04-20T01:05:05.095Z 13 260-279 1 1 1 1 1 1 F F F
4322024-04-20T01:05:05.095Z 14 280-299 1 1 1 1 1 1 F F F
4332024-04-20T01:05:05.095Z 15 300-319 1 1 1 1 1 1 F F F
4342024-04-20T01:05:05.095Z 16 320-339 1 1 1 1 1 1 F F F
4352024-04-20T01:05:05.095Z 17 340-359 1 1 1 1 1 1 F F F
4362024-04-20T01:05:05.095Z 18 360-379 2 2 2 3 3 3 F F F
4372024-04-20T01:05:05.095Z 19 380-399 1 1 1 1 1 1 F F F
4382024-04-20T01:05:05.095Z 20 400-419 2 2 2 2 2 2 F F F
4392024-04-20T01:05:05.096Z 21 420-439 2 2 2 3 3 3 F F F
4402024-04-20T01:05:05.096Z 22 440-459 2 2 2 2 2 2 F F F
4412024-04-20T01:05:05.096Z 23 460-479 1 1 1 1 1 1 F F F
4422024-04-20T01:05:05.096Z 24 480-499 1 1 1 1 1 1 F F F
4432024-04-20T01:05:05.096Z 25 500-519 2 2 2 3 3 3 F F F
4442024-04-20T01:05:05.096Z 26 520-539 2 2 2 2 2 2 F F F
4452024-04-20T01:05:05.096Z 27 540-559 1 1 1 1 1 1 F F F
4462024-04-20T01:05:05.096Z 28 560-579 2 2 2 3 3 3 F F F
4472024-04-20T01:05:05.096Z 29 580-599 1 1 1 1 1 1 F F F
4482024-04-20T01:05:05.096ZMax gen: 3, Max flush: 4
4492024-04-20T01:05:05.096Z
4502024-04-20T01:05:05.096ZBegin loop 2
4512024-04-20T01:05:05.096ZDownstairs to restart: 2
4522024-04-20T01:05:05.102Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614034333Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184}
4532024-04-20T01:05:05.102Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614266393Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184}
4542024-04-20T01:05:05.102Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614283033Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184}
4552024-04-20T01:05:05.108Z{"msg":"Crucible 59e54988-7afd-453a-9902-c8f0cbe840a8 has session id: 7225aa61-60c1-4e96-b64a-618b0c392eef","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614302163Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4562024-04-20T01:05:05.108Z{"msg":"Upstairs opts: Upstairs UUID: 59e54988-7afd-453a-9902-c8f0cbe840a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614353673Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4572024-04-20T01:05:05.108Z{"msg":"Crucible stats registered with UUID: 59e54988-7afd-453a-9902-c8f0cbe840a8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614390383Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"session_id":Crucible runtime is spawned
4582024-04-20T01:05:05.109Z"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4592024-04-20T01:05:05.109Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614519443Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184}
4602024-04-20T01:05:05.109Z{"msg":"59e54988-7afd-453a-9902-c8f0cbe840a8 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614551663Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4612024-04-20T01:05:05.109Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614569544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"0","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4622024-04-20T01:05:05.109Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614595464Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"0","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4632024-04-20T01:05:05.109Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614620744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"1","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4642024-04-20T01:05:05.109Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614644784Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"1","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4652024-04-20T01:05:05.109Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614667544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"2","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4662024-04-20T01:05:05.109Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614683234Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"2","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4672024-04-20T01:05:05.109Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614709684Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4682024-04-20T01:05:05.109Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614727914Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"io task","client":"1","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4692024-04-20T01:05:05.109Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614761304Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"io task","client":"0","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4702024-04-20T01:05:05.109Z{"msg":"ds_connection connected from Ok(127.0.0.1:57533)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.614973444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4712024-04-20T01:05:05.109Z{"msg":"ds_connection connected from Ok(127.0.0.1:39699)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.615175615Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"io task","client":"0","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4722024-04-20T01:05:05.109Z{"msg":"ds_connection connected from Ok(127.0.0.1:41664)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.615215395Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"io task","client":"1","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4732024-04-20T01:05:05.109Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.615693356Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"0","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4742024-04-20T01:05:05.109Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.615742476Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"0","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4752024-04-20T01:05:05.109Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.615769986Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"1","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4762024-04-20T01:05:05.109Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.615802606Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"1","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4772024-04-20T01:05:05.109Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.616137397Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"2","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4782024-04-20T01:05:05.109Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.616188507Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"2","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4792024-04-20T01:05:05.109Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.616222047Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"1","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4802024-04-20T01:05:05.109Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.616462358Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"0","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4812024-04-20T01:05:05.110Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.616751459Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"1","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4822024-04-20T01:05:05.110Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.616792608Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"1","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4832024-04-20T01:05:05.110Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.616814829Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4842024-04-20T01:05:05.110Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.616836169Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"2","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4852024-04-20T01:05:05.110Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.616958359Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"0","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4862024-04-20T01:05:05.110Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.616986419Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"0","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4872024-04-20T01:05:05.110Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.617000499Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4882024-04-20T01:05:05.110Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61721112Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"2","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4892024-04-20T01:05:05.110Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.617237449Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"2","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4902024-04-20T01:05:05.110Z{"msg":"[0]R flush_numbers[0..12]: [4, 4, 3, 1, 1, 1, 1, 1, 3, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61725202Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4912024-04-20T01:05:05.110Z{"msg":"[0]R generation[0..12]: [3, 3, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61727262Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4922024-04-20T01:05:05.110Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61728713Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4932024-04-20T01:05:05.110Z{"msg":"[1]R flush_numbers[0..12]: [4, 4, 3, 1, 1, 1, 1, 1, 3, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61730761Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4942024-04-20T01:05:05.110Z{"msg":"[1]R generation[0..12]: [3, 3, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61732136Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4952024-04-20T01:05:05.110Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61734215Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4962024-04-20T01:05:05.110Z{"msg":"[2]R flush_numbers[0..12]: [4, 4, 3, 1, 1, 1, 1, 1, 3, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61735607Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4972024-04-20T01:05:05.110Z{"msg":"[2]R generation[0..12]: [3, 3, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61737606Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4982024-04-20T01:05:05.110Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61739024Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
4992024-04-20T01:05:05.110Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61741514Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
5002024-04-20T01:05:05.110Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61742861Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
5012024-04-20T01:05:05.110ZWait for a query_work_queue command to finish before sending IO
5022024-04-20T01:05:05.111Z{"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61745193Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
5032024-04-20T01:05:05.111Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61746553Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"":"downstairs","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
5042024-04-20T01:05:05.111Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61748541Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
5052024-04-20T01:05:05.111Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61749779Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"0","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
5062024-04-20T01:05:05.111Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61751733Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"1","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
5072024-04-20T01:05:05.111Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61753086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"client":"2","":"downstairs","session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
5082024-04-20T01:05:05.111Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61755052Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
5092024-04-20T01:05:05.111Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61756306Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
5102024-04-20T01:05:05.111Z{"msg":"59e54988-7afd-453a-9902-c8f0cbe840a8 is now active with session: 7225aa61-60c1-4e96-b64a-618b0c392eef","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61758848Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
5112024-04-20T01:05:05.111Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.61760263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184,"session_id":"7225aa61-60c1-4e96-b64a-618b0c392eef"}
5122024-04-20T01:05:05.111Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:03.617622941Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184}
5132024-04-20T01:05:05.111ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5142024-04-20T01:05:05.118ZLoading write count information from file "/tmp/test_repair_verify.data"
5152024-04-20T01:05:05.118ZRun Repair workload
5162024-04-20T01:05:05.118Z01/30 Write block 288 len 28672 data: 2 2 2 2 2 2 2
5172024-04-20T01:05:05.118Z02/30 Read block 218 len 8192
5182024-04-20T01:05:05.118Z03/30 Read block 432 len 8192
5192024-04-20T01:05:05.118Z04/30 Read block 164 len 24576
5202024-04-20T01:05:05.118Z05/30 Flush
5212024-04-20T01:05:05.118Z06/30 Write block 541 len 36864 data: 2 2 2 2 2 2 2 2 2
5222024-04-20T01:05:05.118Z07/30 Flush
5232024-04-20T01:05:05.122Z08/30 Write block 455 len 36864 data: 2 2 2 2 2 2 2 2 2
5242024-04-20T01:05:05.122Z09/30 Read block 494 len 24576
5252024-04-20T01:05:05.122Z10/30 Read block 146 len 36864
5262024-04-20T01:05:05.122Z11/30 Read block 369 len 12288
5272024-04-20T01:05:05.122Z12/30 Write block 542 len 20480 data: 3 3 3 3 3
5282024-04-20T01:05:05.125Z13/30 Write block 233 len 36864 data: 2 2 2 2 2 2 2 2 2
5292024-04-20T01:05:05.125Z14/30 Read block 446 len 16384
5302024-04-20T01:05:05.125Z15/30 Write block 284 len 16384 data: 2 2 2 2
5312024-04-20T01:05:05.125Z16/30 Flush
5322024-04-20T01:05:05.129Z17/30 Write block 28 len 8192 data: 2 2
5332024-04-20T01:05:05.129Z18/30 Read block 472 len 32768
5342024-04-20T01:05:05.135Z19/30 Write block 237 len 40960 data: 3 3 3 3 3 2 2 2 2 2
5352024-04-20T01:05:05.135Z20/30 Read block 188 len 8192
5362024-04-20T01:05:05.135Z21/30 Flush
5372024-04-20T01:05:05.139Z22/30 Write block 469 len 36864 data: 2 2 2 2 2 2 2 2 2
5382024-04-20T01:05:05.139Z23/30 Read block 169 len 12288
5392024-04-20T01:05:05.139Z24/30 Flush
5402024-04-20T01:05:05.139Z25/30 Write block 47 len 32768 data: 2 2 2 3 3 3 3 3
5412024-04-20T01:05:05.139Z26/30 Read block 80 len 4096
5422024-04-20T01:05:05.139Z27/30 Write block 418 len 24576 data: 2 2 2 2 2 2
5432024-04-20T01:05:05.148Z28/30 Write block 380 len 16384 data: 2 2 2 2
5442024-04-20T01:05:05.148Z29/30 Read block 523 len 4096
5452024-04-20T01:05:05.148Z30/30 Read block 556 len 8192
5462024-04-20T01:05:05.148Z----------------------------------------------------------------
5472024-04-20T01:05:05.148Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30
5482024-04-20T01:05:05.148ZStates: Active Active Active
5492024-04-20T01:05:05.148ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5502024-04-20T01:05:05.148Z 1 Acked 1000 Write 7 Done Done Done false
5512024-04-20T01:05:05.148Z 2 Acked 1001 Read 2 Done Done Sent false
5522024-04-20T01:05:05.148Z 3 Acked 1002 Read 2 Done Done Sent false
5532024-04-20T01:05:05.149Z 4 Acked 1003 Read 6 Done Done Sent false
5542024-04-20T01:05:05.149Z 5 Acked 1004 Flush 0 Done Done Sent false
5552024-04-20T01:05:05.149Z 6 Acked 1005 Write 9 Done Done Sent false
5562024-04-20T01:05:05.149Z 7 Acked 1006 Flush 0 Done Done Sent false
5572024-04-20T01:05:05.149Z 8 Acked 1007 Write 9 Done Done Sent false
5582024-04-20T01:05:05.149Z 9 Acked 1008 Read 6 Done Done Sent false
5592024-04-20T01:05:05.149Z 10 Acked 1009 Read 9 Done Done Sent false
5602024-04-20T01:05:05.149Z 11 Acked 1010 Read 3 Done Done Sent false
5612024-04-20T01:05:05.149Z 12 Acked 1011 Write 5 Done Done Sent false
5622024-04-20T01:05:05.149Z 13 Acked 1012 Write 9 Done Done Sent false
5632024-04-20T01:05:05.149Z 14 Acked 1013 Read 4 Done Done Sent false
5642024-04-20T01:05:05.149Z 15 Acked 1014 Write 4 Done Done Sent false
5652024-04-20T01:05:05.149Z 16 Acked 1015 Flush 0 Done Done Sent false
5662024-04-20T01:05:05.149Z 17 Acked 1016 Write 2 Done Done Sent false
5672024-04-20T01:05:05.149Z 18 Acked 1017 Read 8 Done Done Sent false
5682024-04-20T01:05:05.149Z 19 Acked 1018 Write 10 Done Done Sent false
5692024-04-20T01:05:05.149Z 20 Acked 1019 Read 2 Done Done Sent false
5702024-04-20T01:05:05.149Z 21 Acked 1020 Flush 0 Done Done Sent false
5712024-04-20T01:05:05.149Z 22 Acked 1021 Write 9 Done Done Sent false
5722024-04-20T01:05:05.150Z 23 Acked 1022 Read 3 Done Done Sent false
5732024-04-20T01:05:05.150Z 24 Acked 1023 Flush 0 Done Done Sent false
5742024-04-20T01:05:05.150Z 25 Acked 1024 Write 8 Done Done Sent false
5752024-04-20T01:05:05.150Z 26 Acked 1025 Read 1 Done Done Sent false
5762024-04-20T01:05:05.150Z 27 Acked 1026 Write 6 Done Done Sent false
5772024-04-20T01:05:05.150Z 28 Acked 1027 Write 4 Done Done Sent false
5782024-04-20T01:05:05.150Z 29 Acked 1028 Read 1 Done Done Sent false
5792024-04-20T01:05:05.150Z 30 Acked 1029 Read 2 Done Sent Sent false
5802024-04-20T01:05:05.150Z STATES DS:0 DS:1 DS:2 TOTAL
5812024-04-20T01:05:05.150Z New 0 0 0 0
5822024-04-20T01:05:05.150Z Sent 0 1 29 30
5832024-04-20T01:05:05.150Z Done 30 29 1 60
5842024-04-20T01:05:05.150Z Skipped 0 0 0 0
5852024-04-20T01:05:05.150Z Error 0 0 0 0
5862024-04-20T01:05:05.150ZLast Flush: 1023 1023 0
5872024-04-20T01:05:05.150ZDownstairs last five completed:
5882024-04-20T01:05:05.150ZUpstairs last five completed: 30 29 28 27 26
5892024-04-20T01:05:05.150Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:05:03.653930311Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1184}
5902024-04-20T01:05:05.150ZWrote out file "/tmp/test_repair_verify.data"
5912024-04-20T01:05:05.155Z
5922024-04-20T01:05:05.155ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5932024-04-20T01:05:05.166ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5942024-04-20T01:05:05.166Z 0 000-019 3 3 3 4 4 4 F F F
5952024-04-20T01:05:05.166Z 1 020-039 4 4 3 8 8 4 F F F <---
5962024-04-20T01:05:05.166Z 2 040-059 2 2 2 3 3 3 T T F <---
5972024-04-20T01:05:05.166Z 3 060-079 1 1 1 1 1 1 F F F
5982024-04-20T01:05:05.166Z 4 080-099 1 1 1 1 1 1 F F F
5992024-04-20T01:05:05.166Z 5 100-119 1 1 1 1 1 1 F F F
6002024-04-20T01:05:05.166Z 6 120-139 1 1 1 1 1 1 F F F
6012024-04-20T01:05:05.166Z 7 140-159 1 1 1 1 1 1 F F F
6022024-04-20T01:05:05.166Z 8 160-179 2 2 2 3 3 3 F F F
6032024-04-20T01:05:05.166Z 9 180-199 1 1 1 1 1 1 F F F
6042024-04-20T01:05:05.170Z 10 200-219 1 1 1 1 1 1 F F F
6052024-04-20T01:05:05.170Z 11 220-239 4 4 1 8 8 1 F F F <---
6062024-04-20T01:05:05.170Z 12 240-259 4 4 1 8 8 1 F F F <---
6072024-04-20T01:05:05.170Z 13 260-279 1 1 1 1 1 1 F F F
6082024-04-20T01:05:05.170Z 14 280-299 4 4 1 7 7 1 F F T <---
6092024-04-20T01:05:05.170Z 15 300-319 1 1 1 1 1 1 F F F
6102024-04-20T01:05:05.170Z 16 320-339 1 1 1 1 1 1 F F F
6112024-04-20T01:05:05.170Z 17 340-359 1 1 1 1 1 1 F F F
6122024-04-20T01:05:05.171Z 18 360-379 2 2 2 3 3 3 F F F
6132024-04-20T01:05:05.171Z 19 380-399 1 1 1 1 1 1 T T F <---
6142024-04-20T01:05:05.171Z 20 400-419 2 2 2 2 2 2 T T F <---
6152024-04-20T01:05:05.171Z 21 420-439 2 2 2 3 3 3 T T F <---
6162024-04-20T01:05:05.171Z 22 440-459 4 4 2 7 7 2 F F F <---
6172024-04-20T01:05:05.171Z 23 460-479 4 4 1 9 9 1 F F F <---
6182024-04-20T01:05:05.171Z 24 480-499 1 1 1 1 1 1 F F F
6192024-04-20T01:05:05.171Z 25 500-519 2 2 2 3 3 3 F F F
6202024-04-20T01:05:05.171Z 26 520-539 2 2 2 2 2 2 F F F
6212024-04-20T01:05:05.171Z 27 540-559 4 4 1 7 7 1 F F F <---
6222024-04-20T01:05:05.171Z 28 560-579 2 2 2 3 3 3 F F F
6232024-04-20T01:05:05.171Z 29 580-599 1 1 1 1 1 1 F F F
6242024-04-20T01:05:05.171ZMax gen: 4, Max flush: 9
6252024-04-20T01:05:05.171ZError: Difference in extent metadata found!
6262024-04-20T01:05:05.171ZA Difference in extent metadata is expected here
6272024-04-20T01:05:05.171ZOn loop 2
6282024-04-20T01:05:05.171Z
6292024-04-20T01:05:05.171ZVerifying data now
6302024-04-20T01:05:09.720ZNo change after verify
6312024-04-20T01:05:09.720ZLoop: 2 Downstairs dump after verify (and repair):
6322024-04-20T01:05:10.158ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6332024-04-20T01:05:10.158Z 0 000-019 3 3 3 4 4 4 F F F
6342024-04-20T01:05:10.158Z 1 020-039 4 4 4 8 8 8 F F F
6352024-04-20T01:05:10.158Z 2 040-059 5 5 5 10 10 10 F F F
6362024-04-20T01:05:10.158Z 3 060-079 1 1 1 1 1 1 F F F
6372024-04-20T01:05:10.158Z 4 080-099 1 1 1 1 1 1 F F F
6382024-04-20T01:05:10.158Z 5 100-119 1 1 1 1 1 1 F F F
6392024-04-20T01:05:10.158Z 6 120-139 1 1 1 1 1 1 F F F
6402024-04-20T01:05:10.158Z 7 140-159 1 1 1 1 1 1 F F F
6412024-04-20T01:05:10.158Z 8 160-179 2 2 2 3 3 3 F F F
6422024-04-20T01:05:10.158Z 9 180-199 1 1 1 1 1 1 F F F
6432024-04-20T01:05:10.158Z 10 200-219 1 1 1 1 1 1 F F F
6442024-04-20T01:05:10.158Z 11 220-239 4 4 4 8 8 8 F F F
6452024-04-20T01:05:10.159Z 12 240-259 4 4 4 8 8 8 F F F
6462024-04-20T01:05:10.159Z 13 260-279 1 1 1 1 1 1 F F F
6472024-04-20T01:05:10.159Z 14 280-299 4 4 4 7 7 7 F F F
6482024-04-20T01:05:10.159Z 15 300-319 1 1 1 1 1 1 F F F
6492024-04-20T01:05:10.159Z 16 320-339 1 1 1 1 1 1 F F F
6502024-04-20T01:05:10.159Z 17 340-359 1 1 1 1 1 1 F F F
6512024-04-20T01:05:10.159Z 18 360-379 2 2 2 3 3 3 F F F
6522024-04-20T01:05:10.159Z 19 380-399 5 5 5 10 10 10 F F F
6532024-04-20T01:05:10.159Z 20 400-419 5 5 5 10 10 10 F F F
6542024-04-20T01:05:10.159Z 21 420-439 5 5 5 10 10 10 F F F
6552024-04-20T01:05:10.159Z 22 440-459 4 4 4 7 7 7 F F F
6562024-04-20T01:05:10.159Z 23 460-479 4 4 4 9 9 9 F F F
6572024-04-20T01:05:10.159Z 24 480-499 1 1 1 1 1 1 F F F
6582024-04-20T01:05:10.159Z 25 500-519 2 2 2 3 3 3 F F F
6592024-04-20T01:05:10.159Z 26 520-539 2 2 2 2 2 2 F F F
6602024-04-20T01:05:10.159Z 27 540-559 4 4 4 7 7 7 F F F
6612024-04-20T01:05:10.159Z 28 560-579 2 2 2 3 3 3 F F F
6622024-04-20T01:05:10.159Z 29 580-599 1 1 1 1 1 1 F F F
6632024-04-20T01:05:10.159ZMax gen: 5, Max flush: 10
6642024-04-20T01:05:10.159Z
6652024-04-20T01:05:10.159ZBegin loop 3
6662024-04-20T01:05:10.159ZDownstairs to restart: 1
6672024-04-20T01:05:10.159Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.262784781Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192}
6682024-04-20T01:05:10.160Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263018561Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192}
6692024-04-20T01:05:10.160Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263042111Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192}
6702024-04-20T01:05:10.160Z{"msg":"Crucible 0fa29bb1-9965-445e-b14e-94ccf846ef76 has session id: d2b68102-fd2d-44df-b878-332092f08389","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263077541Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6712024-04-20T01:05:10.160Z{"msg":"Upstairs opts: Upstairs UUID: 0fa29bb1-9965-445e-b14e-94ccf846ef76, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263123002Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6722024-04-20T01:05:10.160ZCrucible runtime is spawned
6732024-04-20T01:05:10.160Z{"msg":"Crucible stats registered with UUID: 0fa29bb1-9965-445e-b14e-94ccf846ef76","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263164082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6742024-04-20T01:05:10.160Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263235132Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192}
6752024-04-20T01:05:10.160Z{"msg":"0fa29bb1-9965-445e-b14e-94ccf846ef76 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263260272Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6762024-04-20T01:05:10.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263274502Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"0","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6772024-04-20T01:05:10.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263296822Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"0","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6782024-04-20T01:05:10.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263324972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"1","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6792024-04-20T01:05:10.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263352262Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"1","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6802024-04-20T01:05:10.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263379222Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"2","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6812024-04-20T01:05:10.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263444192Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"2","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6822024-04-20T01:05:10.160Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263512612Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6832024-04-20T01:05:10.160Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263549843Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6842024-04-20T01:05:10.160Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263574643Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6852024-04-20T01:05:10.160Z{"msg":"ds_connection connected from Ok(127.0.0.1:51501)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263597872Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6862024-04-20T01:05:10.160Z{"msg":"ds_connection connected from Ok(127.0.0.1:51020)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263619532Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6872024-04-20T01:05:10.160Z{"msg":"ds_connection connected from Ok(127.0.0.1:62390)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.263784873Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6882024-04-20T01:05:10.160Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.264296184Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"2","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6892024-04-20T01:05:10.160Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.264363014Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"2","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6902024-04-20T01:05:10.160Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.264410044Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"0","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6912024-04-20T01:05:10.161Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.264470954Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"0","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6922024-04-20T01:05:10.161Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.264650895Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"1","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6932024-04-20T01:05:10.161Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.264685735Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"1","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6942024-04-20T01:05:10.161Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.264950215Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"2","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6952024-04-20T01:05:10.161Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.264988896Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"0","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6962024-04-20T01:05:10.161Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265274296Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"1","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6972024-04-20T01:05:10.161Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265449427Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"0","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6982024-04-20T01:05:10.161Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265484927Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"0","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
6992024-04-20T01:05:10.161Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265499867Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7002024-04-20T01:05:10.161Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265524607Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"2","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7012024-04-20T01:05:10.161Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265545737Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"2","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7022024-04-20T01:05:10.161Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265559617Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7032024-04-20T01:05:10.161Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265771917Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"1","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7042024-04-20T01:05:10.161Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265799457Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"1","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7052024-04-20T01:05:10.161Z{"msg":"[0]R flush_numbers[0..12]: [4, 8, 10, 1, 1, 1, 1, 1, 3, 1, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265813948Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7062024-04-20T01:05:10.161Z{"msg":"[0]R generation[0..12]: [3, 4, 5, 1, 1, 1, 1, 1, 2, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265834717Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7072024-04-20T01:05:10.161Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265849737Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7082024-04-20T01:05:10.161Z{"msg":"[1]R flush_numbers[0..12]: [4, 8, 10, 1, 1, 1, 1, 1, 3, 1, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265869857Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7092024-04-20T01:05:10.161Z{"msg":"[1]R generation[0..12]: [3, 4, 5, 1, 1, 1, 1, 1, 2, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265884138Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7102024-04-20T01:05:10.161Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265904868Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7112024-04-20T01:05:10.161Z{"msg":"[2]R flush_numbers[0..12]: [4, 8, 10, 1, 1, 1, 1, 1, 3, 1, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265918728Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7122024-04-20T01:05:10.162Z{"msgWait for a query_work_queue command to finish before sending IO
7132024-04-20T01:05:10.162Z":"[2]R generation[0..12]: [3, 4, 5, 1, 1, 1, 1, 1, 2, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265941558Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7142024-04-20T01:05:10.162Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265968978Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7152024-04-20T01:05:10.162Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.265986498Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7162024-04-20T01:05:10.162Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.266003088Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7172024-04-20T01:05:10.162Z{"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.266016148Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7182024-04-20T01:05:10.162Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.266036918Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"":"downstairs","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7192024-04-20T01:05:10.162Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.266066728Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7202024-04-20T01:05:10.162Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.266089688Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"0","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7212024-04-20T01:05:10.162Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.266116988Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"1","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7222024-04-20T01:05:10.162Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.266140568Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"client":"2","":"downstairs","session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7232024-04-20T01:05:10.162Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.266168138Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7242024-04-20T01:05:10.162Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7252024-04-20T01:05:10.162Z,"time":"2024-04-20T01:05:08.266190478Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7262024-04-20T01:05:10.162Z{"msg":"0fa29bb1-9965-445e-b14e-94ccf846ef76 is now active with session: d2b68102-fd2d-44df-b878-332092f08389","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.266216678Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7272024-04-20T01:05:10.162Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.266243108Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192,"session_id":"d2b68102-fd2d-44df-b878-332092f08389"}
7282024-04-20T01:05:10.162Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:08.266265538Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192}
7292024-04-20T01:05:10.162ZLoading write count information from file "/tmp/test_repair_verify.data"
7302024-04-20T01:05:10.162ZRun Repair workload
7312024-04-20T01:05:10.163Z01/30 Write block 229 len 36864 data: 2 2 2 2 3 3 3 3 4
7322024-04-20T01:05:10.177Z02/30 Write block 523 len 12288 data: 2 2 2
7332024-04-20T01:05:10.177Z03/30 Read block 299 len 8192
7342024-04-20T01:05:10.178Z04/30 Read block 160 len 20480
7352024-04-20T01:05:10.178Z05/30 Write block 545 len 40960 data: 4 4 3 3 3 2 2 2 2 2
7362024-04-20T01:05:10.178Z06/30 Write block 100 len 32768 data: 2 2 2 2 2 2 2 2
7372024-04-20T01:05:10.178Z07/30 Read block 208 len 12288
7382024-04-20T01:05:10.178Z08/30 Flush
7392024-04-20T01:05:10.178Z09/30 Write block 69 len 16384 data: 2 2 2 2
7402024-04-20T01:05:10.178Z10/30 Read block 528 len 16384
7412024-04-20T01:05:10.178Z11/30 Write block 411 len 20480 data: 2 2 2 2 2
7422024-04-20T01:05:10.178Z12/30 Write block 197 len 12288 data: 2 2 2
7432024-04-20T01:05:10.178Z13/30 Read block 117 len 36864
7442024-04-20T01:05:10.179Z14/30 Read block 392 len 12288
7452024-04-20T01:05:10.179Z15/30 Read block 526 len 20480
7462024-04-20T01:05:10.179Z16/30 Read block 249 len 36864
7472024-04-20T01:05:10.179Z17/30 Write block 373 len 36864 data: 4 4 3 2 2 2 2 3 3
7482024-04-20T01:05:10.179Z18/30 Read block 479 len 32768
7492024-04-20T01:05:10.179Z19/30 Write block 59 len 32768 data: 2 2 2 2 2 2 2 2
7502024-04-20T01:05:10.179Z20/30 Read block 359 len 28672
7512024-04-20T01:05:10.179Z21/30 Write block 51 len 12288 data: 4 4 4
7522024-04-20T01:05:10.179Z22/30 Write block 244 len 8192 data: 3 3
7532024-04-20T01:05:10.179Z23/30 Read block 359 len 20480
7542024-04-20T01:05:10.179Z24/30 Flush
7552024-04-20T01:05:10.179Z25/30 Write block 50 len 16384 data: 4 5 5 5
7562024-04-20T01:05:10.180Z26/30 Read block 320 len 32768
7572024-04-20T01:05:10.180Z27/30 Read block 406 len 28672
7582024-04-20T01:05:10.180Z28/30 Write block 413 len 12288 data: 3 3 3
7592024-04-20T01:05:10.180Z29/30 Write block 318 len 20480 data: 2 2 2 2 2
7602024-04-20T01:05:10.180Z30/30 Read block 276 len 28672
7612024-04-20T01:05:10.180Z----------------------------------------------------------------
7622024-04-20T01:05:10.180Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7632024-04-20T01:05:10.180ZStates: Active Active Active
7642024-04-20T01:05:10.180ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7652024-04-20T01:05:10.180Z 1 Acked 1000 Write 9 Done Done Done false
7662024-04-20T01:05:10.181Z 2 Acked 1001 Write 3 Done Done Done false
7672024-04-20T01:05:10.181Z 3 Acked 1002 Read 2 Done Done Done false
7682024-04-20T01:05:10.181Z 4 Acked 1003 Read 5 Done Sent Done false
7692024-04-20T01:05:10.181Z 5 Acked 1004 Write 10 Done Sent Done false
7702024-04-20T01:05:10.181Z 6 Acked 1005 Write 8 Done Sent Done false
7712024-04-20T01:05:10.181Z 7 Acked 1006 Read 3 Done Sent Done false
7722024-04-20T01:05:10.181Z 8 Acked 1007 Flush 0 Done Sent Done false
7732024-04-20T01:05:10.181Z 9 Acked 1008 Write 4 Done Sent Done false
7742024-04-20T01:05:10.181Z 10 Acked 1009 Read 4 Done Sent Done false
7752024-04-20T01:05:10.181Z 11 Acked 1010 Write 5 Done Sent Done false
7762024-04-20T01:05:10.181Z 12 Acked 1011 Write 3 Done Sent Done false
7772024-04-20T01:05:10.181Z 13 Acked 1012 Read 9 Done Sent Done false
7782024-04-20T01:05:10.182Z 14 Acked 1013 Read 3 Done Sent Done false
7792024-04-20T01:05:10.182Z 15 Acked 1014 Read 5 Done Sent Done false
7802024-04-20T01:05:10.182Z 16 Acked 1015 Read 9 Done Sent Done false
7812024-04-20T01:05:10.182Z 17 Acked 1016 Write 9 Done Sent Done false
7822024-04-20T01:05:10.182Z 18 Acked 1017 Read 8 Done Sent Done false
7832024-04-20T01:05:10.182Z 19 Acked 1018 Write 8 Done Sent Done false
7842024-04-20T01:05:10.182Z 20 Acked 1019 Read 7 Done Sent Done false
7852024-04-20T01:05:10.182Z 21 Acked 1020 Write 3 Done Sent Done false
7862024-04-20T01:05:10.182Z 22 Acked 1021 Write 2 Done Sent Done false
7872024-04-20T01:05:10.182Z 23 Acked 1022 Read 5 Done Sent Done false
7882024-04-20T01:05:10.182Z 24 Acked 1023 Flush 0 Done Sent Done false
7892024-04-20T01:05:10.182Z 25 Acked 1024 Write 4 Done Sent Done false
7902024-04-20T01:05:10.182Z 26 Acked 1025 Read 8 Done Sent Done false
7912024-04-20T01:05:10.182Z 27 Acked 1026 Read 7 Done Sent Done false
7922024-04-20T01:05:10.183Z 28 Acked 1027 Write 3 Done Sent Done false
7932024-04-20T01:05:10.183Z 29 Acked 1028 Write 5 Done Sent Done false
7942024-04-20T01:05:10.183Z 30 Acked 1029 Read 7 Done Sent Done false
7952024-04-20T01:05:10.183Z STATES DS:0 DS:1 DS:2 TOTAL
7962024-04-20T01:05:10.183Z New 0 0 0 0
7972024-04-20T01:05:10.183Z Sent 0 27 0 27
7982024-04-20T01:05:10.183Z Done 30 3 30 63
7992024-04-20T01:05:10.183Z Skipped 0 0 0 0
8002024-04-20T01:05:10.183Z Error 0 0 0 0
8012024-04-20T01:05:10.183ZLast Flush: 1023 0 1023
8022024-04-20T01:05:10.184ZDownstairs last five completed:
8032024-04-20T01:05:10.184ZUpstairs last five completed: 30 29 28 27 26
8042024-04-20T01:05:10.184Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:05:08.297276947Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1192}
8052024-04-20T01:05:10.184ZWrote out file "/tmp/test_repair_verify.data"
8062024-04-20T01:05:10.184Z
8072024-04-20T01:05:10.184ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8082024-04-20T01:05:10.184ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8092024-04-20T01:05:10.184Z 0 000-019 3 3 3 4 4 4 F F F
8102024-04-20T01:05:10.184Z 1 020-039 4 4 4 8 8 8 F F F
8112024-04-20T01:05:10.184Z 2 040-059 6 5 6 12 10 12 T F T <---
8122024-04-20T01:05:10.184Z 3 060-079 6 1 6 12 1 12 F F F <---
8132024-04-20T01:05:10.184Z 4 080-099 1 1 1 1 1 1 F F F
8142024-04-20T01:05:10.184Z 5 100-119 6 1 6 11 1 11 F F F <---
8152024-04-20T01:05:10.184Z 6 120-139 1 1 1 1 1 1 F F F
8162024-04-20T01:05:10.184Z 7 140-159 1 1 1 1 1 1 F F F
8172024-04-20T01:05:10.184Z 8 160-179 2 2 2 3 3 3 F F F
8182024-04-20T01:05:10.185Z 9 180-199 6 1 6 12 1 12 F F F <---
8192024-04-20T01:05:10.185Z 10 200-219 1 1 1 1 1 1 F F F
8202024-04-20T01:05:10.185Z 11 220-239 6 4 6 11 8 11 F T F <---
8212024-04-20T01:05:10.185Z 12 240-259 6 4 6 12 8 12 F F F <---
8222024-04-20T01:05:10.185Z 13 260-279 1 1 1 1 1 1 F F F
8232024-04-20T01:05:10.185Z 14 280-299 4 4 4 7 7 7 F F F
8242024-04-20T01:05:10.185Z 15 300-319 1 1 1 1 1 1 T F T <---
8252024-04-20T01:05:10.185Z 16 320-339 1 1 1 1 1 1 T F T <---
8262024-04-20T01:05:10.185Z 17 340-359 1 1 1 1 1 1 F F F
8272024-04-20T01:05:10.185Z 18 360-379 6 2 6 12 3 12 F F F <---
8282024-04-20T01:05:10.185Z 19 380-399 6 5 6 12 10 12 F F F <---
8292024-04-20T01:05:10.185Z 20 400-419 6 5 6 12 10 12 T F T <---
8302024-04-20T01:05:10.185Z 21 420-439 5 5 5 10 10 10 F F F
8312024-04-20T01:05:10.185Z 22 440-459 4 4 4 7 7 7 F F F
8322024-04-20T01:05:10.627Z 23 460-479 4 4 4 9 9 9 F F F
8332024-04-20T01:05:10.627Z 24 480-499 1 1 1 1 1 1 F F F
8342024-04-20T01:05:10.627Z 25 500-519 2 2 2 3 3 3 F F F
8352024-04-20T01:05:10.628Z 26 520-539 6 2 6 11 2 11 F T F <---
8362024-04-20T01:05:10.628Z 27 540-559 6 4 6 11 7 11 F F F <---
8372024-04-20T01:05:10.628Z 28 560-579 2 2 2 3 3 3 F F F
8382024-04-20T01:05:10.628Z 29 580-599 1 1 1 1 1 1 F F F
8392024-04-20T01:05:10.629ZMax gen: 6, Max flush: 12
8402024-04-20T01:05:10.629ZError: Difference in extent metadata found!
8412024-04-20T01:05:10.629ZA Difference in extent metadata is expected here
8422024-04-20T01:05:10.629ZOn loop 3
8432024-04-20T01:05:10.629Z
8442024-04-20T01:05:10.630ZVerifying data now
8452024-04-20T01:05:24.389ZNo change after verify
8462024-04-20T01:05:24.389ZLoop: 3 Downstairs dump after verify (and repair):
8472024-04-20T01:05:24.403ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8482024-04-20T01:05:24.403Z 0 000-019 3 3 3 4 4 4 F F F
8492024-04-20T01:05:24.403Z 1 020-039 4 4 4 8 8 8 F F F
8502024-04-20T01:05:24.403Z 2 040-059 7 7 7 13 13 13 F F F
8512024-04-20T01:05:24.403Z 3 060-079 6 6 6 12 12 12 F F F
8522024-04-20T01:05:24.403Z 4 080-099 1 1 1 1 1 1 F F F
8532024-04-20T01:05:24.403Z 5 100-119 6 6 6 11 11 11 F F F
8542024-04-20T01:05:24.403Z 6 120-139 1 1 1 1 1 1 F F F
8552024-04-20T01:05:24.403Z 7 140-159 1 1 1 1 1 1 F F F
8562024-04-20T01:05:24.403Z 8 160-179 2 2 2 3 3 3 F F F
8572024-04-20T01:05:24.407Z 9 180-199 6 6 6 12 12 12 F F F
8582024-04-20T01:05:24.407Z 10 200-219 1 1 1 1 1 1 F F F
8592024-04-20T01:05:24.407Z 11 220-239 6 6 6 11 11 11 F F F
8602024-04-20T01:05:24.407Z 12 240-259 6 6 6 12 12 12 F F F
8612024-04-20T01:05:24.407Z 13 260-279 1 1 1 1 1 1 F F F
8622024-04-20T01:05:24.407Z 14 280-299 4 4 4 7 7 7 F F F
8632024-04-20T01:05:24.408Z 15 300-319 7 7 7 13 13 13 F F F
8642024-04-20T01:05:24.408Z 16 320-339 7 7 7 13 13 13 F F F
8652024-04-20T01:05:24.408Z 17 340-359 1 1 1 1 1 1 F F F
8662024-04-20T01:05:24.408Z 18 360-379 6 6 6 12 12 12 F F F
8672024-04-20T01:05:24.408Z 19 380-399 6 6 6 12 12 12 F F F
8682024-04-20T01:05:24.408Z 20 400-419 7 7 7 13 13 13 F F F
8692024-04-20T01:05:24.408Z 21 420-439 5 5 5 10 10 10 F F F
8702024-04-20T01:05:24.408Z 22 440-459 4 4 4 7 7 7 F F F
8712024-04-20T01:05:24.408Z 23 460-479 4 4 4 9 9 9 F F F
8722024-04-20T01:05:24.408Z 24 480-499 1 1 1 1 1 1 F F F
8732024-04-20T01:05:24.408Z 25 500-519 2 2 2 3 3 3 F F F
8742024-04-20T01:05:24.408Z 26 520-539 6 6 6 11 11 11 F F F
8752024-04-20T01:05:24.408Z 27 540-559 6 6 6 11 11 11 F F F
8762024-04-20T01:05:24.408Z 28 560-579 2 2 2 3 3 3 F F F
8772024-04-20T01:05:24.408Z 29 580-599 1 1 1 1 1 1 F F F
8782024-04-20T01:05:24.408ZMax gen: 7, Max flush: 13
8792024-04-20T01:05:24.408Z
8802024-04-20T01:05:24.408ZBegin loop 4
8812024-04-20T01:05:24.408ZDownstairs to restart: 0
8822024-04-20T01:05:24.423Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.934900441Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200}
8832024-04-20T01:05:24.423Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935175631Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200}
8842024-04-20T01:05:24.431Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935210721Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200}
8852024-04-20T01:05:24.431Z{"msg":"Crucible 20721a0a-fac7-45e0-95d2-49ab72a4d0e4 has session id: 00e8669f-2cfb-4bde-b655-b837d2ebc10d","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935233281Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
8862024-04-20T01:05:24.431Z{"msg":"Crucible runtime is spawned
8872024-04-20T01:05:24.432ZUpstairs opts: Upstairs UUID: 20721a0a-fac7-45e0-95d2-49ab72a4d0e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935299571Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
8882024-04-20T01:05:24.432Z{"msg":"Crucible stats registered with UUID: 20721a0a-fac7-45e0-95d2-49ab72a4d0e4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935339391Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
8892024-04-20T01:05:24.432Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935376842Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200}
8902024-04-20T01:05:24.432Z{"msg":"20721a0a-fac7-45e0-95d2-49ab72a4d0e4 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935488902Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
8912024-04-20T01:05:24.432Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935527042Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"0","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
8922024-04-20T01:05:24.432Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935553412Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"0","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
8932024-04-20T01:05:24.432Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935578052Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"1","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
8942024-04-20T01:05:24.432Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935597692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"1","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
8952024-04-20T01:05:24.432Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935618082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"2","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
8962024-04-20T01:05:24.432Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935633332Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"2","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
8972024-04-20T01:05:24.432Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935655592Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
8982024-04-20T01:05:24.432Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935670342Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
8992024-04-20T01:05:24.432Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935687732Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9002024-04-20T01:05:24.432Z{"msg":"ds_connection connected from Ok(127.0.0.1:48837)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935761212Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9012024-04-20T01:05:24.432Z{"msg":"ds_connection connected from Ok(127.0.0.1:55189)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935816163Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9022024-04-20T01:05:24.432Z{"msg":"ds_connection connected from Ok(127.0.0.1:63345)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.935860323Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9032024-04-20T01:05:24.432Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.936506484Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"1","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9042024-04-20T01:05:24.432Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.936553544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"1","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9052024-04-20T01:05:24.432Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.936580004Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"2","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9062024-04-20T01:05:24.432Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.936605774Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"2","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9072024-04-20T01:05:24.432Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.936704614Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"0","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9082024-04-20T01:05:24.432Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.936733015Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"0","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9092024-04-20T01:05:24.433Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937026475Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"2","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9102024-04-20T01:05:24.433Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937070865Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"1","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9112024-04-20T01:05:24.433Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937392616Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"0","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9122024-04-20T01:05:24.433Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937583567Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"1","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9132024-04-20T01:05:24.433Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937630337Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"1","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9142024-04-20T01:05:24.433Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937648367Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9152024-04-20T01:05:24.433Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937691537Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"2","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9162024-04-20T01:05:24.433Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937710287Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"2","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9172024-04-20T01:05:24.433Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937730467Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9182024-04-20T01:05:24.433Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937874867Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"0","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9192024-04-20T01:05:24.433Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937920367Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"0","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9202024-04-20T01:05:24.433Z{"msg":"[0]R flush_numbers[0..12]: [4, 8, 13, 12, 1, 11, 1, 1, 3, 12, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937945837Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9212024-04-20T01:05:24.433Z{"msg":"[0]R generation[0..12]: [3, 4, 7, 6, 1, 6, 1, 1, 2, 6, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.937980388Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9222024-04-20T01:05:24.433Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938007628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9232024-04-20T01:05:24.433Z{"msg":"[1]R flush_numbers[0..12]: [4, 8, 13, 12, 1, 11, 1, 1, 3, 12, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938043617Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9242024-04-20T01:05:24.433Z{"msg":"[1]R generation[0..12]: [3, 4, 7, 6, 1, 6, 1, 1, 2, 6, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938091928Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9252024-04-20T01:05:24.433Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938121668Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs",Wait for a query_work_queue command to finish before sending IO
9262024-04-20T01:05:24.433Z"session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9272024-04-20T01:05:24.433Z{"msg":"[2]R flush_numbers[0..12]: [4, 8, 13, 12, 1, 11, 1, 1, 3, 12, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938159068Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9282024-04-20T01:05:24.433Z{"msg":"[2]R generation[0..12]: [3, 4, 7, 6, 1, 6, 1, 1, 2, 6, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938184818Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9292024-04-20T01:05:24.433Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938220978Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9302024-04-20T01:05:24.433Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938256948Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9312024-04-20T01:05:24.433Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938294668Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9322024-04-20T01:05:24.433Z{"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938318938Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9332024-04-20T01:05:24.434Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938350788Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"":"downstairs","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9342024-04-20T01:05:24.434Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938387648Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9352024-04-20T01:05:24.434Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938422618Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"0","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9362024-04-20T01:05:24.434Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9372024-04-20T01:05:24.434Z,"time":"2024-04-20T01:05:22.938460889Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"1","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9382024-04-20T01:05:24.434Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938499409Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"client":"2","":"downstairs","session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9392024-04-20T01:05:24.434Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938528999Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9402024-04-20T01:05:24.434Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938542449Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9412024-04-20T01:05:24.434Z{"msg":"20721a0a-fac7-45e0-95d2-49ab72a4d0e4 is now active with session: 00e8669f-2cfb-4bde-b655-b837d2ebc10d","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938563259Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9422024-04-20T01:05:24.434Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938575399Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200,"session_id":"00e8669f-2cfb-4bde-b655-b837d2ebc10d"}
9432024-04-20T01:05:24.434Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:22.938594319Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200}
9442024-04-20T01:05:24.434ZLoading write count information from file "/tmp/test_repair_verify.data"
9452024-04-20T01:05:24.434ZRun Repair workload
9462024-04-20T01:05:24.439Z01/30 Write block 0 len 16384 data: 2 2 2 2
9472024-04-20T01:05:24.439Z02/30 Write block 83 len 4096 data: 2
9482024-04-20T01:05:24.439Z03/30 Read block 431 len 12288
9492024-04-20T01:05:24.439Z04/30 Write block 11 len 28672 data: 2 2 2 2 2 2 2
9502024-04-20T01:05:24.439Z05/30 Read block 111 len 4096
9512024-04-20T01:05:24.439Z06/30 Write block 244 len 24576 data: 4 4 3 2 2 2
9522024-04-20T01:05:24.439Z07/30 Write block 222 len 20480 data: 2 2 2 2 2
9532024-04-20T01:05:24.439Z08/30 Write block 168 len 4096 data: 2
9542024-04-20T01:05:24.439Z09/30 Flush
9552024-04-20T01:05:24.439Z10/30 Write block 503 len 12288 data: 3 3 3
9562024-04-20T01:05:24.439Z11/30 Read block 321 len 4096
9572024-04-20T01:05:24.439Z12/30 Flush
9582024-04-20T01:05:24.443Z13/30 Write block 594 len 12288 data: 2 2 2
9592024-04-20T01:05:24.443Z14/30 Read block 464 len 12288
9602024-04-20T01:05:24.443Z15/30 Write block 570 len 16384 data: 4 3 2 2
9612024-04-20T01:05:24.443Z16/30 Write block 177 len 32768 data: 2 2 2 2 2 2 2 2
9622024-04-20T01:05:24.447Z17/30 Write block 340 len 40960 data: 2 2 2 2 2 2 2 2 2 2
9632024-04-20T01:05:24.447Z18/30 Read block 167 len 20480
9642024-04-20T01:05:24.447Z19/30 Write block 252 len 32768 data: 2 2 2 2 2 2 2 2
9652024-04-20T01:05:24.447Z20/30 Read block 458 len 12288
9662024-04-20T01:05:24.450Z21/30 Read block 531 len 28672
9672024-04-20T01:05:24.450Z22/30 Write block 509 len 20480 data: 3 2 2 3 3
9682024-04-20T01:05:24.450Z23/30 Write block 323 len 40960 data: 2 2 2 2 2 2 2 2 2 2
9692024-04-20T01:05:24.454Z24/30 Write block 481 len 16384 data: 2 2 2 2
9702024-04-20T01:05:24.454Z25/30 Write block 289 len 8192 data: 3 3
9712024-04-20T01:05:24.454Z26/30 Write block 144 len 24576 data: 2 2 2 2 2 2
9722024-04-20T01:05:24.454Z27/30 Read block 141 len 28672
9732024-04-20T01:05:24.454Z28/30 Read block 29 len 16384
9742024-04-20T01:05:24.458Z29/30 Write block 439 len 36864 data: 2 2 3 3 4 3 3 3 3
9752024-04-20T01:05:24.458Z30/30 Write block 182 len 4096 data: 3
9762024-04-20T01:05:24.458Z----------------------------------------------------------------
9772024-04-20T01:05:24.459Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9782024-04-20T01:05:24.459ZStates: Active Active Active
9792024-04-20T01:05:24.459ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9802024-04-20T01:05:24.459Z 1 Acked 1000 Write 4 Done Done Done false
9812024-04-20T01:05:24.459Z 2 Acked 1001 Write 1 Done Done Done false
9822024-04-20T01:05:24.459Z 3 Acked 1002 Read 3 Done Done Done false
9832024-04-20T01:05:24.459Z 4 Acked 1003 Write 7 Sent Done Done false
9842024-04-20T01:05:24.459Z 5 Acked 1004 Read 1 Sent Done Done false
9852024-04-20T01:05:24.459Z 6 Acked 1005 Write 6 Sent Done Done false
9862024-04-20T01:05:24.459Z 7 Acked 1006 Write 5 Sent Done Done false
9872024-04-20T01:05:24.459Z 8 Acked 1007 Write 1 Sent Done Done false
9882024-04-20T01:05:24.459Z 9 Acked 1008 Flush 0 Sent Done Done false
9892024-04-20T01:05:24.459Z 10 Acked 1009 Write 3 Sent Done Done false
9902024-04-20T01:05:24.460Z 11 Acked 1010 Read 1 Sent Done Done false
9912024-04-20T01:05:24.460Z 12 Acked 1011 Flush 0 Sent Done Done false
9922024-04-20T01:05:24.460Z 13 Acked 1012 Write 3 Sent Done Done false
9932024-04-20T01:05:24.460Z 14 Acked 1013 Read 3 Sent Done Done false
9942024-04-20T01:05:24.460Z 15 Acked 1014 Write 4 Sent Done Done false
9952024-04-20T01:05:24.460Z 16 Acked 1015 Write 8 Sent Done Done false
9962024-04-20T01:05:24.460Z 17 Acked 1016 Write 10 Sent Done Done false
9972024-04-20T01:05:24.461Z 18 Acked 1017 Read 5 Sent Done Done false
9982024-04-20T01:05:24.461Z 19 Acked 1018 Write 8 Sent Done Done false
9992024-04-20T01:05:24.461Z 20 Acked 1019 Read 3 Sent Done Done false
10002024-04-20T01:05:24.461Z 21 Acked 1020 Read 7 Sent Done Done false
10012024-04-20T01:05:24.461Z 22 Acked 1021 Write 5 Sent Done Done false
10022024-04-20T01:05:24.461Z 23 Acked 1022 Write 10 Sent Done Done false
10032024-04-20T01:05:24.461Z 24 Acked 1023 Write 4 Sent Done Done false
10042024-04-20T01:05:24.462Z 25 Acked 1024 Write 2 Sent Done Done false
10052024-04-20T01:05:24.462Z 26 Acked 1025 Write 6 Sent Done Done false
10062024-04-20T01:05:24.462Z 27 Acked 1026 Read 7 Sent Done Done false
10072024-04-20T01:05:24.462Z 28 Acked 1027 Read 4 Sent Done Done false
10082024-04-20T01:05:24.462Z 29 Acked 1028 Write 9 Sent Sent Sent false
10092024-04-20T01:05:24.462Z 30 Acked 1029 Write 1 Sent Sent Sent false
10102024-04-20T01:05:24.462Z STATES DS:0 DS:1 DS:2 TOTAL
10112024-04-20T01:05:24.462Z New 0 0 0 0
10122024-04-20T01:05:24.462Z Sent 27 2 2 31
10132024-04-20T01:05:24.462Z Done 3 28 28 59
10142024-04-20T01:05:24.463Z Skipped 0 0 0 0
10152024-04-20T01:05:24.463Z Error 0 0 0 0
10162024-04-20T01:05:24.463ZLast Flush: 0 1011 1011
10172024-04-20T01:05:24.463ZDownstairs last five completed:
10182024-04-20T01:05:24.463ZUpstairs last five completed: 30 29 28 27 26
10192024-04-20T01:05:24.464Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:05:22.968423415Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1200}
10202024-04-20T01:05:24.464ZWrote out file "/tmp/test_repair_verify.data"
10212024-04-20T01:05:24.477Z
10222024-04-20T01:05:24.477ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10232024-04-20T01:05:24.488ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10242024-04-20T01:05:24.488Z 0 000-019 3 8 8 4 14 14 T F F <---
10252024-04-20T01:05:24.489Z 1 020-039 4 4 4 8 8 8 F F F
10262024-04-20T01:05:24.489Z 2 040-059 7 7 7 13 13 13 F F F
10272024-04-20T01:05:24.489Z 3 060-079 6 6 6 12 12 12 F F F
10282024-04-20T01:05:24.489Z 4 080-099 1 8 8 1 14 14 T F F <---
10292024-04-20T01:05:24.489Z 5 100-119 6 6 6 11 11 11 F F F
10302024-04-20T01:05:24.489Z 6 120-139 1 1 1 1 1 1 F F F
10312024-04-20T01:05:24.489Z 7 140-159 1 1 1 1 1 1 F T T <---
10322024-04-20T01:05:24.489Z 8 160-179 2 8 8 3 14 14 F T T <---
10332024-04-20T01:05:24.489Z 9 180-199 6 6 6 12 12 12 F T T <---
10342024-04-20T01:05:24.489Z 10 200-219 1 1 1 1 1 1 F F F
10352024-04-20T01:05:24.489Z 11 220-239 6 8 8 11 14 14 F F F <---
10362024-04-20T01:05:24.496Z 12 240-259 6 8 8 12 14 14 F T T <---
10372024-04-20T01:05:24.496Z 13 260-279 1 1 1 1 1 1 F F F
10382024-04-20T01:05:24.496Z 14 280-299 4 4 4 7 7 7 F T T <---
10392024-04-20T01:05:24.496Z 15 300-319 7 7 7 13 13 13 F F F
10402024-04-20T01:05:24.496Z 16 320-339 7 7 7 13 13 13 F T T <---
10412024-04-20T01:05:24.496Z 17 340-359 1 1 1 1 1 1 F T T <---
10422024-04-20T01:05:24.497Z 18 360-379 6 6 6 12 12 12 F F F
10432024-04-20T01:05:24.497Z 19 380-399 6 6 6 12 12 12 F F F
10442024-04-20T01:05:24.497Z 20 400-419 7 7 7 13 13 13 F F F
10452024-04-20T01:05:24.497Z 21 420-439 5 5 5 10 10 10 F T T <---
10462024-04-20T01:05:24.497Z 22 440-459 4 4 4 7 7 7 F T T <---
10472024-04-20T01:05:24.497Z 23 460-479 4 4 4 9 9 9 F F F
10482024-04-20T01:05:24.497Z 24 480-499 1 1 1 1 1 1 F T T <---
10492024-04-20T01:05:24.497Z 25 500-519 2 8 8 3 15 15 F T T <---
10502024-04-20T01:05:24.497Z 26 520-539 6 6 6 11 11 11 F F F
10512024-04-20T01:05:24.497Z 27 540-559 6 6 6 11 11 11 F F F
10522024-04-20T01:05:24.497Z 28 560-579 2 2 2 3 3 3 F T T <---
10532024-04-20T01:05:24.497Z 29 580-599 1 1 1 1 1 1 F T T <---
10542024-04-20T01:05:24.498ZMax gen: 8, Max flush: 15
10552024-04-20T01:05:24.498ZError: Difference in extent metadata found!
10562024-04-20T01:05:24.498ZA Difference in extent metadata is expected here
10572024-04-20T01:05:24.498ZOn loop 4
10582024-04-20T01:05:24.498Z
10592024-04-20T01:05:24.498ZVerifying data now
10602024-04-20T01:05:29.210ZNo change after verify
10612024-04-20T01:05:29.211ZLoop: 4 Downstairs dump after verify (and repair):
10622024-04-20T01:05:29.225ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10632024-04-20T01:05:29.225Z 0 000-019 8 8 8 14 14 14 F F F
10642024-04-20T01:05:29.225Z 1 020-039 4 4 4 8 8 8 F F F
10652024-04-20T01:05:29.225Z 2 040-059 7 7 7 13 13 13 F F F
10662024-04-20T01:05:29.225Z 3 060-079 6 6 6 12 12 12 F F F
10672024-04-20T01:05:29.226Z 4 080-099 8 8 8 14 14 14 F F F
10682024-04-20T01:05:29.226Z 5 100-119 6 6 6 11 11 11 F F F
10692024-04-20T01:05:29.226Z 6 120-139 1 1 1 1 1 1 F F F
10702024-04-20T01:05:29.226Z 7 140-159 9 9 9 16 16 16 F F F
10712024-04-20T01:05:29.226Z 8 160-179 9 9 9 16 16 16 F F F
10722024-04-20T01:05:29.232Z 9 180-199 9 9 9 16 16 16 F F F
10732024-04-20T01:05:29.232Z 10 200-219 1 1 1 1 1 1 F F F
10742024-04-20T01:05:29.232Z 11 220-239 8 8 8 14 14 14 F F F
10752024-04-20T01:05:29.232Z 12 240-259 9 9 9 16 16 16 F F F
10762024-04-20T01:05:29.232Z 13 260-279 1 1 1 1 1 1 F F F
10772024-04-20T01:05:29.233Z 14 280-299 9 9 9 16 16 16 F F F
10782024-04-20T01:05:29.233Z 15 300-319 7 7 7 13 13 13 F F F
10792024-04-20T01:05:29.233Z 16 320-339 9 9 9 16 16 16 F F F
10802024-04-20T01:05:29.233Z 17 340-359 9 9 9 16 16 16 F F F
10812024-04-20T01:05:29.233Z 18 360-379 6 6 6 12 12 12 F F F
10822024-04-20T01:05:29.233Z 19 380-399 6 6 6 12 12 12 F F F
10832024-04-20T01:05:29.234Z 20 400-419 7 7 7 13 13 13 F F F
10842024-04-20T01:05:29.234Z 21 420-439 9 9 9 16 16 16 F F F
10852024-04-20T01:05:29.234Z 22 440-459 9 9 9 16 16 16 F F F
10862024-04-20T01:05:29.234Z 23 460-479 4 4 4 9 9 9 F F F
10872024-04-20T01:05:29.234Z 24 480-499 9 9 9 16 16 16 F F F
10882024-04-20T01:05:29.234Z 25 500-519 9 9 9 16 16 16 F F F
10892024-04-20T01:05:29.234Z 26 520-539 6 6 6 11 11 11 F F F
10902024-04-20T01:05:29.234Z 27 540-559 6 6 6 11 11 11 F F F
10912024-04-20T01:05:29.234Z 28 560-579 9 9 9 16 16 16 F F F
10922024-04-20T01:05:29.234Z 29 580-599 9 9 9 16 16 16 F F F
10932024-04-20T01:05:29.235ZMax gen: 9, Max flush: 16
10942024-04-20T01:05:29.235Z
10952024-04-20T01:05:29.235ZBegin loop 5
10962024-04-20T01:05:29.235ZDownstairs to restart: 2
10972024-04-20T01:05:29.252Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.757434696Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208}
10982024-04-20T01:05:29.252Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.757720846Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208}
10992024-04-20T01:05:29.252Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.757742256Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208}
11002024-04-20T01:05:29.268Z{"msg":"Crucible c18684a0-1078-4d4b-9273-890d3a9f1528 has session id: 9bedcf09-96a7-41de-a3ef-812309b67e53","v":0,"name":"crucible","level":30Crucible runtime is spawned
11012024-04-20T01:05:29.268Z,"time":"2024-04-20T01:05:27.757769566Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11022024-04-20T01:05:29.268Z{"msg":"Upstairs opts: Upstairs UUID: c18684a0-1078-4d4b-9273-890d3a9f1528, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.757873957Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11032024-04-20T01:05:29.269Z{"msg":"Crucible stats registered with UUID: c18684a0-1078-4d4b-9273-890d3a9f1528","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.757927477Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11042024-04-20T01:05:29.269Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.757988387Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208}
11052024-04-20T01:05:29.269Z{"msg":"c18684a0-1078-4d4b-9273-890d3a9f1528 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758075897Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11062024-04-20T01:05:29.269Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758100697Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"0","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11072024-04-20T01:05:29.269Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758126597Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"0","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11082024-04-20T01:05:29.269Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758164027Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"1","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11092024-04-20T01:05:29.269Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758205887Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"1","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11102024-04-20T01:05:29.269Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758241358Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"2","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11112024-04-20T01:05:29.270Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758268538Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"2","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11122024-04-20T01:05:29.270Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758356008Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11132024-04-20T01:05:29.270Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758405828Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11142024-04-20T01:05:29.270Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758457498Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11152024-04-20T01:05:29.270Z{"msg":"ds_connection connected from Ok(127.0.0.1:52633)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758509768Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11162024-04-20T01:05:29.270Z{"msg":"ds_connection connected from Ok(127.0.0.1:49723)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758553608Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11172024-04-20T01:05:29.270Z{"msg":"ds_connection connected from Ok(127.0.0.1:33044)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.758598528Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11182024-04-20T01:05:29.270Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.759140029Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"0","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11192024-04-20T01:05:29.270Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.75919578Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"0","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11202024-04-20T01:05:29.270Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.75922423Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"1","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11212024-04-20T01:05:29.270Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.75926654Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"1","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11222024-04-20T01:05:29.270Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.759681951Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"1","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11232024-04-20T01:05:29.271Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.759824061Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"2","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11242024-04-20T01:05:29.271Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.759856671Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"2","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11252024-04-20T01:05:29.271Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.759930691Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"0","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11262024-04-20T01:05:29.271Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.760130832Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"1","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11272024-04-20T01:05:29.271Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.760188742Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"1","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11282024-04-20T01:05:29.271Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.760267102Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11292024-04-20T01:05:29.271Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.760308622Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"0","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11302024-04-20T01:05:29.271Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.760325352Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"0","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11312024-04-20T01:05:29.271Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.760346932Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11322024-04-20T01:05:29.271Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.760390112Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"2","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11332024-04-20T01:05:29.271Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.760776433Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"2","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11342024-04-20T01:05:29.272Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.760808903Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"2","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11352024-04-20T01:05:29.272Z{"msg":"[0]R flush_numbers[0..12]: [14, 8, 13, 12, 14, 11, 1, 16, 16, 16, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.760885693Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11362024-04-20T01:05:29.272Z{"msg":"[0]R generation[0..12]: [8, 4, 7, 6, 8, 6, 1, 9, 9, 9, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.760912444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11372024-04-20T01:05:29.272Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.760961484Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11382024-04-20T01:05:29.272Z{"msg":"[1]R flush_numbers[0..12]: [14, 8, 13, 12, 14, 11, 1, 16, 16, 16, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761007644Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11392024-04-20T01:05:29.272ZWait for a query_work_queue command to finish before sending IO
11402024-04-20T01:05:29.272Z{"msg":"[1]R generation[0..12]: [8, 4, 7, 6, 8, 6, 1, 9, 9, 9, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761048114Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11412024-04-20T01:05:29.273Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761094384Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11422024-04-20T01:05:29.273Z{"msg":"[2]R flush_numbers[0..12]: [14, 8, 13, 12, 14, 11, 1, 16, 16, 16, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761134184Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11432024-04-20T01:05:29.273Z{"msg":"[2]R generation[0..12]: [8, 4, 7, 6, 8, 6, 1, 9, 9, 9, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761179784Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11442024-04-20T01:05:29.273Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761218474Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11452024-04-20T01:05:29.274Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761254664Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11462024-04-20T01:05:29.274Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761284694Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11472024-04-20T01:05:29.274Z{"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761320574Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11482024-04-20T01:05:29.274Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761356594Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"":"downstairs","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11492024-04-20T01:05:29.274Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11502024-04-20T01:05:29.274Z,"time":"2024-04-20T01:05:27.761392934Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11512024-04-20T01:05:29.275Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761428894Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"0","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11522024-04-20T01:05:29.275Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761455645Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"1","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11532024-04-20T01:05:29.275Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761470535Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"client":"2","":"downstairs","session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11542024-04-20T01:05:29.275Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761491565Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11552024-04-20T01:05:29.275Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761504445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11562024-04-20T01:05:29.275Z{"msg":"c18684a0-1078-4d4b-9273-890d3a9f1528 is now active with session: 9bedcf09-96a7-41de-a3ef-812309b67e53","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761517315Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11572024-04-20T01:05:29.275Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761537075Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208,"session_id":"9bedcf09-96a7-41de-a3ef-812309b67e53"}
11582024-04-20T01:05:29.275Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:27.761549575Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208}
11592024-04-20T01:05:29.276ZLoading write count information from file "/tmp/test_repair_verify.data"
11602024-04-20T01:05:29.276ZRun Repair workload
11612024-04-20T01:05:29.276Z01/30 Write block 433 len 16384 data: 2 2 2 3
11622024-04-20T01:05:29.276Z02/30 Write block 410 len 28672 data: 2 3 3 4 4 4 3
11632024-04-20T01:05:29.276Z03/30 Write block 89 len 8192 data: 2 2
11642024-04-20T01:05:29.276Z04/30 Write block 131 len 24576 data: 2 2 2 2 2 2
11652024-04-20T01:05:29.276Z05/30 Read block 153 len 20480
11662024-04-20T01:05:29.276Z06/30 Read block 20 len 4096
11672024-04-20T01:05:29.292Z07/30 Write block 26 len 20480 data: 2 2 3 3 2
11682024-04-20T01:05:29.292Z08/30 Read block 344 len 24576
11692024-04-20T01:05:29.292Z09/30 Read block 350 len 28672
11702024-04-20T01:05:29.293Z10/30 Write block 591 len 8192 data: 2 2
11712024-04-20T01:05:29.293Z11/30 Read block 379 len 40960
11722024-04-20T01:05:29.293Z12/30 Read block 451 len 12288
11732024-04-20T01:05:29.293Z13/30 Write block 273 len 12288 data: 2 2 2
11742024-04-20T01:05:29.293Z14/30 Write block 318 len 8192 data: 3 3
11752024-04-20T01:05:29.293Z15/30 Read block 552 len 40960
11762024-04-20T01:05:29.293Z16/30 Read block 283 len 8192
11772024-04-20T01:05:29.294Z17/30 Read block 467 len 28672
11782024-04-20T01:05:29.294Z18/30 Flush
11792024-04-20T01:05:29.294Z19/30 Flush
11802024-04-20T01:05:29.294Z20/30 Write block 501 len 40960 data: 3 3 4 4 4 3 3 3 4 3
11812024-04-20T01:05:29.294Z21/30 Read block 292 len 16384
11822024-04-20T01:05:29.295Z22/30 Write block 66 len 16384 data: 3 2 2 3
11832024-04-20T01:05:29.295Z23/30 Write block 557 len 12288 data: 2 2 2
11842024-04-20T01:05:29.295Z24/30 Write block 256 len 16384 data: 3 3 3 3
11852024-04-20T01:05:29.295Z25/30 Write block 513 len 36864 data: 4 3 3 3 3 3 3 3 2
11862024-04-20T01:05:29.295Z26/30 Read block 125 len 24576
11872024-04-20T01:05:29.295Z27/30 Write block 77 len 20480 data: 2 2 2 2 2
11882024-04-20T01:05:29.295Z28/30 Write block 164 len 8192 data: 2 2
11892024-04-20T01:05:29.295Z29/30 Write block 420 len 8192 data: 3 3
11902024-04-20T01:05:29.296Z30/30 Read block 112 len 36864
11912024-04-20T01:05:29.296Z----------------------------------------------------------------
11922024-04-20T01:05:29.296Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
11932024-04-20T01:05:29.296ZStates: Active Active Active
11942024-04-20T01:05:29.296ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11952024-04-20T01:05:29.297Z 1 Acked 1000 Write 4 Done Done Done false
11962024-04-20T01:05:29.297Z 2 Acked 1001 Write 7 Done Done Done false
11972024-04-20T01:05:29.297Z 3 Acked 1002 Write 2 Done Done Done false
11982024-04-20T01:05:29.297Z 4 Acked 1003 Write 6 Done Done Done false
11992024-04-20T01:05:29.297Z 5 Acked 1004 Read 5 Done Done Done false
12002024-04-20T01:05:29.297Z 6 Acked 1005 Read 1 Done Done Done false
12012024-04-20T01:05:29.298Z 7 Acked 1006 Write 5 Done Done Done false
12022024-04-20T01:05:29.298Z 8 Acked 1007 Read 6 Done Done Done false
12032024-04-20T01:05:29.298Z 9 Acked 1008 Read 7 Done Done Sent false
12042024-04-20T01:05:29.298Z 10 Acked 1009 Write 2 Done Done Sent false
12052024-04-20T01:05:29.298Z 11 Acked 1010 Read 10 Done Done Sent false
12062024-04-20T01:05:29.298Z 12 Acked 1011 Read 3 Done Done Sent false
12072024-04-20T01:05:29.298Z 13 Acked 1012 Write 3 Done Done Sent false
12082024-04-20T01:05:29.298Z 14 Acked 1013 Write 2 Done Done Sent false
12092024-04-20T01:05:29.298Z 15 Acked 1014 Read 10 Done Done Sent false
12102024-04-20T01:05:29.298Z 16 Acked 1015 Read 2 Done Done Sent false
12112024-04-20T01:05:29.298Z 17 Acked 1016 Read 7 Done Done Sent false
12122024-04-20T01:05:29.299Z 18 Acked 1017 Flush 0 Done Done Sent false
12132024-04-20T01:05:29.299Z 19 Acked 1018 Flush 0 Done Done Sent false
12142024-04-20T01:05:29.299Z 20 Acked 1019 Write 10 Done Done Sent false
12152024-04-20T01:05:29.299Z 21 Acked 1020 Read 4 Done Done Sent false
12162024-04-20T01:05:29.299Z 22 Acked 1021 Write 4 Done Done Sent false
12172024-04-20T01:05:29.300Z 23 Acked 1022 Write 3 Done Done Sent false
12182024-04-20T01:05:29.300Z 24 Acked 1023 Write 4 Done Done Sent false
12192024-04-20T01:05:29.300Z 25 Acked 1024 Write 9 Done Done Sent false
12202024-04-20T01:05:29.300Z 26 Acked 1025 Read 6 Done Done Sent false
12212024-04-20T01:05:29.300Z 27 Acked 1026 Write 5 Done Done Sent false
12222024-04-20T01:05:29.301Z 28 Acked 1027 Write 2 Done Done Sent false
12232024-04-20T01:05:29.301Z 29 Acked 1028 Write 2 Done Done Sent false
12242024-04-20T01:05:29.301Z 30 Acked 1029 Read 9 Done Done Sent false
12252024-04-20T01:05:29.301Z STATES DS:0 DS:1 DS:2 TOTAL
12262024-04-20T01:05:29.301Z New 0 0 0 0
12272024-04-20T01:05:29.302Z Sent 0 0 22 22
12282024-04-20T01:05:29.302Z Done 30 30 8 68
12292024-04-20T01:05:29.302Z Skipped 0 0 0 0
12302024-04-20T01:05:29.302Z Error 0 0 0 0
12312024-04-20T01:05:29.302ZLast Flush: 1018 1018 0
12322024-04-20T01:05:29.303ZDownstairs last five completed:
12332024-04-20T01:05:29.303ZUpstairs last five completed: 30 29 28 27 26
12342024-04-20T01:05:29.303Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:05:27.789600907Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1208}
12352024-04-20T01:05:29.303ZWrote out file "/tmp/test_repair_verify.data"
12362024-04-20T01:05:29.303Z
12372024-04-20T01:05:29.304ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12382024-04-20T01:05:29.319ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12392024-04-20T01:05:29.320Z 0 000-019 8 8 8 14 14 14 F F F
12402024-04-20T01:05:29.320Z 1 020-039 10 10 4 17 17 8 F F T <---
12412024-04-20T01:05:29.320Z 2 040-059 7 7 7 13 13 13 F F F
12422024-04-20T01:05:29.320Z 3 060-079 6 6 6 12 12 12 T T F <---
12432024-04-20T01:05:29.321Z 4 080-099 10 10 8 17 17 14 T T T <---
12442024-04-20T01:05:29.321Z 5 100-119 6 6 6 11 11 11 F F F
12452024-04-20T01:05:29.321Z 6 120-139 10 10 1 17 17 1 F F T <---
12462024-04-20T01:05:29.321Z 7 140-159 9 9 9 16 16 16 F F F
12472024-04-20T01:05:29.321Z 8 160-179 9 9 9 16 16 16 T T F <---
12482024-04-20T01:05:29.321Z 9 180-199 9 9 9 16 16 16 F F F
12492024-04-20T01:05:29.321Z 10 200-219 1 1 1 1 1 1 F F F
12502024-04-20T01:05:29.321Z 11 220-239 8 8 8 14 14 14 F F F
12512024-04-20T01:05:29.322Z 12 240-259 9 9 9 16 16 16 T T F <---
12522024-04-20T01:05:29.322Z 13 260-279 10 10 1 17 17 1 F F F <---
12532024-04-20T01:05:29.322Z 14 280-299 9 9 9 16 16 16 F F F
12542024-04-20T01:05:29.322Z 15 300-319 10 10 7 17 17 13 F F F <---
12552024-04-20T01:05:29.322Z 16 320-339 9 9 9 16 16 16 F F F
12562024-04-20T01:05:29.322Z 17 340-359 9 9 9 16 16 16 F F F
12572024-04-20T01:05:29.322Z 18 360-379 6 6 6 12 12 12 F F F
12582024-04-20T01:05:29.323Z 19 380-399 6 6 6 12 12 12 F F F
12592024-04-20T01:05:29.323Z 20 400-419 10 10 7 17 17 13 F F T <---
12602024-04-20T01:05:29.323Z 21 420-439 10 10 9 17 17 16 T T T <---
12612024-04-20T01:05:29.323Z 22 440-459 9 9 9 16 16 16 F F F
12622024-04-20T01:05:29.323Z 23 460-479 4 4 4 9 9 9 F F F
12632024-04-20T01:05:29.323Z 24 480-499 9 9 9 16 16 16 F F F
12642024-04-20T01:05:29.324Z 25 500-519 9 9 9 16 16 16 T T F <---
12652024-04-20T01:05:29.324Z 26 520-539 6 6 6 11 11 11 T T F <---
12662024-04-20T01:05:29.324Z 27 540-559 6 6 6 11 11 11 T T F <---
12672024-04-20T01:05:29.324Z 28 560-579 9 9 9 16 16 16 F F F
12682024-04-20T01:05:29.324Z 29 580-599 10 10 9 17 17 16 F F F <---
12692024-04-20T01:05:29.324ZMax gen: 10, Max flush: 17
12702024-04-20T01:05:29.324ZError: Difference in extent metadata found!
12712024-04-20T01:05:29.325ZA Difference in extent metadata is expected here
12722024-04-20T01:05:29.325ZOn loop 5
12732024-04-20T01:05:29.325Z
12742024-04-20T01:05:29.325ZVerifying data now
12752024-04-20T01:05:43.939ZNo change after verify
12762024-04-20T01:05:43.939ZLoop: 5 Downstairs dump after verify (and repair):
12772024-04-20T01:05:43.952ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12782024-04-20T01:05:43.953Z 0 000-019 8 8 8 14 14 14 F F F
12792024-04-20T01:05:43.953Z 1 020-039 10 10 10 17 17 17 F F F
12802024-04-20T01:05:43.953Z 2 040-059 7 7 7 13 13 13 F F F
12812024-04-20T01:05:43.953Z 3 060-079 11 11 11 18 18 18 F F F
12822024-04-20T01:05:43.953Z 4 080-099 11 11 11 18 18 18 F F F
12832024-04-20T01:05:43.953Z 5 100-119 6 6 6 11 11 11 F F F
12842024-04-20T01:05:43.956Z 6 120-139 10 10 10 17 17 17 F F F
12852024-04-20T01:05:43.956Z 7 140-159 9 9 9 16 16 16 F F F
12862024-04-20T01:05:43.957Z 8 160-179 11 11 11 18 18 18 F F F
12872024-04-20T01:05:43.957Z 9 180-199 9 9 9 16 16 16 F F F
12882024-04-20T01:05:43.957Z 10 200-219 1 1 1 1 1 1 F F F
12892024-04-20T01:05:43.957Z 11 220-239 8 8 8 14 14 14 F F F
12902024-04-20T01:05:43.957Z 12 240-259 11 11 11 18 18 18 F F F
12912024-04-20T01:05:43.957Z 13 260-279 10 10 10 17 17 17 F F F
12922024-04-20T01:05:43.957Z 14 280-299 9 9 9 16 16 16 F F F
12932024-04-20T01:05:43.957Z 15 300-319 10 10 10 17 17 17 F F F
12942024-04-20T01:05:43.957Z 16 320-339 9 9 9 16 16 16 F F F
12952024-04-20T01:05:43.957Z 17 340-359 9 9 9 16 16 16 F F F
12962024-04-20T01:05:43.957Z 18 360-379 6 6 6 12 12 12 F F F
12972024-04-20T01:05:43.957Z 19 380-399 6 6 6 12 12 12 F F F
12982024-04-20T01:05:43.957Z 20 400-419 10 10 10 17 17 17 F F F
12992024-04-20T01:05:43.957Z 21 420-439 11 11 11 18 18 18 F F F
13002024-04-20T01:05:43.957Z 22 440-459 9 9 9 16 16 16 F F F
13012024-04-20T01:05:43.957Z 23 460-479 4 4 4 9 9 9 F F F
13022024-04-20T01:05:43.957Z 24 480-499 9 9 9 16 16 16 F F F
13032024-04-20T01:05:43.957Z 25 500-519 11 11 11 18 18 18 F F F
13042024-04-20T01:05:43.957Z 26 520-539 11 11 11 18 18 18 F F F
13052024-04-20T01:05:43.957Z 27 540-559 11 11 11 18 18 18 F F F
13062024-04-20T01:05:43.957Z 28 560-579 9 9 9 16 16 16 F F F
13072024-04-20T01:05:43.957Z 29 580-599 10 10 10 17 17 17 F F F
13082024-04-20T01:05:43.957ZMax gen: 11, Max flush: 18
13092024-04-20T01:05:43.958Z
13102024-04-20T01:05:43.958ZBegin loop 6
13112024-04-20T01:05:43.958ZDownstairs to restart: 1
13122024-04-20T01:05:44.380Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486205902Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216}
13132024-04-20T01:05:44.380Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486429412Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216}
13142024-04-20T01:05:44.380Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486446352Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216}
13152024-04-20T01:05:44.380Z{"msg":"Crucible fdb119ee-b500-4bf7-948f-9c80893d9646 has session id: aba0a414-472b-40e1-9c52-1763de290fc6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486460883Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13162024-04-20T01:05:44.389Z{"msg":"Upstairs opts: Upstairs UUID: fdb119ee-b500-4bf7-948f-9c80893d9646, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486506852Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13172024-04-20T01:05:44.389Z{"msg":"Crucible stats registered with UUID: fdb119ee-b500-4bf7-948f-9c80893d9646","v":0,"name":"crucible","level":30Crucible runtime is spawned
13182024-04-20T01:05:44.389Z,"time":"2024-04-20T01:05:42.486545563Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13192024-04-20T01:05:44.389Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486646733Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216}
13202024-04-20T01:05:44.389Z{"msg":"fdb119ee-b500-4bf7-948f-9c80893d9646 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486732093Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13212024-04-20T01:05:44.389Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486777083Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"0","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13222024-04-20T01:05:44.389Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486822533Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"0","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13232024-04-20T01:05:44.389Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486852183Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"1","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13242024-04-20T01:05:44.389Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486887753Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"1","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13252024-04-20T01:05:44.389Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486903824Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"2","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13262024-04-20T01:05:44.389Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486919664Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"2","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13272024-04-20T01:05:44.389Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486952713Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13282024-04-20T01:05:44.389Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.486980114Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13292024-04-20T01:05:44.390Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.487022814Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13302024-04-20T01:05:44.390Z{"msg":"ds_connection connected from Ok(127.0.0.1:58181)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.487215184Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13312024-04-20T01:05:44.390Z{"msg":"ds_connection connected from Ok(127.0.0.1:42764)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.487271864Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13322024-04-20T01:05:44.390Z{"msg":"ds_connection connected from Ok(127.0.0.1:47517)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.487313444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13332024-04-20T01:05:44.390Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.487944736Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"2","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13342024-04-20T01:05:44.390Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.487973376Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"2","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13352024-04-20T01:05:44.390Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.487988966Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"0","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13362024-04-20T01:05:44.390Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.488007926Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"0","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13372024-04-20T01:05:44.390Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.488059946Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"1","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13382024-04-20T01:05:44.390Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.488103496Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"1","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13392024-04-20T01:05:44.390Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.488545677Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"2","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13402024-04-20T01:05:44.390Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.488588297Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"0","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13412024-04-20T01:05:44.390Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.488610387Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"1","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13422024-04-20T01:05:44.390Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489053608Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"2","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13432024-04-20T01:05:44.390Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489091318Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"2","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13442024-04-20T01:05:44.390Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489108458Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13452024-04-20T01:05:44.390Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489175918Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"1","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13462024-04-20T01:05:44.390Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489212238Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"1","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13472024-04-20T01:05:44.390Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489227219Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13482024-04-20T01:05:44.390Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489240369Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"0","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13492024-04-20T01:05:44.390Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489262489Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"0","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13502024-04-20T01:05:44.390Z{"msg":"[0]R flush_numbers[0..12]: [14, 17, 13, 18, 18, 11, 17, 16, 18, 16, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489276769Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13512024-04-20T01:05:44.391Z{"msg":"[0]R generation[0..12]: [8, 10, 7, 11, 11, 6, 10, 9, 11, 9, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489297609Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13522024-04-20T01:05:44.391Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489312209Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13532024-04-20T01:05:44.391Z{"msg":"[1]R flush_numbers[0..12]: [14, 17, 13, 18, 18, 11, 17, 16, 18, 16, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489334039Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13542024-04-20T01:05:44.391Z{"msg":"[1]R generation[0..12]: [8, 10, 7, 11, 11, 6, 10, 9, 11, 9, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489347979Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13552024-04-20T01:05:44.391Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
13562024-04-20T01:05:44.391Z,"time":"2024-04-20T01:05:42.489369219Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13572024-04-20T01:05:44.391Z{"msg":"[2]R flush_numbers[0..12]: [14, 17, 13, 18, 18, 11, 17, 16, 18, 16, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489386929Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13582024-04-20T01:05:44.391Z{"msg":"[2]R generation[0..12]: [8, 10, 7, 11, 11, 6, 10, 9, 11, 9, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489407609Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13592024-04-20T01:05:44.391Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489422049Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13602024-04-20T01:05:44.391Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489441489Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13612024-04-20T01:05:44.391Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489455119Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13622024-04-20T01:05:44.391Z{"msg":"Next flush: 19","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489474999Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13632024-04-20T01:05:44.391Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489488329Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"":"downstairs","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13642024-04-20T01:05:44.391Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489508359Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13652024-04-20T01:05:44.391Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489520449Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"0","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13662024-04-20T01:05:44.391Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489540389Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"1","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13672024-04-20T01:05:44.391Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489553849Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"client":"2","":"downstairs","session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13682024-04-20T01:05:44.391Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489567679Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13692024-04-20T01:05:44.391Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.48958743Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13702024-04-20T01:05:44.391Z{"msg":"fdb119ee-b500-4bf7-948f-9c80893d9646 is now active with session: aba0a414-472b-40e1-9c52-1763de290fc6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489600479Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13712024-04-20T01:05:44.391Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.48961961Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216,"session_id":"aba0a414-472b-40e1-9c52-1763de290fc6"}
13722024-04-20T01:05:44.391Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:42.489632119Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216}
13732024-04-20T01:05:44.392ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13742024-04-20T01:05:44.392ZLoading write count information from file "/tmp/test_repair_verify.data"
13752024-04-20T01:05:44.392ZRun Repair workload
13762024-04-20T01:05:44.392Z01/30 Write block 574 len 24576 data: 2 2 2 2 2 2
13772024-04-20T01:05:44.392Z02/30 Read block 103 len 4096
13782024-04-20T01:05:44.392Z03/30 Read block 561 len 8192
13792024-04-20T01:05:44.392Z04/30 Read block 8 len 28672
13802024-04-20T01:05:44.392Z05/30 Read block 229 len 4096
13812024-04-20T01:05:44.392Z06/30 Read block 299 len 8192
13822024-04-20T01:05:44.392Z07/30 Flush
13832024-04-20T01:05:44.392Z08/30 Write block 586 len 40960 data: 2 2 2 2 2 3 3 2 3 3
13842024-04-20T01:05:44.392Z09/30 Write block 442 len 20480 data: 4 5 4 4 4
13852024-04-20T01:05:44.392Z10/30 Flush
13862024-04-20T01:05:44.392Z11/30 Write block 519 len 20480 data: 4 4 3 2 3
13872024-04-20T01:05:44.392Z12/30 Read block 412 len 16384
13882024-04-20T01:05:44.392Z13/30 Read block 371 len 20480
13892024-04-20T01:05:44.392Z14/30 Read block 502 len 12288
13902024-04-20T01:05:44.392Z15/30 Write block 64 len 40960 data: 3 3 4 3 3 4 3 3 3 2
13912024-04-20T01:05:44.392Z16/30 Write block 220 len 12288 data: 2 2 3
13922024-04-20T01:05:44.392Z17/30 Write block 431 len 20480 data: 2 2 3 3 3
13932024-04-20T01:05:44.392Z18/30 Write block 343 len 4096 data: 3
13942024-04-20T01:05:44.392Z19/30 Read block 80 len 20480
13952024-04-20T01:05:44.392Z20/30 Write block 134 len 40960 data: 3 3 3 2 2 2 2 2 2 2
13962024-04-20T01:05:44.392Z21/30 Flush
13972024-04-20T01:05:44.393Z22/30 Write block 39 len 12288 data: 2 2 2
13982024-04-20T01:05:44.393Z23/30 Write block 524 len 12288 data: 3 3 2
13992024-04-20T01:05:44.393Z24/30 Read block 383 len 32768
14002024-04-20T01:05:44.393Z25/30 Write block 116 len 12288 data: 2 2 2
14012024-04-20T01:05:44.393Z26/30 Read block 376 len 4096
14022024-04-20T01:05:44.393Z27/30 Write block 135 len 24576 data: 4 4 3 3 3 3
14032024-04-20T01:05:44.393Z28/30 Read block 210 len 40960
14042024-04-20T01:05:44.393Z29/30 Write block 221 len 4096 data: 3
14052024-04-20T01:05:44.393Z30/30 Read block 591 len 4096
14062024-04-20T01:05:44.393Z----------------------------------------------------------------
14072024-04-20T01:05:44.393Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14082024-04-20T01:05:44.393ZStates: Active Active Active
14092024-04-20T01:05:44.393ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14102024-04-20T01:05:44.393Z 1 Acked 1000 Write 6 Done Done Done false
14112024-04-20T01:05:44.393Z 2 Acked 1001 Read 1 Done Sent Done false
14122024-04-20T01:05:44.393Z 3 Acked 1002 Read 2 Done Sent Done false
14132024-04-20T01:05:44.393Z 4 Acked 1003 Read 7 Done Sent Done false
14142024-04-20T01:05:44.393Z 5 Acked 1004 Read 1 Done Sent Done false
14152024-04-20T01:05:44.393Z 6 Acked 1005 Read 2 Done Sent Done false
14162024-04-20T01:05:44.399Z 7 Acked 1006 Flush 0 Done Sent Done false
14172024-04-20T01:05:44.399Z 8 Acked 1007 Write 10 Done Sent Done false
14182024-04-20T01:05:44.399Z 9 Acked 1008 Write 5 Done Sent Done false
14192024-04-20T01:05:44.399Z 10 Acked 1009 Flush 0 Done Sent Done false
14202024-04-20T01:05:44.399Z 11 Acked 1010 Write 5 Done Sent Done false
14212024-04-20T01:05:44.399Z 12 Acked 1011 Read 4 Done Sent Done false
14222024-04-20T01:05:44.400Z 13 Acked 1012 Read 5 Done Sent Done false
14232024-04-20T01:05:44.400Z 14 Acked 1013 Read 3 Done Sent Done false
14242024-04-20T01:05:44.400Z 15 Acked 1014 Write 10 Done Sent Done false
14252024-04-20T01:05:44.400Z 16 Acked 1015 Write 3 Done Sent Done false
14262024-04-20T01:05:44.400Z 17 Acked 1016 Write 5 Done Sent Done false
14272024-04-20T01:05:44.400Z 18 Acked 1017 Write 1 Done Sent Done false
14282024-04-20T01:05:44.400Z 19 Acked 1018 Read 5 Done Sent Done false
14292024-04-20T01:05:44.400Z 20 Acked 1019 Write 10 Done Sent Done false
14302024-04-20T01:05:44.400Z 21 Acked 1020 Flush 0 Done Sent Done false
14312024-04-20T01:05:44.400Z 22 Acked 1021 Write 3 Done Sent Done false
14322024-04-20T01:05:44.400Z 23 Acked 1022 Write 3 Done Sent Done false
14332024-04-20T01:05:44.400Z 24 Acked 1023 Read 8 Done Sent Done false
14342024-04-20T01:05:44.400Z 25 Acked 1024 Write 3 Done Sent Done false
14352024-04-20T01:05:44.400Z 26 Acked 1025 Read 1 Done Sent Done false
14362024-04-20T01:05:44.400Z 27 Acked 1026 Write 6 Done Sent Done false
14372024-04-20T01:05:44.400Z 28 Acked 1027 Read 10 Done Sent Done false
14382024-04-20T01:05:44.400Z 29 Acked 1028 Write 1 Done Sent Done false
14392024-04-20T01:05:44.400Z 30 Acked 1029 Read 1 Done Sent Done false
14402024-04-20T01:05:44.400Z STATES DS:0 DS:1 DS:2 TOTAL
14412024-04-20T01:05:44.400Z New 0 0 0 0
14422024-04-20T01:05:44.400Z Sent 0 29 0 29
14432024-04-20T01:05:44.400Z Done 30 1 30 61
14442024-04-20T01:05:44.401Z Skipped 0 0 0 0
14452024-04-20T01:05:44.401Z Error 0 0 0 0
14462024-04-20T01:05:44.401ZLast Flush: 1020 0 1020
14472024-04-20T01:05:44.401ZDownstairs last five completed:
14482024-04-20T01:05:44.401ZUpstairs last five completed: 30 29 28 27 26
14492024-04-20T01:05:44.401Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:05:42.519738437Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1216}
14502024-04-20T01:05:44.401ZWrote out file "/tmp/test_repair_verify.data"
14512024-04-20T01:05:44.401Z
14522024-04-20T01:05:44.401ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14532024-04-20T01:05:44.401ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14542024-04-20T01:05:44.401Z 0 000-019 8 8 8 14 14 14 F F F
14552024-04-20T01:05:44.401Z 1 020-039 10 10 10 17 17 17 T F T <---
14562024-04-20T01:05:44.401Z 2 040-059 7 7 7 13 13 13 T F T <---
14572024-04-20T01:05:44.401Z 3 060-079 12 11 12 21 18 21 F F F <---
14582024-04-20T01:05:44.401Z 4 080-099 11 11 11 18 18 18 F F F
14592024-04-20T01:05:44.401Z 5 100-119 6 6 6 11 11 11 T F T <---
14602024-04-20T01:05:44.401Z 6 120-139 12 10 12 21 17 21 T F T <---
14612024-04-20T01:05:44.401Z 7 140-159 12 9 12 21 16 21 T F T <---
14622024-04-20T01:05:44.401Z 8 160-179 11 11 11 18 18 18 F F F
14632024-04-20T01:05:44.401Z 9 180-199 9 9 9 16 16 16 F F F
14642024-04-20T01:05:44.401Z 10 200-219 1 1 1 1 1 1 F F F
14652024-04-20T01:05:44.401Z 11 220-239 12 8 12 21 14 21 T F T <---
14662024-04-20T01:05:44.401Z 12 240-259 11 11 11 18 18 18 F F F
14672024-04-20T01:05:44.401Z 13 260-279 10 10 10 17 17 17 F F F
14682024-04-20T01:05:44.402Z 14 280-299 9 9 9 16 16 16 F F F
14692024-04-20T01:05:44.402Z 15 300-319 10 10 10 17 17 17 F F F
14702024-04-20T01:05:44.402Z 16 320-339 9 9 9 16 16 16 F F F
14712024-04-20T01:05:44.402Z 17 340-359 12 9 12 21 16 21 F F F <---
14722024-04-20T01:05:44.402Z 18 360-379 6 6 6 12 12 12 F F F
14732024-04-20T01:05:44.402Z 19 380-399 6 6 6 12 12 12 F F F
14742024-04-20T01:05:44.402Z 20 400-419 10 10 10 17 17 17 F F F
14752024-04-20T01:05:44.402Z 21 420-439 12 11 12 21 18 21 F F F <---
14762024-04-20T01:05:44.402Z 22 440-459 12 9 12 20 16 20 F F F <---
14772024-04-20T01:05:44.402Z 23 460-479 4 4 4 9 9 9 F F F
14782024-04-20T01:05:44.402Z 24 480-499 9 9 9 16 16 16 F F F
14792024-04-20T01:05:44.402Z 25 500-519 12 11 12 21 18 21 F F F <---
14802024-04-20T01:05:44.402Z 26 520-539 12 11 12 21 18 21 T F T <---
14812024-04-20T01:05:44.402Z 27 540-559 11 11 11 18 18 18 F F F
14822024-04-20T01:05:44.402Z 28 560-579 12 9 12 19 16 19 F T F <---
14832024-04-20T01:05:44.402Z 29 580-599 12 10 12 20 17 20 F F F <---
14842024-04-20T01:05:44.402ZMax gen: 12, Max flush: 21
14852024-04-20T01:05:44.402ZError: Difference in extent metadata found!
14862024-04-20T01:05:44.403ZA Difference in extent metadata is expected here
14872024-04-20T01:05:44.403ZOn loop 6
14882024-04-20T01:05:44.403Z
14892024-04-20T01:05:44.403ZVerifying data now
14902024-04-20T01:05:48.657ZNo change after verify
14912024-04-20T01:05:48.657ZLoop: 6 Downstairs dump after verify (and repair):
14922024-04-20T01:05:48.676ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14932024-04-20T01:05:48.677Z 0 000-019 8 8 8 14 14 14 F F F
14942024-04-20T01:05:48.677Z 1 020-039 13 13 13 22 22 22 F F F
14952024-04-20T01:05:48.677Z 2 040-059 13 13 13 22 22 22 F F F
14962024-04-20T01:05:48.677Z 3 060-079 12 12 12 21 21 21 F F F
14972024-04-20T01:05:48.677Z 4 080-099 11 11 11 18 18 18 F F F
14982024-04-20T01:05:48.678Z 5 100-119 13 13 13 22 22 22 F F F
14992024-04-20T01:05:48.678Z 6 120-139 13 13 13 22 22 22 F F F
15002024-04-20T01:05:48.678Z 7 140-159 13 13 13 22 22 22 F F F
15012024-04-20T01:05:48.678Z 8 160-179 11 11 11 18 18 18 F F F
15022024-04-20T01:05:48.678Z 9 180-199 9 9 9 16 16 16 F F F
15032024-04-20T01:05:48.678Z 10 200-219 1 1 1 1 1 1 F F F
15042024-04-20T01:05:48.683Z 11 220-239 13 13 13 22 22 22 F F F
15052024-04-20T01:05:48.684Z 12 240-259 11 11 11 18 18 18 F F F
15062024-04-20T01:05:48.684Z 13 260-279 10 10 10 17 17 17 F F F
15072024-04-20T01:05:48.684Z 14 280-299 9 9 9 16 16 16 F F F
15082024-04-20T01:05:48.684Z 15 300-319 10 10 10 17 17 17 F F F
15092024-04-20T01:05:48.684Z 16 320-339 9 9 9 16 16 16 F F F
15102024-04-20T01:05:48.684Z 17 340-359 12 12 12 21 21 21 F F F
15112024-04-20T01:05:48.684Z 18 360-379 6 6 6 12 12 12 F F F
15122024-04-20T01:05:48.684Z 19 380-399 6 6 6 12 12 12 F F F
15132024-04-20T01:05:48.684Z 20 400-419 10 10 10 17 17 17 F F F
15142024-04-20T01:05:48.684Z 21 420-439 12 12 12 21 21 21 F F F
15152024-04-20T01:05:48.684Z 22 440-459 12 12 12 20 20 20 F F F
15162024-04-20T01:05:48.684Z 23 460-479 4 4 4 9 9 9 F F F
15172024-04-20T01:05:48.684Z 24 480-499 9 9 9 16 16 16 F F F
15182024-04-20T01:05:48.684Z 25 500-519 12 12 12 21 21 21 F F F
15192024-04-20T01:05:48.684Z 26 520-539 13 13 13 22 22 22 F F F
15202024-04-20T01:05:48.684Z 27 540-559 11 11 11 18 18 18 F F F
15212024-04-20T01:05:48.684Z 28 560-579 12 12 12 19 19 19 F F F
15222024-04-20T01:05:48.684Z 29 580-599 12 12 12 20 20 20 F F F
15232024-04-20T01:05:48.684ZMax gen: 13, Max flush: 22
15242024-04-20T01:05:48.685Z
15252024-04-20T01:05:48.685ZBegin loop 7
15262024-04-20T01:05:48.685ZDownstairs to restart: 0
15272024-04-20T01:05:48.694Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.203981914Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224}
15282024-04-20T01:05:48.694Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204212284Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224}
15292024-04-20T01:05:48.694Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204229564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224}
15302024-04-20T01:05:48.702Z{"msg":"Crucible d2647c3a-17c2-45bc-a3cd-4f9ed83a91b3 has session id: 90f67477-b472-4ddd-b0e9-d7f471b02722","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204257714Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15312024-04-20T01:05:48.702Z{"msg":"Upstairs opts: Upstairs UUID: d2647c3a-17c2-45bc-a3cd-4f9ed83a91b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204307545Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15322024-04-20T01:05:48.702Z{"msg":"Crucible stats registered with UUID: d2647c3a-17c2-45bc-a3cd-4f9ed83a91b3","v":0,"name":"crucible","level":Crucible runtime is spawned
15332024-04-20T01:05:48.702Z30,"time":"2024-04-20T01:05:47.204354795Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15342024-04-20T01:05:48.702Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204438175Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224}
15352024-04-20T01:05:48.702Z{"msg":"d2647c3a-17c2-45bc-a3cd-4f9ed83a91b3 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204501795Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15362024-04-20T01:05:48.703Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204536955Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"0","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15372024-04-20T01:05:48.703Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204579315Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"0","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15382024-04-20T01:05:48.703Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204608395Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"1","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15392024-04-20T01:05:48.703Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204724155Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"1","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15402024-04-20T01:05:48.703Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204771295Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"2","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15412024-04-20T01:05:48.703Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204808376Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"2","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15422024-04-20T01:05:48.703Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204842046Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15432024-04-20T01:05:48.703Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204867646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15442024-04-20T01:05:48.703Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204901976Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15452024-04-20T01:05:48.703Z{"msg":"ds_connection connected from Ok(127.0.0.1:52314)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204929656Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15462024-04-20T01:05:48.703Z{"msg":"ds_connection connected from Ok(127.0.0.1:61922)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204955816Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15472024-04-20T01:05:48.703Z{"msg":"ds_connection connected from Ok(127.0.0.1:59619)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.204981196Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15482024-04-20T01:05:48.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.205667128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"1","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15492024-04-20T01:05:48.703Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.205731058Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"1","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15502024-04-20T01:05:48.704Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.205768598Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"2","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15512024-04-20T01:05:48.704Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.205796128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"2","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15522024-04-20T01:05:48.704Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.205922428Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"0","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15532024-04-20T01:05:48.704Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.205970738Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"0","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15542024-04-20T01:05:48.704Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.206382209Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"1","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15552024-04-20T01:05:48.704Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.206418489Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"2","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15562024-04-20T01:05:48.704Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.206480759Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"0","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15572024-04-20T01:05:48.704Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207010551Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"1","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15582024-04-20T01:05:48.704Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207058021Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"1","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15592024-04-20T01:05:48.704Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207076331Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15602024-04-20T01:05:48.704Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207098251Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"0","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15612024-04-20T01:05:48.704Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207112521Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"0","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15622024-04-20T01:05:48.705Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207125781Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15632024-04-20T01:05:48.705Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207145471Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"2","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15642024-04-20T01:05:48.705Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207159431Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"2","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15652024-04-20T01:05:48.705Z{"msg":"[0]R flush_numbers[0..12]: [14, 22, 22, 21, 18, 22, 22, 22, 18, 16, 1, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207172981Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15662024-04-20T01:05:48.705Z{"msg":"[0]R generation[0..12]: [8, 13, 13, 12, 11, 13, 13, 13, 11, 9, 1, 13]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207194081Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15672024-04-20T01:05:48.705Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207208741Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15682024-04-20T01:05:48.705Z{"msg":"[1]R flush_numbers[0..12]: [14, 22, 22, 21, 18, 22, 22, 22, 18, 16, 1, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207223731Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15692024-04-20T01:05:48.705Z{"msg":"[1]R generation[0..12]: [8, 13, 13, 12, 11, 13, 13, 13, 11, 9, 1, 13]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207237321Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15702024-04-20T01:05:48.705Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207258741Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15712024-04-20T01:05:48.705ZWait for a query_work_queue command to finish before sending IO
15722024-04-20T01:05:48.705Z{"msg":"[2]R flush_numbers[0..12]: [14, 22, 22, 21, 18, 22, 22, 22, 18, 16, 1, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207277111Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15732024-04-20T01:05:48.705Z{"msg":"[2]R generation[0..12]: [8, 13, 13, 12, 11, 13, 13, 13, 11, 9, 1, 13]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207292001Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15742024-04-20T01:05:48.705Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207306321Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15752024-04-20T01:05:48.705Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207319301Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15762024-04-20T01:05:48.705Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207349271Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15772024-04-20T01:05:48.705Z{"msg":"Next flush: 23","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207371111Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15782024-04-20T01:05:48.705Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207392671Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"":"downstairs","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15792024-04-20T01:05:48.705Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207415932Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15802024-04-20T01:05:48.705Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207446481Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"0","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15812024-04-20T01:05:48.706Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207471421Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"1","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15822024-04-20T01:05:48.706Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207495262Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"client":"2","":"downstairs","session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15832024-04-20T01:05:48.706Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207508972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15842024-04-20T01:05:48.706Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207521212Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15852024-04-20T01:05:48.706Z{"msg":"d2647c3a-17c2-45bc-a3cd-4f9ed83a91b3 is now active with session: 90f67477-b472-4ddd-b0e9-d7f471b02722","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207541392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15862024-04-20T01:05:48.706Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207553702Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224,"session_id":"90f67477-b472-4ddd-b0e9-d7f471b02722"}
15872024-04-20T01:05:48.706Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:47.207565742Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224}
15882024-04-20T01:05:48.706ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15892024-04-20T01:05:48.706ZLoading write count information from file "/tmp/test_repair_verify.data"
15902024-04-20T01:05:48.706ZRun Repair workload
15912024-04-20T01:05:48.711Z01/30 Write block 82 len 36864 data: 2 3 2 2 2 2 2 3 3
15922024-04-20T01:05:48.711Z02/30 Read block 61 len 12288
15932024-04-20T01:05:48.711Z03/30 Flush
15942024-04-20T01:05:48.711Z04/30 Write block 148 len 32768 data: 3 3 2 2 2 2 2 2
15952024-04-20T01:05:48.711Z05/30 Read block 137 len 28672
15962024-04-20T01:05:48.711Z06/30 Read block 176 len 20480
15972024-04-20T01:05:48.712Z07/30 Read block 373 len 28672
15982024-04-20T01:05:48.712Z08/30 Read block 288 len 32768
15992024-04-20T01:05:48.712Z09/30 Write block 246 len 40960 data: 4 3 3 3 2 2 3 3 3 3
16002024-04-20T01:05:48.712Z10/30 Read block 463 len 40960
16012024-04-20T01:05:48.712Z11/30 Flush
16022024-04-20T01:05:48.716Z12/30 Write block 586 len 36864 data: 3 3 3 3 3 4 4 3 4
16032024-04-20T01:05:48.716Z13/30 Read block 21 len 24576
16042024-04-20T01:05:48.716Z14/30 Read block 586 len 36864
16052024-04-20T01:05:48.716Z15/30 Read block 105 len 12288
16062024-04-20T01:05:48.716Z16/30 Read block 548 len 16384
16072024-04-20T01:05:48.716Z17/30 Read block 387 len 8192
16082024-04-20T01:05:48.716Z18/30 Read block 559 len 32768
16092024-04-20T01:05:48.720Z19/30 Read block 574 len 8192
16102024-04-20T01:05:48.720Z20/30 Write block 452 len 40960 data: 2 2 2 3 3 3 3 3 3 3
16112024-04-20T01:05:48.720Z21/30 Write block 55 len 12288 data: 3 3 2
16122024-04-20T01:05:48.720Z22/30 Read block 552 len 40960
16132024-04-20T01:05:48.720Z23/30 Read block 199 len 8192
16142024-04-20T01:05:48.720Z24/30 Read block 118 len 24576
16152024-04-20T01:05:48.724Z25/30 Write block 27 len 16384 data: 3 4 4 3
16162024-04-20T01:05:48.724Z26/30 Write block 503 len 12288 data: 5 5 5
16172024-04-20T01:05:48.724Z27/30 Write block 386 len 4096 data: 2
16182024-04-20T01:05:48.724Z28/30 Read block 573 len 16384
16192024-04-20T01:05:48.728Z29/30 Write block 313 len 32768 data: 2 2 2 2 2 4 4 3
16202024-04-20T01:05:48.728Z30/30 Read block 346 len 36864
16212024-04-20T01:05:48.728Z----------------------------------------------------------------
16222024-04-20T01:05:48.728Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:27
16232024-04-20T01:05:48.728ZStates: Active Active Active
16242024-04-20T01:05:48.729ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16252024-04-20T01:05:48.729Z 4 Acked 1003 Write 8 Done Done Done false
16262024-04-20T01:05:48.729Z 5 Acked 1004 Read 7 Done Done Done false
16272024-04-20T01:05:48.729Z 6 Acked 1005 Read 5 Done Done Done false
16282024-04-20T01:05:48.729Z 7 Acked 1006 Read 7 Done Done Done false
16292024-04-20T01:05:48.729Z 8 Acked 1007 Read 8 Sent Done Done false
16302024-04-20T01:05:48.729Z 9 Acked 1008 Write 10 Sent Done Done false
16312024-04-20T01:05:48.729Z 10 Acked 1009 Read 10 Sent Done Done false
16322024-04-20T01:05:48.729Z 11 Acked 1010 Flush 0 Sent Done Done false
16332024-04-20T01:05:48.729Z 12 Acked 1011 Write 9 Sent Done Done false
16342024-04-20T01:05:48.729Z 13 Acked 1012 Read 6 Sent Done Done false
16352024-04-20T01:05:48.729Z 14 Acked 1013 Read 9 Sent Done Done false
16362024-04-20T01:05:48.729Z 15 Acked 1014 Read 3 Sent Done Done false
16372024-04-20T01:05:48.729Z 16 Acked 1015 Read 4 Sent Done Done false
16382024-04-20T01:05:48.729Z 17 Acked 1016 Read 2 Sent Done Done false
16392024-04-20T01:05:48.729Z 18 Acked 1017 Read 8 Sent Done Done false
16402024-04-20T01:05:48.729Z 19 Acked 1018 Read 2 Sent Done Done false
16412024-04-20T01:05:48.729Z 20 Acked 1019 Write 10 Sent Done Done false
16422024-04-20T01:05:48.729Z 21 Acked 1020 Write 3 Sent Done Done false
16432024-04-20T01:05:48.729Z 22 Acked 1021 Read 10 Sent Done Done false
16442024-04-20T01:05:48.729Z 23 Acked 1022 Read 2 Sent Done Done false
16452024-04-20T01:05:48.730Z 24 Acked 1023 Read 6 Sent Done Done false
16462024-04-20T01:05:48.730Z 25 Acked 1024 Write 4 Sent Done Done false
16472024-04-20T01:05:48.730Z 26 Acked 1025 Write 3 Sent Done Done false
16482024-04-20T01:05:48.730Z 27 Acked 1026 Write 1 Sent Done Done false
16492024-04-20T01:05:48.730Z 28 Acked 1027 Read 4 Sent Done Done false
16502024-04-20T01:05:48.730Z 29 Acked 1028 Write 8 Sent Done Done false
16512024-04-20T01:05:48.730Z 30 Acked 1029 Read 9 Sent Sent Done false
16522024-04-20T01:05:48.730Z STATES DS:0 DS:1 DS:2 TOTAL
16532024-04-20T01:05:48.730Z New 0 0 0 0
16542024-04-20T01:05:48.730Z Sent 23 1 0 24
16552024-04-20T01:05:48.730Z Done 4 26 27 57
16562024-04-20T01:05:48.730Z Skipped 0 0 0 0
16572024-04-20T01:05:48.730Z Error 0 0 0 0
16582024-04-20T01:05:48.730ZLast Flush: 1002 1010 1010
16592024-04-20T01:05:48.730ZDownstairs last five completed: 1002 1001 1000
16602024-04-20T01:05:48.730ZUpstairs last five completed: 30 29 28 27 26
16612024-04-20T01:05:48.730Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:05:47.237713269Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1224}
16622024-04-20T01:05:48.730ZWrote out file "/tmp/test_repair_verify.data"
16632024-04-20T01:05:48.734Z
16642024-04-20T01:05:48.734ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16652024-04-20T01:05:48.751ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16662024-04-20T01:05:48.751Z 0 000-019 8 8 8 14 14 14 F F F
16672024-04-20T01:05:48.751Z 1 020-039 13 13 13 22 22 22 F T T <---
16682024-04-20T01:05:48.751Z 2 040-059 13 13 13 22 22 22 F T T <---
16692024-04-20T01:05:48.751Z 3 060-079 12 12 12 21 21 21 F F F
16702024-04-20T01:05:48.751Z 4 080-099 14 14 14 23 23 23 F F F
16712024-04-20T01:05:48.752Z 5 100-119 13 13 13 22 22 22 F F F
16722024-04-20T01:05:48.752Z 6 120-139 13 13 13 22 22 22 F F F
16732024-04-20T01:05:48.752Z 7 140-159 13 14 14 22 24 24 T F F <---
16742024-04-20T01:05:48.752Z 8 160-179 11 11 11 18 18 18 F F F
16752024-04-20T01:05:48.752Z 9 180-199 9 9 9 16 16 16 F F F
16762024-04-20T01:05:48.752Z 10 200-219 1 1 1 1 1 1 F F F
16772024-04-20T01:05:48.757Z 11 220-239 13 13 13 22 22 22 F F F
16782024-04-20T01:05:48.757Z 12 240-259 11 14 14 18 24 24 F F F <---
16792024-04-20T01:05:48.757Z 13 260-279 10 10 10 17 17 17 F F F
16802024-04-20T01:05:48.757Z 14 280-299 9 9 9 16 16 16 F F F
16812024-04-20T01:05:48.757Z 15 300-319 10 10 10 17 17 17 F T T <---
16822024-04-20T01:05:48.757Z 16 320-339 9 9 9 16 16 16 F T T <---
16832024-04-20T01:05:48.757Z 17 340-359 12 12 12 21 21 21 F F F
16842024-04-20T01:05:48.757Z 18 360-379 6 6 6 12 12 12 F F F
16852024-04-20T01:05:48.757Z 19 380-399 6 6 6 12 12 12 F T T <---
16862024-04-20T01:05:48.757Z 20 400-419 10 10 10 17 17 17 F F F
16872024-04-20T01:05:48.757Z 21 420-439 12 12 12 21 21 21 F F F
16882024-04-20T01:05:48.757Z 22 440-459 12 12 12 20 20 20 F T T <---
16892024-04-20T01:05:48.758Z 23 460-479 4 4 4 9 9 9 F T T <---
16902024-04-20T01:05:48.758Z 24 480-499 9 9 9 16 16 16 F F F
16912024-04-20T01:05:48.758Z 25 500-519 12 12 12 21 21 21 F T T <---
16922024-04-20T01:05:48.758Z 26 520-539 13 13 13 22 22 22 F F F
16932024-04-20T01:05:48.758Z 27 540-559 11 11 11 18 18 18 F F F
16942024-04-20T01:05:48.758Z 28 560-579 12 12 12 19 19 19 F F F
16952024-04-20T01:05:48.758Z 29 580-599 12 12 12 20 20 20 F T T <---
16962024-04-20T01:05:48.758ZMax gen: 14, Max flush: 24
16972024-04-20T01:05:48.758ZError: Difference in extent metadata found!
16982024-04-20T01:05:48.758ZA Difference in extent metadata is expected here
16992024-04-20T01:05:48.758ZOn loop 7
17002024-04-20T01:05:48.758Z
17012024-04-20T01:05:48.762ZVerifying data now
17022024-04-20T01:05:53.260ZNo change after verify
17032024-04-20T01:05:53.260ZLoop: 7 Downstairs dump after verify (and repair):
17042024-04-20T01:05:53.277ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17052024-04-20T01:05:53.277Z 0 000-019 8 8 8 14 14 14 F F F
17062024-04-20T01:05:53.277Z 1 020-039 15 15 15 25 25 25 F F F
17072024-04-20T01:05:53.277Z 2 040-059 15 15 15 25 25 25 F F F
17082024-04-20T01:05:53.277Z 3 060-079 12 12 12 21 21 21 F F F
17092024-04-20T01:05:53.277Z 4 080-099 14 14 14 23 23 23 F F F
17102024-04-20T01:05:53.277Z 5 100-119 13 13 13 22 22 22 F F F
17112024-04-20T01:05:53.281Z 6 120-139 13 13 13 22 22 22 F F F
17122024-04-20T01:05:53.281Z 7 140-159 14 14 14 24 24 24 F F F
17132024-04-20T01:05:53.281Z 8 160-179 11 11 11 18 18 18 F F F
17142024-04-20T01:05:53.282Z 9 180-199 9 9 9 16 16 16 F F F
17152024-04-20T01:05:53.282Z 10 200-219 1 1 1 1 1 1 F F F
17162024-04-20T01:05:53.282Z 11 220-239 13 13 13 22 22 22 F F F
17172024-04-20T01:05:53.282Z 12 240-259 14 14 14 24 24 24 F F F
17182024-04-20T01:05:53.282Z 13 260-279 10 10 10 17 17 17 F F F
17192024-04-20T01:05:53.282Z 14 280-299 9 9 9 16 16 16 F F F
17202024-04-20T01:05:53.282Z 15 300-319 15 15 15 25 25 25 F F F
17212024-04-20T01:05:53.282Z 16 320-339 15 15 15 25 25 25 F F F
17222024-04-20T01:05:53.282Z 17 340-359 12 12 12 21 21 21 F F F
17232024-04-20T01:05:53.282Z 18 360-379 6 6 6 12 12 12 F F F
17242024-04-20T01:05:53.282Z 19 380-399 15 15 15 25 25 25 F F F
17252024-04-20T01:05:53.282Z 20 400-419 10 10 10 17 17 17 F F F
17262024-04-20T01:05:53.282Z 21 420-439 12 12 12 21 21 21 F F F
17272024-04-20T01:05:53.283Z 22 440-459 15 15 15 25 25 25 F F F
17282024-04-20T01:05:53.283Z 23 460-479 15 15 15 25 25 25 F F F
17292024-04-20T01:05:53.283Z 24 480-499 9 9 9 16 16 16 F F F
17302024-04-20T01:05:53.283Z 25 500-519 15 15 15 25 25 25 F F F
17312024-04-20T01:05:53.283Z 26 520-539 13 13 13 22 22 22 F F F
17322024-04-20T01:05:53.283Z 27 540-559 11 11 11 18 18 18 F F F
17332024-04-20T01:05:53.283Z 28 560-579 12 12 12 19 19 19 F F F
17342024-04-20T01:05:53.283Z 29 580-599 15 15 15 25 25 25 F F F
17352024-04-20T01:05:53.283ZMax gen: 15, Max flush: 25
17362024-04-20T01:05:53.283Z
17372024-04-20T01:05:53.283ZBegin loop 8
17382024-04-20T01:05:53.283ZDownstairs to restart: 1
17392024-04-20T01:05:53.296Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.80717453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232}
17402024-04-20T01:05:53.296Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807400271Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232}
17412024-04-20T01:05:53.296Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807417491Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232}
17422024-04-20T01:05:53.304Z{"msg":"Crucible 5057c645-e887-46a7-aebf-bc6288218855 has session id: d8b2afd1-0d3a-40c1-a5c9-f3960ae30355","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807431571Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17432024-04-20T01:05:53.304Z{"msg":"Upstairs opts: Upstairs UUID: 5057c645-e887-46a7-aebf-bc6288218855, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807470891Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17442024-04-20T01:05:53.304Z{"msg":"Crucible stats registered with UUID: 5057c645-e887-46a7-aebf-bc6288218855","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807506251Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"Crucible runtime is spawned
17452024-04-20T01:05:53.304Zsession_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17462024-04-20T01:05:53.304Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807598041Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232}
17472024-04-20T01:05:53.304Z{"msg":"5057c645-e887-46a7-aebf-bc6288218855 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807701931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17482024-04-20T01:05:53.304Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807727392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"0","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17492024-04-20T01:05:53.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807742332Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"0","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17502024-04-20T01:05:53.304Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807797472Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"1","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17512024-04-20T01:05:53.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807811632Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"1","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17522024-04-20T01:05:53.304Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807824982Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"2","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17532024-04-20T01:05:53.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807847112Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"2","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17542024-04-20T01:05:53.304Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807878522Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17552024-04-20T01:05:53.304Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807904272Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17562024-04-20T01:05:53.305Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.807938802Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17572024-04-20T01:05:53.305Z{"msg":"ds_connection connected from Ok(127.0.0.1:54663)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.808082322Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17582024-04-20T01:05:53.305Z{"msg":"ds_connection connected from Ok(127.0.0.1:46146)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.808137872Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17592024-04-20T01:05:53.305Z{"msg":"ds_connection connected from Ok(127.0.0.1:60500)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.808165193Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17602024-04-20T01:05:53.305Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.808872604Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"0","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17612024-04-20T01:05:53.305Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.808927604Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"0","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17622024-04-20T01:05:53.305Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.809019854Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"2","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17632024-04-20T01:05:53.305Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.809090244Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"2","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17642024-04-20T01:05:53.305Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.809123004Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"1","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17652024-04-20T01:05:53.305Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.809155765Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"1","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17662024-04-20T01:05:53.305Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.809486605Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"0","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17672024-04-20T01:05:53.305Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.809523676Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"2","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17682024-04-20T01:05:53.305Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.809540665Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"1","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17692024-04-20T01:05:53.305Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810015517Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"0","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17702024-04-20T01:05:53.305Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810052887Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"0","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17712024-04-20T01:05:53.305Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810069397Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17722024-04-20T01:05:53.305Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810114167Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"1","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17732024-04-20T01:05:53.305Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810146467Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"1","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17742024-04-20T01:05:53.305Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810176247Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17752024-04-20T01:05:53.305Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810207897Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"2","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17762024-04-20T01:05:53.305Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810239647Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"2","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17772024-04-20T01:05:53.305Z{"msg":"[0]R flush_numbers[0..12]: [14, 25, 25, 21, 23, 22, 22, 24, 18, 16, 1, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810272007Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17782024-04-20T01:05:53.305ZWait for a query_work_queue command to finish before sending IO
17792024-04-20T01:05:53.305Z{"msg":"[0]R generation[0..12]: [8, 15, 15, 12, 14, 13, 13, 14, 11, 9, 1, 13]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810306127Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17802024-04-20T01:05:53.306Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810339687Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17812024-04-20T01:05:53.306Z{"msg":"[1]R flush_numbers[0..12]: [14, 25, 25, 21, 23, 22, 22, 24, 18, 16, 1, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810371077Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17822024-04-20T01:05:53.306Z{"msg":"[1]R generation[0..12]: [8, 15, 15, 12, 14, 13, 13, 14, 11, 9, 1, 13]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810393117Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17832024-04-20T01:05:53.306Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810418478Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17842024-04-20T01:05:53.306Z{"msg":"[2]R flush_numbers[0..12]: [14, 25, 25, 21, 23, 22, 22, 24, 18, 16, 1, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810450427Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17852024-04-20T01:05:53.306Z{"msg":"[2]R generation[0..12]: [8, 15, 15, 12, 14, 13, 13, 14, 11, 9, 1, 13]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810473798Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17862024-04-20T01:05:53.306Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810503608Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17872024-04-20T01:05:53.306Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810526818Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17882024-04-20T01:05:53.306Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810553708Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17892024-04-20T01:05:53.306Z{"msg":"Next flush: 26","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
17902024-04-20T01:05:53.306Z,"time":"2024-04-20T01:05:51.810576538Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17912024-04-20T01:05:53.306Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810610228Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"":"downstairs","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17922024-04-20T01:05:53.306Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810641958Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17932024-04-20T01:05:53.306Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810663668Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"0","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17942024-04-20T01:05:53.306Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810690478Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"1","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17952024-04-20T01:05:53.306Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810713748Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"client":"2","":"downstairs","session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17962024-04-20T01:05:53.306Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810740958Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17972024-04-20T01:05:53.306Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810762208Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17982024-04-20T01:05:53.306Z{"msg":"5057c645-e887-46a7-aebf-bc6288218855 is now active with session: d8b2afd1-0d3a-40c1-a5c9-f3960ae30355","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810790768Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
17992024-04-20T01:05:53.306Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810811998Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232,"session_id":"d8b2afd1-0d3a-40c1-a5c9-f3960ae30355"}
18002024-04-20T01:05:53.306Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:51.810838088Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232}
18012024-04-20T01:05:53.306ZLoading write count information from file "/tmp/test_repair_verify.data"
18022024-04-20T01:05:53.306ZRun Repair workload
18032024-04-20T01:05:53.311Z01/30 Write block 489 len 16384 data: 2 2 2 2
18042024-04-20T01:05:53.311Z02/30 Write block 225 len 32768 data: 3 3 2 2 3 3 3 3
18052024-04-20T01:05:53.311Z03/30 Read block 375 len 8192
18062024-04-20T01:05:53.311Z04/30 Write block 405 len 32768 data: 2 2 2 2 2 3 4 4
18072024-04-20T01:05:53.311Z05/30 Write block 490 len 36864 data: 3 3 3 2 2 2 2 2 2
18082024-04-20T01:05:53.311Z06/30 Read block 92 len 4096
18092024-04-20T01:05:53.311Z07/30 Read block 174 len 8192
18102024-04-20T01:05:53.311Z08/30 Write block 427 len 16384 data: 2 2 2 2
18112024-04-20T01:05:53.311Z09/30 Write block 481 len 8192 data: 3 3
18122024-04-20T01:05:53.311Z10/30 Write block 472 len 28672 data: 3 3 3 3 3 3 2
18132024-04-20T01:05:53.315Z11/30 Write block 348 len 40960 data: 3 3 2 2 2 2 2 2 2 2
18142024-04-20T01:05:53.315Z12/30 Read block 182 len 24576
18152024-04-20T01:05:53.315Z13/30 Write block 204 len 24576 data: 2 2 2 2 2 2
18162024-04-20T01:05:53.315Z14/30 Write block 231 len 40960 data: 4 4 4 4 4 4 5 4 4 4
18172024-04-20T01:05:53.315Z15/30 Read block 337 len 36864
18182024-04-20T01:05:53.319Z16/30 Flush
18192024-04-20T01:05:53.323Z17/30 Write block 31 len 24576 data: 2 2 2 2 2 2
18202024-04-20T01:05:53.323Z18/30 Read block 143 len 40960
18212024-04-20T01:05:53.323Z19/30 Write block 76 len 8192 data: 2 3
18222024-04-20T01:05:53.323Z20/30 Write block 550 len 32768 data: 3 3 3 3 3 2 2 3
18232024-04-20T01:05:53.326Z21/30 Flush
18242024-04-20T01:05:53.326Z22/30 Write block 587 len 36864 data: 4 4 4 4 5 5 4 5 4
18252024-04-20T01:05:53.326Z23/30 Read block 38 len 24576
18262024-04-20T01:05:53.330Z24/30 Read block 39 len 4096
18272024-04-20T01:05:53.330Z25/30 Read block 202 len 32768
18282024-04-20T01:05:53.330Z26/30 Read block 339 len 4096
18292024-04-20T01:05:53.330Z27/30 Write block 173 len 36864 data: 2 2 2 2 3 3 3 3 3
18302024-04-20T01:05:53.334Z28/30 Write block 277 len 8192 data: 2 2
18312024-04-20T01:05:53.334Z29/30 Read block 232 len 12288
18322024-04-20T01:05:53.335Z30/30 Write block 569 len 24576 data: 4 5 4 3 3 3
18332024-04-20T01:05:53.335Z----------------------------------------------------------------
18342024-04-20T01:05:53.335Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18352024-04-20T01:05:53.335ZStates: Active Active Active
18362024-04-20T01:05:53.335ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18372024-04-20T01:05:53.335Z 1 Acked 1000 Write 4 Done Sent Done false
18382024-04-20T01:05:53.335Z 2 Acked 1001 Write 8 Done Sent Done false
18392024-04-20T01:05:53.335Z 3 Acked 1002 Read 2 Done Sent Done false
18402024-04-20T01:05:53.335Z 4 Acked 1003 Write 8 Done Sent Done false
18412024-04-20T01:05:53.335Z 5 Acked 1004 Write 9 Done Sent Done false
18422024-04-20T01:05:53.335Z 6 Acked 1005 Read 1 Done Sent Done false
18432024-04-20T01:05:53.335Z 7 Acked 1006 Read 2 Done Sent Done false
18442024-04-20T01:05:53.335Z 8 Acked 1007 Write 4 Done Sent Done false
18452024-04-20T01:05:53.335Z 9 Acked 1008 Write 2 Done Sent Done false
18462024-04-20T01:05:53.335Z 10 Acked 1009 Write 7 Done Sent Done false
18472024-04-20T01:05:53.335Z 11 Acked 1010 Write 10 Done Sent Done false
18482024-04-20T01:05:53.335Z 12 Acked 1011 Read 6 Done Sent Done false
18492024-04-20T01:05:53.335Z 13 Acked 1012 Write 6 Done Sent Done false
18502024-04-20T01:05:53.335Z 14 Acked 1013 Write 10 Done Sent Done false
18512024-04-20T01:05:53.335Z 15 Acked 1014 Read 9 Done Sent Done false
18522024-04-20T01:05:53.335Z 16 Acked 1015 Flush 0 Done Sent Done false
18532024-04-20T01:05:53.335Z 17 Acked 1016 Write 6 Done Sent Done false
18542024-04-20T01:05:53.335Z 18 Acked 1017 Read 10 Done Sent Done false
18552024-04-20T01:05:53.335Z 19 Acked 1018 Write 2 Done Sent Done false
18562024-04-20T01:05:53.336Z 20 Acked 1019 Write 8 Done Sent Done false
18572024-04-20T01:05:53.336Z 21 Acked 1020 Flush 0 Done Sent Done false
18582024-04-20T01:05:53.336Z 22 Acked 1021 Write 9 Done Sent Done false
18592024-04-20T01:05:53.336Z 23 Acked 1022 Read 6 Done Sent Done false
18602024-04-20T01:05:53.336Z 24 Acked 1023 Read 1 Done Sent Done false
18612024-04-20T01:05:53.336Z 25 Acked 1024 Read 8 Done Sent Done false
18622024-04-20T01:05:53.336Z 26 Acked 1025 Read 1 Done Sent Done false
18632024-04-20T01:05:53.336Z 27 Acked 1026 Write 9 Done Sent Done false
18642024-04-20T01:05:53.336Z 28 Acked 1027 Write 2 Done Sent Done false
18652024-04-20T01:05:53.336Z 29 Acked 1028 Read 3 Done Sent Done false
18662024-04-20T01:05:53.336Z 30 Acked 1029 Write 6 Sent Sent Sent false
18672024-04-20T01:05:53.336Z STATES DS:0 DS:1 DS:2 TOTAL
18682024-04-20T01:05:53.336Z New 0 0 0 0
18692024-04-20T01:05:53.336Z Sent 1 30 1 32
18702024-04-20T01:05:53.336Z Done 29 0 29 58
18712024-04-20T01:05:53.336Z Skipped 0 0 0 0
18722024-04-20T01:05:53.336Z Error 0 0 0 0
18732024-04-20T01:05:53.336ZLast Flush: 1020 0 1020
18742024-04-20T01:05:53.336ZDownstairs last five completed:
18752024-04-20T01:05:53.336ZUpstairs last five completed: 30 29 28 27 26
18762024-04-20T01:05:53.336Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:05:51.844072512Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1232}
18772024-04-20T01:05:53.336ZWrote out file "/tmp/test_repair_verify.data"
18782024-04-20T01:05:53.340Z
18792024-04-20T01:05:53.340ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18802024-04-20T01:05:53.355ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18812024-04-20T01:05:53.355Z 0 000-019 8 8 8 14 14 14 F F F
18822024-04-20T01:05:53.361Z 1 020-039 16 15 16 27 25 27 F F F <---
18832024-04-20T01:05:53.361Z 2 040-059 15 15 15 25 25 25 F F F
18842024-04-20T01:05:53.361Z 3 060-079 16 12 16 27 21 27 F F F <---
18852024-04-20T01:05:53.361Z 4 080-099 14 14 14 23 23 23 F F F
18862024-04-20T01:05:53.361Z 5 100-119 13 13 13 22 22 22 F F F
18872024-04-20T01:05:53.361Z 6 120-139 13 13 13 22 22 22 F F F
18882024-04-20T01:05:53.361Z 7 140-159 14 14 14 24 24 24 F F F
18892024-04-20T01:05:53.361Z 8 160-179 11 11 11 18 18 18 T F T <---
18902024-04-20T01:05:53.361Z 9 180-199 9 9 9 16 16 16 T F T <---
18912024-04-20T01:05:53.361Z 10 200-219 16 1 16 26 1 26 F F F <---
18922024-04-20T01:05:53.361Z 11 220-239 16 13 16 26 22 26 F F F <---
18932024-04-20T01:05:53.361Z 12 240-259 16 14 16 26 24 26 F F F <---
18942024-04-20T01:05:53.361Z 13 260-279 10 10 10 17 17 17 T F T <---
18952024-04-20T01:05:53.361Z 14 280-299 9 9 9 16 16 16 F F F
18962024-04-20T01:05:53.362Z 15 300-319 15 15 15 25 25 25 F F F
18972024-04-20T01:05:53.362Z 16 320-339 15 15 15 25 25 25 F F F
18982024-04-20T01:05:53.362Z 17 340-359 16 12 16 26 21 26 F F F <---
18992024-04-20T01:05:53.362Z 18 360-379 6 6 6 12 12 12 F F F
19002024-04-20T01:05:53.362Z 19 380-399 15 15 15 25 25 25 F F F
19012024-04-20T01:05:53.362Z 20 400-419 16 10 16 26 17 26 F F F <---
19022024-04-20T01:05:53.362Z 21 420-439 16 12 16 26 21 26 F F F <---
19032024-04-20T01:05:53.362Z 22 440-459 15 15 15 25 25 25 F F F
19042024-04-20T01:05:53.362Z 23 460-479 16 15 16 26 25 26 F F F <---
19052024-04-20T01:05:53.362Z 24 480-499 16 9 16 26 16 26 F F F <---
19062024-04-20T01:05:53.362Z 25 500-519 15 15 15 25 25 25 F F F
19072024-04-20T01:05:53.362Z 26 520-539 13 13 13 22 22 22 F F F
19082024-04-20T01:05:53.362Z 27 540-559 16 11 16 27 18 27 F F F <---
19092024-04-20T01:05:53.362Z 28 560-579 12 12 12 19 19 19 T F T <---
19102024-04-20T01:05:53.362Z 29 580-599 15 15 15 25 25 25 T F T <---
19112024-04-20T01:05:53.362ZMax gen: 16, Max flush: 27
19122024-04-20T01:05:53.362ZError: Difference in extent metadata found!
19132024-04-20T01:05:53.362ZA Difference in extent metadata is expected here
19142024-04-20T01:05:53.362ZOn loop 8
19152024-04-20T01:05:53.362Z
19162024-04-20T01:05:53.366ZVerifying data now
19172024-04-20T01:05:58.062ZNo change after verify
19182024-04-20T01:05:58.062ZLoop: 8 Downstairs dump after verify (and repair):
19192024-04-20T01:05:58.076ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19202024-04-20T01:05:58.076Z 0 000-019 8 8 8 14 14 14 F F F
19212024-04-20T01:05:58.076Z 1 020-039 16 16 16 27 27 27 F F F
19222024-04-20T01:05:58.076Z 2 040-059 15 15 15 25 25 25 F F F
19232024-04-20T01:05:58.076Z 3 060-079 16 16 16 27 27 27 F F F
19242024-04-20T01:05:58.076Z 4 080-099 14 14 14 23 23 23 F F F
19252024-04-20T01:05:58.080Z 5 100-119 13 13 13 22 22 22 F F F
19262024-04-20T01:05:58.080Z 6 120-139 13 13 13 22 22 22 F F F
19272024-04-20T01:05:58.080Z 7 140-159 14 14 14 24 24 24 F F F
19282024-04-20T01:05:58.080Z 8 160-179 17 17 17 28 28 28 F F F
19292024-04-20T01:05:58.080Z 9 180-199 17 17 17 28 28 28 F F F
19302024-04-20T01:05:58.080Z 10 200-219 16 16 16 26 26 26 F F F
19312024-04-20T01:05:58.080Z 11 220-239 16 16 16 26 26 26 F F F
19322024-04-20T01:05:58.080Z 12 240-259 16 16 16 26 26 26 F F F
19332024-04-20T01:05:58.080Z 13 260-279 17 17 17 28 28 28 F F F
19342024-04-20T01:05:58.080Z 14 280-299 9 9 9 16 16 16 F F F
19352024-04-20T01:05:58.080Z 15 300-319 15 15 15 25 25 25 F F F
19362024-04-20T01:05:58.080Z 16 320-339 15 15 15 25 25 25 F F F
19372024-04-20T01:05:58.080Z 17 340-359 16 16 16 26 26 26 F F F
19382024-04-20T01:05:58.080Z 18 360-379 6 6 6 12 12 12 F F F
19392024-04-20T01:05:58.080Z 19 380-399 15 15 15 25 25 25 F F F
19402024-04-20T01:05:58.081Z 20 400-419 16 16 16 26 26 26 F F F
19412024-04-20T01:05:58.081Z 21 420-439 16 16 16 26 26 26 F F F
19422024-04-20T01:05:58.081Z 22 440-459 15 15 15 25 25 25 F F F
19432024-04-20T01:05:58.081Z 23 460-479 16 16 16 26 26 26 F F F
19442024-04-20T01:05:58.081Z 24 480-499 16 16 16 26 26 26 F F F
19452024-04-20T01:05:58.081Z 25 500-519 15 15 15 25 25 25 F F F
19462024-04-20T01:05:58.081Z 26 520-539 13 13 13 22 22 22 F F F
19472024-04-20T01:05:58.081Z 27 540-559 16 16 16 27 27 27 F F F
19482024-04-20T01:05:58.081Z 28 560-579 17 17 17 28 28 28 F F F
19492024-04-20T01:05:58.081Z 29 580-599 17 17 17 28 28 28 F F F
19502024-04-20T01:05:58.081ZMax gen: 17, Max flush: 28
19512024-04-20T01:05:58.081Z
19522024-04-20T01:05:58.081ZBegin loop 9
19532024-04-20T01:05:58.081ZDownstairs to restart: 2
19542024-04-20T01:05:58.098Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.608769928Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240}
19552024-04-20T01:05:58.098Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.608999629Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240}
19562024-04-20T01:05:58.098Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.609016039Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240}
19572024-04-20T01:05:58.098Z{"msg":"Crucible 0f5c8bab-f53f-411d-a2fd-ad91b4c376e0 has session id: 2b276a1a-958a-44b4-bab6-41e8a47d4c7a","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.609029739Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19582024-04-20T01:05:58.106Z{"msg":"Upstairs opts: Upstairs UUID: 0f5c8bab-f53f-411d-a2fd-ad91b4c376e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.609071929Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19592024-04-20T01:05:58.106Z{"msg":"Crucible stats registered with UUID: 0f5c8bab-f53f-411d-a2fd-ad91b4c376e0","v":0,"name":"crucible","level":30Crucible runtime is spawned
19602024-04-20T01:05:58.106Z,"time":"2024-04-20T01:05:56.609102699Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19612024-04-20T01:05:58.106Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.6091981Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240}
19622024-04-20T01:05:58.106Z{"msg":"0f5c8bab-f53f-411d-a2fd-ad91b4c376e0 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.60922802Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19632024-04-20T01:05:58.106Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.609242569Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"0","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19642024-04-20T01:05:58.106Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.60925785Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"0","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19652024-04-20T01:05:58.106Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.60927164Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"1","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19662024-04-20T01:05:58.106Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.60928677Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"1","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19672024-04-20T01:05:58.106Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.60931348Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"2","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19682024-04-20T01:05:58.106Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.60932746Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"2","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19692024-04-20T01:05:58.106Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.6093437Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19702024-04-20T01:05:58.106Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.60936589Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19712024-04-20T01:05:58.106Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.60948551Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19722024-04-20T01:05:58.106Z{"msg":"ds_connection connected from Ok(127.0.0.1:33699)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.60954806Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19732024-04-20T01:05:58.106Z{"msg":"ds_connection connected from Ok(127.0.0.1:63388)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.60959462Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19742024-04-20T01:05:58.106Z{"msg":"ds_connection connected from Ok(127.0.0.1:39381)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.609760711Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19752024-04-20T01:05:58.106Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.610331122Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"0","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19762024-04-20T01:05:58.106Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.610372732Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"0","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19772024-04-20T01:05:58.106Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.610436092Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"1","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19782024-04-20T01:05:58.107Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.610470982Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"1","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19792024-04-20T01:05:58.107Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.610749473Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"2","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19802024-04-20T01:05:58.107Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.610785393Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"2","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19812024-04-20T01:05:58.107Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.610802683Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"1","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19822024-04-20T01:05:58.107Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611053814Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"0","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19832024-04-20T01:05:58.107Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611305984Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"1","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19842024-04-20T01:05:58.107Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611344434Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"1","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19852024-04-20T01:05:58.107Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611371324Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19862024-04-20T01:05:58.107Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611441974Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"2","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19872024-04-20T01:05:58.107Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611488405Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"0","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19882024-04-20T01:05:58.107Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611506525Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"0","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19892024-04-20T01:05:58.107Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611528865Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19902024-04-20T01:05:58.107Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611806415Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"2","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19912024-04-20T01:05:58.107Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611838865Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"2","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19922024-04-20T01:05:58.107Z{"msg":"[0]R flush_numbers[0..12]: [14, 27, 25, 27, 23, 22, 22, 24, 28, 28, 26, 26]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611907755Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19932024-04-20T01:05:58.107Z{"msg":"[0]R generation[0..12]: [8, 16, 15, 16, 14, 13, 13, 14, 17, 17, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611933935Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19942024-04-20T01:05:58.107Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611949016Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19952024-04-20T01:05:58.107Z{"msg":"[1]R flush_numbers[0..12]: [14, 27, 25, 27, 23, 22, 22, 24, 28, 28, 26, 26]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611969676Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19962024-04-20T01:05:58.107Z{"msg":"[1]R generation[0..12]: [8, 16, 15, 16, 14, 13, 13, 14, 17, 17, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.611983896Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19972024-04-20T01:05:58.107Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612004516Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19982024-04-20T01:05:58.107Z{"msg":"[2]R flush_numbers[0..12]: [14, 27, 25, 27, 23, 22, 22, 24, 28, 28, 26, 26]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612018496Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
19992024-04-20T01:05:58.107Z{"msg":"[2]R generation[0..12]: [8, 16, 15, 16, 14, 13, 13, 14, 17, 17, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612038646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20002024-04-20T01:05:58.107Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612052936Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20012024-04-20T01:05:58.108ZWait for a query_work_queue command to finish before sending IO
20022024-04-20T01:05:58.108Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612076726Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20032024-04-20T01:05:58.108Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612091066Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20042024-04-20T01:05:58.108Z{"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612110896Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20052024-04-20T01:05:58.108Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612124266Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"":"downstairs","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20062024-04-20T01:05:58.108Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612143666Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20072024-04-20T01:05:58.108Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612156086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"0","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20082024-04-20T01:05:58.108Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612175836Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"1","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20092024-04-20T01:05:58.108Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612189376Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"client":"2","":"downstairs","session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20102024-04-20T01:05:58.108Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612210536Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20112024-04-20T01:05:58.108Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612223566Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20122024-04-20T01:05:58.108Z{"msg":"0f5c8bab-f53f-411d-a2fd-ad91b4c376e0 is now active with session: 2b276a1a-958a-44b4-bab6-41e8a47d4c7a","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612236206Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20132024-04-20T01:05:58.108Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612248206Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240,"session_id":"2b276a1a-958a-44b4-bab6-41e8a47d4c7a"}
20142024-04-20T01:05:58.108Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:05:56.612260566Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240}
20152024-04-20T01:05:58.108ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20162024-04-20T01:05:58.108ZLoading write count information from file "/tmp/test_repair_verify.data"
20172024-04-20T01:05:58.108ZRun Repair workload
20182024-04-20T01:05:58.112Z01/30 Flush
20192024-04-20T01:05:58.112Z02/30 Write block 49 len 16384 data: 3 5 6 6
20202024-04-20T01:05:58.113Z03/30 Read block 399 len 28672
20212024-04-20T01:05:58.113Z04/30 Read block 426 len 12288
20222024-04-20T01:05:58.113Z05/30 Write block 421 len 12288 data: 4 3 3
20232024-04-20T01:05:58.113Z06/30 Read block 215 len 16384
20242024-04-20T01:05:58.113Z07/30 Write block 231 len 36864 data: 5 5 5 5 5 5 6 5 5
20252024-04-20T01:05:58.113Z08/30 Read block 14 len 36864
20262024-04-20T01:05:58.113Z09/30 Write block 119 len 36864 data: 2 2 2 2 2 2 2 2 2
20272024-04-20T01:05:58.113Z10/30 Write block 168 len 20480 data: 3 2 2 2 2
20282024-04-20T01:05:58.113Z11/30 Read block 219 len 20480
20292024-04-20T01:05:58.116Z12/30 Write block 173 len 36864 data: 3 3 3 3 4 4 4 4 4
20302024-04-20T01:05:58.116Z13/30 Read block 334 len 28672
20312024-04-20T01:05:58.117Z14/30 Read block 508 len 4096
20322024-04-20T01:05:58.117Z15/30 Read block 245 len 28672
20332024-04-20T01:05:58.117Z16/30 Flush
20342024-04-20T01:05:58.120Z17/30 Write block 58 len 20480 data: 2 3 3 3 3
20352024-04-20T01:05:58.120Z18/30 Read block 478 len 28672
20362024-04-20T01:05:58.124Z19/30 Write block 365 len 36864 data: 2 3 3 3 3 4 4 4 5
20372024-04-20T01:05:58.124Z20/30 Write block 248 len 16384 data: 4 4 3 3
20382024-04-20T01:05:58.127Z21/30 Write block 455 len 36864 data: 4 4 4 4 4 4 4 3 3
20392024-04-20T01:05:58.127Z22/30 Flush
20402024-04-20T01:05:58.131Z23/30 Write block 100 len 24576 data: 3 3 3 3 3 3
20412024-04-20T01:05:58.131Z24/30 Read block 373 len 40960
20422024-04-20T01:05:58.131Z25/30 Read block 326 len 20480
20432024-04-20T01:05:58.131Z26/30 Write block 402 len 4096 data: 2
20442024-04-20T01:05:58.135Z27/30 Write block 457 len 40960 data: 5 5 5 5 5 4 4 2 2 2
20452024-04-20T01:05:58.135Z28/30 Read block 504 len 4096
20462024-04-20T01:05:58.135Z29/30 Read block 531 len 16384
20472024-04-20T01:05:58.135Z30/30 Read block 183 len 16384
20482024-04-20T01:05:58.135Z----------------------------------------------------------------
20492024-04-20T01:05:58.135Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:29
20502024-04-20T01:05:58.135ZStates: Active Active Active
20512024-04-20T01:05:58.135ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20522024-04-20T01:05:58.135Z 2 Acked 1001 Write 4 Done Done Done false
20532024-04-20T01:05:58.135Z 3 Acked 1002 Read 7 Done Done Done false
20542024-04-20T01:05:58.135Z 4 Acked 1003 Read 3 Done Done Done false
20552024-04-20T01:05:58.135Z 5 Acked 1004 Write 3 Done Done Sent false
20562024-04-20T01:05:58.135Z 6 Acked 1005 Read 4 Done Done Sent false
20572024-04-20T01:05:58.136Z 7 Acked 1006 Write 9 Done Done Sent false
20582024-04-20T01:05:58.136Z 8 Acked 1007 Read 9 Done Done Sent false
20592024-04-20T01:05:58.136Z 9 Acked 1008 Write 9 Done Done Sent false
20602024-04-20T01:05:58.136Z 10 Acked 1009 Write 5 Done Done Sent false
20612024-04-20T01:05:58.136Z 11 Acked 1010 Read 5 Done Done Sent false
20622024-04-20T01:05:58.136Z 12 Acked 1011 Write 9 Done Done Sent false
20632024-04-20T01:05:58.136Z 13 Acked 1012 Read 7 Done Done Sent false
20642024-04-20T01:05:58.136Z 14 Acked 1013 Read 1 Done Done Sent false
20652024-04-20T01:05:58.136Z 15 Acked 1014 Read 7 Done Done Sent false
20662024-04-20T01:05:58.136Z 16 Acked 1015 Flush 0 Done Done Sent false
20672024-04-20T01:05:58.136Z 17 Acked 1016 Write 5 Done Done Sent false
20682024-04-20T01:05:58.136Z 18 Acked 1017 Read 7 Done Done Sent false
20692024-04-20T01:05:58.136Z 19 Acked 1018 Write 9 Done Done Sent false
20702024-04-20T01:05:58.136Z 20 Acked 1019 Write 4 Done Done Sent false
20712024-04-20T01:05:58.136Z 21 Acked 1020 Write 9 Done Done Sent false
20722024-04-20T01:05:58.136Z 22 Acked 1021 Flush 0 Done Done Sent false
20732024-04-20T01:05:58.136Z 23 Acked 1022 Write 6 Done Done Sent false
20742024-04-20T01:05:58.136Z 24 Acked 1023 Read 10 Done Done Sent false
20752024-04-20T01:05:58.136Z 25 Acked 1024 Read 5 Done Done Sent false
20762024-04-20T01:05:58.136Z 26 Acked 1025 Write 1 Done Done Sent false
20772024-04-20T01:05:58.136Z 27 Acked 1026 Write 10 Done Done Sent false
20782024-04-20T01:05:58.136Z 28 Acked 1027 Read 1 Done Done Sent false
20792024-04-20T01:05:58.136Z 29 Acked 1028 Read 4 Done Done Sent false
20802024-04-20T01:05:58.136Z 30 Acked 1029 Read 4 Done Sent Sent false
20812024-04-20T01:05:58.137Z STATES DS:0 DS:1 DS:2 TOTAL
20822024-04-20T01:05:58.137Z New 0 0 0 0
20832024-04-20T01:05:58.137Z Sent 0 1 26 27
20842024-04-20T01:05:58.137Z Done 29 28 3 60
20852024-04-20T01:05:58.137Z Skipped 0 0 0 0
20862024-04-20T01:05:58.137Z Error 0 0 0 0
20872024-04-20T01:05:58.137ZLast Flush: 1021 1021 1000
20882024-04-20T01:05:58.137ZDownstairs last five completed: 1000
20892024-04-20T01:05:58.137ZUpstairs last five completed: 30 29 28 27 26
20902024-04-20T01:05:58.137Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:05:56.645023799Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1240}
20912024-04-20T01:05:58.137ZWrote out file "/tmp/test_repair_verify.data"
20922024-04-20T01:05:58.141Z
20932024-04-20T01:05:58.144ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
20942024-04-20T01:05:58.157ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
20952024-04-20T01:05:58.157Z 0 000-019 8 8 8 14 14 14 F F F
20962024-04-20T01:05:58.157Z 1 020-039 16 16 16 27 27 27 F F F
20972024-04-20T01:05:58.157Z 2 040-059 18 18 15 31 31 25 F F T <---
20982024-04-20T01:05:58.157Z 3 060-079 18 18 16 31 31 27 F F F <---
20992024-04-20T01:05:58.157Z 4 080-099 14 14 14 23 23 23 F F F
21002024-04-20T01:05:58.157Z 5 100-119 18 18 13 30 30 22 T T F <---
21012024-04-20T01:05:58.157Z 6 120-139 18 18 13 30 30 22 F F F <---
21022024-04-20T01:05:58.157Z 7 140-159 14 14 14 24 24 24 F F F
21032024-04-20T01:05:58.157Z 8 160-179 18 18 17 30 30 28 F F F <---
21042024-04-20T01:05:58.157Z 9 180-199 18 18 17 30 30 28 F F F <---
21052024-04-20T01:05:58.158Z 10 200-219 16 16 16 26 26 26 F F F
21062024-04-20T01:05:58.158Z 11 220-239 18 18 16 30 30 26 F F F <---
21072024-04-20T01:05:58.158Z 12 240-259 18 18 16 31 31 26 F F F <---
21082024-04-20T01:05:58.158Z 13 260-279 17 17 17 28 28 28 F F F
21092024-04-20T01:05:58.162Z 14 280-299 9 9 9 16 16 16 F F F
21102024-04-20T01:05:58.162Z 15 300-319 15 15 15 25 25 25 F F F
21112024-04-20T01:05:58.162Z 16 320-339 15 15 15 25 25 25 F F F
21122024-04-20T01:05:58.162Z 17 340-359 16 16 16 26 26 26 F F F
21132024-04-20T01:05:58.162Z 18 360-379 18 18 6 31 31 12 F F F <---
21142024-04-20T01:05:58.162Z 19 380-399 15 15 15 25 25 25 F F F
21152024-04-20T01:05:58.162Z 20 400-419 16 16 16 26 26 26 T T F <---
21162024-04-20T01:05:58.162Z 21 420-439 18 18 16 30 30 26 F F F <---
21172024-04-20T01:05:58.162Z 22 440-459 18 18 15 31 31 25 T T F <---
21182024-04-20T01:05:58.162Z 23 460-479 18 18 16 31 31 26 T T F <---
21192024-04-20T01:05:58.162Z 24 480-499 16 16 16 26 26 26 F F F
21202024-04-20T01:05:58.162Z 25 500-519 15 15 15 25 25 25 F F F
21212024-04-20T01:05:58.162Z 26 520-539 13 13 13 22 22 22 F F F
21222024-04-20T01:05:58.162Z 27 540-559 16 16 16 27 27 27 F F F
21232024-04-20T01:05:58.162Z 28 560-579 17 17 17 28 28 28 F F F
21242024-04-20T01:05:58.162Z 29 580-599 17 17 17 28 28 28 F F F
21252024-04-20T01:05:58.162ZMax gen: 18, Max flush: 31
21262024-04-20T01:05:58.162ZError: Difference in extent metadata found!
21272024-04-20T01:05:58.162ZA Difference in extent metadata is expected here
21282024-04-20T01:05:58.163ZOn loop 9
21292024-04-20T01:05:58.163Z
21302024-04-20T01:05:58.166ZVerifying data now
21312024-04-20T01:06:02.744ZNo change after verify
21322024-04-20T01:06:02.744ZLoop: 9 Downstairs dump after verify (and repair):
21332024-04-20T01:06:02.760ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21342024-04-20T01:06:02.760Z 0 000-019 8 8 8 14 14 14 F F F
21352024-04-20T01:06:02.761Z 1 020-039 16 16 16 27 27 27 F F F
21362024-04-20T01:06:02.761Z 2 040-059 18 18 18 31 31 31 F F F
21372024-04-20T01:06:02.761Z 3 060-079 18 18 18 31 31 31 F F F
21382024-04-20T01:06:02.761Z 4 080-099 14 14 14 23 23 23 F F F
21392024-04-20T01:06:02.761Z 5 100-119 19 19 19 32 32 32 F F F
21402024-04-20T01:06:02.769Z 6 120-139 18 18 18 30 30 30 F F F
21412024-04-20T01:06:02.770Z 7 140-159 14 14 14 24 24 24 F F F
21422024-04-20T01:06:02.770Z 8 160-179 18 18 18 30 30 30 F F F
21432024-04-20T01:06:02.770Z 9 180-199 18 18 18 30 30 30 F F F
21442024-04-20T01:06:02.770Z 10 200-219 16 16 16 26 26 26 F F F
21452024-04-20T01:06:02.770Z 11 220-239 18 18 18 30 30 30 F F F
21462024-04-20T01:06:02.771Z 12 240-259 18 18 18 31 31 31 F F F
21472024-04-20T01:06:02.771Z 13 260-279 17 17 17 28 28 28 F F F
21482024-04-20T01:06:02.771Z 14 280-299 9 9 9 16 16 16 F F F
21492024-04-20T01:06:02.771Z 15 300-319 15 15 15 25 25 25 F F F
21502024-04-20T01:06:02.772Z 16 320-339 15 15 15 25 25 25 F F F
21512024-04-20T01:06:02.772Z 17 340-359 16 16 16 26 26 26 F F F
21522024-04-20T01:06:02.772Z 18 360-379 18 18 18 31 31 31 F F F
21532024-04-20T01:06:02.772Z 19 380-399 15 15 15 25 25 25 F F F
21542024-04-20T01:06:02.772Z 20 400-419 19 19 19 32 32 32 F F F
21552024-04-20T01:06:02.773Z 21 420-439 18 18 18 30 30 30 F F F
21562024-04-20T01:06:02.773Z 22 440-459 19 19 19 32 32 32 F F F
21572024-04-20T01:06:02.773Z 23 460-479 19 19 19 32 32 32 F F F
21582024-04-20T01:06:02.773Z 24 480-499 16 16 16 26 26 26 F F F
21592024-04-20T01:06:02.773Z 25 500-519 15 15 15 25 25 25 F F F
21602024-04-20T01:06:02.774Z 26 520-539 13 13 13 22 22 22 F F F
21612024-04-20T01:06:02.774Z 27 540-559 16 16 16 27 27 27 F F F
21622024-04-20T01:06:02.774Z 28 560-579 17 17 17 28 28 28 F F F
21632024-04-20T01:06:02.774Z 29 580-599 17 17 17 28 28 28 F F F
21642024-04-20T01:06:02.775ZMax gen: 19, Max flush: 32
21652024-04-20T01:06:02.775Z
21662024-04-20T01:06:02.775ZBegin loop 10
21672024-04-20T01:06:02.775ZDownstairs to restart: 1
21682024-04-20T01:06:02.793Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289339018Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248}
21692024-04-20T01:06:02.794Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289602198Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248}
21702024-04-20T01:06:02.794Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289619639Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248}
21712024-04-20T01:06:02.794Z{"msg":"Crucible 25fb2110-5e6a-4f02-a736-6c2661a6ba87 has session id: d8aa1f59-a991-4dd7-8e54-b4b8812b496f","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289633859Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21722024-04-20T01:06:02.794Z{"msg":"Upstairs opts: Upstairs UUID: 25fb2110-5e6a-4f02-a736-6c2661a6ba87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289658628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21732024-04-20T01:06:02.795Z{"msg":"Crucible stats registered with UUID: 25fb2110-5e6a-4f02-a736-6c2661a6ba87","v":0,"name":"crucible","level":30Crucible runtime is spawned
21742024-04-20T01:06:02.795Z,"time":"2024-04-20T01:06:01.289694399Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21752024-04-20T01:06:02.795Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289801869Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248}
21762024-04-20T01:06:02.795Z{"msg":"25fb2110-5e6a-4f02-a736-6c2661a6ba87 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289835589Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21772024-04-20T01:06:02.795Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289850609Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"0","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21782024-04-20T01:06:02.795Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289865709Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"0","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21792024-04-20T01:06:02.796Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289880439Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"1","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21802024-04-20T01:06:02.796Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289903039Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"1","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21812024-04-20T01:06:02.796Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289920499Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"2","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21822024-04-20T01:06:02.796Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289961359Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"2","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21832024-04-20T01:06:02.796Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.289988339Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21842024-04-20T01:06:02.797Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.290020999Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21852024-04-20T01:06:02.797Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.29005564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21862024-04-20T01:06:02.797Z{"msg":"ds_connection connected from Ok(127.0.0.1:49372)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.29024303Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21872024-04-20T01:06:02.797Z{"msg":"ds_connection connected from Ok(127.0.0.1:46337)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.29029308Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21882024-04-20T01:06:02.797Z{"msg":"ds_connection connected from Ok(127.0.0.1:54509)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.29032258Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21892024-04-20T01:06:02.798Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.291030462Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"0","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21902024-04-20T01:06:02.798Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.291069912Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"0","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21912024-04-20T01:06:02.798Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.291096352Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"2","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21922024-04-20T01:06:02.798Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.291121092Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"2","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21932024-04-20T01:06:02.798Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.291293612Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"1","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21942024-04-20T01:06:02.799Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.291333332Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"1","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21952024-04-20T01:06:02.799Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.291604523Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"2","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21962024-04-20T01:06:02.799Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.291654523Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"0","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21972024-04-20T01:06:02.799Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.291885813Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"1","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21982024-04-20T01:06:02.799Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292112404Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"2","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
21992024-04-20T01:06:02.799Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292152254Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"2","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22002024-04-20T01:06:02.800Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292168674Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22012024-04-20T01:06:02.800Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292189904Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"0","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22022024-04-20T01:06:02.800Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292204944Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"0","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22032024-04-20T01:06:02.800Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292218864Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22042024-04-20T01:06:02.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292375074Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"1","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22052024-04-20T01:06:02.824Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292407985Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"1","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22062024-04-20T01:06:02.825Z{"msg":"[0]R flush_numbers[0..12]: [14, 27, 31, 31, 23, 32, 30, 24, 30, 30, 26, 30]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292424135Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22072024-04-20T01:06:02.825Z{"msg":"[0]R generation[0..12]: [8, 16, 18, 18, 14, 19, 18, 14, 18, 18, 16, 18]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292451655Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22082024-04-20T01:06:02.825Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292467085Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22092024-04-20T01:06:02.825Z{"msg":"[1]R flush_numbers[0..12]: [14, 27, 31, 31, 23, 32, 30, 24, 30, 30, 26, 30]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292488395Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22102024-04-20T01:06:02.825Z{"msg":"[1]R generation[0..12]: [8, 16, 18, 18, 14, 19, 18, 14, 18, 18, 16, 18]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292502345Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22112024-04-20T01:06:02.825Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292524165Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22122024-04-20T01:06:02.825Z{"msg":"[2]R flush_numbers[0..12]: [14, 27, 31, 31, 23, 32, 30, 24, 30, 30, 26, 30]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292538055Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22132024-04-20T01:06:02.825Z{"msg":"[2]R generation[0..12]: [8, 16, 18, 18, 14, 19, 18, 14, 18, 18, 16, 18]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292575485Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22142024-04-20T01:06:02.826ZWait for a query_work_queue command to finish before sending IO
22152024-04-20T01:06:02.826Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292616615Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22162024-04-20T01:06:02.826Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292653085Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22172024-04-20T01:06:02.826Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292677285Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22182024-04-20T01:06:02.826Z{"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292691785Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22192024-04-20T01:06:02.826Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292717975Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"":"downstairs","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22202024-04-20T01:06:02.827Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292754466Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22212024-04-20T01:06:02.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292771326Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"0","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22222024-04-20T01:06:02.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292789526Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"1","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22232024-04-20T01:06:02.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292812715Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"client":"2","":"downstairs","session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22242024-04-20T01:06:02.827Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292844916Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22252024-04-20T01:06:02.827Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292868316Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22262024-04-20T01:06:02.828Z{"msg":"25fb2110-5e6a-4f02-a736-6c2661a6ba87 is now active with session: d8aa1f59-a991-4dd7-8e54-b4b8812b496f","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292904246Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22272024-04-20T01:06:02.828Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292921656Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248,"session_id":"d8aa1f59-a991-4dd7-8e54-b4b8812b496f"}
22282024-04-20T01:06:02.828Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:01.292942086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248}
22292024-04-20T01:06:02.828ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22302024-04-20T01:06:02.828ZLoading write count information from file "/tmp/test_repair_verify.data"
22312024-04-20T01:06:02.828ZRun Repair workload
22322024-04-20T01:06:02.828Z01/30 Write block 337 len 20480 data: 2 2 2 3 3
22332024-04-20T01:06:02.828Z02/30 Write block 474 len 36864 data: 4 4 4 4 3 2 2 4 4
22342024-04-20T01:06:02.828Z03/30 Read block 309 len 32768
22352024-04-20T01:06:02.828Z04/30 Read block 135 len 32768
22362024-04-20T01:06:02.828Z05/30 Write block 226 len 12288 data: 4 3 3
22372024-04-20T01:06:02.828Z06/30 Write block 282 len 16384 data: 2 2 3 3
22382024-04-20T01:06:02.829Z07/30 Read block 522 len 16384
22392024-04-20T01:06:02.829Z08/30 Read block 464 len 28672
22402024-04-20T01:06:02.829Z09/30 Write block 447 len 16384 data: 4 3 3 3
22412024-04-20T01:06:02.829Z10/30 Write block 436 len 36864 data: 4 3 2 3 3 4 5 6 5
22422024-04-20T01:06:02.829Z11/30 Write block 587 len 16384 data: 5 5 5 5
22432024-04-20T01:06:02.829Z12/30 Read block 198 len 36864
22442024-04-20T01:06:02.830Z13/30 Write block 473 len 12288 data: 4 5 5
22452024-04-20T01:06:02.830Z14/30 Read block 114 len 4096
22462024-04-20T01:06:02.830Z15/30 Read block 82 len 40960
22472024-04-20T01:06:02.830Z16/30 Read block 140 len 20480
22482024-04-20T01:06:02.830Z17/30 Read block 498 len 40960
22492024-04-20T01:06:02.830Z18/30 Write block 150 len 8192 data: 3 3
22502024-04-20T01:06:02.830Z19/30 Write block 110 len 24576 data: 2 2 2 2 2 2
22512024-04-20T01:06:02.831Z20/30 Read block 441 len 32768
22522024-04-20T01:06:02.831Z21/30 Write block 98 len 8192 data: 2 2
22532024-04-20T01:06:02.831Z22/30 Read block 497 len 24576
22542024-04-20T01:06:02.831Z23/30 Read block 26 len 24576
22552024-04-20T01:06:02.831Z24/30 Read block 524 len 20480
22562024-04-20T01:06:02.831Z25/30 Read block 175 len 24576
22572024-04-20T01:06:02.832Z26/30 Read block 327 len 20480
22582024-04-20T01:06:02.832Z27/30 Read block 371 len 12288
22592024-04-20T01:06:02.832Z28/30 Write block 511 len 20480 data: 3 4 5 4 4
22602024-04-20T01:06:02.832Z29/30 Write block 358 len 32768 data: 2 2 2 3 3 3 2 3
22612024-04-20T01:06:02.832Z30/30 Write block 551 len 8192 data: 4 4
22622024-04-20T01:06:02.832Z----------------------------------------------------------------
22632024-04-20T01:06:02.832Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22642024-04-20T01:06:02.833ZStates: Active Active Active
22652024-04-20T01:06:02.833ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22662024-04-20T01:06:02.833Z 1 Acked 1000 Write 5 Done Done Done false
22672024-04-20T01:06:02.833Z 2 Acked 1001 Write 9 Done Done Done false
22682024-04-20T01:06:02.833Z 3 Acked 1002 Read 8 Done Sent Done false
22692024-04-20T01:06:02.833Z 4 Acked 1003 Read 8 Done Sent Done false
22702024-04-20T01:06:02.833Z 5 Acked 1004 Write 3 Done Sent Done false
22712024-04-20T01:06:02.834Z 6 Acked 1005 Write 4 Done Sent Done false
22722024-04-20T01:06:02.834Z 7 Acked 1006 Read 4 Done Sent Done false
22732024-04-20T01:06:02.834Z 8 Acked 1007 Read 7 Done Sent Done false
22742024-04-20T01:06:02.834Z 9 Acked 1008 Write 4 Done Sent Done false
22752024-04-20T01:06:02.834Z 10 Acked 1009 Write 9 Done Sent Done false
22762024-04-20T01:06:02.834Z 11 Acked 1010 Write 4 Done Sent Done false
22772024-04-20T01:06:02.834Z 12 Acked 1011 Read 9 Done Sent Done false
22782024-04-20T01:06:02.835Z 13 Acked 1012 Write 3 Done Sent Done false
22792024-04-20T01:06:02.835Z 14 Acked 1013 Read 1 Done Sent Done false
22802024-04-20T01:06:02.835Z 15 Acked 1014 Read 10 Done Sent Done false
22812024-04-20T01:06:02.835Z 16 Acked 1015 Read 5 Done Sent Done false
22822024-04-20T01:06:02.835Z 17 Acked 1016 Read 10 Done Sent Done false
22832024-04-20T01:06:02.835Z 18 Acked 1017 Write 2 Done Sent Done false
22842024-04-20T01:06:02.835Z 19 Acked 1018 Write 6 Done Sent Done false
22852024-04-20T01:06:02.835Z 20 Acked 1019 Read 8 Done Sent Done false
22862024-04-20T01:06:02.835Z 21 Acked 1020 Write 2 Done Sent Done false
22872024-04-20T01:06:02.835Z 22 Acked 1021 Read 6 Done Sent Done false
22882024-04-20T01:06:02.835Z 23 Acked 1022 Read 6 Done Sent Done false
22892024-04-20T01:06:02.835Z 24 Acked 1023 Read 5 Done Sent Done false
22902024-04-20T01:06:02.836Z 25 Acked 1024 Read 6 Done Sent Done false
22912024-04-20T01:06:02.836Z 26 Acked 1025 Read 5 Done Sent Done false
22922024-04-20T01:06:02.836Z 27 Acked 1026 Read 3 Done Sent Done false
22932024-04-20T01:06:02.836Z 28 Acked 1027 Write 5 Done Sent Done false
22942024-04-20T01:06:02.836Z 29 Acked 1028 Write 8 Done Sent Done false
22952024-04-20T01:06:02.836Z 30 Acked 1029 Write 2 Sent Sent Sent false
22962024-04-20T01:06:02.836Z STATES DS:0 DS:1 DS:2 TOTAL
22972024-04-20T01:06:02.836Z New 0 0 0 0
22982024-04-20T01:06:02.836Z Sent 1 28 1 30
22992024-04-20T01:06:02.836Z Done 29 2 29 60
23002024-04-20T01:06:02.836Z Skipped 0 0 0 0
23012024-04-20T01:06:02.836Z Error 0 0 0 0
23022024-04-20T01:06:02.836ZLast Flush: 0 0 0
23032024-04-20T01:06:02.837ZDownstairs last five completed:
23042024-04-20T01:06:03.266ZUpstairs last five completed: 30 29 28 27 26
23052024-04-20T01:06:03.266Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:06:01.319022424Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1248}
23062024-04-20T01:06:03.266ZWrote out file "/tmp/test_repair_verify.data"
23072024-04-20T01:06:03.266Z
23082024-04-20T01:06:03.266ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23092024-04-20T01:06:03.266ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23102024-04-20T01:06:03.266Z 0 000-019 8 8 8 14 14 14 F F F
23112024-04-20T01:06:03.266Z 1 020-039 16 16 16 27 27 27 F F F
23122024-04-20T01:06:03.266Z 2 040-059 18 18 18 31 31 31 F F F
23132024-04-20T01:06:03.266Z 3 060-079 18 18 18 31 31 31 F F F
23142024-04-20T01:06:03.266Z 4 080-099 14 14 14 23 23 23 T F T <---
23152024-04-20T01:06:03.266Z 5 100-119 19 19 19 32 32 32 T F T <---
23162024-04-20T01:06:03.266Z 6 120-139 18 18 18 30 30 30 F F F
23172024-04-20T01:06:03.267Z 7 140-159 14 14 14 24 24 24 T F T <---
23182024-04-20T01:06:03.267Z 8 160-179 18 18 18 30 30 30 F F F
23192024-04-20T01:06:03.267Z 9 180-199 18 18 18 30 30 30 F F F
23202024-04-20T01:06:03.267Z 10 200-219 16 16 16 26 26 26 F F F
23212024-04-20T01:06:03.267Z 11 220-239 18 18 18 30 30 30 T F T <---
23222024-04-20T01:06:03.267Z 12 240-259 18 18 18 31 31 31 F F F
23232024-04-20T01:06:03.267Z 13 260-279 17 17 17 28 28 28 F F F
23242024-04-20T01:06:03.267Z 14 280-299 9 9 9 16 16 16 T F T <---
23252024-04-20T01:06:03.267Z 15 300-319 15 15 15 25 25 25 F F F
23262024-04-20T01:06:03.267Z 16 320-339 15 15 15 25 25 25 T T T
23272024-04-20T01:06:03.267Z 17 340-359 16 16 16 26 26 26 T T T
23282024-04-20T01:06:03.267Z 18 360-379 18 18 18 31 31 31 T F T <---
23292024-04-20T01:06:03.267Z 19 380-399 15 15 15 25 25 25 F F F
23302024-04-20T01:06:03.267Z 20 400-419 19 19 19 32 32 32 F F F
23312024-04-20T01:06:03.267Z 21 420-439 18 18 18 30 30 30 T F T <---
23322024-04-20T01:06:03.267Z 22 440-459 19 19 19 32 32 32 T F T <---
23332024-04-20T01:06:03.267Z 23 460-479 19 19 19 32 32 32 T T T
23342024-04-20T01:06:03.267Z 24 480-499 16 16 16 26 26 26 T T T
23352024-04-20T01:06:03.267Z 25 500-519 15 15 15 25 25 25 T F T <---
23362024-04-20T01:06:03.267Z 26 520-539 13 13 13 22 22 22 F F F
23372024-04-20T01:06:03.267Z 27 540-559 16 16 16 27 27 27 T F T <---
23382024-04-20T01:06:03.267Z 28 560-579 17 17 17 28 28 28 F F F
23392024-04-20T01:06:03.267Z 29 580-599 17 17 17 28 28 28 T F T <---
23402024-04-20T01:06:03.267ZMax gen: 19, Max flush: 32
23412024-04-20T01:06:03.268ZError: Difference in extent metadata found!
23422024-04-20T01:06:03.268ZA Difference in extent metadata is expected here
23432024-04-20T01:06:03.268ZOn loop 10
23442024-04-20T01:06:03.268Z
23452024-04-20T01:06:03.268ZVerifying data now
23462024-04-20T01:06:17.524ZNo change after verify
23472024-04-20T01:06:17.528ZLoop: 10 Downstairs dump after verify (and repair):
23482024-04-20T01:06:17.540ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23492024-04-20T01:06:17.540Z 0 000-019 8 8 8 14 14 14 F F F
23502024-04-20T01:06:17.540Z 1 020-039 16 16 16 27 27 27 F F F
23512024-04-20T01:06:17.540Z 2 040-059 18 18 18 31 31 31 F F F
23522024-04-20T01:06:17.540Z 3 060-079 18 18 18 31 31 31 F F F
23532024-04-20T01:06:17.543Z 4 080-099 20 20 20 33 33 33 F F F
23542024-04-20T01:06:17.543Z 5 100-119 20 20 20 33 33 33 F F F
23552024-04-20T01:06:17.544Z 6 120-139 18 18 18 30 30 30 F F F
23562024-04-20T01:06:17.544Z 7 140-159 20 20 20 33 33 33 F F F
23572024-04-20T01:06:17.544Z 8 160-179 18 18 18 30 30 30 F F F
23582024-04-20T01:06:17.544Z 9 180-199 18 18 18 30 30 30 F F F
23592024-04-20T01:06:17.544Z 10 200-219 16 16 16 26 26 26 F F F
23602024-04-20T01:06:17.544Z 11 220-239 20 20 20 33 33 33 F F F
23612024-04-20T01:06:17.544Z 12 240-259 18 18 18 31 31 31 F F F
23622024-04-20T01:06:17.544Z 13 260-279 17 17 17 28 28 28 F F F
23632024-04-20T01:06:17.544Z 14 280-299 20 20 20 33 33 33 F F F
23642024-04-20T01:06:17.544Z 15 300-319 15 15 15 25 25 25 F F F
23652024-04-20T01:06:17.544Z 16 320-339 20 20 20 33 33 33 F F F
23662024-04-20T01:06:17.544Z 17 340-359 20 20 20 33 33 33 F F F
23672024-04-20T01:06:17.544Z 18 360-379 20 20 20 33 33 33 F F F
23682024-04-20T01:06:17.544Z 19 380-399 15 15 15 25 25 25 F F F
23692024-04-20T01:06:17.544Z 20 400-419 19 19 19 32 32 32 F F F
23702024-04-20T01:06:17.544Z 21 420-439 20 20 20 33 33 33 F F F
23712024-04-20T01:06:17.544Z 22 440-459 20 20 20 33 33 33 F F F
23722024-04-20T01:06:17.544Z 23 460-479 20 20 20 33 33 33 F F F
23732024-04-20T01:06:17.544Z 24 480-499 20 20 20 33 33 33 F F F
23742024-04-20T01:06:17.544Z 25 500-519 20 20 20 33 33 33 F F F
23752024-04-20T01:06:17.544Z 26 520-539 13 13 13 22 22 22 F F F
23762024-04-20T01:06:17.544Z 27 540-559 20 20 20 33 33 33 F F F
23772024-04-20T01:06:17.544Z 28 560-579 17 17 17 28 28 28 F F F
23782024-04-20T01:06:17.545Z 29 580-599 20 20 20 33 33 33 F F F
23792024-04-20T01:06:17.545ZMax gen: 20, Max flush: 33
23802024-04-20T01:06:17.545Z
23812024-04-20T01:06:17.545ZBegin loop 11
23822024-04-20T01:06:17.545ZDownstairs to restart: 1
23832024-04-20T01:06:17.561Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.071542653Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256}
23842024-04-20T01:06:17.561Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.071767053Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256}
23852024-04-20T01:06:17.568Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.071802263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256}
23862024-04-20T01:06:17.568Z{"msg":"Crucible 06b420cf-3875-4190-ab14-0e30039daabd has session id: c1653899-140b-4b18-b615-3d41523b6770","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.071829013Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
23872024-04-20T01:06:17.568ZCrucible runtime is spawned
23882024-04-20T01:06:17.568Z{"msg":"Upstairs opts: Upstairs UUID: 06b420cf-3875-4190-ab14-0e30039daabd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.071896484Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
23892024-04-20T01:06:17.569Z{"msg":"Crucible stats registered with UUID: 06b420cf-3875-4190-ab14-0e30039daabd","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.071936014Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
23902024-04-20T01:06:17.569Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.071958494Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256}
23912024-04-20T01:06:17.569Z{"msg":"06b420cf-3875-4190-ab14-0e30039daabd active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.071991994Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
23922024-04-20T01:06:17.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.072088944Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"0","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
23932024-04-20T01:06:17.569Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.072131314Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"0","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
23942024-04-20T01:06:17.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.072152954Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"1","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
23952024-04-20T01:06:17.569Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.072175804Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"1","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
23962024-04-20T01:06:17.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.072198104Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"2","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
23972024-04-20T01:06:17.569Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.072247724Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"2","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
23982024-04-20T01:06:17.569Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.072314205Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
23992024-04-20T01:06:17.569Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.072344965Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24002024-04-20T01:06:17.569Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.072372725Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24012024-04-20T01:06:17.569Z{"msg":"ds_connection connected from Ok(127.0.0.1:62725)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.072547795Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24022024-04-20T01:06:17.569Z{"msg":"ds_connection connected from Ok(127.0.0.1:56231)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.072595505Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24032024-04-20T01:06:17.569Z{"msg":"ds_connection connected from Ok(127.0.0.1:43331)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.072623475Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24042024-04-20T01:06:17.569Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.073111396Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"2","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24052024-04-20T01:06:17.569Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.073165906Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"2","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24062024-04-20T01:06:17.569Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.073195756Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"0","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24072024-04-20T01:06:17.569Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.073230427Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"0","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24082024-04-20T01:06:17.569Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.073269377Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"1","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24092024-04-20T01:06:17.569Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.073300727Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"1","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24102024-04-20T01:06:17.570Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.073609267Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"2","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24112024-04-20T01:06:17.570Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.073771078Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"0","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24122024-04-20T01:06:17.570Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.074093209Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"2","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24132024-04-20T01:06:17.570Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.074142509Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"2","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24142024-04-20T01:06:17.570Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.074170729Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24152024-04-20T01:06:17.570Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.074256969Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"1","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24162024-04-20T01:06:17.570Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.074293099Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"0","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24172024-04-20T01:06:17.570Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.074318349Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"0","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24182024-04-20T01:06:17.570Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.074346319Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24192024-04-20T01:06:17.570Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.074553839Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"1","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24202024-04-20T01:06:17.570Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07459977Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"1","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24212024-04-20T01:06:17.570Z{"msg":"[0]R flush_numbers[0..12]: [14, 27, 31, 31, 33, 33, 30, 33, 30, 30, 26, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07463515Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24222024-04-20T01:06:17.570Z{"msg":"[0]R generation[0..12]: [8, 16, 18, 18, 20, 20, 18, 20, 18, 18, 16, 20]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07466162Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24232024-04-20T01:06:17.570Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07469494Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24242024-04-20T01:06:17.570Z{"msg":"[1]R flush_numbers[0..12]: [14, 27, 31, 31, 33, 33, 30, 33, 30, 30, 26, 33]","v":0,"nameWait for a query_work_queue command to finish before sending IO
24252024-04-20T01:06:17.570Z":"crucible","level":30,"time":"2024-04-20T01:06:16.0747239Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24262024-04-20T01:06:17.570Z{"msg":"[1]R generation[0..12]: [8, 16, 18, 18, 20, 20, 18, 20, 18, 18, 16, 20]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07474909Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24272024-04-20T01:06:17.570Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07477868Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24282024-04-20T01:06:17.570Z{"msg":"[2]R flush_numbers[0..12]: [14, 27, 31, 31, 33, 33, 30, 33, 30, 30, 26, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07480403Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24292024-04-20T01:06:17.570Z{"msg":"[2]R generation[0..12]: [8, 16, 18, 18, 20, 20, 18, 20, 18, 18, 16, 20]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07483538Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24302024-04-20T01:06:17.570Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07486131Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24312024-04-20T01:06:17.570Z{"msg":"Max found gen is 21","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.0748846Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24322024-04-20T01:06:17.570Z{"msg":"Generation requested: 22 >= found:21","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07491338Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24332024-04-20T01:06:17.571Z{"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07493687Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24342024-04-20T01:06:17.571Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07496386Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"":"downstairs","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24352024-04-20T01:06:17.571Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07499414Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24362024-04-20T01:06:17.571ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24372024-04-20T01:06:17.571Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.07502982Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"0","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24382024-04-20T01:06:17.571Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.075055271Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"1","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24392024-04-20T01:06:17.571Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.075084691Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"client":"2","":"downstairs","session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24402024-04-20T01:06:17.571Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.075108491Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24412024-04-20T01:06:17.571Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.075134521Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24422024-04-20T01:06:17.571Z{"msg":"06b420cf-3875-4190-ab14-0e30039daabd is now active with session: c1653899-140b-4b18-b615-3d41523b6770","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.075157891Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24432024-04-20T01:06:17.571Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.075184781Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256,"session_id":"c1653899-140b-4b18-b615-3d41523b6770"}
24442024-04-20T01:06:17.571Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:16.075213291Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256}
24452024-04-20T01:06:17.575ZLoading write count information from file "/tmp/test_repair_verify.data"
24462024-04-20T01:06:17.575ZRun Repair workload
24472024-04-20T01:06:17.575Z01/30 Write block 220 len 4096 data: 3
24482024-04-20T01:06:17.575Z02/30 Write block 561 len 24576 data: 2 2 2 2 2 3
24492024-04-20T01:06:17.575Z03/30 Write block 402 len 8192 data: 3 2
24502024-04-20T01:06:17.575Z04/30 Read block 578 len 4096
24512024-04-20T01:06:17.575Z05/30 Read block 426 len 32768
24522024-04-20T01:06:17.575Z06/30 Read block 153 len 36864
24532024-04-20T01:06:17.575Z07/30 Flush
24542024-04-20T01:06:17.575Z08/30 Write block 47 len 12288 data: 3 3 4
24552024-04-20T01:06:17.575Z09/30 Write block 475 len 16384 data: 6 5 5 4
24562024-04-20T01:06:17.575Z10/30 Read block 452 len 8192
24572024-04-20T01:06:17.576Z11/30 Write block 576 len 28672 data: 3 3 3 3 2 2 2
24582024-04-20T01:06:17.579Z12/30 Write block 138 len 20480 data: 4 4 4 3 3
24592024-04-20T01:06:17.579Z13/30 Write block 461 len 36864 data: 6 5 5 3 3 3 2 2 3
24602024-04-20T01:06:17.579Z14/30 Write block 543 len 16384 data: 4 4 5 5
24612024-04-20T01:06:17.580Z15/30 Write block 49 len 24576 data: 5 6 7 7 6 4
24622024-04-20T01:06:17.583Z16/30 Read block 134 len 20480
24632024-04-20T01:06:17.583Z17/30 Flush
24642024-04-20T01:06:17.587Z18/30 Write block 525 len 40960 data: 4 3 2 2 2 2 2 2 2 2
24652024-04-20T01:06:17.587Z19/30 Read block 376 len 16384
24662024-04-20T01:06:17.587Z20/30 Write block 481 len 40960 data: 5 5 3 3 2 2 2 2 3 4
24672024-04-20T01:06:17.587Z21/30 Read block 196 len 4096
24682024-04-20T01:06:17.590Z22/30 Read block 541 len 24576
24692024-04-20T01:06:17.590Z23/30 Read block 199 len 20480
24702024-04-20T01:06:17.590Z24/30 Read block 358 len 20480
24712024-04-20T01:06:17.590Z25/30 Flush
24722024-04-20T01:06:17.594Z26/30 Write block 212 len 12288 data: 2 2 2
24732024-04-20T01:06:17.594Z27/30 Write block 195 len 32768 data: 2 2 3 3 3 2 2 2
24742024-04-20T01:06:17.594Z28/30 Read block 365 len 20480
24752024-04-20T01:06:17.601Z29/30 Write block 475 len 8192 data: 7 6
24762024-04-20T01:06:17.601Z30/30 Read block 194 len 20480
24772024-04-20T01:06:17.601Z----------------------------------------------------------------
24782024-04-20T01:06:17.601Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30
24792024-04-20T01:06:17.601ZStates: Active Active Active
24802024-04-20T01:06:17.601ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24812024-04-20T01:06:17.601Z 1 Acked 1000 Write 1 Done Done Done false
24822024-04-20T01:06:17.601Z 2 Acked 1001 Write 6 Done Sent Done false
24832024-04-20T01:06:17.601Z 3 Acked 1002 Write 2 Done Sent Done false
24842024-04-20T01:06:17.601Z 4 Acked 1003 Read 1 Done Sent Done false
24852024-04-20T01:06:17.601Z 5 Acked 1004 Read 8 Done Sent Done false
24862024-04-20T01:06:17.601Z 6 Acked 1005 Read 9 Done Sent Done false
24872024-04-20T01:06:17.601Z 7 Acked 1006 Flush 0 Done Sent Done false
24882024-04-20T01:06:17.601Z 8 Acked 1007 Write 3 Done Sent Done false
24892024-04-20T01:06:17.601Z 9 Acked 1008 Write 4 Done Sent Done false
24902024-04-20T01:06:17.601Z 10 Acked 1009 Read 2 Done Sent Done false
24912024-04-20T01:06:17.601Z 11 Acked 1010 Write 7 Done Sent Done false
24922024-04-20T01:06:17.601Z 12 Acked 1011 Write 5 Done Sent Done false
24932024-04-20T01:06:17.602Z 13 Acked 1012 Write 9 Done Sent Done false
24942024-04-20T01:06:17.602Z 14 Acked 1013 Write 4 Done Sent Done false
24952024-04-20T01:06:17.602Z 15 Acked 1014 Write 6 Done Sent Done false
24962024-04-20T01:06:17.602Z 16 Acked 1015 Read 5 Done Sent Done false
24972024-04-20T01:06:17.602Z 17 Acked 1016 Flush 0 Done Sent Done false
24982024-04-20T01:06:17.602Z 18 Acked 1017 Write 10 Done Sent Done false
24992024-04-20T01:06:17.602Z 19 Acked 1018 Read 4 Done Sent Done false
25002024-04-20T01:06:17.602Z 20 Acked 1019 Write 10 Done Sent Done false
25012024-04-20T01:06:17.602Z 21 Acked 1020 Read 1 Done Sent Done false
25022024-04-20T01:06:17.602Z 22 Acked 1021 Read 6 Done Sent Done false
25032024-04-20T01:06:17.602Z 23 Acked 1022 Read 5 Done Sent Done false
25042024-04-20T01:06:17.602Z 24 Acked 1023 Read 5 Done Sent Done false
25052024-04-20T01:06:17.602Z 25 Acked 1024 Flush 0 Done Sent Done false
25062024-04-20T01:06:17.602Z 26 Acked 1025 Write 3 Done Sent Done false
25072024-04-20T01:06:17.602Z 27 Acked 1026 Write 8 Done Sent Done false
25082024-04-20T01:06:17.602Z 28 Acked 1027 Read 5 Done Sent Done false
25092024-04-20T01:06:17.602Z 29 Acked 1028 Write 2 Done Sent Done false
25102024-04-20T01:06:17.602Z 30 Acked 1029 Read 5 Done Sent Done false
25112024-04-20T01:06:17.602Z STATES DS:0 DS:1 DS:2 TOTAL
25122024-04-20T01:06:17.602Z New 0 0 0 0
25132024-04-20T01:06:17.602Z Sent 0 29 0 29
25142024-04-20T01:06:17.602Z Done 30 1 30 61
25152024-04-20T01:06:17.602Z Skipped 0 0 0 0
25162024-04-20T01:06:17.602Z Error 0 0 0 0
25172024-04-20T01:06:17.603ZLast Flush: 1024 0 1024
25182024-04-20T01:06:17.603ZDownstairs last five completed:
25192024-04-20T01:06:17.603ZUpstairs last five completed: 30 29 28 27 26
25202024-04-20T01:06:17.603Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:06:16.106921031Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1256}
25212024-04-20T01:06:17.603ZWrote out file "/tmp/test_repair_verify.data"
25222024-04-20T01:06:17.607Z
25232024-04-20T01:06:17.607ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25242024-04-20T01:06:17.618ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25252024-04-20T01:06:17.618Z 0 000-019 8 8 8 14 14 14 F F F
25262024-04-20T01:06:17.622Z 1 020-039 16 16 16 27 27 27 F F F
25272024-04-20T01:06:17.623Z 2 040-059 22 18 22 35 31 35 F F F <---
25282024-04-20T01:06:17.623Z 3 060-079 18 18 18 31 31 31 F F F
25292024-04-20T01:06:17.623Z 4 080-099 20 20 20 33 33 33 F F F
25302024-04-20T01:06:17.623Z 5 100-119 20 20 20 33 33 33 F F F
25312024-04-20T01:06:17.623Z 6 120-139 22 18 22 35 30 35 F F F <---
25322024-04-20T01:06:17.623Z 7 140-159 22 20 22 35 33 35 F F F <---
25332024-04-20T01:06:17.623Z 8 160-179 18 18 18 30 30 30 F F F
25342024-04-20T01:06:17.623Z 9 180-199 18 18 18 30 30 30 T F T <---
25352024-04-20T01:06:17.623Z 10 200-219 16 16 16 26 26 26 T F T <---
25362024-04-20T01:06:17.623Z 11 220-239 22 20 22 34 33 34 F T F <---
25372024-04-20T01:06:17.623Z 12 240-259 18 18 18 31 31 31 F F F
25382024-04-20T01:06:17.623Z 13 260-279 17 17 17 28 28 28 F F F
25392024-04-20T01:06:17.623Z 14 280-299 20 20 20 33 33 33 F F F
25402024-04-20T01:06:17.623Z 15 300-319 15 15 15 25 25 25 F F F
25412024-04-20T01:06:17.623Z 16 320-339 20 20 20 33 33 33 F F F
25422024-04-20T01:06:17.623Z 17 340-359 20 20 20 33 33 33 F F F
25432024-04-20T01:06:17.623Z 18 360-379 20 20 20 33 33 33 F F F
25442024-04-20T01:06:17.623Z 19 380-399 15 15 15 25 25 25 F F F
25452024-04-20T01:06:17.623Z 20 400-419 22 19 22 34 32 34 F F F <---
25462024-04-20T01:06:17.623Z 21 420-439 20 20 20 33 33 33 F F F
25472024-04-20T01:06:17.623Z 22 440-459 20 20 20 33 33 33 F F F
25482024-04-20T01:06:17.623Z 23 460-479 22 20 22 35 33 35 T F T <---
25492024-04-20T01:06:17.623Z 24 480-499 22 20 22 36 33 36 F F F <---
25502024-04-20T01:06:17.624Z 25 500-519 20 20 20 33 33 33 F F F
25512024-04-20T01:06:17.624Z 26 520-539 22 13 22 36 22 36 F F F <---
25522024-04-20T01:06:17.624Z 27 540-559 22 20 22 35 33 35 F F F <---
25532024-04-20T01:06:17.624Z 28 560-579 22 17 22 35 28 35 F F F <---
25542024-04-20T01:06:17.624Z 29 580-599 22 20 22 35 33 35 F F F <---
25552024-04-20T01:06:17.624ZMax gen: 22, Max flush: 36
25562024-04-20T01:06:17.624ZError: Difference in extent metadata found!
25572024-04-20T01:06:17.624ZA Difference in extent metadata is expected here
25582024-04-20T01:06:17.624ZOn loop 11
25592024-04-20T01:06:17.624Z
25602024-04-20T01:06:17.628ZVerifying data now
25612024-04-20T01:06:22.195ZNo change after verify
25622024-04-20T01:06:22.196ZLoop: 11 Downstairs dump after verify (and repair):
25632024-04-20T01:06:22.210ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25642024-04-20T01:06:22.210Z 0 000-019 8 8 8 14 14 14 F F F
25652024-04-20T01:06:22.214Z 1 020-039 16 16 16 27 27 27 F F F
25662024-04-20T01:06:22.214Z 2 040-059 22 22 22 35 35 35 F F F
25672024-04-20T01:06:22.214Z 3 060-079 18 18 18 31 31 31 F F F
25682024-04-20T01:06:22.214Z 4 080-099 20 20 20 33 33 33 F F F
25692024-04-20T01:06:22.215Z 5 100-119 20 20 20 33 33 33 F F F
25702024-04-20T01:06:22.215Z 6 120-139 22 22 22 35 35 35 F F F
25712024-04-20T01:06:22.215Z 7 140-159 22 22 22 35 35 35 F F F
25722024-04-20T01:06:22.215Z 8 160-179 18 18 18 30 30 30 F F F
25732024-04-20T01:06:22.215Z 9 180-199 23 23 23 37 37 37 F F F
25742024-04-20T01:06:22.215Z 10 200-219 23 23 23 37 37 37 F F F
25752024-04-20T01:06:22.215Z 11 220-239 22 22 22 34 34 34 F F F
25762024-04-20T01:06:22.215Z 12 240-259 18 18 18 31 31 31 F F F
25772024-04-20T01:06:22.215Z 13 260-279 17 17 17 28 28 28 F F F
25782024-04-20T01:06:22.215Z 14 280-299 20 20 20 33 33 33 F F F
25792024-04-20T01:06:22.215Z 15 300-319 15 15 15 25 25 25 F F F
25802024-04-20T01:06:22.215Z 16 320-339 20 20 20 33 33 33 F F F
25812024-04-20T01:06:22.215Z 17 340-359 20 20 20 33 33 33 F F F
25822024-04-20T01:06:22.215Z 18 360-379 20 20 20 33 33 33 F F F
25832024-04-20T01:06:22.215Z 19 380-399 15 15 15 25 25 25 F F F
25842024-04-20T01:06:22.215Z 20 400-419 22 22 22 34 34 34 F F F
25852024-04-20T01:06:22.215Z 21 420-439 20 20 20 33 33 33 F F F
25862024-04-20T01:06:22.215Z 22 440-459 20 20 20 33 33 33 F F F
25872024-04-20T01:06:22.216Z 23 460-479 23 23 23 37 37 37 F F F
25882024-04-20T01:06:22.216Z 24 480-499 22 22 22 36 36 36 F F F
25892024-04-20T01:06:22.216Z 25 500-519 20 20 20 33 33 33 F F F
25902024-04-20T01:06:22.216Z 26 520-539 22 22 22 36 36 36 F F F
25912024-04-20T01:06:22.216Z 27 540-559 22 22 22 35 35 35 F F F
25922024-04-20T01:06:22.216Z 28 560-579 22 22 22 35 35 35 F F F
25932024-04-20T01:06:22.216Z 29 580-599 22 22 22 35 35 35 F F F
25942024-04-20T01:06:22.216ZMax gen: 23, Max flush: 37
25952024-04-20T01:06:22.216Z
25962024-04-20T01:06:22.216ZBegin loop 12
25972024-04-20T01:06:22.216ZDownstairs to restart: 0
25982024-04-20T01:06:22.236Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.741801909Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264}
25992024-04-20T01:06:22.236Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.742065259Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264}
26002024-04-20T01:06:22.236Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.742082859Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264}
26012024-04-20T01:06:22.236Z{"msg":"Crucible 24bb8f2d-22eb-48a8-a124-0de5586cb76c has session id: 20eecabe-1077-4c53-bb62-a60f4cdc4c9a","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.742096399Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26022024-04-20T01:06:22.244Z{"msg":"Upstairs opts: Upstairs UUID: 24bb8f2d-22eb-48a8-a124-0de5586cb76c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.742123859Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26032024-04-20T01:06:22.244Z{"msg":"Crucible stats registered with UUID: 24bb8f2d-22eb-48a8-a124-0de5586cb76c","v":0,"name":"crucible","level":30Crucible runtime is spawned
26042024-04-20T01:06:22.244Z,"time":"2024-04-20T01:06:20.742182369Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26052024-04-20T01:06:22.244Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.74228242Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264}
26062024-04-20T01:06:22.244Z{"msg":"24bb8f2d-22eb-48a8-a124-0de5586cb76c active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.74236768Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26072024-04-20T01:06:22.244Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.74239794Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"0","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26082024-04-20T01:06:22.244Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.74241391Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"0","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26092024-04-20T01:06:22.245Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.74244688Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"1","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26102024-04-20T01:06:22.245Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.74248102Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"1","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26112024-04-20T01:06:22.245Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.74250053Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"2","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26122024-04-20T01:06:22.245Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.74252433Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"2","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26132024-04-20T01:06:22.245Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.742592441Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26142024-04-20T01:06:22.245Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.74262487Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26152024-04-20T01:06:22.245Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.74265131Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26162024-04-20T01:06:22.245Z{"msg":"ds_connection connected from Ok(127.0.0.1:59621)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.742784301Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26172024-04-20T01:06:22.245Z{"msg":"ds_connection connected from Ok(127.0.0.1:34799)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.742985831Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26182024-04-20T01:06:22.245Z{"msg":"ds_connection connected from Ok(127.0.0.1:33910)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.743051062Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26192024-04-20T01:06:22.245Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.743487952Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"2","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26202024-04-20T01:06:22.245Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.743531603Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"2","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26212024-04-20T01:06:22.245Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.743560553Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"1","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26222024-04-20T01:06:22.245Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.743586773Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"1","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26232024-04-20T01:06:22.245Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.743773703Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"0","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26242024-04-20T01:06:22.245Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.743829333Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"0","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26252024-04-20T01:06:22.245Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.744293644Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"1","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26262024-04-20T01:06:22.245Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.744331024Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"0","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26272024-04-20T01:06:22.245Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.744348194Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"2","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26282024-04-20T01:06:22.246Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.744930156Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"1","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26292024-04-20T01:06:22.246Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.744969546Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"1","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26302024-04-20T01:06:22.246Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.744984726Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26312024-04-20T01:06:22.246Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745004756Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"2","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26322024-04-20T01:06:22.246Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745019086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"2","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26332024-04-20T01:06:22.246Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745032536Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26342024-04-20T01:06:22.246Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745044826Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"0","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26352024-04-20T01:06:22.246Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745058096Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"0","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26362024-04-20T01:06:22.246Z{"msg":"[0]R flush_numbers[0..12]: [14, 27, 35, 31, 33, 33, 35, 35, 30, 37, 37, 34]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745071696Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26372024-04-20T01:06:22.246Z{"msg":"[0]R generation[0..12]: [8, 16, 22, 18, 20, 20, 22, 22, 18, 23, 23, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745085976Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26382024-04-20T01:06:22.246Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745099976Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26392024-04-20T01:06:22.246Z{"msg":"[1]R flush_numbers[0..12]: [14, 27, 35, 31, 33, 33, 35, 35, 30, 37, 37, 34]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745120456Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26402024-04-20T01:06:22.247Z{"msg":"[1]R generation[0..12]: [8, 16, 22, 18, 20, 20, 22, 22, 18, 23, 23, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745135126Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26412024-04-20T01:06:22.247Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745162026Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26422024-04-20T01:06:22.247Z{"msg":"[2]R flush_numbers[0..12]: [14, 27, 35, 31, 33, 33, 35, 35, 30, 37, 37, 34]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745176036Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26432024-04-20T01:06:22.247Z{"msg":"[2]R generation[0..12]: [8, 16, 22, 18, 20, 20, 22, 22, 18, 23, 23, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745189426Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264Wait for a query_work_queue command to finish before sending IO
26442024-04-20T01:06:22.247Z,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26452024-04-20T01:06:22.247Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745222836Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26462024-04-20T01:06:22.247Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745235756Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26472024-04-20T01:06:22.247Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745248876Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26482024-04-20T01:06:22.247Z{"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745261736Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26492024-04-20T01:06:22.247Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745274516Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"":"downstairs","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26502024-04-20T01:06:22.247Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745287596Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26512024-04-20T01:06:22.247Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745306636Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"0","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26522024-04-20T01:06:22.247Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745320296Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"1","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26532024-04-20T01:06:22.247Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745339836Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"client":"2","":"downstairs","session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26542024-04-20T01:06:22.248Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745353287Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26552024-04-20T01:06:22.248Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745365927Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26562024-04-20T01:06:22.248Z{"msg":"24bb8f2d-22eb-48a8-a124-0de5586cb76c is now active with session: 20eecabe-1077-4c53-bb62-a60f4cdc4c9a","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745384997Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26572024-04-20T01:06:22.248Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745397267Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264,"session_id":"20eecabe-1077-4c53-bb62-a60f4cdc4c9a"}
26582024-04-20T01:06:22.248Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:20.745410507Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264}
26592024-04-20T01:06:22.248ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26602024-04-20T01:06:22.248ZLoading write count information from file "/tmp/test_repair_verify.data"
26612024-04-20T01:06:22.248ZRun Repair workload
26622024-04-20T01:06:22.248Z01/30 Write block 543 len 8192 data: 5 5
26632024-04-20T01:06:22.248Z02/30 Read block 161 len 32768
26642024-04-20T01:06:22.248Z03/30 Read block 491 len 36864
26652024-04-20T01:06:22.248Z04/30 Write block 153 len 12288 data: 3 3 3
26662024-04-20T01:06:22.248Z05/30 Read block 83 len 8192
26672024-04-20T01:06:22.248Z06/30 Write block 24 len 16384 data: 2 2 3 4
26682024-04-20T01:06:22.253Z07/30 Write block 106 len 4096 data: 3
26692024-04-20T01:06:22.253Z08/30 Read block 415 len 40960
26702024-04-20T01:06:22.253Z09/30 Write block 231 len 36864 data: 6 6 6 6 6 6 7 6 6
26712024-04-20T01:06:22.253Z10/30 Read block 247 len 20480
26722024-04-20T01:06:22.253Z11/30 Read block 409 len 32768
26732024-04-20T01:06:22.253Z12/30 Read block 106 len 4096
26742024-04-20T01:06:22.253Z13/30 Write block 217 len 40960 data: 2 2 2 4 4 4 3 3 4 5
26752024-04-20T01:06:22.253Z14/30 Flush
26762024-04-20T01:06:22.253Z15/30 Write block 217 len 32768 data: 3 3 3 5 5 5 4 4
26772024-04-20T01:06:22.253Z16/30 Read block 49 len 32768
26782024-04-20T01:06:22.253Z17/30 Read block 225 len 28672
26792024-04-20T01:06:22.257Z18/30 Write block 158 len 32768 data: 2 2 3 3 3 3 3 3
26802024-04-20T01:06:22.257Z19/30 Read block 456 len 4096
26812024-04-20T01:06:22.258Z20/30 Write block 391 len 24576 data: 2 2 2 2 2 2
26822024-04-20T01:06:22.258Z21/30 Write block 230 len 20480 data: 4 7 7 7 7
26832024-04-20T01:06:22.258Z22/30 Read block 505 len 16384
26842024-04-20T01:06:22.258Z23/30 Read block 175 len 24576
26852024-04-20T01:06:22.261Z24/30 Write block 15 len 20480 data: 3 3 3 3 3
26862024-04-20T01:06:22.261Z25/30 Write block 216 len 40960 data: 2 4 4 4 6 6 6 5 5 5
26872024-04-20T01:06:22.261Z26/30 Read block 221 len 20480
26882024-04-20T01:06:22.268Z27/30 Write block 18 len 16384 data: 4 4 3 3
26892024-04-20T01:06:22.268Z28/30 Write block 521 len 12288 data: 4 3 4
26902024-04-20T01:06:22.268Z29/30 Read block 424 len 12288
26912024-04-20T01:06:22.268Z30/30 Read block 200 len 24576
26922024-04-20T01:06:22.268Z----------------------------------------------------------------
26932024-04-20T01:06:22.268Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
26942024-04-20T01:06:22.268ZStates: Active Active Active
26952024-04-20T01:06:22.268ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
26962024-04-20T01:06:22.268Z 1 Acked 1000 Write 2 Sent Done Done false
26972024-04-20T01:06:22.268Z 2 Acked 1001 Read 8 Sent Done Done false
26982024-04-20T01:06:22.268Z 3 Acked 1002 Read 9 Sent Done Done false
26992024-04-20T01:06:22.268Z 4 Acked 1003 Write 3 Sent Done Done false
27002024-04-20T01:06:22.269Z 5 Acked 1004 Read 2 Sent Done Done false
27012024-04-20T01:06:22.269Z 6 Acked 1005 Write 4 Sent Done Done false
27022024-04-20T01:06:22.269Z 7 Acked 1006 Write 1 Sent Done Done false
27032024-04-20T01:06:22.269Z 8 Acked 1007 Read 10 Sent Done Done false
27042024-04-20T01:06:22.269Z 9 Acked 1008 Write 9 Sent Done Done false
27052024-04-20T01:06:22.269Z 10 Acked 1009 Read 5 Sent Done Done false
27062024-04-20T01:06:22.269Z 11 Acked 1010 Read 8 Sent Done Done false
27072024-04-20T01:06:22.269Z 12 Acked 1011 Read 1 Sent Done Done false
27082024-04-20T01:06:22.269Z 13 Acked 1012 Write 10 Sent Done Done false
27092024-04-20T01:06:22.269Z 14 Acked 1013 Flush 0 Sent Done Done false
27102024-04-20T01:06:22.269Z 15 Acked 1014 Write 8 Sent Done Done false
27112024-04-20T01:06:22.269Z 16 Acked 1015 Read 8 Sent Done Done false
27122024-04-20T01:06:22.270Z 17 Acked 1016 Read 7 Sent Done Done false
27132024-04-20T01:06:22.270Z 18 Acked 1017 Write 8 Sent Done Done false
27142024-04-20T01:06:22.270Z 19 Acked 1018 Read 1 Sent Done Done false
27152024-04-20T01:06:22.270Z 20 Acked 1019 Write 6 Sent Done Done false
27162024-04-20T01:06:22.270Z 21 Acked 1020 Write 5 Sent Done Done false
27172024-04-20T01:06:22.270Z 22 Acked 1021 Read 4 Sent Done Done false
27182024-04-20T01:06:22.270Z 23 Acked 1022 Read 6 Sent Done Done false
27192024-04-20T01:06:22.270Z 24 Acked 1023 Write 5 Sent Done Done false
27202024-04-20T01:06:22.270Z 25 Acked 1024 Write 10 Sent Done Done false
27212024-04-20T01:06:22.270Z 26 Acked 1025 Read 5 Sent Done Done false
27222024-04-20T01:06:22.270Z 27 Acked 1026 Write 4 Sent Done Done false
27232024-04-20T01:06:22.271Z 28 Acked 1027 Write 3 Sent Done Done false
27242024-04-20T01:06:22.271Z 29 Acked 1028 Read 3 Sent Done Done false
27252024-04-20T01:06:22.271Z 30 Acked 1029 Read 6 Sent Sent Done false
27262024-04-20T01:06:22.271Z STATES DS:0 DS:1 DS:2 TOTAL
27272024-04-20T01:06:22.271Z New 0 0 0 0
27282024-04-20T01:06:22.271Z Sent 30 1 0 31
27292024-04-20T01:06:22.271Z Done 0 29 30 59
27302024-04-20T01:06:22.271Z Skipped 0 0 0 0
27312024-04-20T01:06:22.271Z Error 0 0 0 0
27322024-04-20T01:06:22.271ZLast Flush: 0 1013 1013
27332024-04-20T01:06:22.271ZDownstairs last five completed:
27342024-04-20T01:06:22.271ZUpstairs last five completed: 30 29 28 27 26
27352024-04-20T01:06:22.271Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:06:20.774974733Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1264}
27362024-04-20T01:06:22.272ZWrote out file "/tmp/test_repair_verify.data"
27372024-04-20T01:06:22.276Z
27382024-04-20T01:06:22.276ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27392024-04-20T01:06:22.288ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27402024-04-20T01:06:22.288Z 0 000-019 8 8 8 14 14 14 F T T <---
27412024-04-20T01:06:22.288Z 1 020-039 16 24 24 27 38 38 F T T <---
27422024-04-20T01:06:22.289Z 2 040-059 22 22 22 35 35 35 F F F
27432024-04-20T01:06:22.289Z 3 060-079 18 18 18 31 31 31 F F F
27442024-04-20T01:06:22.289Z 4 080-099 20 20 20 33 33 33 F F F
27452024-04-20T01:06:22.289Z 5 100-119 20 24 24 33 38 38 F F F <---
27462024-04-20T01:06:22.289Z 6 120-139 22 22 22 35 35 35 F F F
27472024-04-20T01:06:22.293Z 7 140-159 22 24 24 35 38 38 F T T <---
27482024-04-20T01:06:22.293Z 8 160-179 18 18 18 30 30 30 F T T <---
27492024-04-20T01:06:22.293Z 9 180-199 23 23 23 37 37 37 F F F
27502024-04-20T01:06:22.293Z 10 200-219 23 24 24 37 38 38 F T T <---
27512024-04-20T01:06:22.293Z 11 220-239 22 24 24 34 38 38 F T T <---
27522024-04-20T01:06:22.293Z 12 240-259 18 18 18 31 31 31 F F F
27532024-04-20T01:06:22.293Z 13 260-279 17 17 17 28 28 28 F F F
27542024-04-20T01:06:22.293Z 14 280-299 20 20 20 33 33 33 F F F
27552024-04-20T01:06:22.293Z 15 300-319 15 15 15 25 25 25 F F F
27562024-04-20T01:06:22.294Z 16 320-339 20 20 20 33 33 33 F F F
27572024-04-20T01:06:22.294Z 17 340-359 20 20 20 33 33 33 F F F
27582024-04-20T01:06:22.294Z 18 360-379 20 20 20 33 33 33 F F F
27592024-04-20T01:06:22.294Z 19 380-399 15 15 15 25 25 25 F T T <---
27602024-04-20T01:06:22.294Z 20 400-419 22 22 22 34 34 34 F F F
27612024-04-20T01:06:22.294Z 21 420-439 20 20 20 33 33 33 F F F
27622024-04-20T01:06:22.294Z 22 440-459 20 20 20 33 33 33 F F F
27632024-04-20T01:06:22.294Z 23 460-479 23 23 23 37 37 37 F F F
27642024-04-20T01:06:22.294Z 24 480-499 22 22 22 36 36 36 F F F
27652024-04-20T01:06:22.294Z 25 500-519 20 20 20 33 33 33 F F F
27662024-04-20T01:06:22.294Z 26 520-539 22 22 22 36 36 36 F T T <---
27672024-04-20T01:06:22.294Z 27 540-559 22 24 24 35 38 38 F F F <---
27682024-04-20T01:06:22.294Z 28 560-579 22 22 22 35 35 35 F F F
27692024-04-20T01:06:22.294Z 29 580-599 22 22 22 35 35 35 F F F
27702024-04-20T01:06:22.294ZMax gen: 24, Max flush: 38
27712024-04-20T01:06:22.294ZError: Difference in extent metadata found!
27722024-04-20T01:06:22.295ZA Difference in extent metadata is expected here
27732024-04-20T01:06:22.295ZOn loop 12
27742024-04-20T01:06:22.295Z
27752024-04-20T01:06:22.301ZVerifying data now
27762024-04-20T01:06:26.763ZNo change after verify
27772024-04-20T01:06:26.763ZLoop: 12 Downstairs dump after verify (and repair):
27782024-04-20T01:06:26.777ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27792024-04-20T01:06:26.777Z 0 000-019 25 25 25 39 39 39 F F F
27802024-04-20T01:06:26.780Z 1 020-039 25 25 25 39 39 39 F F F
27812024-04-20T01:06:26.780Z 2 040-059 22 22 22 35 35 35 F F F
27822024-04-20T01:06:26.781Z 3 060-079 18 18 18 31 31 31 F F F
27832024-04-20T01:06:26.781Z 4 080-099 20 20 20 33 33 33 F F F
27842024-04-20T01:06:26.781Z 5 100-119 24 24 24 38 38 38 F F F
27852024-04-20T01:06:26.781Z 6 120-139 22 22 22 35 35 35 F F F
27862024-04-20T01:06:26.781Z 7 140-159 25 25 25 39 39 39 F F F
27872024-04-20T01:06:26.781Z 8 160-179 25 25 25 39 39 39 F F F
27882024-04-20T01:06:26.781Z 9 180-199 23 23 23 37 37 37 F F F
27892024-04-20T01:06:26.781Z 10 200-219 25 25 25 39 39 39 F F F
27902024-04-20T01:06:26.781Z 11 220-239 25 25 25 39 39 39 F F F
27912024-04-20T01:06:26.781Z 12 240-259 18 18 18 31 31 31 F F F
27922024-04-20T01:06:26.781Z 13 260-279 17 17 17 28 28 28 F F F
27932024-04-20T01:06:26.781Z 14 280-299 20 20 20 33 33 33 F F F
27942024-04-20T01:06:26.781Z 15 300-319 15 15 15 25 25 25 F F F
27952024-04-20T01:06:26.781Z 16 320-339 20 20 20 33 33 33 F F F
27962024-04-20T01:06:26.781Z 17 340-359 20 20 20 33 33 33 F F F
27972024-04-20T01:06:26.781Z 18 360-379 20 20 20 33 33 33 F F F
27982024-04-20T01:06:26.781Z 19 380-399 25 25 25 39 39 39 F F F
27992024-04-20T01:06:26.781Z 20 400-419 22 22 22 34 34 34 F F F
28002024-04-20T01:06:26.782Z 21 420-439 20 20 20 33 33 33 F F F
28012024-04-20T01:06:26.782Z 22 440-459 20 20 20 33 33 33 F F F
28022024-04-20T01:06:26.782Z 23 460-479 23 23 23 37 37 37 F F F
28032024-04-20T01:06:26.782Z 24 480-499 22 22 22 36 36 36 F F F
28042024-04-20T01:06:26.782Z 25 500-519 20 20 20 33 33 33 F F F
28052024-04-20T01:06:26.782Z 26 520-539 25 25 25 39 39 39 F F F
28062024-04-20T01:06:26.782Z 27 540-559 24 24 24 38 38 38 F F F
28072024-04-20T01:06:26.782Z 28 560-579 22 22 22 35 35 35 F F F
28082024-04-20T01:06:26.782Z 29 580-599 22 22 22 35 35 35 F F F
28092024-04-20T01:06:26.782ZMax gen: 25, Max flush: 39
28102024-04-20T01:06:26.782Z
28112024-04-20T01:06:26.782ZBegin loop 13
28122024-04-20T01:06:26.782ZDownstairs to restart: 2
28132024-04-20T01:06:26.800Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310039168Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272}
28142024-04-20T01:06:26.800Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310291018Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272}
28152024-04-20T01:06:26.800Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310325698Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272}
28162024-04-20T01:06:26.808Z{"msg":"Crucible 3b5064cb-1759-456c-adc8-f4b5a9488241 has session id: e5b03dd6-04a8-4108-82e4-8d373bbdd276","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310357358Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28172024-04-20T01:06:26.808Z{"msg":"Upstairs opts: Upstairs UUID: 3b5064cb-1759-456c-adc8-f4b5a9488241, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
28182024-04-20T01:06:26.808Z,"time":"2024-04-20T01:06:25.310404369Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28192024-04-20T01:06:26.808Z{"msg":"Crucible stats registered with UUID: 3b5064cb-1759-456c-adc8-f4b5a9488241","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310448379Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28202024-04-20T01:06:26.808Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310472589Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272}
28212024-04-20T01:06:26.809Z{"msg":"3b5064cb-1759-456c-adc8-f4b5a9488241 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310555149Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28222024-04-20T01:06:26.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310594799Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"0","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28232024-04-20T01:06:26.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310622329Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"0","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28242024-04-20T01:06:26.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310646439Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"1","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28252024-04-20T01:06:26.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310694149Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"1","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28262024-04-20T01:06:26.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310757019Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"2","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28272024-04-20T01:06:26.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.310848019Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"2","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28282024-04-20T01:06:26.809Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.31089987Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28292024-04-20T01:06:26.809Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.31092872Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28302024-04-20T01:06:26.809Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.31096453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28312024-04-20T01:06:26.809Z{"msg":"ds_connection connected from Ok(127.0.0.1:38758)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.31099105Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28322024-04-20T01:06:26.809Z{"msg":"ds_connection connected from Ok(127.0.0.1:48300)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.31101676Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28332024-04-20T01:06:26.809Z{"msg":"ds_connection connected from Ok(127.0.0.1:33289)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.31104512Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28342024-04-20T01:06:26.809Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.311823952Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"0","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28352024-04-20T01:06:26.809Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.311931392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"0","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28362024-04-20T01:06:26.809Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.312015502Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"1","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28372024-04-20T01:06:26.809Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.312045802Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"1","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28382024-04-20T01:06:26.809Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.312079352Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"2","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28392024-04-20T01:06:26.809Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.312105112Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"2","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28402024-04-20T01:06:26.809Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.312392293Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"1","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28412024-04-20T01:06:26.809Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.312506033Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"2","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28422024-04-20T01:06:26.810Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.312570203Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"0","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28432024-04-20T01:06:26.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.312905824Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"1","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28442024-04-20T01:06:26.810Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.312946214Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"1","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28452024-04-20T01:06:26.810Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.312971864Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28462024-04-20T01:06:26.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313091615Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"2","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28472024-04-20T01:06:26.810Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313129795Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"2","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28482024-04-20T01:06:26.810Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313154715Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28492024-04-20T01:06:26.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313181995Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"0","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28502024-04-20T01:06:26.810Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313205945Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"0","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28512024-04-20T01:06:26.810Z{"msg":"[0]R flush_numbers[0..12]: [39, 39, 35, 31, 33, 38, 35, 39, 39, 37, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313234125Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28522024-04-20T01:06:26.810Z{"msg":"[0]R generation[0..12]: [25, 25, 22, 18, 20, 24, 22, 25, 25, 23, 25, 25]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
28532024-04-20T01:06:26.810Z,"time":"2024-04-20T01:06:25.313259795Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28542024-04-20T01:06:26.810Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313286325Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28552024-04-20T01:06:26.810Z{"msg":"[1]R flush_numbers[0..12]: [39, 39, 35, 31, 33, 38, 35, 39, 39, 37, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313317565Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28562024-04-20T01:06:26.810Z{"msg":"[1]R generation[0..12]: [25, 25, 22, 18, 20, 24, 22, 25, 25, 23, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313341815Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28572024-04-20T01:06:26.810Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313370815Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28582024-04-20T01:06:26.810Z{"msg":"[2]R flush_numbers[0..12]: [39, 39, 35, 31, 33, 38, 35, 39, 39, 37, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313396115Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28592024-04-20T01:06:26.810Z{"msg":"[2]R generation[0..12]: [25, 25, 22, 18, 20, 24, 22, 25, 25, 23, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313419805Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28602024-04-20T01:06:26.810Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313452505Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28612024-04-20T01:06:26.810Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313476235Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28622024-04-20T01:06:26.810Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313498845Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28632024-04-20T01:06:26.810Z{"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313520885Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"":"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28642024-04-20T01:06:26.810Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313551416Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,""Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28652024-04-20T01:06:26.811Z:"downstairs","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28662024-04-20T01:06:26.811Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313580686Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28672024-04-20T01:06:26.811Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313601616Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"0","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28682024-04-20T01:06:26.811Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313632536Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"1","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28692024-04-20T01:06:26.811Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313655866Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"client":"2","":"downstairs","session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28702024-04-20T01:06:26.811Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313682706Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28712024-04-20T01:06:26.811Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313704806Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28722024-04-20T01:06:26.811Z{"msg":"3b5064cb-1759-456c-adc8-f4b5a9488241 is now active with session: e5b03dd6-04a8-4108-82e4-8d373bbdd276","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313732646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28732024-04-20T01:06:26.811Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313753826Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272,"session_id":"e5b03dd6-04a8-4108-82e4-8d373bbdd276"}
28742024-04-20T01:06:26.811Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:25.313775866Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272}
28752024-04-20T01:06:26.811ZLoading write count information from file "/tmp/test_repair_verify.data"
28762024-04-20T01:06:26.811ZRun Repair workload
28772024-04-20T01:06:26.815Z01/30 Write block 52 len 28672 data: 8 7 5 4 4 3 3
28782024-04-20T01:06:26.815Z02/30 Read block 585 len 28672
28792024-04-20T01:06:26.815Z03/30 Flush
28802024-04-20T01:06:26.815Z04/30 Write block 510 len 32768 data: 4 4 5 6 5 5 4 4
28812024-04-20T01:06:26.815Z05/30 Read block 356 len 8192
28822024-04-20T01:06:26.815Z06/30 Read block 231 len 36864
28832024-04-20T01:06:26.815Z07/30 Write block 4 len 4096 data: 2
28842024-04-20T01:06:26.815Z08/30 Read block 67 len 16384
28852024-04-20T01:06:26.815Z09/30 Write block 119 len 32768 data: 3 3 3 3 3 3 3 3
28862024-04-20T01:06:26.815Z10/30 Read block 380 len 32768
28872024-04-20T01:06:26.819Z11/30 Write block 424 len 36864 data: 2 2 2 3 3 3 3 3 3
28882024-04-20T01:06:26.819Z12/30 Read block 159 len 28672
28892024-04-20T01:06:26.819Z13/30 Write block 586 len 12288 data: 4 6 6
28902024-04-20T01:06:26.819Z14/30 Write block 155 len 36864 data: 4 2 2 3 3 4 4 4 4
28912024-04-20T01:06:26.819Z15/30 Write block 248 len 12288 data: 5 5 4
28922024-04-20T01:06:26.823Z16/30 Read block 75 len 12288
28932024-04-20T01:06:26.823Z17/30 Write block 318 len 40960 data: 5 5 4 3 3 3 3 3 3 3
28942024-04-20T01:06:26.823Z18/30 Write block 128 len 32768 data: 2 2 2 3 3 3 4 5
28952024-04-20T01:06:26.823Z19/30 Read block 155 len 36864
28962024-04-20T01:06:26.827Z20/30 Read block 265 len 8192
28972024-04-20T01:06:26.827Z21/30 Write block 26 len 8192 data: 4 5
28982024-04-20T01:06:26.827Z22/30 Flush
28992024-04-20T01:06:26.830Z23/30 Write block 124 len 40960 data: 4 4 4 3 3 3 3 4 4 4
29002024-04-20T01:06:26.834Z24/30 Write block 386 len 12288 data: 3 2 2
29012024-04-20T01:06:26.834Z25/30 Write block 493 len 32768 data: 3 3 3 3 3 3 2 2
29022024-04-20T01:06:26.834Z26/30 Flush
29032024-04-20T01:06:26.837Z27/30 Write block 209 len 4096 data: 3
29042024-04-20T01:06:26.837Z28/30 Read block 456 len 32768
29052024-04-20T01:06:26.837Z29/30 Read block 442 len 24576
29062024-04-20T01:06:26.837Z30/30 Read block 549 len 36864
29072024-04-20T01:06:26.842Z----------------------------------------------------------------
29082024-04-20T01:06:26.842Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29092024-04-20T01:06:26.842ZStates: Active Active Active
29102024-04-20T01:06:26.842ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29112024-04-20T01:06:26.842Z 1 Acked 1000 Write 7 Done Done Sent false
29122024-04-20T01:06:26.842Z 2 Acked 1001 Read 7 Done Done Sent false
29132024-04-20T01:06:26.842Z 3 Acked 1002 Flush 0 Done Done Sent false
29142024-04-20T01:06:26.842Z 4 Acked 1003 Write 8 Done Done Sent false
29152024-04-20T01:06:26.842Z 5 Acked 1004 Read 2 Done Done Sent false
29162024-04-20T01:06:26.842Z 6 Acked 1005 Read 9 Done Done Sent false
29172024-04-20T01:06:26.842Z 7 Acked 1006 Write 1 Done Done Sent false
29182024-04-20T01:06:26.842Z 8 Acked 1007 Read 4 Done Done Sent false
29192024-04-20T01:06:26.842Z 9 Acked 1008 Write 8 Done Done Sent false
29202024-04-20T01:06:26.842Z 10 Acked 1009 Read 8 Done Done Sent false
29212024-04-20T01:06:26.842Z 11 Acked 1010 Write 9 Done Done Sent false
29222024-04-20T01:06:26.842Z 12 Acked 1011 Read 7 Done Done Sent false
29232024-04-20T01:06:26.842Z 13 Acked 1012 Write 3 Done Done Sent false
29242024-04-20T01:06:26.842Z 14 Acked 1013 Write 9 Done Done Sent false
29252024-04-20T01:06:26.842Z 15 Acked 1014 Write 3 Done Done Sent false
29262024-04-20T01:06:26.843Z 16 Acked 1015 Read 3 Done Done Sent false
29272024-04-20T01:06:26.843Z 17 Acked 1016 Write 10 Done Done Sent false
29282024-04-20T01:06:26.843Z 18 Acked 1017 Write 8 Done Done Sent false
29292024-04-20T01:06:26.843Z 19 Acked 1018 Read 9 Done Done Sent false
29302024-04-20T01:06:26.843Z 20 Acked 1019 Read 2 Done Done Sent false
29312024-04-20T01:06:26.843Z 21 Acked 1020 Write 2 Done Done Sent false
29322024-04-20T01:06:26.843Z 22 Acked 1021 Flush 0 Done Done Sent false
29332024-04-20T01:06:26.843Z 23 Acked 1022 Write 10 Done Done Sent false
29342024-04-20T01:06:26.843Z 24 Acked 1023 Write 3 Done Done Sent false
29352024-04-20T01:06:26.843Z 25 Acked 1024 Write 8 Done Done Sent false
29362024-04-20T01:06:26.843Z 26 Acked 1025 Flush 0 Done Done Sent false
29372024-04-20T01:06:26.843Z 27 Acked 1026 Write 1 Done Done Sent false
29382024-04-20T01:06:26.843Z 28 Acked 1027 Read 8 Done Done Sent false
29392024-04-20T01:06:26.843Z 29 Acked 1028 Read 6 Done Done Sent false
29402024-04-20T01:06:26.843Z 30 Acked 1029 Read 9 Done Sent Sent false
29412024-04-20T01:06:26.843Z STATES DS:0 DS:1 DS:2 TOTAL
29422024-04-20T01:06:26.843Z New 0 0 0 0
29432024-04-20T01:06:26.843Z Sent 0 1 30 31
29442024-04-20T01:06:26.843Z Done 30 29 0 59
29452024-04-20T01:06:26.843Z Skipped 0 0 0 0
29462024-04-20T01:06:26.843Z Error 0 0 0 0
29472024-04-20T01:06:26.843ZLast Flush: 1025 1025 0
29482024-04-20T01:06:26.843ZDownstairs last five completed:
29492024-04-20T01:06:26.844ZUpstairs last five completed: 30 29 28 27 26
29502024-04-20T01:06:26.844Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:06:25.348949974Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1272}
29512024-04-20T01:06:26.844ZWrote out file "/tmp/test_repair_verify.data"
29522024-04-20T01:06:26.847Z
29532024-04-20T01:06:26.847ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29542024-04-20T01:06:26.860ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29552024-04-20T01:06:26.860Z 0 000-019 26 26 25 41 41 39 F F F <---
29562024-04-20T01:06:26.860Z 1 020-039 26 26 25 41 41 39 F F F <---
29572024-04-20T01:06:26.860Z 2 040-059 26 26 22 40 40 35 F F F <---
29582024-04-20T01:06:26.860Z 3 060-079 18 18 18 31 31 31 F F F
29592024-04-20T01:06:26.860Z 4 080-099 20 20 20 33 33 33 F F F
29602024-04-20T01:06:26.860Z 5 100-119 26 26 24 41 41 38 F F F <---
29612024-04-20T01:06:26.860Z 6 120-139 26 26 22 42 42 35 F F F <---
29622024-04-20T01:06:26.864Z 7 140-159 26 26 25 41 41 39 F F F <---
29632024-04-20T01:06:26.864Z 8 160-179 26 26 25 41 41 39 F F F <---
29642024-04-20T01:06:26.864Z 9 180-199 23 23 23 37 37 37 F F F
29652024-04-20T01:06:26.864Z 10 200-219 25 25 25 39 39 39 T T F <---
29662024-04-20T01:06:26.864Z 11 220-239 25 25 25 39 39 39 F F F
29672024-04-20T01:06:26.864Z 12 240-259 26 26 18 41 41 31 F F F <---
29682024-04-20T01:06:26.864Z 13 260-279 17 17 17 28 28 28 F F F
29692024-04-20T01:06:26.864Z 14 280-299 20 20 20 33 33 33 F F F
29702024-04-20T01:06:26.864Z 15 300-319 26 26 15 41 41 25 F F F <---
29712024-04-20T01:06:26.864Z 16 320-339 26 26 20 41 41 33 F F F <---
29722024-04-20T01:06:26.864Z 17 340-359 20 20 20 33 33 33 F F F
29732024-04-20T01:06:26.864Z 18 360-379 20 20 20 33 33 33 F F F
29742024-04-20T01:06:26.864Z 19 380-399 26 26 25 42 42 39 F F F <---
29752024-04-20T01:06:26.864Z 20 400-419 22 22 22 34 34 34 F F F
29762024-04-20T01:06:26.864Z 21 420-439 26 26 20 41 41 33 F F F <---
29772024-04-20T01:06:26.864Z 22 440-459 20 20 20 33 33 33 F F F
29782024-04-20T01:06:26.864Z 23 460-479 23 23 23 37 37 37 F F F
29792024-04-20T01:06:26.865Z 24 480-499 26 26 22 42 42 36 F F F <---
29802024-04-20T01:06:26.865Z 25 500-519 26 26 20 42 42 33 F F F <---
29812024-04-20T01:06:26.865Z 26 520-539 25 25 25 39 39 39 F F F
29822024-04-20T01:06:26.865Z 27 540-559 24 24 24 38 38 38 F F F
29832024-04-20T01:06:26.865Z 28 560-579 22 22 22 35 35 35 F F F
29842024-04-20T01:06:26.865Z 29 580-599 26 26 22 41 41 35 F F F <---
29852024-04-20T01:06:26.865ZMax gen: 26, Max flush: 42
29862024-04-20T01:06:26.865ZError: Difference in extent metadata found!
29872024-04-20T01:06:26.865ZA Difference in extent metadata is expected here
29882024-04-20T01:06:26.865ZOn loop 13
29892024-04-20T01:06:26.865Z
29902024-04-20T01:06:26.870ZVerifying data now
29912024-04-20T01:06:31.555ZNo change after verify
29922024-04-20T01:06:31.555ZLoop: 13 Downstairs dump after verify (and repair):
29932024-04-20T01:06:31.569ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29942024-04-20T01:06:31.569Z 0 000-019 26 26 26 41 41 41 F F F
29952024-04-20T01:06:31.573Z 1 020-039 26 26 26 41 41 41 F F F
29962024-04-20T01:06:31.574Z 2 040-059 26 26 26 40 40 40 F F F
29972024-04-20T01:06:31.574Z 3 060-079 18 18 18 31 31 31 F F F
29982024-04-20T01:06:31.574Z 4 080-099 20 20 20 33 33 33 F F F
29992024-04-20T01:06:31.574Z 5 100-119 26 26 26 41 41 41 F F F
30002024-04-20T01:06:31.574Z 6 120-139 26 26 26 42 42 42 F F F
30012024-04-20T01:06:31.574Z 7 140-159 26 26 26 41 41 41 F F F
30022024-04-20T01:06:31.574Z 8 160-179 26 26 26 41 41 41 F F F
30032024-04-20T01:06:31.574Z 9 180-199 23 23 23 37 37 37 F F F
30042024-04-20T01:06:31.574Z 10 200-219 27 27 27 43 43 43 F F F
30052024-04-20T01:06:31.574Z 11 220-239 25 25 25 39 39 39 F F F
30062024-04-20T01:06:31.574Z 12 240-259 26 26 26 41 41 41 F F F
30072024-04-20T01:06:31.574Z 13 260-279 17 17 17 28 28 28 F F F
30082024-04-20T01:06:31.574Z 14 280-299 20 20 20 33 33 33 F F F
30092024-04-20T01:06:31.574Z 15 300-319 26 26 26 41 41 41 F F F
30102024-04-20T01:06:31.574Z 16 320-339 26 26 26 41 41 41 F F F
30112024-04-20T01:06:31.574Z 17 340-359 20 20 20 33 33 33 F F F
30122024-04-20T01:06:31.574Z 18 360-379 20 20 20 33 33 33 F F F
30132024-04-20T01:06:31.574Z 19 380-399 26 26 26 42 42 42 F F F
30142024-04-20T01:06:31.574Z 20 400-419 22 22 22 34 34 34 F F F
30152024-04-20T01:06:31.574Z 21 420-439 26 26 26 41 41 41 F F F
30162024-04-20T01:06:31.574Z 22 440-459 20 20 20 33 33 33 F F F
30172024-04-20T01:06:31.575Z 23 460-479 23 23 23 37 37 37 F F F
30182024-04-20T01:06:31.575Z 24 480-499 26 26 26 42 42 42 F F F
30192024-04-20T01:06:31.575Z 25 500-519 26 26 26 42 42 42 F F F
30202024-04-20T01:06:31.575Z 26 520-539 25 25 25 39 39 39 F F F
30212024-04-20T01:06:31.575Z 27 540-559 24 24 24 38 38 38 F F F
30222024-04-20T01:06:31.575Z 28 560-579 22 22 22 35 35 35 F F F
30232024-04-20T01:06:31.575Z 29 580-599 26 26 26 41 41 41 F F F
30242024-04-20T01:06:31.575ZMax gen: 27, Max flush: 43
30252024-04-20T01:06:31.575Z
30262024-04-20T01:06:31.575ZBegin loop 14
30272024-04-20T01:06:31.575ZDownstairs to restart: 2
30282024-04-20T01:06:31.591Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.101364003Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280}
30292024-04-20T01:06:31.591Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.101605734Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280}
30302024-04-20T01:06:31.598Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.101641524Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280}
30312024-04-20T01:06:31.598Z{"msg":"Crucible dcd2e31b-da34-4b2b-a550-f7dbabe5256e has session id: 4cb56545-1c9d-40e9-8a2a-dacba2cadf04","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.101674524Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30322024-04-20T01:06:31.599Z{"msg":"Crucible runtime is spawned
30332024-04-20T01:06:31.599ZUpstairs opts: Upstairs UUID: dcd2e31b-da34-4b2b-a550-f7dbabe5256e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.101797344Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30342024-04-20T01:06:31.599Z{"msg":"Crucible stats registered with UUID: dcd2e31b-da34-4b2b-a550-f7dbabe5256e","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.101826294Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30352024-04-20T01:06:31.599Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.101851995Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280}
30362024-04-20T01:06:31.599Z{"msg":"dcd2e31b-da34-4b2b-a550-f7dbabe5256e active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.101918795Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30372024-04-20T01:06:31.599Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.101949545Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"0","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30382024-04-20T01:06:31.599Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.101988105Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"0","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30392024-04-20T01:06:31.599Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.102032915Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"1","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30402024-04-20T01:06:31.599Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.102075585Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"1","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30412024-04-20T01:06:31.599Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.102103545Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"2","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30422024-04-20T01:06:31.599Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.102130935Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"2","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30432024-04-20T01:06:31.599Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.102163695Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30442024-04-20T01:06:31.599Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.102188145Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30452024-04-20T01:06:31.599Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.102222445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30462024-04-20T01:06:31.599Z{"msg":"ds_connection connected from Ok(127.0.0.1:46442)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.102247895Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30472024-04-20T01:06:31.599Z{"msg":"ds_connection connected from Ok(127.0.0.1:40445)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.102284365Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30482024-04-20T01:06:31.599Z{"msg":"ds_connection connected from Ok(127.0.0.1:35467)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.102313405Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30492024-04-20T01:06:31.599Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.102993627Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"1","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30502024-04-20T01:06:31.599Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.103039017Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"1","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30512024-04-20T01:06:31.599Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.103080627Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"0","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30522024-04-20T01:06:31.599Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.103118327Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"0","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30532024-04-20T01:06:31.600Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.103553538Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"2","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30542024-04-20T01:06:31.600Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.103583688Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"2","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30552024-04-20T01:06:31.600Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.103599268Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"0","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30562024-04-20T01:06:31.600Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.103743189Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"1","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30572024-04-20T01:06:31.600Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.10421502Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"0","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30582024-04-20T01:06:31.600Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.10426856Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"0","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30592024-04-20T01:06:31.600Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.10431374Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30602024-04-20T01:06:31.600Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.10435259Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"2","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30612024-04-20T01:06:31.600Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.10439212Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"1","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30622024-04-20T01:06:31.600Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.104423Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"1","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30632024-04-20T01:06:31.600Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.10444403Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30642024-04-20T01:06:31.600Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.104797311Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"2","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30652024-04-20T01:06:31.600Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.104825181Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"2","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30662024-04-20T01:06:31.600Z{"msg":"[0]R flush_numbers[0..12]: [41, 41, 40, 31, 33, 41, 42, 41, 41, 37, 43, 39]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.104840671Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30672024-04-20T01:06:31.600Z{"msg":"[0]R generation[0..12]: [26, 26, 26, 18, 20, 26, 26, 26, 26, 23, 27, 25]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.104862331Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30682024-04-20T01:06:31.600Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.104876781Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30692024-04-20T01:06:31.600Z{"msg":"[1]R flush_numbers[0..12]: [41, 41, 40, 31, 33, 41, 42, 41, 41, 37, 43, 39]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.104897621Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30702024-04-20T01:06:31.600Z{"msg":"[1]R generation[0..12]: [26, 26, 26, 18, 20, 26, 26, 26, 26, 23, 27, 25]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.104912431Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30712024-04-20T01:06:31.600Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.104934411Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30722024-04-20T01:06:31.600Z{"msg":"[2]R flush_numbers[0..12]: [41, 41, 40, 31, 33, 41, 42, 41, 41, 37, 43, 39]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.104948181Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30732024-04-20T01:06:31.600Z{"msg":"[2]R generation[0..12]: [26, 26, 26, 18, 20, 26, 26, 26, 26, 23, 27, 25]",Wait for a query_work_queue command to finish before sending IO
30742024-04-20T01:06:31.600Z"v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.104971741Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30752024-04-20T01:06:31.600Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.104986791Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30762024-04-20T01:06:31.601Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.105014931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30772024-04-20T01:06:31.601Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.105031042Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30782024-04-20T01:06:31.601Z{"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.105064071Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30792024-04-20T01:06:31.601Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.105080102Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"":"downstairs","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30802024-04-20T01:06:31.601Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.105114042Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30812024-04-20T01:06:31.601Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.105146392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"0","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30822024-04-20T01:06:31.601Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.105184042Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"1","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30832024-04-20T01:06:31.601Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.105211692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"client":"2","":"downstairs","session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30842024-04-20T01:06:31.601Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.105247422Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30852024-04-20T01:06:31.601Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.105272872Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30862024-04-20T01:06:31.601Z{"msg":"dcd2e31b-da34-4b2b-a550-f7dbabe5256e is now active with session: 4cb56545-1c9d-40e9-8a2a-dacba2cadf04","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
30872024-04-20T01:06:31.601Z,"time":"2024-04-20T01:06:30.105307002Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30882024-04-20T01:06:31.601Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.105345892Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280,"session_id":"4cb56545-1c9d-40e9-8a2a-dacba2cadf04"}
30892024-04-20T01:06:31.601Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:30.105364032Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280}
30902024-04-20T01:06:31.605ZLoading write count information from file "/tmp/test_repair_verify.data"
30912024-04-20T01:06:31.606ZRun Repair workload
30922024-04-20T01:06:31.606Z01/30 Write block 129 len 40960 data: 4 4 5 5 5 5 6 5 4 5
30932024-04-20T01:06:31.606Z02/30 Write block 541 len 12288 data: 3 4 6
30942024-04-20T01:06:31.606Z03/30 Flush
30952024-04-20T01:06:31.606Z04/30 Write block 106 len 16384 data: 4 3 2 2
30962024-04-20T01:06:31.606Z05/30 Read block 164 len 40960
30972024-04-20T01:06:31.606Z06/30 Write block 321 len 8192 data: 4 4
30982024-04-20T01:06:31.606Z07/30 Read block 108 len 16384
30992024-04-20T01:06:31.606Z08/30 Read block 452 len 12288
31002024-04-20T01:06:31.606Z09/30 Read block 114 len 36864
31012024-04-20T01:06:31.606Z10/30 Flush
31022024-04-20T01:06:31.610Z11/30 Write block 25 len 8192 data: 3 5
31032024-04-20T01:06:31.610Z12/30 Flush
31042024-04-20T01:06:31.611Z13/30 Write block 371 len 36864 data: 5 5 6 5 4 3 3 3 3
31052024-04-20T01:06:31.615Z14/30 Read block 79 len 8192
31062024-04-20T01:06:31.615Z15/30 Write block 301 len 32768 data: 2 2 2 2 2 2 2 2
31072024-04-20T01:06:31.615Z16/30 Write block 270 len 16384 data: 2 2 2 3
31082024-04-20T01:06:31.615Z17/30 Read block 123 len 16384
31092024-04-20T01:06:31.618Z18/30 Write block 318 len 24576 data: 6 6 5 5 5 4
31102024-04-20T01:06:31.618Z19/30 Read block 325 len 40960
31112024-04-20T01:06:31.618Z20/30 Read block 589 len 4096
31122024-04-20T01:06:31.618Z21/30 Read block 520 len 32768
31132024-04-20T01:06:31.618Z22/30 Read block 382 len 28672
31142024-04-20T01:06:31.618Z23/30 Flush
31152024-04-20T01:06:31.627Z24/30 Flush
31162024-04-20T01:06:31.627Z25/30 Write block 519 len 24576 data: 5 5 5 4 5 4
31172024-04-20T01:06:31.627Z26/30 Write block 505 len 32768 data: 6 4 4 4 5 5 5 6
31182024-04-20T01:06:31.627Z27/30 Write block 579 len 40960 data: 4 3 3 3 2 2 2 5 7 7
31192024-04-20T01:06:31.627Z28/30 Read block 236 len 16384
31202024-04-20T01:06:31.633Z29/30 Write block 34 len 28672 data: 3 3 3 2 2 3 3
31212024-04-20T01:06:31.633Z30/30 Write block 493 len 12288 data: 4 4 4
31222024-04-20T01:06:31.633Z----------------------------------------------------------------
31232024-04-20T01:06:31.633Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:27
31242024-04-20T01:06:31.633ZStates: Active Active Active
31252024-04-20T01:06:31.633ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31262024-04-20T01:06:31.633Z 4 Acked 1003 Write 4 Done Done Done false
31272024-04-20T01:06:31.633Z 5 Acked 1004 Read 10 Done Done Done false
31282024-04-20T01:06:31.633Z 6 Acked 1005 Write 2 Done Done Sent false
31292024-04-20T01:06:31.633Z 7 Acked 1006 Read 4 Done Done Sent false
31302024-04-20T01:06:31.633Z 8 Acked 1007 Read 3 Done Done Sent false
31312024-04-20T01:06:31.633Z 9 Acked 1008 Read 9 Done Done Sent false
31322024-04-20T01:06:31.633Z 10 Acked 1009 Flush 0 Done Done Sent false
31332024-04-20T01:06:31.633Z 11 Acked 1010 Write 2 Done Done Sent false
31342024-04-20T01:06:31.633Z 12 Acked 1011 Flush 0 Done Done Sent false
31352024-04-20T01:06:31.633Z 13 Acked 1012 Write 9 Done Done Sent false
31362024-04-20T01:06:31.633Z 14 Acked 1013 Read 2 Done Done Sent false
31372024-04-20T01:06:31.634Z 15 Acked 1014 Write 8 Done Done Sent false
31382024-04-20T01:06:31.634Z 16 Acked 1015 Write 4 Done Done Sent false
31392024-04-20T01:06:31.634Z 17 Acked 1016 Read 4 Done Done Sent false
31402024-04-20T01:06:31.634Z 18 Acked 1017 Write 6 Done Done Sent false
31412024-04-20T01:06:31.634Z 19 Acked 1018 Read 10 Done Done Sent false
31422024-04-20T01:06:31.634Z 20 Acked 1019 Read 1 Done Done Sent false
31432024-04-20T01:06:31.634Z 21 Acked 1020 Read 8 Done Done Sent false
31442024-04-20T01:06:31.634Z 22 Acked 1021 Read 7 Done Done Sent false
31452024-04-20T01:06:31.634Z 23 Acked 1022 Flush 0 Done Done Sent false
31462024-04-20T01:06:31.634Z 24 Acked 1023 Flush 0 Done Done Sent false
31472024-04-20T01:06:31.634Z 25 Acked 1024 Write 6 Done Done Sent false
31482024-04-20T01:06:31.634Z 26 Acked 1025 Write 8 Done Done Sent false
31492024-04-20T01:06:31.634Z 27 Acked 1026 Write 10 Done Done Sent false
31502024-04-20T01:06:31.634Z 28 Acked 1027 Read 4 Done Done Sent false
31512024-04-20T01:06:31.634Z 29 Acked 1028 Write 7 Done Done Sent false
31522024-04-20T01:06:31.634Z 30 Acked 1029 Write 3 Sent Sent Sent false
31532024-04-20T01:06:31.634Z STATES DS:0 DS:1 DS:2 TOTAL
31542024-04-20T01:06:31.634Z New 0 0 0 0
31552024-04-20T01:06:31.634Z Sent 1 1 25 27
31562024-04-20T01:06:31.634Z Done 26 26 2 54
31572024-04-20T01:06:31.634Z Skipped 0 0 0 0
31582024-04-20T01:06:31.634Z Error 0 0 0 0
31592024-04-20T01:06:31.635ZLast Flush: 1023 1023 1002
31602024-04-20T01:06:31.635ZDownstairs last five completed: 1002 1001 1000
31612024-04-20T01:06:31.635ZUpstairs last five completed: 30 29 28 27 26
31622024-04-20T01:06:31.635Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:06:30.139022927Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1280}
31632024-04-20T01:06:31.635ZWrote out file "/tmp/test_repair_verify.data"
31642024-04-20T01:06:31.639Z
31652024-04-20T01:06:31.639ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31662024-04-20T01:06:31.653ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31672024-04-20T01:06:31.653Z 0 000-019 26 26 26 41 41 41 F F F
31682024-04-20T01:06:31.653Z 1 020-039 28 28 26 46 46 41 T T F <---
31692024-04-20T01:06:31.653Z 2 040-059 26 26 26 40 40 40 T T F <---
31702024-04-20T01:06:31.654Z 3 060-079 18 18 18 31 31 31 F F F
31712024-04-20T01:06:31.654Z 4 080-099 20 20 20 33 33 33 F F F
31722024-04-20T01:06:31.654Z 5 100-119 28 28 26 45 45 41 F F T <---
31732024-04-20T01:06:31.658Z 6 120-139 28 28 28 44 44 44 F F F
31742024-04-20T01:06:31.658Z 7 140-159 26 26 26 41 41 41 F F F
31752024-04-20T01:06:31.658Z 8 160-179 26 26 26 41 41 41 F F F
31762024-04-20T01:06:31.658Z 9 180-199 23 23 23 37 37 37 F F F
31772024-04-20T01:06:31.658Z 10 200-219 27 27 27 43 43 43 F F F
31782024-04-20T01:06:31.658Z 11 220-239 25 25 25 39 39 39 F F F
31792024-04-20T01:06:31.658Z 12 240-259 26 26 26 41 41 41 F F F
31802024-04-20T01:06:31.658Z 13 260-279 28 28 17 47 47 28 F F F <---
31812024-04-20T01:06:31.658Z 14 280-299 20 20 20 33 33 33 F F F
31822024-04-20T01:06:31.658Z 15 300-319 28 28 26 47 47 41 F F F <---
31832024-04-20T01:06:31.658Z 16 320-339 28 28 26 47 47 41 F F F <---
31842024-04-20T01:06:31.659Z 17 340-359 20 20 20 33 33 33 F F F
31852024-04-20T01:06:31.659Z 18 360-379 28 28 20 47 47 33 F F F <---
31862024-04-20T01:06:31.659Z 19 380-399 26 26 26 42 42 42 F F F
31872024-04-20T01:06:31.659Z 20 400-419 22 22 22 34 34 34 F F F
31882024-04-20T01:06:31.659Z 21 420-439 26 26 26 41 41 41 F F F
31892024-04-20T01:06:31.659Z 22 440-459 20 20 20 33 33 33 F F F
31902024-04-20T01:06:31.659Z 23 460-479 23 23 23 37 37 37 F F F
31912024-04-20T01:06:31.659Z 24 480-499 26 26 26 42 42 42 T T F <---
31922024-04-20T01:06:31.659Z 25 500-519 26 26 26 42 42 42 T T F <---
31932024-04-20T01:06:31.659Z 26 520-539 25 25 25 39 39 39 T T F <---
31942024-04-20T01:06:31.659Z 27 540-559 28 28 28 44 44 44 F F F
31952024-04-20T01:06:31.659Z 28 560-579 22 22 22 35 35 35 T T F <---
31962024-04-20T01:06:31.659Z 29 580-599 26 26 26 41 41 41 T T F <---
31972024-04-20T01:06:31.659ZMax gen: 28, Max flush: 47
31982024-04-20T01:06:31.659ZError: Difference in extent metadata found!
31992024-04-20T01:06:31.659ZA Difference in extent metadata is expected here
32002024-04-20T01:06:31.659ZOn loop 14
32012024-04-20T01:06:31.659Z
32022024-04-20T01:06:31.664ZVerifying data now
32032024-04-20T01:06:36.219ZNo change after verify
32042024-04-20T01:06:36.220ZLoop: 14 Downstairs dump after verify (and repair):
32052024-04-20T01:06:36.236ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32062024-04-20T01:06:36.236Z 0 000-019 26 26 26 41 41 41 F F F
32072024-04-20T01:06:36.236Z 1 020-039 29 29 29 48 48 48 F F F
32082024-04-20T01:06:36.236Z 2 040-059 29 29 29 48 48 48 F F F
32092024-04-20T01:06:36.237Z 3 060-079 18 18 18 31 31 31 F F F
32102024-04-20T01:06:36.237Z 4 080-099 20 20 20 33 33 33 F F F
32112024-04-20T01:06:36.237Z 5 100-119 28 28 28 45 45 45 F F F
32122024-04-20T01:06:36.237Z 6 120-139 28 28 28 44 44 44 F F F
32132024-04-20T01:06:36.237Z 7 140-159 26 26 26 41 41 41 F F F
32142024-04-20T01:06:36.241Z 8 160-179 26 26 26 41 41 41 F F F
32152024-04-20T01:06:36.241Z 9 180-199 23 23 23 37 37 37 F F F
32162024-04-20T01:06:36.241Z 10 200-219 27 27 27 43 43 43 F F F
32172024-04-20T01:06:36.241Z 11 220-239 25 25 25 39 39 39 F F F
32182024-04-20T01:06:36.242Z 12 240-259 26 26 26 41 41 41 F F F
32192024-04-20T01:06:36.242Z 13 260-279 28 28 28 47 47 47 F F F
32202024-04-20T01:06:36.242Z 14 280-299 20 20 20 33 33 33 F F F
32212024-04-20T01:06:36.242Z 15 300-319 28 28 28 47 47 47 F F F
32222024-04-20T01:06:36.242Z 16 320-339 28 28 28 47 47 47 F F F
32232024-04-20T01:06:36.242Z 17 340-359 20 20 20 33 33 33 F F F
32242024-04-20T01:06:36.242Z 18 360-379 28 28 28 47 47 47 F F F
32252024-04-20T01:06:36.242Z 19 380-399 26 26 26 42 42 42 F F F
32262024-04-20T01:06:36.242Z 20 400-419 22 22 22 34 34 34 F F F
32272024-04-20T01:06:36.242Z 21 420-439 26 26 26 41 41 41 F F F
32282024-04-20T01:06:36.242Z 22 440-459 20 20 20 33 33 33 F F F
32292024-04-20T01:06:36.242Z 23 460-479 23 23 23 37 37 37 F F F
32302024-04-20T01:06:36.242Z 24 480-499 29 29 29 48 48 48 F F F
32312024-04-20T01:06:36.242Z 25 500-519 29 29 29 48 48 48 F F F
32322024-04-20T01:06:36.242Z 26 520-539 29 29 29 48 48 48 F F F
32332024-04-20T01:06:36.242Z 27 540-559 28 28 28 44 44 44 F F F
32342024-04-20T01:06:36.243Z 28 560-579 29 29 29 48 48 48 F F F
32352024-04-20T01:06:36.243Z 29 580-599 29 29 29 48 48 48 F F F
32362024-04-20T01:06:36.243ZMax gen: 29, Max flush: 48
32372024-04-20T01:06:36.243Z
32382024-04-20T01:06:36.243ZBegin loop 15
32392024-04-20T01:06:36.243ZDownstairs to restart: 2
32402024-04-20T01:06:36.263Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.766688378Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288}
32412024-04-20T01:06:36.272Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.766935439Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288}
32422024-04-20T01:06:36.272Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.766970499Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288}
32432024-04-20T01:06:36.272Z{"msg":"Crucible ebee81c2-8891-4049-bc91-91c1c800776d has session id: b0d4940b-2882-44cc-8749-d35aca90b8d4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.767003069Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32442024-04-20T01:06:36.272Z{"msg":"Upstairs opts: Upstairs UUID: ebee81c2-8891-4049-bc91-91c1c800776d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
32452024-04-20T01:06:36.272Z,"time":"2024-04-20T01:06:34.767064729Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32462024-04-20T01:06:36.272Z{"msg":"Crucible stats registered with UUID: ebee81c2-8891-4049-bc91-91c1c800776d","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.767096769Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32472024-04-20T01:06:36.272Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.767141129Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288}
32482024-04-20T01:06:36.272Z{"msg":"ebee81c2-8891-4049-bc91-91c1c800776d active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.767199849Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32492024-04-20T01:06:36.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.767243909Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"0","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32502024-04-20T01:06:36.272Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.767319629Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"0","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32512024-04-20T01:06:36.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.767347789Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"1","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32522024-04-20T01:06:36.272Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.76737908Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"1","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32532024-04-20T01:06:36.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.76740383Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"2","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32542024-04-20T01:06:36.272Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.76743662Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"2","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32552024-04-20T01:06:36.272Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.76746589Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32562024-04-20T01:06:36.272Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.76749155Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32572024-04-20T01:06:36.272Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.7675262Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32582024-04-20T01:06:36.272Z{"msg":"ds_connection connected from Ok(127.0.0.1:51318)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.76756377Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32592024-04-20T01:06:36.272Z{"msg":"ds_connection connected from Ok(127.0.0.1:42392)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.76760113Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32602024-04-20T01:06:36.272Z{"msg":"ds_connection connected from Ok(127.0.0.1:47955)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.7676358Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32612024-04-20T01:06:36.273Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.768251771Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"1","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32622024-04-20T01:06:36.273Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.768316932Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"1","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32632024-04-20T01:06:36.273Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.768391492Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"0","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32642024-04-20T01:06:36.273Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.768440342Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"0","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32652024-04-20T01:06:36.273Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.768766573Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"2","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32662024-04-20T01:06:36.273Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.768820143Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"2","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32672024-04-20T01:06:36.273Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.768898173Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"0","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32682024-04-20T01:06:36.273Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.768935023Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"1","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32692024-04-20T01:06:36.273Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.769327304Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"1","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32702024-04-20T01:06:36.273Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.769367984Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"1","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32712024-04-20T01:06:36.273Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.769383484Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32722024-04-20T01:06:36.273Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.769457714Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"2","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32732024-04-20T01:06:36.273Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.769484344Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"0","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32742024-04-20T01:06:36.273Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.769499184Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"0","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32752024-04-20T01:06:36.273Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.769512284Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32762024-04-20T01:06:36.273Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.769913125Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"2","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32772024-04-20T01:06:36.273Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.769949605Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"2","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32782024-04-20T01:06:36.273Z{"msg":"[0]R flush_numbers[0..12]: [41, 48, 48, 31, 33, 45, 44, 41, 41, 37, 43, 39]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.769964855Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32792024-04-20T01:06:36.273Z{"msg":"[0]R generation[0..12]: [26, 29, 29, 18, 20, 28, 28, 26, 26, 23, 27, 25]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.769986526Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32802024-04-20T01:06:36.273Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770001326Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32812024-04-20T01:06:36.273Z{"msg":"[1]R flush_numbers[0..12]: [41, 48, 48, 31, 33, 45, 44, 41, 41, 37, 43, 39]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770021995Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32822024-04-20T01:06:36.273Z{"msg":"[1]R generation[0..12]: [26, 29, 29, 18, 20, 28, 28, 26, 26, 23, 27, 25]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770036106Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32832024-04-20T01:06:36.273Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770049826Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32842024-04-20T01:06:36.274Z{"msg":"[2]R flush_numbers[0..12]: [41, 48, 48, 31, 33, 45, 44, 41, 41, 37, 43, 39]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770070376Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32852024-04-20T01:06:36.274Z{"msg":"[2]R generation[0..12]: [26, 29, 29, 18, 20, 28, 28, 26, 26, 23, 27, 25]","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO
32862024-04-20T01:06:36.274Z"time":"2024-04-20T01:06:34.770084246Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32872024-04-20T01:06:36.274Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770108316Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32882024-04-20T01:06:36.274Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770121996Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32892024-04-20T01:06:36.274Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770141566Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32902024-04-20T01:06:36.274Z{"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770154586Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32912024-04-20T01:06:36.274Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770167516Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"":"downstairs","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32922024-04-20T01:06:36.274Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770180846Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32932024-04-20T01:06:36.274Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770192796Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"0","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32942024-04-20T01:06:36.274Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770213156Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"1","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32952024-04-20T01:06:36.274Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770226776Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"client":"2","":"downstairs","session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32962024-04-20T01:06:36.274Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770240376Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32972024-04-20T01:06:36.274Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770259426Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32982024-04-20T01:06:36.274Z{"msg":"ebee81c2-8891-4049-bc91-91c1c800776d is now active with session: b0d4940b-2882-44cc-8749-d35aca90b8d4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770272386Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
32992024-04-20T01:06:36.274Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770284216Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288,"session_id":"b0d4940b-2882-44cc-8749-d35aca90b8d4"}
33002024-04-20T01:06:36.274Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:34.770303126Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288}
33012024-04-20T01:06:36.274ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33022024-04-20T01:06:36.274ZLoading write count information from file "/tmp/test_repair_verify.data"
33032024-04-20T01:06:36.274ZRun Repair workload
33042024-04-20T01:06:36.274Z01/30 Flush
33052024-04-20T01:06:36.274Z02/30 Write block 233 len 28672 data: 8 8 7 7 8 7 7
33062024-04-20T01:06:36.274Z03/30 Read block 143 len 28672
33072024-04-20T01:06:36.275Z04/30 Read block 331 len 16384
33082024-04-20T01:06:36.275Z05/30 Write block 248 len 4096 data: 6
33092024-04-20T01:06:36.275Z06/30 Write block 499 len 8192 data: 3 3
33102024-04-20T01:06:36.275Z07/30 Read block 513 len 8192
33112024-04-20T01:06:36.275Z08/30 Read block 478 len 8192
33122024-04-20T01:06:36.279Z09/30 Read block 458 len 40960
33132024-04-20T01:06:36.279Z10/30 Write block 332 len 40960 data: 3 2 2 2 2 3 3 3 4 4
33142024-04-20T01:06:36.279Z11/30 Write block 345 len 16384 data: 3 3 3 4
33152024-04-20T01:06:36.279Z12/30 Write block 474 len 32768 data: 6 8 7 6 5 3 3 6
33162024-04-20T01:06:36.279Z13/30 Write block 475 len 16384 data: 9 8 7 6
33172024-04-20T01:06:36.279Z14/30 Write block 489 len 16384 data: 4 5 4 4
33182024-04-20T01:06:36.279Z15/30 Write block 449 len 16384 data: 4 4 2 3
33192024-04-20T01:06:36.280Z16/30 Flush
33202024-04-20T01:06:36.280Z17/30 Write block 362 len 24576 data: 4 4 3 4 4 4
33212024-04-20T01:06:36.283Z18/30 Write block 393 len 28672 data: 3 3 3 3 2 2 2
33222024-04-20T01:06:36.283Z19/30 Read block 426 len 16384
33232024-04-20T01:06:36.283Z20/30 Flush
33242024-04-20T01:06:36.283Z21/30 Write block 380 len 20480 data: 4 4 3 3 2
33252024-04-20T01:06:36.287Z22/30 Write block 326 len 20480 data: 4 4 3 3 3
33262024-04-20T01:06:36.287Z23/30 Write block 241 len 4096 data: 4
33272024-04-20T01:06:36.287Z24/30 Read block 264 len 4096
33282024-04-20T01:06:36.287Z25/30 Read block 18 len 12288
33292024-04-20T01:06:36.287Z26/30 Read block 500 len 8192
33302024-04-20T01:06:36.287Z27/30 Write block 293 len 20480 data: 3 3 2 2 2
33312024-04-20T01:06:36.291Z28/30 Read block 223 len 4096
33322024-04-20T01:06:36.291Z29/30 Read block 14 len 12288
33332024-04-20T01:06:36.291Z30/30 Read block 562 len 20480
33342024-04-20T01:06:36.292Z----------------------------------------------------------------
33352024-04-20T01:06:36.292Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:29
33362024-04-20T01:06:36.292ZStates: Active Active Active
33372024-04-20T01:06:36.292ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33382024-04-20T01:06:36.292Z 2 Acked 1001 Write 7 Done Done Sent false
33392024-04-20T01:06:36.292Z 3 Acked 1002 Read 7 Done Done Sent false
33402024-04-20T01:06:36.292Z 4 Acked 1003 Read 4 Done Done Sent false
33412024-04-20T01:06:36.292Z 5 Acked 1004 Write 1 Done Done Sent false
33422024-04-20T01:06:36.292Z 6 Acked 1005 Write 2 Done Done Sent false
33432024-04-20T01:06:36.292Z 7 Acked 1006 Read 2 Done Done Sent false
33442024-04-20T01:06:36.292Z 8 Acked 1007 Read 2 Done Done Sent false
33452024-04-20T01:06:36.292Z 9 Acked 1008 Read 10 Done Done Sent false
33462024-04-20T01:06:36.292Z 10 Acked 1009 Write 10 Done Done Sent false
33472024-04-20T01:06:36.292Z 11 Acked 1010 Write 4 Done Done Sent false
33482024-04-20T01:06:36.292Z 12 Acked 1011 Write 8 Done Done Sent false
33492024-04-20T01:06:36.292Z 13 Acked 1012 Write 4 Done Done Sent false
33502024-04-20T01:06:36.292Z 14 Acked 1013 Write 4 Done Done Sent false
33512024-04-20T01:06:36.292Z 15 Acked 1014 Write 4 Done Done Sent false
33522024-04-20T01:06:36.292Z 16 Acked 1015 Flush 0 Done Done Sent false
33532024-04-20T01:06:36.292Z 17 Acked 1016 Write 6 Done Done Sent false
33542024-04-20T01:06:36.292Z 18 Acked 1017 Write 7 Done Done Sent false
33552024-04-20T01:06:36.292Z 19 Acked 1018 Read 4 Done Done Sent false
33562024-04-20T01:06:36.292Z 20 Acked 1019 Flush 0 Done Done Sent false
33572024-04-20T01:06:36.292Z 21 Acked 1020 Write 5 Done Done Sent false
33582024-04-20T01:06:36.293Z 22 Acked 1021 Write 5 Done Done Sent false
33592024-04-20T01:06:36.293Z 23 Acked 1022 Write 1 Done Done Sent false
33602024-04-20T01:06:36.293Z 24 Acked 1023 Read 1 Done Done Sent false
33612024-04-20T01:06:36.293Z 25 Acked 1024 Read 3 Done Done Sent false
33622024-04-20T01:06:36.293Z 26 Acked 1025 Read 2 Done Done Sent false
33632024-04-20T01:06:36.293Z 27 Acked 1026 Write 5 Done Done Sent false
33642024-04-20T01:06:36.293Z 28 Acked 1027 Read 1 Done Done Sent false
33652024-04-20T01:06:36.293Z 29 Acked 1028 Read 3 Done Done Sent false
33662024-04-20T01:06:36.293Z 30 Acked 1029 Read 5 Done Sent Sent false
33672024-04-20T01:06:36.293Z STATES DS:0 DS:1 DS:2 TOTAL
33682024-04-20T01:06:36.293Z New 0 0 0 0
33692024-04-20T01:06:36.293Z Sent 0 1 29 30
33702024-04-20T01:06:36.293Z Done 29 28 0 57
33712024-04-20T01:06:36.293Z Skipped 0 0 0 0
33722024-04-20T01:06:36.293Z Error 0 0 0 0
33732024-04-20T01:06:36.293ZLast Flush: 1019 1019 1000
33742024-04-20T01:06:36.293ZDownstairs last five completed: 1000
33752024-04-20T01:06:36.293ZUpstairs last five completed: 30 29 28 27 26
33762024-04-20T01:06:36.293Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:06:34.800047432Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1288}
33772024-04-20T01:06:36.293ZWrote out file "/tmp/test_repair_verify.data"
33782024-04-20T01:06:36.297Z
33792024-04-20T01:06:36.297ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33802024-04-20T01:06:36.313ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33812024-04-20T01:06:36.313Z 0 000-019 26 26 26 41 41 41 F F F
33822024-04-20T01:06:36.313Z 1 020-039 29 29 29 48 48 48 F F F
33832024-04-20T01:06:36.313Z 2 040-059 29 29 29 48 48 48 F F F
33842024-04-20T01:06:36.313Z 3 060-079 18 18 18 31 31 31 F F F
33852024-04-20T01:06:36.313Z 4 080-099 20 20 20 33 33 33 F F F
33862024-04-20T01:06:36.313Z 5 100-119 28 28 28 45 45 45 F F F
33872024-04-20T01:06:36.313Z 6 120-139 28 28 28 44 44 44 F F F
33882024-04-20T01:06:36.313Z 7 140-159 26 26 26 41 41 41 F F F
33892024-04-20T01:06:36.317Z 8 160-179 26 26 26 41 41 41 F F F
33902024-04-20T01:06:36.317Z 9 180-199 23 23 23 37 37 37 F F F
33912024-04-20T01:06:36.317Z 10 200-219 27 27 27 43 43 43 F F F
33922024-04-20T01:06:36.317Z 11 220-239 30 30 25 50 50 39 F F F <---
33932024-04-20T01:06:36.317Z 12 240-259 30 30 26 50 50 41 T T F <---
33942024-04-20T01:06:36.317Z 13 260-279 28 28 28 47 47 47 F F F
33952024-04-20T01:06:36.317Z 14 280-299 20 20 20 33 33 33 T T F <---
33962024-04-20T01:06:36.317Z 15 300-319 28 28 28 47 47 47 F F F
33972024-04-20T01:06:36.317Z 16 320-339 30 30 28 50 50 47 T T F <---
33982024-04-20T01:06:36.317Z 17 340-359 30 30 20 50 50 33 F F F <---
33992024-04-20T01:06:36.317Z 18 360-379 30 30 28 51 51 47 F F F <---
34002024-04-20T01:06:36.317Z 19 380-399 30 30 26 51 51 42 T T F <---
34012024-04-20T01:06:36.317Z 20 400-419 22 22 22 34 34 34 F F F
34022024-04-20T01:06:36.317Z 21 420-439 26 26 26 41 41 41 F F F
34032024-04-20T01:06:36.318Z 22 440-459 30 30 20 50 50 33 F F F <---
34042024-04-20T01:06:36.318Z 23 460-479 30 30 23 50 50 37 F F F <---
34052024-04-20T01:06:36.318Z 24 480-499 30 30 29 50 50 48 F F F <---
34062024-04-20T01:06:36.318Z 25 500-519 30 30 29 50 50 48 F F F <---
34072024-04-20T01:06:36.318Z 26 520-539 29 29 29 48 48 48 F F F
34082024-04-20T01:06:36.318Z 27 540-559 28 28 28 44 44 44 F F F
34092024-04-20T01:06:36.318Z 28 560-579 29 29 29 48 48 48 F F F
34102024-04-20T01:06:36.318Z 29 580-599 29 29 29 48 48 48 F F F
34112024-04-20T01:06:36.318ZMax gen: 30, Max flush: 51
34122024-04-20T01:06:36.318ZError: Difference in extent metadata found!
34132024-04-20T01:06:36.318ZA Difference in extent metadata is expected here
34142024-04-20T01:06:36.318ZOn loop 15
34152024-04-20T01:06:36.318Z
34162024-04-20T01:06:36.322ZVerifying data now
34172024-04-20T01:06:40.818ZNo change after verify
34182024-04-20T01:06:40.818ZLoop: 15 Downstairs dump after verify (and repair):
34192024-04-20T01:06:40.829ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34202024-04-20T01:06:40.829Z 0 000-019 26 26 26 41 41 41 F F F
34212024-04-20T01:06:40.829Z 1 020-039 29 29 29 48 48 48 F F F
34222024-04-20T01:06:40.829Z 2 040-059 29 29 29 48 48 48 F F F
34232024-04-20T01:06:40.829Z 3 060-079 18 18 18 31 31 31 F F F
34242024-04-20T01:06:40.829Z 4 080-099 20 20 20 33 33 33 F F F
34252024-04-20T01:06:40.829Z 5 100-119 28 28 28 45 45 45 F F F
34262024-04-20T01:06:40.829Z 6 120-139 28 28 28 44 44 44 F F F
34272024-04-20T01:06:40.830Z 7 140-159 26 26 26 41 41 41 F F F
34282024-04-20T01:06:40.834Z 8 160-179 26 26 26 41 41 41 F F F
34292024-04-20T01:06:40.834Z 9 180-199 23 23 23 37 37 37 F F F
34302024-04-20T01:06:40.834Z 10 200-219 27 27 27 43 43 43 F F F
34312024-04-20T01:06:40.834Z 11 220-239 30 30 30 50 50 50 F F F
34322024-04-20T01:06:40.834Z 12 240-259 31 31 31 52 52 52 F F F
34332024-04-20T01:06:40.834Z 13 260-279 28 28 28 47 47 47 F F F
34342024-04-20T01:06:40.834Z 14 280-299 31 31 31 52 52 52 F F F
34352024-04-20T01:06:40.834Z 15 300-319 28 28 28 47 47 47 F F F
34362024-04-20T01:06:40.834Z 16 320-339 31 31 31 52 52 52 F F F
34372024-04-20T01:06:40.834Z 17 340-359 30 30 30 50 50 50 F F F
34382024-04-20T01:06:40.834Z 18 360-379 30 30 30 51 51 51 F F F
34392024-04-20T01:06:40.834Z 19 380-399 31 31 31 52 52 52 F F F
34402024-04-20T01:06:40.834Z 20 400-419 22 22 22 34 34 34 F F F
34412024-04-20T01:06:40.834Z 21 420-439 26 26 26 41 41 41 F F F
34422024-04-20T01:06:40.834Z 22 440-459 30 30 30 50 50 50 F F F
34432024-04-20T01:06:40.834Z 23 460-479 30 30 30 50 50 50 F F F
34442024-04-20T01:06:40.834Z 24 480-499 30 30 30 50 50 50 F F F
34452024-04-20T01:06:40.834Z 25 500-519 30 30 30 50 50 50 F F F
34462024-04-20T01:06:40.834Z 26 520-539 29 29 29 48 48 48 F F F
34472024-04-20T01:06:40.834Z 27 540-559 28 28 28 44 44 44 F F F
34482024-04-20T01:06:40.835Z 28 560-579 29 29 29 48 48 48 F F F
34492024-04-20T01:06:40.835Z 29 580-599 29 29 29 48 48 48 F F F
34502024-04-20T01:06:40.835ZMax gen: 31, Max flush: 52
34512024-04-20T01:06:40.835Z
34522024-04-20T01:06:40.835ZBegin loop 16
34532024-04-20T01:06:40.835ZDownstairs to restart: 1
34542024-04-20T01:06:40.851Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.361421016Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296}
34552024-04-20T01:06:40.859Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.361703807Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296}
34562024-04-20T01:06:40.859Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.361737637Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296}
34572024-04-20T01:06:40.859Z{"msg":"Crucible 360519c5-fbf1-4a34-83c6-5dd70c8eb7cd has session id: 29766ddd-0b56-41a9-9bac-db3f11964ad3","v":0,"name":"crucible","level":30Crucible runtime is spawned
34582024-04-20T01:06:40.859Z,"time":"2024-04-20T01:06:39.361761147Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34592024-04-20T01:06:40.859Z{"msg":"Upstairs opts: Upstairs UUID: 360519c5-fbf1-4a34-83c6-5dd70c8eb7cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.361806797Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34602024-04-20T01:06:40.859Z{"msg":"Crucible stats registered with UUID: 360519c5-fbf1-4a34-83c6-5dd70c8eb7cd","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.361840807Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34612024-04-20T01:06:40.859Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.361864667Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296}
34622024-04-20T01:06:40.859Z{"msg":"360519c5-fbf1-4a34-83c6-5dd70c8eb7cd active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.361964197Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34632024-04-20T01:06:40.859Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.361994997Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"0","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34642024-04-20T01:06:40.859Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.362020797Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"0","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34652024-04-20T01:06:40.859Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.362058017Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"1","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34662024-04-20T01:06:40.859Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.362085428Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"1","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34672024-04-20T01:06:40.859Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.362121568Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"2","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34682024-04-20T01:06:40.859Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.362145528Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"2","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34692024-04-20T01:06:40.859Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.362178318Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34702024-04-20T01:06:40.859Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.362207058Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34712024-04-20T01:06:40.859Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.362235668Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34722024-04-20T01:06:40.859Z{"msg":"ds_connection connected from Ok(127.0.0.1:41456)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.362261378Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34732024-04-20T01:06:40.859Z{"msg":"ds_connection connected from Ok(127.0.0.1:45325)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.362287518Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34742024-04-20T01:06:40.859Z{"msg":"ds_connection connected from Ok(127.0.0.1:54847)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.362322828Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34752024-04-20T01:06:40.860Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.36304793Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"0","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34762024-04-20T01:06:40.860Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.3631089Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"0","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34772024-04-20T01:06:40.860Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.36313464Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"2","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34782024-04-20T01:06:40.860Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.36320163Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"2","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34792024-04-20T01:06:40.860Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.363598211Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"1","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34802024-04-20T01:06:40.860Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.363646081Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"1","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34812024-04-20T01:06:40.860Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.363803112Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"2","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34822024-04-20T01:06:40.860Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.363842681Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"0","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34832024-04-20T01:06:40.860Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364137532Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"1","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34842024-04-20T01:06:40.860Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364269853Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"0","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34852024-04-20T01:06:40.860Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364317133Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"0","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34862024-04-20T01:06:40.860Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364343613Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34872024-04-20T01:06:40.860Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364370283Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"2","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34882024-04-20T01:06:40.860Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364395503Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"2","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34892024-04-20T01:06:40.860Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364422593Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34902024-04-20T01:06:40.860Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364683813Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"1","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34912024-04-20T01:06:40.860Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364715594Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"1","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34922024-04-20T01:06:40.860Z{"msg":"[0]R flush_numbers[0..12]: [41, 48, 48, 31, 33, 45, 44, 41, 41, 37, 43, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364741163Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34932024-04-20T01:06:40.860Z{"msg":"[0]R generation[0..12]: [26, 29, 29, 18, 20, 28, 28, 26, 26, 23, 27, 30]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364771374Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34942024-04-20T01:06:40.860Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364797354Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34952024-04-20T01:06:40.860Z{"msg":"[1]R flush_numbers[0..12]: [41, 48, 48, 31, 33, 45, 44, 41, 41, 37, 43, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364825504Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34962024-04-20T01:06:40.860Z{"msg":"[1]R generation[0..12]: [26, 29, 29, 18, 20, 28, 28, 26, 26, 23, 27, 30]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364850564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34972024-04-20T01:06:40.860ZWait for a query_work_queue command to finish before sending IO
34982024-04-20T01:06:40.861Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364885074Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
34992024-04-20T01:06:40.861Z{"msg":"[2]R flush_numbers[0..12]: [41, 48, 48, 31, 33, 45, 44, 41, 41, 37, 43, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364908154Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35002024-04-20T01:06:40.861Z{"msg":"[2]R generation[0..12]: [26, 29, 29, 18, 20, 28, 28, 26, 26, 23, 27, 30]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364932634Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35012024-04-20T01:06:40.861Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364966124Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35022024-04-20T01:06:40.861Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.364990294Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35032024-04-20T01:06:40.861Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.365021114Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35042024-04-20T01:06:40.861Z{"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.365044474Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35052024-04-20T01:06:40.861Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.365070534Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"":"downstairs","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35062024-04-20T01:06:40.861Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.365093094Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35072024-04-20T01:06:40.861Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.365119144Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"0","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35082024-04-20T01:06:40.861Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.365142334Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"1","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35092024-04-20T01:06:40.861ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35102024-04-20T01:06:40.861Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.365174304Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"client":"2","":"downstairs","session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35112024-04-20T01:06:40.861Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.365198695Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35122024-04-20T01:06:40.861Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.365219984Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35132024-04-20T01:06:40.861Z{"msg":"360519c5-fbf1-4a34-83c6-5dd70c8eb7cd is now active with session: 29766ddd-0b56-41a9-9bac-db3f11964ad3","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.365242165Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35142024-04-20T01:06:40.861Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.365271485Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296,"session_id":"29766ddd-0b56-41a9-9bac-db3f11964ad3"}
35152024-04-20T01:06:40.861Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:39.365293665Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296}
35162024-04-20T01:06:40.865ZLoading write count information from file "/tmp/test_repair_verify.data"
35172024-04-20T01:06:40.865ZRun Repair workload
35182024-04-20T01:06:40.865Z01/30 Write block 454 len 32768 data: 3 5 5 6 6 6 6 7
35192024-04-20T01:06:40.865Z02/30 Read block 167 len 4096
35202024-04-20T01:06:40.865Z03/30 Write block 413 len 28672 data: 5 5 5 4 2 3 3
35212024-04-20T01:06:40.865Z04/30 Write block 516 len 28672 data: 5 5 4 6 6 6 5
35222024-04-20T01:06:40.865Z05/30 Write block 465 len 40960 data: 4 4 3 3 4 3 3 4 5 7
35232024-04-20T01:06:40.865Z06/30 Read block 165 len 20480
35242024-04-20T01:06:40.865Z07/30 Write block 410 len 20480 data: 4 5 5 6 6
35252024-04-20T01:06:40.866Z08/30 Read block 157 len 32768
35262024-04-20T01:06:40.866Z09/30 Read block 167 len 40960
35272024-04-20T01:06:40.869Z10/30 Read block 539 len 16384
35282024-04-20T01:06:40.869Z11/30 Flush
35292024-04-20T01:06:40.872Z12/30 Write block 416 len 40960 data: 5 3 4 4 4 5 4 4 3 3
35302024-04-20T01:06:40.872Z13/30 Flush
35312024-04-20T01:06:40.876Z14/30 Write block 326 len 40960 data: 5 5 4 4 4 3 4 3 3 3
35322024-04-20T01:06:40.876Z15/30 Read block 305 len 4096
35332024-04-20T01:06:40.876Z16/30 Read block 198 len 32768
35342024-04-20T01:06:40.880Z17/30 Write block 338 len 40960 data: 4 4 5 5 3 4 3 4 4 4
35352024-04-20T01:06:40.880Z18/30 Write block 213 len 24576 data: 3 3 2 3 5 5
35362024-04-20T01:06:40.880Z19/30 Write block 562 len 40960 data: 3 3 3 3 4 3 3 5 6 5
35372024-04-20T01:06:40.884Z20/30 Read block 362 len 36864
35382024-04-20T01:06:40.884Z21/30 Write block 205 len 40960 data: 3 3 3 3 4 2 2 3 4 4
35392024-04-20T01:06:40.884Z22/30 Read block 390 len 32768
35402024-04-20T01:06:40.888Z23/30 Write block 179 len 12288 data: 5 5 5
35412024-04-20T01:06:40.888Z24/30 Write block 11 len 28672 data: 3 3 3 3 4 4 4
35422024-04-20T01:06:40.888Z25/30 Write block 195 len 20480 data: 3 3 4 4 4
35432024-04-20T01:06:40.888Z26/30 Read block 251 len 8192
35442024-04-20T01:06:40.888Z27/30 Read block 230 len 32768
35452024-04-20T01:06:40.893Z28/30 Write block 130 len 20480 data: 5 6 6 6 6
35462024-04-20T01:06:40.893Z29/30 Read block 211 len 32768
35472024-04-20T01:06:40.893Z30/30 Read block 269 len 20480
35482024-04-20T01:06:40.893Z----------------------------------------------------------------
35492024-04-20T01:06:40.894Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35502024-04-20T01:06:40.894ZStates: Active Active Active
35512024-04-20T01:06:40.894ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35522024-04-20T01:06:40.894Z 1 Acked 1000 Write 8 Done Done Done false
35532024-04-20T01:06:40.894Z 2 Acked 1001 Read 1 Done Sent Done false
35542024-04-20T01:06:40.894Z 3 Acked 1002 Write 7 Done Sent Done false
35552024-04-20T01:06:40.894Z 4 Acked 1003 Write 7 Done Sent Done false
35562024-04-20T01:06:40.894Z 5 Acked 1004 Write 10 Done Sent Done false
35572024-04-20T01:06:40.894Z 6 Acked 1005 Read 5 Done Sent Done false
35582024-04-20T01:06:40.894Z 7 Acked 1006 Write 5 Done Sent Done false
35592024-04-20T01:06:40.894Z 8 Acked 1007 Read 8 Done Sent Done false
35602024-04-20T01:06:40.894Z 9 Acked 1008 Read 10 Done Sent Done false
35612024-04-20T01:06:40.894Z 10 Acked 1009 Read 4 Done Sent Done false
35622024-04-20T01:06:40.894Z 11 Acked 1010 Flush 0 Done Sent Done false
35632024-04-20T01:06:40.894Z 12 Acked 1011 Write 10 Done Sent Done false
35642024-04-20T01:06:40.894Z 13 Acked 1012 Flush 0 Done Sent Done false
35652024-04-20T01:06:40.894Z 14 Acked 1013 Write 10 Done Sent Done false
35662024-04-20T01:06:40.894Z 15 Acked 1014 Read 1 Done Sent Done false
35672024-04-20T01:06:40.894Z 16 Acked 1015 Read 8 Done Sent Done false
35682024-04-20T01:06:40.894Z 17 Acked 1016 Write 10 Done Sent Done false
35692024-04-20T01:06:40.895Z 18 Acked 1017 Write 6 Done Sent Done false
35702024-04-20T01:06:40.895Z 19 Acked 1018 Write 10 Done Sent Done false
35712024-04-20T01:06:40.895Z 20 Acked 1019 Read 9 Done Sent Done false
35722024-04-20T01:06:40.895Z 21 Acked 1020 Write 10 Done Sent Done false
35732024-04-20T01:06:40.895Z 22 Acked 1021 Read 8 Done Sent Done false
35742024-04-20T01:06:40.895Z 23 Acked 1022 Write 3 Done Sent Done false
35752024-04-20T01:06:40.895Z 24 Acked 1023 Write 7 Done Sent Done false
35762024-04-20T01:06:40.895Z 25 Acked 1024 Write 5 Done Sent Done false
35772024-04-20T01:06:40.895Z 26 Acked 1025 Read 2 Done Sent Done false
35782024-04-20T01:06:40.895Z 27 Acked 1026 Read 8 Done Sent Done false
35792024-04-20T01:06:40.895Z 28 Acked 1027 Write 5 Done Sent Done false
35802024-04-20T01:06:40.895Z 29 Acked 1028 Read 8 Done Sent Done false
35812024-04-20T01:06:40.895Z 30 Acked 1029 Read 5 Done Sent Done false
35822024-04-20T01:06:40.895Z STATES DS:0 DS:1 DS:2 TOTAL
35832024-04-20T01:06:40.895Z New 0 0 0 0
35842024-04-20T01:06:40.895Z Sent 0 29 0 29
35852024-04-20T01:06:40.895Z Done 30 1 30 61
35862024-04-20T01:06:40.895Z Skipped 0 0 0 0
35872024-04-20T01:06:40.895Z Error 0 0 0 0
35882024-04-20T01:06:40.895ZLast Flush: 1012 0 1012
35892024-04-20T01:06:40.895ZDownstairs last five completed:
35902024-04-20T01:06:40.895ZUpstairs last five completed: 30 29 28 27 26
35912024-04-20T01:06:40.895Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:06:39.400792454Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1296}
35922024-04-20T01:06:40.895ZWrote out file "/tmp/test_repair_verify.data"
35932024-04-20T01:06:40.899Z
35942024-04-20T01:06:40.899ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
35952024-04-20T01:06:40.915ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
35962024-04-20T01:06:40.915Z 0 000-019 26 26 26 41 41 41 T F T <---
35972024-04-20T01:06:40.915Z 1 020-039 29 29 29 48 48 48 F F F
35982024-04-20T01:06:40.915Z 2 040-059 29 29 29 48 48 48 F F F
35992024-04-20T01:06:40.915Z 3 060-079 18 18 18 31 31 31 F F F
36002024-04-20T01:06:40.915Z 4 080-099 20 20 20 33 33 33 F F F
36012024-04-20T01:06:40.915Z 5 100-119 28 28 28 45 45 45 F F F
36022024-04-20T01:06:40.920Z 6 120-139 28 28 28 44 44 44 T F T <---
36032024-04-20T01:06:40.920Z 7 140-159 26 26 26 41 41 41 F F F
36042024-04-20T01:06:40.920Z 8 160-179 26 26 26 41 41 41 T F T <---
36052024-04-20T01:06:40.920Z 9 180-199 23 23 23 37 37 37 T F T <---
36062024-04-20T01:06:40.920Z 10 200-219 27 27 27 43 43 43 T F T <---
36072024-04-20T01:06:40.920Z 11 220-239 30 30 30 50 50 50 F F F
36082024-04-20T01:06:40.921Z 12 240-259 31 31 31 52 52 52 F F F
36092024-04-20T01:06:40.921Z 13 260-279 28 28 28 47 47 47 F F F
36102024-04-20T01:06:40.921Z 14 280-299 31 31 31 52 52 52 F F F
36112024-04-20T01:06:40.921Z 15 300-319 28 28 28 47 47 47 F F F
36122024-04-20T01:06:40.921Z 16 320-339 31 31 31 52 52 52 T F T <---
36132024-04-20T01:06:40.921Z 17 340-359 30 30 30 50 50 50 T F T <---
36142024-04-20T01:06:40.921Z 18 360-379 30 30 30 51 51 51 F F F
36152024-04-20T01:06:40.921Z 19 380-399 31 31 31 52 52 52 F F F
36162024-04-20T01:06:40.921Z 20 400-419 32 22 32 54 34 54 F F F <---
36172024-04-20T01:06:40.921Z 21 420-439 32 26 32 54 41 54 F F F <---
36182024-04-20T01:06:40.921Z 22 440-459 32 30 32 53 50 53 F T F <---
36192024-04-20T01:06:40.921Z 23 460-479 32 30 32 53 50 53 F T F <---
36202024-04-20T01:06:40.921Z 24 480-499 30 30 30 50 50 50 F F F
36212024-04-20T01:06:40.921Z 25 500-519 32 30 32 53 50 53 F F F <---
36222024-04-20T01:06:40.921Z 26 520-539 32 29 32 53 48 53 F F F <---
36232024-04-20T01:06:40.921Z 27 540-559 28 28 28 44 44 44 F F F
36242024-04-20T01:06:40.921Z 28 560-579 29 29 29 48 48 48 T F T <---
36252024-04-20T01:06:40.921Z 29 580-599 29 29 29 48 48 48 F F F
36262024-04-20T01:06:40.921ZMax gen: 32, Max flush: 54
36272024-04-20T01:06:40.921ZError: Difference in extent metadata found!
36282024-04-20T01:06:40.921ZA Difference in extent metadata is expected here
36292024-04-20T01:06:40.921ZOn loop 16
36302024-04-20T01:06:40.921Z
36312024-04-20T01:06:40.925ZVerifying data now
36322024-04-20T01:06:55.550ZNo change after verify
36332024-04-20T01:06:55.550ZLoop: 16 Downstairs dump after verify (and repair):
36342024-04-20T01:06:55.564ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36352024-04-20T01:06:55.564Z 0 000-019 33 33 33 55 55 55 F F F
36362024-04-20T01:06:55.568Z 1 020-039 29 29 29 48 48 48 F F F
36372024-04-20T01:06:55.568Z 2 040-059 29 29 29 48 48 48 F F F
36382024-04-20T01:06:55.568Z 3 060-079 18 18 18 31 31 31 F F F
36392024-04-20T01:06:55.568Z 4 080-099 20 20 20 33 33 33 F F F
36402024-04-20T01:06:55.568Z 5 100-119 28 28 28 45 45 45 F F F
36412024-04-20T01:06:55.568Z 6 120-139 33 33 33 55 55 55 F F F
36422024-04-20T01:06:55.568Z 7 140-159 26 26 26 41 41 41 F F F
36432024-04-20T01:06:55.568Z 8 160-179 33 33 33 55 55 55 F F F
36442024-04-20T01:06:55.568Z 9 180-199 33 33 33 55 55 55 F F F
36452024-04-20T01:06:55.568Z 10 200-219 33 33 33 55 55 55 F F F
36462024-04-20T01:06:55.568Z 11 220-239 30 30 30 50 50 50 F F F
36472024-04-20T01:06:55.568Z 12 240-259 31 31 31 52 52 52 F F F
36482024-04-20T01:06:55.568Z 13 260-279 28 28 28 47 47 47 F F F
36492024-04-20T01:06:55.568Z 14 280-299 31 31 31 52 52 52 F F F
36502024-04-20T01:06:55.568Z 15 300-319 28 28 28 47 47 47 F F F
36512024-04-20T01:06:55.568Z 16 320-339 33 33 33 55 55 55 F F F
36522024-04-20T01:06:55.568Z 17 340-359 33 33 33 55 55 55 F F F
36532024-04-20T01:06:55.568Z 18 360-379 30 30 30 51 51 51 F F F
36542024-04-20T01:06:55.568Z 19 380-399 31 31 31 52 52 52 F F F
36552024-04-20T01:06:55.568Z 20 400-419 32 32 32 54 54 54 F F F
36562024-04-20T01:06:55.569Z 21 420-439 32 32 32 54 54 54 F F F
36572024-04-20T01:06:55.569Z 22 440-459 32 32 32 53 53 53 F F F
36582024-04-20T01:06:55.569Z 23 460-479 32 32 32 53 53 53 F F F
36592024-04-20T01:06:55.569Z 24 480-499 30 30 30 50 50 50 F F F
36602024-04-20T01:06:55.569Z 25 500-519 32 32 32 53 53 53 F F F
36612024-04-20T01:06:55.569Z 26 520-539 32 32 32 53 53 53 F F F
36622024-04-20T01:06:55.569Z 27 540-559 28 28 28 44 44 44 F F F
36632024-04-20T01:06:55.569Z 28 560-579 33 33 33 55 55 55 F F F
36642024-04-20T01:06:55.569Z 29 580-599 29 29 29 48 48 48 F F F
36652024-04-20T01:06:55.569ZMax gen: 33, Max flush: 55
36662024-04-20T01:06:55.569Z
36672024-04-20T01:06:55.569ZBegin loop 17
36682024-04-20T01:06:55.569ZDownstairs to restart: 0
36692024-04-20T01:06:55.586Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096300646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304}
36702024-04-20T01:06:55.587Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096554887Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304}
36712024-04-20T01:06:55.594Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096580947Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304}
36722024-04-20T01:06:55.594Z{"msg":"Crucible 22e7188d-ad6e-43e2-8d99-4d4894660f40 has session id: 5f16b389-4de7-4d78-b051-199bbfef13b9","v":0,"name":"crucible","level":30Crucible runtime is spawned
36732024-04-20T01:06:55.594Z,"time":"2024-04-20T01:06:54.096616136Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36742024-04-20T01:06:55.594Z{"msg":"Upstairs opts: Upstairs UUID: 22e7188d-ad6e-43e2-8d99-4d4894660f40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096659857Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36752024-04-20T01:06:55.594Z{"msg":"Crucible stats registered with UUID: 22e7188d-ad6e-43e2-8d99-4d4894660f40","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096692357Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36762024-04-20T01:06:55.594Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096706957Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304}
36772024-04-20T01:06:55.594Z{"msg":"22e7188d-ad6e-43e2-8d99-4d4894660f40 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096718897Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36782024-04-20T01:06:55.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096742057Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"0","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36792024-04-20T01:06:55.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096766917Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"0","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36802024-04-20T01:06:55.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096792147Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"1","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36812024-04-20T01:06:55.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096828707Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"1","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36822024-04-20T01:06:55.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096854537Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"2","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36832024-04-20T01:06:55.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.096948317Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"2","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36842024-04-20T01:06:55.595Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.097032728Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36852024-04-20T01:06:55.595Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.097073958Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36862024-04-20T01:06:55.595Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.097098158Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36872024-04-20T01:06:55.595Z{"msg":"ds_connection connected from Ok(127.0.0.1:47378)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.097127288Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36882024-04-20T01:06:55.595Z{"msg":"ds_connection connected from Ok(127.0.0.1:40480)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.097152748Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36892024-04-20T01:06:55.595Z{"msg":"ds_connection connected from Ok(127.0.0.1:48571)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.097197488Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36902024-04-20T01:06:55.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.097869639Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"1","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36912024-04-20T01:06:55.595Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.09791607Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"1","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36922024-04-20T01:06:55.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.09794269Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"2","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36932024-04-20T01:06:55.595Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.097965659Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"2","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36942024-04-20T01:06:55.595Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.098430971Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"1","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36952024-04-20T01:06:55.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.098471791Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"0","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36962024-04-20T01:06:55.595Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.098494071Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"0","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36972024-04-20T01:06:55.595Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.098611381Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"2","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36982024-04-20T01:06:55.595Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.098995662Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"1","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
36992024-04-20T01:06:55.595Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099030762Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"1","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37002024-04-20T01:06:55.595Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099046242Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37012024-04-20T01:06:55.595Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099123452Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"2","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37022024-04-20T01:06:55.595Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099163762Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"2","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37032024-04-20T01:06:55.595Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099199172Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37042024-04-20T01:06:55.595Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099381943Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"0","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37052024-04-20T01:06:55.595Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099779804Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"0","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37062024-04-20T01:06:55.596Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099823294Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"0","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37072024-04-20T01:06:55.596Z{"msg":"[0]R flush_numbers[0..12]: [55, 48, 48, 31, 33, 45, 55, 41, 55, 55, 55, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099841034Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37082024-04-20T01:06:55.596Z{"msg":"[0]R generation[0..12]: [33, 29, 29, 18, 20, 28, 33, 26, 33, 33, 33, 30]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099863514Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37092024-04-20T01:06:55.596Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099878644Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37102024-04-20T01:06:55.596Z{"msg":"[1]R flush_numbers[0..12]: [55, 48, 48, 31, 33, 45, 55, 41, 55, 55, 55, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099893044Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37112024-04-20T01:06:55.596Z{"msg":"[1]R generation[0..12]: [33, 29, 29, 18, 20, 28, 33, 26, 33, 33, 33, 30]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099919414Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37122024-04-20T01:06:55.596Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099934474Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37132024-04-20T01:06:55.596ZWait for a query_work_queue command to finish before sending IO
37142024-04-20T01:06:55.596Z{"msg":"[2]R flush_numbers[0..12]: [55, 48, 48, 31, 33, 45, 55, 41, 55, 55, 55, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099959304Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37152024-04-20T01:06:55.596Z{"msg":"[2]R generation[0..12]: [33, 29, 29, 18, 20, 28, 33, 26, 33, 33, 33, 30]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.099974074Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37162024-04-20T01:06:55.596Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100029674Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37172024-04-20T01:06:55.596Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100044924Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37182024-04-20T01:06:55.596Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100058334Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37192024-04-20T01:06:55.596Z{"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100078544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37202024-04-20T01:06:55.596Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100091934Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"":"downstairs","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37212024-04-20T01:06:55.596Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100112004Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37222024-04-20T01:06:55.596Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100124234Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"0","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37232024-04-20T01:06:55.596Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100137435Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"1","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37242024-04-20T01:06:55.596Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100157344Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"client":"2","":"downstairs","session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37252024-04-20T01:06:55.596Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100170955Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37262024-04-20T01:06:55.596Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100184095Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37272024-04-20T01:06:55.596Z{"msg":"22e7188d-ad6e-43e2-8d99-4d4894660f40 is now active with session: 5f16b389-4de7-4d78-b051-199bbfef13b9","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100204744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37282024-04-20T01:06:55.596Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100217065Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304,"session_id":"5f16b389-4de7-4d78-b051-199bbfef13b9"}
37292024-04-20T01:06:55.596Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:54.100229825Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304}
37302024-04-20T01:06:55.597ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37312024-04-20T01:06:55.601ZLoading write count information from file "/tmp/test_repair_verify.data"
37322024-04-20T01:06:55.601ZRun Repair workload
37332024-04-20T01:06:55.601Z01/30 Write block 470 len 4096 data: 4
37342024-04-20T01:06:55.601Z02/30 Write block 37 len 32768 data: 3 3 4 4 3 2 2 2
37352024-04-20T01:06:55.601Z03/30 Write block 483 len 24576 data: 4 4 3 3 3 3
37362024-04-20T01:06:55.601Z04/30 Write block 511 len 32768 data: 6 7 7 6 6 6 6 5
37372024-04-20T01:06:55.601Z05/30 Read block 530 len 28672
37382024-04-20T01:06:55.601Z06/30 Write block 538 len 24576 data: 2 2 2 4 5 7
37392024-04-20T01:06:55.601Z07/30 Read block 503 len 8192
37402024-04-20T01:06:55.601Z08/30 Read block 91 len 8192
37412024-04-20T01:06:55.601Z09/30 Read block 205 len 32768
37422024-04-20T01:06:55.601Z10/30 Write block 179 len 40960 data: 6 6 6 4 3 3 2 2 2 2
37432024-04-20T01:06:55.602Z11/30 Read block 46 len 12288
37442024-04-20T01:06:55.605Z12/30 Read block 209 len 28672
37452024-04-20T01:06:55.605Z13/30 Write block 38 len 8192 data: 4 5
37462024-04-20T01:06:55.605Z14/30 Read block 192 len 28672
37472024-04-20T01:06:55.605Z15/30 Write block 90 len 16384 data: 4 2 2 2
37482024-04-20T01:06:55.605Z16/30 Write block 294 len 4096 data: 4
37492024-04-20T01:06:55.619Z17/30 Write block 96 len 28672 data: 2 2 3 3 4 4 4
37502024-04-20T01:06:55.619Z18/30 Read block 131 len 12288
37512024-04-20T01:06:55.619Z19/30 Flush
37522024-04-20T01:06:55.619Z20/30 Write block 28 len 8192 data: 5 5
37532024-04-20T01:06:55.619Z21/30 Read block 469 len 8192
37542024-04-20T01:06:55.619Z22/30 Flush
37552024-04-20T01:06:55.619Z23/30 Flush
37562024-04-20T01:06:55.619Z24/30 Write block 403 len 16384 data: 3 2 3 3
37572024-04-20T01:06:55.619Z25/30 Read block 86 len 20480
37582024-04-20T01:06:55.619Z26/30 Read block 480 len 24576
37592024-04-20T01:06:55.619Z27/30 Read block 424 len 4096
37602024-04-20T01:06:55.619Z28/30 Write block 155 len 40960 data: 5 3 3 4 4 5 5 5 5 4
37612024-04-20T01:06:55.619Z29/30 Write block 105 len 8192 data: 4 5
37622024-04-20T01:06:55.619Z30/30 Read block 561 len 32768
37632024-04-20T01:06:55.624Z----------------------------------------------------------------
37642024-04-20T01:06:55.624Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
37652024-04-20T01:06:55.624ZStates: Active Active Active
37662024-04-20T01:06:55.624ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37672024-04-20T01:06:55.624Z 1 Acked 1000 Write 1 Done Done Done false
37682024-04-20T01:06:55.624Z 2 Acked 1001 Write 8 Sent Done Done false
37692024-04-20T01:06:55.624Z 3 Acked 1002 Write 6 Sent Done Done false
37702024-04-20T01:06:55.624Z 4 Acked 1003 Write 8 Sent Done Done false
37712024-04-20T01:06:55.624Z 5 Acked 1004 Read 7 Sent Done Done false
37722024-04-20T01:06:55.624Z 6 Acked 1005 Write 6 Sent Done Done false
37732024-04-20T01:06:55.624Z 7 Acked 1006 Read 2 Sent Done Done false
37742024-04-20T01:06:55.625Z 8 Acked 1007 Read 2 Sent Done Done false
37752024-04-20T01:06:55.625Z 9 Acked 1008 Read 8 Sent Done Done false
37762024-04-20T01:06:55.625Z 10 Acked 1009 Write 10 Sent Done Done false
37772024-04-20T01:06:55.625Z 11 Acked 1010 Read 3 Sent Done Done false
37782024-04-20T01:06:55.625Z 12 Acked 1011 Read 7 Sent Done Done false
37792024-04-20T01:06:55.625Z 13 Acked 1012 Write 2 Sent Done Done false
37802024-04-20T01:06:55.625Z 14 Acked 1013 Read 7 Sent Done Done false
37812024-04-20T01:06:55.625Z 15 Acked 1014 Write 4 Sent Done Done false
37822024-04-20T01:06:55.625Z 16 Acked 1015 Write 1 Sent Done Done false
37832024-04-20T01:06:55.625Z 17 Acked 1016 Write 7 Sent Done Done false
37842024-04-20T01:06:55.625Z 18 Acked 1017 Read 3 Sent Done Done false
37852024-04-20T01:06:55.625Z 19 Acked 1018 Flush 0 Sent Done Done false
37862024-04-20T01:06:55.625Z 20 Acked 1019 Write 2 Sent Done Done false
37872024-04-20T01:06:55.625Z 21 Acked 1020 Read 2 Sent Done Done false
37882024-04-20T01:06:55.625Z 22 Acked 1021 Flush 0 Sent Done Done false
37892024-04-20T01:06:55.625Z 23 Acked 1022 Flush 0 Sent Done Done false
37902024-04-20T01:06:55.625Z 24 Acked 1023 Write 4 Sent Done Done false
37912024-04-20T01:06:55.625Z 25 Acked 1024 Read 5 Sent Done Done false
37922024-04-20T01:06:55.625Z 26 Acked 1025 Read 6 Sent Done Done false
37932024-04-20T01:06:55.625Z 27 Acked 1026 Read 1 Sent Done Done false
37942024-04-20T01:06:55.625Z 28 Acked 1027 Write 10 Sent Done Done false
37952024-04-20T01:06:55.625Z 29 Acked 1028 Write 2 Sent Done Done false
37962024-04-20T01:06:55.625Z 30 Acked 1029 Read 8 Sent Done Done false
37972024-04-20T01:06:55.626Z STATES DS:0 DS:1 DS:2 TOTAL
37982024-04-20T01:06:55.626Z New 0 0 0 0
37992024-04-20T01:06:55.626Z Sent 29 0 0 29
38002024-04-20T01:06:55.626Z Done 1 30 30 61
38012024-04-20T01:06:55.626Z Skipped 0 0 0 0
38022024-04-20T01:06:55.626Z Error 0 0 0 0
38032024-04-20T01:06:55.626ZLast Flush: 0 1022 1022
38042024-04-20T01:06:55.626ZDownstairs last five completed:
38052024-04-20T01:06:55.626ZUpstairs last five completed: 30 29 28 27 26
38062024-04-20T01:06:55.626Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:06:54.129879871Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1304}
38072024-04-20T01:06:55.626ZWrote out file "/tmp/test_repair_verify.data"
38082024-04-20T01:06:55.626Z
38092024-04-20T01:06:55.630ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38102024-04-20T01:06:55.643ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38112024-04-20T01:06:55.643Z 0 000-019 33 33 33 55 55 55 F F F
38122024-04-20T01:06:55.643Z 1 020-039 29 34 34 48 57 57 F F F <---
38132024-04-20T01:06:55.643Z 2 040-059 29 34 34 48 56 56 F F F <---
38142024-04-20T01:06:55.643Z 3 060-079 18 18 18 31 31 31 F F F
38152024-04-20T01:06:55.643Z 4 080-099 20 34 34 33 56 56 F F F <---
38162024-04-20T01:06:55.643Z 5 100-119 28 34 34 45 56 56 F T T <---
38172024-04-20T01:06:55.643Z 6 120-139 33 33 33 55 55 55 F F F
38182024-04-20T01:06:55.643Z 7 140-159 26 26 26 41 41 41 F T T <---
38192024-04-20T01:06:55.643Z 8 160-179 33 34 34 55 56 56 F T T <---
38202024-04-20T01:06:55.643Z 9 180-199 33 34 34 55 56 56 F F F <---
38212024-04-20T01:06:55.643Z 10 200-219 33 33 33 55 55 55 F F F
38222024-04-20T01:06:55.650Z 11 220-239 30 30 30 50 50 50 F F F
38232024-04-20T01:06:55.650Z 12 240-259 31 31 31 52 52 52 F F F
38242024-04-20T01:06:55.650Z 13 260-279 28 28 28 47 47 47 F F F
38252024-04-20T01:06:55.650Z 14 280-299 31 34 34 52 56 56 F F F <---
38262024-04-20T01:06:55.650Z 15 300-319 28 28 28 47 47 47 F F F
38272024-04-20T01:06:55.650Z 16 320-339 33 33 33 55 55 55 F F F
38282024-04-20T01:06:55.650Z 17 340-359 33 33 33 55 55 55 F F F
38292024-04-20T01:06:55.650Z 18 360-379 30 30 30 51 51 51 F F F
38302024-04-20T01:06:55.650Z 19 380-399 31 31 31 52 52 52 F F F
38312024-04-20T01:06:55.650Z 20 400-419 32 32 32 54 54 54 F T T <---
38322024-04-20T01:06:55.650Z 21 420-439 32 32 32 54 54 54 F F F
38332024-04-20T01:06:55.650Z 22 440-459 32 32 32 53 53 53 F F F
38342024-04-20T01:06:55.650Z 23 460-479 32 34 34 53 56 56 T F F <---
38352024-04-20T01:06:55.650Z 24 480-499 30 34 34 50 56 56 F F F <---
38362024-04-20T01:06:55.650Z 25 500-519 32 34 34 53 56 56 F F F <---
38372024-04-20T01:06:55.650Z 26 520-539 32 34 34 53 56 56 F F F <---
38382024-04-20T01:06:55.650Z 27 540-559 28 34 34 44 56 56 F F F <---
38392024-04-20T01:06:55.650Z 28 560-579 33 33 33 55 55 55 F F F
38402024-04-20T01:06:55.651Z 29 580-599 29 29 29 48 48 48 F F F
38412024-04-20T01:06:55.651ZMax gen: 34, Max flush: 57
38422024-04-20T01:06:55.651ZError: Difference in extent metadata found!
38432024-04-20T01:06:55.651ZA Difference in extent metadata is expected here
38442024-04-20T01:06:55.651ZOn loop 17
38452024-04-20T01:06:55.651Z
38462024-04-20T01:06:55.654ZVerifying data now
38472024-04-20T01:07:00.273ZNo change after verify
38482024-04-20T01:07:00.273ZLoop: 17 Downstairs dump after verify (and repair):
38492024-04-20T01:07:00.287ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38502024-04-20T01:07:00.287Z 0 000-019 33 33 33 55 55 55 F F F
38512024-04-20T01:07:00.287Z 1 020-039 34 34 34 57 57 57 F F F
38522024-04-20T01:07:00.287Z 2 040-059 34 34 34 56 56 56 F F F
38532024-04-20T01:07:00.287Z 3 060-079 18 18 18 31 31 31 F F F
38542024-04-20T01:07:00.287Z 4 080-099 34 34 34 56 56 56 F F F
38552024-04-20T01:07:00.287Z 5 100-119 35 35 35 58 58 58 F F F
38562024-04-20T01:07:00.287Z 6 120-139 33 33 33 55 55 55 F F F
38572024-04-20T01:07:00.287Z 7 140-159 35 35 35 58 58 58 F F F
38582024-04-20T01:07:00.291Z 8 160-179 35 35 35 58 58 58 F F F
38592024-04-20T01:07:00.291Z 9 180-199 34 34 34 56 56 56 F F F
38602024-04-20T01:07:00.291Z 10 200-219 33 33 33 55 55 55 F F F
38612024-04-20T01:07:00.291Z 11 220-239 30 30 30 50 50 50 F F F
38622024-04-20T01:07:00.291Z 12 240-259 31 31 31 52 52 52 F F F
38632024-04-20T01:07:00.291Z 13 260-279 28 28 28 47 47 47 F F F
38642024-04-20T01:07:00.291Z 14 280-299 34 34 34 56 56 56 F F F
38652024-04-20T01:07:00.291Z 15 300-319 28 28 28 47 47 47 F F F
38662024-04-20T01:07:00.291Z 16 320-339 33 33 33 55 55 55 F F F
38672024-04-20T01:07:00.291Z 17 340-359 33 33 33 55 55 55 F F F
38682024-04-20T01:07:00.291Z 18 360-379 30 30 30 51 51 51 F F F
38692024-04-20T01:07:00.292Z 19 380-399 31 31 31 52 52 52 F F F
38702024-04-20T01:07:00.292Z 20 400-419 35 35 35 58 58 58 F F F
38712024-04-20T01:07:00.292Z 21 420-439 32 32 32 54 54 54 F F F
38722024-04-20T01:07:00.292Z 22 440-459 32 32 32 53 53 53 F F F
38732024-04-20T01:07:00.292Z 23 460-479 34 34 34 56 56 56 F F F
38742024-04-20T01:07:00.292Z 24 480-499 34 34 34 56 56 56 F F F
38752024-04-20T01:07:00.292Z 25 500-519 34 34 34 56 56 56 F F F
38762024-04-20T01:07:00.292Z 26 520-539 34 34 34 56 56 56 F F F
38772024-04-20T01:07:00.292Z 27 540-559 34 34 34 56 56 56 F F F
38782024-04-20T01:07:00.292Z 28 560-579 33 33 33 55 55 55 F F F
38792024-04-20T01:07:00.292Z 29 580-599 29 29 29 48 48 48 F F F
38802024-04-20T01:07:00.292ZMax gen: 35, Max flush: 58
38812024-04-20T01:07:00.292Z
38822024-04-20T01:07:00.292ZBegin loop 18
38832024-04-20T01:07:00.292ZDownstairs to restart: 2
38842024-04-20T01:07:00.309Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818233807Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312}
38852024-04-20T01:07:00.309Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818463267Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312}
38862024-04-20T01:07:00.309Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818480507Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312}
38872024-04-20T01:07:00.309Z{"msg":"Crucible 0d9d9219-b9c6-4552-aed9-10c6d634bad9 has session id: e05a7ab7-fdc1-4139-afdd-e29ae3589822","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818512438Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
38882024-04-20T01:07:00.316Z{"msg":"Upstairs opts: Upstairs UUID: 0d9d9219-b9c6-4552-aed9-10c6d634bad9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818567418Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
38892024-04-20T01:07:00.316ZCrucible runtime is spawned
38902024-04-20T01:07:00.316Z{"msg":"Crucible stats registered with UUID: 0d9d9219-b9c6-4552-aed9-10c6d634bad9","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818612258Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
38912024-04-20T01:07:00.316Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818656968Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312}
38922024-04-20T01:07:00.316Z{"msg":"0d9d9219-b9c6-4552-aed9-10c6d634bad9 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818778938Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
38932024-04-20T01:07:00.316Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818812978Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"0","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
38942024-04-20T01:07:00.316Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818829338Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"0","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
38952024-04-20T01:07:00.317Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818843738Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"1","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
38962024-04-20T01:07:00.317Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818860108Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"1","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
38972024-04-20T01:07:00.317Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818880208Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"2","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
38982024-04-20T01:07:00.317Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818899679Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"2","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
38992024-04-20T01:07:00.317Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818935679Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39002024-04-20T01:07:00.317Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818957889Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39012024-04-20T01:07:00.317Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.818994239Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39022024-04-20T01:07:00.317Z{"msg":"ds_connection connected from Ok(127.0.0.1:54120)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.819159259Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39032024-04-20T01:07:00.317Z{"msg":"ds_connection connected from Ok(127.0.0.1:52076)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.819206299Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39042024-04-20T01:07:00.317Z{"msg":"ds_connection connected from Ok(127.0.0.1:41899)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.819232309Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39052024-04-20T01:07:00.317Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.819959481Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"0","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39062024-04-20T01:07:00.317Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.819999151Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"0","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39072024-04-20T01:07:00.317Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.820016571Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"2","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39082024-04-20T01:07:00.317Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.820034111Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"2","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39092024-04-20T01:07:00.317Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.820075591Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"1","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39102024-04-20T01:07:00.317Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.820124681Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"1","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39112024-04-20T01:07:00.317Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.820557132Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"1","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39122024-04-20T01:07:00.317Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.820648352Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"0","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39132024-04-20T01:07:00.317Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.820690652Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"2","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39142024-04-20T01:07:00.317Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821047453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"0","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39152024-04-20T01:07:00.317Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821084283Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"0","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39162024-04-20T01:07:00.317Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821110424Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39172024-04-20T01:07:00.317Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821143924Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"1","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39182024-04-20T01:07:00.318Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821174824Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"1","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39192024-04-20T01:07:00.318Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821189754Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39202024-04-20T01:07:00.318Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821202134Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"2","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39212024-04-20T01:07:00.318Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821229994Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"2","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39222024-04-20T01:07:00.318Z{"msg":"[0]R flush_numbers[0..12]: [55, 57, 56, 31, 56, 58, 55, 58, 58, 56, 55, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821258304Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39232024-04-20T01:07:00.318Z{"msg":"[0]R generation[0..12]: [33, 34, 34, 18, 34, 35, 33, 35, 35, 34, 33, 30]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821283694Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39242024-04-20T01:07:00.318Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821314664Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39252024-04-20T01:07:00.318Z{"msg":"[1]R flush_numbers[0..12]: [55, 57, 56, 31, 56, 58, 55, 58, 58, 56, 55, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821339494Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39262024-04-20T01:07:00.318Z{"msg":"[1]R generation[0..12]: [33, 34, 34, 18, 34, 35, 33, 35, 35, 34, 33, 30]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821369574Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39272024-04-20T01:07:00.318ZWait for a query_work_queue command to finish before sending IO
39282024-04-20T01:07:00.318Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821406524Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39292024-04-20T01:07:00.318Z{"msg":"[2]R flush_numbers[0..12]: [55, 57, 56, 31, 56, 58, 55, 58, 58, 56, 55, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821436394Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39302024-04-20T01:07:00.318Z{"msg":"[2]R generation[0..12]: [33, 34, 34, 18, 34, 35, 33, 35, 35, 34, 33, 30]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821461444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39312024-04-20T01:07:00.318Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821492104Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39322024-04-20T01:07:00.318Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821515514Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39332024-04-20T01:07:00.318Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821548714Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39342024-04-20T01:07:00.318Z{"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821572434Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39352024-04-20T01:07:00.318Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821598685Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"":"downstairs","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39362024-04-20T01:07:00.318Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821621825Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39372024-04-20T01:07:00.318Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821648185Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"0","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39382024-04-20T01:07:00.318Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821671665Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39392024-04-20T01:07:00.318Zw-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"1","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39402024-04-20T01:07:00.318Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821709865Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"client":"2","":"downstairs","session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39412024-04-20T01:07:00.319Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821742135Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39422024-04-20T01:07:00.319Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821764675Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39432024-04-20T01:07:00.319Z{"msg":"0d9d9219-b9c6-4552-aed9-10c6d634bad9 is now active with session: e05a7ab7-fdc1-4139-afdd-e29ae3589822","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821791495Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39442024-04-20T01:07:00.319Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821812945Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312,"session_id":"e05a7ab7-fdc1-4139-afdd-e29ae3589822"}
39452024-04-20T01:07:00.319Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:06:58.821838965Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312}
39462024-04-20T01:07:00.319ZLoading write count information from file "/tmp/test_repair_verify.data"
39472024-04-20T01:07:00.319ZRun Repair workload
39482024-04-20T01:07:00.323Z01/30 Write block 45 len 36864 data: 2 2 4 4 6 7 8 9 8
39492024-04-20T01:07:00.323Z02/30 Read block 212 len 4096
39502024-04-20T01:07:00.323Z03/30 Flush
39512024-04-20T01:07:00.323Z04/30 Write block 68 len 28672 data: 4 5 4 4 4 3 2
39522024-04-20T01:07:00.323Z05/30 Flush
39532024-04-20T01:07:00.323Z06/30 Write block 226 len 4096 data: 6
39542024-04-20T01:07:00.323Z07/30 Read block 249 len 36864
39552024-04-20T01:07:00.323Z08/30 Write block 388 len 12288 data: 3 2 2
39562024-04-20T01:07:00.323Z09/30 Write block 399 len 8192 data: 3 2
39572024-04-20T01:07:00.323Z10/30 Read block 48 len 36864
39582024-04-20T01:07:00.327Z11/30 Read block 49 len 12288
39592024-04-20T01:07:00.327Z12/30 Write block 582 len 16384 data: 4 3 3 3
39602024-04-20T01:07:00.327Z13/30 Read block 406 len 40960
39612024-04-20T01:07:00.327Z14/30 Write block 244 len 36864 data: 5 5 5 4 7 6 5 4 4
39622024-04-20T01:07:00.330Z15/30 Write block 300 len 24576 data: 2 3 3 3 3 3
39632024-04-20T01:07:00.331Z16/30 Read block 174 len 40960
39642024-04-20T01:07:00.331Z17/30 Write block 185 len 28672 data: 3 3 3 3 2 2 2
39652024-04-20T01:07:00.334Z18/30 Write block 548 len 12288 data: 4 4 4
39662024-04-20T01:07:00.334Z19/30 Read block 124 len 36864
39672024-04-20T01:07:00.334Z20/30 Read block 279 len 4096
39682024-04-20T01:07:00.334Z21/30 Write block 124 len 28672 data: 5 5 5 4 4 5 6
39692024-04-20T01:07:00.334Z22/30 Write block 368 len 16384 data: 4 4 5 6
39702024-04-20T01:07:00.340Z23/30 Write block 555 len 32768 data: 3 3 4 3 3 2 3 4
39712024-04-20T01:07:00.340Z24/30 Flush
39722024-04-20T01:07:00.343Z25/30 Write block 117 len 24576 data: 3 3 4 4 4 4
39732024-04-20T01:07:00.343Z26/30 Read block 330 len 24576
39742024-04-20T01:07:00.343Z27/30 Write block 84 len 28672 data: 3 3 3 3 3 4 5
39752024-04-20T01:07:00.343Z28/30 Read block 511 len 20480
39762024-04-20T01:07:00.348Z29/30 Write block 372 len 16384 data: 6 7 6 5
39772024-04-20T01:07:00.348Z30/30 Read block 27 len 32768
39782024-04-20T01:07:00.348Z----------------------------------------------------------------
39792024-04-20T01:07:00.348Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
39802024-04-20T01:07:00.348ZStates: Active Active Active
39812024-04-20T01:07:00.348ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
39822024-04-20T01:07:00.348Z 1 Acked 1000 Write 9 Done Done Done false
39832024-04-20T01:07:00.348Z 2 Acked 1001 Read 1 Done Done Done false
39842024-04-20T01:07:00.348Z 3 Acked 1002 Flush 0 Done Done Sent false
39852024-04-20T01:07:00.348Z 4 Acked 1003 Write 7 Done Done Sent false
39862024-04-20T01:07:00.348Z 5 Acked 1004 Flush 0 Done Done Sent false
39872024-04-20T01:07:00.348Z 6 Acked 1005 Write 1 Done Done Sent false
39882024-04-20T01:07:00.348Z 7 Acked 1006 Read 9 Done Done Sent false
39892024-04-20T01:07:00.348Z 8 Acked 1007 Write 3 Done Done Sent false
39902024-04-20T01:07:00.348Z 9 Acked 1008 Write 2 Done Done Sent false
39912024-04-20T01:07:00.348Z 10 Acked 1009 Read 9 Done Done Sent false
39922024-04-20T01:07:00.348Z 11 Acked 1010 Read 3 Done Done Sent false
39932024-04-20T01:07:00.348Z 12 Acked 1011 Write 4 Done Done Sent false
39942024-04-20T01:07:00.348Z 13 Acked 1012 Read 10 Done Done Sent false
39952024-04-20T01:07:00.348Z 14 Acked 1013 Write 9 Done Done Sent false
39962024-04-20T01:07:00.348Z 15 Acked 1014 Write 6 Done Done Sent false
39972024-04-20T01:07:00.348Z 16 Acked 1015 Read 10 Done Done Sent false
39982024-04-20T01:07:00.349Z 17 Acked 1016 Write 7 Done Done Sent false
39992024-04-20T01:07:00.349Z 18 Acked 1017 Write 3 Done Done Sent false
40002024-04-20T01:07:00.349Z 19 Acked 1018 Read 9 Done Done Sent false
40012024-04-20T01:07:00.349Z 20 Acked 1019 Read 1 Done Done Sent false
40022024-04-20T01:07:00.349Z 21 Acked 1020 Write 7 Done Done Sent false
40032024-04-20T01:07:00.349Z 22 Acked 1021 Write 4 Done Done Sent false
40042024-04-20T01:07:00.349Z 23 Acked 1022 Write 8 Done Done Sent false
40052024-04-20T01:07:00.349Z 24 Acked 1023 Flush 0 Done Done Sent false
40062024-04-20T01:07:00.349Z 25 Acked 1024 Write 6 Done Done Sent false
40072024-04-20T01:07:00.349Z 26 Acked 1025 Read 6 Done Done Sent false
40082024-04-20T01:07:00.349Z 27 Acked 1026 Write 7 Done Done Sent false
40092024-04-20T01:07:00.349Z 28 Acked 1027 Read 5 Done Done Sent false
40102024-04-20T01:07:00.349Z 29 Acked 1028 Write 4 Done Done Sent false
40112024-04-20T01:07:00.349Z 30 Acked 1029 Read 8 Done Done Sent false
40122024-04-20T01:07:00.349Z STATES DS:0 DS:1 DS:2 TOTAL
40132024-04-20T01:07:00.349Z New 0 0 0 0
40142024-04-20T01:07:00.349Z Sent 0 0 28 28
40152024-04-20T01:07:00.349Z Done 30 30 2 62
40162024-04-20T01:07:00.349Z Skipped 0 0 0 0
40172024-04-20T01:07:00.349Z Error 0 0 0 0
40182024-04-20T01:07:00.349ZLast Flush: 1023 1023 0
40192024-04-20T01:07:00.349ZDownstairs last five completed:
40202024-04-20T01:07:00.350ZUpstairs last five completed: 30 29 28 27 26
40212024-04-20T01:07:00.350Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:06:58.85559368Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1312}
40222024-04-20T01:07:00.350ZWrote out file "/tmp/test_repair_verify.data"
40232024-04-20T01:07:00.353Z
40242024-04-20T01:07:00.354ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40252024-04-20T01:07:00.369ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40262024-04-20T01:07:00.369Z 0 000-019 33 33 33 55 55 55 F F F
40272024-04-20T01:07:00.369Z 1 020-039 34 34 34 57 57 57 F F F
40282024-04-20T01:07:00.369Z 2 040-059 36 36 34 59 59 56 F F T <---
40292024-04-20T01:07:00.369Z 3 060-079 36 36 18 60 60 31 F F F <---
40302024-04-20T01:07:00.369Z 4 080-099 34 34 34 56 56 56 T T F <---
40312024-04-20T01:07:00.369Z 5 100-119 35 35 35 58 58 58 T T F <---
40322024-04-20T01:07:00.369Z 6 120-139 36 36 33 61 61 55 T T F <---
40332024-04-20T01:07:00.369Z 7 140-159 35 35 35 58 58 58 F F F
40342024-04-20T01:07:00.370Z 8 160-179 35 35 35 58 58 58 F F F
40352024-04-20T01:07:00.373Z 9 180-199 36 36 34 61 61 56 F F F <---
40362024-04-20T01:07:00.373Z 10 200-219 33 33 33 55 55 55 F F F
40372024-04-20T01:07:00.373Z 11 220-239 36 36 30 61 61 50 F F F <---
40382024-04-20T01:07:00.373Z 12 240-259 36 36 31 61 61 52 F F F <---
40392024-04-20T01:07:00.373Z 13 260-279 28 28 28 47 47 47 F F F
40402024-04-20T01:07:00.373Z 14 280-299 34 34 34 56 56 56 F F F
40412024-04-20T01:07:00.373Z 15 300-319 36 36 28 61 61 47 F F F <---
40422024-04-20T01:07:00.373Z 16 320-339 33 33 33 55 55 55 F F F
40432024-04-20T01:07:00.373Z 17 340-359 33 33 33 55 55 55 F F F
40442024-04-20T01:07:00.373Z 18 360-379 36 36 30 61 61 51 T T F <---
40452024-04-20T01:07:00.373Z 19 380-399 36 36 31 61 61 52 F F F <---
40462024-04-20T01:07:00.373Z 20 400-419 36 36 35 61 61 58 F F F <---
40472024-04-20T01:07:00.374Z 21 420-439 32 32 32 54 54 54 F F F
40482024-04-20T01:07:00.374Z 22 440-459 32 32 32 53 53 53 F F F
40492024-04-20T01:07:00.374Z 23 460-479 34 34 34 56 56 56 F F F
40502024-04-20T01:07:00.374Z 24 480-499 34 34 34 56 56 56 F F F
40512024-04-20T01:07:00.374Z 25 500-519 34 34 34 56 56 56 F F F
40522024-04-20T01:07:00.374Z 26 520-539 34 34 34 56 56 56 F F F
40532024-04-20T01:07:00.374Z 27 540-559 36 36 34 61 61 56 F F F <---
40542024-04-20T01:07:00.374Z 28 560-579 36 36 33 61 61 55 F F F <---
40552024-04-20T01:07:00.374Z 29 580-599 36 36 29 61 61 48 F F F <---
40562024-04-20T01:07:00.374ZMax gen: 36, Max flush: 61
40572024-04-20T01:07:00.374ZError: Difference in extent metadata found!
40582024-04-20T01:07:00.374ZA Difference in extent metadata is expected here
40592024-04-20T01:07:00.374ZOn loop 18
40602024-04-20T01:07:00.374Z
40612024-04-20T01:07:00.378ZVerifying data now
40622024-04-20T01:07:15.083ZNo change after verify
40632024-04-20T01:07:15.084ZLoop: 18 Downstairs dump after verify (and repair):
40642024-04-20T01:07:15.100ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40652024-04-20T01:07:15.101Z 0 000-019 33 33 33 55 55 55 F F F
40662024-04-20T01:07:15.101Z 1 020-039 34 34 34 57 57 57 F F F
40672024-04-20T01:07:15.101Z 2 040-059 36 36 36 59 59 59 F F F
40682024-04-20T01:07:15.101Z 3 060-079 36 36 36 60 60 60 F F F
40692024-04-20T01:07:15.101Z 4 080-099 37 37 37 62 62 62 F F F
40702024-04-20T01:07:15.101Z 5 100-119 37 37 37 62 62 62 F F F
40712024-04-20T01:07:15.104Z 6 120-139 37 37 37 62 62 62 F F F
40722024-04-20T01:07:15.105Z 7 140-159 35 35 35 58 58 58 F F F
40732024-04-20T01:07:15.105Z 8 160-179 35 35 35 58 58 58 F F F
40742024-04-20T01:07:15.105Z 9 180-199 36 36 36 61 61 61 F F F
40752024-04-20T01:07:15.105Z 10 200-219 33 33 33 55 55 55 F F F
40762024-04-20T01:07:15.105Z 11 220-239 36 36 36 61 61 61 F F F
40772024-04-20T01:07:15.105Z 12 240-259 36 36 36 61 61 61 F F F
40782024-04-20T01:07:15.105Z 13 260-279 28 28 28 47 47 47 F F F
40792024-04-20T01:07:15.105Z 14 280-299 34 34 34 56 56 56 F F F
40802024-04-20T01:07:15.105Z 15 300-319 36 36 36 61 61 61 F F F
40812024-04-20T01:07:15.105Z 16 320-339 33 33 33 55 55 55 F F F
40822024-04-20T01:07:15.105Z 17 340-359 33 33 33 55 55 55 F F F
40832024-04-20T01:07:15.105Z 18 360-379 37 37 37 62 62 62 F F F
40842024-04-20T01:07:15.105Z 19 380-399 36 36 36 61 61 61 F F F
40852024-04-20T01:07:15.105Z 20 400-419 36 36 36 61 61 61 F F F
40862024-04-20T01:07:15.105Z 21 420-439 32 32 32 54 54 54 F F F
40872024-04-20T01:07:15.105Z 22 440-459 32 32 32 53 53 53 F F F
40882024-04-20T01:07:15.105Z 23 460-479 34 34 34 56 56 56 F F F
40892024-04-20T01:07:15.105Z 24 480-499 34 34 34 56 56 56 F F F
40902024-04-20T01:07:15.105Z 25 500-519 34 34 34 56 56 56 F F F
40912024-04-20T01:07:15.105Z 26 520-539 34 34 34 56 56 56 F F F
40922024-04-20T01:07:15.105Z 27 540-559 36 36 36 61 61 61 F F F
40932024-04-20T01:07:15.105Z 28 560-579 36 36 36 61 61 61 F F F
40942024-04-20T01:07:15.105Z 29 580-599 36 36 36 61 61 61 F F F
40952024-04-20T01:07:15.106ZMax gen: 37, Max flush: 62
40962024-04-20T01:07:15.106Z
40972024-04-20T01:07:15.106ZBegin loop 19
40982024-04-20T01:07:15.106ZDownstairs to restart: 2
40992024-04-20T01:07:15.120Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.629980628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320}
41002024-04-20T01:07:15.121Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.630247438Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320}
41012024-04-20T01:07:15.121Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.630264298Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320}
41022024-04-20T01:07:15.121Z{"msg":"Crucible 8af41ee6-de13-4c52-a052-b7ac49e22eaa has session id: ad0a0a5a-62e7-4288-8ff2-b1929f580ad0","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.630278309Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41032024-04-20T01:07:15.128Z{"msg":"Upstairs opts: Upstairs UUID: 8af41ee6-de13-4c52-a052-b7ac49e22eaa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.630325829Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41042024-04-20T01:07:15.128Z{"msg":"Crucible stats registered with UUID: 8af41ee6-de13-4c52-a052-b7ac49e22eaa","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.630364559Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41052024-04-20T01:07:15.128ZCrucible runtime is spawned
41062024-04-20T01:07:15.128Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.630502289Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320}
41072024-04-20T01:07:15.128Z{"msg":"8af41ee6-de13-4c52-a052-b7ac49e22eaa active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.630633139Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41082024-04-20T01:07:15.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.63066167Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"0","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41092024-04-20T01:07:15.128Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.630696619Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"0","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41102024-04-20T01:07:15.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.630723639Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"1","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41112024-04-20T01:07:15.128Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.63076263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"1","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41122024-04-20T01:07:15.129Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.63078925Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"2","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41132024-04-20T01:07:15.129Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.63081334Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"2","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41142024-04-20T01:07:15.129Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.63083528Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41152024-04-20T01:07:15.129Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.63085878Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41162024-04-20T01:07:15.130Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.63088967Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41172024-04-20T01:07:15.130Z{"msg":"ds_connection connected from Ok(127.0.0.1:46051)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.63105174Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41182024-04-20T01:07:15.130Z{"msg":"ds_connection connected from Ok(127.0.0.1:49512)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.63109808Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41192024-04-20T01:07:15.131Z{"msg":"ds_connection connected from Ok(127.0.0.1:60015)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.63112562Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41202024-04-20T01:07:15.131Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.631878452Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"0","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41212024-04-20T01:07:15.131Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.631919412Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"0","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41222024-04-20T01:07:15.131Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.631958162Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"1","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41232024-04-20T01:07:15.131Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.632013493Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"1","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41242024-04-20T01:07:15.131Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.632046382Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"2","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41252024-04-20T01:07:15.131Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.632088063Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"2","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41262024-04-20T01:07:15.132Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.632404083Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"1","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41272024-04-20T01:07:15.132Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.632592524Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"0","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41282024-04-20T01:07:15.132Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.632747574Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"2","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41292024-04-20T01:07:15.132Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.632904404Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"1","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41302024-04-20T01:07:15.132Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.632949724Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"1","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41312024-04-20T01:07:15.132Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.632984845Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41322024-04-20T01:07:15.132Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633011635Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"0","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41332024-04-20T01:07:15.132Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633035645Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"0","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41342024-04-20T01:07:15.132Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633068855Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41352024-04-20T01:07:15.132Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633254315Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"2","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41362024-04-20T01:07:15.133Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633299825Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"2","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41372024-04-20T01:07:15.133Z{"msg":"[0]R flush_numbers[0..12]: [55, 57, 59, 60, 62, 62, 62, 58, 58, 61, 55, 61]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633355716Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41382024-04-20T01:07:15.133Z{"msg":"[0]R generation[0..12]: [33, 34, 36, 36, 37, 37, 37, 35, 35, 36, 33, 36]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633380336Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41392024-04-20T01:07:15.133Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633395365Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41402024-04-20T01:07:15.133Z{"msg":"[1]R flush_numbers[0..12]: [55, 57, 59, 60, 62, 62, 62, 58, 58, 61, 55, 61]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
41412024-04-20T01:07:15.133Z,"time":"2024-04-20T01:07:13.633417825Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41422024-04-20T01:07:15.133Z{"msg":"[1]R generation[0..12]: [33, 34, 36, 36, 37, 37, 37, 35, 35, 36, 33, 36]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633433156Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41432024-04-20T01:07:15.133Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633448846Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41442024-04-20T01:07:15.133Z{"msg":"[2]R flush_numbers[0..12]: [55, 57, 59, 60, 62, 62, 62, 58, 58, 61, 55, 61]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633472196Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41452024-04-20T01:07:15.133Z{"msg":"[2]R generation[0..12]: [33, 34, 36, 36, 37, 37, 37, 35, 35, 36, 33, 36]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633486466Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41462024-04-20T01:07:15.133Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633507906Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41472024-04-20T01:07:15.133Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633522426Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41482024-04-20T01:07:15.133Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633542586Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41492024-04-20T01:07:15.133Z{"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633555906Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41502024-04-20T01:07:15.134Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633569216Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"":"downstairs","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41512024-04-20T01:07:15.134Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633589566Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41522024-04-20T01:07:15.134Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633601866Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"0","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41532024-04-20T01:07:15.134Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633621896Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"1","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41542024-04-20T01:07:15.134Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633635476Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"client":"2","":"downstairs","session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41552024-04-20T01:07:15.134Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633655736Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41562024-04-20T01:07:15.134Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633668586Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41572024-04-20T01:07:15.134Z{"msg":"8af41ee6-de13-4c52-a052-b7ac49e22eaa is now active with session: ad0a0a5a-62e7-4288-8ff2-b1929f580ad0","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633687976Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41582024-04-20T01:07:15.134Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633700376Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320,"session_id":"ad0a0a5a-62e7-4288-8ff2-b1929f580ad0"}
41592024-04-20T01:07:15.134ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41602024-04-20T01:07:15.134Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:13.633725016Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320}
41612024-04-20T01:07:15.145ZLoading write count information from file "/tmp/test_repair_verify.data"
41622024-04-20T01:07:15.145ZRun Repair workload
41632024-04-20T01:07:15.145Z01/30 Write block 482 len 28672 data: 6 5 5 4 4 4 4
41642024-04-20T01:07:15.145Z02/30 Read block 366 len 24576
41652024-04-20T01:07:15.145Z03/30 Write block 328 len 28672 data: 5 5 5 4 5 4 4
41662024-04-20T01:07:15.145Z04/30 Read block 421 len 24576
41672024-04-20T01:07:15.146Z05/30 Write block 451 len 36864 data: 3 4 3 4 6 6 7 7 7
41682024-04-20T01:07:15.146Z06/30 Write block 448 len 40960 data: 4 5 5 4 5 4 5 7 7 8
41692024-04-20T01:07:15.146Z07/30 Read block 443 len 4096
41702024-04-20T01:07:15.146Z08/30 Read block 276 len 16384
41712024-04-20T01:07:15.146Z09/30 Read block 480 len 8192
41722024-04-20T01:07:15.146Z10/30 Read block 312 len 36864
41732024-04-20T01:07:15.146Z11/30 Read block 78 len 36864
41742024-04-20T01:07:15.146Z12/30 Read block 371 len 32768
41752024-04-20T01:07:15.146Z13/30 Write block 315 len 4096 data: 3
41762024-04-20T01:07:15.146Z14/30 Write block 521 len 28672 data: 7 6 6 5 5 4 3
41772024-04-20T01:07:15.146Z15/30 Read block 387 len 20480
41782024-04-20T01:07:15.151Z16/30 Read block 236 len 36864
41792024-04-20T01:07:15.151Z17/30 Write block 217 len 4096 data: 6
41802024-04-20T01:07:15.152Z18/30 Write block 355 len 40960 data: 3 3 3 3 3 3 4 5 5 4
41812024-04-20T01:07:15.152Z19/30 Write block 125 len 36864 data: 6 6 5 5 6 7 7 7 7
41822024-04-20T01:07:15.152Z20/30 Flush
41832024-04-20T01:07:15.152Z21/30 Write block 461 len 4096 data: 8
41842024-04-20T01:07:15.152Z22/30 Read block 349 len 24576
41852024-04-20T01:07:15.152Z23/30 Read block 479 len 16384
41862024-04-20T01:07:15.152Z24/30 Read block 68 len 24576
41872024-04-20T01:07:15.152Z25/30 Flush
41882024-04-20T01:07:15.158Z26/30 Write block 50 len 20480 data: 8 9 10 9 6
41892024-04-20T01:07:15.158Z27/30 Write block 39 len 12288 data: 6 5 4
41902024-04-20T01:07:15.158Z28/30 Read block 444 len 32768
41912024-04-20T01:07:15.158Z29/30 Read block 484 len 24576
41922024-04-20T01:07:15.158Z30/30 Write block 142 len 28672 data: 4 3 3 3 3 3 4
41932024-04-20T01:07:15.158Z----------------------------------------------------------------
41942024-04-20T01:07:15.158Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
41952024-04-20T01:07:15.158ZStates: Active Active Active
41962024-04-20T01:07:15.158ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
41972024-04-20T01:07:15.158Z 1 Acked 1000 Write 7 Done Done Sent false
41982024-04-20T01:07:15.158Z 2 Acked 1001 Read 6 Done Done Sent false
41992024-04-20T01:07:15.158Z 3 Acked 1002 Write 7 Done Done Sent false
42002024-04-20T01:07:15.158Z 4 Acked 1003 Read 6 Done Done Sent false
42012024-04-20T01:07:15.158Z 5 Acked 1004 Write 9 Done Done Sent false
42022024-04-20T01:07:15.158Z 6 Acked 1005 Write 10 Done Done Sent false
42032024-04-20T01:07:15.158Z 7 Acked 1006 Read 1 Done Done Sent false
42042024-04-20T01:07:15.158Z 8 Acked 1007 Read 4 Done Done Sent false
42052024-04-20T01:07:15.158Z 9 Acked 1008 Read 2 Done Done Sent false
42062024-04-20T01:07:15.158Z 10 Acked 1009 Read 9 Done Done Sent false
42072024-04-20T01:07:15.159Z 11 Acked 1010 Read 9 Done Done Sent false
42082024-04-20T01:07:15.159Z 12 Acked 1011 Read 8 Done Done Sent false
42092024-04-20T01:07:15.159Z 13 Acked 1012 Write 1 Done Done Sent false
42102024-04-20T01:07:15.159Z 14 Acked 1013 Write 7 Done Done Sent false
42112024-04-20T01:07:15.159Z 15 Acked 1014 Read 5 Done Done Sent false
42122024-04-20T01:07:15.159Z 16 Acked 1015 Read 9 Done Done Sent false
42132024-04-20T01:07:15.159Z 17 Acked 1016 Write 1 Done Done Sent false
42142024-04-20T01:07:15.159Z 18 Acked 1017 Write 10 Done Done Sent false
42152024-04-20T01:07:15.159Z 19 Acked 1018 Write 9 Done Done Sent false
42162024-04-20T01:07:15.159Z 20 Acked 1019 Flush 0 Done Done Sent false
42172024-04-20T01:07:15.159Z 21 Acked 1020 Write 1 Done Done Sent false
42182024-04-20T01:07:15.159Z 22 Acked 1021 Read 6 Done Done Sent false
42192024-04-20T01:07:15.159Z 23 Acked 1022 Read 4 Done Done Sent false
42202024-04-20T01:07:15.159Z 24 Acked 1023 Read 6 Done Done Sent false
42212024-04-20T01:07:15.159Z 25 Acked 1024 Flush 0 Done Done Sent false
42222024-04-20T01:07:15.159Z 26 Acked 1025 Write 5 Done Done Sent false
42232024-04-20T01:07:15.159Z 27 Acked 1026 Write 3 Done Done Sent false
42242024-04-20T01:07:15.159Z 28 Acked 1027 Read 8 Done Done Sent false
42252024-04-20T01:07:15.159Z 29 Acked 1028 Read 6 Done Done Sent false
42262024-04-20T01:07:15.159Z 30 Acked 1029 Write 7 Sent Sent Sent false
42272024-04-20T01:07:15.159Z STATES DS:0 DS:1 DS:2 TOTAL
42282024-04-20T01:07:15.159Z New 0 0 0 0
42292024-04-20T01:07:15.159Z Sent 1 1 30 32
42302024-04-20T01:07:15.159Z Done 29 29 0 58
42312024-04-20T01:07:15.160Z Skipped 0 0 0 0
42322024-04-20T01:07:15.160Z Error 0 0 0 0
42332024-04-20T01:07:15.160ZLast Flush: 1024 1024 0
42342024-04-20T01:07:15.160ZDownstairs last five completed:
42352024-04-20T01:07:15.160ZUpstairs last five completed: 30 29 28 27 26
42362024-04-20T01:07:15.160Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:07:13.665886048Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1320}
42372024-04-20T01:07:15.160ZWrote out file "/tmp/test_repair_verify.data"
42382024-04-20T01:07:15.164Z
42392024-04-20T01:07:15.164ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42402024-04-20T01:07:15.179ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42412024-04-20T01:07:15.179Z 0 000-019 33 33 33 55 55 55 F F F
42422024-04-20T01:07:15.179Z 1 020-039 34 34 34 57 57 57 T T F <---
42432024-04-20T01:07:15.179Z 2 040-059 36 36 36 59 59 59 T T F <---
42442024-04-20T01:07:15.179Z 3 060-079 36 36 36 60 60 60 F F F
42452024-04-20T01:07:15.179Z 4 080-099 37 37 37 62 62 62 F F F
42462024-04-20T01:07:15.179Z 5 100-119 37 37 37 62 62 62 F F F
42472024-04-20T01:07:15.179Z 6 120-139 38 38 37 63 63 62 F F F <---
42482024-04-20T01:07:15.179Z 7 140-159 35 35 35 58 58 58 T T F <---
42492024-04-20T01:07:15.179Z 8 160-179 35 35 35 58 58 58 F F F
42502024-04-20T01:07:15.179Z 9 180-199 36 36 36 61 61 61 F F F
42512024-04-20T01:07:15.179Z 10 200-219 38 38 33 63 63 55 F F F <---
42522024-04-20T01:07:15.180Z 11 220-239 36 36 36 61 61 61 F F F
42532024-04-20T01:07:15.184Z 12 240-259 36 36 36 61 61 61 F F F
42542024-04-20T01:07:15.184Z 13 260-279 28 28 28 47 47 47 F F F
42552024-04-20T01:07:15.184Z 14 280-299 34 34 34 56 56 56 F F F
42562024-04-20T01:07:15.184Z 15 300-319 38 38 36 63 63 61 F F F <---
42572024-04-20T01:07:15.184Z 16 320-339 38 38 33 63 63 55 F F F <---
42582024-04-20T01:07:15.184Z 17 340-359 38 38 33 63 63 55 F F F <---
42592024-04-20T01:07:15.184Z 18 360-379 38 38 37 63 63 62 F F F <---
42602024-04-20T01:07:15.184Z 19 380-399 36 36 36 61 61 61 F F F
42612024-04-20T01:07:15.184Z 20 400-419 36 36 36 61 61 61 F F F
42622024-04-20T01:07:15.184Z 21 420-439 32 32 32 54 54 54 F F F
42632024-04-20T01:07:15.184Z 22 440-459 38 38 32 63 63 53 F F F <---
42642024-04-20T01:07:15.184Z 23 460-479 38 38 34 64 64 56 F F F <---
42652024-04-20T01:07:15.184Z 24 480-499 38 38 34 63 63 56 F F F <---
42662024-04-20T01:07:15.184Z 25 500-519 34 34 34 56 56 56 F F F
42672024-04-20T01:07:15.184Z 26 520-539 38 38 34 63 63 56 F F F <---
42682024-04-20T01:07:15.184Z 27 540-559 36 36 36 61 61 61 F F F
42692024-04-20T01:07:15.184Z 28 560-579 36 36 36 61 61 61 F F F
42702024-04-20T01:07:15.184Z 29 580-599 36 36 36 61 61 61 F F F
42712024-04-20T01:07:15.184ZMax gen: 38, Max flush: 64
42722024-04-20T01:07:15.184ZError: Difference in extent metadata found!
42732024-04-20T01:07:15.185ZA Difference in extent metadata is expected here
42742024-04-20T01:07:15.185ZOn loop 19
42752024-04-20T01:07:15.185Z
42762024-04-20T01:07:15.188ZVerifying data now
42772024-04-20T01:07:19.763ZNo change after verify
42782024-04-20T01:07:19.763ZLoop: 19 Downstairs dump after verify (and repair):
42792024-04-20T01:07:19.777ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42802024-04-20T01:07:19.778Z 0 000-019 33 33 33 55 55 55 F F F
42812024-04-20T01:07:19.782Z 1 020-039 39 39 39 65 65 65 F F F
42822024-04-20T01:07:19.782Z 2 040-059 39 39 39 65 65 65 F F F
42832024-04-20T01:07:19.782Z 3 060-079 36 36 36 60 60 60 F F F
42842024-04-20T01:07:19.782Z 4 080-099 37 37 37 62 62 62 F F F
42852024-04-20T01:07:19.782Z 5 100-119 37 37 37 62 62 62 F F F
42862024-04-20T01:07:19.782Z 6 120-139 38 38 38 63 63 63 F F F
42872024-04-20T01:07:19.782Z 7 140-159 39 39 39 65 65 65 F F F
42882024-04-20T01:07:19.782Z 8 160-179 35 35 35 58 58 58 F F F
42892024-04-20T01:07:19.782Z 9 180-199 36 36 36 61 61 61 F F F
42902024-04-20T01:07:19.782Z 10 200-219 38 38 38 63 63 63 F F F
42912024-04-20T01:07:19.782Z 11 220-239 36 36 36 61 61 61 F F F
42922024-04-20T01:07:19.782Z 12 240-259 36 36 36 61 61 61 F F F
42932024-04-20T01:07:19.783Z 13 260-279 28 28 28 47 47 47 F F F
42942024-04-20T01:07:19.783Z 14 280-299 34 34 34 56 56 56 F F F
42952024-04-20T01:07:19.783Z 15 300-319 38 38 38 63 63 63 F F F
42962024-04-20T01:07:19.783Z 16 320-339 38 38 38 63 63 63 F F F
42972024-04-20T01:07:19.783Z 17 340-359 38 38 38 63 63 63 F F F
42982024-04-20T01:07:19.783Z 18 360-379 38 38 38 63 63 63 F F F
42992024-04-20T01:07:19.783Z 19 380-399 36 36 36 61 61 61 F F F
43002024-04-20T01:07:19.783Z 20 400-419 36 36 36 61 61 61 F F F
43012024-04-20T01:07:19.783Z 21 420-439 32 32 32 54 54 54 F F F
43022024-04-20T01:07:19.783Z 22 440-459 38 38 38 63 63 63 F F F
43032024-04-20T01:07:19.783Z 23 460-479 38 38 38 64 64 64 F F F
43042024-04-20T01:07:19.783Z 24 480-499 38 38 38 63 63 63 F F F
43052024-04-20T01:07:19.783Z 25 500-519 34 34 34 56 56 56 F F F
43062024-04-20T01:07:19.783Z 26 520-539 38 38 38 63 63 63 F F F
43072024-04-20T01:07:19.783Z 27 540-559 36 36 36 61 61 61 F F F
43082024-04-20T01:07:19.783Z 28 560-579 36 36 36 61 61 61 F F F
43092024-04-20T01:07:19.783Z 29 580-599 36 36 36 61 61 61 F F F
43102024-04-20T01:07:19.783ZMax gen: 39, Max flush: 65
43112024-04-20T01:07:19.783Z
43122024-04-20T01:07:19.783ZBegin loop 20
43132024-04-20T01:07:19.783ZDownstairs to restart: 0
43142024-04-20T01:07:19.800Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309020353Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328}
43152024-04-20T01:07:19.800Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309252924Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328}
43162024-04-20T01:07:19.800Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309278984Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328}
43172024-04-20T01:07:19.808Z{"msg":"Crucible 0233560f-173b-4a39-89bc-0b3079a6f751 has session id: b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309304114Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43182024-04-20T01:07:19.808Z{"msg":"Crucible runtime is spawned
43192024-04-20T01:07:19.808ZUpstairs opts: Upstairs UUID: 0233560f-173b-4a39-89bc-0b3079a6f751, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309432334Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43202024-04-20T01:07:19.808Z{"msg":"Crucible stats registered with UUID: 0233560f-173b-4a39-89bc-0b3079a6f751","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309482574Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43212024-04-20T01:07:19.808Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309506525Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328}
43222024-04-20T01:07:19.808Z{"msg":"0233560f-173b-4a39-89bc-0b3079a6f751 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309548325Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43232024-04-20T01:07:19.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309569185Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"0","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43242024-04-20T01:07:19.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309601485Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"0","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43252024-04-20T01:07:19.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309635945Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"1","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43262024-04-20T01:07:19.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309659695Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"1","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43272024-04-20T01:07:19.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309684565Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"2","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43282024-04-20T01:07:19.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309770225Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"2","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43292024-04-20T01:07:19.809Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309803505Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43302024-04-20T01:07:19.809Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309846405Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43312024-04-20T01:07:19.809Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309898815Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43322024-04-20T01:07:19.809Z{"msg":"ds_connection connected from Ok(127.0.0.1:40081)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309938455Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43332024-04-20T01:07:19.809Z{"msg":"ds_connection connected from Ok(127.0.0.1:55199)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.309967306Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43342024-04-20T01:07:19.809Z{"msg":"ds_connection connected from Ok(127.0.0.1:58952)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.310004186Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43352024-04-20T01:07:19.809Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.310664287Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"1","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43362024-04-20T01:07:19.809Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.310701497Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"1","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43372024-04-20T01:07:19.809Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.310718877Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"2","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43382024-04-20T01:07:19.809Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.310736127Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"2","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43392024-04-20T01:07:19.809Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.311155858Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"0","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43402024-04-20T01:07:19.809Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.311195238Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"0","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43412024-04-20T01:07:19.809Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.311218058Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"2","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43422024-04-20T01:07:19.809Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.311372519Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"1","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43432024-04-20T01:07:19.809Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.311709819Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"2","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43442024-04-20T01:07:19.809Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.311742699Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"2","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43452024-04-20T01:07:19.809Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.311759299Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43462024-04-20T01:07:19.810Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.31182132Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"0","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43472024-04-20T01:07:19.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.31194629Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"1","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43482024-04-20T01:07:19.810Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.31198449Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"1","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43492024-04-20T01:07:19.810Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.31201486Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43502024-04-20T01:07:19.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.31222086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"0","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43512024-04-20T01:07:19.810Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312257511Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"0","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43522024-04-20T01:07:19.810Z{"msg":"[0]R flush_numbers[0..12]: [55, 65, 65, 60, 62, 62, 63, 65, 58, 61, 63, 61]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312283411Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43532024-04-20T01:07:19.810Z{"msg":"[0]R generation[0..12]: [33, 39, 39, 36, 37, 37, 38, 39, 35, 36, 38, 36]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312314141Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43542024-04-20T01:07:19.810Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312340431Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43552024-04-20T01:07:19.810Z{"msg":"[1]R flush_numbers[0..12]: [55, 65, 65, 60, 62, 62, 63, 65, 58, 61, 63, 61]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
43562024-04-20T01:07:19.810Z,"time":"2024-04-20T01:07:18.312368721Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43572024-04-20T01:07:19.810Z{"msg":"[1]R generation[0..12]: [33, 39, 39, 36, 37, 37, 38, 39, 35, 36, 38, 36]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312393351Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43582024-04-20T01:07:19.810Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312430011Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43592024-04-20T01:07:19.810Z{"msg":"[2]R flush_numbers[0..12]: [55, 65, 65, 60, 62, 62, 63, 65, 58, 61, 63, 61]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312452851Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43602024-04-20T01:07:19.810Z{"msg":"[2]R generation[0..12]: [33, 39, 39, 36, 37, 37, 38, 39, 35, 36, 38, 36]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312470741Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43612024-04-20T01:07:19.810Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312486591Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43622024-04-20T01:07:19.810Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312507711Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43632024-04-20T01:07:19.810Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312521761Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43642024-04-20T01:07:19.810Z{"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312542101Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43652024-04-20T01:07:19.810Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312555931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"":"downstairs","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43662024-04-20T01:07:19.810Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312577081Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43672024-04-20T01:07:19.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312589581Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"0","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43682024-04-20T01:07:19.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312610212Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"1","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43692024-04-20T01:07:19.811Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312623692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"client":"2","":"downstairs","session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43702024-04-20T01:07:19.811Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312637012Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43712024-04-20T01:07:19.811Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312656181Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43722024-04-20T01:07:19.811Z{"msg":"0233560f-173b-4a39-89bc-0b3079a6f751 is now active with session: b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
43732024-04-20T01:07:19.811Z,"time":"2024-04-20T01:07:18.312670032Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43742024-04-20T01:07:19.811Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312685362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328,"session_id":"b2134fb3-6ce2-4ad8-aea4-caa464ffb7b8"}
43752024-04-20T01:07:19.811Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:18.312705022Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328}
43762024-04-20T01:07:19.811ZLoading write count information from file "/tmp/test_repair_verify.data"
43772024-04-20T01:07:19.811ZRun Repair workload
43782024-04-20T01:07:19.815Z01/30 Write block 40 len 12288 data: 6 5 3
43792024-04-20T01:07:19.815Z02/30 Read block 337 len 8192
43802024-04-20T01:07:19.815Z03/30 Read block 495 len 16384
43812024-04-20T01:07:19.815Z04/30 Write block 150 len 28672 data: 4 4 3 4 4 6 4
43822024-04-20T01:07:19.815Z05/30 Write block 347 len 24576 data: 5 5 4 3 3 3
43832024-04-20T01:07:19.815Z06/30 Write block 580 len 12288 data: 4 4 5
43842024-04-20T01:07:19.815Z07/30 Read block 288 len 12288
43852024-04-20T01:07:19.815Z08/30 Flush
43862024-04-20T01:07:19.815Z09/30 Write block 218 len 4096 data: 6
43872024-04-20T01:07:19.815Z10/30 Flush
43882024-04-20T01:07:19.819Z11/30 Write block 128 len 4096 data: 6
43892024-04-20T01:07:19.819Z12/30 Write block 62 len 24576 data: 4 3 4 4 5 4
43902024-04-20T01:07:19.819Z13/30 Write block 103 len 4096 data: 4
43912024-04-20T01:07:19.819Z14/30 Flush
43922024-04-20T01:07:19.823Z15/30 Write block 73 len 20480 data: 4 3 2 3 4
43932024-04-20T01:07:19.823Z16/30 Read block 515 len 12288
43942024-04-20T01:07:19.823Z17/30 Read block 23 len 4096
43952024-04-20T01:07:19.823Z18/30 Read block 306 len 24576
43962024-04-20T01:07:19.826Z19/30 Write block 379 len 16384 data: 4 5 5 4
43972024-04-20T01:07:19.826Z20/30 Read block 276 len 24576
43982024-04-20T01:07:19.827Z21/30 Read block 368 len 16384
43992024-04-20T01:07:19.827Z22/30 Write block 260 len 32768 data: 2 2 2 2 2 2 2 2
44002024-04-20T01:07:19.831Z23/30 Write block 143 len 16384 data: 4 4 4 4
44012024-04-20T01:07:19.832Z24/30 Write block 178 len 24576 data: 5 7 7 7 5 4
44022024-04-20T01:07:19.832Z25/30 Write block 389 len 36864 data: 3 3 3 3 4 4 4 4 3
44032024-04-20T01:07:19.832Z26/30 Read block 562 len 8192
44042024-04-20T01:07:19.832Z27/30 Read block 371 len 32768
44052024-04-20T01:07:19.839Z28/30 Write block 579 len 12288 data: 5 5 5
44062024-04-20T01:07:19.840Z29/30 Read block 235 len 28672
44072024-04-20T01:07:19.840Z30/30 Read block 30 len 36864
44082024-04-20T01:07:19.840Z----------------------------------------------------------------
44092024-04-20T01:07:19.840Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
44102024-04-20T01:07:19.840ZStates: Active Active Active
44112024-04-20T01:07:19.840ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44122024-04-20T01:07:19.840Z 1 Acked 1000 Write 3 Done Done Done false
44132024-04-20T01:07:19.841Z 2 Acked 1001 Read 2 Done Done Done false
44142024-04-20T01:07:19.841Z 3 Acked 1002 Read 4 Done Done Done false
44152024-04-20T01:07:19.841Z 4 Acked 1003 Write 7 Done Done Done false
44162024-04-20T01:07:19.841Z 5 Acked 1004 Write 6 Done Done Done false
44172024-04-20T01:07:19.841Z 6 Acked 1005 Write 3 Done Done Done false
44182024-04-20T01:07:19.841Z 7 Acked 1006 Read 3 Done Done Done false
44192024-04-20T01:07:19.841Z 8 Acked 1007 Flush 0 Sent Done Done false
44202024-04-20T01:07:19.841Z 9 Acked 1008 Write 1 Sent Done Done false
44212024-04-20T01:07:19.841Z 10 Acked 1009 Flush 0 Sent Done Done false
44222024-04-20T01:07:19.841Z 11 Acked 1010 Write 1 Sent Done Done false
44232024-04-20T01:07:19.842Z 12 Acked 1011 Write 6 Sent Done Done false
44242024-04-20T01:07:19.842Z 13 Acked 1012 Write 1 Sent Done Done false
44252024-04-20T01:07:19.842Z 14 Acked 1013 Flush 0 Sent Done Done false
44262024-04-20T01:07:19.842Z 15 Acked 1014 Write 5 Sent Done Done false
44272024-04-20T01:07:19.842Z 16 Acked 1015 Read 3 Sent Done Done false
44282024-04-20T01:07:19.842Z 17 Acked 1016 Read 1 Sent Done Done false
44292024-04-20T01:07:19.842Z 18 Acked 1017 Read 6 Sent Done Done false
44302024-04-20T01:07:19.842Z 19 Acked 1018 Write 4 Sent Done Done false
44312024-04-20T01:07:19.843Z 20 Acked 1019 Read 6 Sent Done Done false
44322024-04-20T01:07:19.843Z 21 Acked 1020 Read 4 Sent Done Done false
44332024-04-20T01:07:19.843Z 22 Acked 1021 Write 8 Sent Done Done false
44342024-04-20T01:07:19.843Z 23 Acked 1022 Write 4 Sent Done Done false
44352024-04-20T01:07:19.843Z 24 Acked 1023 Write 6 Sent Done Done false
44362024-04-20T01:07:19.843Z 25 Acked 1024 Write 9 Sent Done Done false
44372024-04-20T01:07:19.843Z 26 Acked 1025 Read 2 Sent Done Done false
44382024-04-20T01:07:19.843Z 27 Acked 1026 Read 8 Sent Done Done false
44392024-04-20T01:07:19.844Z 28 Acked 1027 Write 3 Sent Done Done false
44402024-04-20T01:07:19.844Z 29 Acked 1028 Read 7 Sent Done Done false
44412024-04-20T01:07:19.844Z 30 Acked 1029 Read 9 Sent Sent Done false
44422024-04-20T01:07:19.844Z STATES DS:0 DS:1 DS:2 TOTAL
44432024-04-20T01:07:19.844Z New 0 0 0 0
44442024-04-20T01:07:19.844Z Sent 23 1 0 24
44452024-04-20T01:07:19.845Z Done 7 29 30 66
44462024-04-20T01:07:19.845Z Skipped 0 0 0 0
44472024-04-20T01:07:19.845Z Error 0 0 0 0
44482024-04-20T01:07:19.845ZLast Flush: 0 1013 1013
44492024-04-20T01:07:19.845ZDownstairs last five completed:
44502024-04-20T01:07:19.845ZUpstairs last five completed: 30 29 28 27 26
44512024-04-20T01:07:19.845Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:07:18.342832379Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1328}
44522024-04-20T01:07:19.845ZWrote out file "/tmp/test_repair_verify.data"
44532024-04-20T01:07:19.870Z
44542024-04-20T01:07:19.871ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44552024-04-20T01:07:20.412ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44562024-04-20T01:07:20.412Z 0 000-019 33 33 33 55 55 55 F F F
44572024-04-20T01:07:20.412Z 1 020-039 39 39 39 65 65 65 F F F
44582024-04-20T01:07:20.412Z 2 040-059 39 40 40 65 66 66 T F F <---
44592024-04-20T01:07:20.412Z 3 060-079 36 40 40 60 68 68 F T T <---
44602024-04-20T01:07:20.412Z 4 080-099 37 37 37 62 62 62 F F F
44612024-04-20T01:07:20.412Z 5 100-119 37 40 40 62 68 68 F F F <---
44622024-04-20T01:07:20.412Z 6 120-139 38 40 40 63 68 68 F F F <---
44632024-04-20T01:07:20.412Z 7 140-159 39 40 40 65 66 66 T T T <---
44642024-04-20T01:07:20.412Z 8 160-179 35 35 35 58 58 58 F T T <---
44652024-04-20T01:07:20.413Z 9 180-199 36 36 36 61 61 61 F T T <---
44662024-04-20T01:07:20.413Z 10 200-219 38 40 40 63 67 67 F F F <---
44672024-04-20T01:07:20.413Z 11 220-239 36 36 36 61 61 61 F F F
44682024-04-20T01:07:20.413Z 12 240-259 36 36 36 61 61 61 F F F
44692024-04-20T01:07:20.413Z 13 260-279 28 28 28 47 47 47 F T T <---
44702024-04-20T01:07:20.413Z 14 280-299 34 34 34 56 56 56 F F F
44712024-04-20T01:07:20.413Z 15 300-319 38 38 38 63 63 63 F F F
44722024-04-20T01:07:20.413Z 16 320-339 38 38 38 63 63 63 F F F
44732024-04-20T01:07:20.413Z 17 340-359 38 40 40 63 66 66 T F F <---
44742024-04-20T01:07:20.413Z 18 360-379 38 38 38 63 63 63 F T T <---
44752024-04-20T01:07:20.413Z 19 380-399 36 36 36 61 61 61 F T T <---
44762024-04-20T01:07:20.413Z 20 400-419 36 36 36 61 61 61 F F F
44772024-04-20T01:07:20.413Z 21 420-439 32 32 32 54 54 54 F F F
44782024-04-20T01:07:20.413Z 22 440-459 38 38 38 63 63 63 F F F
44792024-04-20T01:07:20.413Z 23 460-479 38 38 38 64 64 64 F F F
44802024-04-20T01:07:20.413Z 24 480-499 38 38 38 63 63 63 F F F
44812024-04-20T01:07:20.413Z 25 500-519 34 34 34 56 56 56 F F F
44822024-04-20T01:07:20.413Z 26 520-539 38 38 38 63 63 63 F F F
44832024-04-20T01:07:20.413Z 27 540-559 36 36 36 61 61 61 F F F
44842024-04-20T01:07:20.413Z 28 560-579 36 36 36 61 61 61 F T T <---
44852024-04-20T01:07:20.413Z 29 580-599 36 40 40 61 66 66 T T T <---
44862024-04-20T01:07:20.414ZMax gen: 40, Max flush: 68
44872024-04-20T01:07:20.414ZError: Difference in extent metadata found!
44882024-04-20T01:07:20.414ZA Difference in extent metadata is expected here
44892024-04-20T01:07:20.414ZOn loop 20
44902024-04-20T01:07:20.414Z
44912024-04-20T01:07:20.414ZVerifying data now
44922024-04-20T01:07:34.503ZNo change after verify
44932024-04-20T01:07:34.503ZLoop: 20 Downstairs dump after verify (and repair):
44942024-04-20T01:07:34.516ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44952024-04-20T01:07:34.516Z 0 000-019 33 33 33 55 55 55 F F F
44962024-04-20T01:07:34.516Z 1 020-039 39 39 39 65 65 65 F F F
44972024-04-20T01:07:34.516Z 2 040-059 40 40 40 66 66 66 F F F
44982024-04-20T01:07:34.516Z 3 060-079 41 41 41 69 69 69 F F F
44992024-04-20T01:07:34.517Z 4 080-099 37 37 37 62 62 62 F F F
45002024-04-20T01:07:34.517Z 5 100-119 40 40 40 68 68 68 F F F
45012024-04-20T01:07:34.517Z 6 120-139 40 40 40 68 68 68 F F F
45022024-04-20T01:07:34.517Z 7 140-159 41 41 41 69 69 69 F F F
45032024-04-20T01:07:34.520Z 8 160-179 41 41 41 69 69 69 F F F
45042024-04-20T01:07:34.521Z 9 180-199 41 41 41 69 69 69 F F F
45052024-04-20T01:07:34.521Z 10 200-219 40 40 40 67 67 67 F F F
45062024-04-20T01:07:34.521Z 11 220-239 36 36 36 61 61 61 F F F
45072024-04-20T01:07:34.521Z 12 240-259 36 36 36 61 61 61 F F F
45082024-04-20T01:07:34.521Z 13 260-279 41 41 41 69 69 69 F F F
45092024-04-20T01:07:34.521Z 14 280-299 34 34 34 56 56 56 F F F
45102024-04-20T01:07:34.521Z 15 300-319 38 38 38 63 63 63 F F F
45112024-04-20T01:07:34.521Z 16 320-339 38 38 38 63 63 63 F F F
45122024-04-20T01:07:34.521Z 17 340-359 40 40 40 66 66 66 F F F
45132024-04-20T01:07:34.521Z 18 360-379 41 41 41 69 69 69 F F F
45142024-04-20T01:07:34.521Z 19 380-399 41 41 41 69 69 69 F F F
45152024-04-20T01:07:34.521Z 20 400-419 36 36 36 61 61 61 F F F
45162024-04-20T01:07:34.521Z 21 420-439 32 32 32 54 54 54 F F F
45172024-04-20T01:07:34.521Z 22 440-459 38 38 38 63 63 63 F F F
45182024-04-20T01:07:34.521Z 23 460-479 38 38 38 64 64 64 F F F
45192024-04-20T01:07:34.521Z 24 480-499 38 38 38 63 63 63 F F F
45202024-04-20T01:07:34.521Z 25 500-519 34 34 34 56 56 56 F F F
45212024-04-20T01:07:34.521Z 26 520-539 38 38 38 63 63 63 F F F
45222024-04-20T01:07:34.521Z 27 540-559 36 36 36 61 61 61 F F F
45232024-04-20T01:07:34.521Z 28 560-579 41 41 41 69 69 69 F F F
45242024-04-20T01:07:34.521Z 29 580-599 41 41 41 69 69 69 F F F
45252024-04-20T01:07:34.521ZMax gen: 41, Max flush: 69
45262024-04-20T01:07:34.522Z
45272024-04-20T01:07:34.522ZBegin loop 21
45282024-04-20T01:07:34.522ZDownstairs to restart: 2
45292024-04-20T01:07:34.538Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047145511Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336}
45302024-04-20T01:07:34.538Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047383961Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336}
45312024-04-20T01:07:34.546Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047428541Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336}
45322024-04-20T01:07:34.546Z{"msg":"Crucible ca65fe99-2bc2-41d1-a0c9-d9c3d2bd7acd has session id: 709c276b-7263-4397-8be0-6bec779b7dc4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047470331Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45332024-04-20T01:07:34.546Z{Crucible runtime is spawned
45342024-04-20T01:07:34.546Z"msg":"Upstairs opts: Upstairs UUID: ca65fe99-2bc2-41d1-a0c9-d9c3d2bd7acd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047529701Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45352024-04-20T01:07:34.546Z{"msg":"Crucible stats registered with UUID: ca65fe99-2bc2-41d1-a0c9-d9c3d2bd7acd","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047569501Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45362024-04-20T01:07:34.546Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047584252Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336}
45372024-04-20T01:07:34.547Z{"msg":"ca65fe99-2bc2-41d1-a0c9-d9c3d2bd7acd active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047731082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45382024-04-20T01:07:34.547Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047762382Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"0","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45392024-04-20T01:07:34.547Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047821362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"0","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45402024-04-20T01:07:34.547Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047857282Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"1","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45412024-04-20T01:07:34.547Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047873942Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"1","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45422024-04-20T01:07:34.547Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047901172Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"2","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45432024-04-20T01:07:34.547Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047918492Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"2","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45442024-04-20T01:07:34.547Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047958022Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45452024-04-20T01:07:34.547Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.047985502Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45462024-04-20T01:07:34.547Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.048159713Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45472024-04-20T01:07:34.547Z{"msg":"ds_connection connected from Ok(127.0.0.1:58341)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.048202473Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45482024-04-20T01:07:34.547Z{"msg":"ds_connection connected from Ok(127.0.0.1:47108)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.048227023Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45492024-04-20T01:07:34.547Z{"msg":"ds_connection connected from Ok(127.0.0.1:53770)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.048263123Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45502024-04-20T01:07:34.547Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.048804324Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"0","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45512024-04-20T01:07:34.547Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.048859684Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"0","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45522024-04-20T01:07:34.547Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.048890364Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"1","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45532024-04-20T01:07:34.547Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.048916744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"1","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45542024-04-20T01:07:34.547Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.049060855Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"2","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45552024-04-20T01:07:34.547Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.049107055Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"2","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45562024-04-20T01:07:34.547Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.049486016Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"0","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45572024-04-20T01:07:34.547Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.049522526Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"1","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45582024-04-20T01:07:34.547Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.049648146Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"2","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45592024-04-20T01:07:34.547Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050036237Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"0","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45602024-04-20T01:07:34.548Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050091207Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"0","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45612024-04-20T01:07:34.548Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050129617Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45622024-04-20T01:07:34.548Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050171817Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"1","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45632024-04-20T01:07:34.548Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050201507Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"1","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45642024-04-20T01:07:34.548Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050231857Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45652024-04-20T01:07:34.548Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050245397Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"2","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45662024-04-20T01:07:34.548Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050265737Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"2","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45672024-04-20T01:07:34.548Z{"msg":"[0]R flush_numbers[0..12]: [55, 65, 66, 69, 62, 68, 68, 69, 69, 69, 67, 61]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050279907Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45682024-04-20T01:07:34.548Z{"msg":"[0]R generation[0..12]: [33, 39, 40, 41, 37, 40, 40, 41, 41, 41, 40, 36]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050300578Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45692024-04-20T01:07:34.548Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
45702024-04-20T01:07:34.548Zcrucible","level":30,"time":"2024-04-20T01:07:33.050318178Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45712024-04-20T01:07:34.548Z{"msg":"[1]R flush_numbers[0..12]: [55, 65, 66, 69, 62, 68, 68, 69, 69, 69, 67, 61]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050339218Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45722024-04-20T01:07:34.548Z{"msg":"[1]R generation[0..12]: [33, 39, 40, 41, 37, 40, 40, 41, 41, 41, 40, 36]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050353107Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45732024-04-20T01:07:34.548Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050373898Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45742024-04-20T01:07:34.548Z{"msg":"[2]R flush_numbers[0..12]: [55, 65, 66, 69, 62, 68, 68, 69, 69, 69, 67, 61]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050387898Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45752024-04-20T01:07:34.548Z{"msg":"[2]R generation[0..12]: [33, 39, 40, 41, 37, 40, 40, 41, 41, 41, 40, 36]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050408098Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45762024-04-20T01:07:34.548Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050423008Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45772024-04-20T01:07:34.548Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050443068Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45782024-04-20T01:07:34.548Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050456568Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45792024-04-20T01:07:34.548Z{"msg":"Next flush: 70","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050476528Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45802024-04-20T01:07:34.548Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050490178Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"":"downstairs","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45812024-04-20T01:07:34.548Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050510178Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45822024-04-20T01:07:34.549Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050522648Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"0","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45832024-04-20T01:07:34.549Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050536398Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"1","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45842024-04-20T01:07:34.549Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050549458Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"client":"2","":"downstairs","session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45852024-04-20T01:07:34.549Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050569168Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45862024-04-20T01:07:34.549ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
45872024-04-20T01:07:34.549Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050587278Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45882024-04-20T01:07:34.549Z{"msg":"ca65fe99-2bc2-41d1-a0c9-d9c3d2bd7acd is now active with session: 709c276b-7263-4397-8be0-6bec779b7dc4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050606658Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45892024-04-20T01:07:34.549Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050623308Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336,"session_id":"709c276b-7263-4397-8be0-6bec779b7dc4"}
45902024-04-20T01:07:34.549Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:33.050636388Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336}
45912024-04-20T01:07:34.549ZLoading write count information from file "/tmp/test_repair_verify.data"
45922024-04-20T01:07:34.549ZRun Repair workload
45932024-04-20T01:07:34.553Z01/30 Write block 588 len 4096 data: 8
45942024-04-20T01:07:34.553Z02/30 Read block 526 len 12288
45952024-04-20T01:07:34.553Z03/30 Write block 232 len 36864 data: 8 9 9 8 8 9 8 8 5
45962024-04-20T01:07:34.553Z04/30 Read block 289 len 16384
45972024-04-20T01:07:34.553Z05/30 Write block 330 len 16384 data: 6 5 6 5
45982024-04-20T01:07:34.553Z06/30 Flush
45992024-04-20T01:07:34.553Z07/30 Write block 568 len 8192 data: 4 6
46002024-04-20T01:07:34.553Z08/30 Flush
46012024-04-20T01:07:34.553Z09/30 Write block 542 len 40960 data: 6 8 6 6 6 4 5 5 5 5
46022024-04-20T01:07:34.556Z10/30 Write block 347 len 12288 data: 6 6 5
46032024-04-20T01:07:34.556Z11/30 Write block 369 len 40960 data: 5 6 7 7 8 7 6 4 4 4
46042024-04-20T01:07:34.557Z12/30 Read block 483 len 8192
46052024-04-20T01:07:34.557Z13/30 Read block 347 len 20480
46062024-04-20T01:07:34.557Z14/30 Read block 223 len 12288
46072024-04-20T01:07:34.560Z15/30 Write block 412 len 36864 data: 6 7 7 6 6 4 5 5 5
46082024-04-20T01:07:34.560Z16/30 Read block 261 len 12288
46092024-04-20T01:07:34.560Z17/30 Flush
46102024-04-20T01:07:34.567Z18/30 Write block 485 len 8192 data: 5 5
46112024-04-20T01:07:34.567Z19/30 Write block 324 len 32768 data: 4 4 6 6 6 6 7 6
46122024-04-20T01:07:34.570Z20/30 Write block 571 len 16384 data: 6 4 4 4
46132024-04-20T01:07:34.570Z21/30 Write block 254 len 36864 data: 4 4 4 4 4 4 3 3 3
46142024-04-20T01:07:34.571Z22/30 Write block 407 len 32768 data: 3 3 3 5 6 7 8 8
46152024-04-20T01:07:34.571Z23/30 Read block 347 len 12288
46162024-04-20T01:07:34.571Z24/30 Read block 438 len 12288
46172024-04-20T01:07:34.571Z25/30 Read block 541 len 32768
46182024-04-20T01:07:34.571Z26/30 Read block 103 len 12288
46192024-04-20T01:07:34.575Z27/30 Read block 289 len 8192
46202024-04-20T01:07:34.575Z28/30 Read block 81 len 16384
46212024-04-20T01:07:34.575Z29/30 Read block 567 len 32768
46222024-04-20T01:07:34.575Z30/30 Write block 8 len 40960 data: 2 2 2 4 4 4 4 5 5 5
46232024-04-20T01:07:34.575Z----------------------------------------------------------------
46242024-04-20T01:07:34.575Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46252024-04-20T01:07:34.575ZStates: Active Active Active
46262024-04-20T01:07:34.575ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46272024-04-20T01:07:34.575Z 1 Acked 1000 Write 1 Done Done Done false
46282024-04-20T01:07:34.575Z 2 Acked 1001 Read 3 Done Done Done false
46292024-04-20T01:07:34.576Z 3 Acked 1002 Write 9 Done Done Done false
46302024-04-20T01:07:34.576Z 4 Acked 1003 Read 4 Done Done Done false
46312024-04-20T01:07:34.576Z 5 Acked 1004 Write 4 Done Done Sent false
46322024-04-20T01:07:34.576Z 6 Acked 1005 Flush 0 Done Done Sent false
46332024-04-20T01:07:34.576Z 7 Acked 1006 Write 2 Done Done Sent false
46342024-04-20T01:07:34.576Z 8 Acked 1007 Flush 0 Done Done Sent false
46352024-04-20T01:07:34.576Z 9 Acked 1008 Write 10 Done Done Sent false
46362024-04-20T01:07:34.576Z 10 Acked 1009 Write 3 Done Done Sent false
46372024-04-20T01:07:34.576Z 11 Acked 1010 Write 10 Done Done Sent false
46382024-04-20T01:07:34.576Z 12 Acked 1011 Read 2 Done Done Sent false
46392024-04-20T01:07:34.576Z 13 Acked 1012 Read 5 Done Done Sent false
46402024-04-20T01:07:34.576Z 14 Acked 1013 Read 3 Done Done Sent false
46412024-04-20T01:07:34.576Z 15 Acked 1014 Write 9 Done Done Sent false
46422024-04-20T01:07:34.576Z 16 Acked 1015 Read 3 Done Done Sent false
46432024-04-20T01:07:34.576Z 17 Acked 1016 Flush 0 Done Done Sent false
46442024-04-20T01:07:34.576Z 18 Acked 1017 Write 2 Done Done Sent false
46452024-04-20T01:07:34.576Z 19 Acked 1018 Write 8 Done Done Sent false
46462024-04-20T01:07:34.576Z 20 Acked 1019 Write 4 Done Done Sent false
46472024-04-20T01:07:34.576Z 21 Acked 1020 Write 9 Done Done Sent false
46482024-04-20T01:07:34.576Z 22 Acked 1021 Write 8 Done Done Sent false
46492024-04-20T01:07:34.576Z 23 Acked 1022 Read 3 Done Done Sent false
46502024-04-20T01:07:34.576Z 24 Acked 1023 Read 3 Done Done Sent false
46512024-04-20T01:07:34.576Z 25 Acked 1024 Read 8 Done Done Sent false
46522024-04-20T01:07:34.576Z 26 Acked 1025 Read 3 Done Done Sent false
46532024-04-20T01:07:34.577Z 27 Acked 1026 Read 2 Done Done Sent false
46542024-04-20T01:07:34.577Z 28 Acked 1027 Read 4 Done Done Sent false
46552024-04-20T01:07:34.577Z 29 Acked 1028 Read 8 Done Done Sent false
46562024-04-20T01:07:34.577Z 30 Acked 1029 Write 10 Sent Sent Sent false
46572024-04-20T01:07:34.577Z STATES DS:0 DS:1 DS:2 TOTAL
46582024-04-20T01:07:34.577Z New 0 0 0 0
46592024-04-20T01:07:34.577Z Sent 1 1 26 28
46602024-04-20T01:07:34.577Z Done 29 29 4 62
46612024-04-20T01:07:34.577Z Skipped 0 0 0 0
46622024-04-20T01:07:34.577Z Error 0 0 0 0
46632024-04-20T01:07:34.577ZLast Flush: 1016 1016 0
46642024-04-20T01:07:34.577ZDownstairs last five completed:
46652024-04-20T01:07:34.577ZUpstairs last five completed: 30 29 28 27 26
46662024-04-20T01:07:34.577Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:07:33.083586261Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1336}
46672024-04-20T01:07:34.582ZWrote out file "/tmp/test_repair_verify.data"
46682024-04-20T01:07:34.582Z
46692024-04-20T01:07:34.586ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46702024-04-20T01:07:34.599ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46712024-04-20T01:07:34.599Z 0 000-019 33 33 33 55 55 55 T T F <---
46722024-04-20T01:07:34.599Z 1 020-039 39 39 39 65 65 65 F F F
46732024-04-20T01:07:34.599Z 2 040-059 40 40 40 66 66 66 F F F
46742024-04-20T01:07:34.599Z 3 060-079 41 41 41 69 69 69 F F F
46752024-04-20T01:07:34.599Z 4 080-099 37 37 37 62 62 62 F F F
46762024-04-20T01:07:34.599Z 5 100-119 40 40 40 68 68 68 F F F
46772024-04-20T01:07:34.599Z 6 120-139 40 40 40 68 68 68 F F F
46782024-04-20T01:07:34.599Z 7 140-159 41 41 41 69 69 69 F F F
46792024-04-20T01:07:34.599Z 8 160-179 41 41 41 69 69 69 F F F
46802024-04-20T01:07:34.599Z 9 180-199 41 41 41 69 69 69 F F F
46812024-04-20T01:07:34.599Z 10 200-219 40 40 40 67 67 67 F F F
46822024-04-20T01:07:34.599Z 11 220-239 42 42 36 70 70 61 F F T <---
46832024-04-20T01:07:34.599Z 12 240-259 42 42 36 70 70 61 T T T <---
46842024-04-20T01:07:34.603Z 13 260-279 41 41 41 69 69 69 T T F <---
46852024-04-20T01:07:34.603Z 14 280-299 34 34 34 56 56 56 F F F
46862024-04-20T01:07:34.604Z 15 300-319 38 38 38 63 63 63 F F F
46872024-04-20T01:07:34.604Z 16 320-339 42 42 38 70 70 63 T T F <---
46882024-04-20T01:07:34.604Z 17 340-359 42 42 40 72 72 66 F F F <---
46892024-04-20T01:07:34.604Z 18 360-379 42 42 41 72 72 69 F F F <---
46902024-04-20T01:07:34.604Z 19 380-399 41 41 41 69 69 69 F F F
46912024-04-20T01:07:34.604Z 20 400-419 42 42 36 72 72 61 T T F <---
46922024-04-20T01:07:34.604Z 21 420-439 42 42 32 72 72 54 F F F <---
46932024-04-20T01:07:34.604Z 22 440-459 38 38 38 63 63 63 F F F
46942024-04-20T01:07:34.604Z 23 460-479 38 38 38 64 64 64 F F F
46952024-04-20T01:07:34.604Z 24 480-499 38 38 38 63 63 63 T T F <---
46962024-04-20T01:07:34.604Z 25 500-519 34 34 34 56 56 56 F F F
46972024-04-20T01:07:34.604Z 26 520-539 38 38 38 63 63 63 F F F
46982024-04-20T01:07:34.604Z 27 540-559 42 42 36 72 72 61 F F F <---
46992024-04-20T01:07:34.604Z 28 560-579 42 42 41 71 71 69 T T F <---
47002024-04-20T01:07:34.604Z 29 580-599 42 42 41 70 70 69 F F T <---
47012024-04-20T01:07:34.604ZMax gen: 42, Max flush: 72
47022024-04-20T01:07:34.604ZError: Difference in extent metadata found!
47032024-04-20T01:07:34.604ZA Difference in extent metadata is expected here
47042024-04-20T01:07:34.604ZOn loop 21
47052024-04-20T01:07:34.604Z
47062024-04-20T01:07:34.608ZVerifying data now
47072024-04-20T01:07:49.200ZNo change after verify
47082024-04-20T01:07:49.200ZLoop: 21 Downstairs dump after verify (and repair):
47092024-04-20T01:07:49.215ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47102024-04-20T01:07:49.215Z 0 000-019 43 43 43 73 73 73 F F F
47112024-04-20T01:07:49.215Z 1 020-039 39 39 39 65 65 65 F F F
47122024-04-20T01:07:49.215Z 2 040-059 40 40 40 66 66 66 F F F
47132024-04-20T01:07:49.215Z 3 060-079 41 41 41 69 69 69 F F F
47142024-04-20T01:07:49.215Z 4 080-099 37 37 37 62 62 62 F F F
47152024-04-20T01:07:49.215Z 5 100-119 40 40 40 68 68 68 F F F
47162024-04-20T01:07:49.221Z 6 120-139 40 40 40 68 68 68 F F F
47172024-04-20T01:07:49.221Z 7 140-159 41 41 41 69 69 69 F F F
47182024-04-20T01:07:49.221Z 8 160-179 41 41 41 69 69 69 F F F
47192024-04-20T01:07:49.221Z 9 180-199 41 41 41 69 69 69 F F F
47202024-04-20T01:07:49.221Z 10 200-219 40 40 40 67 67 67 F F F
47212024-04-20T01:07:49.221Z 11 220-239 42 42 42 70 70 70 F F F
47222024-04-20T01:07:49.221Z 12 240-259 43 43 43 73 73 73 F F F
47232024-04-20T01:07:49.221Z 13 260-279 43 43 43 73 73 73 F F F
47242024-04-20T01:07:49.221Z 14 280-299 34 34 34 56 56 56 F F F
47252024-04-20T01:07:49.221Z 15 300-319 38 38 38 63 63 63 F F F
47262024-04-20T01:07:49.221Z 16 320-339 43 43 43 73 73 73 F F F
47272024-04-20T01:07:49.222Z 17 340-359 42 42 42 72 72 72 F F F
47282024-04-20T01:07:49.222Z 18 360-379 42 42 42 72 72 72 F F F
47292024-04-20T01:07:49.222Z 19 380-399 41 41 41 69 69 69 F F F
47302024-04-20T01:07:49.222Z 20 400-419 43 43 43 73 73 73 F F F
47312024-04-20T01:07:49.222Z 21 420-439 42 42 42 72 72 72 F F F
47322024-04-20T01:07:49.222Z 22 440-459 38 38 38 63 63 63 F F F
47332024-04-20T01:07:49.222Z 23 460-479 38 38 38 64 64 64 F F F
47342024-04-20T01:07:49.222Z 24 480-499 43 43 43 73 73 73 F F F
47352024-04-20T01:07:49.222Z 25 500-519 34 34 34 56 56 56 F F F
47362024-04-20T01:07:49.222Z 26 520-539 38 38 38 63 63 63 F F F
47372024-04-20T01:07:49.222Z 27 540-559 42 42 42 72 72 72 F F F
47382024-04-20T01:07:49.222Z 28 560-579 43 43 43 73 73 73 F F F
47392024-04-20T01:07:49.222Z 29 580-599 42 42 42 70 70 70 F F F
47402024-04-20T01:07:49.222ZMax gen: 43, Max flush: 73
47412024-04-20T01:07:49.222Z
47422024-04-20T01:07:49.222ZBegin loop 22
47432024-04-20T01:07:49.222ZDownstairs to restart: 0
47442024-04-20T01:07:49.236Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745041958Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344}
47452024-04-20T01:07:49.236Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745267189Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344}
47462024-04-20T01:07:49.244Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745292619Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344}
47472024-04-20T01:07:49.244Z{"msg":"Crucible 4c5643ab-7825-4686-b866-d59e0313ea85 has session id: dc10379d-b44e-4d4e-9c06-6acdbd18f8f4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745322849Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47482024-04-20T01:07:49.244Z{"msg":"Upstairs opts: Upstairs UUID: 4c5643ab-7825-4686-b866-d59e0313ea85, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745363129Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47492024-04-20T01:07:49.244ZCrucible runtime is spawned
47502024-04-20T01:07:49.244Z{"msg":"Crucible stats registered with UUID: 4c5643ab-7825-4686-b866-d59e0313ea85","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745417759Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47512024-04-20T01:07:49.244Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745460079Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344}
47522024-04-20T01:07:49.244Z{"msg":"4c5643ab-7825-4686-b866-d59e0313ea85 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745497799Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47532024-04-20T01:07:49.244Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745531249Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"0","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47542024-04-20T01:07:49.245Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745562479Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"0","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47552024-04-20T01:07:49.245Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745587889Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"1","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47562024-04-20T01:07:49.245Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745628949Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"1","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47572024-04-20T01:07:49.245Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.74565489Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"2","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47582024-04-20T01:07:49.245Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.745684979Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"2","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47592024-04-20T01:07:49.245Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.74570887Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47602024-04-20T01:07:49.245Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.74578743Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47612024-04-20T01:07:49.245Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.74587461Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47622024-04-20T01:07:49.245Z{"msg":"ds_connection connected from Ok(127.0.0.1:44897)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.74591827Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47632024-04-20T01:07:49.245Z{"msg":"ds_connection connected from Ok(127.0.0.1:49877)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.74594026Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47642024-04-20T01:07:49.245Z{"msg":"ds_connection connected from Ok(127.0.0.1:52314)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.7459707Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47652024-04-20T01:07:49.245Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.746637262Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"1","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47662024-04-20T01:07:49.245Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.746684152Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"1","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47672024-04-20T01:07:49.245Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.746746962Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"2","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47682024-04-20T01:07:49.245Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.746779032Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"2","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47692024-04-20T01:07:49.245Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.747163303Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"0","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47702024-04-20T01:07:49.245Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.747233593Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"0","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47712024-04-20T01:07:49.245Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.747258693Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"2","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47722024-04-20T01:07:49.245Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.747375913Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"1","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47732024-04-20T01:07:49.245Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.747824464Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"0","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47742024-04-20T01:07:49.245Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.747988655Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"2","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47752024-04-20T01:07:49.245Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748042495Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"2","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47762024-04-20T01:07:49.245Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748078585Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47772024-04-20T01:07:49.246Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748093185Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"1","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47782024-04-20T01:07:49.246Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748113885Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"1","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47792024-04-20T01:07:49.246Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748127785Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47802024-04-20T01:07:49.246Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748245625Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"0","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47812024-04-20T01:07:49.246Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748277175Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"0","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47822024-04-20T01:07:49.246Z{"msg":"[0]R flush_numbers[0..12]: [73, 65, 66, 69, 62, 68, 68, 69, 69, 69, 67, 70]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748292145Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47832024-04-20T01:07:49.246Z{"msg":"[0]R generation[0..12]: [43, 39, 40, 41, 37, 40, 40, 41, 41, 41, 40, 42]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748306315Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47842024-04-20T01:07:49.246Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748376056Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47852024-04-20T01:07:49.246Z{"msg":"[1]R flush_numbers[0..12]: [73, 65, 66, 69, 62, 68, 68, 69, 69, 69, 67, 70]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748403245Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47862024-04-20T01:07:49.246Z{"msg":"[1]R generation[0..12]: [43, 39, 40, 41, 37, 40, 40, 41, 41, 41, 40, 42]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748417826Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47872024-04-20T01:07:49.246Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748439506Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47882024-04-20T01:07:49.246Z{"msg":"[2]R flush_numbers[0..12]: [73, 65, 66, 69, 62, 68, 68, 69, 69, 69, 67, 70]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748458846Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47892024-04-20T01:07:49.246Z{"msg":"[2]R generation[0..12]: [43, 39, 40, 41, 37, 40, 40, 41, 41, 41, 40, 42]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748491876Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47902024-04-20T01:07:49.246Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
47912024-04-20T01:07:49.246Z,"time":"2024-04-20T01:07:47.748508186Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47922024-04-20T01:07:49.246Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748521706Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47932024-04-20T01:07:49.246Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748541986Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47942024-04-20T01:07:49.246Z{"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748555356Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47952024-04-20T01:07:49.246Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748568566Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"":"downstairs","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47962024-04-20T01:07:49.246Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748588956Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47972024-04-20T01:07:49.246Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748601456Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"0","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47982024-04-20T01:07:49.246Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748614856Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"1","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
47992024-04-20T01:07:49.246Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748627966Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"client":"2","":"downstairs","session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
48002024-04-20T01:07:49.247Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748653726Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
48012024-04-20T01:07:49.247Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748666476Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
48022024-04-20T01:07:49.247Z{"msg":"4c5643ab-7825-4686-b866-d59e0313ea85 is now active with session: dc10379d-b44e-4d4e-9c06-6acdbd18f8f4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748679936Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
48032024-04-20T01:07:49.247Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748700456Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344,"session_id":"dc10379d-b44e-4d4e-9c06-6acdbd18f8f4"}
48042024-04-20T01:07:49.247Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:47.748713056Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344}
48052024-04-20T01:07:49.247ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
48062024-04-20T01:07:49.251ZLoading write count information from file "/tmp/test_repair_verify.data"
48072024-04-20T01:07:49.251ZRun Repair workload
48082024-04-20T01:07:49.251Z01/30 Write block 322 len 12288 data: 6 5 5
48092024-04-20T01:07:49.251Z02/30 Read block 363 len 24576
48102024-04-20T01:07:49.251Z03/30 Write block 451 len 32768 data: 5 6 5 6 8 8 9 8
48112024-04-20T01:07:49.251Z04/30 Write block 212 len 36864 data: 4 5 5 3 4 7 7 5 7
48122024-04-20T01:07:49.251Z05/30 Write block 24 len 36864 data: 3 4 6 6 6 6 4 3 3
48132024-04-20T01:07:49.251Z06/30 Read block 113 len 24576
48142024-04-20T01:07:49.251Z07/30 Read block 420 len 12288
48152024-04-20T01:07:49.252Z08/30 Read block 439 len 20480
48162024-04-20T01:07:49.252Z09/30 Write block 399 len 32768 data: 4 3 2 4 4 3 4 4
48172024-04-20T01:07:49.252Z10/30 Read block 504 len 8192
48182024-04-20T01:07:49.252Z11/30 Read block 57 len 16384
48192024-04-20T01:07:49.256Z12/30 Write block 407 len 28672 data: 4 4 4 6 7 8 9
48202024-04-20T01:07:49.256Z13/30 Read block 289 len 36864
48212024-04-20T01:07:49.256Z14/30 Read block 215 len 4096
48222024-04-20T01:07:49.256Z15/30 Read block 435 len 24576
48232024-04-20T01:07:49.256Z16/30 Flush
48242024-04-20T01:07:49.259Z17/30 Flush
48252024-04-20T01:07:49.259Z18/30 Write block 134 len 32768 data: 7 7 6 5 6 5 5 4
48262024-04-20T01:07:49.263Z19/30 Write block 416 len 16384 data: 7 5 6 6
48272024-04-20T01:07:49.263Z20/30 Read block 470 len 28672
48282024-04-20T01:07:49.263Z21/30 Read block 182 len 32768
48292024-04-20T01:07:49.263Z22/30 Write block 107 len 16384 data: 4 3 3 3
48302024-04-20T01:07:49.263Z23/30 Flush
48312024-04-20T01:07:49.267Z24/30 Write block 585 len 8192 data: 4 6
48322024-04-20T01:07:49.267Z25/30 Read block 487 len 8192
48332024-04-20T01:07:49.270Z26/30 Flush
48342024-04-20T01:07:49.270Z27/30 Write block 584 len 24576 data: 4 5 7 8 9 6
48352024-04-20T01:07:49.270Z28/30 Write block 371 len 4096 data: 8
48362024-04-20T01:07:49.274Z29/30 Write block 101 len 12288 data: 5 5 5
48372024-04-20T01:07:49.274Z30/30 Read block 172 len 24576
48382024-04-20T01:07:49.274Z----------------------------------------------------------------
48392024-04-20T01:07:49.274Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48402024-04-20T01:07:49.274ZStates: Active Active Active
48412024-04-20T01:07:49.274ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48422024-04-20T01:07:49.274Z 1 Acked 1000 Write 3 Done Done Done false
48432024-04-20T01:07:49.274Z 2 Acked 1001 Read 6 Sent Done Done false
48442024-04-20T01:07:49.274Z 3 Acked 1002 Write 8 Sent Done Done false
48452024-04-20T01:07:49.274Z 4 Acked 1003 Write 9 Sent Done Done false
48462024-04-20T01:07:49.275Z 5 Acked 1004 Write 9 Sent Done Done false
48472024-04-20T01:07:49.275Z 6 Acked 1005 Read 6 Sent Done Done false
48482024-04-20T01:07:49.275Z 7 Acked 1006 Read 3 Sent Done Done false
48492024-04-20T01:07:49.275Z 8 Acked 1007 Read 5 Sent Done Done false
48502024-04-20T01:07:49.275Z 9 Acked 1008 Write 8 Sent Done Done false
48512024-04-20T01:07:49.275Z 10 Acked 1009 Read 2 Sent Done Done false
48522024-04-20T01:07:49.275Z 11 Acked 1010 Read 4 Sent Done Done false
48532024-04-20T01:07:49.275Z 12 Acked 1011 Write 7 Sent Done Done false
48542024-04-20T01:07:49.275Z 13 Acked 1012 Read 9 Sent Done Done false
48552024-04-20T01:07:49.275Z 14 Acked 1013 Read 1 Sent Done Done false
48562024-04-20T01:07:49.275Z 15 Acked 1014 Read 6 Sent Done Done false
48572024-04-20T01:07:49.275Z 16 Acked 1015 Flush 0 Sent Done Done false
48582024-04-20T01:07:49.275Z 17 Acked 1016 Flush 0 Sent Done Done false
48592024-04-20T01:07:49.275Z 18 Acked 1017 Write 8 Sent Done Done false
48602024-04-20T01:07:49.275Z 19 Acked 1018 Write 4 Sent Done Done false
48612024-04-20T01:07:49.275Z 20 Acked 1019 Read 7 Sent Done Done false
48622024-04-20T01:07:49.275Z 21 Acked 1020 Read 8 Sent Done Done false
48632024-04-20T01:07:49.275Z 22 Acked 1021 Write 4 Sent Done Done false
48642024-04-20T01:07:49.275Z 23 Acked 1022 Flush 0 Sent Done Done false
48652024-04-20T01:07:49.275Z 24 Acked 1023 Write 2 Sent Done Done false
48662024-04-20T01:07:49.275Z 25 Acked 1024 Read 2 Sent Done Done false
48672024-04-20T01:07:49.275Z 26 Acked 1025 Flush 0 Sent Done Done false
48682024-04-20T01:07:49.275Z 27 Acked 1026 Write 6 Sent Done Done false
48692024-04-20T01:07:49.275Z 28 Acked 1027 Write 1 Sent Done Done false
48702024-04-20T01:07:49.276Z 29 Acked 1028 Write 3 Sent Done Done false
48712024-04-20T01:07:49.276Z 30 Acked 1029 Read 6 Sent Done Sent false
48722024-04-20T01:07:49.276Z STATES DS:0 DS:1 DS:2 TOTAL
48732024-04-20T01:07:49.276Z New 0 0 0 0
48742024-04-20T01:07:49.276Z Sent 29 0 1 30
48752024-04-20T01:07:49.276Z Done 1 30 29 60
48762024-04-20T01:07:49.276Z Skipped 0 0 0 0
48772024-04-20T01:07:49.276Z Error 0 0 0 0
48782024-04-20T01:07:49.276ZLast Flush: 0 1025 1025
48792024-04-20T01:07:49.276ZDownstairs last five completed:
48802024-04-20T01:07:49.276ZUpstairs last five completed: 30 29 28 27 26
48812024-04-20T01:07:49.276Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:07:47.781038188Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1344}
48822024-04-20T01:07:49.276ZWrote out file "/tmp/test_repair_verify.data"
48832024-04-20T01:07:49.283Z
48842024-04-20T01:07:49.283ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48852024-04-20T01:07:49.294ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48862024-04-20T01:07:49.294Z 0 000-019 43 43 43 73 73 73 F F F
48872024-04-20T01:07:49.295Z 1 020-039 39 44 44 65 74 74 F F F <---
48882024-04-20T01:07:49.295Z 2 040-059 40 40 40 66 66 66 F F F
48892024-04-20T01:07:49.295Z 3 060-079 41 41 41 69 69 69 F F F
48902024-04-20T01:07:49.295Z 4 080-099 37 37 37 62 62 62 F F F
48912024-04-20T01:07:49.295Z 5 100-119 40 44 44 68 76 76 F T T <---
48922024-04-20T01:07:49.295Z 6 120-139 40 44 44 68 76 76 F F F <---
48932024-04-20T01:07:49.295Z 7 140-159 41 44 44 69 76 76 F F F <---
48942024-04-20T01:07:49.295Z 8 160-179 41 41 41 69 69 69 F F F
48952024-04-20T01:07:49.295Z 9 180-199 41 41 41 69 69 69 F F F
48962024-04-20T01:07:49.295Z 10 200-219 40 44 44 67 74 74 F F F <---
48972024-04-20T01:07:49.295Z 11 220-239 42 44 44 70 74 74 F F F <---
48982024-04-20T01:07:49.295Z 12 240-259 43 43 43 73 73 73 F F F
48992024-04-20T01:07:49.295Z 13 260-279 43 43 43 73 73 73 F F F
49002024-04-20T01:07:49.299Z 14 280-299 34 34 34 56 56 56 F F F
49012024-04-20T01:07:49.299Z 15 300-319 38 38 38 63 63 63 F F F
49022024-04-20T01:07:49.299Z 16 320-339 43 44 44 73 74 74 T F F <---
49032024-04-20T01:07:49.299Z 17 340-359 42 42 42 72 72 72 F F F
49042024-04-20T01:07:49.300Z 18 360-379 42 42 42 72 72 72 F T T <---
49052024-04-20T01:07:49.300Z 19 380-399 41 44 44 69 74 74 F F F <---
49062024-04-20T01:07:49.300Z 20 400-419 43 44 44 73 76 76 F F F <---
49072024-04-20T01:07:49.300Z 21 420-439 42 42 42 72 72 72 F F F
49082024-04-20T01:07:49.300Z 22 440-459 38 44 44 63 74 74 F F F <---
49092024-04-20T01:07:49.300Z 23 460-479 38 38 38 64 64 64 F F F
49102024-04-20T01:07:49.300Z 24 480-499 43 43 43 73 73 73 F F F
49112024-04-20T01:07:49.300Z 25 500-519 34 34 34 56 56 56 F F F
49122024-04-20T01:07:49.300Z 26 520-539 38 38 38 63 63 63 F F F
49132024-04-20T01:07:49.300Z 27 540-559 42 42 42 72 72 72 F F F
49142024-04-20T01:07:49.300Z 28 560-579 43 43 43 73 73 73 F F F
49152024-04-20T01:07:49.300Z 29 580-599 42 44 44 70 77 77 F T T <---
49162024-04-20T01:07:49.300ZMax gen: 44, Max flush: 77
49172024-04-20T01:07:49.300ZError: Difference in extent metadata found!
49182024-04-20T01:07:49.300ZA Difference in extent metadata is expected here
49192024-04-20T01:07:49.300ZOn loop 22
49202024-04-20T01:07:49.301Z
49212024-04-20T01:07:49.305ZVerifying data now
49222024-04-20T01:07:49.844ZNo change after verify
49232024-04-20T01:07:49.844ZLoop: 22 Downstairs dump after verify (and repair):
49242024-04-20T01:07:49.856ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49252024-04-20T01:07:49.857Z 0 000-019 43 43 43 73 73 73 F F F
49262024-04-20T01:07:49.857Z 1 020-039 44 44 44 74 74 74 F F F
49272024-04-20T01:07:49.857Z 2 040-059 40 40 40 66 66 66 F F F
49282024-04-20T01:07:49.857Z 3 060-079 41 41 41 69 69 69 F F F
49292024-04-20T01:07:49.857Z 4 080-099 37 37 37 62 62 62 F F F
49302024-04-20T01:07:49.857Z 5 100-119 45 45 45 78 78 78 F F F
49312024-04-20T01:07:49.857Z 6 120-139 44 44 44 76 76 76 F F F
49322024-04-20T01:07:49.858Z 7 140-159 44 44 44 76 76 76 F F F
49332024-04-20T01:07:49.858Z 8 160-179 41 41 41 69 69 69 F F F
49342024-04-20T01:07:49.858Z 9 180-199 41 41 41 69 69 69 F F F
49352024-04-20T01:07:49.858Z 10 200-219 44 44 44 74 74 74 F F F
49362024-04-20T01:07:49.858Z 11 220-239 44 44 44 74 74 74 F F F
49372024-04-20T01:07:49.858Z 12 240-259 43 43 43 73 73 73 F F F
49382024-04-20T01:07:49.858Z 13 260-279 43 43 43 73 73 73 F F F
49392024-04-20T01:07:49.858Z 14 280-299 34 34 34 56 56 56 F F F
49402024-04-20T01:07:49.858Z 15 300-319 38 38 38 63 63 63 F F F
49412024-04-20T01:07:49.858Z 16 320-339 44 44 44 74 74 74 F F F
49422024-04-20T01:07:49.859Z 17 340-359 42 42 42 72 72 72 F F F
49432024-04-20T01:07:49.859Z 18 360-379 45 45 45 78 78 78 F F F
49442024-04-20T01:07:49.859Z 19 380-399 44 44 44 74 74 74 F F F
49452024-04-20T01:07:49.859Z 20 400-419 44 44 44 76 76 76 F F F
49462024-04-20T01:07:49.859Z 21 420-439 42 42 42 72 72 72 F F F
49472024-04-20T01:07:49.859Z 22 440-459 44 44 44 74 74 74 F F F
49482024-04-20T01:07:49.859Z 23 460-479 38 38 38 64 64 64 F F F
49492024-04-20T01:07:49.859Z 24 480-499 43 43 43 73 73 73 F F F
49502024-04-20T01:07:49.859Z 25 500-519 34 34 34 56 56 56 F F F
49512024-04-20T01:07:49.859Z 26 520-539 38 38 38 63 63 63 F F F
49522024-04-20T01:07:49.859Z 27 540-559 42 42 42 72 72 72 F F F
49532024-04-20T01:07:49.859Z 28 560-579 43 43 43 73 73 73 F F F
49542024-04-20T01:07:49.859Z 29 580-599 45 45 45 78 78 78 F F F
49552024-04-20T01:07:49.859ZMax gen: 45, Max flush: 78
49562024-04-20T01:07:49.863Z
49572024-04-20T01:07:49.863ZBegin loop 23
49582024-04-20T01:07:49.863ZDownstairs to restart: 1
49592024-04-20T01:07:49.872Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.380555203Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352}
49602024-04-20T01:07:49.872Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.380777483Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352}
49612024-04-20T01:07:49.872Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.380802213Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352}
49622024-04-20T01:07:49.880Z{"msg":"Crucible 2c5cac1b-5703-4b7c-8911-09113a583a65 has session id: 10c92d14-28d5-4ec0-9bc5-acb1397f76c8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.380823544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49632024-04-20T01:07:49.880ZCrucible runtime is spawned
49642024-04-20T01:07:49.880Z{"msg":"Upstairs opts: Upstairs UUID: 2c5cac1b-5703-4b7c-8911-09113a583a65, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.380878984Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49652024-04-20T01:07:49.880Z{"msg":"Crucible stats registered with UUID: 2c5cac1b-5703-4b7c-8911-09113a583a65","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.380923544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49662024-04-20T01:07:49.880Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.380947564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352}
49672024-04-20T01:07:49.880Z{"msg":"2c5cac1b-5703-4b7c-8911-09113a583a65 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381014094Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49682024-04-20T01:07:49.880Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381068474Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"0","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49692024-04-20T01:07:49.880Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381112844Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"0","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49702024-04-20T01:07:49.880Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381131404Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"1","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49712024-04-20T01:07:49.880Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381153824Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"1","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49722024-04-20T01:07:49.880Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381167734Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"2","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49732024-04-20T01:07:49.880Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381188694Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"2","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49742024-04-20T01:07:49.880Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381202554Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49752024-04-20T01:07:49.880Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381216434Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49762024-04-20T01:07:49.880Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381266685Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49772024-04-20T01:07:49.880Z{"msg":"ds_connection connected from Ok(127.0.0.1:54342)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381455925Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49782024-04-20T01:07:49.880Z{"msg":"ds_connection connected from Ok(127.0.0.1:44541)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381495165Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49792024-04-20T01:07:49.880Z{"msg":"ds_connection connected from Ok(127.0.0.1:47041)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.381511365Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49802024-04-20T01:07:49.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.382294547Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"2","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49812024-04-20T01:07:49.881Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.382329967Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"2","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49822024-04-20T01:07:49.881Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.382415757Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"0","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49832024-04-20T01:07:49.881Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.382457167Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"0","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49842024-04-20T01:07:49.881Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.382677528Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"1","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49852024-04-20T01:07:49.881Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.382711628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"1","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49862024-04-20T01:07:49.881Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.382813538Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"2","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49872024-04-20T01:07:49.881Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.383127439Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"0","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49882024-04-20T01:07:49.881Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.383301869Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"1","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49892024-04-20T01:07:49.881Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.383404719Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"2","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49902024-04-20T01:07:49.881Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.383498759Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"2","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49912024-04-20T01:07:49.881Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.383542039Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49922024-04-20T01:07:49.881Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.383559239Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"0","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49932024-04-20T01:07:49.881Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38358623Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"0","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49942024-04-20T01:07:49.881Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38360231Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49952024-04-20T01:07:49.881Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38367399Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"1","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49962024-04-20T01:07:49.881Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38370334Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"1","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49972024-04-20T01:07:49.881Z{"msg":"[0]R flush_numbers[0..12]: [73, 74, 66, 69, 62, 78, 76, 76, 69, 69, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38371891Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49982024-04-20T01:07:49.881Z{"msg":"[0]R generation[0..12]: [43, 44, 40, 41, 37, 45, 44, 44, 41, 41, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38374173Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
49992024-04-20T01:07:49.881Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38375718Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50002024-04-20T01:07:49.881Z{"msg":"[1]R flush_numbers[0..12]: [73, 74, 66, 69, 62, 78, 76, 76, 69, 69, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38377897Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50012024-04-20T01:07:49.881Z{"msg":"[1]R generation[0..12]: [43, 44, 40, 41, 37, 45, 44, 44, 41, 41, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38379357Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50022024-04-20T01:07:49.881Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38381563Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50032024-04-20T01:07:49.882Z{"msg":"[2]R flush_numbers[0..12]: [73, 74, 66, 69, 62, 78, 76, 76, 69, 69, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38383037Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50042024-04-20T01:07:49.882Z{"msg":"[2]R generation[0..12]: [43, 44, 40, 41, 37, 45, 44, 44, 41, 41, 44, 44]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
50052024-04-20T01:07:49.882Z,"time":"2024-04-20T01:07:48.38385217Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50062024-04-20T01:07:49.882Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38386871Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50072024-04-20T01:07:49.882Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.3838835Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50082024-04-20T01:07:49.882Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38390426Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50092024-04-20T01:07:49.882Z{"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38391892Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50102024-04-20T01:07:49.882Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.383939331Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"":"downstairs","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50112024-04-20T01:07:49.882Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.38395816Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50122024-04-20T01:07:49.882Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.383988671Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"0","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50132024-04-20T01:07:49.882Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.384023201Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"1","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50142024-04-20T01:07:49.882Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.384051211Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"client":"2","":"downstairs","session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50152024-04-20T01:07:49.882Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.384086741Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50162024-04-20T01:07:49.882Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.384118791Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50172024-04-20T01:07:49.882Z{"msg":"2c5cac1b-5703-4b7c-8911-09113a583a65 is now active with session: 10c92d14-28d5-4ec0-9bc5-acb1397f76c8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.384135491Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50182024-04-20T01:07:49.882Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.384156801Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352,"session_id":"10c92d14-28d5-4ec0-9bc5-acb1397f76c8"}
50192024-04-20T01:07:49.882Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:48.384188971Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352}
50202024-04-20T01:07:49.882ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50212024-04-20T01:07:49.882ZLoading write count information from file "/tmp/test_repair_verify.data"
50222024-04-20T01:07:49.882ZRun Repair workload
50232024-04-20T01:07:49.886Z01/30 Write block 527 len 20480 data: 4 3 3 3 3
50242024-04-20T01:07:49.886Z02/30 Read block 511 len 36864
50252024-04-20T01:07:49.886Z03/30 Read block 15 len 40960
50262024-04-20T01:07:49.887Z04/30 Read block 575 len 8192
50272024-04-20T01:07:49.887Z05/30 Write block 128 len 4096 data: 7
50282024-04-20T01:07:49.887Z06/30 Read block 151 len 28672
50292024-04-20T01:07:49.887Z07/30 Read block 275 len 8192
50302024-04-20T01:07:49.887Z08/30 Write block 122 len 24576 data: 5 4 6 7 7 6
50312024-04-20T01:07:49.887Z09/30 Write block 449 len 4096 data: 6
50322024-04-20T01:07:49.887Z10/30 Read block 511 len 8192
50332024-04-20T01:07:49.887Z11/30 Write block 489 len 20480 data: 5 6 5 5 5
50342024-04-20T01:07:49.887Z12/30 Write block 51 len 4096 data: 10
50352024-04-20T01:07:49.887Z13/30 Read block 324 len 4096
50362024-04-20T01:07:49.887Z14/30 Write block 244 len 24576 data: 6 6 6 5 8 7
50372024-04-20T01:07:49.892Z15/30 Read block 308 len 24576
50382024-04-20T01:07:49.892Z16/30 Read block 550 len 36864
50392024-04-20T01:07:49.892Z17/30 Read block 24 len 36864
50402024-04-20T01:07:49.892Z18/30 Flush
50412024-04-20T01:07:49.892Z19/30 Write block 74 len 4096 data: 4
50422024-04-20T01:07:49.892Z20/30 Read block 348 len 20480
50432024-04-20T01:07:49.896Z21/30 Read block 246 len 16384
50442024-04-20T01:07:49.896Z22/30 Read block 247 len 40960
50452024-04-20T01:07:49.896Z23/30 Write block 425 len 8192 data: 4 3
50462024-04-20T01:07:49.896Z24/30 Write block 371 len 36864 data: 9 8 9 8 7 5 5 5 5
50472024-04-20T01:07:49.899Z25/30 Read block 396 len 16384
50482024-04-20T01:07:49.899Z26/30 Read block 542 len 40960
50492024-04-20T01:07:49.899Z27/30 Read block 57 len 24576
50502024-04-20T01:07:49.900Z28/30 Read block 320 len 28672
50512024-04-20T01:07:49.900Z29/30 Read block 42 len 28672
50522024-04-20T01:07:49.904Z30/30 Write block 22 len 16384 data: 3 3 4 5
50532024-04-20T01:07:49.904Z----------------------------------------------------------------
50542024-04-20T01:07:49.904Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
50552024-04-20T01:07:49.904ZStates: Active Active Active
50562024-04-20T01:07:49.904ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50572024-04-20T01:07:49.904Z 1 Acked 1000 Write 5 Done Done Done false
50582024-04-20T01:07:49.904Z 2 Acked 1001 Read 9 Done Done Done false
50592024-04-20T01:07:49.904Z 3 Acked 1002 Read 10 Done Done Done false
50602024-04-20T01:07:49.904Z 4 Acked 1003 Read 2 Done Done Done false
50612024-04-20T01:07:49.904Z 5 Acked 1004 Write 1 Done Sent Done false
50622024-04-20T01:07:49.904Z 6 Acked 1005 Read 7 Done Sent Done false
50632024-04-20T01:07:49.904Z 7 Acked 1006 Read 2 Done Sent Done false
50642024-04-20T01:07:49.904Z 8 Acked 1007 Write 6 Done Sent Done false
50652024-04-20T01:07:49.904Z 9 Acked 1008 Write 1 Done Sent Done false
50662024-04-20T01:07:49.904Z 10 Acked 1009 Read 2 Done Sent Done false
50672024-04-20T01:07:49.905Z 11 Acked 1010 Write 5 Done Sent Done false
50682024-04-20T01:07:49.905Z 12 Acked 1011 Write 1 Done Sent Done false
50692024-04-20T01:07:49.905Z 13 Acked 1012 Read 1 Done Sent Done false
50702024-04-20T01:07:49.905Z 14 Acked 1013 Write 6 Done Sent Done false
50712024-04-20T01:07:49.905Z 15 Acked 1014 Read 6 Done Sent Done false
50722024-04-20T01:07:49.905Z 16 Acked 1015 Read 9 Done Sent Done false
50732024-04-20T01:07:49.905Z 17 Acked 1016 Read 9 Done Sent Done false
50742024-04-20T01:07:49.905Z 18 Acked 1017 Flush 0 Done Sent Done false
50752024-04-20T01:07:49.905Z 19 Acked 1018 Write 1 Done Sent Done false
50762024-04-20T01:07:49.905Z 20 Acked 1019 Read 5 Done Sent Done false
50772024-04-20T01:07:49.905Z 21 Acked 1020 Read 4 Done Sent Done false
50782024-04-20T01:07:49.905Z 22 Acked 1021 Read 10 Done Sent Done false
50792024-04-20T01:07:49.905Z 23 Acked 1022 Write 2 Done Sent Done false
50802024-04-20T01:07:49.905Z 24 Acked 1023 Write 9 Done Sent Done false
50812024-04-20T01:07:49.905Z 25 Acked 1024 Read 4 Done Sent Done false
50822024-04-20T01:07:49.905Z 26 Acked 1025 Read 10 Done Sent Done false
50832024-04-20T01:07:49.905Z 27 Acked 1026 Read 6 Done Sent Done false
50842024-04-20T01:07:49.905Z 28 Acked 1027 Read 7 Done Sent Done false
50852024-04-20T01:07:49.905Z 29 Acked 1028 Read 7 Done Sent Done false
50862024-04-20T01:07:49.905Z 30 Acked 1029 Write 4 Sent Sent Sent false
50872024-04-20T01:07:49.905Z STATES DS:0 DS:1 DS:2 TOTAL
50882024-04-20T01:07:49.905Z New 0 0 0 0
50892024-04-20T01:07:49.905Z Sent 1 26 1 28
50902024-04-20T01:07:49.906Z Done 29 4 29 62
50912024-04-20T01:07:49.906Z Skipped 0 0 0 0
50922024-04-20T01:07:49.906Z Error 0 0 0 0
50932024-04-20T01:07:49.906ZLast Flush: 1017 0 1017
50942024-04-20T01:07:49.906ZDownstairs last five completed:
50952024-04-20T01:07:49.906ZUpstairs last five completed: 30 29 28 27 26
50962024-04-20T01:07:49.906Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:07:48.410070179Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1352}
50972024-04-20T01:07:49.906ZWrote out file "/tmp/test_repair_verify.data"
50982024-04-20T01:07:49.921Z
50992024-04-20T01:07:49.921ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51002024-04-20T01:07:49.925ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51012024-04-20T01:07:49.925Z 0 000-019 43 43 43 73 73 73 F F F
51022024-04-20T01:07:49.925Z 1 020-039 44 44 44 74 74 74 T F T <---
51032024-04-20T01:07:49.925Z 2 040-059 46 40 46 79 66 79 F F F <---
51042024-04-20T01:07:49.925Z 3 060-079 41 41 41 69 69 69 T F T <---
51052024-04-20T01:07:49.925Z 4 080-099 37 37 37 62 62 62 F F F
51062024-04-20T01:07:49.925Z 5 100-119 45 45 45 78 78 78 F F F
51072024-04-20T01:07:49.925Z 6 120-139 46 44 46 79 76 79 F F F <---
51082024-04-20T01:07:49.925Z 7 140-159 44 44 44 76 76 76 F F F
51092024-04-20T01:07:49.925Z 8 160-179 41 41 41 69 69 69 F F F
51102024-04-20T01:07:49.925Z 9 180-199 41 41 41 69 69 69 F F F
51112024-04-20T01:07:49.925Z 10 200-219 44 44 44 74 74 74 F F F
51122024-04-20T01:07:49.925Z 11 220-239 44 44 44 74 74 74 F F F
51132024-04-20T01:07:49.926Z 12 240-259 46 43 46 79 73 79 F F F <---
51142024-04-20T01:07:49.926Z 13 260-279 43 43 43 73 73 73 F F F
51152024-04-20T01:07:49.926Z 14 280-299 34 34 34 56 56 56 F F F
51162024-04-20T01:07:49.926Z 15 300-319 38 38 38 63 63 63 F F F
51172024-04-20T01:07:49.926Z 16 320-339 44 44 44 74 74 74 F F F
51182024-04-20T01:07:49.926Z 17 340-359 42 42 42 72 72 72 F F F
51192024-04-20T01:07:49.926Z 18 360-379 45 45 45 78 78 78 T F T <---
51202024-04-20T01:07:49.926Z 19 380-399 44 44 44 74 74 74 F F F
51212024-04-20T01:07:49.926Z 20 400-419 44 44 44 76 76 76 F F F
51222024-04-20T01:07:49.926Z 21 420-439 42 42 42 72 72 72 T F T <---
51232024-04-20T01:07:49.926Z 22 440-459 46 44 46 79 74 79 F F F <---
51242024-04-20T01:07:49.926Z 23 460-479 38 38 38 64 64 64 F F F
51252024-04-20T01:07:49.926Z 24 480-499 46 43 46 79 73 79 F F F <---
51262024-04-20T01:07:49.926Z 25 500-519 34 34 34 56 56 56 F F F
51272024-04-20T01:07:49.926Z 26 520-539 46 38 46 79 63 79 F T F <---
51282024-04-20T01:07:49.926Z 27 540-559 42 42 42 72 72 72 F F F
51292024-04-20T01:07:49.926Z 28 560-579 43 43 43 73 73 73 F F F
51302024-04-20T01:07:49.926Z 29 580-599 45 45 45 78 78 78 F F F
51312024-04-20T01:07:49.926ZMax gen: 46, Max flush: 79
51322024-04-20T01:07:49.933ZError: Difference in extent metadata found!
51332024-04-20T01:07:49.933ZA Difference in extent metadata is expected here
51342024-04-20T01:07:49.933ZOn loop 23
51352024-04-20T01:07:49.933Z
51362024-04-20T01:07:49.933ZVerifying data now
51372024-04-20T01:07:54.401ZNo change after verify
51382024-04-20T01:07:54.402ZLoop: 23 Downstairs dump after verify (and repair):
51392024-04-20T01:07:54.416ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51402024-04-20T01:07:54.416Z 0 000-019 43 43 43 73 73 73 F F F
51412024-04-20T01:07:54.416Z 1 020-039 47 47 47 80 80 80 F F F
51422024-04-20T01:07:54.416Z 2 040-059 46 46 46 79 79 79 F F F
51432024-04-20T01:07:54.416Z 3 060-079 47 47 47 80 80 80 F F F
51442024-04-20T01:07:54.416Z 4 080-099 37 37 37 62 62 62 F F F
51452024-04-20T01:07:54.424Z 5 100-119 45 45 45 78 78 78 F F F
51462024-04-20T01:07:54.424Z 6 120-139 46 46 46 79 79 79 F F F
51472024-04-20T01:07:54.425Z 7 140-159 44 44 44 76 76 76 F F F
51482024-04-20T01:07:54.425Z 8 160-179 41 41 41 69 69 69 F F F
51492024-04-20T01:07:54.425Z 9 180-199 41 41 41 69 69 69 F F F
51502024-04-20T01:07:54.425Z 10 200-219 44 44 44 74 74 74 F F F
51512024-04-20T01:07:54.425Z 11 220-239 44 44 44 74 74 74 F F F
51522024-04-20T01:07:54.425Z 12 240-259 46 46 46 79 79 79 F F F
51532024-04-20T01:07:54.426Z 13 260-279 43 43 43 73 73 73 F F F
51542024-04-20T01:07:54.426Z 14 280-299 34 34 34 56 56 56 F F F
51552024-04-20T01:07:54.426Z 15 300-319 38 38 38 63 63 63 F F F
51562024-04-20T01:07:54.426Z 16 320-339 44 44 44 74 74 74 F F F
51572024-04-20T01:07:54.426Z 17 340-359 42 42 42 72 72 72 F F F
51582024-04-20T01:07:54.426Z 18 360-379 47 47 47 80 80 80 F F F
51592024-04-20T01:07:54.426Z 19 380-399 44 44 44 74 74 74 F F F
51602024-04-20T01:07:54.427Z 20 400-419 44 44 44 76 76 76 F F F
51612024-04-20T01:07:54.427Z 21 420-439 47 47 47 80 80 80 F F F
51622024-04-20T01:07:54.427Z 22 440-459 46 46 46 79 79 79 F F F
51632024-04-20T01:07:54.427Z 23 460-479 38 38 38 64 64 64 F F F
51642024-04-20T01:07:54.427Z 24 480-499 46 46 46 79 79 79 F F F
51652024-04-20T01:07:54.427Z 25 500-519 34 34 34 56 56 56 F F F
51662024-04-20T01:07:54.427Z 26 520-539 46 46 46 79 79 79 F F F
51672024-04-20T01:07:54.427Z 27 540-559 42 42 42 72 72 72 F F F
51682024-04-20T01:07:54.428Z 28 560-579 43 43 43 73 73 73 F F F
51692024-04-20T01:07:54.428Z 29 580-599 45 45 45 78 78 78 F F F
51702024-04-20T01:07:54.428ZMax gen: 47, Max flush: 80
51712024-04-20T01:07:54.428Z
51722024-04-20T01:07:54.428ZBegin loop 24
51732024-04-20T01:07:54.428ZDownstairs to restart: 1
51742024-04-20T01:07:54.438Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.943062519Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360}
51752024-04-20T01:07:54.438Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.943352869Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360}
51762024-04-20T01:07:54.438Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.94338952Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360}
51772024-04-20T01:07:54.438Z{"msg":"Crucible 542b2b4d-02a3-4299-8463-ef293814fcce has session id: 3d5a7359-044a-4dd1-8b47-88113314cdbd","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.94341243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51782024-04-20T01:07:54.439Z{"msg":"Upstairs opts: Upstairs UUID: 542b2b4d-02a3-4299-8463-ef293814fcce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
51792024-04-20T01:07:54.439Z,"time":"2024-04-20T01:07:52.94347243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51802024-04-20T01:07:54.439Z{"msg":"Crucible stats registered with UUID: 542b2b4d-02a3-4299-8463-ef293814fcce","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.94351003Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51812024-04-20T01:07:54.468Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.94354978Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360}
51822024-04-20T01:07:54.469Z{"msg":"542b2b4d-02a3-4299-8463-ef293814fcce active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.94365218Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51832024-04-20T01:07:54.469Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.94368698Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"0","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51842024-04-20T01:07:54.469Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.943746861Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"0","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51852024-04-20T01:07:54.469Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.943776241Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"1","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51862024-04-20T01:07:54.469Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.943812501Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"1","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51872024-04-20T01:07:54.469Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.943829131Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"2","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51882024-04-20T01:07:54.469Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.943846701Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"2","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51892024-04-20T01:07:54.469Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.943887701Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51902024-04-20T01:07:54.469Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.943934911Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51912024-04-20T01:07:54.469Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.943962971Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51922024-04-20T01:07:54.469Z{"msg":"ds_connection connected from Ok(127.0.0.1:50373)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.944008911Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51932024-04-20T01:07:54.470Z{"msg":"ds_connection connected from Ok(127.0.0.1:36681)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.944049941Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51942024-04-20T01:07:54.470Z{"msg":"ds_connection connected from Ok(127.0.0.1:53344)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.944148931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51952024-04-20T01:07:54.470Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.944772983Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"2","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51962024-04-20T01:07:54.470Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.944814323Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"2","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51972024-04-20T01:07:54.470Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.944841453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"0","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51982024-04-20T01:07:54.470Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.944874643Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"0","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
51992024-04-20T01:07:54.471Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.944901533Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"1","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52002024-04-20T01:07:54.471Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.944939133Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"1","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52012024-04-20T01:07:54.471Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.945336644Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"2","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52022024-04-20T01:07:54.471Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.945372374Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"1","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52032024-04-20T01:07:54.471Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.945393894Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"0","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52042024-04-20T01:07:54.471Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.945938445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"1","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52052024-04-20T01:07:54.471Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.945973286Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"1","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52062024-04-20T01:07:54.471Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.945988415Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52072024-04-20T01:07:54.471Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946008075Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"0","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52082024-04-20T01:07:54.471Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946022306Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"0","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52092024-04-20T01:07:54.472Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946042326Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52102024-04-20T01:07:54.472Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946054646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"2","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52112024-04-20T01:07:54.472Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946074476Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"2","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52122024-04-20T01:07:54.472Z{"msg":"[0]R flush_numbers[0..12]: [73, 80, 79, 80, 62, 78, 79, 76, 69, 69, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946088406Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52132024-04-20T01:07:54.472Z{"msg":"[0]R generation[0..12]: [43, 47, 46, 47, 37, 45, 46, 44, 41, 41, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946109376Z","hostname"Wait for a query_work_queue command to finish before sending IO
52142024-04-20T01:07:54.472Z:"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52152024-04-20T01:07:54.473Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946126676Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52162024-04-20T01:07:54.473Z{"msg":"[1]R flush_numbers[0..12]: [73, 80, 79, 80, 62, 78, 79, 76, 69, 69, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946141996Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52172024-04-20T01:07:54.473Z{"msg":"[1]R generation[0..12]: [43, 47, 46, 47, 37, 45, 46, 44, 41, 41, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946155436Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52182024-04-20T01:07:54.473Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946177326Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52192024-04-20T01:07:54.473Z{"msg":"[2]R flush_numbers[0..12]: [73, 80, 79, 80, 62, 78, 79, 76, 69, 69, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946191176Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52202024-04-20T01:07:54.473Z{"msg":"[2]R generation[0..12]: [43, 47, 46, 47, 37, 45, 46, 44, 41, 41, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946204926Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52212024-04-20T01:07:54.473Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946220486Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52222024-04-20T01:07:54.473Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946242016Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52232024-04-20T01:07:54.474Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946255476Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52242024-04-20T01:07:54.474Z{"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946268256Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52252024-04-20T01:07:54.474Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946287736Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"":"downstairs","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52262024-04-20T01:07:54.474Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946301176Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52272024-04-20T01:07:54.474Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946313536Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"0","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52282024-04-20T01:07:54.474Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946334476Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"1","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52292024-04-20T01:07:54.474Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52302024-04-20T01:07:54.474Z,"time":"2024-04-20T01:07:52.946347706Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"client":"2","":"downstairs","session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52312024-04-20T01:07:54.474Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946371576Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52322024-04-20T01:07:54.474Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946384186Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52332024-04-20T01:07:54.474Z{"msg":"542b2b4d-02a3-4299-8463-ef293814fcce is now active with session: 3d5a7359-044a-4dd1-8b47-88113314cdbd","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946397726Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52342024-04-20T01:07:54.474Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946416266Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360,"session_id":"3d5a7359-044a-4dd1-8b47-88113314cdbd"}
52352024-04-20T01:07:54.475Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:52.946428587Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360}
52362024-04-20T01:07:54.475ZLoading write count information from file "/tmp/test_repair_verify.data"
52372024-04-20T01:07:54.475ZRun Repair workload
52382024-04-20T01:07:54.475Z01/30 Write block 202 len 40960 data: 3 2 3 4 4 4 4 5 3 3
52392024-04-20T01:07:54.475Z02/30 Read block 561 len 4096
52402024-04-20T01:07:54.475Z03/30 Read block 584 len 4096
52412024-04-20T01:07:54.475Z04/30 Write block 367 len 36864 data: 5 5 6 7 10 9 10 9 8
52422024-04-20T01:07:54.475Z05/30 Read block 33 len 32768
52432024-04-20T01:07:54.476Z06/30 Read block 136 len 32768
52442024-04-20T01:07:54.476Z07/30 Read block 193 len 24576
52452024-04-20T01:07:54.476Z08/30 Read block 103 len 12288
52462024-04-20T01:07:54.476Z09/30 Flush
52472024-04-20T01:07:54.485Z10/30 Write block 191 len 4096 data: 3
52482024-04-20T01:07:54.485Z11/30 Read block 519 len 32768
52492024-04-20T01:07:54.485Z12/30 Read block 189 len 20480
52502024-04-20T01:07:54.485Z13/30 Write block 100 len 4096 data: 5
52512024-04-20T01:07:54.485Z14/30 Write block 534 len 28672 data: 3 2 2 2 3 3 3
52522024-04-20T01:07:54.485Z15/30 Flush
52532024-04-20T01:07:54.486Z16/30 Write block 221 len 32768 data: 7 7 6 6 6 7 4 4
52542024-04-20T01:07:54.486Z17/30 Read block 172 len 4096
52552024-04-20T01:07:54.486Z18/30 Read block 258 len 40960
52562024-04-20T01:07:54.486Z19/30 Read block 506 len 36864
52572024-04-20T01:07:54.486Z20/30 Write block 421 len 8192 data: 6 5
52582024-04-20T01:07:54.486Z21/30 Write block 111 len 20480 data: 3 3 3 3 3
52592024-04-20T01:07:54.486Z22/30 Flush
52602024-04-20T01:07:54.486Z23/30 Write block 280 len 40960 data: 2 2 3 3 4 4 3 3 3 4
52612024-04-20T01:07:54.486Z24/30 Write block 166 len 4096 data: 2
52622024-04-20T01:07:54.486Z25/30 Read block 400 len 16384
52632024-04-20T01:07:54.486Z26/30 Write block 94 len 8192 data: 2 2
52642024-04-20T01:07:54.486Z27/30 Read block 349 len 20480
52652024-04-20T01:07:54.486Z28/30 Read block 549 len 40960
52662024-04-20T01:07:54.486Z29/30 Write block 277 len 20480 data: 3 3 2 3 3
52672024-04-20T01:07:54.486Z30/30 Write block 582 len 12288 data: 6 4 5
52682024-04-20T01:07:54.487Z----------------------------------------------------------------
52692024-04-20T01:07:54.487Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
52702024-04-20T01:07:54.487ZStates: Active Active Active
52712024-04-20T01:07:54.487ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52722024-04-20T01:07:54.487Z 1 Acked 1000 Write 10 Done Sent Done false
52732024-04-20T01:07:54.487Z 2 Acked 1001 Read 1 Done Sent Done false
52742024-04-20T01:07:54.487Z 3 Acked 1002 Read 1 Done Sent Done false
52752024-04-20T01:07:54.487Z 4 Acked 1003 Write 9 Done Sent Done false
52762024-04-20T01:07:54.487Z 5 Acked 1004 Read 8 Done Sent Done false
52772024-04-20T01:07:54.487Z 6 Acked 1005 Read 8 Done Sent Done false
52782024-04-20T01:07:54.487Z 7 Acked 1006 Read 6 Done Sent Done false
52792024-04-20T01:07:54.487Z 8 Acked 1007 Read 3 Done Sent Done false
52802024-04-20T01:07:54.487Z 9 Acked 1008 Flush 0 Done Sent Done false
52812024-04-20T01:07:54.488Z 10 Acked 1009 Write 1 Done Sent Done false
52822024-04-20T01:07:54.488Z 11 Acked 1010 Read 8 Done Sent Done false
52832024-04-20T01:07:54.488Z 12 Acked 1011 Read 5 Done Sent Done false
52842024-04-20T01:07:54.488Z 13 Acked 1012 Write 1 Done Sent Done false
52852024-04-20T01:07:54.488Z 14 Acked 1013 Write 7 Done Sent Done false
52862024-04-20T01:07:54.488Z 15 Acked 1014 Flush 0 Done Sent Done false
52872024-04-20T01:07:54.488Z 16 Acked 1015 Write 8 Done Sent Done false
52882024-04-20T01:07:54.488Z 17 Acked 1016 Read 1 Done Sent Done false
52892024-04-20T01:07:54.488Z 18 Acked 1017 Read 10 Done Sent Done false
52902024-04-20T01:07:54.488Z 19 Acked 1018 Read 9 Done Sent Done false
52912024-04-20T01:07:54.488Z 20 Acked 1019 Write 2 Done Sent Done false
52922024-04-20T01:07:54.488Z 21 Acked 1020 Write 5 Done Sent Done false
52932024-04-20T01:07:54.488Z 22 Acked 1021 Flush 0 Done Sent Done false
52942024-04-20T01:07:54.489Z 23 Acked 1022 Write 10 Done Sent Done false
52952024-04-20T01:07:54.489Z 24 Acked 1023 Write 1 Done Sent Done false
52962024-04-20T01:07:54.489Z 25 Acked 1024 Read 4 Done Sent Done false
52972024-04-20T01:07:54.489Z 26 Acked 1025 Write 2 Done Sent Done false
52982024-04-20T01:07:54.489Z 27 Acked 1026 Read 5 Done Sent Done false
52992024-04-20T01:07:54.489Z 28 Acked 1027 Read 10 Done Sent Done false
53002024-04-20T01:07:54.489Z 29 Acked 1028 Write 5 Done Sent Done false
53012024-04-20T01:07:54.489Z 30 Acked 1029 Write 3 Sent Sent Sent false
53022024-04-20T01:07:54.489Z STATES DS:0 DS:1 DS:2 TOTAL
53032024-04-20T01:07:54.489Z New 0 0 0 0
53042024-04-20T01:07:54.489Z Sent 1 30 1 32
53052024-04-20T01:07:54.490Z Done 29 0 29 58
53062024-04-20T01:07:54.490Z Skipped 0 0 0 0
53072024-04-20T01:07:54.490Z Error 0 0 0 0
53082024-04-20T01:07:54.490ZLast Flush: 1021 0 1021
53092024-04-20T01:07:54.490ZDownstairs last five completed:
53102024-04-20T01:07:54.490ZUpstairs last five completed: 30 29 28 27 26
53112024-04-20T01:07:54.490Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:07:52.978038567Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1360}
53122024-04-20T01:07:54.491ZWrote out file "/tmp/test_repair_verify.data"
53132024-04-20T01:07:54.491Z
53142024-04-20T01:07:54.491ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53152024-04-20T01:07:54.502ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53162024-04-20T01:07:54.502Z 0 000-019 43 43 43 73 73 73 F F F
53172024-04-20T01:07:54.503Z 1 020-039 47 47 47 80 80 80 F F F
53182024-04-20T01:07:54.503Z 2 040-059 46 46 46 79 79 79 F F F
53192024-04-20T01:07:54.503Z 3 060-079 47 47 47 80 80 80 F F F
53202024-04-20T01:07:54.503Z 4 080-099 37 37 37 62 62 62 T F T <---
53212024-04-20T01:07:54.503Z 5 100-119 48 45 48 83 78 83 F F F <---
53222024-04-20T01:07:54.503Z 6 120-139 46 46 46 79 79 79 F F F
53232024-04-20T01:07:54.503Z 7 140-159 44 44 44 76 76 76 F F F
53242024-04-20T01:07:54.503Z 8 160-179 41 41 41 69 69 69 T F T <---
53252024-04-20T01:07:54.504Z 9 180-199 48 41 48 82 69 82 F F F <---
53262024-04-20T01:07:54.504Z 10 200-219 48 44 48 81 74 81 F F F <---
53272024-04-20T01:07:54.504Z 11 220-239 48 44 48 83 74 83 F F F <---
53282024-04-20T01:07:54.504Z 12 240-259 46 46 46 79 79 79 F F F
53292024-04-20T01:07:54.504Z 13 260-279 43 43 43 73 73 73 T F T <---
53302024-04-20T01:07:54.504Z 14 280-299 34 34 34 56 56 56 T F T <---
53312024-04-20T01:07:54.504Z 15 300-319 38 38 38 63 63 63 F F F
53322024-04-20T01:07:54.505Z 16 320-339 44 44 44 74 74 74 F F F
53332024-04-20T01:07:54.505Z 17 340-359 42 42 42 72 72 72 F F F
53342024-04-20T01:07:54.505Z 18 360-379 48 47 48 81 80 81 F F F <---
53352024-04-20T01:07:54.505Z 19 380-399 44 44 44 74 74 74 F F F
53362024-04-20T01:07:54.505Z 20 400-419 44 44 44 76 76 76 F F F
53372024-04-20T01:07:54.505Z 21 420-439 48 47 48 83 80 83 F F F <---
53382024-04-20T01:07:54.505Z 22 440-459 46 46 46 79 79 79 F F F
53392024-04-20T01:07:54.505Z 23 460-479 38 38 38 64 64 64 F F F
53402024-04-20T01:07:54.506Z 24 480-499 46 46 46 79 79 79 F F F
53412024-04-20T01:07:54.506Z 25 500-519 34 34 34 56 56 56 F F F
53422024-04-20T01:07:54.506Z 26 520-539 48 46 48 82 79 82 F F F <---
53432024-04-20T01:07:54.506Z 27 540-559 48 42 48 82 72 82 F F F <---
53442024-04-20T01:07:54.506Z 28 560-579 43 43 43 73 73 73 F F F
53452024-04-20T01:07:54.506Z 29 580-599 45 45 45 78 78 78 T F T <---
53462024-04-20T01:07:54.506ZMax gen: 48, Max flush: 83
53472024-04-20T01:07:54.506ZError: Difference in extent metadata found!
53482024-04-20T01:07:54.507ZA Difference in extent metadata is expected here
53492024-04-20T01:07:54.507ZOn loop 24
53502024-04-20T01:07:54.507Z
53512024-04-20T01:07:54.507ZVerifying data now
53522024-04-20T01:07:59.091ZNo change after verify
53532024-04-20T01:07:59.091ZLoop: 24 Downstairs dump after verify (and repair):
53542024-04-20T01:07:59.104ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53552024-04-20T01:07:59.104Z 0 000-019 43 43 43 73 73 73 F F F
53562024-04-20T01:07:59.109Z 1 020-039 47 47 47 80 80 80 F F F
53572024-04-20T01:07:59.109Z 2 040-059 46 46 46 79 79 79 F F F
53582024-04-20T01:07:59.109Z 3 060-079 47 47 47 80 80 80 F F F
53592024-04-20T01:07:59.109Z 4 080-099 49 49 49 84 84 84 F F F
53602024-04-20T01:07:59.109Z 5 100-119 48 48 48 83 83 83 F F F
53612024-04-20T01:07:59.109Z 6 120-139 46 46 46 79 79 79 F F F
53622024-04-20T01:07:59.109Z 7 140-159 44 44 44 76 76 76 F F F
53632024-04-20T01:07:59.109Z 8 160-179 49 49 49 84 84 84 F F F
53642024-04-20T01:07:59.109Z 9 180-199 48 48 48 82 82 82 F F F
53652024-04-20T01:07:59.109Z 10 200-219 48 48 48 81 81 81 F F F
53662024-04-20T01:07:59.109Z 11 220-239 48 48 48 83 83 83 F F F
53672024-04-20T01:07:59.109Z 12 240-259 46 46 46 79 79 79 F F F
53682024-04-20T01:07:59.109Z 13 260-279 49 49 49 84 84 84 F F F
53692024-04-20T01:07:59.109Z 14 280-299 49 49 49 84 84 84 F F F
53702024-04-20T01:07:59.109Z 15 300-319 38 38 38 63 63 63 F F F
53712024-04-20T01:07:59.109Z 16 320-339 44 44 44 74 74 74 F F F
53722024-04-20T01:07:59.109Z 17 340-359 42 42 42 72 72 72 F F F
53732024-04-20T01:07:59.109Z 18 360-379 48 48 48 81 81 81 F F F
53742024-04-20T01:07:59.109Z 19 380-399 44 44 44 74 74 74 F F F
53752024-04-20T01:07:59.110Z 20 400-419 44 44 44 76 76 76 F F F
53762024-04-20T01:07:59.110Z 21 420-439 48 48 48 83 83 83 F F F
53772024-04-20T01:07:59.110Z 22 440-459 46 46 46 79 79 79 F F F
53782024-04-20T01:07:59.110Z 23 460-479 38 38 38 64 64 64 F F F
53792024-04-20T01:07:59.110Z 24 480-499 46 46 46 79 79 79 F F F
53802024-04-20T01:07:59.110Z 25 500-519 34 34 34 56 56 56 F F F
53812024-04-20T01:07:59.110Z 26 520-539 48 48 48 82 82 82 F F F
53822024-04-20T01:07:59.110Z 27 540-559 48 48 48 82 82 82 F F F
53832024-04-20T01:07:59.110Z 28 560-579 43 43 43 73 73 73 F F F
53842024-04-20T01:07:59.110Z 29 580-599 49 49 49 84 84 84 F F F
53852024-04-20T01:07:59.110ZMax gen: 49, Max flush: 84
53862024-04-20T01:07:59.110Z
53872024-04-20T01:07:59.110ZBegin loop 25
53882024-04-20T01:07:59.110ZDownstairs to restart: 1
53892024-04-20T01:07:59.127Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635165333Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368}
53902024-04-20T01:07:59.127Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635406324Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368}
53912024-04-20T01:07:59.134Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635436994Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368}
53922024-04-20T01:07:59.134Z{"msg":"Crucible 8ecb21bc-01e2-4d65-8101-eb317155243a has session id: 2e722897-9402-4c16-922e-37359696d8ec","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635461614Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
53932024-04-20T01:07:59.134Z{"msg":"Upstairs opts: Upstairs UUID: 8ecb21bc-01e2-4d65-8101-eb317155243a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635499334Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
53942024-04-20T01:07:59.134Z{"msg":"Crucible stats registered with UUID: 8ecb21bc-01e2-4d65-8101-eb317155243a","v":0,"name":"crucible","level":30Crucible runtime is spawned
53952024-04-20T01:07:59.134Z,"time":"2024-04-20T01:07:57.635542464Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
53962024-04-20T01:07:59.134Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635613745Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368}
53972024-04-20T01:07:59.134Z{"msg":"8ecb21bc-01e2-4d65-8101-eb317155243a active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635724295Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
53982024-04-20T01:07:59.134Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635751635Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"0","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
53992024-04-20T01:07:59.134Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635771185Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"0","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54002024-04-20T01:07:59.134Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635797015Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"1","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54012024-04-20T01:07:59.134Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635825895Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"1","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54022024-04-20T01:07:59.134Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635861055Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"2","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54032024-04-20T01:07:59.134Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635888505Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"2","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54042024-04-20T01:07:59.134Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635922995Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54052024-04-20T01:07:59.134Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.635965365Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54062024-04-20T01:07:59.134Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.636007725Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54072024-04-20T01:07:59.134Z{"msg":"ds_connection connected from Ok(127.0.0.1:39245)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.636209546Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54082024-04-20T01:07:59.135Z{"msg":"ds_connection connected from Ok(127.0.0.1:56921)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.636354556Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54092024-04-20T01:07:59.135Z{"msg":"ds_connection connected from Ok(127.0.0.1:52949)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.636423686Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54102024-04-20T01:07:59.135Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.636915887Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"2","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54112024-04-20T01:07:59.135Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.636953117Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"2","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54122024-04-20T01:07:59.135Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.636979048Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"0","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54132024-04-20T01:07:59.135Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.637000297Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"0","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54142024-04-20T01:07:59.135Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.637021288Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"1","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54152024-04-20T01:07:59.135Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.637045688Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"1","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54162024-04-20T01:07:59.135Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.637457498Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"2","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54172024-04-20T01:07:59.135Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.637619199Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"0","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54182024-04-20T01:07:59.135Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.637674079Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"1","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54192024-04-20T01:07:59.135Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.63801644Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"2","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54202024-04-20T01:07:59.135Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.63807825Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"2","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54212024-04-20T01:07:59.135Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.6381182Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54222024-04-20T01:07:59.135Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.6381421Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"0","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54232024-04-20T01:07:59.135Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.63817134Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"0","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54242024-04-20T01:07:59.135Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.63818623Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54252024-04-20T01:07:59.135Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.63820045Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"1","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54262024-04-20T01:07:59.135Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.63822926Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"1","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54272024-04-20T01:07:59.135Z{"msg":"[0]R flush_numbers[0..12]: [73, 80, 79, 80, 84, 83, 79, 76, 84, 82, 81, 83]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.63824982Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54282024-04-20T01:07:59.135Z{"msg":"[0]R generation[0..12]: [43, 47, 46, 47, 49, 48, 46, 44, 49, 48, 48, 48]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.63826899Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54292024-04-20T01:07:59.135Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638284671Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54302024-04-20T01:07:59.135Z{"msg":"[1]R flush_numbers[0..12]: [73, 80, 79, 80, 84, 83, 79, 76, 84, 82, 81, 83]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.63829904Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54312024-04-20T01:07:59.136Z{"msg":"[1]R generation[0..12]: [43, 47, 46, 47, 49, 48, 46, 44, 49, 48, 48, 48]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638321211Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54322024-04-20T01:07:59.136Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.63833584Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54332024-04-20T01:07:59.136Z{"msg":"[2]R flush_numbers[0..12]: [73, 80, 79, 80, 84, 83, 79, 76, 84, 82, 81, 83]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638365561Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54342024-04-20T01:07:59.136Z{"msg":"[2]R generation[0..12]: [43, 47, 46, 47, 49, 48, 46, 44, 49, 48, 48, 48]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
54352024-04-20T01:07:59.136Zcrucible","level":30,"time":"2024-04-20T01:07:57.638393831Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54362024-04-20T01:07:59.136Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638433621Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54372024-04-20T01:07:59.136Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638470861Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54382024-04-20T01:07:59.136Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638510261Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54392024-04-20T01:07:59.136Z{"msg":"Next flush: 85","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638547611Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54402024-04-20T01:07:59.136Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638580091Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"":"downstairs","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54412024-04-20T01:07:59.136Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638608691Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54422024-04-20T01:07:59.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638642061Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"0","":"downstairs","session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54432024-04-20T01:07:59.136Z2e722897-9402-4c16-922e-37359696d8ec"}
54442024-04-20T01:07:59.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638681371Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"1","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54452024-04-20T01:07:59.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638715371Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"client":"2","":"downstairs","session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54462024-04-20T01:07:59.136Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638750141Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54472024-04-20T01:07:59.136Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638781032Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54482024-04-20T01:07:59.136Z{"msg":"8ecb21bc-01e2-4d65-8101-eb317155243a is now active with session: 2e722897-9402-4c16-922e-37359696d8ec","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638813102Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54492024-04-20T01:07:59.136Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638841882Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368,"session_id":"2e722897-9402-4c16-922e-37359696d8ec"}
54502024-04-20T01:07:59.136Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:07:57.638871762Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368}
54512024-04-20T01:07:59.140ZLoading write count information from file "/tmp/test_repair_verify.data"
54522024-04-20T01:07:59.141ZRun Repair workload
54532024-04-20T01:07:59.141Z01/30 Write block 151 len 28672 data: 5 4 5 5 7 5 4
54542024-04-20T01:07:59.141Z02/30 Write block 81 len 16384 data: 3 3 4 4
54552024-04-20T01:07:59.141Z03/30 Write block 492 len 20480 data: 6 6 5 5 4
54562024-04-20T01:07:59.141Z04/30 Write block 62 len 20480 data: 5 4 5 5 6
54572024-04-20T01:07:59.141Z05/30 Write block 226 len 36864 data: 8 5 5 4 5 8 9 10 10
54582024-04-20T01:07:59.141Z06/30 Write block 78 len 32768 data: 3 3 3 4 4 5 5 4
54592024-04-20T01:07:59.141Z07/30 Read block 105 len 28672
54602024-04-20T01:07:59.141Z08/30 Read block 13 len 24576
54612024-04-20T01:07:59.141Z09/30 Read block 101 len 24576
54622024-04-20T01:07:59.145Z10/30 Write block 415 len 32768 data: 7 8 6 7 7 6 7 6
54632024-04-20T01:07:59.145Z11/30 Read block 27 len 16384
54642024-04-20T01:07:59.145Z12/30 Read block 46 len 32768
54652024-04-20T01:07:59.145Z13/30 Read block 136 len 28672
54662024-04-20T01:07:59.145Z14/30 Read block 514 len 40960
54672024-04-20T01:07:59.148Z15/30 Write block 53 len 36864 data: 10 7 5 5 4 4 4 4 4
54682024-04-20T01:07:59.149Z16/30 Read block 460 len 24576
54692024-04-20T01:07:59.149Z17/30 Read block 281 len 28672
54702024-04-20T01:07:59.149Z18/30 Read block 305 len 32768
54712024-04-20T01:07:59.152Z19/30 Write block 521 len 24576 data: 8 7 7 6 6 5
54722024-04-20T01:07:59.152Z20/30 Flush
54732024-04-20T01:07:59.156Z21/30 Write block 592 len 28672 data: 6 5 6 5 3 2 2
54742024-04-20T01:07:59.156Z22/30 Read block 177 len 12288
54752024-04-20T01:07:59.161Z23/30 Read block 531 len 40960
54762024-04-20T01:07:59.161Z24/30 Write block 301 len 16384 data: 4 4 4 4
54772024-04-20T01:07:59.161Z25/30 Flush
54782024-04-20T01:07:59.165Z26/30 Write block 66 len 16384 data: 7 5 5 6
54792024-04-20T01:07:59.165Z27/30 Read block 252 len 24576
54802024-04-20T01:07:59.165Z28/30 Read block 487 len 20480
54812024-04-20T01:07:59.165Z29/30 Read block 172 len 16384
54822024-04-20T01:07:59.165Z30/30 Write block 154 len 20480 data: 6 8 6 5 5
54832024-04-20T01:07:59.165Z----------------------------------------------------------------
54842024-04-20T01:07:59.166Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
54852024-04-20T01:07:59.166ZStates: Active Active Active
54862024-04-20T01:07:59.166ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
54872024-04-20T01:07:59.166Z 1 Acked 1000 Write 7 Done Done Done false
54882024-04-20T01:07:59.166Z 2 Acked 1001 Write 4 Done Done Done false
54892024-04-20T01:07:59.166Z 3 Acked 1002 Write 5 Done Done Done false
54902024-04-20T01:07:59.166Z 4 Acked 1003 Write 5 Done Done Done false
54912024-04-20T01:07:59.166Z 5 Acked 1004 Write 9 Done Done Done false
54922024-04-20T01:07:59.166Z 6 Acked 1005 Write 8 Done Done Done false
54932024-04-20T01:07:59.166Z 7 Acked 1006 Read 7 Done Sent Done false
54942024-04-20T01:07:59.166Z 8 Acked 1007 Read 6 Done Sent Done false
54952024-04-20T01:07:59.166Z 9 Acked 1008 Read 6 Done Sent Done false
54962024-04-20T01:07:59.166Z 10 Acked 1009 Write 8 Done Sent Done false
54972024-04-20T01:07:59.166Z 11 Acked 1010 Read 4 Done Sent Done false
54982024-04-20T01:07:59.166Z 12 Acked 1011 Read 8 Done Sent Done false
54992024-04-20T01:07:59.166Z 13 Acked 1012 Read 7 Done Sent Done false
55002024-04-20T01:07:59.166Z 14 Acked 1013 Read 10 Done Sent Done false
55012024-04-20T01:07:59.166Z 15 Acked 1014 Write 9 Done Sent Done false
55022024-04-20T01:07:59.166Z 16 Acked 1015 Read 6 Done Sent Done false
55032024-04-20T01:07:59.166Z 17 Acked 1016 Read 7 Done Sent Done false
55042024-04-20T01:07:59.166Z 18 Acked 1017 Read 8 Done Sent Done false
55052024-04-20T01:07:59.166Z 19 Acked 1018 Write 6 Done Sent Done false
55062024-04-20T01:07:59.166Z 20 Acked 1019 Flush 0 Done Sent Done false
55072024-04-20T01:07:59.167Z 21 Acked 1020 Write 7 Done Sent Done false
55082024-04-20T01:07:59.167Z 22 Acked 1021 Read 3 Done Sent Done false
55092024-04-20T01:07:59.167Z 23 Acked 1022 Read 10 Done Sent Done false
55102024-04-20T01:07:59.167Z 24 Acked 1023 Write 4 Done Sent Done false
55112024-04-20T01:07:59.167Z 25 Acked 1024 Flush 0 Done Sent Done false
55122024-04-20T01:07:59.167Z 26 Acked 1025 Write 4 Done Sent Done false
55132024-04-20T01:07:59.167Z 27 Acked 1026 Read 6 Done Sent Done false
55142024-04-20T01:07:59.167Z 28 Acked 1027 Read 5 Done Sent Done false
55152024-04-20T01:07:59.167Z 29 Acked 1028 Read 4 Done Sent Done false
55162024-04-20T01:07:59.167Z 30 Acked 1029 Write 5 Sent Sent Sent false
55172024-04-20T01:07:59.167Z STATES DS:0 DS:1 DS:2 TOTAL
55182024-04-20T01:07:59.167Z New 0 0 0 0
55192024-04-20T01:07:59.167Z Sent 1 24 1 26
55202024-04-20T01:07:59.167Z Done 29 6 29 64
55212024-04-20T01:07:59.167Z Skipped 0 0 0 0
55222024-04-20T01:07:59.167Z Error 0 0 0 0
55232024-04-20T01:07:59.167ZLast Flush: 1024 0 1024
55242024-04-20T01:07:59.167ZDownstairs last five completed:
55252024-04-20T01:07:59.167ZUpstairs last five completed: 30 29 28 27 26
55262024-04-20T01:07:59.167Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:07:57.673546259Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1368}
55272024-04-20T01:07:59.171ZWrote out file "/tmp/test_repair_verify.data"
55282024-04-20T01:07:59.171Z
55292024-04-20T01:07:59.177ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55302024-04-20T01:07:59.187ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55312024-04-20T01:07:59.187Z 0 000-019 43 43 43 73 73 73 F F F
55322024-04-20T01:07:59.188Z 1 020-039 47 47 47 80 80 80 F F F
55332024-04-20T01:07:59.188Z 2 040-059 50 46 50 85 79 85 F F F <---
55342024-04-20T01:07:59.188Z 3 060-079 50 47 50 85 80 85 T T T <---
55352024-04-20T01:07:59.188Z 4 080-099 50 49 50 85 84 85 F T F <---
55362024-04-20T01:07:59.188Z 5 100-119 48 48 48 83 83 83 F F F
55372024-04-20T01:07:59.188Z 6 120-139 46 46 46 79 79 79 F F F
55382024-04-20T01:07:59.188Z 7 140-159 50 44 50 85 76 85 T T T <---
55392024-04-20T01:07:59.188Z 8 160-179 49 49 49 84 84 84 F F F
55402024-04-20T01:07:59.188Z 9 180-199 48 48 48 82 82 82 F F F
55412024-04-20T01:07:59.192Z 10 200-219 48 48 48 81 81 81 F F F
55422024-04-20T01:07:59.192Z 11 220-239 50 48 50 85 83 85 F T F <---
55432024-04-20T01:07:59.192Z 12 240-259 46 46 46 79 79 79 F F F
55442024-04-20T01:07:59.192Z 13 260-279 49 49 49 84 84 84 F F F
55452024-04-20T01:07:59.192Z 14 280-299 49 49 49 84 84 84 F F F
55462024-04-20T01:07:59.192Z 15 300-319 50 38 50 86 63 86 F F F <---
55472024-04-20T01:07:59.192Z 16 320-339 44 44 44 74 74 74 F F F
55482024-04-20T01:07:59.192Z 17 340-359 42 42 42 72 72 72 F F F
55492024-04-20T01:07:59.192Z 18 360-379 48 48 48 81 81 81 F F F
55502024-04-20T01:07:59.192Z 19 380-399 44 44 44 74 74 74 F F F
55512024-04-20T01:07:59.192Z 20 400-419 50 44 50 85 76 85 F F F <---
55522024-04-20T01:07:59.192Z 21 420-439 50 48 50 85 83 85 F F F <---
55532024-04-20T01:07:59.192Z 22 440-459 46 46 46 79 79 79 F F F
55542024-04-20T01:07:59.192Z 23 460-479 38 38 38 64 64 64 F F F
55552024-04-20T01:07:59.192Z 24 480-499 50 46 50 85 79 85 F T F <---
55562024-04-20T01:07:59.192Z 25 500-519 34 34 34 56 56 56 F F F
55572024-04-20T01:07:59.192Z 26 520-539 50 48 50 85 82 85 F F F <---
55582024-04-20T01:07:59.192Z 27 540-559 48 48 48 82 82 82 F F F
55592024-04-20T01:07:59.192Z 28 560-579 43 43 43 73 73 73 F F F
55602024-04-20T01:07:59.192Z 29 580-599 50 49 50 86 84 86 F F F <---
55612024-04-20T01:07:59.192ZMax gen: 50, Max flush: 86
55622024-04-20T01:07:59.193ZError: Difference in extent metadata found!
55632024-04-20T01:07:59.193ZA Difference in extent metadata is expected here
55642024-04-20T01:07:59.193ZOn loop 25
55652024-04-20T01:07:59.193Z
55662024-04-20T01:07:59.196ZVerifying data now
55672024-04-20T01:08:13.703ZNo change after verify
55682024-04-20T01:08:13.703ZLoop: 25 Downstairs dump after verify (and repair):
55692024-04-20T01:08:13.716ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55702024-04-20T01:08:13.716Z 0 000-019 43 43 43 73 73 73 F F F
55712024-04-20T01:08:13.716Z 1 020-039 47 47 47 80 80 80 F F F
55722024-04-20T01:08:13.716Z 2 040-059 50 50 50 85 85 85 F F F
55732024-04-20T01:08:13.716Z 3 060-079 51 51 51 87 87 87 F F F
55742024-04-20T01:08:13.716Z 4 080-099 50 50 50 85 85 85 F F F
55752024-04-20T01:08:13.716Z 5 100-119 48 48 48 83 83 83 F F F
55762024-04-20T01:08:13.720Z 6 120-139 46 46 46 79 79 79 F F F
55772024-04-20T01:08:13.720Z 7 140-159 51 51 51 87 87 87 F F F
55782024-04-20T01:08:13.720Z 8 160-179 49 49 49 84 84 84 F F F
55792024-04-20T01:08:13.720Z 9 180-199 48 48 48 82 82 82 F F F
55802024-04-20T01:08:13.720Z 10 200-219 48 48 48 81 81 81 F F F
55812024-04-20T01:08:13.720Z 11 220-239 50 50 50 85 85 85 F F F
55822024-04-20T01:08:13.720Z 12 240-259 46 46 46 79 79 79 F F F
55832024-04-20T01:08:13.720Z 13 260-279 49 49 49 84 84 84 F F F
55842024-04-20T01:08:13.720Z 14 280-299 49 49 49 84 84 84 F F F
55852024-04-20T01:08:13.720Z 15 300-319 50 50 50 86 86 86 F F F
55862024-04-20T01:08:13.720Z 16 320-339 44 44 44 74 74 74 F F F
55872024-04-20T01:08:13.720Z 17 340-359 42 42 42 72 72 72 F F F
55882024-04-20T01:08:13.721Z 18 360-379 48 48 48 81 81 81 F F F
55892024-04-20T01:08:13.721Z 19 380-399 44 44 44 74 74 74 F F F
55902024-04-20T01:08:13.721Z 20 400-419 50 50 50 85 85 85 F F F
55912024-04-20T01:08:13.721Z 21 420-439 50 50 50 85 85 85 F F F
55922024-04-20T01:08:13.721Z 22 440-459 46 46 46 79 79 79 F F F
55932024-04-20T01:08:13.721Z 23 460-479 38 38 38 64 64 64 F F F
55942024-04-20T01:08:13.721Z 24 480-499 50 50 50 85 85 85 F F F
55952024-04-20T01:08:13.721Z 25 500-519 34 34 34 56 56 56 F F F
55962024-04-20T01:08:13.721Z 26 520-539 50 50 50 85 85 85 F F F
55972024-04-20T01:08:13.721Z 27 540-559 48 48 48 82 82 82 F F F
55982024-04-20T01:08:13.721Z 28 560-579 43 43 43 73 73 73 F F F
55992024-04-20T01:08:13.721Z 29 580-599 50 50 50 86 86 86 F F F
56002024-04-20T01:08:13.721ZMax gen: 51, Max flush: 87
56012024-04-20T01:08:13.721Z
56022024-04-20T01:08:13.721ZBegin loop 26
56032024-04-20T01:08:13.721ZDownstairs to restart: 2
56042024-04-20T01:08:13.738Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.245949417Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376}
56052024-04-20T01:08:13.738Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246223308Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376}
56062024-04-20T01:08:13.746Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246257838Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376}
56072024-04-20T01:08:13.746Z{"msg":"Crucible 10915b22-19f7-42f4-b612-2cbcc3c0109d has session id: 68cffba3-2a81-4500-bd53-22279713045a","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246290208Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56082024-04-20T01:08:13.746Z{"msg":"Crucible runtime is spawned
56092024-04-20T01:08:13.746ZUpstairs opts: Upstairs UUID: 10915b22-19f7-42f4-b612-2cbcc3c0109d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246435008Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56102024-04-20T01:08:13.746Z{"msg":"Crucible stats registered with UUID: 10915b22-19f7-42f4-b612-2cbcc3c0109d","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246465088Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56112024-04-20T01:08:13.746Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246515989Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376}
56122024-04-20T01:08:13.746Z{"msg":"10915b22-19f7-42f4-b612-2cbcc3c0109d active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246629569Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56132024-04-20T01:08:13.746Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246664689Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"0","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56142024-04-20T01:08:13.746Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246694159Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"0","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56152024-04-20T01:08:13.746Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246733859Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"1","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56162024-04-20T01:08:13.746Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246757849Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"1","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56172024-04-20T01:08:13.746Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246829149Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"2","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56182024-04-20T01:08:13.746Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246866569Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"2","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56192024-04-20T01:08:13.746Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.246949219Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56202024-04-20T01:08:13.746Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.24703046Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56212024-04-20T01:08:13.746Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.24707111Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56222024-04-20T01:08:13.746Z{"msg":"ds_connection connected from Ok(127.0.0.1:49909)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.24711161Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56232024-04-20T01:08:13.746Z{"msg":"ds_connection connected from Ok(127.0.0.1:53151)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.24713897Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56242024-04-20T01:08:13.746Z{"msg":"ds_connection connected from Ok(127.0.0.1:40686)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.24717737Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56252024-04-20T01:08:13.746Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.247710521Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"1","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56262024-04-20T01:08:13.747Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.247750481Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"1","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56272024-04-20T01:08:13.747Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.247766241Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"0","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56282024-04-20T01:08:13.747Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.247781451Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"0","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56292024-04-20T01:08:13.747Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.247852492Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"2","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56302024-04-20T01:08:13.747Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.247882332Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"2","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56312024-04-20T01:08:13.747Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.248267202Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"1","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56322024-04-20T01:08:13.747Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.248305382Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"0","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56332024-04-20T01:08:13.747Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.248635973Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"2","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56342024-04-20T01:08:13.747Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.248690103Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"1","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56352024-04-20T01:08:13.747Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.248714313Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"1","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56362024-04-20T01:08:13.747Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.248750334Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56372024-04-20T01:08:13.747Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.248870314Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"0","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56382024-04-20T01:08:13.747Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.248903824Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"0","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56392024-04-20T01:08:13.747Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.248920194Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56402024-04-20T01:08:13.747Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249015494Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"2","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56412024-04-20T01:08:13.747Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249046884Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"2","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56422024-04-20T01:08:13.747Z{"msg":"[0]R flush_numbers[0..12]: [73, 80, 85, 87, 85, 83, 79, 87, 84, 82, 81, 85]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249062094Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56432024-04-20T01:08:13.747Z{"msg":"[0]R generation[0..12]: [43, 47, 50, 51, 50, 48, 46, 51, 49, 48, 48, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249076244Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56442024-04-20T01:08:13.747Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249091274Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56452024-04-20T01:08:13.747Z{"msg":"[1]R flush_numbers[0..12]: [73, 80, 85, 87, 85, 83, 79, 87, 84, 82, 81, 85]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249112424Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56462024-04-20T01:08:13.747Z{"msg":"[1]R generation[0..12]: [43, 47, 50, 51, 50, 48, 46, 51, 49, 48, 48, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249127484Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56472024-04-20T01:08:13.747Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249163264Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56482024-04-20T01:08:13.747ZWait for a query_work_queue command to finish before sending IO
56492024-04-20T01:08:13.748Z{"msg":"[2]R flush_numbers[0..12]: [73, 80, 85, 87, 85, 83, 79, 87, 84, 82, 81, 85]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249202224Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56502024-04-20T01:08:13.748Z{"msg":"[2]R generation[0..12]: [43, 47, 50, 51, 50, 48, 46, 51, 49, 48, 48, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249240935Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56512024-04-20T01:08:13.748Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249257915Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56522024-04-20T01:08:13.748Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249274125Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56532024-04-20T01:08:13.748Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249316845Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56542024-04-20T01:08:13.748Z{"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249353835Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56552024-04-20T01:08:13.748Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249380445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"":"downstairs","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56562024-04-20T01:08:13.748Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249412405Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56572024-04-20T01:08:13.748Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249434385Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"0","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56582024-04-20T01:08:13.748Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249464825Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"1","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56592024-04-20T01:08:13.748Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249498285Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"client":"2","":"downstairs","session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56602024-04-20T01:08:13.748Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56612024-04-20T01:08:13.748Z30,"time":"2024-04-20T01:08:12.249537225Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56622024-04-20T01:08:13.748Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249563155Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56632024-04-20T01:08:13.748Z{"msg":"10915b22-19f7-42f4-b612-2cbcc3c0109d is now active with session: 68cffba3-2a81-4500-bd53-22279713045a","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249586385Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56642024-04-20T01:08:13.748Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249598655Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376,"session_id":"68cffba3-2a81-4500-bd53-22279713045a"}
56652024-04-20T01:08:13.748Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:12.249617575Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376}
56662024-04-20T01:08:13.752ZLoading write count information from file "/tmp/test_repair_verify.data"
56672024-04-20T01:08:13.752ZRun Repair workload
56682024-04-20T01:08:13.752Z01/30 Write block 294 len 16384 data: 5 3 3 3
56692024-04-20T01:08:13.752Z02/30 Write block 181 len 24576 data: 8 6 5 4 4 4
56702024-04-20T01:08:13.752Z03/30 Read block 200 len 20480
56712024-04-20T01:08:13.752Z04/30 Read block 371 len 24576
56722024-04-20T01:08:13.752Z05/30 Read block 277 len 8192
56732024-04-20T01:08:13.752Z06/30 Read block 135 len 32768
56742024-04-20T01:08:13.753Z07/30 Write block 531 len 20480 data: 4 3 3 4 3
56752024-04-20T01:08:13.753Z08/30 Write block 190 len 40960 data: 3 4 2 2 2 4 4 5 5 5
56762024-04-20T01:08:13.753Z09/30 Read block 465 len 12288
56772024-04-20T01:08:13.753Z10/30 Read block 38 len 4096
56782024-04-20T01:08:13.753Z11/30 Read block 72 len 40960
56792024-04-20T01:08:13.756Z12/30 Write block 58 len 36864 data: 5 5 5 5 6 5 6 6 8
56802024-04-20T01:08:13.756Z13/30 Flush
56812024-04-20T01:08:13.760Z14/30 Write block 64 len 24576 data: 7 7 9 6 6 7
56822024-04-20T01:08:13.760Z15/30 Write block 469 len 20480 data: 5 5 4 5 6
56832024-04-20T01:08:13.760Z16/30 Write block 330 len 12288 data: 8 7 7
56842024-04-20T01:08:13.760Z17/30 Read block 112 len 4096
56852024-04-20T01:08:13.760Z18/30 Flush
56862024-04-20T01:08:13.763Z19/30 Write block 280 len 8192 data: 4 4
56872024-04-20T01:08:13.763Z20/30 Read block 461 len 28672
56882024-04-20T01:08:13.767Z21/30 Write block 340 len 24576 data: 6 6 4 5 4 5
56892024-04-20T01:08:13.767Z22/30 Write block 577 len 24576 data: 4 4 6 6 6 7
56902024-04-20T01:08:13.767Z23/30 Write block 467 len 20480 data: 4 4 6 6 5
56912024-04-20T01:08:13.767Z24/30 Write block 165 len 4096 data: 4
56922024-04-20T01:08:13.770Z25/30 Write block 186 len 36864 data: 5 4 4 3 4 5 3 3 3
56932024-04-20T01:08:13.770Z26/30 Read block 493 len 32768
56942024-04-20T01:08:13.770Z27/30 Write block 550 len 32768 data: 6 6 5 4 4 4 4 5
56952024-04-20T01:08:13.774Z28/30 Read block 461 len 20480
56962024-04-20T01:08:13.774Z29/30 Write block 107 len 40960 data: 5 4 4 4 4 4 4 4 4 3
56972024-04-20T01:08:13.774Z30/30 Read block 290 len 28672
56982024-04-20T01:08:13.780Z----------------------------------------------------------------
56992024-04-20T01:08:13.780Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
57002024-04-20T01:08:13.780ZStates: Active Active Active
57012024-04-20T01:08:13.780ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
57022024-04-20T01:08:13.781Z 1 Acked 1000 Write 4 Done Done Sent false
57032024-04-20T01:08:13.781Z 2 Acked 1001 Write 6 Done Done Sent false
57042024-04-20T01:08:13.781Z 3 Acked 1002 Read 5 Done Done Sent false
57052024-04-20T01:08:13.781Z 4 Acked 1003 Read 6 Done Done Sent false
57062024-04-20T01:08:13.781Z 5 Acked 1004 Read 2 Done Done Sent false
57072024-04-20T01:08:13.781Z 6 Acked 1005 Read 8 Done Done Sent false
57082024-04-20T01:08:13.781Z 7 Acked 1006 Write 5 Done Done Sent false
57092024-04-20T01:08:13.781Z 8 Acked 1007 Write 10 Done Done Sent false
57102024-04-20T01:08:13.781Z 9 Acked 1008 Read 3 Done Done Sent false
57112024-04-20T01:08:13.781Z 10 Acked 1009 Read 1 Done Done Sent false
57122024-04-20T01:08:13.781Z 11 Acked 1010 Read 10 Done Done Sent false
57132024-04-20T01:08:13.781Z 12 Acked 1011 Write 9 Done Done Sent false
57142024-04-20T01:08:13.781Z 13 Acked 1012 Flush 0 Done Done Sent false
57152024-04-20T01:08:13.781Z 14 Acked 1013 Write 6 Done Done Sent false
57162024-04-20T01:08:13.781Z 15 Acked 1014 Write 5 Done Done Sent false
57172024-04-20T01:08:13.781Z 16 Acked 1015 Write 3 Done Done Sent false
57182024-04-20T01:08:13.781Z 17 Acked 1016 Read 1 Done Done Sent false
57192024-04-20T01:08:13.781Z 18 Acked 1017 Flush 0 Done Done Sent false
57202024-04-20T01:08:13.781Z 19 Acked 1018 Write 2 Done Done Sent false
57212024-04-20T01:08:13.781Z 20 Acked 1019 Read 7 Done Done Sent false
57222024-04-20T01:08:13.781Z 21 Acked 1020 Write 6 Done Done Sent false
57232024-04-20T01:08:13.781Z 22 Acked 1021 Write 6 Done Done Sent false
57242024-04-20T01:08:13.781Z 23 Acked 1022 Write 5 Done Done Sent false
57252024-04-20T01:08:13.782Z 24 Acked 1023 Write 1 Done Done Sent false
57262024-04-20T01:08:13.782Z 25 Acked 1024 Write 9 Done Done Sent false
57272024-04-20T01:08:13.782Z 26 Acked 1025 Read 8 Done Done Sent false
57282024-04-20T01:08:13.782Z 27 Acked 1026 Write 8 Done Done Sent false
57292024-04-20T01:08:13.782Z 28 Acked 1027 Read 5 Done Done Sent false
57302024-04-20T01:08:13.782Z 29 Acked 1028 Write 10 Done Done Sent false
57312024-04-20T01:08:13.782Z 30 Acked 1029 Read 7 Done Done Sent false
57322024-04-20T01:08:13.782Z STATES DS:0 DS:1 DS:2 TOTAL
57332024-04-20T01:08:13.782Z New 0 0 0 0
57342024-04-20T01:08:13.782Z Sent 0 0 30 30
57352024-04-20T01:08:13.782Z Done 30 30 0 60
57362024-04-20T01:08:13.782Z Skipped 0 0 0 0
57372024-04-20T01:08:13.782Z Error 0 0 0 0
57382024-04-20T01:08:13.782ZLast Flush: 1017 1017 0
57392024-04-20T01:08:13.782ZDownstairs last five completed:
57402024-04-20T01:08:13.782ZUpstairs last five completed: 30 29 28 27 26
57412024-04-20T01:08:13.782Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:08:12.282730719Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1376}
57422024-04-20T01:08:13.782ZWrote out file "/tmp/test_repair_verify.data"
57432024-04-20T01:08:13.786Z
57442024-04-20T01:08:13.786ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57452024-04-20T01:08:13.798ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57462024-04-20T01:08:13.798Z 0 000-019 43 43 43 73 73 73 F F F
57472024-04-20T01:08:13.798Z 1 020-039 47 47 47 80 80 80 F F F
57482024-04-20T01:08:13.798Z 2 040-059 52 52 50 88 88 85 F F F <---
57492024-04-20T01:08:13.798Z 3 060-079 52 52 51 89 89 87 F F F <---
57502024-04-20T01:08:13.798Z 4 080-099 50 50 50 85 85 85 F F F
57512024-04-20T01:08:13.798Z 5 100-119 48 48 48 83 83 83 T T F <---
57522024-04-20T01:08:13.802Z 6 120-139 46 46 46 79 79 79 F F F
57532024-04-20T01:08:13.802Z 7 140-159 51 51 51 87 87 87 F F F
57542024-04-20T01:08:13.802Z 8 160-179 49 49 49 84 84 84 T T F <---
57552024-04-20T01:08:13.802Z 9 180-199 52 52 48 88 88 82 T T F <---
57562024-04-20T01:08:13.802Z 10 200-219 48 48 48 81 81 81 F F F
57572024-04-20T01:08:13.803Z 11 220-239 50 50 50 85 85 85 F F F
57582024-04-20T01:08:13.803Z 12 240-259 46 46 46 79 79 79 F F F
57592024-04-20T01:08:13.803Z 13 260-279 49 49 49 84 84 84 F F F
57602024-04-20T01:08:13.803Z 14 280-299 52 52 49 88 88 84 T T F <---
57612024-04-20T01:08:13.803Z 15 300-319 50 50 50 86 86 86 F F F
57622024-04-20T01:08:13.803Z 16 320-339 52 52 44 89 89 74 F F F <---
57632024-04-20T01:08:13.803Z 17 340-359 42 42 42 72 72 72 T T F <---
57642024-04-20T01:08:13.803Z 18 360-379 48 48 48 81 81 81 F F F
57652024-04-20T01:08:13.803Z 19 380-399 44 44 44 74 74 74 F F F
57662024-04-20T01:08:13.803Z 20 400-419 50 50 50 85 85 85 F F F
57672024-04-20T01:08:13.803Z 21 420-439 50 50 50 85 85 85 F F F
57682024-04-20T01:08:13.803Z 22 440-459 46 46 46 79 79 79 F F F
57692024-04-20T01:08:13.803Z 23 460-479 52 52 38 89 89 64 T T F <---
57702024-04-20T01:08:13.803Z 24 480-499 50 50 50 85 85 85 F F F
57712024-04-20T01:08:13.803Z 25 500-519 34 34 34 56 56 56 F F F
57722024-04-20T01:08:13.803Z 26 520-539 52 52 50 88 88 85 F F F <---
57732024-04-20T01:08:13.803Z 27 540-559 48 48 48 82 82 82 T T F <---
57742024-04-20T01:08:13.803Z 28 560-579 43 43 43 73 73 73 T T F <---
57752024-04-20T01:08:13.803Z 29 580-599 50 50 50 86 86 86 T T F <---
57762024-04-20T01:08:13.803ZMax gen: 52, Max flush: 89
57772024-04-20T01:08:13.803ZError: Difference in extent metadata found!
57782024-04-20T01:08:13.803ZA Difference in extent metadata is expected here
57792024-04-20T01:08:13.803ZOn loop 26
57802024-04-20T01:08:13.804Z
57812024-04-20T01:08:13.807ZVerifying data now
57822024-04-20T01:08:18.403ZNo change after verify
57832024-04-20T01:08:18.403ZLoop: 26 Downstairs dump after verify (and repair):
57842024-04-20T01:08:18.417ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57852024-04-20T01:08:18.417Z 0 000-019 43 43 43 73 73 73 F F F
57862024-04-20T01:08:18.417Z 1 020-039 47 47 47 80 80 80 F F F
57872024-04-20T01:08:18.417Z 2 040-059 52 52 52 88 88 88 F F F
57882024-04-20T01:08:18.417Z 3 060-079 52 52 52 89 89 89 F F F
57892024-04-20T01:08:18.417Z 4 080-099 50 50 50 85 85 85 F F F
57902024-04-20T01:08:18.417Z 5 100-119 53 53 53 90 90 90 F F F
57912024-04-20T01:08:18.422Z 6 120-139 46 46 46 79 79 79 F F F
57922024-04-20T01:08:18.422Z 7 140-159 51 51 51 87 87 87 F F F
57932024-04-20T01:08:18.422Z 8 160-179 53 53 53 90 90 90 F F F
57942024-04-20T01:08:18.422Z 9 180-199 53 53 53 90 90 90 F F F
57952024-04-20T01:08:18.422Z 10 200-219 48 48 48 81 81 81 F F F
57962024-04-20T01:08:18.422Z 11 220-239 50 50 50 85 85 85 F F F
57972024-04-20T01:08:18.422Z 12 240-259 46 46 46 79 79 79 F F F
57982024-04-20T01:08:18.422Z 13 260-279 49 49 49 84 84 84 F F F
57992024-04-20T01:08:18.422Z 14 280-299 53 53 53 90 90 90 F F F
58002024-04-20T01:08:18.422Z 15 300-319 50 50 50 86 86 86 F F F
58012024-04-20T01:08:18.422Z 16 320-339 52 52 52 89 89 89 F F F
58022024-04-20T01:08:18.423Z 17 340-359 53 53 53 90 90 90 F F F
58032024-04-20T01:08:18.423Z 18 360-379 48 48 48 81 81 81 F F F
58042024-04-20T01:08:18.423Z 19 380-399 44 44 44 74 74 74 F F F
58052024-04-20T01:08:18.423Z 20 400-419 50 50 50 85 85 85 F F F
58062024-04-20T01:08:18.423Z 21 420-439 50 50 50 85 85 85 F F F
58072024-04-20T01:08:18.423Z 22 440-459 46 46 46 79 79 79 F F F
58082024-04-20T01:08:18.423Z 23 460-479 53 53 53 90 90 90 F F F
58092024-04-20T01:08:18.423Z 24 480-499 50 50 50 85 85 85 F F F
58102024-04-20T01:08:18.423Z 25 500-519 34 34 34 56 56 56 F F F
58112024-04-20T01:08:18.423Z 26 520-539 52 52 52 88 88 88 F F F
58122024-04-20T01:08:18.423Z 27 540-559 53 53 53 90 90 90 F F F
58132024-04-20T01:08:18.423Z 28 560-579 53 53 53 90 90 90 F F F
58142024-04-20T01:08:18.423Z 29 580-599 53 53 53 90 90 90 F F F
58152024-04-20T01:08:18.423ZMax gen: 53, Max flush: 90
58162024-04-20T01:08:18.423Z
58172024-04-20T01:08:18.423ZBegin loop 27
58182024-04-20T01:08:18.423ZDownstairs to restart: 0
58192024-04-20T01:08:18.447Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.94644686Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384}
58202024-04-20T01:08:18.457Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.946758081Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384}
58212024-04-20T01:08:18.457Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
58222024-04-20T01:08:18.457Z,"time":"2024-04-20T01:08:16.946834721Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384}
58232024-04-20T01:08:18.458Z{"msg":"Crucible 916d7af6-7873-40c7-992b-27cb5873e598 has session id: 5ccc0786-6621-4d76-be0b-d6af757a53d7","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.946877251Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58242024-04-20T01:08:18.458Z{"msg":"Upstairs opts: Upstairs UUID: 916d7af6-7873-40c7-992b-27cb5873e598, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.946941141Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58252024-04-20T01:08:18.458Z{"msg":"Crucible stats registered with UUID: 916d7af6-7873-40c7-992b-27cb5873e598","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.946987841Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58262024-04-20T01:08:18.458Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947017912Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384}
58272024-04-20T01:08:18.458Z{"msg":"916d7af6-7873-40c7-992b-27cb5873e598 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947056422Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58282024-04-20T01:08:18.458Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947084472Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"0","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58292024-04-20T01:08:18.458Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947116252Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"0","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58302024-04-20T01:08:18.458Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947165392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"1","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58312024-04-20T01:08:18.458Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947212422Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"1","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58322024-04-20T01:08:18.458Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947269362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"2","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58332024-04-20T01:08:18.458Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947319212Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"2","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58342024-04-20T01:08:18.459Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947351192Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58352024-04-20T01:08:18.459Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947380832Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58362024-04-20T01:08:18.459Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947425853Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58372024-04-20T01:08:18.459Z{"msg":"ds_connection connected from Ok(127.0.0.1:44451)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947476082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58382024-04-20T01:08:18.459Z{"msg":"ds_connection connected from Ok(127.0.0.1:36524)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947526743Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58392024-04-20T01:08:18.459Z{"msg":"ds_connection connected from Ok(127.0.0.1:54765)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.947574063Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58402024-04-20T01:08:18.459Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.948211854Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"2","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58412024-04-20T01:08:18.459Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.948245524Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"2","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58422024-04-20T01:08:18.459Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.948261024Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"1","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58432024-04-20T01:08:18.459Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.948297024Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"1","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58442024-04-20T01:08:18.459Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.948683245Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"0","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58452024-04-20T01:08:18.459Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.948761685Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"0","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58462024-04-20T01:08:18.460Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.948864886Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"1","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58472024-04-20T01:08:18.460Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.948964486Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"2","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58482024-04-20T01:08:18.460Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949260657Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"0","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58492024-04-20T01:08:18.460Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949442567Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"1","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58502024-04-20T01:08:18.460Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949481177Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"1","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58512024-04-20T01:08:18.460Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949506747Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58522024-04-20T01:08:18.460Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949529477Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"2","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58532024-04-20T01:08:18.460Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949552397Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"2","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58542024-04-20T01:08:18.460Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949576257Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58552024-04-20T01:08:18.460Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949748458Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"0","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58562024-04-20T01:08:18.460Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949784128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"0","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58572024-04-20T01:08:18.460Z{"msg":"[0]R flush_numbers[0..12]: [73, 80, 88, 89, 85, 90, 79, 87, 90, 90, 81, 85]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949800418Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58582024-04-20T01:08:18.461Z{"msg":"[0]R generation[0..12]: [43, 47, 52, 52, 50, 53, 46, 51, 53, 53, 48, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949814938Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58592024-04-20T01:08:18.461Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949837088Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58602024-04-20T01:08:18.461Z{"msg":"[1]R flush_numbers[0..12]: [73, 80, 88, 89, 85, 90, 79, 87, 90, 90, 81, 85]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949851108Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58612024-04-20T01:08:18.461Z{"msg":"[1]R generation[0..12]: [43, 47, 52, 52, 50, 53, 46, 51, 53, 53, 48, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949864718Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58622024-04-20T01:08:18.461Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949879378Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58632024-04-20T01:08:18.461Z{"msg":"[2]R flush_numbers[0..12]: [73, 80, 88, 89, 85, 90, 79, 87, 90, 90, 81, 85]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949893208Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58642024-04-20T01:08:18.461Z{"msg":"[2]R generation[0..12]: [43, 47, 52, 52, 50, 53, 46, 51, 53, 53, 48, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949914238Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58652024-04-20T01:08:18.461Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
58662024-04-20T01:08:18.461Z,"time":"2024-04-20T01:08:16.949928438Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58672024-04-20T01:08:18.461Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949942678Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58682024-04-20T01:08:18.461Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949963658Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58692024-04-20T01:08:18.461Z{"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949977088Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58702024-04-20T01:08:18.462Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.949990198Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"":"downstairs","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58712024-04-20T01:08:18.462Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.950010668Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58722024-04-20T01:08:18.462Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.950023238Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"0","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58732024-04-20T01:08:18.462Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.950037138Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"1","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58742024-04-20T01:08:18.462Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.950050908Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"client":"2","":"downstairs","session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58752024-04-20T01:08:18.462Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.950064108Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58762024-04-20T01:08:18.462Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.950076618Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58772024-04-20T01:08:18.462Z{"msg":"916d7af6-7873-40c7-992b-27cb5873e598 is now active with session: 5ccc0786-6621-4d76-be0b-d6af757a53d7","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.950101079Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58782024-04-20T01:08:18.462Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.950116979Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384,"session_id":"5ccc0786-6621-4d76-be0b-d6af757a53d7"}
58792024-04-20T01:08:18.462Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:16.950129388Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384}
58802024-04-20T01:08:18.462ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58812024-04-20T01:08:18.462ZLoading write count information from file "/tmp/test_repair_verify.data"
58822024-04-20T01:08:18.463ZRun Repair workload
58832024-04-20T01:08:18.463Z01/30 Write block 455 len 16384 data: 9 9 10 9
58842024-04-20T01:08:18.463Z02/30 Write block 1 len 28672 data: 3 3 3 3 2 2 2
58852024-04-20T01:08:18.463Z03/30 Write block 550 len 8192 data: 7 7
58862024-04-20T01:08:18.463Z04/30 Write block 531 len 8192 data: 5 4
58872024-04-20T01:08:18.463Z05/30 Read block 479 len 16384
58882024-04-20T01:08:18.463Z06/30 Write block 471 len 16384 data: 6 6 7 8
58892024-04-20T01:08:18.463Z07/30 Write block 26 len 40960 data: 7 7 7 7 5 4 4 3 4 4
58902024-04-20T01:08:18.463Z08/30 Read block 234 len 20480
58912024-04-20T01:08:18.463Z09/30 Write block 485 len 32768 data: 6 6 5 5 6 7 6 7
58922024-04-20T01:08:18.463Z10/30 Read block 2 len 28672
58932024-04-20T01:08:18.469Z11/30 Read block 316 len 28672
58942024-04-20T01:08:18.469Z12/30 Read block 273 len 40960
58952024-04-20T01:08:18.469Z13/30 Flush
58962024-04-20T01:08:18.469Z14/30 Write block 269 len 24576 data: 2 3 3 3 4 3
58972024-04-20T01:08:18.469Z15/30 Write block 94 len 28672 data: 3 3 3 3 4 4 6
58982024-04-20T01:08:18.469Z16/30 Write block 198 len 24576 data: 6 6 3 3 4 3
58992024-04-20T01:08:18.469Z17/30 Read block 429 len 40960
59002024-04-20T01:08:18.469Z18/30 Write block 523 len 8192 data: 8 7
59012024-04-20T01:08:18.469Z19/30 Write block 571 len 8192 data: 7 5
59022024-04-20T01:08:18.469Z20/30 Write block 44 len 12288 data: 3 3 3
59032024-04-20T01:08:18.469Z21/30 Write block 154 len 20480 data: 7 9 7 6 6
59042024-04-20T01:08:18.470Z22/30 Write block 596 len 12288 data: 4 3 3
59052024-04-20T01:08:18.470Z23/30 Write block 21 len 8192 data: 4 4
59062024-04-20T01:08:18.470Z24/30 Read block 523 len 36864
59072024-04-20T01:08:18.470Z25/30 Read block 510 len 32768
59082024-04-20T01:08:18.474Z26/30 Write block 320 len 36864 data: 6 6 7 6 6 5 7 7 7
59092024-04-20T01:08:18.474Z27/30 Write block 181 len 4096 data: 9
59102024-04-20T01:08:18.475Z28/30 Read block 502 len 12288
59112024-04-20T01:08:18.475Z29/30 Read block 382 len 20480
59122024-04-20T01:08:18.475Z30/30 Read block 445 len 20480
59132024-04-20T01:08:18.475Z----------------------------------------------------------------
59142024-04-20T01:08:18.475Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
59152024-04-20T01:08:18.475ZStates: Active Active Active
59162024-04-20T01:08:18.475ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59172024-04-20T01:08:18.475Z 1 Acked 1000 Write 4 Done Done Done false
59182024-04-20T01:08:18.475Z 2 Acked 1001 Write 7 Done Done Done false
59192024-04-20T01:08:18.475Z 3 Acked 1002 Write 2 Sent Done Done false
59202024-04-20T01:08:18.475Z 4 Acked 1003 Write 2 Sent Done Done false
59212024-04-20T01:08:18.475Z 5 Acked 1004 Read 4 Sent Done Done false
59222024-04-20T01:08:18.475Z 6 Acked 1005 Write 4 Sent Done Done false
59232024-04-20T01:08:18.475Z 7 Acked 1006 Write 10 Sent Done Done false
59242024-04-20T01:08:18.475Z 8 Acked 1007 Read 5 Sent Done Done false
59252024-04-20T01:08:18.475Z 9 Acked 1008 Write 8 Sent Done Done false
59262024-04-20T01:08:18.475Z 10 Acked 1009 Read 7 Sent Done Done false
59272024-04-20T01:08:18.476Z 11 Acked 1010 Read 7 Sent Done Done false
59282024-04-20T01:08:18.476Z 12 Acked 1011 Read 10 Sent Done Done false
59292024-04-20T01:08:18.476Z 13 Acked 1012 Flush 0 Sent Done Done false
59302024-04-20T01:08:18.476Z 14 Acked 1013 Write 6 Sent Done Done false
59312024-04-20T01:08:18.476Z 15 Acked 1014 Write 7 Sent Done Done false
59322024-04-20T01:08:18.476Z 16 Acked 1015 Write 6 Sent Done Done false
59332024-04-20T01:08:18.476Z 17 Acked 1016 Read 10 Sent Done Done false
59342024-04-20T01:08:18.476Z 18 Acked 1017 Write 2 Sent Done Done false
59352024-04-20T01:08:18.476Z 19 Acked 1018 Write 2 Sent Done Done false
59362024-04-20T01:08:18.476Z 20 Acked 1019 Write 3 Sent Done Done false
59372024-04-20T01:08:18.476Z 21 Acked 1020 Write 5 Sent Done Done false
59382024-04-20T01:08:18.476Z 22 Acked 1021 Write 3 Sent Done Done false
59392024-04-20T01:08:18.476Z 23 Acked 1022 Write 2 Sent Done Done false
59402024-04-20T01:08:18.476Z 24 Acked 1023 Read 9 Sent Done Done false
59412024-04-20T01:08:18.476Z 25 Acked 1024 Read 8 Sent Done Done false
59422024-04-20T01:08:18.476Z 26 Acked 1025 Write 9 Sent Done Done false
59432024-04-20T01:08:18.476Z 27 Acked 1026 Write 1 Sent Done Done false
59442024-04-20T01:08:18.476Z 28 Acked 1027 Read 3 Sent Done Done false
59452024-04-20T01:08:18.476Z 29 Acked 1028 Read 5 Sent Done Done false
59462024-04-20T01:08:18.476Z 30 Acked 1029 Read 5 Sent Done Sent false
59472024-04-20T01:08:18.476Z STATES DS:0 DS:1 DS:2 TOTAL
59482024-04-20T01:08:18.477Z New 0 0 0 0
59492024-04-20T01:08:18.477Z Sent 28 0 1 29
59502024-04-20T01:08:18.477Z Done 2 30 29 61
59512024-04-20T01:08:18.477Z Skipped 0 0 0 0
59522024-04-20T01:08:18.477Z Error 0 0 0 0
59532024-04-20T01:08:18.477ZLast Flush: 0 1012 1012
59542024-04-20T01:08:18.477ZDownstairs last five completed:
59552024-04-20T01:08:18.477ZUpstairs last five completed: 30 29 28 27 26
59562024-04-20T01:08:18.477Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:08:16.980187045Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1384}
59572024-04-20T01:08:18.477ZWrote out file "/tmp/test_repair_verify.data"
59582024-04-20T01:08:18.484Z
59592024-04-20T01:08:18.484ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59602024-04-20T01:08:18.497ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59612024-04-20T01:08:18.498Z 0 000-019 43 54 54 73 91 91 T F F <---
59622024-04-20T01:08:18.498Z 1 020-039 47 54 54 80 91 91 F T T <---
59632024-04-20T01:08:18.498Z 2 040-059 52 52 52 88 88 88 F T T <---
59642024-04-20T01:08:18.498Z 3 060-079 52 52 52 89 89 89 F F F
59652024-04-20T01:08:18.498Z 4 080-099 50 50 50 85 85 85 F T T <---
59662024-04-20T01:08:18.498Z 5 100-119 53 53 53 90 90 90 F T T <---
59672024-04-20T01:08:18.498Z 6 120-139 46 46 46 79 79 79 F F F
59682024-04-20T01:08:18.502Z 7 140-159 51 51 51 87 87 87 F T T <---
59692024-04-20T01:08:18.502Z 8 160-179 53 53 53 90 90 90 F F F
59702024-04-20T01:08:18.502Z 9 180-199 53 53 53 90 90 90 F T T <---
59712024-04-20T01:08:18.502Z 10 200-219 48 48 48 81 81 81 F T T <---
59722024-04-20T01:08:18.502Z 11 220-239 50 50 50 85 85 85 F F F
59732024-04-20T01:08:18.502Z 12 240-259 46 46 46 79 79 79 F F F
59742024-04-20T01:08:18.502Z 13 260-279 49 49 49 84 84 84 F T T <---
59752024-04-20T01:08:18.502Z 14 280-299 53 53 53 90 90 90 F F F
59762024-04-20T01:08:18.502Z 15 300-319 50 50 50 86 86 86 F F F
59772024-04-20T01:08:18.502Z 16 320-339 52 52 52 89 89 89 F T T <---
59782024-04-20T01:08:18.502Z 17 340-359 53 53 53 90 90 90 F F F
59792024-04-20T01:08:18.502Z 18 360-379 48 48 48 81 81 81 F F F
59802024-04-20T01:08:18.502Z 19 380-399 44 44 44 74 74 74 F F F
59812024-04-20T01:08:18.502Z 20 400-419 50 50 50 85 85 85 F F F
59822024-04-20T01:08:18.502Z 21 420-439 50 50 50 85 85 85 F F F
59832024-04-20T01:08:18.502Z 22 440-459 46 54 54 79 91 91 T F F <---
59842024-04-20T01:08:18.502Z 23 460-479 53 54 54 90 91 91 F F F <---
59852024-04-20T01:08:18.502Z 24 480-499 50 54 54 85 91 91 F F F <---
59862024-04-20T01:08:18.502Z 25 500-519 34 34 34 56 56 56 F F F
59872024-04-20T01:08:18.502Z 26 520-539 52 54 54 88 91 91 F T T <---
59882024-04-20T01:08:18.502Z 27 540-559 53 54 54 90 91 91 F F F <---
59892024-04-20T01:08:18.502Z 28 560-579 53 53 53 90 90 90 F T T <---
59902024-04-20T01:08:18.502Z 29 580-599 53 53 53 90 90 90 F T T <---
59912024-04-20T01:08:18.503ZMax gen: 54, Max flush: 91
59922024-04-20T01:08:18.503ZError: Difference in extent metadata found!
59932024-04-20T01:08:18.503ZA Difference in extent metadata is expected here
59942024-04-20T01:08:18.503ZOn loop 27
59952024-04-20T01:08:18.503Z
59962024-04-20T01:08:18.506ZVerifying data now
59972024-04-20T01:08:33.271ZNo change after verify
59982024-04-20T01:08:33.271ZLoop: 27 Downstairs dump after verify (and repair):
59992024-04-20T01:08:33.291ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60002024-04-20T01:08:33.291Z 0 000-019 54 54 54 91 91 91 F F F
60012024-04-20T01:08:33.291Z 1 020-039 55 55 55 92 92 92 F F F
60022024-04-20T01:08:33.291Z 2 040-059 55 55 55 92 92 92 F F F
60032024-04-20T01:08:33.291Z 3 060-079 52 52 52 89 89 89 F F F
60042024-04-20T01:08:33.295Z 4 080-099 55 55 55 92 92 92 F F F
60052024-04-20T01:08:33.295Z 5 100-119 55 55 55 92 92 92 F F F
60062024-04-20T01:08:33.295Z 6 120-139 46 46 46 79 79 79 F F F
60072024-04-20T01:08:33.295Z 7 140-159 55 55 55 92 92 92 F F F
60082024-04-20T01:08:33.295Z 8 160-179 53 53 53 90 90 90 F F F
60092024-04-20T01:08:33.296Z 9 180-199 55 55 55 92 92 92 F F F
60102024-04-20T01:08:33.296Z 10 200-219 55 55 55 92 92 92 F F F
60112024-04-20T01:08:33.296Z 11 220-239 50 50 50 85 85 85 F F F
60122024-04-20T01:08:33.296Z 12 240-259 46 46 46 79 79 79 F F F
60132024-04-20T01:08:33.296Z 13 260-279 55 55 55 92 92 92 F F F
60142024-04-20T01:08:33.296Z 14 280-299 53 53 53 90 90 90 F F F
60152024-04-20T01:08:33.296Z 15 300-319 50 50 50 86 86 86 F F F
60162024-04-20T01:08:33.296Z 16 320-339 55 55 55 92 92 92 F F F
60172024-04-20T01:08:33.296Z 17 340-359 53 53 53 90 90 90 F F F
60182024-04-20T01:08:33.296Z 18 360-379 48 48 48 81 81 81 F F F
60192024-04-20T01:08:33.296Z 19 380-399 44 44 44 74 74 74 F F F
60202024-04-20T01:08:33.296Z 20 400-419 50 50 50 85 85 85 F F F
60212024-04-20T01:08:33.296Z 21 420-439 50 50 50 85 85 85 F F F
60222024-04-20T01:08:33.296Z 22 440-459 54 54 54 91 91 91 F F F
60232024-04-20T01:08:33.296Z 23 460-479 54 54 54 91 91 91 F F F
60242024-04-20T01:08:33.296Z 24 480-499 54 54 54 91 91 91 F F F
60252024-04-20T01:08:33.296Z 25 500-519 34 34 34 56 56 56 F F F
60262024-04-20T01:08:33.296Z 26 520-539 55 55 55 92 92 92 F F F
60272024-04-20T01:08:33.297Z 27 540-559 54 54 54 91 91 91 F F F
60282024-04-20T01:08:33.297Z 28 560-579 55 55 55 92 92 92 F F F
60292024-04-20T01:08:33.297Z 29 580-599 55 55 55 92 92 92 F F F
60302024-04-20T01:08:33.297ZMax gen: 55, Max flush: 92
60312024-04-20T01:08:33.297Z
60322024-04-20T01:08:33.297ZBegin loop 28
60332024-04-20T01:08:33.297ZDownstairs to restart: 2
60342024-04-20T01:08:33.313Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815116398Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392}
60352024-04-20T01:08:33.313Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815324818Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392}
60362024-04-20T01:08:33.313Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815341629Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392}
60372024-04-20T01:08:33.313Z{"msg":"Crucible aa16625e-5d80-4b55-a8ff-b947b12534f9 has session id: 5a16a9e8-5b88-428a-9802-47f27e4c2aeb","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815363999Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60382024-04-20T01:08:33.321Z{"msg":"Upstairs opts: Upstairs UUID: aa16625e-5d80-4b55-a8ff-b947b12534f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","Crucible runtime is spawned
60392024-04-20T01:08:33.321Zlevel":30,"time":"2024-04-20T01:08:31.815419599Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60402024-04-20T01:08:33.321Z{"msg":"Crucible stats registered with UUID: aa16625e-5d80-4b55-a8ff-b947b12534f9","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815458269Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60412024-04-20T01:08:33.321Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815484559Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392}
60422024-04-20T01:08:33.321Z{"msg":"aa16625e-5d80-4b55-a8ff-b947b12534f9 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815620439Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60432024-04-20T01:08:33.321Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815646149Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"0","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60442024-04-20T01:08:33.321Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815661319Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"0","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60452024-04-20T01:08:33.322Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815675139Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"1","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60462024-04-20T01:08:33.322Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815705149Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"1","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60472024-04-20T01:08:33.322Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815725499Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"2","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60482024-04-20T01:08:33.322Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815749739Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"2","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60492024-04-20T01:08:33.322Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.815771599Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60502024-04-20T01:08:33.322Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.81580081Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60512024-04-20T01:08:33.322Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.81581753Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60522024-04-20T01:08:33.322Z{"msg":"ds_connection connected from Ok(127.0.0.1:41129)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.81604023Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60532024-04-20T01:08:33.322Z{"msg":"ds_connection connected from Ok(127.0.0.1:35531)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.81607907Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60542024-04-20T01:08:33.322Z{"msg":"ds_connection connected from Ok(127.0.0.1:39712)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.81609488Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60552024-04-20T01:08:33.322Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.816692442Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"0","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60562024-04-20T01:08:33.322Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.816759462Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"0","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60572024-04-20T01:08:33.322Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.816818332Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"1","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60582024-04-20T01:08:33.322Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.816880072Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"1","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60592024-04-20T01:08:33.322Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.817273563Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"1","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60602024-04-20T01:08:33.322Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.817312363Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"0","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60612024-04-20T01:08:33.322Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.817423843Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"2","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60622024-04-20T01:08:33.322Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.817466223Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"2","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60632024-04-20T01:08:33.322Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.817835914Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"1","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60642024-04-20T01:08:33.323Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.817877344Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"1","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60652024-04-20T01:08:33.323Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.817892824Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60662024-04-20T01:08:33.323Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.817906314Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"0","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60672024-04-20T01:08:33.323Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.817927954Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"0","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60682024-04-20T01:08:33.323Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.817952855Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60692024-04-20T01:08:33.323Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818080795Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"2","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60702024-04-20T01:08:33.323Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818520956Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"2","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60712024-04-20T01:08:33.323Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818574046Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"2","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60722024-04-20T01:08:33.323Z{"msg":"[0]R flush_numbers[0..12]: [91, 92, 92, 89, 92, 92, 79, 92, 90, 92, 92, 85]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818609296Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60732024-04-20T01:08:33.323Z{"msg":"[0]R generation[0..12]: [54, 55, 55, 52, 55, 55, 46, 55, 53, 55, 55, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818625776Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60742024-04-20T01:08:33.323Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818648016Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60752024-04-20T01:08:33.323Z{"msg":"[1]R flush_numbers[0..12]: [91, 92, 92, 89, 92, 92, 79, 92, 90, 92, 92, 85]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818662346Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60762024-04-20T01:08:33.323Z{"msg":"[1]R generation[0..12]: [54, 55, 55, 52, 55, 55, 46, 55, 53, 55, 55, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818683366Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60772024-04-20T01:08:33.323Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818698696Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
60782024-04-20T01:08:33.323Z5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60792024-04-20T01:08:33.323Z{"msg":"[2]R flush_numbers[0..12]: [91, 92, 92, 89, 92, 92, 79, 92, 90, 92, 92, 85]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818715086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60802024-04-20T01:08:33.323Z{"msg":"[2]R generation[0..12]: [54, 55, 55, 52, 55, 55, 46, 55, 53, 55, 55, 50]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818737076Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60812024-04-20T01:08:33.323Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818752816Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60822024-04-20T01:08:33.323Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818773746Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60832024-04-20T01:08:33.323Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818787456Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60842024-04-20T01:08:33.323Z{"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818801066Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60852024-04-20T01:08:33.323Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818814696Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"":"downstairs","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60862024-04-20T01:08:33.323Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818827956Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60872024-04-20T01:08:33.324Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818847326Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"0","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60882024-04-20T01:08:33.324Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818860817Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"1","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60892024-04-20T01:08:33.324Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818873826Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"client":"2","":"downstairs","session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60902024-04-20T01:08:33.324Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818894186Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60912024-04-20T01:08:33.324Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818906826Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60922024-04-20T01:08:33.324Z{"msg":"aa16625e-5d80-4b55-a8ff-b947b12534f9 is now active with session: 5a16a9e8-5b88-428a-9802-47f27e4c2aeb","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818920277Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60932024-04-20T01:08:33.324Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818938967Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392,"session_id":"5a16a9e8-5b88-428a-9802-47f27e4c2aeb"}
60942024-04-20T01:08:33.324Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:31.818951527Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392}
60952024-04-20T01:08:33.324ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
60962024-04-20T01:08:33.324ZLoading write count information from file "/tmp/test_repair_verify.data"
60972024-04-20T01:08:33.324ZRun Repair workload
60982024-04-20T01:08:33.324Z01/30 Write block 83 len 4096 data: 6
60992024-04-20T01:08:33.324Z02/30 Write block 137 len 40960 data: 6 7 6 6 5 5 5 5 5 5
61002024-04-20T01:08:33.324Z03/30 Read block 397 len 32768
61012024-04-20T01:08:33.324Z04/30 Write block 434 len 32768 data: 4 4 5 4 3 4 4 5
61022024-04-20T01:08:33.324Z05/30 Write block 101 len 4096 data: 6
61032024-04-20T01:08:33.329Z06/30 Write block 483 len 4096 data: 6
61042024-04-20T01:08:33.329Z07/30 Write block 134 len 32768 data: 8 8 7 7 8 7 7 6
61052024-04-20T01:08:33.329Z08/30 Read block 339 len 28672
61062024-04-20T01:08:33.329Z09/30 Write block 589 len 40960 data: 7 6 6 7 6 7 6 5 4 4
61072024-04-20T01:08:33.329Z10/30 Flush
61082024-04-20T01:08:33.329Z11/30 Write block 85 len 32768 data: 5 4 4 4 5 6 3 3
61092024-04-20T01:08:33.329Z12/30 Write block 330 len 4096 data: 9
61102024-04-20T01:08:33.329Z13/30 Read block 61 len 40960
61112024-04-20T01:08:33.329Z14/30 Write block 477 len 40960 data: 8 7 4 4 7 7 7 6 7 7
61122024-04-20T01:08:33.329Z15/30 Read block 275 len 16384
61132024-04-20T01:08:33.334Z16/30 Read block 535 len 40960
61142024-04-20T01:08:33.334Z17/30 Read block 397 len 28672
61152024-04-20T01:08:33.334Z18/30 Flush
61162024-04-20T01:08:33.338Z19/30 Write block 401 len 24576 data: 3 5 5 4 5 5
61172024-04-20T01:08:33.338Z20/30 Read block 19 len 12288
61182024-04-20T01:08:33.338Z21/30 Flush
61192024-04-20T01:08:33.338Z22/30 Write block 222 len 20480 data: 8 7 7 7 9
61202024-04-20T01:08:33.338Z23/30 Read block 590 len 24576
61212024-04-20T01:08:33.342Z24/30 Read block 32 len 40960
61222024-04-20T01:08:33.342Z25/30 Read block 275 len 24576
61232024-04-20T01:08:33.342Z26/30 Write block 228 len 4096 data: 6
61242024-04-20T01:08:33.342Z27/30 Write block 307 len 20480 data: 3 3 2 2 2
61252024-04-20T01:08:33.346Z28/30 Write block 78 len 32768 data: 4 4 4 5 5 7 6 6
61262024-04-20T01:08:33.346Z29/30 Read block 474 len 8192
61272024-04-20T01:08:33.346Z30/30 Read block 382 len 24576
61282024-04-20T01:08:33.346Z----------------------------------------------------------------
61292024-04-20T01:08:33.346Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
61302024-04-20T01:08:33.346ZStates: Active Active Active
61312024-04-20T01:08:33.346ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61322024-04-20T01:08:33.346Z 1 Acked 1000 Write 1 Done Done Done false
61332024-04-20T01:08:33.346Z 2 Acked 1001 Write 10 Done Done Done false
61342024-04-20T01:08:33.346Z 3 Acked 1002 Read 8 Done Done Done false
61352024-04-20T01:08:33.346Z 4 Acked 1003 Write 8 Done Done Done false
61362024-04-20T01:08:33.346Z 5 Acked 1004 Write 1 Done Done Done false
61372024-04-20T01:08:33.346Z 6 Acked 1005 Write 1 Done Done Sent false
61382024-04-20T01:08:33.346Z 7 Acked 1006 Write 8 Done Done Sent false
61392024-04-20T01:08:33.347Z 8 Acked 1007 Read 7 Done Done Sent false
61402024-04-20T01:08:33.347Z 9 Acked 1008 Write 10 Done Done Sent false
61412024-04-20T01:08:33.347Z 10 Acked 1009 Flush 0 Done Done Sent false
61422024-04-20T01:08:33.347Z 11 Acked 1010 Write 8 Done Done Sent false
61432024-04-20T01:08:33.347Z 12 Acked 1011 Write 1 Done Done Sent false
61442024-04-20T01:08:33.347Z 13 Acked 1012 Read 10 Done Done Sent false
61452024-04-20T01:08:33.347Z 14 Acked 1013 Write 10 Done Done Sent false
61462024-04-20T01:08:33.347Z 15 Acked 1014 Read 4 Done Done Sent false
61472024-04-20T01:08:33.347Z 16 Acked 1015 Read 10 Done Done Sent false
61482024-04-20T01:08:33.347Z 17 Acked 1016 Read 7 Done Done Sent false
61492024-04-20T01:08:33.347Z 18 Acked 1017 Flush 0 Done Done Sent false
61502024-04-20T01:08:33.347Z 19 Acked 1018 Write 6 Done Done Sent false
61512024-04-20T01:08:33.347Z 20 Acked 1019 Read 3 Done Done Sent false
61522024-04-20T01:08:33.347Z 21 Acked 1020 Flush 0 Done Done Sent false
61532024-04-20T01:08:33.347Z 22 Acked 1021 Write 5 Done Done Sent false
61542024-04-20T01:08:33.347Z 23 Acked 1022 Read 6 Done Done Sent false
61552024-04-20T01:08:33.347Z 24 Acked 1023 Read 10 Done Done Sent false
61562024-04-20T01:08:33.347Z 25 Acked 1024 Read 6 Done Done Sent false
61572024-04-20T01:08:33.347Z 26 Acked 1025 Write 1 Done Done Sent false
61582024-04-20T01:08:33.347Z 27 Acked 1026 Write 5 Done Done Sent false
61592024-04-20T01:08:33.347Z 28 Acked 1027 Write 8 Done Done Sent false
61602024-04-20T01:08:33.347Z 29 Acked 1028 Read 2 Done Done Sent false
61612024-04-20T01:08:33.347Z 30 Acked 1029 Read 6 Done Sent Sent false
61622024-04-20T01:08:33.348Z STATES DS:0 DS:1 DS:2 TOTAL
61632024-04-20T01:08:33.348Z New 0 0 0 0
61642024-04-20T01:08:33.348Z Sent 0 1 25 26
61652024-04-20T01:08:33.348Z Done 30 29 5 64
61662024-04-20T01:08:33.348Z Skipped 0 0 0 0
61672024-04-20T01:08:33.348Z Error 0 0 0 0
61682024-04-20T01:08:33.348ZLast Flush: 1020 1020 0
61692024-04-20T01:08:33.348ZDownstairs last five completed:
61702024-04-20T01:08:33.348ZUpstairs last five completed: 30 29 28 27 26
61712024-04-20T01:08:33.348Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:08:31.853078583Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1392}
61722024-04-20T01:08:33.348ZWrote out file "/tmp/test_repair_verify.data"
61732024-04-20T01:08:33.352Z
61742024-04-20T01:08:33.358ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61752024-04-20T01:08:33.369ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61762024-04-20T01:08:33.369Z 0 000-019 54 54 54 91 91 91 F F F
61772024-04-20T01:08:33.375Z 1 020-039 55 55 55 92 92 92 F F F
61782024-04-20T01:08:33.375Z 2 040-059 55 55 55 92 92 92 F F F
61792024-04-20T01:08:33.375Z 3 060-079 52 52 52 89 89 89 T T F <---
61802024-04-20T01:08:33.375Z 4 080-099 56 56 55 94 94 92 T T T <---
61812024-04-20T01:08:33.375Z 5 100-119 56 56 55 93 93 92 F F T <---
61822024-04-20T01:08:33.375Z 6 120-139 56 56 46 93 93 79 F F T <---
61832024-04-20T01:08:33.375Z 7 140-159 56 56 55 93 93 92 F F T <---
61842024-04-20T01:08:33.375Z 8 160-179 53 53 53 90 90 90 F F F
61852024-04-20T01:08:33.375Z 9 180-199 55 55 55 92 92 92 F F F
61862024-04-20T01:08:33.376Z 10 200-219 55 55 55 92 92 92 F F F
61872024-04-20T01:08:33.376Z 11 220-239 50 50 50 85 85 85 T T F <---
61882024-04-20T01:08:33.376Z 12 240-259 46 46 46 79 79 79 F F F
61892024-04-20T01:08:33.376Z 13 260-279 55 55 55 92 92 92 F F F
61902024-04-20T01:08:33.376Z 14 280-299 53 53 53 90 90 90 F F F
61912024-04-20T01:08:33.376Z 15 300-319 50 50 50 86 86 86 T T F <---
61922024-04-20T01:08:33.376Z 16 320-339 56 56 55 94 94 92 F F F <---
61932024-04-20T01:08:33.376Z 17 340-359 53 53 53 90 90 90 F F F
61942024-04-20T01:08:33.376Z 18 360-379 48 48 48 81 81 81 F F F
61952024-04-20T01:08:33.376Z 19 380-399 44 44 44 74 74 74 F F F
61962024-04-20T01:08:33.376Z 20 400-419 56 56 50 95 95 85 F F F <---
61972024-04-20T01:08:33.376Z 21 420-439 56 56 50 93 93 85 F F T <---
61982024-04-20T01:08:33.376Z 22 440-459 56 56 54 93 93 91 F F T <---
61992024-04-20T01:08:33.376Z 23 460-479 56 56 54 94 94 91 F F F <---
62002024-04-20T01:08:33.377Z 24 480-499 56 56 54 94 94 91 F F F <---
62012024-04-20T01:08:33.377Z 25 500-519 34 34 34 56 56 56 F F F
62022024-04-20T01:08:33.377Z 26 520-539 55 55 55 92 92 92 F F F
62032024-04-20T01:08:33.377Z 27 540-559 54 54 54 91 91 91 F F F
62042024-04-20T01:08:33.377Z 28 560-579 55 55 55 92 92 92 F F F
62052024-04-20T01:08:33.377Z 29 580-599 56 56 55 93 93 92 F F F <---
62062024-04-20T01:08:33.377ZMax gen: 56, Max flush: 95
62072024-04-20T01:08:33.377ZError: Difference in extent metadata found!
62082024-04-20T01:08:33.377ZA Difference in extent metadata is expected here
62092024-04-20T01:08:33.377ZOn loop 28
62102024-04-20T01:08:33.377Z
62112024-04-20T01:08:33.382ZVerifying data now
62122024-04-20T01:08:48.009ZNo change after verify
62132024-04-20T01:08:48.009ZLoop: 28 Downstairs dump after verify (and repair):
62142024-04-20T01:08:48.023ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62152024-04-20T01:08:48.023Z 0 000-019 54 54 54 91 91 91 F F F
62162024-04-20T01:08:48.023Z 1 020-039 55 55 55 92 92 92 F F F
62172024-04-20T01:08:48.023Z 2 040-059 55 55 55 92 92 92 F F F
62182024-04-20T01:08:48.023Z 3 060-079 57 57 57 96 96 96 F F F
62192024-04-20T01:08:48.023Z 4 080-099 57 57 57 96 96 96 F F F
62202024-04-20T01:08:48.027Z 5 100-119 56 56 56 93 93 93 F F F
62212024-04-20T01:08:48.027Z 6 120-139 56 56 56 93 93 93 F F F
62222024-04-20T01:08:48.027Z 7 140-159 56 56 56 93 93 93 F F F
62232024-04-20T01:08:48.027Z 8 160-179 53 53 53 90 90 90 F F F
62242024-04-20T01:08:48.027Z 9 180-199 55 55 55 92 92 92 F F F
62252024-04-20T01:08:48.027Z 10 200-219 55 55 55 92 92 92 F F F
62262024-04-20T01:08:48.027Z 11 220-239 57 57 57 96 96 96 F F F
62272024-04-20T01:08:48.027Z 12 240-259 46 46 46 79 79 79 F F F
62282024-04-20T01:08:48.027Z 13 260-279 55 55 55 92 92 92 F F F
62292024-04-20T01:08:48.027Z 14 280-299 53 53 53 90 90 90 F F F
62302024-04-20T01:08:48.027Z 15 300-319 57 57 57 96 96 96 F F F
62312024-04-20T01:08:48.027Z 16 320-339 56 56 56 94 94 94 F F F
62322024-04-20T01:08:48.027Z 17 340-359 53 53 53 90 90 90 F F F
62332024-04-20T01:08:48.027Z 18 360-379 48 48 48 81 81 81 F F F
62342024-04-20T01:08:48.027Z 19 380-399 44 44 44 74 74 74 F F F
62352024-04-20T01:08:48.027Z 20 400-419 56 56 56 95 95 95 F F F
62362024-04-20T01:08:48.027Z 21 420-439 56 56 56 93 93 93 F F F
62372024-04-20T01:08:48.027Z 22 440-459 56 56 56 93 93 93 F F F
62382024-04-20T01:08:48.028Z 23 460-479 56 56 56 94 94 94 F F F
62392024-04-20T01:08:48.028Z 24 480-499 56 56 56 94 94 94 F F F
62402024-04-20T01:08:48.028Z 25 500-519 34 34 34 56 56 56 F F F
62412024-04-20T01:08:48.028Z 26 520-539 55 55 55 92 92 92 F F F
62422024-04-20T01:08:48.028Z 27 540-559 54 54 54 91 91 91 F F F
62432024-04-20T01:08:48.028Z 28 560-579 55 55 55 92 92 92 F F F
62442024-04-20T01:08:48.028Z 29 580-599 56 56 56 93 93 93 F F F
62452024-04-20T01:08:48.028ZMax gen: 57, Max flush: 96
62462024-04-20T01:08:48.028Z
62472024-04-20T01:08:48.028ZBegin loop 29
62482024-04-20T01:08:48.028ZDownstairs to restart: 0
62492024-04-20T01:08:48.045Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.552349273Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400}
62502024-04-20T01:08:48.045Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.552614964Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400}
62512024-04-20T01:08:48.052Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.552670694Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400}
62522024-04-20T01:08:48.052Z{"msg":"Crucible 52572e67-d968-4a13-8a37-b03e6fee87e6 has session id: f0c49d42-030d-4c12-b1b6-2c28eddc6292","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.552696814Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"session_id":"Crucible runtime is spawned
62532024-04-20T01:08:48.052Zf0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62542024-04-20T01:08:48.052Z{"msg":"Upstairs opts: Upstairs UUID: 52572e67-d968-4a13-8a37-b03e6fee87e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.552756324Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62552024-04-20T01:08:48.052Z{"msg":"Crucible stats registered with UUID: 52572e67-d968-4a13-8a37-b03e6fee87e6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.552793324Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62562024-04-20T01:08:48.052Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.552815744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400}
62572024-04-20T01:08:48.052Z{"msg":"52572e67-d968-4a13-8a37-b03e6fee87e6 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.552885104Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62582024-04-20T01:08:48.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.552915055Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"0","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62592024-04-20T01:08:48.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.552947675Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"0","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62602024-04-20T01:08:48.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.552982875Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"1","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62612024-04-20T01:08:48.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.553031155Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"1","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62622024-04-20T01:08:48.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.553060995Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"2","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62632024-04-20T01:08:48.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.553087475Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"2","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62642024-04-20T01:08:48.053Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.553124705Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62652024-04-20T01:08:48.053Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.553159085Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62662024-04-20T01:08:48.053Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.553182555Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62672024-04-20T01:08:48.053Z{"msg":"ds_connection connected from Ok(127.0.0.1:63546)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.553209645Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62682024-04-20T01:08:48.053Z{"msg":"ds_connection connected from Ok(127.0.0.1:49715)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.553245875Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62692024-04-20T01:08:48.053Z{"msg":"ds_connection connected from Ok(127.0.0.1:58711)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.553329945Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62702024-04-20T01:08:48.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.554048267Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"1","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62712024-04-20T01:08:48.053Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.554098157Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"1","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62722024-04-20T01:08:48.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.554124927Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"2","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62732024-04-20T01:08:48.053Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.554146857Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"2","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62742024-04-20T01:08:48.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.554554928Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"0","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62752024-04-20T01:08:48.053Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.554617448Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"0","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62762024-04-20T01:08:48.053Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.554659668Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"1","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62772024-04-20T01:08:48.053Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.554704248Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"2","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62782024-04-20T01:08:48.053Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.55518296Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"2","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62792024-04-20T01:08:48.053Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.55523136Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"2","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62802024-04-20T01:08:48.053Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.55524936Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62812024-04-20T01:08:48.053Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.55527798Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"0","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62822024-04-20T01:08:48.053Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.55529377Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"1","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62832024-04-20T01:08:48.053Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.55530744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"1","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62842024-04-20T01:08:48.053Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.5553206Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62852024-04-20T01:08:48.053Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555714141Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"0","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62862024-04-20T01:08:48.053Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555747481Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"0","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62872024-04-20T01:08:48.054Z{"msg":"[0]R flush_numbers[0..12]: [91, 92, 92, 96, 96, 93, 93, 93, 90, 92, 92, 96]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555767521Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62882024-04-20T01:08:48.054Z{"msg":"[0]R generation[0..12]: [54, 55, 55, 57, 57, 56, 56, 56, 53, 55, 55, 57]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555782091Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62892024-04-20T01:08:48.054Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555796911Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62902024-04-20T01:08:48.054Z{"msg":"[1]R flush_numbers[0..12]: [91, 92, 92, 96, 96, 93, 93, 93, 90, 92, 92, 96]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555820641Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62912024-04-20T01:08:48.054Z{"msg":"[1]R generation[0..12]: [54, 55, 55, 57, 57, 56, 56, 56, 53, 55, 55, 57]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555835431Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62922024-04-20T01:08:48.054Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555857581Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62932024-04-20T01:08:48.054Z{"msg":"[2]R flush_numbers[0..12]: [91, 92, 92, 96, 96, 93, 93, 93, 90, 92, 92, 96]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555871791Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62942024-04-20T01:08:48.054Z{"msg":"[2]R generation[0..12]: [54, 55, 55, 57, 57, 56, 56, 56, 53, 55, 55, 57]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555892981Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62952024-04-20T01:08:48.054Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555908941Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62962024-04-20T01:08:48.054ZWait for a query_work_queue command to finish before sending IO
62972024-04-20T01:08:48.054Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555932931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62982024-04-20T01:08:48.054Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555947831Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
62992024-04-20T01:08:48.054Z{"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555968091Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
63002024-04-20T01:08:48.054Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.555981591Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"":"downstairs","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
63012024-04-20T01:08:48.054Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.556001801Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
63022024-04-20T01:08:48.054Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.556014331Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"0","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
63032024-04-20T01:08:48.054Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.556040501Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"1","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
63042024-04-20T01:08:48.054Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.556054141Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"client":"2","":"downstairs","session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
63052024-04-20T01:08:48.054Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.556069382Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
63062024-04-20T01:08:48.054Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.556081731Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
63072024-04-20T01:08:48.054Z{"msg":"52572e67-d968-4a13-8a37-b03e6fee87e6 is now active with session: f0c49d42-030d-4c12-b1b6-2c28eddc6292","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.556095502Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
63082024-04-20T01:08:48.054Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.556115122Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400,"session_id":"f0c49d42-030d-4c12-b1b6-2c28eddc6292"}
63092024-04-20T01:08:48.054Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:46.556128762Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400}
63102024-04-20T01:08:48.058ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
63112024-04-20T01:08:48.059ZLoading write count information from file "/tmp/test_repair_verify.data"
63122024-04-20T01:08:48.059ZRun Repair workload
63132024-04-20T01:08:48.059Z01/30 Write block 585 len 20480 data: 6 8 9 10 8
63142024-04-20T01:08:48.059Z02/30 Read block 558 len 24576
63152024-04-20T01:08:48.059Z03/30 Write block 429 len 20480 data: 4 4 4 4 4
63162024-04-20T01:08:48.059Z04/30 Read block 154 len 16384
63172024-04-20T01:08:48.059Z05/30 Write block 358 len 40960 data: 4 4 4 5 6 6 5 5 5 6
63182024-04-20T01:08:48.059Z06/30 Read block 545 len 8192
63192024-04-20T01:08:48.059Z07/30 Write block 434 len 36864 data: 5 5 6 5 4 5 5 6 6
63202024-04-20T01:08:48.059Z08/30 Write block 366 len 8192 data: 6 7
63212024-04-20T01:08:48.062Z09/30 Write block 549 len 16384 data: 6 8 8 6
63222024-04-20T01:08:48.062Z10/30 Write block 272 len 28672 data: 4 5 4 3 2 4 4
63232024-04-20T01:08:48.062Z11/30 Flush
63242024-04-20T01:08:48.066Z12/30 Write block 29 len 20480 data: 8 6 5 5 4
63252024-04-20T01:08:48.066Z13/30 Read block 72 len 28672
63262024-04-20T01:08:48.066Z14/30 Read block 406 len 12288
63272024-04-20T01:08:48.069Z15/30 Write block 506 len 28672 data: 5 5 5 6 6 7 8
63282024-04-20T01:08:48.070Z16/30 Read block 313 len 16384
63292024-04-20T01:08:48.070Z17/30 Write block 15 len 8192 data: 6 6
63302024-04-20T01:08:48.070Z18/30 Write block 187 len 32768 data: 5 5 4 5 6 4 4 4
63312024-04-20T01:08:48.073Z19/30 Read block 441 len 20480
63322024-04-20T01:08:48.073Z20/30 Write block 340 len 24576 data: 7 7 5 6 5 6
63332024-04-20T01:08:48.073Z21/30 Write block 476 len 8192 data: 9 9
63342024-04-20T01:08:48.073Z22/30 Read block 235 len 4096
63352024-04-20T01:08:48.073Z23/30 Flush
63362024-04-20T01:08:48.077Z24/30 Flush
63372024-04-20T01:08:48.080Z25/30 Write block 549 len 28672 data: 7 9 9 7 5 5 5
63382024-04-20T01:08:48.080Z26/30 Flush
63392024-04-20T01:08:48.084Z27/30 Write block 320 len 28672 data: 7 7 8 7 7 6 8
63402024-04-20T01:08:48.084Z28/30 Write block 583 len 36864 data: 5 6 7 9 10 11 9 7 7
63412024-04-20T01:08:48.084Z29/30 Write block 516 len 20480 data: 7 7 6 7 7
63422024-04-20T01:08:48.088Z30/30 Write block 444 len 40960 data: 6 5 5 5 5 7 6 6 7 6
63432024-04-20T01:08:48.088Z----------------------------------------------------------------
63442024-04-20T01:08:48.088Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
63452024-04-20T01:08:48.088ZStates: Active Active Active
63462024-04-20T01:08:48.088ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63472024-04-20T01:08:48.089Z 1 Acked 1000 Write 5 Done Done Done false
63482024-04-20T01:08:48.089Z 2 Acked 1001 Read 6 Done Done Done false
63492024-04-20T01:08:48.089Z 3 Acked 1002 Write 5 Done Done Done false
63502024-04-20T01:08:48.089Z 4 Acked 1003 Read 4 Done Done Done false
63512024-04-20T01:08:48.089Z 5 Acked 1004 Write 10 Done Done Done false
63522024-04-20T01:08:48.089Z 6 Acked 1005 Read 2 Done Done Done false
63532024-04-20T01:08:48.089Z 7 Acked 1006 Write 9 Done Done Done false
63542024-04-20T01:08:48.089Z 8 Acked 1007 Write 2 Sent Done Done false
63552024-04-20T01:08:48.089Z 9 Acked 1008 Write 4 Sent Done Done false
63562024-04-20T01:08:48.089Z 10 Acked 1009 Write 7 Sent Done Done false
63572024-04-20T01:08:48.089Z 11 Acked 1010 Flush 0 Sent Done Done false
63582024-04-20T01:08:48.089Z 12 Acked 1011 Write 5 Sent Done Done false
63592024-04-20T01:08:48.089Z 13 Acked 1012 Read 7 Sent Done Done false
63602024-04-20T01:08:48.089Z 14 Acked 1013 Read 3 Sent Done Done false
63612024-04-20T01:08:48.089Z 15 Acked 1014 Write 7 Sent Done Done false
63622024-04-20T01:08:48.089Z 16 Acked 1015 Read 4 Sent Done Done false
63632024-04-20T01:08:48.089Z 17 Acked 1016 Write 2 Sent Done Done false
63642024-04-20T01:08:48.089Z 18 Acked 1017 Write 8 Sent Done Done false
63652024-04-20T01:08:48.089Z 19 Acked 1018 Read 5 Sent Done Done false
63662024-04-20T01:08:48.089Z 20 Acked 1019 Write 6 Sent Done Done false
63672024-04-20T01:08:48.089Z 21 Acked 1020 Write 2 Sent Done Done false
63682024-04-20T01:08:48.089Z 22 Acked 1021 Read 1 Sent Done Done false
63692024-04-20T01:08:48.089Z 23 Acked 1022 Flush 0 Sent Done Done false
63702024-04-20T01:08:48.090Z 24 Acked 1023 Flush 0 Sent Done Done false
63712024-04-20T01:08:48.090Z 25 Acked 1024 Write 7 Sent Done Done false
63722024-04-20T01:08:48.090Z 26 Acked 1025 Flush 0 Sent Done Done false
63732024-04-20T01:08:48.090Z 27 Acked 1026 Write 7 Sent Done Done false
63742024-04-20T01:08:48.090Z 28 Acked 1027 Write 9 Sent Done Done false
63752024-04-20T01:08:48.090Z 29 Acked 1028 Write 5 Sent Done Done false
63762024-04-20T01:08:48.090Z 30 Acked 1029 Write 10 Sent Sent Sent false
63772024-04-20T01:08:48.090Z STATES DS:0 DS:1 DS:2 TOTAL
63782024-04-20T01:08:48.090Z New 0 0 0 0
63792024-04-20T01:08:48.090Z Sent 23 1 1 25
63802024-04-20T01:08:48.090Z Done 7 29 29 65
63812024-04-20T01:08:48.090Z Skipped 0 0 0 0
63822024-04-20T01:08:48.090Z Error 0 0 0 0
63832024-04-20T01:08:48.090ZLast Flush: 0 1025 1025
63842024-04-20T01:08:48.090ZDownstairs last five completed:
63852024-04-20T01:08:48.090ZUpstairs last five completed: 30 29 28 27 26
63862024-04-20T01:08:48.090Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:08:46.593579565Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1400}
63872024-04-20T01:08:48.090ZWrote out file "/tmp/test_repair_verify.data"
63882024-04-20T01:08:48.097Z
63892024-04-20T01:08:48.097ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63902024-04-20T01:08:48.110ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63912024-04-20T01:08:48.110Z 0 000-019 54 58 58 91 98 98 F F F <---
63922024-04-20T01:08:48.114Z 1 020-039 55 58 58 92 98 98 F F F <---
63932024-04-20T01:08:48.114Z 2 040-059 55 55 55 92 92 92 F F F
63942024-04-20T01:08:48.114Z 3 060-079 57 57 57 96 96 96 F F F
63952024-04-20T01:08:48.114Z 4 080-099 57 57 57 96 96 96 F F F
63962024-04-20T01:08:48.115Z 5 100-119 56 56 56 93 93 93 F F F
63972024-04-20T01:08:48.115Z 6 120-139 56 56 56 93 93 93 F F F
63982024-04-20T01:08:48.115Z 7 140-159 56 56 56 93 93 93 F F F
63992024-04-20T01:08:48.115Z 8 160-179 53 53 53 90 90 90 F F F
64002024-04-20T01:08:48.115Z 9 180-199 55 58 58 92 98 98 F F F <---
64012024-04-20T01:08:48.115Z 10 200-219 55 55 55 92 92 92 F F F
64022024-04-20T01:08:48.115Z 11 220-239 57 57 57 96 96 96 F F F
64032024-04-20T01:08:48.115Z 12 240-259 46 46 46 79 79 79 F F F
64042024-04-20T01:08:48.115Z 13 260-279 55 58 58 92 97 97 F F F <---
64052024-04-20T01:08:48.115Z 14 280-299 53 53 53 90 90 90 F F F
64062024-04-20T01:08:48.115Z 15 300-319 57 57 57 96 96 96 F F F
64072024-04-20T01:08:48.115Z 16 320-339 56 56 56 94 94 94 F T T <---
64082024-04-20T01:08:48.115Z 17 340-359 53 58 58 90 98 98 T F F <---
64092024-04-20T01:08:48.115Z 18 360-379 48 58 58 81 97 97 T F F <---
64102024-04-20T01:08:48.115Z 19 380-399 44 44 44 74 74 74 F F F
64112024-04-20T01:08:48.115Z 20 400-419 56 56 56 95 95 95 F F F
64122024-04-20T01:08:48.115Z 21 420-439 56 58 58 93 97 97 T F F <---
64132024-04-20T01:08:48.115Z 22 440-459 56 58 58 93 97 97 T T T <---
64142024-04-20T01:08:48.115Z 23 460-479 56 58 58 94 98 98 F F F <---
64152024-04-20T01:08:48.115Z 24 480-499 56 56 56 94 94 94 F F F
64162024-04-20T01:08:48.115Z 25 500-519 34 58 58 56 98 98 F T T <---
64172024-04-20T01:08:48.115Z 26 520-539 55 55 55 92 92 92 F T T <---
64182024-04-20T01:08:48.115Z 27 540-559 54 58 58 91 100 100 F F F <---
64192024-04-20T01:08:48.115Z 28 560-579 55 55 55 92 92 92 F F F
64202024-04-20T01:08:48.116Z 29 580-599 56 58 58 93 97 97 T T T <---
64212024-04-20T01:08:48.116ZMax gen: 58, Max flush: 100
64222024-04-20T01:08:48.116ZError: Difference in extent metadata found!
64232024-04-20T01:08:48.116ZA Difference in extent metadata is expected here
64242024-04-20T01:08:48.116ZOn loop 29
64252024-04-20T01:08:48.116Z
64262024-04-20T01:08:48.119ZVerifying data now
64272024-04-20T01:09:02.752ZNo change after verify
64282024-04-20T01:09:02.752ZLoop: 29 Downstairs dump after verify (and repair):
64292024-04-20T01:09:02.768ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64302024-04-20T01:09:02.768Z 0 000-019 58 58 58 98 98 98 F F F
64312024-04-20T01:09:02.768Z 1 020-039 58 58 58 98 98 98 F F F
64322024-04-20T01:09:02.772Z 2 040-059 55 55 55 92 92 92 F F F
64332024-04-20T01:09:02.772Z 3 060-079 57 57 57 96 96 96 F F F
64342024-04-20T01:09:02.772Z 4 080-099 57 57 57 96 96 96 F F F
64352024-04-20T01:09:02.772Z 5 100-119 56 56 56 93 93 93 F F F
64362024-04-20T01:09:02.772Z 6 120-139 56 56 56 93 93 93 F F F
64372024-04-20T01:09:02.772Z 7 140-159 56 56 56 93 93 93 F F F
64382024-04-20T01:09:02.772Z 8 160-179 53 53 53 90 90 90 F F F
64392024-04-20T01:09:02.772Z 9 180-199 58 58 58 98 98 98 F F F
64402024-04-20T01:09:02.772Z 10 200-219 55 55 55 92 92 92 F F F
64412024-04-20T01:09:02.772Z 11 220-239 57 57 57 96 96 96 F F F
64422024-04-20T01:09:02.772Z 12 240-259 46 46 46 79 79 79 F F F
64432024-04-20T01:09:02.772Z 13 260-279 58 58 58 97 97 97 F F F
64442024-04-20T01:09:02.773Z 14 280-299 53 53 53 90 90 90 F F F
64452024-04-20T01:09:02.773Z 15 300-319 57 57 57 96 96 96 F F F
64462024-04-20T01:09:02.773Z 16 320-339 59 59 59 101 101 101 F F F
64472024-04-20T01:09:02.773Z 17 340-359 58 58 58 98 98 98 F F F
64482024-04-20T01:09:02.773Z 18 360-379 58 58 58 97 97 97 F F F
64492024-04-20T01:09:02.773Z 19 380-399 44 44 44 74 74 74 F F F
64502024-04-20T01:09:02.773Z 20 400-419 56 56 56 95 95 95 F F F
64512024-04-20T01:09:02.773Z 21 420-439 58 58 58 97 97 97 F F F
64522024-04-20T01:09:02.773Z 22 440-459 59 59 59 101 101 101 F F F
64532024-04-20T01:09:02.773Z 23 460-479 58 58 58 98 98 98 F F F
64542024-04-20T01:09:02.773Z 24 480-499 56 56 56 94 94 94 F F F
64552024-04-20T01:09:02.773Z 25 500-519 59 59 59 101 101 101 F F F
64562024-04-20T01:09:02.773Z 26 520-539 59 59 59 101 101 101 F F F
64572024-04-20T01:09:02.773Z 27 540-559 58 58 58 100 100 100 F F F
64582024-04-20T01:09:02.773Z 28 560-579 55 55 55 92 92 92 F F F
64592024-04-20T01:09:02.773Z 29 580-599 59 59 59 101 101 101 F F F
64602024-04-20T01:09:02.773ZMax gen: 59, Max flush: 101
64612024-04-20T01:09:02.773Z
64622024-04-20T01:09:02.773ZBegin loop 30
64632024-04-20T01:09:02.773ZDownstairs to restart: 0
64642024-04-20T01:09:02.788Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295308531Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408}
64652024-04-20T01:09:02.796Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295540452Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408}
64662024-04-20T01:09:02.796Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295583392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408}
64672024-04-20T01:09:02.796Z{"msg":"Crucible d04053c1-cc40-445d-9d87-bc2713b866ab has session id: 5aef2b37-5f00-4dca-a98e-bd42c688977b","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295608452Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64682024-04-20T01:09:02.796Z{"msg":"Upstairs opts: Upstairs UUID: d04053c1-cc40-445d-9d87-bc2713b866ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false"Crucible runtime is spawned
64692024-04-20T01:09:02.796Z,"v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295693592Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64702024-04-20T01:09:02.796Z{"msg":"Crucible stats registered with UUID: d04053c1-cc40-445d-9d87-bc2713b866ab","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295733972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64712024-04-20T01:09:02.796Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295769872Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408}
64722024-04-20T01:09:02.796Z{"msg":"d04053c1-cc40-445d-9d87-bc2713b866ab active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295841972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64732024-04-20T01:09:02.796Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295877152Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"0","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64742024-04-20T01:09:02.796Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295908382Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"0","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64752024-04-20T01:09:02.796Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295934042Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"1","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64762024-04-20T01:09:02.796Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295959512Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"1","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64772024-04-20T01:09:02.796Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.295990633Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"2","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64782024-04-20T01:09:02.796Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.296006053Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"2","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64792024-04-20T01:09:02.797Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.296024202Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64802024-04-20T01:09:02.797Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.296061343Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64812024-04-20T01:09:02.797Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.296087783Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64822024-04-20T01:09:02.797Z{"msg":"ds_connection connected from Ok(127.0.0.1:42719)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.296246703Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64832024-04-20T01:09:02.797Z{"msg":"ds_connection connected from Ok(127.0.0.1:51524)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.296286143Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64842024-04-20T01:09:02.797Z{"msg":"ds_connection connected from Ok(127.0.0.1:37444)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.296315073Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64852024-04-20T01:09:02.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.297046935Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"1","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64862024-04-20T01:09:02.797Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.297075905Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"1","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64872024-04-20T01:09:02.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.297091965Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"2","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64882024-04-20T01:09:02.797Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.297133195Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"2","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64892024-04-20T01:09:02.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.297288895Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"0","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64902024-04-20T01:09:02.797Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.297329056Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"0","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64912024-04-20T01:09:02.797Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.297793746Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"1","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64922024-04-20T01:09:02.797Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.297856797Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"2","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64932024-04-20T01:09:02.797Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.297917747Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"0","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64942024-04-20T01:09:02.797Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298464488Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"1","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64952024-04-20T01:09:02.797Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298516598Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"1","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64962024-04-20T01:09:02.797Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298533748Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64972024-04-20T01:09:02.797Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298546348Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"0","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64982024-04-20T01:09:02.797Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298559778Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"0","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
64992024-04-20T01:09:02.797Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298594178Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65002024-04-20T01:09:02.797Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298608618Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"2","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65012024-04-20T01:09:02.798Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298639888Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"2","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65022024-04-20T01:09:02.798Z{"msg":"[0]R flush_numbers[0..12]: [98, 98, 92, 96, 96, 93, 93, 93, 90, 98, 92, 96]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298656468Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65032024-04-20T01:09:02.798Z{"msg":"[0]R generation[0..12]: [58, 58, 55, 57, 57, 56, 56, 56, 53, 58, 55, 57]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298687068Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65042024-04-20T01:09:02.798Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298703588Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65052024-04-20T01:09:02.798Z{"msg":"[1]R flush_numbers[0..12]: [98, 98, 92, 96, 96, 93, 93, 93, 90, 98, 92, 96]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
65062024-04-20T01:09:02.798Z,"time":"2024-04-20T01:09:01.298734359Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65072024-04-20T01:09:02.798Z{"msg":"[1]R generation[0..12]: [58, 58, 55, 57, 57, 56, 56, 56, 53, 58, 55, 57]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298750099Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65082024-04-20T01:09:02.798Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298782569Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65092024-04-20T01:09:02.798Z{"msg":"[2]R flush_numbers[0..12]: [98, 98, 92, 96, 96, 93, 93, 93, 90, 98, 92, 96]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298799269Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65102024-04-20T01:09:02.798Z{"msg":"[2]R generation[0..12]: [58, 58, 55, 57, 57, 56, 56, 56, 53, 58, 55, 57]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298823409Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65112024-04-20T01:09:02.798Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298844909Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65122024-04-20T01:09:02.798Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298872719Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65132024-04-20T01:09:02.798Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298887099Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65142024-04-20T01:09:02.798Z{"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298903409Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65152024-04-20T01:09:02.798Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298932599Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"":"downstairs","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65162024-04-20T01:09:02.798Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298947099Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65172024-04-20T01:09:02.798Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298963499Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"0","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65182024-04-20T01:09:02.798Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.298992019Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"1","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65192024-04-20T01:09:02.798Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.299006479Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"client":"2","":"downstairs","session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65202024-04-20T01:09:02.798Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
65212024-04-20T01:09:02.798Z,"time":"2024-04-20T01:09:01.299026169Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65222024-04-20T01:09:02.798Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.299056289Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65232024-04-20T01:09:02.798Z{"msg":"d04053c1-cc40-445d-9d87-bc2713b866ab is now active with session: 5aef2b37-5f00-4dca-a98e-bd42c688977b","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.299070229Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65242024-04-20T01:09:02.798Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.299101099Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408,"session_id":"5aef2b37-5f00-4dca-a98e-bd42c688977b"}
65252024-04-20T01:09:02.799Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:01.299120019Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408}
65262024-04-20T01:09:02.802ZLoading write count information from file "/tmp/test_repair_verify.data"
65272024-04-20T01:09:02.803ZRun Repair workload
65282024-04-20T01:09:02.803Z01/30 Write block 188 len 12288 data: 6 5 6
65292024-04-20T01:09:02.803Z02/30 Read block 63 len 16384
65302024-04-20T01:09:02.803Z03/30 Write block 74 len 32768 data: 5 3 4 5 5 5 5 6
65312024-04-20T01:09:02.803Z04/30 Write block 189 len 28672 data: 6 7 7 5 5 5 5
65322024-04-20T01:09:02.803Z05/30 Read block 311 len 32768
65332024-04-20T01:09:02.803Z06/30 Write block 91 len 4096 data: 4
65342024-04-20T01:09:02.803Z07/30 Read block 92 len 28672
65352024-04-20T01:09:02.803Z08/30 Read block 435 len 4096
65362024-04-20T01:09:02.803Z09/30 Read block 496 len 36864
65372024-04-20T01:09:02.803Z10/30 Write block 215 len 24576 data: 4 5 8 8 6 8
65382024-04-20T01:09:02.803Z11/30 Flush
65392024-04-20T01:09:02.807Z12/30 Flush
65402024-04-20T01:09:02.807Z13/30 Write block 67 len 20480 data: 7 7 8 5 5
65412024-04-20T01:09:02.807Z14/30 Flush
65422024-04-20T01:09:02.810Z15/30 Write block 172 len 8192 data: 3 4
65432024-04-20T01:09:02.811Z16/30 Write block 123 len 40960 data: 5 7 8 8 7 8 7 8 8 8
65442024-04-20T01:09:02.817Z17/30 Read block 198 len 32768
65452024-04-20T01:09:02.817Z18/30 Read block 508 len 32768
65462024-04-20T01:09:02.817Z19/30 Write block 334 len 8192 data: 5 4
65472024-04-20T01:09:02.817Z20/30 Write block 506 len 20480 data: 6 6 6 7 7
65482024-04-20T01:09:02.821Z21/30 Write block 86 len 32768 data: 5 5 5 6 7 5 4 3
65492024-04-20T01:09:02.821Z22/30 Write block 73 len 28672 data: 5 6 4 5 6 6 6
65502024-04-20T01:09:02.821Z23/30 Read block 393 len 36864
65512024-04-20T01:09:02.821Z24/30 Read block 588 len 36864
65522024-04-20T01:09:02.821Z25/30 Read block 588 len 4096
65532024-04-20T01:09:02.821Z26/30 Read block 329 len 16384
65542024-04-20T01:09:02.821Z27/30 Write block 269 len 4096 data: 3
65552024-04-20T01:09:02.821Z28/30 Read block 326 len 20480
65562024-04-20T01:09:02.821Z29/30 Read block 363 len 36864
65572024-04-20T01:09:02.825Z30/30 Read block 488 len 28672
65582024-04-20T01:09:02.825Z----------------------------------------------------------------
65592024-04-20T01:09:02.825Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
65602024-04-20T01:09:02.826ZStates: Active Active Active
65612024-04-20T01:09:02.826ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65622024-04-20T01:09:02.826Z 1 Acked 1000 Write 3 Done Done Done false
65632024-04-20T01:09:02.826Z 2 Acked 1001 Read 4 Done Done Done false
65642024-04-20T01:09:02.826Z 3 Acked 1002 Write 8 Done Done Done false
65652024-04-20T01:09:02.826Z 4 Acked 1003 Write 7 Done Done Done false
65662024-04-20T01:09:02.826Z 5 Acked 1004 Read 8 Done Done Done false
65672024-04-20T01:09:02.826Z 6 Acked 1005 Write 1 Sent Done Done false
65682024-04-20T01:09:02.826Z 7 Acked 1006 Read 7 Sent Done Done false
65692024-04-20T01:09:02.826Z 8 Acked 1007 Read 1 Sent Done Done false
65702024-04-20T01:09:02.826Z 9 Acked 1008 Read 9 Sent Done Done false
65712024-04-20T01:09:02.826Z 10 Acked 1009 Write 6 Sent Done Done false
65722024-04-20T01:09:02.826Z 11 Acked 1010 Flush 0 Sent Done Done false
65732024-04-20T01:09:02.826Z 12 Acked 1011 Flush 0 Sent Done Done false
65742024-04-20T01:09:02.826Z 13 Acked 1012 Write 5 Sent Done Done false
65752024-04-20T01:09:02.826Z 14 Acked 1013 Flush 0 Sent Done Done false
65762024-04-20T01:09:02.826Z 15 Acked 1014 Write 2 Sent Done Done false
65772024-04-20T01:09:02.826Z 16 Acked 1015 Write 10 Sent Done Done false
65782024-04-20T01:09:02.826Z 17 Acked 1016 Read 8 Sent Done Done false
65792024-04-20T01:09:02.826Z 18 Acked 1017 Read 8 Sent Done Done false
65802024-04-20T01:09:02.826Z 19 Acked 1018 Write 2 Sent Done Done false
65812024-04-20T01:09:02.826Z 20 Acked 1019 Write 5 Sent Done Done false
65822024-04-20T01:09:02.827Z 21 Acked 1020 Write 8 Sent Done Done false
65832024-04-20T01:09:02.827Z 22 Acked 1021 Write 7 Sent Done Done false
65842024-04-20T01:09:02.827Z 23 Acked 1022 Read 9 Sent Done Done false
65852024-04-20T01:09:02.827Z 24 Acked 1023 Read 9 Sent Done Done false
65862024-04-20T01:09:02.827Z 25 Acked 1024 Read 1 Sent Done Done false
65872024-04-20T01:09:02.827Z 26 Acked 1025 Read 4 Sent Done Done false
65882024-04-20T01:09:02.827Z 27 Acked 1026 Write 1 Sent Done Done false
65892024-04-20T01:09:02.827Z 28 Acked 1027 Read 5 Sent Done Done false
65902024-04-20T01:09:02.827Z 29 Acked 1028 Read 9 Sent Done Done false
65912024-04-20T01:09:02.827Z 30 Acked 1029 Read 7 Sent Sent Done false
65922024-04-20T01:09:02.827Z STATES DS:0 DS:1 DS:2 TOTAL
65932024-04-20T01:09:02.827Z New 0 0 0 0
65942024-04-20T01:09:02.827Z Sent 25 1 0 26
65952024-04-20T01:09:02.827Z Done 5 29 30 64
65962024-04-20T01:09:02.827Z Skipped 0 0 0 0
65972024-04-20T01:09:02.827Z Error 0 0 0 0
65982024-04-20T01:09:02.827ZLast Flush: 0 1013 1013
65992024-04-20T01:09:02.827ZDownstairs last five completed:
66002024-04-20T01:09:02.827ZUpstairs last five completed: 30 29 28 27 26
66012024-04-20T01:09:02.827Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:09:01.329534297Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1408}
66022024-04-20T01:09:02.827ZWrote out file "/tmp/test_repair_verify.data"
66032024-04-20T01:09:02.831Z
66042024-04-20T01:09:02.831ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
66052024-04-20T01:09:02.847ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66062024-04-20T01:09:02.847Z 0 000-019 58 58 58 98 98 98 F F F
66072024-04-20T01:09:02.847Z 1 020-039 58 58 58 98 98 98 F F F
66082024-04-20T01:09:02.847Z 2 040-059 55 55 55 92 92 92 F F F
66092024-04-20T01:09:02.847Z 3 060-079 57 60 60 96 104 104 T T T <---
66102024-04-20T01:09:02.847Z 4 080-099 57 60 60 96 102 102 T T T <---
66112024-04-20T01:09:02.851Z 5 100-119 56 56 56 93 93 93 F F F
66122024-04-20T01:09:02.851Z 6 120-139 56 56 56 93 93 93 F T T <---
66132024-04-20T01:09:02.851Z 7 140-159 56 56 56 93 93 93 F F F
66142024-04-20T01:09:02.851Z 8 160-179 53 53 53 90 90 90 F T T <---
66152024-04-20T01:09:02.851Z 9 180-199 58 60 60 98 102 102 T F F <---
66162024-04-20T01:09:02.851Z 10 200-219 55 60 60 92 102 102 F F F <---
66172024-04-20T01:09:02.851Z 11 220-239 57 60 60 96 102 102 F F F <---
66182024-04-20T01:09:02.851Z 12 240-259 46 46 46 79 79 79 F F F
66192024-04-20T01:09:02.852Z 13 260-279 58 58 58 97 97 97 F T T <---
66202024-04-20T01:09:02.852Z 14 280-299 53 53 53 90 90 90 F F F
66212024-04-20T01:09:02.852Z 15 300-319 57 57 57 96 96 96 F F F
66222024-04-20T01:09:02.852Z 16 320-339 59 59 59 101 101 101 F T T <---
66232024-04-20T01:09:02.852Z 17 340-359 58 58 58 98 98 98 F F F
66242024-04-20T01:09:02.852Z 18 360-379 58 58 58 97 97 97 F F F
66252024-04-20T01:09:02.852Z 19 380-399 44 44 44 74 74 74 F F F
66262024-04-20T01:09:02.852Z 20 400-419 56 56 56 95 95 95 F F F
66272024-04-20T01:09:02.852Z 21 420-439 58 58 58 97 97 97 F F F
66282024-04-20T01:09:02.852Z 22 440-459 59 59 59 101 101 101 F F F
66292024-04-20T01:09:02.852Z 23 460-479 58 58 58 98 98 98 F F F
66302024-04-20T01:09:02.853Z 24 480-499 56 56 56 94 94 94 F F F
66312024-04-20T01:09:02.853Z 25 500-519 59 59 59 101 101 101 F T T <---
66322024-04-20T01:09:02.853Z 26 520-539 59 59 59 101 101 101 F F F
66332024-04-20T01:09:02.853Z 27 540-559 58 58 58 100 100 100 F F F
66342024-04-20T01:09:02.853Z 28 560-579 55 55 55 92 92 92 F F F
66352024-04-20T01:09:02.853Z 29 580-599 59 59 59 101 101 101 F F F
66362024-04-20T01:09:02.853ZMax gen: 60, Max flush: 104
66372024-04-20T01:09:02.853ZError: Difference in extent metadata found!
66382024-04-20T01:09:02.853ZA Difference in extent metadata is expected here
66392024-04-20T01:09:02.853ZOn loop 30
66402024-04-20T01:09:02.853Z
66412024-04-20T01:09:02.865ZVerifying data now
66422024-04-20T01:09:17.334ZNo change after verify
66432024-04-20T01:09:17.334ZLoop: 30 Downstairs dump after verify (and repair):
66442024-04-20T01:09:17.348ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66452024-04-20T01:09:17.348Z 0 000-019 58 58 58 98 98 98 F F F
66462024-04-20T01:09:17.348Z 1 020-039 58 58 58 98 98 98 F F F
66472024-04-20T01:09:17.352Z 2 040-059 55 55 55 92 92 92 F F F
66482024-04-20T01:09:17.352Z 3 060-079 61 61 61 105 105 105 F F F
66492024-04-20T01:09:17.352Z 4 080-099 61 61 61 105 105 105 F F F
66502024-04-20T01:09:17.352Z 5 100-119 56 56 56 93 93 93 F F F
66512024-04-20T01:09:17.352Z 6 120-139 61 61 61 105 105 105 F F F
66522024-04-20T01:09:17.352Z 7 140-159 56 56 56 93 93 93 F F F
66532024-04-20T01:09:17.352Z 8 160-179 61 61 61 105 105 105 F F F
66542024-04-20T01:09:17.352Z 9 180-199 60 60 60 102 102 102 F F F
66552024-04-20T01:09:17.352Z 10 200-219 60 60 60 102 102 102 F F F
66562024-04-20T01:09:17.352Z 11 220-239 60 60 60 102 102 102 F F F
66572024-04-20T01:09:17.352Z 12 240-259 46 46 46 79 79 79 F F F
66582024-04-20T01:09:17.352Z 13 260-279 61 61 61 105 105 105 F F F
66592024-04-20T01:09:17.352Z 14 280-299 53 53 53 90 90 90 F F F
66602024-04-20T01:09:17.352Z 15 300-319 57 57 57 96 96 96 F F F
66612024-04-20T01:09:17.352Z 16 320-339 61 61 61 105 105 105 F F F
66622024-04-20T01:09:17.352Z 17 340-359 58 58 58 98 98 98 F F F
66632024-04-20T01:09:17.353Z 18 360-379 58 58 58 97 97 97 F F F
66642024-04-20T01:09:17.353Z 19 380-399 44 44 44 74 74 74 F F F
66652024-04-20T01:09:17.353Z 20 400-419 56 56 56 95 95 95 F F F
66662024-04-20T01:09:17.353Z 21 420-439 58 58 58 97 97 97 F F F
66672024-04-20T01:09:17.353Z 22 440-459 59 59 59 101 101 101 F F F
66682024-04-20T01:09:17.353Z 23 460-479 58 58 58 98 98 98 F F F
66692024-04-20T01:09:17.353Z 24 480-499 56 56 56 94 94 94 F F F
66702024-04-20T01:09:17.353Z 25 500-519 61 61 61 105 105 105 F F F
66712024-04-20T01:09:17.353Z 26 520-539 59 59 59 101 101 101 F F F
66722024-04-20T01:09:17.353Z 27 540-559 58 58 58 100 100 100 F F F
66732024-04-20T01:09:17.353Z 28 560-579 55 55 55 92 92 92 F F F
66742024-04-20T01:09:17.353Z 29 580-599 59 59 59 101 101 101 F F F
66752024-04-20T01:09:17.353ZMax gen: 61, Max flush: 105
66762024-04-20T01:09:17.357Z
66772024-04-20T01:09:17.357ZBegin loop 31
66782024-04-20T01:09:17.357ZDownstairs to restart: 0
66792024-04-20T01:09:17.370Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.87704753Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416}
66802024-04-20T01:09:17.370Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.87727979Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416}
66812024-04-20T01:09:17.370Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.87730588Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416}
66822024-04-20T01:09:17.370Z{"msg":"Crucible 2d302f3a-074b-4ef4-bad4-027eb61c0273 has session id: 4a8d1bf1-0021-41bb-b166-6d30de15588f","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877341211Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66832024-04-20T01:09:17.378Z{Crucible runtime is spawned
66842024-04-20T01:09:17.378Z"msg":"Upstairs opts: Upstairs UUID: 2d302f3a-074b-4ef4-bad4-027eb61c0273, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.87739524Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66852024-04-20T01:09:17.378Z{"msg":"Crucible stats registered with UUID: 2d302f3a-074b-4ef4-bad4-027eb61c0273","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877431251Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66862024-04-20T01:09:17.378Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877446251Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416}
66872024-04-20T01:09:17.378Z{"msg":"2d302f3a-074b-4ef4-bad4-027eb61c0273 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877484871Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66882024-04-20T01:09:17.378Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877499141Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"0","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66892024-04-20T01:09:17.378Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877530461Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"0","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66902024-04-20T01:09:17.378Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877552951Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"1","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66912024-04-20T01:09:17.378Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877578091Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"1","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66922024-04-20T01:09:17.378Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877613501Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"2","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66932024-04-20T01:09:17.378Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877630981Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"2","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66942024-04-20T01:09:17.378Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877655441Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66952024-04-20T01:09:17.378Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877685741Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66962024-04-20T01:09:17.378Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877716021Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66972024-04-20T01:09:17.378Z{"msg":"ds_connection connected from Ok(127.0.0.1:47246)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.877981102Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66982024-04-20T01:09:17.378Z{"msg":"ds_connection connected from Ok(127.0.0.1:48342)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.878051362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
66992024-04-20T01:09:17.378Z{"msg":"ds_connection connected from Ok(127.0.0.1:45988)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.878078312Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67002024-04-20T01:09:17.379Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.878826974Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"1","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67012024-04-20T01:09:17.379Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.878869004Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"1","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67022024-04-20T01:09:17.379Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.878887874Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"2","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67032024-04-20T01:09:17.379Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.878911694Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"2","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67042024-04-20T01:09:17.379Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.879258245Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"0","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67052024-04-20T01:09:17.379Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.879305425Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"0","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67062024-04-20T01:09:17.379Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.879364315Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"2","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67072024-04-20T01:09:17.379Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.879480095Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"1","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67082024-04-20T01:09:17.379Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.879859076Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"0","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67092024-04-20T01:09:17.379Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.879990276Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"1","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67102024-04-20T01:09:17.379Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880043386Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"1","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67112024-04-20T01:09:17.379Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880088546Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67122024-04-20T01:09:17.379Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880122717Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"2","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67132024-04-20T01:09:17.379Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880153267Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"2","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67142024-04-20T01:09:17.379Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880186367Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67152024-04-20T01:09:17.379Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880256537Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"0","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67162024-04-20T01:09:17.379Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880300057Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"0","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67172024-04-20T01:09:17.379Z{"msg":"[0]R flush_numbers[0..12]: [98, 98, 92, 105, 105, 93, 105, 93, 105, 102, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880335377Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67182024-04-20T01:09:17.379Z{"msg":"[0]R generation[0..12]: [58, 58, 55, 61, 61, 56, 61, 56, 61, 60, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880372877Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67192024-04-20T01:09:17.379Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880408947Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67202024-04-20T01:09:17.379Z{"msg":"[1]R flush_numbers[0..12]: [98, 98, 92, 105, 105, 93, 105, 93, 105, 102, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880444037Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67212024-04-20T01:09:17.379ZWait for a query_work_queue command to finish before sending IO
67222024-04-20T01:09:17.379Z{"msg":"[1]R generation[0..12]: [58, 58, 55, 61, 61, 56, 61, 56, 61, 60, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880483107Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67232024-04-20T01:09:17.380Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880522678Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67242024-04-20T01:09:17.380Z{"msg":"[2]R flush_numbers[0..12]: [98, 98, 92, 105, 105, 93, 105, 93, 105, 102, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880561408Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67252024-04-20T01:09:17.380Z{"msg":"[2]R generation[0..12]: [58, 58, 55, 61, 61, 56, 61, 56, 61, 60, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880610488Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67262024-04-20T01:09:17.380Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880649508Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67272024-04-20T01:09:17.380Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880685518Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67282024-04-20T01:09:17.380Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880723508Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67292024-04-20T01:09:17.380Z{"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880760918Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67302024-04-20T01:09:17.380Z{"msg":"All extents match","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67312024-04-20T01:09:17.380Zv":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880804268Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"":"downstairs","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67322024-04-20T01:09:17.380Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880840078Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67332024-04-20T01:09:17.380Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880854628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"0","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67342024-04-20T01:09:17.380Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880876588Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"1","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67352024-04-20T01:09:17.380Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880890268Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"client":"2","":"downstairs","session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67362024-04-20T01:09:17.380Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880910768Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67372024-04-20T01:09:17.380Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880923418Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67382024-04-20T01:09:17.380Z{"msg":"2d302f3a-074b-4ef4-bad4-027eb61c0273 is now active with session: 4a8d1bf1-0021-41bb-b166-6d30de15588f","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880944379Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67392024-04-20T01:09:17.380Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880957319Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416,"session_id":"4a8d1bf1-0021-41bb-b166-6d30de15588f"}
67402024-04-20T01:09:17.380Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:15.880977139Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416}
67412024-04-20T01:09:17.384ZLoading write count information from file "/tmp/test_repair_verify.data"
67422024-04-20T01:09:17.385ZRun Repair workload
67432024-04-20T01:09:17.385Z01/30 Write block 412 len 36864 data: 9 10 9 8 9 7 8 8 7
67442024-04-20T01:09:17.385Z02/30 Read block 415 len 24576
67452024-04-20T01:09:17.385Z03/30 Write block 188 len 32768 data: 7 7 8 8 6 6 6 6
67462024-04-20T01:09:17.385Z04/30 Write block 428 len 32768 data: 4 5 5 5 5 5 6 6
67472024-04-20T01:09:17.385Z05/30 Read block 530 len 8192
67482024-04-20T01:09:17.385Z06/30 Read block 361 len 4096
67492024-04-20T01:09:17.385Z07/30 Write block 457 len 4096 data: 11
67502024-04-20T01:09:17.385Z08/30 Flush
67512024-04-20T01:09:17.385Z09/30 Write block 197 len 24576 data: 6 7 7 4 4 5
67522024-04-20T01:09:17.389Z10/30 Write block 519 len 20480 data: 8 8 9 8 9
67532024-04-20T01:09:17.389Z11/30 Read block 362 len 8192
67542024-04-20T01:09:17.389Z12/30 Read block 68 len 4096
67552024-04-20T01:09:17.389Z13/30 Read block 500 len 12288
67562024-04-20T01:09:17.389Z14/30 Read block 308 len 4096
67572024-04-20T01:09:17.389Z15/30 Read block 496 len 32768
67582024-04-20T01:09:17.395Z16/30 Read block 219 len 8192
67592024-04-20T01:09:17.395Z17/30 Write block 564 len 20480 data: 4 4 5 4 5
67602024-04-20T01:09:17.395Z18/30 Write block 425 len 24576 data: 5 4 4 5 6 6
67612024-04-20T01:09:17.396Z19/30 Read block 483 len 40960
67622024-04-20T01:09:17.396Z20/30 Flush
67632024-04-20T01:09:17.399Z21/30 Write block 461 len 24576 data: 9 6 6 4 5 5
67642024-04-20T01:09:17.399Z22/30 Write block 444 len 8192 data: 7 6
67652024-04-20T01:09:17.399Z23/30 Flush
67662024-04-20T01:09:17.403Z24/30 Write block 202 len 28672 data: 6 4 4 5 5 5 5
67672024-04-20T01:09:17.403Z25/30 Write block 110 len 28672 data: 5 5 5 5 5 5 4
67682024-04-20T01:09:17.403Z26/30 Write block 328 len 28672 data: 8 7 10 8 8 6 6
67692024-04-20T01:09:17.407Z27/30 Write block 33 len 24576 data: 5 5 5 4 4 5
67702024-04-20T01:09:17.407Z28/30 Read block 35 len 16384
67712024-04-20T01:09:17.407Z29/30 Write block 97 len 32768 data: 4 5 5 7 7 6 6 4
67722024-04-20T01:09:17.407Z30/30 Write block 390 len 36864 data: 4 4 4 5 5 5 5 4 3
67732024-04-20T01:09:17.407Z----------------------------------------------------------------
67742024-04-20T01:09:17.407Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
67752024-04-20T01:09:17.407ZStates: Active Active Active
67762024-04-20T01:09:17.407ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67772024-04-20T01:09:17.407Z 1 Acked 1000 Write 9 Done Done Done false
67782024-04-20T01:09:17.407Z 2 Acked 1001 Read 6 Done Done Done false
67792024-04-20T01:09:17.408Z 3 Acked 1002 Write 8 Done Done Done false
67802024-04-20T01:09:17.408Z 4 Acked 1003 Write 8 Sent Done Done false
67812024-04-20T01:09:17.408Z 5 Acked 1004 Read 2 Sent Done Done false
67822024-04-20T01:09:17.408Z 6 Acked 1005 Read 1 Sent Done Done false
67832024-04-20T01:09:17.408Z 7 Acked 1006 Write 1 Sent Done Done false
67842024-04-20T01:09:17.408Z 8 Acked 1007 Flush 0 Sent Done Done false
67852024-04-20T01:09:17.408Z 9 Acked 1008 Write 6 Sent Done Done false
67862024-04-20T01:09:17.408Z 10 Acked 1009 Write 5 Sent Done Done false
67872024-04-20T01:09:17.408Z 11 Acked 1010 Read 2 Sent Done Done false
67882024-04-20T01:09:17.408Z 12 Acked 1011 Read 1 Sent Done Done false
67892024-04-20T01:09:17.412Z 13 Acked 1012 Read 3 Sent Done Done false
67902024-04-20T01:09:17.412Z 14 Acked 1013 Read 1 Sent Done Done false
67912024-04-20T01:09:17.412Z 15 Acked 1014 Read 8 Sent Done Done false
67922024-04-20T01:09:17.412Z 16 Acked 1015 Read 2 Sent Done Done false
67932024-04-20T01:09:17.412Z 17 Acked 1016 Write 5 Sent Done Done false
67942024-04-20T01:09:17.413Z 18 Acked 1017 Write 6 Sent Done Done false
67952024-04-20T01:09:17.413Z 19 Acked 1018 Read 10 Sent Done Done false
67962024-04-20T01:09:17.413Z 20 Acked 1019 Flush 0 Sent Done Done false
67972024-04-20T01:09:17.413Z 21 Acked 1020 Write 6 Sent Done Done false
67982024-04-20T01:09:17.413Z 22 Acked 1021 Write 2 Sent Done Done false
67992024-04-20T01:09:17.413Z 23 Acked 1022 Flush 0 Sent Done Done false
68002024-04-20T01:09:17.413Z 24 Acked 1023 Write 7 Sent Done Done false
68012024-04-20T01:09:17.413Z 25 Acked 1024 Write 7 Sent Done Done false
68022024-04-20T01:09:17.413Z 26 Acked 1025 Write 7 Sent Done Done false
68032024-04-20T01:09:17.413Z 27 Acked 1026 Write 6 Sent Done Done false
68042024-04-20T01:09:17.413Z 28 Acked 1027 Read 4 Sent Done Done false
68052024-04-20T01:09:17.413Z 29 Acked 1028 Write 8 Sent Done Done false
68062024-04-20T01:09:17.413Z 30 Acked 1029 Write 9 Sent Sent Sent false
68072024-04-20T01:09:17.413Z STATES DS:0 DS:1 DS:2 TOTAL
68082024-04-20T01:09:17.413Z New 0 0 0 0
68092024-04-20T01:09:17.413Z Sent 27 1 1 29
68102024-04-20T01:09:17.413Z Done 3 29 29 61
68112024-04-20T01:09:17.413Z Skipped 0 0 0 0
68122024-04-20T01:09:17.413Z Error 0 0 0 0
68132024-04-20T01:09:17.413ZLast Flush: 0 1022 1022
68142024-04-20T01:09:17.413ZDownstairs last five completed:
68152024-04-20T01:09:17.413ZUpstairs last five completed: 30 29 28 27 26
68162024-04-20T01:09:17.413Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:09:15.914485793Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1416}
68172024-04-20T01:09:17.414ZWrote out file "/tmp/test_repair_verify.data"
68182024-04-20T01:09:17.414Z
68192024-04-20T01:09:17.417ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
68202024-04-20T01:09:17.432ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68212024-04-20T01:09:17.432Z 0 000-019 58 58 58 98 98 98 F F F
68222024-04-20T01:09:17.432Z 1 020-039 58 58 58 98 98 98 F T T <---
68232024-04-20T01:09:17.432Z 2 040-059 55 55 55 92 92 92 F F F
68242024-04-20T01:09:17.432Z 3 060-079 61 61 61 105 105 105 F F F
68252024-04-20T01:09:17.436Z 4 080-099 61 61 61 105 105 105 F T T <---
68262024-04-20T01:09:17.436Z 5 100-119 56 56 56 93 93 93 F T T <---
68272024-04-20T01:09:17.436Z 6 120-139 61 61 61 105 105 105 F F F
68282024-04-20T01:09:17.436Z 7 140-159 56 56 56 93 93 93 F F F
68292024-04-20T01:09:17.436Z 8 160-179 61 61 61 105 105 105 F F F
68302024-04-20T01:09:17.436Z 9 180-199 60 62 62 102 107 107 T F F <---
68312024-04-20T01:09:17.436Z 10 200-219 60 62 62 102 107 107 F T T <---
68322024-04-20T01:09:17.436Z 11 220-239 60 60 60 102 102 102 F F F
68332024-04-20T01:09:17.436Z 12 240-259 46 46 46 79 79 79 F F F
68342024-04-20T01:09:17.436Z 13 260-279 61 61 61 105 105 105 F F F
68352024-04-20T01:09:17.436Z 14 280-299 53 53 53 90 90 90 F F F
68362024-04-20T01:09:17.436Z 15 300-319 57 57 57 96 96 96 F F F
68372024-04-20T01:09:17.436Z 16 320-339 61 61 61 105 105 105 F T T <---
68382024-04-20T01:09:17.436Z 17 340-359 58 58 58 98 98 98 F F F
68392024-04-20T01:09:17.436Z 18 360-379 58 58 58 97 97 97 F F F
68402024-04-20T01:09:17.436Z 19 380-399 44 44 44 74 74 74 F T T <---
68412024-04-20T01:09:17.436Z 20 400-419 56 62 62 95 106 106 T F F <---
68422024-04-20T01:09:17.436Z 21 420-439 58 62 62 97 107 107 T F F <---
68432024-04-20T01:09:17.436Z 22 440-459 59 62 62 101 108 108 F F F <---
68442024-04-20T01:09:17.436Z 23 460-479 58 62 62 98 108 108 F F F <---
68452024-04-20T01:09:17.437Z 24 480-499 56 56 56 94 94 94 F F F
68462024-04-20T01:09:17.437Z 25 500-519 61 62 62 105 107 107 F F F <---
68472024-04-20T01:09:17.437Z 26 520-539 59 62 62 101 107 107 F F F <---
68482024-04-20T01:09:17.437Z 27 540-559 58 58 58 100 100 100 F F F
68492024-04-20T01:09:17.437Z 28 560-579 55 62 62 92 107 107 F F F <---
68502024-04-20T01:09:17.437Z 29 580-599 59 59 59 101 101 101 F F F
68512024-04-20T01:09:17.437ZMax gen: 62, Max flush: 108
68522024-04-20T01:09:17.437ZError: Difference in extent metadata found!
68532024-04-20T01:09:17.437ZA Difference in extent metadata is expected here
68542024-04-20T01:09:17.437ZOn loop 31
68552024-04-20T01:09:17.437Z
68562024-04-20T01:09:17.441ZVerifying data now
68572024-04-20T01:09:32.076ZNo change after verify
68582024-04-20T01:09:32.076ZLoop: 31 Downstairs dump after verify (and repair):
68592024-04-20T01:09:32.086ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68602024-04-20T01:09:32.087Z 0 000-019 58 58 58 98 98 98 F F F
68612024-04-20T01:09:32.087Z 1 020-039 63 63 63 109 109 109 F F F
68622024-04-20T01:09:32.087Z 2 040-059 55 55 55 92 92 92 F F F
68632024-04-20T01:09:32.087Z 3 060-079 61 61 61 105 105 105 F F F
68642024-04-20T01:09:32.087Z 4 080-099 63 63 63 109 109 109 F F F
68652024-04-20T01:09:32.087Z 5 100-119 63 63 63 109 109 109 F F F
68662024-04-20T01:09:32.087Z 6 120-139 61 61 61 105 105 105 F F F
68672024-04-20T01:09:32.087Z 7 140-159 56 56 56 93 93 93 F F F
68682024-04-20T01:09:32.087Z 8 160-179 61 61 61 105 105 105 F F F
68692024-04-20T01:09:32.091Z 9 180-199 62 62 62 107 107 107 F F F
68702024-04-20T01:09:32.091Z 10 200-219 63 63 63 109 109 109 F F F
68712024-04-20T01:09:32.091Z 11 220-239 60 60 60 102 102 102 F F F
68722024-04-20T01:09:32.092Z 12 240-259 46 46 46 79 79 79 F F F
68732024-04-20T01:09:32.092Z 13 260-279 61 61 61 105 105 105 F F F
68742024-04-20T01:09:32.092Z 14 280-299 53 53 53 90 90 90 F F F
68752024-04-20T01:09:32.092Z 15 300-319 57 57 57 96 96 96 F F F
68762024-04-20T01:09:32.092Z 16 320-339 63 63 63 109 109 109 F F F
68772024-04-20T01:09:32.092Z 17 340-359 58 58 58 98 98 98 F F F
68782024-04-20T01:09:32.092Z 18 360-379 58 58 58 97 97 97 F F F
68792024-04-20T01:09:32.092Z 19 380-399 63 63 63 109 109 109 F F F
68802024-04-20T01:09:32.092Z 20 400-419 62 62 62 106 106 106 F F F
68812024-04-20T01:09:32.092Z 21 420-439 62 62 62 107 107 107 F F F
68822024-04-20T01:09:32.092Z 22 440-459 62 62 62 108 108 108 F F F
68832024-04-20T01:09:32.092Z 23 460-479 62 62 62 108 108 108 F F F
68842024-04-20T01:09:32.092Z 24 480-499 56 56 56 94 94 94 F F F
68852024-04-20T01:09:32.092Z 25 500-519 62 62 62 107 107 107 F F F
68862024-04-20T01:09:32.092Z 26 520-539 62 62 62 107 107 107 F F F
68872024-04-20T01:09:32.092Z 27 540-559 58 58 58 100 100 100 F F F
68882024-04-20T01:09:32.092Z 28 560-579 62 62 62 107 107 107 F F F
68892024-04-20T01:09:32.092Z 29 580-599 59 59 59 101 101 101 F F F
68902024-04-20T01:09:32.092ZMax gen: 63, Max flush: 109
68912024-04-20T01:09:32.092Z
68922024-04-20T01:09:32.092ZBegin loop 32
68932024-04-20T01:09:32.092ZDownstairs to restart: 2
68942024-04-20T01:09:32.108Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615081266Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424}
68952024-04-20T01:09:32.108Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615305417Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424}
68962024-04-20T01:09:32.119Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615340037Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424}
68972024-04-20T01:09:32.119Z{"msg":"Crucible 14b215a3-0dd8-4cbc-9808-8e0f21d498e9 has session id: f30a6f0f-e2c2-489b-a739-d8ed086b7a7f","v":Crucible runtime is spawned
68982024-04-20T01:09:32.119Z0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615383917Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
68992024-04-20T01:09:32.119Z{"msg":"Upstairs opts: Upstairs UUID: 14b215a3-0dd8-4cbc-9808-8e0f21d498e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615438397Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69002024-04-20T01:09:32.119Z{"msg":"Crucible stats registered with UUID: 14b215a3-0dd8-4cbc-9808-8e0f21d498e9","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615480307Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69012024-04-20T01:09:32.119Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615498087Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424}
69022024-04-20T01:09:32.119Z{"msg":"14b215a3-0dd8-4cbc-9808-8e0f21d498e9 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615510258Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69032024-04-20T01:09:32.119Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615522727Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"0","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69042024-04-20T01:09:32.119Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615551928Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"0","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69052024-04-20T01:09:32.119Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615571828Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"1","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69062024-04-20T01:09:32.119Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615589948Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"1","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69072024-04-20T01:09:32.119Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615612638Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"2","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69082024-04-20T01:09:32.119Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615638228Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"2","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69092024-04-20T01:09:32.119Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615675628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69102024-04-20T01:09:32.119Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615721738Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69112024-04-20T01:09:32.119Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615739628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69122024-04-20T01:09:32.119Z{"msg":"ds_connection connected from Ok(127.0.0.1:56935)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.615920539Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69132024-04-20T01:09:32.119Z{"msg":"ds_connection connected from Ok(127.0.0.1:48846)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.616124269Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69142024-04-20T01:09:32.119Z{"msg":"ds_connection connected from Ok(127.0.0.1:60300)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.616196939Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69152024-04-20T01:09:32.119Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.6165217Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"1","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69162024-04-20T01:09:32.120Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.61656886Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"1","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69172024-04-20T01:09:32.120Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.61668169Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"0","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69182024-04-20T01:09:32.120Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.61671867Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"0","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69192024-04-20T01:09:32.120Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.617037861Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"1","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69202024-04-20T01:09:32.120Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.617190351Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"0","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69212024-04-20T01:09:32.120Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.617294491Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"2","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69222024-04-20T01:09:32.120Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.617322571Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"2","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69232024-04-20T01:09:32.120Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.617508972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"1","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69242024-04-20T01:09:32.120Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.617562742Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"1","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69252024-04-20T01:09:32.120Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.617603512Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69262024-04-20T01:09:32.120Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.617629212Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"0","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69272024-04-20T01:09:32.120Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.617657802Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"0","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69282024-04-20T01:09:32.120Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.617681692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69292024-04-20T01:09:32.120Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.617893433Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"2","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69302024-04-20T01:09:32.120Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618320624Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"2","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69312024-04-20T01:09:32.120Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618357114Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"2","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69322024-04-20T01:09:32.120Z{"msg":"[0]R flush_numbers[0..12]: [98, 109, 92, 105, 109, 109, 105, 93, 105, 107, 109, 102]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618373784Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69332024-04-20T01:09:32.120Z{"msg":"[0]R generation[0..12]: [58, 63, 55, 61, 63, 63, 61, 56, 61, 62, 63, 60]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618395684Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69342024-04-20T01:09:32.120Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618416544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69352024-04-20T01:09:32.120Z{"msg":"[1]R flush_numbers[0..12]: [98, 109, 92, 105, 109, 109, 105, 93, 105, 107, 109, 102]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618430434Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69362024-04-20T01:09:32.120Z{"msg":"[1]R generation[0..12]: [58, 63, 55, 61, 63, 63, 61, 56, 61, 62, 63, 60]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618450664Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69372024-04-20T01:09:32.120Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618465164Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69382024-04-20T01:09:32.120Z{"msg":"[2]R flush_numbers[0..12]: [98, 109, 92, 105, 109, 109, 105, 93, 105, 107, 109, 102]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618485734Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
69392024-04-20T01:09:32.121Zsession_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69402024-04-20T01:09:32.121Z{"msg":"[2]R generation[0..12]: [58, 63, 55, 61, 63, 63, 61, 56, 61, 62, 63, 60]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618503144Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69412024-04-20T01:09:32.121Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618524584Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69422024-04-20T01:09:32.121Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618537794Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69432024-04-20T01:09:32.121Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618558014Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69442024-04-20T01:09:32.121Z{"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618571544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69452024-04-20T01:09:32.121Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618591454Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"":"downstairs","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69462024-04-20T01:09:32.121Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618605045Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69472024-04-20T01:09:32.121Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618616805Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"0","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69482024-04-20T01:09:32.121Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618637134Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"1","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69492024-04-20T01:09:32.121Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618650345Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"client":"2","":"downstairs","session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69502024-04-20T01:09:32.121Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618663805Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69512024-04-20T01:09:32.121Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618676374Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69522024-04-20T01:09:32.121Z{"msg":"14b215a3-0dd8-4cbc-9808-8e0f21d498e9 is now active with session: f30a6f0f-e2c2-489b-a739-d8ed086b7a7f","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618689005Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69532024-04-20T01:09:32.121Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618707515Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424,"session_id":"f30a6f0f-e2c2-489b-a739-d8ed086b7a7f"}
69542024-04-20T01:09:32.121Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:30.618719815Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424}
69552024-04-20T01:09:32.121ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69562024-04-20T01:09:32.121ZLoading write count information from file "/tmp/test_repair_verify.data"
69572024-04-20T01:09:32.121ZRun Repair workload
69582024-04-20T01:09:32.121Z01/30 Write block 389 len 4096 data: 4
69592024-04-20T01:09:32.125Z02/30 Write block 234 len 36864 data: 11 9 9 10 9 9 6 5 3
69602024-04-20T01:09:32.126Z03/30 Write block 244 len 36864 data: 7 7 7 6 9 8 6 5 5
69612024-04-20T01:09:32.126Z04/30 Write block 269 len 28672 data: 4 4 4 5 6 5 4
69622024-04-20T01:09:32.126Z05/30 Write block 37 len 28672 data: 5 6 7 7 6 4 3
69632024-04-20T01:09:32.126Z06/30 Read block 515 len 16384
69642024-04-20T01:09:32.126Z07/30 Write block 356 len 36864 data: 4 4 5 5 5 6 7 7 6
69652024-04-20T01:09:32.126Z08/30 Read block 217 len 36864
69662024-04-20T01:09:32.126Z09/30 Write block 370 len 24576 data: 8 11 10 11 10 9
69672024-04-20T01:09:32.126Z10/30 Read block 124 len 8192
69682024-04-20T01:09:32.126Z11/30 Read block 5 len 12288
69692024-04-20T01:09:32.126Z12/30 Write block 596 len 4096 data: 6
69702024-04-20T01:09:32.126Z13/30 Read block 531 len 28672
69712024-04-20T01:09:32.130Z14/30 Write block 313 len 8192 data: 3 3
69722024-04-20T01:09:32.130Z15/30 Read block 9 len 32768
69732024-04-20T01:09:32.130Z16/30 Write block 34 len 36864 data: 6 6 5 6 7 8 8 7 5
69742024-04-20T01:09:32.130Z17/30 Flush
69752024-04-20T01:09:32.134Z18/30 Write block 185 len 32768 data: 5 6 6 8 8 9 9 7
69762024-04-20T01:09:32.138Z19/30 Read block 216 len 4096
69772024-04-20T01:09:32.138Z20/30 Read block 94 len 40960
69782024-04-20T01:09:32.138Z21/30 Read block 141 len 16384
69792024-04-20T01:09:32.138Z22/30 Write block 585 len 8192 data: 8 10
69802024-04-20T01:09:32.138Z23/30 Flush
69812024-04-20T01:09:32.142Z24/30 Write block 228 len 28672 data: 7 5 6 9 10 11 12
69822024-04-20T01:09:32.142Z25/30 Write block 517 len 4096 data: 8
69832024-04-20T01:09:32.145Z26/30 Write block 512 len 24576 data: 9 8 7 7 8 9
69842024-04-20T01:09:32.145Z27/30 Read block 352 len 40960
69852024-04-20T01:09:32.145Z28/30 Write block 190 len 20480 data: 10 10 8 7 7
69862024-04-20T01:09:32.145Z29/30 Write block 506 len 16384 data: 7 7 7 8
69872024-04-20T01:09:32.149Z30/30 Write block 249 len 12288 data: 9 7 6
69882024-04-20T01:09:32.150Z----------------------------------------------------------------
69892024-04-20T01:09:32.150Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
69902024-04-20T01:09:32.150ZStates: Active Active Active
69912024-04-20T01:09:32.150ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69922024-04-20T01:09:32.150Z 1 Acked 1000 Write 1 Done Done Done false
69932024-04-20T01:09:32.150Z 2 Acked 1001 Write 9 Done Done Sent false
69942024-04-20T01:09:32.150Z 3 Acked 1002 Write 9 Done Done Sent false
69952024-04-20T01:09:32.150Z 4 Acked 1003 Write 7 Done Done Sent false
69962024-04-20T01:09:32.150Z 5 Acked 1004 Write 7 Done Done Sent false
69972024-04-20T01:09:32.150Z 6 Acked 1005 Read 4 Done Done Sent false
69982024-04-20T01:09:32.150Z 7 Acked 1006 Write 9 Done Done Sent false
69992024-04-20T01:09:32.150Z 8 Acked 1007 Read 9 Done Done Sent false
70002024-04-20T01:09:32.150Z 9 Acked 1008 Write 6 Done Done Sent false
70012024-04-20T01:09:32.150Z 10 Acked 1009 Read 2 Done Done Sent false
70022024-04-20T01:09:32.150Z 11 Acked 1010 Read 3 Done Done Sent false
70032024-04-20T01:09:32.150Z 12 Acked 1011 Write 1 Done Done Sent false
70042024-04-20T01:09:32.150Z 13 Acked 1012 Read 7 Done Done Sent false
70052024-04-20T01:09:32.150Z 14 Acked 1013 Write 2 Done Done Sent false
70062024-04-20T01:09:32.150Z 15 Acked 1014 Read 8 Done Done Sent false
70072024-04-20T01:09:32.150Z 16 Acked 1015 Write 9 Done Done Sent false
70082024-04-20T01:09:32.150Z 17 Acked 1016 Flush 0 Done Done Sent false
70092024-04-20T01:09:32.150Z 18 Acked 1017 Write 8 Done Done Sent false
70102024-04-20T01:09:32.150Z 19 Acked 1018 Read 1 Done Done Sent false
70112024-04-20T01:09:32.151Z 20 Acked 1019 Read 10 Done Done Sent false
70122024-04-20T01:09:32.151Z 21 Acked 1020 Read 4 Done Done Sent false
70132024-04-20T01:09:32.151Z 22 Acked 1021 Write 2 Done Done Sent false
70142024-04-20T01:09:32.151Z 23 Acked 1022 Flush 0 Done Done Sent false
70152024-04-20T01:09:32.151Z 24 Acked 1023 Write 7 Done Done Sent false
70162024-04-20T01:09:32.151Z 25 Acked 1024 Write 1 Done Done Sent false
70172024-04-20T01:09:32.151Z 26 Acked 1025 Write 6 Done Done Sent false
70182024-04-20T01:09:32.151Z 27 Acked 1026 Read 10 Done Done Sent false
70192024-04-20T01:09:32.151Z 28 Acked 1027 Write 5 Done Done Sent false
70202024-04-20T01:09:32.151Z 29 Acked 1028 Write 4 Done Done Sent false
70212024-04-20T01:09:32.151Z 30 Acked 1029 Write 3 Sent Sent Sent false
70222024-04-20T01:09:32.151Z STATES DS:0 DS:1 DS:2 TOTAL
70232024-04-20T01:09:32.151Z New 0 0 0 0
70242024-04-20T01:09:32.151Z Sent 1 1 29 31
70252024-04-20T01:09:32.151Z Done 29 29 1 59
70262024-04-20T01:09:32.151Z Skipped 0 0 0 0
70272024-04-20T01:09:32.151Z Error 0 0 0 0
70282024-04-20T01:09:32.151ZLast Flush: 1022 1022 0
70292024-04-20T01:09:32.151ZDownstairs last five completed:
70302024-04-20T01:09:32.151ZUpstairs last five completed: 30 29 28 27 26
70312024-04-20T01:09:32.151Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:09:30.652976541Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1424}
70322024-04-20T01:09:32.151ZWrote out file "/tmp/test_repair_verify.data"
70332024-04-20T01:09:32.155Z
70342024-04-20T01:09:32.155ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70352024-04-20T01:09:32.168ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70362024-04-20T01:09:32.168Z 0 000-019 58 58 58 98 98 98 F F F
70372024-04-20T01:09:32.168Z 1 020-039 64 64 63 110 110 109 F F F <---
70382024-04-20T01:09:32.169Z 2 040-059 64 64 55 110 110 92 F F F <---
70392024-04-20T01:09:32.169Z 3 060-079 61 61 61 105 105 105 F F F
70402024-04-20T01:09:32.169Z 4 080-099 63 63 63 109 109 109 F F F
70412024-04-20T01:09:32.169Z 5 100-119 63 63 63 109 109 109 F F F
70422024-04-20T01:09:32.173Z 6 120-139 61 61 61 105 105 105 F F F
70432024-04-20T01:09:32.173Z 7 140-159 56 56 56 93 93 93 F F F
70442024-04-20T01:09:32.173Z 8 160-179 61 61 61 105 105 105 F F F
70452024-04-20T01:09:32.173Z 9 180-199 64 64 62 111 111 107 T T F <---
70462024-04-20T01:09:32.173Z 10 200-219 63 63 63 109 109 109 F F F
70472024-04-20T01:09:32.173Z 11 220-239 64 64 60 110 110 102 T T F <---
70482024-04-20T01:09:32.173Z 12 240-259 64 64 46 110 110 79 T T F <---
70492024-04-20T01:09:32.173Z 13 260-279 64 64 61 110 110 105 F F F <---
70502024-04-20T01:09:32.173Z 14 280-299 53 53 53 90 90 90 F F F
70512024-04-20T01:09:32.173Z 15 300-319 64 64 57 110 110 96 F F F <---
70522024-04-20T01:09:32.173Z 16 320-339 63 63 63 109 109 109 F F F
70532024-04-20T01:09:32.173Z 17 340-359 64 64 58 110 110 98 F F F <---
70542024-04-20T01:09:32.173Z 18 360-379 64 64 58 110 110 97 F F F <---
70552024-04-20T01:09:32.173Z 19 380-399 64 64 63 110 110 109 F F T <---
70562024-04-20T01:09:32.173Z 20 400-419 62 62 62 106 106 106 F F F
70572024-04-20T01:09:32.173Z 21 420-439 62 62 62 107 107 107 F F F
70582024-04-20T01:09:32.173Z 22 440-459 62 62 62 108 108 108 F F F
70592024-04-20T01:09:32.173Z 23 460-479 62 62 62 108 108 108 F F F
70602024-04-20T01:09:32.174Z 24 480-499 56 56 56 94 94 94 F F F
70612024-04-20T01:09:32.174Z 25 500-519 62 62 62 107 107 107 T T F <---
70622024-04-20T01:09:32.174Z 26 520-539 62 62 62 107 107 107 F F F
70632024-04-20T01:09:32.174Z 27 540-559 58 58 58 100 100 100 F F F
70642024-04-20T01:09:32.174Z 28 560-579 62 62 62 107 107 107 F F F
70652024-04-20T01:09:32.174Z 29 580-599 64 64 59 111 111 101 F F F <---
70662024-04-20T01:09:32.174ZMax gen: 64, Max flush: 111
70672024-04-20T01:09:32.174ZError: Difference in extent metadata found!
70682024-04-20T01:09:32.174ZA Difference in extent metadata is expected here
70692024-04-20T01:09:32.174ZOn loop 32
70702024-04-20T01:09:32.174Z
70712024-04-20T01:09:32.178ZVerifying data now
70722024-04-20T01:09:36.724ZNo change after verify
70732024-04-20T01:09:36.724ZLoop: 32 Downstairs dump after verify (and repair):
70742024-04-20T01:09:36.744ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70752024-04-20T01:09:36.745Z 0 000-019 58 58 58 98 98 98 F F F
70762024-04-20T01:09:36.745Z 1 020-039 64 64 64 110 110 110 F F F
70772024-04-20T01:09:36.745Z 2 040-059 64 64 64 110 110 110 F F F
70782024-04-20T01:09:36.745Z 3 060-079 61 61 61 105 105 105 F F F
70792024-04-20T01:09:36.746Z 4 080-099 63 63 63 109 109 109 F F F
70802024-04-20T01:09:36.746Z 5 100-119 63 63 63 109 109 109 F F F
70812024-04-20T01:09:36.746Z 6 120-139 61 61 61 105 105 105 F F F
70822024-04-20T01:09:36.756Z 7 140-159 56 56 56 93 93 93 F F F
70832024-04-20T01:09:36.756Z 8 160-179 61 61 61 105 105 105 F F F
70842024-04-20T01:09:36.756Z 9 180-199 65 65 65 112 112 112 F F F
70852024-04-20T01:09:36.756Z 10 200-219 63 63 63 109 109 109 F F F
70862024-04-20T01:09:36.756Z 11 220-239 65 65 65 112 112 112 F F F
70872024-04-20T01:09:36.756Z 12 240-259 65 65 65 112 112 112 F F F
70882024-04-20T01:09:36.757Z 13 260-279 64 64 64 110 110 110 F F F
70892024-04-20T01:09:36.757Z 14 280-299 53 53 53 90 90 90 F F F
70902024-04-20T01:09:36.757Z 15 300-319 64 64 64 110 110 110 F F F
70912024-04-20T01:09:36.757Z 16 320-339 63 63 63 109 109 109 F F F
70922024-04-20T01:09:36.758Z 17 340-359 64 64 64 110 110 110 F F F
70932024-04-20T01:09:36.758Z 18 360-379 64 64 64 110 110 110 F F F
70942024-04-20T01:09:36.758Z 19 380-399 64 64 64 110 110 110 F F F
70952024-04-20T01:09:36.758Z 20 400-419 62 62 62 106 106 106 F F F
70962024-04-20T01:09:36.758Z 21 420-439 62 62 62 107 107 107 F F F
70972024-04-20T01:09:36.759Z 22 440-459 62 62 62 108 108 108 F F F
70982024-04-20T01:09:36.759Z 23 460-479 62 62 62 108 108 108 F F F
70992024-04-20T01:09:36.759Z 24 480-499 56 56 56 94 94 94 F F F
71002024-04-20T01:09:36.759Z 25 500-519 65 65 65 112 112 112 F F F
71012024-04-20T01:09:36.759Z 26 520-539 62 62 62 107 107 107 F F F
71022024-04-20T01:09:36.760Z 27 540-559 58 58 58 100 100 100 F F F
71032024-04-20T01:09:36.760Z 28 560-579 62 62 62 107 107 107 F F F
71042024-04-20T01:09:36.760Z 29 580-599 64 64 64 111 111 111 F F F
71052024-04-20T01:09:36.760ZMax gen: 65, Max flush: 112
71062024-04-20T01:09:36.761Z
71072024-04-20T01:09:36.761ZBegin loop 33
71082024-04-20T01:09:36.761ZDownstairs to restart: 0
71092024-04-20T01:09:36.778Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.264498986Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432}
71102024-04-20T01:09:36.778Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.264705047Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432}
71112024-04-20T01:09:36.778Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.264740347Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432}
71122024-04-20T01:09:36.778Z{"msg":"Crucible ae5b8550-592d-40ff-9da6-d8ced3b9f4ef has session id: 4ecd1921-fc9e-4c13-b52e-4077f38e227c","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.264764577Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71132024-04-20T01:09:36.778Z{"msg":"Upstairs opts: Upstairs UUID: ae5b8550-592d-40ff-9da6-d8ced3b9f4ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.264812597Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71142024-04-20T01:09:36.778Z{"msg":"Crucible stats registered with UUID: ae5b8550-592d-40ff-9da6-d8ced3b9f4ef","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.264836317Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71152024-04-20T01:09:36.778ZCrucible runtime is spawned
71162024-04-20T01:09:36.779Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.264982717Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432}
71172024-04-20T01:09:36.779Z{"msg":"ae5b8550-592d-40ff-9da6-d8ced3b9f4ef active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265024128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71182024-04-20T01:09:36.779Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265044068Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"0","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71192024-04-20T01:09:36.779Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265096188Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"0","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71202024-04-20T01:09:36.779Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265137498Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"1","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71212024-04-20T01:09:36.779Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265162968Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"1","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71222024-04-20T01:09:36.779Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265189758Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"2","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71232024-04-20T01:09:36.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265207138Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"2","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71242024-04-20T01:09:36.780Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265230518Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71252024-04-20T01:09:36.780Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265299578Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71262024-04-20T01:09:36.780Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265346558Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71272024-04-20T01:09:36.781Z{"msg":"ds_connection connected from Ok(127.0.0.1:35505)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265380728Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71282024-04-20T01:09:36.781Z{"msg":"ds_connection connected from Ok(127.0.0.1:33325)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265428868Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71292024-04-20T01:09:36.781Z{"msg":"ds_connection connected from Ok(127.0.0.1:34388)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.265474158Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71302024-04-20T01:09:36.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.26620516Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"1","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71312024-04-20T01:09:36.781Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.26624037Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"1","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71322024-04-20T01:09:36.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.26625583Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"2","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71332024-04-20T01:09:36.782Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.2662697Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"2","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71342024-04-20T01:09:36.782Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.266815221Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"1","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71352024-04-20T01:09:36.782Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.266976162Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"2","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71362024-04-20T01:09:36.782Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.267300182Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"1","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71372024-04-20T01:09:36.782Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.267336423Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"1","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71382024-04-20T01:09:36.783Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.267362543Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71392024-04-20T01:09:36.783Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.267408823Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"0","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71402024-04-20T01:09:36.783Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.267438683Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"0","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71412024-04-20T01:09:36.783Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.267510013Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"2","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71422024-04-20T01:09:36.783Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.267564423Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"2","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71432024-04-20T01:09:36.784Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.267599183Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71442024-04-20T01:09:36.784Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.267798004Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"0","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71452024-04-20T01:09:36.784Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268214245Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"0","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71462024-04-20T01:09:36.784Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268249505Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"0","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71472024-04-20T01:09:36.784Z{"msg":"[0]R flush_numbers[0..12]: [98, 110, 110, 105, 109, 109, 105, 93, 105, 112, 109, 112]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268265475Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71482024-04-20T01:09:36.784Z{"msg":"[0]R generation[0..12]: [58, 64, 64, 61, 63, 63, 61, 56, 61, 65, 63, 65]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268280365Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71492024-04-20T01:09:36.785Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268294575Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71502024-04-20T01:09:36.785Z{"msg":"[1]R flush_numbers[0..12]: [98, 110, 110, 105, 109, 109, 105, 93, 105, 112, 109, 112]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268308575Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71512024-04-20T01:09:36.785Z{"msg":"[1]R generation[0..12]: [58, 64, 64, 61, 63, 63, 61, 56, 61, 65, 63, 65]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268323265Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71522024-04-20T01:09:36.785Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268351945Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71532024-04-20T01:09:36.785ZWait for a query_work_queue command to finish before sending IO
71542024-04-20T01:09:36.786Z{"msg":"[2]R flush_numbers[0..12]: [98, 110, 110, 105, 109, 109, 105, 93, 105, 112, 109, 112]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268389005Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71552024-04-20T01:09:36.786Z{"msg":"[2]R generation[0..12]: [58, 64, 64, 61, 63, 63, 61, 56, 61, 65, 63, 65]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268426875Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71562024-04-20T01:09:36.786Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268465495Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71572024-04-20T01:09:36.786Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268502125Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71582024-04-20T01:09:36.787Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268533795Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71592024-04-20T01:09:36.787Z{"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268563835Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71602024-04-20T01:09:36.787Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268598005Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"":"downstairs","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71612024-04-20T01:09:36.787Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268624796Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71622024-04-20T01:09:36.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268638056Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"0","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71632024-04-20T01:09:36.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268671166Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"1","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71642024-04-20T01:09:36.788ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71652024-04-20T01:09:36.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268709846Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"client":"2","":"downstairs","session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71662024-04-20T01:09:36.789Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268744146Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71672024-04-20T01:09:36.789Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268777346Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71682024-04-20T01:09:36.789Z{"msg":"ae5b8550-592d-40ff-9da6-d8ced3b9f4ef is now active with session: 4ecd1921-fc9e-4c13-b52e-4077f38e227c","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268810146Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71692024-04-20T01:09:36.789Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268840066Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432,"session_id":"4ecd1921-fc9e-4c13-b52e-4077f38e227c"}
71702024-04-20T01:09:36.790Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:35.268869536Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432}
71712024-04-20T01:09:36.790ZLoading write count information from file "/tmp/test_repair_verify.data"
71722024-04-20T01:09:36.790ZRun Repair workload
71732024-04-20T01:09:36.791Z01/30 Write block 348 len 32768 data: 7 6 4 4 4 3 3 4
71742024-04-20T01:09:36.791Z02/30 Flush
71752024-04-20T01:09:36.791Z03/30 Flush
71762024-04-20T01:09:36.791Z04/30 Write block 20 len 12288 data: 4 5 5
71772024-04-20T01:09:36.809Z05/30 Read block 309 len 20480
71782024-04-20T01:09:36.809Z06/30 Read block 150 len 32768
71792024-04-20T01:09:36.810Z07/30 Write block 123 len 40960 data: 6 8 9 9 8 9 8 9 9 9
71802024-04-20T01:09:36.810Z08/30 Write block 399 len 8192 data: 5 4
71812024-04-20T01:09:36.810Z09/30 Read block 149 len 40960
71822024-04-20T01:09:36.810Z10/30 Write block 435 len 12288 data: 7 7 6
71832024-04-20T01:09:36.810Z11/30 Read block 410 len 24576
71842024-04-20T01:09:36.811Z12/30 Read block 67 len 4096
71852024-04-20T01:09:36.811Z13/30 Write block 245 len 24576 data: 8 8 7 10 10 8
71862024-04-20T01:09:36.811Z14/30 Write block 233 len 20480 data: 12 13 10 10 11
71872024-04-20T01:09:36.811Z15/30 Write block 513 len 8192 data: 9 8
71882024-04-20T01:09:36.811Z16/30 Read block 62 len 40960
71892024-04-20T01:09:36.811Z17/30 Read block 224 len 20480
71902024-04-20T01:09:36.811Z18/30 Read block 222 len 20480
71912024-04-20T01:09:36.811Z19/30 Write block 86 len 36864 data: 6 6 6 7 8 6 5 4 4
71922024-04-20T01:09:36.811Z20/30 Write block 156 len 28672 data: 8 7 7 5 6 6 6
71932024-04-20T01:09:36.812Z21/30 Write block 226 len 20480 data: 10 6 8 6 7
71942024-04-20T01:09:36.812Z22/30 Read block 307 len 16384
71952024-04-20T01:09:36.812Z23/30 Write block 430 len 4096 data: 7
71962024-04-20T01:09:36.812Z24/30 Write block 171 len 40960 data: 3 4 5 4 4 4 5 6 8 8
71972024-04-20T01:09:36.812Z25/30 Write block 366 len 40960 data: 7 8 6 7 9 12 11 12 11 10
71982024-04-20T01:09:36.812Z26/30 Write block 330 len 24576 data: 11 9 9 7 7 5
71992024-04-20T01:09:36.812Z27/30 Write block 318 len 32768 data: 7 7 8 8 9 8 8 7
72002024-04-20T01:09:36.813Z28/30 Read block 368 len 20480
72012024-04-20T01:09:36.813Z29/30 Read block 55 len 16384
72022024-04-20T01:09:36.813Z30/30 Write block 433 len 28672 data: 6 7 8 8 7 5 6
72032024-04-20T01:09:36.813Z----------------------------------------------------------------
72042024-04-20T01:09:36.813Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:27
72052024-04-20T01:09:36.814ZStates: Active Active Active
72062024-04-20T01:09:36.814ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
72072024-04-20T01:09:36.814Z 4 Acked 1003 Write 3 Done Done Done false
72082024-04-20T01:09:36.814Z 5 Acked 1004 Read 5 Done Done Done false
72092024-04-20T01:09:36.814Z 6 Acked 1005 Read 8 Done Done Done false
72102024-04-20T01:09:36.814Z 7 Acked 1006 Write 10 Done Done Done false
72112024-04-20T01:09:36.814Z 8 Acked 1007 Write 2 Done Done Done false
72122024-04-20T01:09:36.815Z 9 Acked 1008 Read 10 Done Done Done false
72132024-04-20T01:09:36.815Z 10 Acked 1009 Write 3 Done Done Done false
72142024-04-20T01:09:36.815Z 11 Acked 1010 Read 6 Done Done Done false
72152024-04-20T01:09:36.815Z 12 Acked 1011 Read 1 Done Done Done false
72162024-04-20T01:09:36.815Z 13 Acked 1012 Write 6 Done Done Done false
72172024-04-20T01:09:36.815Z 14 Acked 1013 Write 5 Sent Done Done false
72182024-04-20T01:09:36.816Z 15 Acked 1014 Write 2 Sent Done Done false
72192024-04-20T01:09:36.816Z 16 Acked 1015 Read 10 Sent Done Done false
72202024-04-20T01:09:36.816Z 17 Acked 1016 Read 5 Sent Done Done false
72212024-04-20T01:09:36.816Z 18 Acked 1017 Read 5 Sent Done Done false
72222024-04-20T01:09:36.816Z 19 Acked 1018 Write 9 Sent Done Done false
72232024-04-20T01:09:36.816Z 20 Acked 1019 Write 7 Sent Done Done false
72242024-04-20T01:09:36.816Z 21 Acked 1020 Write 5 Sent Done Done false
72252024-04-20T01:09:36.817Z 22 Acked 1021 Read 4 Sent Done Done false
72262024-04-20T01:09:36.817Z 23 Acked 1022 Write 1 Sent Done Done false
72272024-04-20T01:09:36.817Z 24 Acked 1023 Write 10 Sent Done Done false
72282024-04-20T01:09:36.817Z 25 Acked 1024 Write 10 Sent Done Done false
72292024-04-20T01:09:36.817Z 26 Acked 1025 Write 6 Sent Done Done false
72302024-04-20T01:09:36.818Z 27 Acked 1026 Write 8 Sent Done Done false
72312024-04-20T01:09:36.818Z 28 Acked 1027 Read 5 Sent Done Done false
72322024-04-20T01:09:36.818Z 29 Acked 1028 Read 4 Sent Done Done false
72332024-04-20T01:09:36.818Z 30 Acked 1029 Write 7 Sent Sent Sent false
72342024-04-20T01:09:36.818Z STATES DS:0 DS:1 DS:2 TOTAL
72352024-04-20T01:09:36.819Z New 0 0 0 0
72362024-04-20T01:09:36.819Z Sent 17 1 1 19
72372024-04-20T01:09:36.819Z Done 10 26 26 62
72382024-04-20T01:09:36.819Z Skipped 0 0 0 0
72392024-04-20T01:09:36.819Z Error 0 0 0 0
72402024-04-20T01:09:36.819ZLast Flush: 1002 1002 1002
72412024-04-20T01:09:36.819ZDownstairs last five completed: 1002 1001 1000
72422024-04-20T01:09:36.819ZUpstairs last five completed: 30 29 28 27 26
72432024-04-20T01:09:36.820Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:09:35.298956993Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1432}
72442024-04-20T01:09:36.820ZWrote out file "/tmp/test_repair_verify.data"
72452024-04-20T01:09:36.820Z
72462024-04-20T01:09:36.828ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72472024-04-20T01:09:36.828ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72482024-04-20T01:09:36.828Z 0 000-019 58 58 58 98 98 98 F F F
72492024-04-20T01:09:36.828Z 1 020-039 64 64 64 110 110 110 T T T
72502024-04-20T01:09:36.828Z 2 040-059 64 64 64 110 110 110 F F F
72512024-04-20T01:09:36.828Z 3 060-079 61 61 61 105 105 105 F F F
72522024-04-20T01:09:36.828Z 4 080-099 63 63 63 109 109 109 F T T <---
72532024-04-20T01:09:36.829Z 5 100-119 63 63 63 109 109 109 F F F
72542024-04-20T01:09:36.829Z 6 120-139 61 61 61 105 105 105 T T T
72552024-04-20T01:09:36.829Z 7 140-159 56 56 56 93 93 93 F T T <---
72562024-04-20T01:09:36.829Z 8 160-179 61 61 61 105 105 105 F T T <---
72572024-04-20T01:09:36.830Z 9 180-199 65 65 65 112 112 112 F T T <---
72582024-04-20T01:09:36.830Z 10 200-219 63 63 63 109 109 109 F F F
72592024-04-20T01:09:36.830Z 11 220-239 65 65 65 112 112 112 F T T <---
72602024-04-20T01:09:36.830Z 12 240-259 65 65 65 112 112 112 T T T
72612024-04-20T01:09:36.831Z 13 260-279 64 64 64 110 110 110 F F F
72622024-04-20T01:09:36.831Z 14 280-299 53 53 53 90 90 90 F F F
72632024-04-20T01:09:36.831Z 15 300-319 64 64 64 110 110 110 F T T <---
72642024-04-20T01:09:36.831Z 16 320-339 63 63 63 109 109 109 F T T <---
72652024-04-20T01:09:36.831Z 17 340-359 66 66 66 113 113 113 F F F
72662024-04-20T01:09:36.831Z 18 360-379 64 64 64 110 110 110 F T T <---
72672024-04-20T01:09:36.832Z 19 380-399 64 64 64 110 110 110 T T T
72682024-04-20T01:09:36.832Z 20 400-419 62 62 62 106 106 106 T T T
72692024-04-20T01:09:36.832Z 21 420-439 62 62 62 107 107 107 T T T
72702024-04-20T01:09:36.832Z 22 440-459 62 62 62 108 108 108 F F F
72712024-04-20T01:09:36.832Z 23 460-479 62 62 62 108 108 108 F F F
72722024-04-20T01:09:36.833Z 24 480-499 56 56 56 94 94 94 F F F
72732024-04-20T01:09:36.833Z 25 500-519 65 65 65 112 112 112 F T T <---
72742024-04-20T01:09:36.833Z 26 520-539 62 62 62 107 107 107 F F F
72752024-04-20T01:09:36.833Z 27 540-559 58 58 58 100 100 100 F F F
72762024-04-20T01:09:36.833Z 28 560-579 62 62 62 107 107 107 F F F
72772024-04-20T01:09:36.834Z 29 580-599 64 64 64 111 111 111 F F F
72782024-04-20T01:09:36.834ZMax gen: 66, Max flush: 113
72792024-04-20T01:09:36.834ZError: Difference in extent metadata found!
72802024-04-20T01:09:36.834ZA Difference in extent metadata is expected here
72812024-04-20T01:09:36.835ZOn loop 33
72822024-04-20T01:09:36.835Z
72832024-04-20T01:09:36.844ZVerifying data now
72842024-04-20T01:09:51.586ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
72852024-04-20T01:09:51.590Z433c433
72862024-04-20T01:09:51.590Z< 6,
72872024-04-20T01:09:51.590Z---
72882024-04-20T01:09:51.590Z> 7,
72892024-04-20T01:09:51.590Z436d435
72902024-04-20T01:09:51.590Z< 5,
72912024-04-20T01:09:51.590Z438a438,439
72922024-04-20T01:09:51.590Z> 8,
72932024-04-20T01:09:51.590Z> 8,
72942024-04-20T01:09:51.590Z439a441
72952024-04-20T01:09:51.590Z> 5,
72962024-04-20T01:09:51.590Z441d442
72972024-04-20T01:09:51.590Z< 4,
72982024-04-20T01:09:51.591Z443d443
72992024-04-20T01:09:51.591Z< 5,
73002024-04-20T01:09:51.591Zdiff found after verify
73012024-04-20T01:09:51.595ZLoop: 33 Downstairs dump after verify (and repair):
73022024-04-20T01:09:51.605ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
73032024-04-20T01:09:51.605Z 0 000-019 58 58 58 98 98 98 F F F
73042024-04-20T01:09:51.605Z 1 020-039 67 67 67 114 114 114 F F F
73052024-04-20T01:09:51.605Z 2 040-059 64 64 64 110 110 110 F F F
73062024-04-20T01:09:51.605Z 3 060-079 61 61 61 105 105 105 F F F
73072024-04-20T01:09:51.605Z 4 080-099 67 67 67 114 114 114 F F F
73082024-04-20T01:09:51.606Z 5 100-119 63 63 63 109 109 109 F F F
73092024-04-20T01:09:51.606Z 6 120-139 67 67 67 114 114 114 F F F
73102024-04-20T01:09:51.606Z 7 140-159 67 67 67 114 114 114 F F F
73112024-04-20T01:09:51.606Z 8 160-179 67 67 67 114 114 114 F F F
73122024-04-20T01:09:51.606Z 9 180-199 67 67 67 114 114 114 F F F
73132024-04-20T01:09:51.606Z 10 200-219 63 63 63 109 109 109 F F F
73142024-04-20T01:09:51.606Z 11 220-239 67 67 67 114 114 114 F F F
73152024-04-20T01:09:51.610Z 12 240-259 67 67 67 114 114 114 F F F
73162024-04-20T01:09:51.610Z 13 260-279 64 64 64 110 110 110 F F F
73172024-04-20T01:09:51.610Z 14 280-299 53 53 53 90 90 90 F F F
73182024-04-20T01:09:51.610Z 15 300-319 67 67 67 114 114 114 F F F
73192024-04-20T01:09:51.610Z 16 320-339 67 67 67 114 114 114 F F F
73202024-04-20T01:09:51.611Z 17 340-359 66 66 66 113 113 113 F F F
73212024-04-20T01:09:51.611Z 18 360-379 67 67 67 114 114 114 F F F
73222024-04-20T01:09:51.611Z 19 380-399 67 67 67 114 114 114 F F F
73232024-04-20T01:09:51.611Z 20 400-419 67 67 67 114 114 114 F F F
73242024-04-20T01:09:51.611Z 21 420-439 67 67 67 114 114 114 F F F
73252024-04-20T01:09:51.611Z 22 440-459 62 62 62 108 108 108 F F F
73262024-04-20T01:09:51.611Z 23 460-479 62 62 62 108 108 108 F F F
73272024-04-20T01:09:51.611Z 24 480-499 56 56 56 94 94 94 F F F
73282024-04-20T01:09:51.611Z 25 500-519 67 67 67 114 114 114 F F F
73292024-04-20T01:09:51.611Z 26 520-539 62 62 62 107 107 107 F F F
73302024-04-20T01:09:51.611Z 27 540-559 58 58 58 100 100 100 F F F
73312024-04-20T01:09:51.611Z 28 560-579 62 62 62 107 107 107 F F F
73322024-04-20T01:09:51.612Z 29 580-599 64 64 64 111 111 111 F F F
73332024-04-20T01:09:51.612ZMax gen: 67, Max flush: 114
73342024-04-20T01:09:51.612Z
73352024-04-20T01:09:51.612ZBegin loop 34
73362024-04-20T01:09:51.612ZDownstairs to restart: 0
73372024-04-20T01:09:51.630Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.133689125Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443}
73382024-04-20T01:09:51.630Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.133895976Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443}
73392024-04-20T01:09:51.630Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.133917996Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443}
73402024-04-20T01:09:51.638Z{"msg":"Crucible 3c86153e-4b31-4d75-8453-92c855515248 has session id: c8f37007-02e5-4a53-aad6-57f57b7ddbe7","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.133941016Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73412024-04-20T01:09:51.638Z{"msg":"Upstairs opts: Upstairs UUID: 3c86153e-4b31-4d75-8453-92c855515248, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.133986106Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73422024-04-20T01:09:51.638Z{"msg":"Crucible stats registered with UUID: 3c86153e-4b31-4d75-8453-92c855515248","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134018306Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73432024-04-20T01:09:51.638ZCrucible runtime is spawned
73442024-04-20T01:09:51.638Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134201427Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443}
73452024-04-20T01:09:51.638Z{"msg":"3c86153e-4b31-4d75-8453-92c855515248 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134287287Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73462024-04-20T01:09:51.638Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134313107Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"0","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73472024-04-20T01:09:51.638Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134331097Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"0","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73482024-04-20T01:09:51.638Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134363737Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"1","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73492024-04-20T01:09:51.638Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134392827Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"1","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73502024-04-20T01:09:51.638Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134409237Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"2","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73512024-04-20T01:09:51.638Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134436697Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"2","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73522024-04-20T01:09:51.638Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134460147Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73532024-04-20T01:09:51.638Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134487217Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73542024-04-20T01:09:51.639Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134513307Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73552024-04-20T01:09:51.639Z{"msg":"ds_connection connected from Ok(127.0.0.1:53331)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134605607Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73562024-04-20T01:09:51.639Z{"msg":"ds_connection connected from Ok(127.0.0.1:43039)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134681688Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73572024-04-20T01:09:51.639Z{"msg":"ds_connection connected from Ok(127.0.0.1:62385)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.134722338Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73582024-04-20T01:09:51.639Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.135274159Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"1","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73592024-04-20T01:09:51.639Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.135320529Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"1","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73602024-04-20T01:09:51.639Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.135357189Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"2","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73612024-04-20T01:09:51.639Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.135383109Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"2","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73622024-04-20T01:09:51.639Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.13582957Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"2","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73632024-04-20T01:09:51.639Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.136019191Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"1","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73642024-04-20T01:09:51.639Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.136171721Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"0","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73652024-04-20T01:09:51.639Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.136220871Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"0","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73662024-04-20T01:09:51.639Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.136310221Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"2","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73672024-04-20T01:09:51.639Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.136351101Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"2","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73682024-04-20T01:09:51.639Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.136392431Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73692024-04-20T01:09:51.639Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.136427571Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"1","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73702024-04-20T01:09:51.639Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.136443621Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"1","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73712024-04-20T01:09:51.639Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.136461432Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73722024-04-20T01:09:51.639Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.136634852Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"0","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73732024-04-20T01:09:51.639Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137036353Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"0","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73742024-04-20T01:09:51.639Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137058843Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"0","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73752024-04-20T01:09:51.639Z{"msg":"[0]R flush_numbers[0..12]: [98, 114, 110, 105, 114, 109, 114, 114, 114, 114, 109, 114]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137073283Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73762024-04-20T01:09:51.639Z{"msg":"[0]R generation[0..12]: [58, 67, 64, 61, 67, 63, 67, 67, 67, 67, 63, 67]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137107083Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73772024-04-20T01:09:51.640Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137129733Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73782024-04-20T01:09:51.640Z{"msg":"[1]R flush_numbers[0..12]: [98, 114, 110, 105, 114, 109, 114, 114, 114, 114, 109, 114]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137143993Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73792024-04-20T01:09:51.640Z{"msg":"[1]R generation[0..12]: [58, 67, 64, 61, 67, 63, 67, 67, 67, 67, 63, 67]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137158413Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73802024-04-20T01:09:51.640Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137179713Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73812024-04-20T01:09:51.640Z{"msg":"[2]R flush_numbers[0..12]: [98, 114, 110, 105, 114, 109, 114, 114, 114, 114, 109, 114]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137193783Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73822024-04-20T01:09:51.640Z{"msg":"Wait for a query_work_queue command to finish before sending IO
73832024-04-20T01:09:51.640Z[2]R generation[0..12]: [58, 67, 64, 61, 67, 63, 67, 67, 67, 67, 63, 67]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137210343Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73842024-04-20T01:09:51.640Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137224973Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73852024-04-20T01:09:51.640Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137238463Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73862024-04-20T01:09:51.640Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137251983Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73872024-04-20T01:09:51.640Z{"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137271933Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73882024-04-20T01:09:51.640Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137285203Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"":"downstairs","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73892024-04-20T01:09:51.640Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137303293Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73902024-04-20T01:09:51.640Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137331614Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"0","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73912024-04-20T01:09:51.640Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137352454Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"client":"1","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73922024-04-20T01:09:51.640Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137387083Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73932024-04-20T01:09:51.640Z1443,"client":"2","":"downstairs","session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73942024-04-20T01:09:51.640Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137419064Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73952024-04-20T01:09:51.640Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137451624Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73962024-04-20T01:09:51.640Z{"msg":"3c86153e-4b31-4d75-8453-92c855515248 is now active with session: c8f37007-02e5-4a53-aad6-57f57b7ddbe7","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137485804Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73972024-04-20T01:09:51.640Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137516884Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443,"session_id":"c8f37007-02e5-4a53-aad6-57f57b7ddbe7"}
73982024-04-20T01:09:51.640Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:09:50.137547024Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443}
73992024-04-20T01:09:51.640ZLoading write count information from file "/tmp/test_repair_verify.data"
74002024-04-20T01:09:51.641ZRun Repair workload
74012024-04-20T01:09:51.641Z01/30 Write block 509 len 20480 data: 9 8 8 10 10
74022024-04-20T01:09:51.655Z02/30 Write block 197 len 28672 data: 7 8 8 5 5 7 5
74032024-04-20T01:09:51.656Z03/30 Read block 243 len 20480
74042024-04-20T01:09:51.656Z04/30 Read block 500 len 40960
74052024-04-20T01:09:51.656Z05/30 Read block 454 len 36864
74062024-04-20T01:09:51.656Z06/30 Read block 143 len 28672
74072024-04-20T01:09:51.656Z07/30 Write block 538 len 28672 data: 4 4 4 5 7 9 7
74082024-04-20T01:09:51.656Z08/30 Write block 353 len 28672 data: 4 4 5 5 5 6 6
74092024-04-20T01:09:51.656Z09/30 Flush
74102024-04-20T01:09:51.660Z10/30 Write block 411 len 4096 data: 8
74112024-04-20T01:09:51.660Z11/30 Write block 538 len 28672 data: 5 5 5 6 8 10 8
74122024-04-20T01:09:51.660Z12/30 Read block 82 len 28672
74132024-04-20T01:09:51.661Z13/30 Read block 30 len 36864
74142024-04-20T01:09:51.661Z14/30 Read block 565 len 8192
74152024-04-20T01:09:51.661Z15/30 Read block 17 len 12288
74162024-04-20T01:09:51.661Z16/30 Read block 88 len 12288
74172024-04-20T01:09:51.661Z17/30 Write block 351 len 8192 data: 5 5
74182024-04-20T01:09:51.661Z18/30 Write block 398 len 16384 data: 4 6 5 4
74192024-04-20T01:09:51.661Z19/30 Read block 581 len 8192
74202024-04-20T01:09:51.661Z20/30 Read block 557 len 4096
74212024-04-20T01:09:51.661Z21/30 Flush
74222024-04-20T01:09:51.666Z22/30 Write block 10 len 16384 data: 3 5 5 5
74232024-04-20T01:09:51.666Z23/30 Read block 81 len 28672
74242024-04-20T01:09:51.666Z24/30 Read block 521 len 40960
74252024-04-20T01:09:51.670Z25/30 Read block 3 len 28672
74262024-04-20T01:09:51.670Z26/30 Read block 553 len 36864
74272024-04-20T01:09:51.670Z27/30 Write block 466 len 40960 data: 6 5 5 7 7 7 7 8 9 10
74282024-04-20T01:09:51.674Z28/30 Write block 593 len 24576 data: 7 8 7 7 5 5
74292024-04-20T01:09:51.675Z29/30 Write block 62 len 4096 data: 7
74302024-04-20T01:09:51.675Z30/30 Write block 155 len 4096 data: 10
74312024-04-20T01:09:51.675Z----------------------------------------------------------------
74322024-04-20T01:09:51.675Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
74332024-04-20T01:09:51.675ZStates: Active Active Active
74342024-04-20T01:09:51.675ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74352024-04-20T01:09:51.675Z 1 Acked 1000 Write 5 Done Done Done false
74362024-04-20T01:09:51.675Z 2 Acked 1001 Write 7 Done Done Done false
74372024-04-20T01:09:51.675Z 3 Acked 1002 Read 5 Done Done Done false
74382024-04-20T01:09:51.675Z 4 Acked 1003 Read 10 Sent Done Done false
74392024-04-20T01:09:51.675Z 5 Acked 1004 Read 9 Sent Done Done false
74402024-04-20T01:09:51.675Z 6 Acked 1005 Read 7 Sent Done Done false
74412024-04-20T01:09:51.676Z 7 Acked 1006 Write 7 Sent Done Done false
74422024-04-20T01:09:51.676Z 8 Acked 1007 Write 7 Sent Done Done false
74432024-04-20T01:09:51.676Z 9 Acked 1008 Flush 0 Sent Done Done false
74442024-04-20T01:09:51.676Z 10 Acked 1009 Write 1 Sent Done Done false
74452024-04-20T01:09:51.676Z 11 Acked 1010 Write 7 Sent Done Done false
74462024-04-20T01:09:51.676Z 12 Acked 1011 Read 7 Sent Done Done false
74472024-04-20T01:09:51.676Z 13 Acked 1012 Read 9 Sent Done Done false
74482024-04-20T01:09:51.676Z 14 Acked 1013 Read 2 Sent Done Done false
74492024-04-20T01:09:51.676Z 15 Acked 1014 Read 3 Sent Done Done false
74502024-04-20T01:09:51.676Z 16 Acked 1015 Read 3 Sent Done Done false
74512024-04-20T01:09:51.676Z 17 Acked 1016 Write 2 Sent Done Done false
74522024-04-20T01:09:51.676Z 18 Acked 1017 Write 4 Sent Done Done false
74532024-04-20T01:09:51.677Z 19 Acked 1018 Read 2 Sent Done Done false
74542024-04-20T01:09:51.677Z 20 Acked 1019 Read 1 Sent Done Done false
74552024-04-20T01:09:51.677Z 21 Acked 1020 Flush 0 Sent Done Done false
74562024-04-20T01:09:51.677Z 22 Acked 1021 Write 4 Sent Done Done false
74572024-04-20T01:09:51.677Z 23 Acked 1022 Read 7 Sent Done Done false
74582024-04-20T01:09:51.677Z 24 Acked 1023 Read 10 Sent Done Done false
74592024-04-20T01:09:51.677Z 25 Acked 1024 Read 7 Sent Done Done false
74602024-04-20T01:09:51.677Z 26 Acked 1025 Read 9 Sent Done Done false
74612024-04-20T01:09:51.677Z 27 Acked 1026 Write 10 Sent Done Done false
74622024-04-20T01:09:51.677Z 28 Acked 1027 Write 6 Sent Done Done false
74632024-04-20T01:09:51.677Z 29 Acked 1028 Write 1 Sent Done Done false
74642024-04-20T01:09:51.677Z 30 Acked 1029 Write 1 Sent Sent Sent false
74652024-04-20T01:09:51.678Z STATES DS:0 DS:1 DS:2 TOTAL
74662024-04-20T01:09:51.678Z New 0 0 0 0
74672024-04-20T01:09:51.678Z Sent 27 1 1 29
74682024-04-20T01:09:51.678Z Done 3 29 29 61
74692024-04-20T01:09:51.678Z Skipped 0 0 0 0
74702024-04-20T01:09:51.678Z Error 0 0 0 0
74712024-04-20T01:09:51.678ZLast Flush: 0 1020 1020
74722024-04-20T01:09:51.678ZDownstairs last five completed:
74732024-04-20T01:09:51.678ZUpstairs last five completed: 30 29 28 27 26
74742024-04-20T01:09:51.678Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:09:50.178165104Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1443}
74752024-04-20T01:09:51.678ZWrote out file "/tmp/test_repair_verify.data"
74762024-04-20T01:09:51.682Z
74772024-04-20T01:09:51.683ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74782024-04-20T01:09:51.696ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74792024-04-20T01:09:51.696Z 0 000-019 58 58 58 98 98 98 F T T <---
74802024-04-20T01:09:51.696Z 1 020-039 67 67 67 114 114 114 F F F
74812024-04-20T01:09:51.696Z 2 040-059 64 64 64 110 110 110 F F F
74822024-04-20T01:09:51.696Z 3 060-079 61 61 61 105 105 105 F T T <---
74832024-04-20T01:09:51.696Z 4 080-099 67 67 67 114 114 114 F F F
74842024-04-20T01:09:51.696Z 5 100-119 63 63 63 109 109 109 F F F
74852024-04-20T01:09:51.696Z 6 120-139 67 67 67 114 114 114 F F F
74862024-04-20T01:09:51.696Z 7 140-159 67 67 67 114 114 114 F T T <---
74872024-04-20T01:09:51.697Z 8 160-179 67 67 67 114 114 114 F F F
74882024-04-20T01:09:51.700Z 9 180-199 67 68 68 114 115 115 T F F <---
74892024-04-20T01:09:51.700Z 10 200-219 63 68 68 109 115 115 T F F <---
74902024-04-20T01:09:51.701Z 11 220-239 67 67 67 114 114 114 F F F
74912024-04-20T01:09:51.701Z 12 240-259 67 67 67 114 114 114 F F F
74922024-04-20T01:09:51.701Z 13 260-279 64 64 64 110 110 110 F F F
74932024-04-20T01:09:51.701Z 14 280-299 53 53 53 90 90 90 F F F
74942024-04-20T01:09:51.701Z 15 300-319 67 67 67 114 114 114 F F F
74952024-04-20T01:09:51.701Z 16 320-339 67 67 67 114 114 114 F F F
74962024-04-20T01:09:51.701Z 17 340-359 66 68 68 113 116 116 F F F <---
74972024-04-20T01:09:51.701Z 18 360-379 67 67 67 114 114 114 F F F
74982024-04-20T01:09:51.701Z 19 380-399 67 68 68 114 116 116 F F F <---
74992024-04-20T01:09:51.701Z 20 400-419 67 68 68 114 116 116 F F F <---
75002024-04-20T01:09:51.701Z 21 420-439 67 67 67 114 114 114 F F F
75012024-04-20T01:09:51.701Z 22 440-459 62 62 62 108 108 108 F F F
75022024-04-20T01:09:51.702Z 23 460-479 62 62 62 108 108 108 F T T <---
75032024-04-20T01:09:51.702Z 24 480-499 56 56 56 94 94 94 F F F
75042024-04-20T01:09:51.702Z 25 500-519 67 68 68 114 115 115 T F F <---
75052024-04-20T01:09:51.702Z 26 520-539 62 68 68 107 116 116 F F F <---
75062024-04-20T01:09:51.702Z 27 540-559 58 68 68 100 116 116 F F F <---
75072024-04-20T01:09:51.702Z 28 560-579 62 62 62 107 107 107 F F F
75082024-04-20T01:09:51.702Z 29 580-599 64 64 64 111 111 111 F T T <---
75092024-04-20T01:09:51.702ZMax gen: 68, Max flush: 116
75102024-04-20T01:09:51.702ZError: Difference in extent metadata found!
75112024-04-20T01:09:51.702ZA Difference in extent metadata is expected here
75122024-04-20T01:09:51.702ZOn loop 34
75132024-04-20T01:09:51.702Z
75142024-04-20T01:09:51.706ZVerifying data now
75152024-04-20T01:10:06.310ZNo change after verify
75162024-04-20T01:10:06.310ZLoop: 34 Downstairs dump after verify (and repair):
75172024-04-20T01:10:06.323ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75182024-04-20T01:10:06.323Z 0 000-019 69 69 69 117 117 117 F F F
75192024-04-20T01:10:06.323Z 1 020-039 67 67 67 114 114 114 F F F
75202024-04-20T01:10:06.323Z 2 040-059 64 64 64 110 110 110 F F F
75212024-04-20T01:10:06.323Z 3 060-079 69 69 69 117 117 117 F F F
75222024-04-20T01:10:06.323Z 4 080-099 67 67 67 114 114 114 F F F
75232024-04-20T01:10:06.323Z 5 100-119 63 63 63 109 109 109 F F F
75242024-04-20T01:10:06.323Z 6 120-139 67 67 67 114 114 114 F F F
75252024-04-20T01:10:06.328Z 7 140-159 69 69 69 117 117 117 F F F
75262024-04-20T01:10:06.328Z 8 160-179 67 67 67 114 114 114 F F F
75272024-04-20T01:10:06.328Z 9 180-199 68 68 68 115 115 115 F F F
75282024-04-20T01:10:06.328Z 10 200-219 68 68 68 115 115 115 F F F
75292024-04-20T01:10:06.328Z 11 220-239 67 67 67 114 114 114 F F F
75302024-04-20T01:10:06.328Z 12 240-259 67 67 67 114 114 114 F F F
75312024-04-20T01:10:06.328Z 13 260-279 64 64 64 110 110 110 F F F
75322024-04-20T01:10:06.328Z 14 280-299 53 53 53 90 90 90 F F F
75332024-04-20T01:10:06.328Z 15 300-319 67 67 67 114 114 114 F F F
75342024-04-20T01:10:06.328Z 16 320-339 67 67 67 114 114 114 F F F
75352024-04-20T01:10:06.328Z 17 340-359 68 68 68 116 116 116 F F F
75362024-04-20T01:10:06.328Z 18 360-379 67 67 67 114 114 114 F F F
75372024-04-20T01:10:06.328Z 19 380-399 68 68 68 116 116 116 F F F
75382024-04-20T01:10:06.328Z 20 400-419 68 68 68 116 116 116 F F F
75392024-04-20T01:10:06.328Z 21 420-439 67 67 67 114 114 114 F F F
75402024-04-20T01:10:06.328Z 22 440-459 62 62 62 108 108 108 F F F
75412024-04-20T01:10:06.328Z 23 460-479 69 69 69 117 117 117 F F F
75422024-04-20T01:10:06.328Z 24 480-499 56 56 56 94 94 94 F F F
75432024-04-20T01:10:06.328Z 25 500-519 68 68 68 115 115 115 F F F
75442024-04-20T01:10:06.328Z 26 520-539 68 68 68 116 116 116 F F F
75452024-04-20T01:10:06.328Z 27 540-559 68 68 68 116 116 116 F F F
75462024-04-20T01:10:06.329Z 28 560-579 62 62 62 107 107 107 F F F
75472024-04-20T01:10:06.329Z 29 580-599 69 69 69 117 117 117 F F F
75482024-04-20T01:10:06.329ZMax gen: 69, Max flush: 117
75492024-04-20T01:10:06.329Z
75502024-04-20T01:10:06.329ZBegin loop 35
75512024-04-20T01:10:06.329ZDownstairs to restart: 2
75522024-04-20T01:10:06.345Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.850572175Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451}
75532024-04-20T01:10:06.354Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.850802875Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451}
75542024-04-20T01:10:06.354Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.850848126Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451}
75552024-04-20T01:10:06.354Z{"msg":"Crucible bb4af6c3-dd00-4cfa-a99b-c357d0762dcf has session id: 70e80408-3c50-409f-a4e8-959a7317bdc3","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.850870195Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75562024-04-20T01:10:06.354Z{"msg":"Upstairs opts: Upstairs UUID: bb4af6c3-dd00-4cfa-a99b-c357d0762dcf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","Crucible runtime is spawned
75572024-04-20T01:10:06.354Zlevel":30,"time":"2024-04-20T01:10:04.850929936Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75582024-04-20T01:10:06.354Z{"msg":"Crucible stats registered with UUID: bb4af6c3-dd00-4cfa-a99b-c357d0762dcf","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.850974346Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75592024-04-20T01:10:06.354Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851030136Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451}
75602024-04-20T01:10:06.354Z{"msg":"bb4af6c3-dd00-4cfa-a99b-c357d0762dcf active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851072086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75612024-04-20T01:10:06.354Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851095146Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"0","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75622024-04-20T01:10:06.354Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851139316Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"0","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75632024-04-20T01:10:06.354Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851176936Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"1","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75642024-04-20T01:10:06.354Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851255826Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"1","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75652024-04-20T01:10:06.354Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851294176Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"2","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75662024-04-20T01:10:06.354Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851321096Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"2","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75672024-04-20T01:10:06.354Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851355036Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75682024-04-20T01:10:06.354Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851382327Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75692024-04-20T01:10:06.354Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851406337Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75702024-04-20T01:10:06.354Z{"msg":"ds_connection connected from Ok(127.0.0.1:36193)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851429077Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75712024-04-20T01:10:06.354Z{"msg":"ds_connection connected from Ok(127.0.0.1:40860)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851605767Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75722024-04-20T01:10:06.354Z{"msg":"ds_connection connected from Ok(127.0.0.1:59029)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.851657757Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75732024-04-20T01:10:06.354Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.852236438Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"0","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75742024-04-20T01:10:06.355Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.852290649Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"0","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75752024-04-20T01:10:06.355Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.852324399Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"1","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75762024-04-20T01:10:06.355Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.852380269Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"1","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75772024-04-20T01:10:06.355Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.852429759Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"2","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75782024-04-20T01:10:06.355Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.852461699Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"2","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75792024-04-20T01:10:06.355Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.8528448Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"1","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75802024-04-20T01:10:06.355Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.85290485Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"2","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75812024-04-20T01:10:06.355Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.85300731Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"0","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75822024-04-20T01:10:06.355Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853440521Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"0","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75832024-04-20T01:10:06.355Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853475901Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"0","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75842024-04-20T01:10:06.355Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853491471Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75852024-04-20T01:10:06.355Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853504341Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"2","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75862024-04-20T01:10:06.355Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853525272Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"2","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75872024-04-20T01:10:06.355Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853538672Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75882024-04-20T01:10:06.355Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853551152Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"1","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75892024-04-20T01:10:06.355Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853564232Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"1","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75902024-04-20T01:10:06.355Z{"msg":"[0]R flush_numbers[0..12]: [117, 114, 110, 117, 114, 109, 114, 117, 114, 115, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853578601Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75912024-04-20T01:10:06.355Z{"msg":"[0]R generation[0..12]: [69, 67, 64, 69, 67, 63, 67, 69, 67, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853593152Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75922024-04-20T01:10:06.355Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853614162Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75932024-04-20T01:10:06.355Z{"msg":"[1]R flush_numbers[0..12]: [117, 114, 110, 117, 114, 109, 114, 117, 114, 115, 115, 114]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
75942024-04-20T01:10:06.355Z,"time":"2024-04-20T01:10:04.853628492Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75952024-04-20T01:10:06.355Z{"msg":"[1]R generation[0..12]: [69, 67, 64, 69, 67, 63, 67, 69, 67, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853646432Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75962024-04-20T01:10:06.355Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853667462Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75972024-04-20T01:10:06.356Z{"msg":"[2]R flush_numbers[0..12]: [117, 114, 110, 117, 114, 109, 114, 117, 114, 115, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853681712Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75982024-04-20T01:10:06.356Z{"msg":"[2]R generation[0..12]: [69, 67, 64, 69, 67, 63, 67, 69, 67, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853702182Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
75992024-04-20T01:10:06.356Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853716552Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76002024-04-20T01:10:06.356Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853736842Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76012024-04-20T01:10:06.356Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853750342Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76022024-04-20T01:10:06.356Z{"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853769852Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76032024-04-20T01:10:06.356Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853783092Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"":"downstairs","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76042024-04-20T01:10:06.356Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853803352Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76052024-04-20T01:10:06.356Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853815672Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"0","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76062024-04-20T01:10:06.356Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853829162Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"1","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76072024-04-20T01:10:06.356Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853848772Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"client":"2","":"downstairs","session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76082024-04-20T01:10:06.356Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853862202Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76092024-04-20T01:10:06.356Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853881342Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76102024-04-20T01:10:06.356Z{"msg":"bb4af6c3-dd00-4cfa-a99b-c357d0762dcf is now active with session: 70e80408-3c50-409f-a4e8-959a7317bdc3","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853894532Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76112024-04-20T01:10:06.356Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853906972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451,"session_id":"70e80408-3c50-409f-a4e8-959a7317bdc3"}
76122024-04-20T01:10:06.356Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:04.853925912Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451}
76132024-04-20T01:10:06.356ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
76142024-04-20T01:10:06.356ZLoading write count information from file "/tmp/test_repair_verify.data"
76152024-04-20T01:10:06.356ZRun Repair workload
76162024-04-20T01:10:06.361Z01/30 Write block 498 len 28672 data: 4 4 4 4 4 6 6
76172024-04-20T01:10:06.361Z02/30 Read block 489 len 8192
76182024-04-20T01:10:06.361Z03/30 Write block 270 len 24576 data: 5 5 6 7 6 5
76192024-04-20T01:10:06.361Z04/30 Read block 328 len 24576
76202024-04-20T01:10:06.361Z05/30 Read block 146 len 20480
76212024-04-20T01:10:06.361Z06/30 Write block 99 len 24576 data: 6 8 8 7 7 5
76222024-04-20T01:10:06.361Z07/30 Read block 230 len 40960
76232024-04-20T01:10:06.361Z08/30 Write block 585 len 8192 data: 9 11
76242024-04-20T01:10:06.361Z09/30 Write block 86 len 28672 data: 7 7 7 8 9 7 6
76252024-04-20T01:10:06.361Z10/30 Read block 242 len 28672
76262024-04-20T01:10:06.361Z11/30 Flush
76272024-04-20T01:10:06.365Z12/30 Write block 50 len 24576 data: 9 11 11 11 8 6
76282024-04-20T01:10:06.365Z13/30 Read block 359 len 28672
76292024-04-20T01:10:06.365Z14/30 Flush
76302024-04-20T01:10:06.368Z15/30 Write block 524 len 36864 data: 8 7 6 5 4 4 4 6 5
76312024-04-20T01:10:06.368Z16/30 Read block 489 len 8192
76322024-04-20T01:10:06.368Z17/30 Flush
76332024-04-20T01:10:06.368Z18/30 Write block 77 len 16384 data: 7 7 7 6
76342024-04-20T01:10:06.372Z19/30 Write block 118 len 32768 data: 4 5 5 5 6 7 9 10
76352024-04-20T01:10:06.372Z20/30 Write block 136 len 40960 data: 8 8 9 8 8 7 6 6 6 6
76362024-04-20T01:10:06.372Z21/30 Write block 577 len 20480 data: 5 5 7 7 7
76372024-04-20T01:10:06.376Z22/30 Write block 532 len 24576 data: 6 4 5 4 3 3
76382024-04-20T01:10:06.376Z23/30 Read block 130 len 32768
76392024-04-20T01:10:06.376Z24/30 Flush
76402024-04-20T01:10:06.384Z25/30 Write block 271 len 40960 data: 6 7 8 7 6 3 5 5 3 5
76412024-04-20T01:10:06.384Z26/30 Write block 301 len 24576 data: 5 5 5 5 4 3
76422024-04-20T01:10:06.388Z27/30 Write block 132 len 8192 data: 10 8
76432024-04-20T01:10:06.388Z28/30 Read block 206 len 8192
76442024-04-20T01:10:06.388Z29/30 Read block 183 len 24576
76452024-04-20T01:10:06.388Z30/30 Write block 495 len 24576 data: 6 5 4 5 5 5
76462024-04-20T01:10:06.388Z----------------------------------------------------------------
76472024-04-20T01:10:06.388Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
76482024-04-20T01:10:06.388ZStates: Active Active Active
76492024-04-20T01:10:06.388ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76502024-04-20T01:10:06.388Z 1 Acked 1000 Write 7 Done Done Sent false
76512024-04-20T01:10:06.388Z 2 Acked 1001 Read 2 Done Done Sent false
76522024-04-20T01:10:06.388Z 3 Acked 1002 Write 6 Done Done Sent false
76532024-04-20T01:10:06.388Z 4 Acked 1003 Read 6 Done Done Sent false
76542024-04-20T01:10:06.389Z 5 Acked 1004 Read 5 Done Done Sent false
76552024-04-20T01:10:06.389Z 6 Acked 1005 Write 6 Done Done Sent false
76562024-04-20T01:10:06.389Z 7 Acked 1006 Read 10 Done Done Sent false
76572024-04-20T01:10:06.389Z 8 Acked 1007 Write 2 Done Done Sent false
76582024-04-20T01:10:06.389Z 9 Acked 1008 Write 7 Done Done Sent false
76592024-04-20T01:10:06.389Z 10 Acked 1009 Read 7 Done Done Sent false
76602024-04-20T01:10:06.389Z 11 Acked 1010 Flush 0 Done Done Sent false
76612024-04-20T01:10:06.389Z 12 Acked 1011 Write 6 Done Done Sent false
76622024-04-20T01:10:06.389Z 13 Acked 1012 Read 7 Done Done Sent false
76632024-04-20T01:10:06.389Z 14 Acked 1013 Flush 0 Done Done Sent false
76642024-04-20T01:10:06.389Z 15 Acked 1014 Write 9 Done Done Sent false
76652024-04-20T01:10:06.389Z 16 Acked 1015 Read 2 Done Done Sent false
76662024-04-20T01:10:06.389Z 17 Acked 1016 Flush 0 Done Done Sent false
76672024-04-20T01:10:06.389Z 18 Acked 1017 Write 4 Done Done Sent false
76682024-04-20T01:10:06.389Z 19 Acked 1018 Write 8 Done Done Sent false
76692024-04-20T01:10:06.389Z 20 Acked 1019 Write 10 Done Done Sent false
76702024-04-20T01:10:06.389Z 21 Acked 1020 Write 5 Done Done Sent false
76712024-04-20T01:10:06.389Z 22 Acked 1021 Write 6 Done Done Sent false
76722024-04-20T01:10:06.389Z 23 Acked 1022 Read 8 Done Done Sent false
76732024-04-20T01:10:06.389Z 24 Acked 1023 Flush 0 Done Done Sent false
76742024-04-20T01:10:06.389Z 25 Acked 1024 Write 10 Done Done Sent false
76752024-04-20T01:10:06.389Z 26 Acked 1025 Write 6 Done Done Sent false
76762024-04-20T01:10:06.389Z 27 Acked 1026 Write 2 Done Done Sent false
76772024-04-20T01:10:06.389Z 28 Acked 1027 Read 2 Done Done Sent false
76782024-04-20T01:10:06.390Z 29 Acked 1028 Read 6 Done Done Sent false
76792024-04-20T01:10:06.390Z 30 Acked 1029 Write 6 Sent Sent Sent false
76802024-04-20T01:10:06.390Z STATES DS:0 DS:1 DS:2 TOTAL
76812024-04-20T01:10:06.390Z New 0 0 0 0
76822024-04-20T01:10:06.390Z Sent 1 1 30 32
76832024-04-20T01:10:06.390Z Done 29 29 0 58
76842024-04-20T01:10:06.390Z Skipped 0 0 0 0
76852024-04-20T01:10:06.390Z Error 0 0 0 0
76862024-04-20T01:10:06.390ZLast Flush: 1023 1023 0
76872024-04-20T01:10:06.390ZDownstairs last five completed:
76882024-04-20T01:10:06.390ZUpstairs last five completed: 30 29 28 27 26
76892024-04-20T01:10:06.390Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:10:04.889393351Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1451}
76902024-04-20T01:10:06.390ZWrote out file "/tmp/test_repair_verify.data"
76912024-04-20T01:10:06.390Z
76922024-04-20T01:10:06.394ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76932024-04-20T01:10:06.407ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76942024-04-20T01:10:06.407Z 0 000-019 69 69 69 117 117 117 F F F
76952024-04-20T01:10:06.407Z 1 020-039 67 67 67 114 114 114 F F F
76962024-04-20T01:10:06.407Z 2 040-059 70 70 64 119 119 110 F F F <---
76972024-04-20T01:10:06.412Z 3 060-079 70 70 69 121 121 117 F F F <---
76982024-04-20T01:10:06.412Z 4 080-099 70 70 67 121 121 114 F F F <---
76992024-04-20T01:10:06.412Z 5 100-119 70 70 63 121 121 109 F F F <---
77002024-04-20T01:10:06.412Z 6 120-139 70 70 67 121 121 114 T T F <---
77012024-04-20T01:10:06.412Z 7 140-159 70 70 69 121 121 117 F F F <---
77022024-04-20T01:10:06.412Z 8 160-179 67 67 67 114 114 114 F F F
77032024-04-20T01:10:06.412Z 9 180-199 68 68 68 115 115 115 F F F
77042024-04-20T01:10:06.412Z 10 200-219 68 68 68 115 115 115 F F F
77052024-04-20T01:10:06.412Z 11 220-239 67 67 67 114 114 114 F F F
77062024-04-20T01:10:06.412Z 12 240-259 67 67 67 114 114 114 F F F
77072024-04-20T01:10:06.412Z 13 260-279 70 70 64 118 118 110 T T F <---
77082024-04-20T01:10:06.412Z 14 280-299 53 53 53 90 90 90 T T F <---
77092024-04-20T01:10:06.413Z 15 300-319 67 67 67 114 114 114 T T F <---
77102024-04-20T01:10:06.413Z 16 320-339 67 67 67 114 114 114 F F F
77112024-04-20T01:10:06.413Z 17 340-359 68 68 68 116 116 116 F F F
77122024-04-20T01:10:06.413Z 18 360-379 67 67 67 114 114 114 F F F
77132024-04-20T01:10:06.413Z 19 380-399 68 68 68 116 116 116 F F F
77142024-04-20T01:10:06.413Z 20 400-419 68 68 68 116 116 116 F F F
77152024-04-20T01:10:06.413Z 21 420-439 67 67 67 114 114 114 F F F
77162024-04-20T01:10:06.413Z 22 440-459 62 62 62 108 108 108 F F F
77172024-04-20T01:10:06.413Z 23 460-479 69 69 69 117 117 117 F F F
77182024-04-20T01:10:06.413Z 24 480-499 70 70 56 118 118 94 T T F <---
77192024-04-20T01:10:06.413Z 25 500-519 70 70 68 118 118 115 T T F <---
77202024-04-20T01:10:06.413Z 26 520-539 70 70 68 121 121 116 F F F <---
77212024-04-20T01:10:06.413Z 27 540-559 68 68 68 116 116 116 F F F
77222024-04-20T01:10:06.413Z 28 560-579 70 70 62 121 121 107 F F F <---
77232024-04-20T01:10:06.413Z 29 580-599 70 70 69 121 121 117 F F F <---
77242024-04-20T01:10:06.413ZMax gen: 70, Max flush: 121
77252024-04-20T01:10:06.413ZError: Difference in extent metadata found!
77262024-04-20T01:10:06.413ZA Difference in extent metadata is expected here
77272024-04-20T01:10:06.413ZOn loop 35
77282024-04-20T01:10:06.413Z
77292024-04-20T01:10:06.418ZVerifying data now
77302024-04-20T01:10:11.051ZNo change after verify
77312024-04-20T01:10:11.051ZLoop: 35 Downstairs dump after verify (and repair):
77322024-04-20T01:10:11.065ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77332024-04-20T01:10:11.065Z 0 000-019 69 69 69 117 117 117 F F F
77342024-04-20T01:10:11.065Z 1 020-039 67 67 67 114 114 114 F F F
77352024-04-20T01:10:11.065Z 2 040-059 70 70 70 119 119 119 F F F
77362024-04-20T01:10:11.065Z 3 060-079 70 70 70 121 121 121 F F F
77372024-04-20T01:10:11.066Z 4 080-099 70 70 70 121 121 121 F F F
77382024-04-20T01:10:11.069Z 5 100-119 70 70 70 121 121 121 F F F
77392024-04-20T01:10:11.069Z 6 120-139 71 71 71 122 122 122 F F F
77402024-04-20T01:10:11.070Z 7 140-159 70 70 70 121 121 121 F F F
77412024-04-20T01:10:11.070Z 8 160-179 67 67 67 114 114 114 F F F
77422024-04-20T01:10:11.070Z 9 180-199 68 68 68 115 115 115 F F F
77432024-04-20T01:10:11.070Z 10 200-219 68 68 68 115 115 115 F F F
77442024-04-20T01:10:11.070Z 11 220-239 67 67 67 114 114 114 F F F
77452024-04-20T01:10:11.070Z 12 240-259 67 67 67 114 114 114 F F F
77462024-04-20T01:10:11.070Z 13 260-279 71 71 71 122 122 122 F F F
77472024-04-20T01:10:11.070Z 14 280-299 71 71 71 122 122 122 F F F
77482024-04-20T01:10:11.070Z 15 300-319 71 71 71 122 122 122 F F F
77492024-04-20T01:10:11.070Z 16 320-339 67 67 67 114 114 114 F F F
77502024-04-20T01:10:11.070Z 17 340-359 68 68 68 116 116 116 F F F
77512024-04-20T01:10:11.070Z 18 360-379 67 67 67 114 114 114 F F F
77522024-04-20T01:10:11.070Z 19 380-399 68 68 68 116 116 116 F F F
77532024-04-20T01:10:11.070Z 20 400-419 68 68 68 116 116 116 F F F
77542024-04-20T01:10:11.070Z 21 420-439 67 67 67 114 114 114 F F F
77552024-04-20T01:10:11.070Z 22 440-459 62 62 62 108 108 108 F F F
77562024-04-20T01:10:11.070Z 23 460-479 69 69 69 117 117 117 F F F
77572024-04-20T01:10:11.070Z 24 480-499 71 71 71 122 122 122 F F F
77582024-04-20T01:10:11.070Z 25 500-519 71 71 71 122 122 122 F F F
77592024-04-20T01:10:11.070Z 26 520-539 70 70 70 121 121 121 F F F
77602024-04-20T01:10:11.070Z 27 540-559 68 68 68 116 116 116 F F F
77612024-04-20T01:10:11.070Z 28 560-579 70 70 70 121 121 121 F F F
77622024-04-20T01:10:11.070Z 29 580-599 70 70 70 121 121 121 F F F
77632024-04-20T01:10:11.071ZMax gen: 71, Max flush: 122
77642024-04-20T01:10:11.071Z
77652024-04-20T01:10:11.071ZBegin loop 36
77662024-04-20T01:10:11.071ZDownstairs to restart: 0
77672024-04-20T01:10:11.088Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.593478153Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459}
77682024-04-20T01:10:11.095Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.593742263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459}
77692024-04-20T01:10:11.095Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.593785353Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459}
77702024-04-20T01:10:11.095Z{"msg":"Crucible d76f5af5-519f-4dd5-aaf4-28e8a29a1209 has session id: 297f6f8b-37f7-42e6-b65c-0c0bd7c636ba","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.593808993Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77712024-04-20T01:10:11.095Z{"msg":"Upstairs opts: Upstairs UUID: d76f5af5-519f-4dd5-aaf4-28e8a29a1209, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
77722024-04-20T01:10:11.095Z,"time":"2024-04-20T01:10:09.593873344Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77732024-04-20T01:10:11.095Z{"msg":"Crucible stats registered with UUID: d76f5af5-519f-4dd5-aaf4-28e8a29a1209","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.593899944Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77742024-04-20T01:10:11.095Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.593932774Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459}
77752024-04-20T01:10:11.095Z{"msg":"d76f5af5-519f-4dd5-aaf4-28e8a29a1209 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594007884Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77762024-04-20T01:10:11.095Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594032844Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"0","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77772024-04-20T01:10:11.095Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594048164Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"0","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77782024-04-20T01:10:11.095Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594067234Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"1","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77792024-04-20T01:10:11.095Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594103314Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"1","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77802024-04-20T01:10:11.095Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594129064Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"2","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77812024-04-20T01:10:11.095Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594188924Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"2","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77822024-04-20T01:10:11.095Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594219224Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77832024-04-20T01:10:11.096Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594245114Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77842024-04-20T01:10:11.096Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594284854Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77852024-04-20T01:10:11.096Z{"msg":"ds_connection connected from Ok(127.0.0.1:60985)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594363155Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77862024-04-20T01:10:11.096Z{"msg":"ds_connection connected from Ok(127.0.0.1:52177)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594409905Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77872024-04-20T01:10:11.096Z{"msg":"ds_connection connected from Ok(127.0.0.1:46871)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.594427835Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77882024-04-20T01:10:11.096Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.595152986Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"1","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77892024-04-20T01:10:11.096Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.595197047Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"1","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77902024-04-20T01:10:11.096Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.595223527Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"2","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77912024-04-20T01:10:11.096Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.595268087Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"2","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77922024-04-20T01:10:11.096Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.595816178Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"2","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77932024-04-20T01:10:11.096Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.595861248Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"1","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77942024-04-20T01:10:11.096Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.595991368Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"0","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77952024-04-20T01:10:11.096Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.596036688Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"0","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77962024-04-20T01:10:11.096Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.596264489Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"2","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77972024-04-20T01:10:11.096Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.596302179Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"2","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77982024-04-20T01:10:11.096Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.596347309Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
77992024-04-20T01:10:11.096Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.596392389Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"1","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78002024-04-20T01:10:11.096Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.596408709Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"1","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78012024-04-20T01:10:11.096Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.596430549Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78022024-04-20T01:10:11.096Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.59664642Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"0","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78032024-04-20T01:10:11.096Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597060691Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"0","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78042024-04-20T01:10:11.096Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597101151Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"0","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78052024-04-20T01:10:11.097Z{"msg":"[0]R flush_numbers[0..12]: [117, 114, 119, 121, 121, 121, 122, 121, 114, 115, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597117991Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78062024-04-20T01:10:11.097Z{"msg":"[0]R generation[0..12]: [69, 67, 70, 70, 70, 70, 71, 70, 67, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597139371Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78072024-04-20T01:10:11.097Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597154431Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78082024-04-20T01:10:11.097Z{"msg":"[1]R flush_numbers[0..12]: [117, 114, 119, 121, 121, 121, 122, 121, 114, 115, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597175031Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78092024-04-20T01:10:11.097Z{"msg":"[1]R generation[0..12]: [69, 67, 70, 70, 70, 70, 71, 70, 67, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597189281Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78102024-04-20T01:10:11.097ZWait for a query_work_queue command to finish before sending IO
78112024-04-20T01:10:11.097Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597214931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78122024-04-20T01:10:11.097Z{"msg":"[2]R flush_numbers[0..12]: [117, 114, 119, 121, 121, 121, 122, 121, 114, 115, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597230261Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78132024-04-20T01:10:11.097Z{"msg":"[2]R generation[0..12]: [69, 67, 70, 70, 70, 70, 71, 70, 67, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597244341Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78142024-04-20T01:10:11.097Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597266481Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78152024-04-20T01:10:11.097Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597279751Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78162024-04-20T01:10:11.097Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597299541Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78172024-04-20T01:10:11.097Z{"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597312821Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78182024-04-20T01:10:11.097Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597332381Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"":"downstairs","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78192024-04-20T01:10:11.097Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597345831Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78202024-04-20T01:10:11.097Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597364051Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"0","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78212024-04-20T01:10:11.097Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597377591Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"1","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78222024-04-20T01:10:11.097Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597397261Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"client":"2","":"downstairs","session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78232024-04-20T01:10:11.097Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597411021Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78242024-04-20T01:10:11.097Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597430172Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78252024-04-20T01:10:11.101Z{"msg":"d76f5af5-519f-4dd5-aaf4-28e8a29a1209 is now active with session: 297f6f8b-37f7-42e6-b65c-0c0bd7c636ba","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597444112Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78262024-04-20T01:10:11.101Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597456041Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459,"session_id":"297f6f8b-37f7-42e6-b65c-0c0bd7c636ba"}
78272024-04-20T01:10:11.101Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:09.597468012Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459}
78282024-04-20T01:10:11.101ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
78292024-04-20T01:10:11.101ZLoading write count information from file "/tmp/test_repair_verify.data"
78302024-04-20T01:10:11.102ZRun Repair workload
78312024-04-20T01:10:11.102Z01/30 Write block 147 len 32768 data: 4 5 4 5 6 5 6 8
78322024-04-20T01:10:11.102Z02/30 Read block 258 len 40960
78332024-04-20T01:10:11.102Z03/30 Write block 454 len 40960 data: 7 10 10 12 10 8 7 10 7 7
78342024-04-20T01:10:11.102Z04/30 Write block 323 len 16384 data: 9 9 8 9
78352024-04-20T01:10:11.102Z05/30 Write block 142 len 32768 data: 7 7 7 7 6 5 6 5
78362024-04-20T01:10:11.102Z06/30 Read block 493 len 32768
78372024-04-20T01:10:11.102Z07/30 Write block 157 len 20480 data: 8 8 6 7 7
78382024-04-20T01:10:11.102Z08/30 Write block 558 len 20480 data: 4 4 3 4 5
78392024-04-20T01:10:11.105Z09/30 Read block 235 len 20480
78402024-04-20T01:10:11.105Z10/30 Write block 540 len 12288 data: 6 7 9
78412024-04-20T01:10:11.106Z11/30 Flush
78422024-04-20T01:10:11.109Z12/30 Write block 223 len 16384 data: 8 8 8 11
78432024-04-20T01:10:11.109Z13/30 Read block 135 len 32768
78442024-04-20T01:10:11.109Z14/30 Read block 97 len 4096
78452024-04-20T01:10:11.113Z15/30 Write block 560 len 40960 data: 4 5 6 4 5 5 6 5 6 7
78462024-04-20T01:10:11.113Z16/30 Write block 330 len 4096 data: 12
78472024-04-20T01:10:11.113Z17/30 Read block 182 len 36864
78482024-04-20T01:10:11.113Z18/30 Write block 509 len 16384 data: 10 9 9 11
78492024-04-20T01:10:11.113Z19/30 Read block 216 len 40960
78502024-04-20T01:10:11.117Z20/30 Write block 584 len 12288 data: 7 10 12
78512024-04-20T01:10:11.117Z21/30 Read block 247 len 40960
78522024-04-20T01:10:11.117Z22/30 Read block 468 len 8192
78532024-04-20T01:10:11.117Z23/30 Read block 575 len 4096
78542024-04-20T01:10:11.117Z24/30 Write block 68 len 16384 data: 8 9 6 6
78552024-04-20T01:10:11.117Z25/30 Read block 115 len 24576
78562024-04-20T01:10:11.117Z26/30 Read block 568 len 28672
78572024-04-20T01:10:11.122Z27/30 Write block 100 len 36864 data: 9 9 8 8 6 5 6 6 5
78582024-04-20T01:10:11.122Z28/30 Write block 415 len 20480 data: 9 10 8 9 9
78592024-04-20T01:10:11.122Z29/30 Write block 279 len 12288 data: 4 6 5
78602024-04-20T01:10:11.122Z30/30 Read block 41 len 20480
78612024-04-20T01:10:11.126Z----------------------------------------------------------------
78622024-04-20T01:10:11.126Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
78632024-04-20T01:10:11.126ZStates: Active Active Active
78642024-04-20T01:10:11.126ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78652024-04-20T01:10:11.126Z 1 Acked 1000 Write 8 Done Done Done false
78662024-04-20T01:10:11.127Z 2 Acked 1001 Read 10 Done Done Done false
78672024-04-20T01:10:11.127Z 3 Acked 1002 Write 10 Done Done Done false
78682024-04-20T01:10:11.127Z 4 Acked 1003 Write 4 Done Done Done false
78692024-04-20T01:10:11.127Z 5 Acked 1004 Write 8 Done Done Done false
78702024-04-20T01:10:11.127Z 6 Acked 1005 Read 8 Sent Done Done false
78712024-04-20T01:10:11.127Z 7 Acked 1006 Write 5 Sent Done Done false
78722024-04-20T01:10:11.127Z 8 Acked 1007 Write 5 Sent Done Done false
78732024-04-20T01:10:11.127Z 9 Acked 1008 Read 5 Sent Done Done false
78742024-04-20T01:10:11.127Z 10 Acked 1009 Write 3 Sent Done Done false
78752024-04-20T01:10:11.127Z 11 Acked 1010 Flush 0 Sent Done Done false
78762024-04-20T01:10:11.127Z 12 Acked 1011 Write 4 Sent Done Done false
78772024-04-20T01:10:11.127Z 13 Acked 1012 Read 8 Sent Done Done false
78782024-04-20T01:10:11.127Z 14 Acked 1013 Read 1 Sent Done Done false
78792024-04-20T01:10:11.127Z 15 Acked 1014 Write 10 Sent Done Done false
78802024-04-20T01:10:11.127Z 16 Acked 1015 Write 1 Sent Done Done false
78812024-04-20T01:10:11.127Z 17 Acked 1016 Read 9 Sent Done Done false
78822024-04-20T01:10:11.127Z 18 Acked 1017 Write 4 Sent Done Done false
78832024-04-20T01:10:11.127Z 19 Acked 1018 Read 10 Sent Done Done false
78842024-04-20T01:10:11.127Z 20 Acked 1019 Write 3 Sent Done Done false
78852024-04-20T01:10:11.127Z 21 Acked 1020 Read 10 Sent Done Done false
78862024-04-20T01:10:11.127Z 22 Acked 1021 Read 2 Sent Done Done false
78872024-04-20T01:10:11.127Z 23 Acked 1022 Read 1 Sent Done Done false
78882024-04-20T01:10:11.128Z 24 Acked 1023 Write 4 Sent Done Done false
78892024-04-20T01:10:11.128Z 25 Acked 1024 Read 6 Sent Done Done false
78902024-04-20T01:10:11.128Z 26 Acked 1025 Read 7 Sent Done Done false
78912024-04-20T01:10:11.128Z 27 Acked 1026 Write 9 Sent Done Done false
78922024-04-20T01:10:11.128Z 28 Acked 1027 Write 5 Sent Done Done false
78932024-04-20T01:10:11.128Z 29 Acked 1028 Write 3 Sent Done Done false
78942024-04-20T01:10:11.128Z 30 Acked 1029 Read 5 Sent Sent Done false
78952024-04-20T01:10:11.128Z STATES DS:0 DS:1 DS:2 TOTAL
78962024-04-20T01:10:11.128Z New 0 0 0 0
78972024-04-20T01:10:11.128Z Sent 25 1 0 26
78982024-04-20T01:10:11.128Z Done 5 29 30 64
78992024-04-20T01:10:11.128Z Skipped 0 0 0 0
79002024-04-20T01:10:11.128Z Error 0 0 0 0
79012024-04-20T01:10:11.128ZLast Flush: 0 1010 1010
79022024-04-20T01:10:11.128ZDownstairs last five completed:
79032024-04-20T01:10:11.128ZUpstairs last five completed: 30 29 28 27 26
79042024-04-20T01:10:11.128Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:10:09.627698759Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1459}
79052024-04-20T01:10:11.128ZWrote out file "/tmp/test_repair_verify.data"
79062024-04-20T01:10:11.128Z
79072024-04-20T01:10:11.132ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
79082024-04-20T01:10:11.148ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79092024-04-20T01:10:11.148Z 0 000-019 69 69 69 117 117 117 F F F
79102024-04-20T01:10:11.148Z 1 020-039 67 67 67 114 114 114 F F F
79112024-04-20T01:10:11.148Z 2 040-059 70 70 70 119 119 119 F F F
79122024-04-20T01:10:11.148Z 3 060-079 70 70 70 121 121 121 F T T <---
79132024-04-20T01:10:11.148Z 4 080-099 70 70 70 121 121 121 F F F
79142024-04-20T01:10:11.148Z 5 100-119 70 70 70 121 121 121 F T T <---
79152024-04-20T01:10:11.148Z 6 120-139 71 71 71 122 122 122 F F F
79162024-04-20T01:10:11.148Z 7 140-159 70 72 72 121 123 123 T F F <---
79172024-04-20T01:10:11.155Z 8 160-179 67 72 72 114 123 123 F F F <---
79182024-04-20T01:10:11.155Z 9 180-199 68 68 68 115 115 115 F F F
79192024-04-20T01:10:11.155Z 10 200-219 68 68 68 115 115 115 F F F
79202024-04-20T01:10:11.155Z 11 220-239 67 67 67 114 114 114 F T T <---
79212024-04-20T01:10:11.155Z 12 240-259 67 67 67 114 114 114 F F F
79222024-04-20T01:10:11.155Z 13 260-279 71 71 71 122 122 122 F T T <---
79232024-04-20T01:10:11.155Z 14 280-299 71 71 71 122 122 122 F T T <---
79242024-04-20T01:10:11.155Z 15 300-319 71 71 71 122 122 122 F F F
79252024-04-20T01:10:11.155Z 16 320-339 67 72 72 114 123 123 T T T <---
79262024-04-20T01:10:11.155Z 17 340-359 68 68 68 116 116 116 F F F
79272024-04-20T01:10:11.155Z 18 360-379 67 67 67 114 114 114 F F F
79282024-04-20T01:10:11.155Z 19 380-399 68 68 68 116 116 116 F F F
79292024-04-20T01:10:11.155Z 20 400-419 68 68 68 116 116 116 F T T <---
79302024-04-20T01:10:11.155Z 21 420-439 67 67 67 114 114 114 F F F
79312024-04-20T01:10:11.155Z 22 440-459 62 72 72 108 123 123 T F F <---
79322024-04-20T01:10:11.155Z 23 460-479 69 72 72 117 123 123 T F F <---
79332024-04-20T01:10:11.155Z 24 480-499 71 71 71 122 122 122 F F F
79342024-04-20T01:10:11.155Z 25 500-519 71 71 71 122 122 122 F T T <---
79352024-04-20T01:10:11.155Z 26 520-539 70 70 70 121 121 121 F F F
79362024-04-20T01:10:11.155Z 27 540-559 68 72 72 116 123 123 F F F <---
79372024-04-20T01:10:11.156Z 28 560-579 70 72 72 121 123 123 F T T <---
79382024-04-20T01:10:11.156Z 29 580-599 70 70 70 121 121 121 F T T <---
79392024-04-20T01:10:11.156ZMax gen: 72, Max flush: 123
79402024-04-20T01:10:11.156ZError: Difference in extent metadata found!
79412024-04-20T01:10:11.156ZA Difference in extent metadata is expected here
79422024-04-20T01:10:11.156ZOn loop 36
79432024-04-20T01:10:11.156Z
79442024-04-20T01:10:11.156ZVerifying data now
79452024-04-20T01:10:25.855ZNo change after verify
79462024-04-20T01:10:25.856ZLoop: 36 Downstairs dump after verify (and repair):
79472024-04-20T01:10:25.867ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79482024-04-20T01:10:25.867Z 0 000-019 69 69 69 117 117 117 F F F
79492024-04-20T01:10:25.867Z 1 020-039 67 67 67 114 114 114 F F F
79502024-04-20T01:10:25.867Z 2 040-059 70 70 70 119 119 119 F F F
79512024-04-20T01:10:25.867Z 3 060-079 73 73 73 124 124 124 F F F
79522024-04-20T01:10:25.867Z 4 080-099 70 70 70 121 121 121 F F F
79532024-04-20T01:10:25.873Z 5 100-119 73 73 73 124 124 124 F F F
79542024-04-20T01:10:25.873Z 6 120-139 71 71 71 122 122 122 F F F
79552024-04-20T01:10:25.874Z 7 140-159 72 72 72 123 123 123 F F F
79562024-04-20T01:10:25.874Z 8 160-179 72 72 72 123 123 123 F F F
79572024-04-20T01:10:25.874Z 9 180-199 68 68 68 115 115 115 F F F
79582024-04-20T01:10:25.874Z 10 200-219 68 68 68 115 115 115 F F F
79592024-04-20T01:10:25.874Z 11 220-239 73 73 73 124 124 124 F F F
79602024-04-20T01:10:25.874Z 12 240-259 67 67 67 114 114 114 F F F
79612024-04-20T01:10:25.874Z 13 260-279 73 73 73 124 124 124 F F F
79622024-04-20T01:10:25.874Z 14 280-299 73 73 73 124 124 124 F F F
79632024-04-20T01:10:25.874Z 15 300-319 71 71 71 122 122 122 F F F
79642024-04-20T01:10:25.874Z 16 320-339 73 73 73 124 124 124 F F F
79652024-04-20T01:10:25.874Z 17 340-359 68 68 68 116 116 116 F F F
79662024-04-20T01:10:25.874Z 18 360-379 67 67 67 114 114 114 F F F
79672024-04-20T01:10:25.874Z 19 380-399 68 68 68 116 116 116 F F F
79682024-04-20T01:10:25.874Z 20 400-419 73 73 73 124 124 124 F F F
79692024-04-20T01:10:25.874Z 21 420-439 67 67 67 114 114 114 F F F
79702024-04-20T01:10:25.874Z 22 440-459 72 72 72 123 123 123 F F F
79712024-04-20T01:10:25.874Z 23 460-479 72 72 72 123 123 123 F F F
79722024-04-20T01:10:25.874Z 24 480-499 71 71 71 122 122 122 F F F
79732024-04-20T01:10:25.874Z 25 500-519 73 73 73 124 124 124 F F F
79742024-04-20T01:10:25.874Z 26 520-539 70 70 70 121 121 121 F F F
79752024-04-20T01:10:25.874Z 27 540-559 72 72 72 123 123 123 F F F
79762024-04-20T01:10:25.874Z 28 560-579 73 73 73 124 124 124 F F F
79772024-04-20T01:10:25.875Z 29 580-599 73 73 73 124 124 124 F F F
79782024-04-20T01:10:25.875ZMax gen: 73, Max flush: 124
79792024-04-20T01:10:25.875Z
79802024-04-20T01:10:25.875ZBegin loop 37
79812024-04-20T01:10:25.875ZDownstairs to restart: 0
79822024-04-20T01:10:25.888Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.394026759Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467}
79832024-04-20T01:10:25.889Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.39429401Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467}
79842024-04-20T01:10:25.889Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.39431615Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467}
79852024-04-20T01:10:25.889Z{"msg":"Crucible 07e0dd4f-24ba-46af-9015-1267c93995cd has session id: 234c8fc7-ccc7-49b6-a808-8d5c3efe5b27","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.39435034Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
79862024-04-20T01:10:25.896Z{"msg":"Upstairs opts: Upstairs UUID: 07e0dd4f-24ba-46af-9015-1267c93995cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.3943952Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
79872024-04-20T01:10:25.896Z{"msg":"Crucible stats registered with UUID: 07e0dd4f-24ba-46af-9015-1267c93995cd","v":0,"name":"crucible","level":30Crucible runtime is spawned
79882024-04-20T01:10:25.896Z,"time":"2024-04-20T01:10:24.39443553Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
79892024-04-20T01:10:25.896Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.39451573Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467}
79902024-04-20T01:10:25.896Z{"msg":"07e0dd4f-24ba-46af-9015-1267c93995cd active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.39454073Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
79912024-04-20T01:10:25.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.39455411Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"0","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
79922024-04-20T01:10:25.896Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.39456854Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"0","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
79932024-04-20T01:10:25.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.39458965Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"1","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
79942024-04-20T01:10:25.896Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.39460933Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"1","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
79952024-04-20T01:10:25.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.39462906Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"2","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
79962024-04-20T01:10:25.896Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.394655Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"2","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
79972024-04-20T01:10:25.896Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.39468403Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
79982024-04-20T01:10:25.896Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.3947161Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
79992024-04-20T01:10:25.896Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.39473108Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80002024-04-20T01:10:25.896Z{"msg":"ds_connection connected from Ok(127.0.0.1:58846)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.394979241Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80012024-04-20T01:10:25.896Z{"msg":"ds_connection connected from Ok(127.0.0.1:61948)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.395056941Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80022024-04-20T01:10:25.896Z{"msg":"ds_connection connected from Ok(127.0.0.1:52295)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.395116771Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80032024-04-20T01:10:25.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.395780033Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"2","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80042024-04-20T01:10:25.896Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.395823453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"2","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80052024-04-20T01:10:25.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.395845583Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"1","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80062024-04-20T01:10:25.896Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.395876993Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"1","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80072024-04-20T01:10:25.897Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.395902343Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"0","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80082024-04-20T01:10:25.897Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.395929853Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"0","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80092024-04-20T01:10:25.897Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.396325404Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"2","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80102024-04-20T01:10:25.897Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.396370514Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"1","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80112024-04-20T01:10:25.897Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.396567924Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"0","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80122024-04-20T01:10:25.897Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.396829095Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"2","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80132024-04-20T01:10:25.897Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.396880435Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"2","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80142024-04-20T01:10:25.897Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.396916375Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80152024-04-20T01:10:25.897Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.396931815Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"1","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80162024-04-20T01:10:25.897Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.396948615Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"1","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80172024-04-20T01:10:25.897Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.396971976Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80182024-04-20T01:10:25.897Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397073606Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"0","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80192024-04-20T01:10:25.897Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397128986Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"0","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80202024-04-20T01:10:25.897Z{"msg":"[0]R flush_numbers[0..12]: [117, 114, 119, 124, 121, 124, 122, 123, 123, 115, 115, 124]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397154816Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80212024-04-20T01:10:25.897Z{"msg":"[0]R generation[0..12]: [69, 67, 70, 73, 70, 73, 71, 72, 72, 68, 68, 73]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397183106Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80222024-04-20T01:10:25.897Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397199416Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80232024-04-20T01:10:25.897Z{"msg":"[1]R flush_numbers[0..12]: [117, 114, 119, 124, 121, 124, 122, 123, 123, 115, 115, 124]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397220906Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80242024-04-20T01:10:25.897Z{"msg":"[1]R generation[0..12]: [69, 67, 70, 73, 70, 73, 71, 72, 72, 68, 68, 73]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397235356Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80252024-04-20T01:10:25.897Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397250506Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80262024-04-20T01:10:25.897Z{"msg":"[2]R flush_numbers[0..12]: [117, 114, 119, 124, 121, 124, 122, 123, 123, 115, 115, 124]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397276776Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80272024-04-20T01:10:25.897ZWait for a query_work_queue command to finish before sending IO
80282024-04-20T01:10:25.897Z{"msg":"[2]R generation[0..12]: [69, 67, 70, 73, 70, 73, 71, 72, 72, 68, 68, 73]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397303246Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80292024-04-20T01:10:25.897Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397318346Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80302024-04-20T01:10:25.898Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397338176Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80312024-04-20T01:10:25.898Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397352026Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80322024-04-20T01:10:25.898Z{"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397372406Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80332024-04-20T01:10:25.898Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397386026Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"":"downstairs","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80342024-04-20T01:10:25.898Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397406496Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80352024-04-20T01:10:25.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397418857Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"0","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80362024-04-20T01:10:25.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397438986Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"1","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80372024-04-20T01:10:25.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397458857Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"client":"2","":"downstairs","session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80382024-04-20T01:10:25.898Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397479537Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80392024-04-20T01:10:25.898Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397493007Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80402024-04-20T01:10:25.898Z{"msg":"07e0dd4f-24ba-46af-9015-1267c93995cd is now active with session: 234c8fc7-ccc7-49b6-a808-8d5c3efe5b27","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397505987Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80412024-04-20T01:10:25.898Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397517877Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467,"session_id":"234c8fc7-ccc7-49b6-a808-8d5c3efe5b27"}
80422024-04-20T01:10:25.898Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:24.397530177Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467}
80432024-04-20T01:10:25.898ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80442024-04-20T01:10:25.902ZLoading write count information from file "/tmp/test_repair_verify.data"
80452024-04-20T01:10:25.902ZRun Repair workload
80462024-04-20T01:10:25.902Z01/30 Write block 297 len 40960 data: 4 2 2 3 6 6 6 6 5 4
80472024-04-20T01:10:25.902Z02/30 Read block 557 len 32768
80482024-04-20T01:10:25.902Z03/30 Write block 500 len 28672 data: 6 5 5 7 7 7 8
80492024-04-20T01:10:25.902Z04/30 Read block 326 len 36864
80502024-04-20T01:10:25.902Z05/30 Write block 206 len 4096 data: 6
80512024-04-20T01:10:25.902Z06/30 Read block 376 len 36864
80522024-04-20T01:10:25.903Z07/30 Read block 69 len 32768
80532024-04-20T01:10:25.903Z08/30 Write block 217 len 16384 data: 9 9 7 9
80542024-04-20T01:10:25.903Z09/30 Write block 356 len 4096 data: 6
80552024-04-20T01:10:25.903Z10/30 Read block 396 len 20480
80562024-04-20T01:10:25.906Z11/30 Write block 582 len 24576 data: 8 6 8 11 13 11
80572024-04-20T01:10:25.906Z12/30 Write block 466 len 40960 data: 7 6 6 8 8 8 8 9 10 11
80582024-04-20T01:10:25.906Z13/30 Read block 462 len 16384
80592024-04-20T01:10:25.912Z14/30 Write block 112 len 32768 data: 6 6 6 6 5 4 5 6
80602024-04-20T01:10:25.912Z15/30 Read block 555 len 32768
80612024-04-20T01:10:25.912Z16/30 Write block 210 len 28672 data: 4 4 5 6 6 5 6
80622024-04-20T01:10:25.912Z17/30 Flush
80632024-04-20T01:10:25.916Z18/30 Write block 59 len 12288 data: 6 6 6
80642024-04-20T01:10:25.916Z19/30 Read block 262 len 8192
80652024-04-20T01:10:25.916Z20/30 Read block 204 len 40960
80662024-04-20T01:10:25.916Z21/30 Write block 98 len 4096 data: 6
80672024-04-20T01:10:25.920Z22/30 Write block 2 len 36864 data: 4 4 4 3 3 3 3 3 4
80682024-04-20T01:10:25.920Z23/30 Read block 29 len 32768
80692024-04-20T01:10:25.920Z24/30 Write block 59 len 8192 data: 7 7
80702024-04-20T01:10:25.920Z25/30 Write block 248 len 28672 data: 11 11 9 7 6 4 5
80712024-04-20T01:10:25.920Z26/30 Read block 250 len 16384
80722024-04-20T01:10:25.923Z27/30 Write block 489 len 20480 data: 7 8 7 8 7
80732024-04-20T01:10:25.923Z28/30 Write block 565 len 24576 data: 6 7 6 7 8 7
80742024-04-20T01:10:25.923Z29/30 Read block 99 len 4096
80752024-04-20T01:10:25.924Z30/30 Read block 137 len 36864
80762024-04-20T01:10:25.927Z----------------------------------------------------------------
80772024-04-20T01:10:25.928Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
80782024-04-20T01:10:25.928ZStates: Active Active Active
80792024-04-20T01:10:25.928ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80802024-04-20T01:10:25.928Z 1 Acked 1000 Write 10 Done Done Done false
80812024-04-20T01:10:25.928Z 2 Acked 1001 Read 8 Done Done Done false
80822024-04-20T01:10:25.928Z 3 Acked 1002 Write 7 Done Done Done false
80832024-04-20T01:10:25.928Z 4 Acked 1003 Read 9 Done Done Done false
80842024-04-20T01:10:25.928Z 5 Acked 1004 Write 1 Done Done Done false
80852024-04-20T01:10:25.928Z 6 Acked 1005 Read 9 Done Done Done false
80862024-04-20T01:10:25.928Z 7 Acked 1006 Read 8 Done Done Done false
80872024-04-20T01:10:25.928Z 8 Acked 1007 Write 4 Done Done Done false
80882024-04-20T01:10:25.928Z 9 Acked 1008 Write 1 Done Done Done false
80892024-04-20T01:10:25.928Z 10 Acked 1009 Read 5 Sent Done Done false
80902024-04-20T01:10:25.928Z 11 Acked 1010 Write 6 Sent Done Done false
80912024-04-20T01:10:25.928Z 12 Acked 1011 Write 10 Sent Done Done false
80922024-04-20T01:10:25.928Z 13 Acked 1012 Read 4 Sent Done Done false
80932024-04-20T01:10:25.928Z 14 Acked 1013 Write 8 Sent Done Done false
80942024-04-20T01:10:25.928Z 15 Acked 1014 Read 8 Sent Done Done false
80952024-04-20T01:10:25.928Z 16 Acked 1015 Write 7 Sent Done Done false
80962024-04-20T01:10:25.928Z 17 Acked 1016 Flush 0 Sent Done Done false
80972024-04-20T01:10:25.928Z 18 Acked 1017 Write 3 Sent Done Done false
80982024-04-20T01:10:25.928Z 19 Acked 1018 Read 2 Sent Done Done false
80992024-04-20T01:10:25.928Z 20 Acked 1019 Read 10 Sent Done Done false
81002024-04-20T01:10:25.929Z 21 Acked 1020 Write 1 Sent Done Done false
81012024-04-20T01:10:25.929Z 22 Acked 1021 Write 9 Sent Done Done false
81022024-04-20T01:10:25.929Z 23 Acked 1022 Read 8 Sent Done Done false
81032024-04-20T01:10:25.929Z 24 Acked 1023 Write 2 Sent Done Done false
81042024-04-20T01:10:25.929Z 25 Acked 1024 Write 7 Sent Done Done false
81052024-04-20T01:10:25.929Z 26 Acked 1025 Read 4 Sent Done Done false
81062024-04-20T01:10:25.929Z 27 Acked 1026 Write 5 Sent Done Done false
81072024-04-20T01:10:25.929Z 28 Acked 1027 Write 6 Sent Done Done false
81082024-04-20T01:10:25.929Z 29 Acked 1028 Read 1 Sent Done Done false
81092024-04-20T01:10:25.929Z 30 Acked 1029 Read 9 Sent Done Done false
81102024-04-20T01:10:25.929Z STATES DS:0 DS:1 DS:2 TOTAL
81112024-04-20T01:10:25.929Z New 0 0 0 0
81122024-04-20T01:10:25.929Z Sent 21 0 0 21
81132024-04-20T01:10:25.929Z Done 9 30 30 69
81142024-04-20T01:10:25.929Z Skipped 0 0 0 0
81152024-04-20T01:10:25.929Z Error 0 0 0 0
81162024-04-20T01:10:25.929ZLast Flush: 0 1016 1016
81172024-04-20T01:10:25.929ZDownstairs last five completed:
81182024-04-20T01:10:25.929ZUpstairs last five completed: 30 29 28 27 26
81192024-04-20T01:10:25.929Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:10:24.430013629Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1467}
81202024-04-20T01:10:25.929ZWrote out file "/tmp/test_repair_verify.data"
81212024-04-20T01:10:25.933Z
81222024-04-20T01:10:25.933ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
81232024-04-20T01:10:25.949ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81242024-04-20T01:10:25.949Z 0 000-019 69 69 69 117 117 117 F T T <---
81252024-04-20T01:10:25.949Z 1 020-039 67 67 67 114 114 114 F F F
81262024-04-20T01:10:25.949Z 2 040-059 70 70 70 119 119 119 F T T <---
81272024-04-20T01:10:25.949Z 3 060-079 73 73 73 124 124 124 F T T <---
81282024-04-20T01:10:25.949Z 4 080-099 70 70 70 121 121 121 F T T <---
81292024-04-20T01:10:25.949Z 5 100-119 73 74 74 124 125 125 F F F <---
81302024-04-20T01:10:25.949Z 6 120-139 71 71 71 122 122 122 F F F
81312024-04-20T01:10:25.949Z 7 140-159 72 72 72 123 123 123 F F F
81322024-04-20T01:10:25.950Z 8 160-179 72 72 72 123 123 123 F F F
81332024-04-20T01:10:25.950Z 9 180-199 68 68 68 115 115 115 F F F
81342024-04-20T01:10:25.954Z 10 200-219 68 74 74 115 125 125 T F F <---
81352024-04-20T01:10:25.954Z 11 220-239 73 74 74 124 125 125 T F F <---
81362024-04-20T01:10:25.954Z 12 240-259 67 67 67 114 114 114 F T T <---
81372024-04-20T01:10:25.955Z 13 260-279 73 73 73 124 124 124 F F F
81382024-04-20T01:10:25.955Z 14 280-299 73 74 74 124 125 125 T F F <---
81392024-04-20T01:10:25.955Z 15 300-319 71 74 74 122 125 125 T F F <---
81402024-04-20T01:10:25.955Z 16 320-339 73 73 73 124 124 124 F F F
81412024-04-20T01:10:25.955Z 17 340-359 68 74 74 116 125 125 T F F <---
81422024-04-20T01:10:25.955Z 18 360-379 67 67 67 114 114 114 F F F
81432024-04-20T01:10:25.955Z 19 380-399 68 68 68 116 116 116 F F F
81442024-04-20T01:10:25.955Z 20 400-419 73 73 73 124 124 124 F F F
81452024-04-20T01:10:25.955Z 21 420-439 67 67 67 114 114 114 F F F
81462024-04-20T01:10:25.955Z 22 440-459 72 72 72 123 123 123 F F F
81472024-04-20T01:10:25.955Z 23 460-479 72 74 74 123 125 125 F F F <---
81482024-04-20T01:10:25.955Z 24 480-499 71 71 71 122 122 122 F T T <---
81492024-04-20T01:10:25.956Z 25 500-519 73 74 74 124 125 125 T F F <---
81502024-04-20T01:10:25.956Z 26 520-539 70 70 70 121 121 121 F F F
81512024-04-20T01:10:25.956Z 27 540-559 72 72 72 123 123 123 F F F
81522024-04-20T01:10:25.956Z 28 560-579 73 73 73 124 124 124 F T T <---
81532024-04-20T01:10:25.956Z 29 580-599 73 74 74 124 125 125 F F F <---
81542024-04-20T01:10:25.956ZMax gen: 74, Max flush: 125
81552024-04-20T01:10:25.956ZError: Difference in extent metadata found!
81562024-04-20T01:10:25.956ZA Difference in extent metadata is expected here
81572024-04-20T01:10:25.956ZOn loop 37
81582024-04-20T01:10:25.956Z
81592024-04-20T01:10:25.960ZVerifying data now
81602024-04-20T01:10:40.667ZNo change after verify
81612024-04-20T01:10:40.667ZLoop: 37 Downstairs dump after verify (and repair):
81622024-04-20T01:10:40.681ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81632024-04-20T01:10:40.681Z 0 000-019 75 75 75 126 126 126 F F F
81642024-04-20T01:10:40.681Z 1 020-039 67 67 67 114 114 114 F F F
81652024-04-20T01:10:40.681Z 2 040-059 75 75 75 126 126 126 F F F
81662024-04-20T01:10:40.681Z 3 060-079 75 75 75 126 126 126 F F F
81672024-04-20T01:10:40.681Z 4 080-099 75 75 75 126 126 126 F F F
81682024-04-20T01:10:40.681Z 5 100-119 74 74 74 125 125 125 F F F
81692024-04-20T01:10:40.681Z 6 120-139 71 71 71 122 122 122 F F F
81702024-04-20T01:10:40.681Z 7 140-159 72 72 72 123 123 123 F F F
81712024-04-20T01:10:40.681Z 8 160-179 72 72 72 123 123 123 F F F
81722024-04-20T01:10:40.681Z 9 180-199 68 68 68 115 115 115 F F F
81732024-04-20T01:10:40.681Z 10 200-219 74 74 74 125 125 125 F F F
81742024-04-20T01:10:40.681Z 11 220-239 74 74 74 125 125 125 F F F
81752024-04-20T01:10:40.681Z 12 240-259 75 75 75 126 126 126 F F F
81762024-04-20T01:10:40.685Z 13 260-279 73 73 73 124 124 124 F F F
81772024-04-20T01:10:40.685Z 14 280-299 74 74 74 125 125 125 F F F
81782024-04-20T01:10:40.685Z 15 300-319 74 74 74 125 125 125 F F F
81792024-04-20T01:10:40.685Z 16 320-339 73 73 73 124 124 124 F F F
81802024-04-20T01:10:40.685Z 17 340-359 74 74 74 125 125 125 F F F
81812024-04-20T01:10:40.685Z 18 360-379 67 67 67 114 114 114 F F F
81822024-04-20T01:10:40.685Z 19 380-399 68 68 68 116 116 116 F F F
81832024-04-20T01:10:40.685Z 20 400-419 73 73 73 124 124 124 F F F
81842024-04-20T01:10:40.685Z 21 420-439 67 67 67 114 114 114 F F F
81852024-04-20T01:10:40.685Z 22 440-459 72 72 72 123 123 123 F F F
81862024-04-20T01:10:40.685Z 23 460-479 74 74 74 125 125 125 F F F
81872024-04-20T01:10:40.685Z 24 480-499 75 75 75 126 126 126 F F F
81882024-04-20T01:10:40.685Z 25 500-519 74 74 74 125 125 125 F F F
81892024-04-20T01:10:40.685Z 26 520-539 70 70 70 121 121 121 F F F
81902024-04-20T01:10:40.686Z 27 540-559 72 72 72 123 123 123 F F F
81912024-04-20T01:10:40.686Z 28 560-579 75 75 75 126 126 126 F F F
81922024-04-20T01:10:40.686Z 29 580-599 74 74 74 125 125 125 F F F
81932024-04-20T01:10:40.686ZMax gen: 75, Max flush: 126
81942024-04-20T01:10:40.686Z
81952024-04-20T01:10:40.686ZBegin loop 38
81962024-04-20T01:10:40.686ZDownstairs to restart: 2
81972024-04-20T01:10:40.703Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.208600086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475}
81982024-04-20T01:10:40.703Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.208823496Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475}
81992024-04-20T01:10:40.710Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.208860736Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475}
82002024-04-20T01:10:40.710ZCrucible runtime is spawned
82012024-04-20T01:10:40.710Z{"msg":"Crucible 79181978-f25e-4b92-89c6-2942d41bf2e0 has session id: 9ef2303e-41e0-489d-a8f5-cb037ecaf877","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.208933117Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82022024-04-20T01:10:40.710Z{"msg":"Upstairs opts: Upstairs UUID: 79181978-f25e-4b92-89c6-2942d41bf2e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.208977747Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82032024-04-20T01:10:40.711Z{"msg":"Crucible stats registered with UUID: 79181978-f25e-4b92-89c6-2942d41bf2e0","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209008737Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82042024-04-20T01:10:40.711Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209031687Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475}
82052024-04-20T01:10:40.711Z{"msg":"79181978-f25e-4b92-89c6-2942d41bf2e0 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209052147Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82062024-04-20T01:10:40.711Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209080647Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"0","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82072024-04-20T01:10:40.711Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209105897Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"0","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82082024-04-20T01:10:40.711Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209147357Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"1","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82092024-04-20T01:10:40.711Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209173577Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"1","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82102024-04-20T01:10:40.711Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209204487Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"2","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82112024-04-20T01:10:40.711Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209230967Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"2","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82122024-04-20T01:10:40.711Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209255957Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82132024-04-20T01:10:40.711Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209282988Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82142024-04-20T01:10:40.711Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209310708Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82152024-04-20T01:10:40.711Z{"msg":"ds_connection connected from Ok(127.0.0.1:45898)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209346788Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82162024-04-20T01:10:40.711Z{"msg":"ds_connection connected from Ok(127.0.0.1:44885)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209372368Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82172024-04-20T01:10:40.711Z{"msg":"ds_connection connected from Ok(127.0.0.1:64826)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.209405398Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82182024-04-20T01:10:40.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.210111449Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"0","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82192024-04-20T01:10:40.711Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.210154779Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"0","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82202024-04-20T01:10:40.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.21018503Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"1","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82212024-04-20T01:10:40.711Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.21021481Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"1","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82222024-04-20T01:10:40.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.21054234Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"2","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82232024-04-20T01:10:40.711Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.21059216Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"2","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82242024-04-20T01:10:40.711Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.21065066Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"1","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82252024-04-20T01:10:40.711Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.210834471Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"0","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82262024-04-20T01:10:40.712Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211120521Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"2","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82272024-04-20T01:10:40.712Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211282832Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"1","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82282024-04-20T01:10:40.712Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211329882Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"1","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82292024-04-20T01:10:40.712Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211365992Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82302024-04-20T01:10:40.712Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211399142Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"0","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82312024-04-20T01:10:40.712Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211433342Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"0","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82322024-04-20T01:10:40.712Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211456492Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82332024-04-20T01:10:40.712Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211643723Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"2","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82342024-04-20T01:10:40.712Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211680173Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"2","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82352024-04-20T01:10:40.712Z{"msg":"[0]R flush_numbers[0..12]: [126, 114, 126, 126, 126, 125, 122, 123, 123, 115, 125, 125]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211700173Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82362024-04-20T01:10:40.712Z{"msg":"[0]R generation[0..12]: [75, 67, 75, 75, 75, 74, 71, 72, 72, 68, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211732023Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82372024-04-20T01:10:40.712Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211754653Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82382024-04-20T01:10:40.712Z{"msg":"[1]R flush_numbers[0..12]: [126, 114, 126, 126, 126, 125, 122, 123, 123, 115, 125, 125]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211780703Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82392024-04-20T01:10:40.712Z{"msg":"[1]R generation[0..12]: [75, 67, 75, 75, 75, 74, 71, 72, 72, 68, 74, 74]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
82402024-04-20T01:10:40.712Z,"level":30,"time":"2024-04-20T01:10:39.211829963Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82412024-04-20T01:10:40.712Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211868353Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82422024-04-20T01:10:40.712Z{"msg":"[2]R flush_numbers[0..12]: [126, 114, 126, 126, 126, 125, 122, 123, 123, 115, 125, 125]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211909593Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82432024-04-20T01:10:40.712Z{"msg":"[2]R generation[0..12]: [75, 67, 75, 75, 75, 74, 71, 72, 72, 68, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211939723Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82442024-04-20T01:10:40.712Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211974463Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82452024-04-20T01:10:40.712Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.211991063Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82462024-04-20T01:10:40.712Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.212042903Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82472024-04-20T01:10:40.712Z{"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.212079944Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82482024-04-20T01:10:40.712Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.212116704Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"":"downstairs","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82492024-04-20T01:10:40.713Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.212151884Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82502024-04-20T01:10:40.713Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.212167244Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"0","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82512024-04-20T01:10:40.713Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.212192444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"1","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82522024-04-20T01:10:40.713ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82532024-04-20T01:10:40.713Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.212229244Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"client":"2","":"downstairs","session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82542024-04-20T01:10:40.713Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.212262594Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82552024-04-20T01:10:40.713Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.212296634Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82562024-04-20T01:10:40.717Z{"msg":"79181978-f25e-4b92-89c6-2942d41bf2e0 is now active with session: 9ef2303e-41e0-489d-a8f5-cb037ecaf877","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.212329744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82572024-04-20T01:10:40.717Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.212359374Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475,"session_id":"9ef2303e-41e0-489d-a8f5-cb037ecaf877"}
82582024-04-20T01:10:40.717Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:39.212380214Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475}
82592024-04-20T01:10:40.717ZLoading write count information from file "/tmp/test_repair_verify.data"
82602024-04-20T01:10:40.717ZRun Repair workload
82612024-04-20T01:10:40.717Z01/30 Write block 15 len 24576 data: 7 7 6 5 5 5
82622024-04-20T01:10:40.717Z02/30 Read block 48 len 20480
82632024-04-20T01:10:40.717Z03/30 Read block 161 len 16384
82642024-04-20T01:10:40.717Z04/30 Write block 34 len 20480 data: 7 7 6 7 8
82652024-04-20T01:10:40.717Z05/30 Write block 398 len 16384 data: 5 7 6 5
82662024-04-20T01:10:40.717Z06/30 Flush
82672024-04-20T01:10:40.717Z07/30 Write block 579 len 12288 data: 8 8 8
82682024-04-20T01:10:40.717Z08/30 Read block 179 len 16384
82692024-04-20T01:10:40.721Z09/30 Read block 65 len 32768
82702024-04-20T01:10:40.721Z10/30 Write block 52 len 28672 data: 12 12 9 7 6 5 6
82712024-04-20T01:10:40.721Z11/30 Read block 393 len 28672
82722024-04-20T01:10:40.721Z12/30 Read block 156 len 12288
82732024-04-20T01:10:40.721Z13/30 Write block 491 len 16384 data: 8 9 8 6
82742024-04-20T01:10:40.725Z14/30 Write block 372 len 36864 data: 12 13 12 11 6 6 6 6 6
82752024-04-20T01:10:40.725Z15/30 Read block 178 len 8192
82762024-04-20T01:10:40.725Z16/30 Flush
82772024-04-20T01:10:40.729Z17/30 Write block 182 len 36864 data: 7 6 5 6 7 7 9 9 11
82782024-04-20T01:10:40.732Z18/30 Write block 235 len 32768 data: 11 11 12 10 10 7 6 4
82792024-04-20T01:10:40.732Z19/30 Read block 203 len 20480
82802024-04-20T01:10:40.732Z20/30 Write block 406 len 36864 data: 6 5 5 5 7 9 10 11 10
82812024-04-20T01:10:40.732Z21/30 Read block 189 len 40960
82822024-04-20T01:10:40.736Z22/30 Read block 510 len 24576
82832024-04-20T01:10:40.736Z23/30 Write block 376 len 20480 data: 7 7 7 7 7
82842024-04-20T01:10:40.736Z24/30 Read block 6 len 36864
82852024-04-20T01:10:40.740Z25/30 Write block 314 len 40960 data: 4 4 3 3 8 8 9 9 10 10
82862024-04-20T01:10:40.740Z26/30 Read block 77 len 16384
82872024-04-20T01:10:40.740Z27/30 Read block 572 len 40960
82882024-04-20T01:10:40.740Z28/30 Write block 465 len 24576 data: 6 8 7 7 9 9
82892024-04-20T01:10:40.743Z29/30 Write block 516 len 12288 data: 9 10 7
82902024-04-20T01:10:40.744Z30/30 Write block 429 len 12288 data: 7 7 6
82912024-04-20T01:10:40.744Z----------------------------------------------------------------
82922024-04-20T01:10:40.744Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:14
82932024-04-20T01:10:40.744ZStates: Active Active Active
82942024-04-20T01:10:40.744ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82952024-04-20T01:10:40.744Z 17 Acked 1016 Write 9 Done Done Done false
82962024-04-20T01:10:40.744Z 18 Acked 1017 Write 8 Done Done Sent false
82972024-04-20T01:10:40.744Z 19 Acked 1018 Read 5 Done Done Sent false
82982024-04-20T01:10:40.744Z 20 Acked 1019 Write 9 Done Done Sent false
82992024-04-20T01:10:40.744Z 21 Acked 1020 Read 10 Done Done Sent false
83002024-04-20T01:10:40.744Z 22 Acked 1021 Read 6 Done Done Sent false
83012024-04-20T01:10:40.744Z 23 Acked 1022 Write 5 Done Done Sent false
83022024-04-20T01:10:40.744Z 24 Acked 1023 Read 9 Done Done Sent false
83032024-04-20T01:10:40.744Z 25 Acked 1024 Write 10 Done Done Sent false
83042024-04-20T01:10:40.744Z 26 Acked 1025 Read 4 Done Done Sent false
83052024-04-20T01:10:40.744Z 27 Acked 1026 Read 10 Done Done Sent false
83062024-04-20T01:10:40.744Z 28 Acked 1027 Write 6 Done Done Sent false
83072024-04-20T01:10:40.744Z 29 Acked 1028 Write 3 Done Done Sent false
83082024-04-20T01:10:40.744Z 30 Acked 1029 Write 3 Sent Sent Sent false
83092024-04-20T01:10:40.744Z STATES DS:0 DS:1 DS:2 TOTAL
83102024-04-20T01:10:40.744Z New 0 0 0 0
83112024-04-20T01:10:40.744Z Sent 1 1 13 15
83122024-04-20T01:10:40.745Z Done 13 13 1 27
83132024-04-20T01:10:40.745Z Skipped 0 0 0 0
83142024-04-20T01:10:40.745Z Error 0 0 0 0
83152024-04-20T01:10:40.745ZLast Flush: 1015 1015 1015
83162024-04-20T01:10:40.745ZDownstairs last five completed: 1015 1014 1013 1012 1011
83172024-04-20T01:10:40.745ZUpstairs last five completed: 30 29 28 27 26
83182024-04-20T01:10:40.745Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:10:39.246848361Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1475}
83192024-04-20T01:10:40.745ZWrote out file "/tmp/test_repair_verify.data"
83202024-04-20T01:10:40.749Z
83212024-04-20T01:10:40.750ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83222024-04-20T01:10:40.767ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83232024-04-20T01:10:40.767Z 0 000-019 76 76 76 127 127 127 F F F
83242024-04-20T01:10:40.767Z 1 020-039 76 76 76 127 127 127 F F F
83252024-04-20T01:10:40.767Z 2 040-059 76 76 76 128 128 128 F F F
83262024-04-20T01:10:40.767Z 3 060-079 75 75 75 126 126 126 F F F
83272024-04-20T01:10:40.772Z 4 080-099 75 75 75 126 126 126 F F F
83282024-04-20T01:10:40.772Z 5 100-119 74 74 74 125 125 125 F F F
83292024-04-20T01:10:40.772Z 6 120-139 71 71 71 122 122 122 F F F
83302024-04-20T01:10:40.772Z 7 140-159 72 72 72 123 123 123 F F F
83312024-04-20T01:10:40.772Z 8 160-179 72 72 72 123 123 123 F F F
83322024-04-20T01:10:40.772Z 9 180-199 68 68 68 115 115 115 T T T
83332024-04-20T01:10:40.772Z 10 200-219 74 74 74 125 125 125 F F F
83342024-04-20T01:10:40.772Z 11 220-239 74 74 74 125 125 125 T T F <---
83352024-04-20T01:10:40.772Z 12 240-259 75 75 75 126 126 126 T T F <---
83362024-04-20T01:10:40.772Z 13 260-279 73 73 73 124 124 124 F F F
83372024-04-20T01:10:40.772Z 14 280-299 74 74 74 125 125 125 F F F
83382024-04-20T01:10:40.772Z 15 300-319 74 74 74 125 125 125 T T F <---
83392024-04-20T01:10:40.772Z 16 320-339 73 73 73 124 124 124 T T F <---
83402024-04-20T01:10:40.772Z 17 340-359 74 74 74 125 125 125 F F F
83412024-04-20T01:10:40.772Z 18 360-379 76 76 76 128 128 128 T T F <---
83422024-04-20T01:10:40.772Z 19 380-399 76 76 76 128 128 128 T T F <---
83432024-04-20T01:10:40.772Z 20 400-419 76 76 76 127 127 127 T T F <---
83442024-04-20T01:10:40.772Z 21 420-439 67 67 67 114 114 114 T T F <---
83452024-04-20T01:10:40.772Z 22 440-459 72 72 72 123 123 123 F F F
83462024-04-20T01:10:40.773Z 23 460-479 74 74 74 125 125 125 T T F <---
83472024-04-20T01:10:40.773Z 24 480-499 76 76 76 128 128 128 F F F
83482024-04-20T01:10:40.773Z 25 500-519 74 74 74 125 125 125 T T F <---
83492024-04-20T01:10:40.773Z 26 520-539 70 70 70 121 121 121 F F F
83502024-04-20T01:10:40.773Z 27 540-559 72 72 72 123 123 123 F F F
83512024-04-20T01:10:40.773Z 28 560-579 76 76 76 128 128 128 F F F
83522024-04-20T01:10:40.773Z 29 580-599 76 76 76 128 128 128 F F F
83532024-04-20T01:10:40.773ZMax gen: 76, Max flush: 128
83542024-04-20T01:10:40.773ZError: Difference in extent metadata found!
83552024-04-20T01:10:40.773ZA Difference in extent metadata is expected here
83562024-04-20T01:10:40.773ZOn loop 38
83572024-04-20T01:10:40.773Z
83582024-04-20T01:10:40.777ZVerifying data now
83592024-04-20T01:10:45.295ZNo change after verify
83602024-04-20T01:10:45.295ZLoop: 38 Downstairs dump after verify (and repair):
83612024-04-20T01:10:45.309ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83622024-04-20T01:10:45.309Z 0 000-019 76 76 76 127 127 127 F F F
83632024-04-20T01:10:45.310Z 1 020-039 76 76 76 127 127 127 F F F
83642024-04-20T01:10:45.310Z 2 040-059 76 76 76 128 128 128 F F F
83652024-04-20T01:10:45.310Z 3 060-079 75 75 75 126 126 126 F F F
83662024-04-20T01:10:45.310Z 4 080-099 75 75 75 126 126 126 F F F
83672024-04-20T01:10:45.310Z 5 100-119 74 74 74 125 125 125 F F F
83682024-04-20T01:10:45.310Z 6 120-139 71 71 71 122 122 122 F F F
83692024-04-20T01:10:45.310Z 7 140-159 72 72 72 123 123 123 F F F
83702024-04-20T01:10:45.310Z 8 160-179 72 72 72 123 123 123 F F F
83712024-04-20T01:10:45.314Z 9 180-199 77 77 77 129 129 129 F F F
83722024-04-20T01:10:45.314Z 10 200-219 74 74 74 125 125 125 F F F
83732024-04-20T01:10:45.314Z 11 220-239 77 77 77 129 129 129 F F F
83742024-04-20T01:10:45.314Z 12 240-259 77 77 77 129 129 129 F F F
83752024-04-20T01:10:45.314Z 13 260-279 73 73 73 124 124 124 F F F
83762024-04-20T01:10:45.314Z 14 280-299 74 74 74 125 125 125 F F F
83772024-04-20T01:10:45.314Z 15 300-319 77 77 77 129 129 129 F F F
83782024-04-20T01:10:45.314Z 16 320-339 77 77 77 129 129 129 F F F
83792024-04-20T01:10:45.314Z 17 340-359 74 74 74 125 125 125 F F F
83802024-04-20T01:10:45.314Z 18 360-379 77 77 77 129 129 129 F F F
83812024-04-20T01:10:45.314Z 19 380-399 77 77 77 129 129 129 F F F
83822024-04-20T01:10:45.314Z 20 400-419 77 77 77 129 129 129 F F F
83832024-04-20T01:10:45.314Z 21 420-439 77 77 77 129 129 129 F F F
83842024-04-20T01:10:45.314Z 22 440-459 72 72 72 123 123 123 F F F
83852024-04-20T01:10:45.314Z 23 460-479 77 77 77 129 129 129 F F F
83862024-04-20T01:10:45.314Z 24 480-499 76 76 76 128 128 128 F F F
83872024-04-20T01:10:45.314Z 25 500-519 77 77 77 129 129 129 F F F
83882024-04-20T01:10:45.314Z 26 520-539 70 70 70 121 121 121 F F F
83892024-04-20T01:10:45.314Z 27 540-559 72 72 72 123 123 123 F F F
83902024-04-20T01:10:45.315Z 28 560-579 76 76 76 128 128 128 F F F
83912024-04-20T01:10:45.315Z 29 580-599 76 76 76 128 128 128 F F F
83922024-04-20T01:10:45.315ZMax gen: 77, Max flush: 129
83932024-04-20T01:10:45.315Z
83942024-04-20T01:10:45.315ZBegin loop 39
83952024-04-20T01:10:45.315ZDownstairs to restart: 2
83962024-04-20T01:10:45.331Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836109807Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483}
83972024-04-20T01:10:45.331Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836375917Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483}
83982024-04-20T01:10:45.339Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836396847Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483}
83992024-04-20T01:10:45.339Z{"msg":"Crucible 3ce75ef4-9adf-467e-a60f-ebf733cad2ee has session id: cdff2d7b-63b3-429e-9ca6-4130d39c959f","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836431647Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84002024-04-20T01:10:45.339Z{"msg":"Upstairs opts: Upstairs UUID: 3ce75ef4-9adf-467e-a60f-ebf733cad2ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836475737Z","hostnameCrucible runtime is spawned
84012024-04-20T01:10:45.339Z":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84022024-04-20T01:10:45.339Z{"msg":"Crucible stats registered with UUID: 3ce75ef4-9adf-467e-a60f-ebf733cad2ee","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836518268Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84032024-04-20T01:10:45.339Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836558538Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483}
84042024-04-20T01:10:45.339Z{"msg":"3ce75ef4-9adf-467e-a60f-ebf733cad2ee active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836672118Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84052024-04-20T01:10:45.339Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836714178Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"0","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84062024-04-20T01:10:45.339Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836740328Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"0","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84072024-04-20T01:10:45.339Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836778698Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"1","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84082024-04-20T01:10:45.339Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836805698Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"1","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84092024-04-20T01:10:45.339Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836840798Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"2","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84102024-04-20T01:10:45.339Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836865058Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"2","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84112024-04-20T01:10:45.340Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836895678Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84122024-04-20T01:10:45.340Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836931889Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84132024-04-20T01:10:45.340Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.836959209Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84142024-04-20T01:10:45.340Z{"msg":"ds_connection connected from Ok(127.0.0.1:42450)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.837080479Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84152024-04-20T01:10:45.340Z{"msg":"ds_connection connected from Ok(127.0.0.1:36419)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.837117859Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84162024-04-20T01:10:45.340Z{"msg":"ds_connection connected from Ok(127.0.0.1:48170)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.837144289Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84172024-04-20T01:10:45.340Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.837820811Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"0","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84182024-04-20T01:10:45.340Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.837869951Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"0","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84192024-04-20T01:10:45.340Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.837912471Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"1","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84202024-04-20T01:10:45.340Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.837945101Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"1","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84212024-04-20T01:10:45.340Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.838408692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"0","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84222024-04-20T01:10:45.340Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.838555172Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"1","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84232024-04-20T01:10:45.340Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.838595512Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"2","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84242024-04-20T01:10:45.340Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.838624822Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"2","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84252024-04-20T01:10:45.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.838835233Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"0","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84262024-04-20T01:10:45.340Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.838884103Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"0","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84272024-04-20T01:10:45.340Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.838910193Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84282024-04-20T01:10:45.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839074133Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"1","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84292024-04-20T01:10:45.340Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839109753Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"1","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84302024-04-20T01:10:45.340Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839125953Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84312024-04-20T01:10:45.340Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839245694Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"2","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84322024-04-20T01:10:45.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839624844Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"2","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84332024-04-20T01:10:45.340Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839658135Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"2","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84342024-04-20T01:10:45.341Z{"msg":"[0]R flush_numbers[0..12]: [127, 127, 128, 126, 126, 125, 122, 123, 123, 129, 125, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839673324Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84352024-04-20T01:10:45.341Z{"msg":"[0]R generation[0..12]: [76, 76, 76, 75, 75, 74, 71, 72, 72, 77, 74, 77]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839696245Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84362024-04-20T01:10:45.341Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839711835Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84372024-04-20T01:10:45.341Z{"msg":"[1]R flush_numbers[0..12]: [127, 127, 128, 126, 126, 125, 122, 123, 123, 129, 125, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839725755Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84382024-04-20T01:10:45.341Z{"msg":"[1]R generation[0..12]: [76, 76, 76, 75, 75, 74, 71, 72, 72, 77, 74, 77]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839747295Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84392024-04-20T01:10:45.341Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839762115Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84402024-04-20T01:10:45.341Z{"msg":"[2]R flush_numbers[0..12]: [127, 127, 128, 126, 126, 125, 122, 123, 123, 129, 125, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839776785Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84412024-04-20T01:10:45.341Z{"msg":"[2]R generation[0..12]: [76, 76, 76, 75, 75, 74, 71, 72, 72, 77, 74, 77]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839790385Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84422024-04-20T01:10:45.341Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839811565Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84432024-04-20T01:10:45.341Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839824855Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84442024-04-20T01:10:45.341Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839837985Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"Wait for a query_work_queue command to finish before sending IO
84452024-04-20T01:10:45.341Z":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84462024-04-20T01:10:45.341Z{"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839862555Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84472024-04-20T01:10:45.341Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839875705Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"":"downstairs","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84482024-04-20T01:10:45.341Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839889055Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84492024-04-20T01:10:45.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839908775Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"0","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84502024-04-20T01:10:45.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839922295Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"1","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84512024-04-20T01:10:45.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839935435Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"client":"2","":"downstairs","session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84522024-04-20T01:10:45.341Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839959615Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84532024-04-20T01:10:45.341Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.839982025Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84542024-04-20T01:10:45.341Z{"msg":"3ce75ef4-9adf-467e-a60f-ebf733cad2ee is now active with session: cdff2d7b-63b3-429e-9ca6-4130d39c959f","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.840040775Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84552024-04-20T01:10:45.341Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.840064956Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483,"session_id":"cdff2d7b-63b3-429e-9ca6-4130d39c959f"}
84562024-04-20T01:10:45.341Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:43.840087325Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483}
84572024-04-20T01:10:45.342ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84582024-04-20T01:10:45.345ZLoading write count information from file "/tmp/test_repair_verify.data"
84592024-04-20T01:10:45.345ZRun Repair workload
84602024-04-20T01:10:45.345Z01/30 Write block 434 len 40960 data: 7 8 8 7 5 6 6 7 7 7
84612024-04-20T01:10:45.345Z02/30 Read block 267 len 12288
84622024-04-20T01:10:45.346Z03/30 Read block 47 len 4096
84632024-04-20T01:10:45.346Z04/30 Flush
84642024-04-20T01:10:45.346Z05/30 Write block 254 len 36864 data: 6 5 5 5 5 5 4 4 4
84652024-04-20T01:10:45.346Z06/30 Read block 54 len 36864
84662024-04-20T01:10:45.346Z07/30 Read block 127 len 28672
84672024-04-20T01:10:45.349Z08/30 Write block 176 len 24576 data: 5 6 7 9 9 10
84682024-04-20T01:10:45.349Z09/30 Read block 585 len 24576
84692024-04-20T01:10:45.349Z10/30 Write block 292 len 12288 data: 3 4 6
84702024-04-20T01:10:45.349Z11/30 Read block 423 len 28672
84712024-04-20T01:10:45.349Z12/30 Flush
84722024-04-20T01:10:45.353Z13/30 Flush
84732024-04-20T01:10:45.353Z14/30 Write block 503 len 16384 data: 8 8 8 9
84742024-04-20T01:10:45.356Z15/30 Write block 520 len 32768 data: 9 10 9 10 9 8 7 6
84752024-04-20T01:10:45.356Z16/30 Read block 125 len 8192
84762024-04-20T01:10:45.356Z17/30 Read block 192 len 4096
84772024-04-20T01:10:45.356Z18/30 Read block 271 len 12288
84782024-04-20T01:10:45.356Z19/30 Read block 36 len 20480
84792024-04-20T01:10:45.356Z20/30 Read block 526 len 16384
84802024-04-20T01:10:45.360Z21/30 Read block 250 len 16384
84812024-04-20T01:10:45.360Z22/30 Write block 379 len 20480 data: 8 8 6 5 4
84822024-04-20T01:10:45.360Z23/30 Write block 138 len 40960 data: 10 9 9 8 8 8 8 8 7 6
84832024-04-20T01:10:45.363Z24/30 Read block 399 len 36864
84842024-04-20T01:10:45.363Z25/30 Read block 592 len 28672
84852024-04-20T01:10:45.363Z26/30 Write block 58 len 40960 data: 7 8 8 7 8 6 8 8 10 8
84862024-04-20T01:10:45.363Z27/30 Read block 422 len 36864
84872024-04-20T01:10:45.368Z28/30 Read block 15 len 24576
84882024-04-20T01:10:45.368Z29/30 Read block 480 len 28672
84892024-04-20T01:10:45.368Z30/30 Write block 359 len 20480 data: 7 6 7 8 8
84902024-04-20T01:10:45.368Z----------------------------------------------------------------
84912024-04-20T01:10:45.368Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
84922024-04-20T01:10:45.368ZStates: Active Active Active
84932024-04-20T01:10:45.368ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
84942024-04-20T01:10:45.368Z 1 Acked 1000 Write 10 Done Done Done false
84952024-04-20T01:10:45.368Z 2 Acked 1001 Read 3 Done Done Done false
84962024-04-20T01:10:45.368Z 3 Acked 1002 Read 1 Done Done Sent false
84972024-04-20T01:10:45.368Z 4 Acked 1003 Flush 0 Done Done Sent false
84982024-04-20T01:10:45.368Z 5 Acked 1004 Write 9 Done Done Sent false
84992024-04-20T01:10:45.369Z 6 Acked 1005 Read 9 Done Done Sent false
85002024-04-20T01:10:45.369Z 7 Acked 1006 Read 7 Done Done Sent false
85012024-04-20T01:10:45.369Z 8 Acked 1007 Write 6 Done Done Sent false
85022024-04-20T01:10:45.369Z 9 Acked 1008 Read 6 Done Done Sent false
85032024-04-20T01:10:45.369Z 10 Acked 1009 Write 3 Done Done Sent false
85042024-04-20T01:10:45.369Z 11 Acked 1010 Read 7 Done Done Sent false
85052024-04-20T01:10:45.369Z 12 Acked 1011 Flush 0 Done Done Sent false
85062024-04-20T01:10:45.369Z 13 Acked 1012 Flush 0 Done Done Sent false
85072024-04-20T01:10:45.369Z 14 Acked 1013 Write 4 Done Done Sent false
85082024-04-20T01:10:45.369Z 15 Acked 1014 Write 8 Done Done Sent false
85092024-04-20T01:10:45.369Z 16 Acked 1015 Read 2 Done Done Sent false
85102024-04-20T01:10:45.369Z 17 Acked 1016 Read 1 Done Done Sent false
85112024-04-20T01:10:45.369Z 18 Acked 1017 Read 3 Done Done Sent false
85122024-04-20T01:10:45.369Z 19 Acked 1018 Read 5 Done Done Sent false
85132024-04-20T01:10:45.369Z 20 Acked 1019 Read 4 Done Done Sent false
85142024-04-20T01:10:45.369Z 21 Acked 1020 Read 4 Done Done Sent false
85152024-04-20T01:10:45.369Z 22 Acked 1021 Write 5 Done Done Sent false
85162024-04-20T01:10:45.369Z 23 Acked 1022 Write 10 Done Done Sent false
85172024-04-20T01:10:45.369Z 24 Acked 1023 Read 9 Done Done Sent false
85182024-04-20T01:10:45.370Z 25 Acked 1024 Read 7 Done Done Sent false
85192024-04-20T01:10:45.370Z 26 Acked 1025 Write 10 Done Done Sent false
85202024-04-20T01:10:45.370Z 27 Acked 1026 Read 9 Done Done Sent false
85212024-04-20T01:10:45.370Z 28 Acked 1027 Read 6 Done Done Sent false
85222024-04-20T01:10:45.370Z 29 Acked 1028 Read 7 Done Done Sent false
85232024-04-20T01:10:45.370Z 30 Acked 1029 Write 5 Sent Sent Sent false
85242024-04-20T01:10:45.370Z STATES DS:0 DS:1 DS:2 TOTAL
85252024-04-20T01:10:45.370Z New 0 0 0 0
85262024-04-20T01:10:45.370Z Sent 1 1 28 30
85272024-04-20T01:10:45.370Z Done 29 29 2 60
85282024-04-20T01:10:45.370Z Skipped 0 0 0 0
85292024-04-20T01:10:45.370Z Error 0 0 0 0
85302024-04-20T01:10:45.370ZLast Flush: 1012 1012 0
85312024-04-20T01:10:45.370ZDownstairs last five completed:
85322024-04-20T01:10:45.370ZUpstairs last five completed: 30 29 28 27 26
85332024-04-20T01:10:45.370Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:10:43.872120017Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1483}
85342024-04-20T01:10:45.370ZWrote out file "/tmp/test_repair_verify.data"
85352024-04-20T01:10:45.374Z
85362024-04-20T01:10:45.374ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
85372024-04-20T01:10:45.391ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85382024-04-20T01:10:45.391Z 0 000-019 76 76 76 127 127 127 F F F
85392024-04-20T01:10:45.391Z 1 020-039 76 76 76 127 127 127 F F F
85402024-04-20T01:10:45.392Z 2 040-059 76 76 76 128 128 128 T T F <---
85412024-04-20T01:10:45.392Z 3 060-079 75 75 75 126 126 126 T T F <---
85422024-04-20T01:10:45.392Z 4 080-099 75 75 75 126 126 126 F F F
85432024-04-20T01:10:45.392Z 5 100-119 74 74 74 125 125 125 F F F
85442024-04-20T01:10:45.396Z 6 120-139 71 71 71 122 122 122 T T F <---
85452024-04-20T01:10:45.396Z 7 140-159 72 72 72 123 123 123 T T F <---
85462024-04-20T01:10:45.396Z 8 160-179 78 78 72 131 131 123 F F F <---
85472024-04-20T01:10:45.396Z 9 180-199 78 78 77 131 131 129 F F F <---
85482024-04-20T01:10:45.396Z 10 200-219 74 74 74 125 125 125 F F F
85492024-04-20T01:10:45.396Z 11 220-239 77 77 77 129 129 129 F F F
85502024-04-20T01:10:45.397Z 12 240-259 78 78 77 131 131 129 F F F <---
85512024-04-20T01:10:45.397Z 13 260-279 78 78 73 131 131 124 F F F <---
85522024-04-20T01:10:45.397Z 14 280-299 78 78 74 131 131 125 F F F <---
85532024-04-20T01:10:45.397Z 15 300-319 77 77 77 129 129 129 F F F
85542024-04-20T01:10:45.397Z 16 320-339 77 77 77 129 129 129 F F F
85552024-04-20T01:10:45.397Z 17 340-359 74 74 74 125 125 125 T T F <---
85562024-04-20T01:10:45.397Z 18 360-379 77 77 77 129 129 129 T T F <---
85572024-04-20T01:10:45.397Z 19 380-399 77 77 77 129 129 129 T T F <---
85582024-04-20T01:10:45.397Z 20 400-419 77 77 77 129 129 129 F F F
85592024-04-20T01:10:45.397Z 21 420-439 78 78 77 130 130 129 F F T <---
85602024-04-20T01:10:45.397Z 22 440-459 78 78 72 130 130 123 F F T <---
85612024-04-20T01:10:45.397Z 23 460-479 77 77 77 129 129 129 F F F
85622024-04-20T01:10:45.397Z 24 480-499 76 76 76 128 128 128 F F F
85632024-04-20T01:10:45.397Z 25 500-519 77 77 77 129 129 129 T T F <---
85642024-04-20T01:10:45.397Z 26 520-539 70 70 70 121 121 121 T T F <---
85652024-04-20T01:10:45.397Z 27 540-559 72 72 72 123 123 123 F F F
85662024-04-20T01:10:45.397Z 28 560-579 76 76 76 128 128 128 F F F
85672024-04-20T01:10:45.397Z 29 580-599 76 76 76 128 128 128 F F F
85682024-04-20T01:10:45.397ZMax gen: 78, Max flush: 131
85692024-04-20T01:10:45.397ZError: Difference in extent metadata found!
85702024-04-20T01:10:45.397ZA Difference in extent metadata is expected here
85712024-04-20T01:10:45.397ZOn loop 39
85722024-04-20T01:10:45.397Z
85732024-04-20T01:10:45.401ZVerifying data now
85742024-04-20T01:11:00.116ZNo change after verify
85752024-04-20T01:11:00.116ZLoop: 39 Downstairs dump after verify (and repair):
85762024-04-20T01:11:00.130ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85772024-04-20T01:11:00.130Z 0 000-019 76 76 76 127 127 127 F F F
85782024-04-20T01:11:00.130Z 1 020-039 76 76 76 127 127 127 F F F
85792024-04-20T01:11:00.130Z 2 040-059 79 79 79 132 132 132 F F F
85802024-04-20T01:11:00.130Z 3 060-079 79 79 79 132 132 132 F F F
85812024-04-20T01:11:00.130Z 4 080-099 75 75 75 126 126 126 F F F
85822024-04-20T01:11:00.130Z 5 100-119 74 74 74 125 125 125 F F F
85832024-04-20T01:11:00.134Z 6 120-139 79 79 79 132 132 132 F F F
85842024-04-20T01:11:00.134Z 7 140-159 79 79 79 132 132 132 F F F
85852024-04-20T01:11:00.134Z 8 160-179 78 78 78 131 131 131 F F F
85862024-04-20T01:11:00.134Z 9 180-199 78 78 78 131 131 131 F F F
85872024-04-20T01:11:00.134Z 10 200-219 74 74 74 125 125 125 F F F
85882024-04-20T01:11:00.134Z 11 220-239 77 77 77 129 129 129 F F F
85892024-04-20T01:11:00.134Z 12 240-259 78 78 78 131 131 131 F F F
85902024-04-20T01:11:00.134Z 13 260-279 78 78 78 131 131 131 F F F
85912024-04-20T01:11:00.134Z 14 280-299 78 78 78 131 131 131 F F F
85922024-04-20T01:11:00.134Z 15 300-319 77 77 77 129 129 129 F F F
85932024-04-20T01:11:00.134Z 16 320-339 77 77 77 129 129 129 F F F
85942024-04-20T01:11:00.134Z 17 340-359 79 79 79 132 132 132 F F F
85952024-04-20T01:11:00.134Z 18 360-379 79 79 79 132 132 132 F F F
85962024-04-20T01:11:00.134Z 19 380-399 79 79 79 132 132 132 F F F
85972024-04-20T01:11:00.134Z 20 400-419 77 77 77 129 129 129 F F F
85982024-04-20T01:11:00.134Z 21 420-439 78 78 78 130 130 130 F F F
85992024-04-20T01:11:00.134Z 22 440-459 78 78 78 130 130 130 F F F
86002024-04-20T01:11:00.134Z 23 460-479 77 77 77 129 129 129 F F F
86012024-04-20T01:11:00.134Z 24 480-499 76 76 76 128 128 128 F F F
86022024-04-20T01:11:00.134Z 25 500-519 79 79 79 132 132 132 F F F
86032024-04-20T01:11:00.135Z 26 520-539 79 79 79 132 132 132 F F F
86042024-04-20T01:11:00.135Z 27 540-559 72 72 72 123 123 123 F F F
86052024-04-20T01:11:00.135Z 28 560-579 76 76 76 128 128 128 F F F
86062024-04-20T01:11:00.135Z 29 580-599 76 76 76 128 128 128 F F F
86072024-04-20T01:11:00.135ZMax gen: 79, Max flush: 132
86082024-04-20T01:11:00.135Z
86092024-04-20T01:11:00.135ZBegin loop 40
86102024-04-20T01:11:00.135ZDownstairs to restart: 0
86112024-04-20T01:11:00.152Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.656572847Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491}
86122024-04-20T01:11:00.152Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.656780958Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491}
86132024-04-20T01:11:00.152Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.656797178Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491}
86142024-04-20T01:11:00.160Z{"msg":"Crucible 57c5b180-b183-4050-be68-04c901797a29 has session id: a4a4074e-5a85-4fdf-b834-4f23cdb2fe04","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.656810208Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86152024-04-20T01:11:00.160Z{"msg":"Upstairs opts: Upstairs UUID: 57c5b180-b183-4050-be68-04c901797a29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time"Crucible runtime is spawned
86162024-04-20T01:11:00.160Z:"2024-04-20T01:10:58.656855828Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86172024-04-20T01:11:00.160Z{"msg":"Crucible stats registered with UUID: 57c5b180-b183-4050-be68-04c901797a29","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.656902008Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86182024-04-20T01:11:00.160Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.656942608Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491}
86192024-04-20T01:11:00.160Z{"msg":"57c5b180-b183-4050-be68-04c901797a29 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.656983828Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86202024-04-20T01:11:00.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.657007218Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"0","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86212024-04-20T01:11:00.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.657039038Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"0","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86222024-04-20T01:11:00.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.657063958Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"1","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86232024-04-20T01:11:00.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.657088668Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"1","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86242024-04-20T01:11:00.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.657118678Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"2","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86252024-04-20T01:11:00.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.657143428Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"2","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86262024-04-20T01:11:00.160Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.657200339Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86272024-04-20T01:11:00.160Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.657247959Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86282024-04-20T01:11:00.160Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.657266499Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86292024-04-20T01:11:00.160Z{"msg":"ds_connection connected from Ok(127.0.0.1:51545)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.657433089Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86302024-04-20T01:11:00.160Z{"msg":"ds_connection connected from Ok(127.0.0.1:33779)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.657487429Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86312024-04-20T01:11:00.161Z{"msg":"ds_connection connected from Ok(127.0.0.1:46452)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.657513209Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86322024-04-20T01:11:00.161Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.658228951Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"1","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86332024-04-20T01:11:00.161Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.658267031Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"1","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86342024-04-20T01:11:00.161Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.658290411Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"2","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86352024-04-20T01:11:00.161Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.658326761Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"2","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86362024-04-20T01:11:00.161Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.658609082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"0","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86372024-04-20T01:11:00.161Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.658650862Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"0","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86382024-04-20T01:11:00.161Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.658799012Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"2","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86392024-04-20T01:11:00.161Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.658847372Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"1","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86402024-04-20T01:11:00.161Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659278623Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"0","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86412024-04-20T01:11:00.161Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659344613Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"2","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86422024-04-20T01:11:00.161Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659396913Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"2","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86432024-04-20T01:11:00.161Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659425913Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86442024-04-20T01:11:00.161Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659461704Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"1","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86452024-04-20T01:11:00.161Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659479054Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"1","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86462024-04-20T01:11:00.161Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659492344Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86472024-04-20T01:11:00.161Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659660084Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"0","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86482024-04-20T01:11:00.161Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659699304Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"0","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86492024-04-20T01:11:00.161Z{"msg":"[0]R flush_numbers[0..12]: [127, 127, 132, 132, 126, 125, 132, 132, 131, 131, 125, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659714944Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86502024-04-20T01:11:00.161Z{"msg":"[0]R generation[0..12]: [76, 76, 79, 79, 75, 74, 79, 79, 78, 78, 74, 77]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659736474Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86512024-04-20T01:11:00.161Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659751444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86522024-04-20T01:11:00.161Z{"msg":"[1]R flush_numbers[0..12]: [127, 127, 132, 132, 126, 125, 132, 132, 131, 131, 125, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659765134Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86532024-04-20T01:11:00.161Z{"msg":"[1]R generation[0..12]: [76, 76, 79, 79, 75, 74, 79, 79, 78, 78, 74, 77]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659785434Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86542024-04-20T01:11:00.161Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659799724Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86552024-04-20T01:11:00.162Z{"msg":"[2]R flush_numbers[0..12]: [127, 127, 132, 132, 126, 125, 132, 132, 131, 131, 125, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659826194Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86562024-04-20T01:11:00.162Z{"msg":"[2]R generation[0..12]: [76, 76, 79, 79, 75, 74, 79, 79, 78, 78, 74, 77]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659840254Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86572024-04-20T01:11:00.162ZWait for a query_work_queue command to finish before sending IO
86582024-04-20T01:11:00.162Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659866194Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86592024-04-20T01:11:00.162Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659880445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86602024-04-20T01:11:00.162Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659900385Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86612024-04-20T01:11:00.162Z{"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659913545Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86622024-04-20T01:11:00.162Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659945495Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"":"downstairs","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86632024-04-20T01:11:00.162Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.659976735Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86642024-04-20T01:11:00.162Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.660013435Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"0","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86652024-04-20T01:11:00.162Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.660049205Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"1","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86662024-04-20T01:11:00.162Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.660084685Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"client":"2","":"downstairs","session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86672024-04-20T01:11:00.162Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.660119645Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86682024-04-20T01:11:00.162Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.660155445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86692024-04-20T01:11:00.162ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86702024-04-20T01:11:00.162Z{"msg":"57c5b180-b183-4050-be68-04c901797a29 is now active with session: a4a4074e-5a85-4fdf-b834-4f23cdb2fe04","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.660195035Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86712024-04-20T01:11:00.162Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.660226675Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491,"session_id":"a4a4074e-5a85-4fdf-b834-4f23cdb2fe04"}
86722024-04-20T01:11:00.162Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:10:58.660240285Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491}
86732024-04-20T01:11:00.162ZLoading write count information from file "/tmp/test_repair_verify.data"
86742024-04-20T01:11:00.162ZRun Repair workload
86752024-04-20T01:11:00.166Z01/30 Write block 460 len 32768 data: 8 11 8 8 5 7 9 8
86762024-04-20T01:11:00.166Z02/30 Read block 190 len 4096
86772024-04-20T01:11:00.166Z03/30 Read block 276 len 24576
86782024-04-20T01:11:00.166Z04/30 Flush
86792024-04-20T01:11:00.166Z05/30 Write block 576 len 28672 data: 4 6 6 9 9 9 9
86802024-04-20T01:11:00.167Z06/30 Write block 509 len 32768 data: 11 10 10 12 11 9 8 10
86812024-04-20T01:11:00.167Z07/30 Read block 172 len 28672
86822024-04-20T01:11:00.167Z08/30 Write block 569 len 16384 data: 9 8 8 6
86832024-04-20T01:11:00.167Z09/30 Read block 244 len 32768
86842024-04-20T01:11:00.170Z10/30 Write block 565 len 36864 data: 7 8 7 8 10 9 9 7 5
86852024-04-20T01:11:00.170Z11/30 Read block 218 len 16384
86862024-04-20T01:11:00.170Z12/30 Read block 59 len 28672
86872024-04-20T01:11:00.170Z13/30 Read block 76 len 32768
86882024-04-20T01:11:00.174Z14/30 Flush
86892024-04-20T01:11:00.174Z15/30 Write block 536 len 8192 data: 4 4
86902024-04-20T01:11:00.174Z16/30 Read block 585 len 28672
86912024-04-20T01:11:00.177Z17/30 Write block 526 len 40960 data: 8 7 5 5 5 7 7 5 6 5
86922024-04-20T01:11:00.177Z18/30 Read block 348 len 28672
86932024-04-20T01:11:00.181Z19/30 Write block 86 len 40960 data: 8 8 8 9 10 8 7 5 5 4
86942024-04-20T01:11:00.181Z20/30 Read block 491 len 40960
86952024-04-20T01:11:00.181Z21/30 Write block 318 len 8192 data: 9 9
86962024-04-20T01:11:00.181Z22/30 Read block 234 len 36864
86972024-04-20T01:11:00.181Z23/30 Read block 6 len 16384
86982024-04-20T01:11:00.185Z24/30 Read block 586 len 4096
86992024-04-20T01:11:00.185Z25/30 Write block 338 len 8192 data: 5 5
87002024-04-20T01:11:00.185Z26/30 Read block 37 len 8192
87012024-04-20T01:11:00.185Z27/30 Write block 139 len 40960 data: 10 10 9 9 9 9 9 8 7 7
87022024-04-20T01:11:00.185Z28/30 Read block 32 len 28672
87032024-04-20T01:11:00.189Z29/30 Read block 6 len 28672
87042024-04-20T01:11:00.189Z30/30 Write block 517 len 16384 data: 11 8 9 10
87052024-04-20T01:11:00.189Z----------------------------------------------------------------
87062024-04-20T01:11:00.189Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
87072024-04-20T01:11:00.189ZStates: Active Active Active
87082024-04-20T01:11:00.189ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
87092024-04-20T01:11:00.189Z 1 Acked 1000 Write 8 Done Done Done false
87102024-04-20T01:11:00.189Z 2 Acked 1001 Read 1 Done Done Done false
87112024-04-20T01:11:00.189Z 3 Acked 1002 Read 6 Done Done Done false
87122024-04-20T01:11:00.190Z 4 Acked 1003 Flush 0 Sent Done Done false
87132024-04-20T01:11:00.190Z 5 Acked 1004 Write 7 Sent Done Done false
87142024-04-20T01:11:00.190Z 6 Acked 1005 Write 8 Sent Done Done false
87152024-04-20T01:11:00.190Z 7 Acked 1006 Read 7 Sent Done Done false
87162024-04-20T01:11:00.190Z 8 Acked 1007 Write 4 Sent Done Done false
87172024-04-20T01:11:00.190Z 9 Acked 1008 Read 8 Sent Done Done false
87182024-04-20T01:11:00.190Z 10 Acked 1009 Write 9 Sent Done Done false
87192024-04-20T01:11:00.190Z 11 Acked 1010 Read 4 Sent Done Done false
87202024-04-20T01:11:00.190Z 12 Acked 1011 Read 7 Sent Done Done false
87212024-04-20T01:11:00.190Z 13 Acked 1012 Read 8 Sent Done Done false
87222024-04-20T01:11:00.190Z 14 Acked 1013 Flush 0 Sent Done Done false
87232024-04-20T01:11:00.190Z 15 Acked 1014 Write 2 Sent Done Done false
87242024-04-20T01:11:00.190Z 16 Acked 1015 Read 7 Sent Done Done false
87252024-04-20T01:11:00.190Z 17 Acked 1016 Write 10 Sent Done Done false
87262024-04-20T01:11:00.190Z 18 Acked 1017 Read 7 Sent Done Done false
87272024-04-20T01:11:00.190Z 19 Acked 1018 Write 10 Sent Done Done false
87282024-04-20T01:11:00.190Z 20 Acked 1019 Read 10 Sent Done Done false
87292024-04-20T01:11:00.190Z 21 Acked 1020 Write 2 Sent Done Done false
87302024-04-20T01:11:00.190Z 22 Acked 1021 Read 9 Sent Done Done false
87312024-04-20T01:11:00.190Z 23 Acked 1022 Read 4 Sent Done Done false
87322024-04-20T01:11:00.190Z 24 Acked 1023 Read 1 Sent Done Done false
87332024-04-20T01:11:00.190Z 25 Acked 1024 Write 2 Sent Done Done false
87342024-04-20T01:11:00.190Z 26 Acked 1025 Read 2 Sent Done Done false
87352024-04-20T01:11:00.191Z 27 Acked 1026 Write 10 Sent Done Done false
87362024-04-20T01:11:00.191Z 28 Acked 1027 Read 7 Sent Done Done false
87372024-04-20T01:11:00.191Z 29 Acked 1028 Read 7 Sent Done Done false
87382024-04-20T01:11:00.191Z 30 Acked 1029 Write 4 Sent Sent Sent false
87392024-04-20T01:11:00.191Z STATES DS:0 DS:1 DS:2 TOTAL
87402024-04-20T01:11:00.191Z New 0 0 0 0
87412024-04-20T01:11:00.191Z Sent 27 1 1 29
87422024-04-20T01:11:00.191Z Done 3 29 29 61
87432024-04-20T01:11:00.191Z Skipped 0 0 0 0
87442024-04-20T01:11:00.191Z Error 0 0 0 0
87452024-04-20T01:11:00.191ZLast Flush: 0 1013 1013
87462024-04-20T01:11:00.191ZDownstairs last five completed:
87472024-04-20T01:11:00.191ZUpstairs last five completed: 30 29 28 27 26
87482024-04-20T01:11:00.191Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:10:58.692030856Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1491}
87492024-04-20T01:11:00.191ZWrote out file "/tmp/test_repair_verify.data"
87502024-04-20T01:11:00.198Z
87512024-04-20T01:11:00.198ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87522024-04-20T01:11:00.211ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87532024-04-20T01:11:00.211Z 0 000-019 76 76 76 127 127 127 F F F
87542024-04-20T01:11:00.211Z 1 020-039 76 76 76 127 127 127 F F F
87552024-04-20T01:11:00.211Z 2 040-059 79 79 79 132 132 132 F F F
87562024-04-20T01:11:00.211Z 3 060-079 79 79 79 132 132 132 F F F
87572024-04-20T01:11:00.211Z 4 080-099 75 75 75 126 126 126 F T T <---
87582024-04-20T01:11:00.215Z 5 100-119 74 74 74 125 125 125 F F F
87592024-04-20T01:11:00.215Z 6 120-139 79 79 79 132 132 132 F T T <---
87602024-04-20T01:11:00.215Z 7 140-159 79 79 79 132 132 132 F T T <---
87612024-04-20T01:11:00.215Z 8 160-179 78 78 78 131 131 131 F F F
87622024-04-20T01:11:00.215Z 9 180-199 78 78 78 131 131 131 F F F
87632024-04-20T01:11:00.215Z 10 200-219 74 74 74 125 125 125 F F F
87642024-04-20T01:11:00.215Z 11 220-239 77 77 77 129 129 129 F F F
87652024-04-20T01:11:00.215Z 12 240-259 78 78 78 131 131 131 F F F
87662024-04-20T01:11:00.215Z 13 260-279 78 78 78 131 131 131 F F F
87672024-04-20T01:11:00.215Z 14 280-299 78 78 78 131 131 131 F F F
87682024-04-20T01:11:00.215Z 15 300-319 77 77 77 129 129 129 F T T <---
87692024-04-20T01:11:00.215Z 16 320-339 77 77 77 129 129 129 F T T <---
87702024-04-20T01:11:00.215Z 17 340-359 79 79 79 132 132 132 F F F
87712024-04-20T01:11:00.215Z 18 360-379 79 79 79 132 132 132 F F F
87722024-04-20T01:11:00.215Z 19 380-399 79 79 79 132 132 132 F F F
87732024-04-20T01:11:00.215Z 20 400-419 77 77 77 129 129 129 F F F
87742024-04-20T01:11:00.215Z 21 420-439 78 78 78 130 130 130 F F F
87752024-04-20T01:11:00.215Z 22 440-459 78 78 78 130 130 130 F F F
87762024-04-20T01:11:00.215Z 23 460-479 77 80 80 129 133 133 T F F <---
87772024-04-20T01:11:00.215Z 24 480-499 76 76 76 128 128 128 F F F
87782024-04-20T01:11:00.215Z 25 500-519 79 80 80 132 134 134 F T T <---
87792024-04-20T01:11:00.215Z 26 520-539 79 79 79 132 132 132 F T T <---
87802024-04-20T01:11:00.216Z 27 540-559 72 72 72 123 123 123 F F F
87812024-04-20T01:11:00.216Z 28 560-579 76 80 80 128 134 134 F F F <---
87822024-04-20T01:11:00.216Z 29 580-599 76 80 80 128 134 134 F F F <---
87832024-04-20T01:11:00.216ZMax gen: 80, Max flush: 134
87842024-04-20T01:11:00.216ZError: Difference in extent metadata found!
87852024-04-20T01:11:00.216ZA Difference in extent metadata is expected here
87862024-04-20T01:11:00.216ZOn loop 40
87872024-04-20T01:11:00.216Z
87882024-04-20T01:11:00.219ZVerifying data now
87892024-04-20T01:11:14.698ZNo change after verify
87902024-04-20T01:11:14.698ZLoop: 40 Downstairs dump after verify (and repair):
87912024-04-20T01:11:14.713ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87922024-04-20T01:11:14.713Z 0 000-019 76 76 76 127 127 127 F F F
87932024-04-20T01:11:14.713Z 1 020-039 76 76 76 127 127 127 F F F
87942024-04-20T01:11:14.713Z 2 040-059 79 79 79 132 132 132 F F F
87952024-04-20T01:11:14.713Z 3 060-079 79 79 79 132 132 132 F F F
87962024-04-20T01:11:14.713Z 4 080-099 81 81 81 135 135 135 F F F
87972024-04-20T01:11:14.713Z 5 100-119 74 74 74 125 125 125 F F F
87982024-04-20T01:11:14.717Z 6 120-139 81 81 81 135 135 135 F F F
87992024-04-20T01:11:14.717Z 7 140-159 81 81 81 135 135 135 F F F
88002024-04-20T01:11:14.717Z 8 160-179 78 78 78 131 131 131 F F F
88012024-04-20T01:11:14.717Z 9 180-199 78 78 78 131 131 131 F F F
88022024-04-20T01:11:14.717Z 10 200-219 74 74 74 125 125 125 F F F
88032024-04-20T01:11:14.717Z 11 220-239 77 77 77 129 129 129 F F F
88042024-04-20T01:11:14.717Z 12 240-259 78 78 78 131 131 131 F F F
88052024-04-20T01:11:14.717Z 13 260-279 78 78 78 131 131 131 F F F
88062024-04-20T01:11:14.717Z 14 280-299 78 78 78 131 131 131 F F F
88072024-04-20T01:11:14.717Z 15 300-319 81 81 81 135 135 135 F F F
88082024-04-20T01:11:14.717Z 16 320-339 81 81 81 135 135 135 F F F
88092024-04-20T01:11:14.717Z 17 340-359 79 79 79 132 132 132 F F F
88102024-04-20T01:11:14.717Z 18 360-379 79 79 79 132 132 132 F F F
88112024-04-20T01:11:14.717Z 19 380-399 79 79 79 132 132 132 F F F
88122024-04-20T01:11:14.717Z 20 400-419 77 77 77 129 129 129 F F F
88132024-04-20T01:11:14.717Z 21 420-439 78 78 78 130 130 130 F F F
88142024-04-20T01:11:14.717Z 22 440-459 78 78 78 130 130 130 F F F
88152024-04-20T01:11:14.717Z 23 460-479 80 80 80 133 133 133 F F F
88162024-04-20T01:11:14.717Z 24 480-499 76 76 76 128 128 128 F F F
88172024-04-20T01:11:14.718Z 25 500-519 81 81 81 135 135 135 F F F
88182024-04-20T01:11:14.718Z 26 520-539 81 81 81 135 135 135 F F F
88192024-04-20T01:11:14.718Z 27 540-559 72 72 72 123 123 123 F F F
88202024-04-20T01:11:14.718Z 28 560-579 80 80 80 134 134 134 F F F
88212024-04-20T01:11:14.718Z 29 580-599 80 80 80 134 134 134 F F F
88222024-04-20T01:11:14.718ZMax gen: 81, Max flush: 135
88232024-04-20T01:11:14.718Z
88242024-04-20T01:11:14.718ZBegin loop 41
88252024-04-20T01:11:14.718ZDownstairs to restart: 2
88262024-04-20T01:11:14.733Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238358826Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499}
88272024-04-20T01:11:14.743Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238628727Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499}
88282024-04-20T01:11:14.743ZCrucible runtime is spawned
88292024-04-20T01:11:14.743Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238709607Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499}
88302024-04-20T01:11:14.743Z{"msg":"Crucible 9e2b0f79-a81d-4e86-aa7e-8dadf92cf6e0 has session id: f16c9d1b-0753-42e6-a885-fee25cdd1f3c","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238744367Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88312024-04-20T01:11:14.743Z{"msg":"Upstairs opts: Upstairs UUID: 9e2b0f79-a81d-4e86-aa7e-8dadf92cf6e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238771727Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88322024-04-20T01:11:14.743Z{"msg":"Crucible stats registered with UUID: 9e2b0f79-a81d-4e86-aa7e-8dadf92cf6e0","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238795667Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88332024-04-20T01:11:14.743Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238809297Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499}
88342024-04-20T01:11:14.743Z{"msg":"9e2b0f79-a81d-4e86-aa7e-8dadf92cf6e0 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238823327Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88352024-04-20T01:11:14.743Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238855157Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"0","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88362024-04-20T01:11:14.743Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238900057Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"0","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88372024-04-20T01:11:14.743Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238926357Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"1","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88382024-04-20T01:11:14.743Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238963758Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"1","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88392024-04-20T01:11:14.743Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.238989667Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"2","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88402024-04-20T01:11:14.743Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.239022188Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"2","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88412024-04-20T01:11:14.743Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.239060748Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88422024-04-20T01:11:14.743Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.239116798Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88432024-04-20T01:11:14.743Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.239209808Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88442024-04-20T01:11:14.743Z{"msg":"ds_connection connected from Ok(127.0.0.1:53328)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.239263408Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88452024-04-20T01:11:14.743Z{"msg":"ds_connection connected from Ok(127.0.0.1:54040)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.239295938Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88462024-04-20T01:11:14.743Z{"msg":"ds_connection connected from Ok(127.0.0.1:55886)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.239326518Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88472024-04-20T01:11:14.744Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.23988954Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"0","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88482024-04-20T01:11:14.744Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.23993522Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"0","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88492024-04-20T01:11:14.744Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.23995059Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"1","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88502024-04-20T01:11:14.744Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.23997958Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"1","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88512024-04-20T01:11:14.744Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.240412501Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"2","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88522024-04-20T01:11:14.744Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.240472371Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"2","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88532024-04-20T01:11:14.744Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.240500941Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"1","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88542024-04-20T01:11:14.744Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.240548001Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"0","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88552024-04-20T01:11:14.744Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.240989702Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"1","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88562024-04-20T01:11:14.744Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241056032Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"1","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88572024-04-20T01:11:14.744Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241095512Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88582024-04-20T01:11:14.744Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241122122Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"2","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88592024-04-20T01:11:14.744Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241165972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"0","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88602024-04-20T01:11:14.744Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241199952Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"0","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88612024-04-20T01:11:14.744Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241227463Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88622024-04-20T01:11:14.744Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241561483Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"2","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88632024-04-20T01:11:14.744Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241597403Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"2","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88642024-04-20T01:11:14.744Z{"msg":"[0]R flush_numbers[0..12]: [127, 127, 132, 132, 135, 125, 135, 135, 131, 131, 125, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241613513Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88652024-04-20T01:11:14.744Z{"msg":"[0]R generation[0..12]: [76, 76, 79, 79, 81, 74, 81, 81, 78, 78, 74, 77]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241628533Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88662024-04-20T01:11:14.744Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241642784Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88672024-04-20T01:11:14.744Z{"msg":"[1]R flush_numbers[0..12]: [127, 127, 132, 132, 135, 125, 135, 135, 131, 131, 125, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241664444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88682024-04-20T01:11:14.744Z{"msg":"[1]R generation[0..12]: [76, 76, 79, 79, 81, 74, 81, 81, 78, 78, 74, 77]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241678453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88692024-04-20T01:11:14.745Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241692624Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88702024-04-20T01:11:14.745Z{"msg":"[2]R flush_numbers[0..12]: [127, 127, 132, 132, 135, 125, 135, 135, 131, 131, 125, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241706344Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88712024-04-20T01:11:14.745Z{"msg":"[2]R generation[0..12]: [76, 76, 79, 79, 81, 74, 81, 81, 78, 78, 74, 77]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241733484Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88722024-04-20T01:11:14.745Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241747734Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88732024-04-20T01:11:14.745Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
88742024-04-20T01:11:14.745Z,"time":"2024-04-20T01:11:13.241761034Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88752024-04-20T01:11:14.745Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241775504Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88762024-04-20T01:11:14.745Z{"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241788894Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88772024-04-20T01:11:14.745Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241808954Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"":"downstairs","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88782024-04-20T01:11:14.745Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241822564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88792024-04-20T01:11:14.745Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241846504Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"0","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88802024-04-20T01:11:14.745Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241860144Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"1","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88812024-04-20T01:11:14.745Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241873784Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"client":"2","":"downstairs","session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88822024-04-20T01:11:14.745Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241893774Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88832024-04-20T01:11:14.745Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241906444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88842024-04-20T01:11:14.745Z{"msg":"9e2b0f79-a81d-4e86-aa7e-8dadf92cf6e0 is now active with session: f16c9d1b-0753-42e6-a885-fee25cdd1f3c","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241926434Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88852024-04-20T01:11:14.745Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241938774Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499,"session_id":"f16c9d1b-0753-42e6-a885-fee25cdd1f3c"}
88862024-04-20T01:11:14.745Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:13.241950754Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499}
88872024-04-20T01:11:14.745ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88882024-04-20T01:11:14.749ZLoading write count information from file "/tmp/test_repair_verify.data"
88892024-04-20T01:11:14.749ZRun Repair workload
88902024-04-20T01:11:14.750Z01/30 Write block 68 len 28672 data: 9 10 7 7 5 6 7
88912024-04-20T01:11:14.750Z02/30 Read block 519 len 4096
88922024-04-20T01:11:14.750Z03/30 Write block 222 len 12288 data: 9 9 9
88932024-04-20T01:11:14.750Z04/30 Write block 455 len 40960 data: 11 11 13 11 9 9 12 9 9 6
88942024-04-20T01:11:14.750Z05/30 Read block 54 len 36864
88952024-04-20T01:11:14.750Z06/30 Write block 465 len 36864 data: 8 10 9 8 10 10 9 9 10
88962024-04-20T01:11:14.750Z07/30 Write block 356 len 16384 data: 7 6 7 8
88972024-04-20T01:11:14.750Z08/30 Flush
88982024-04-20T01:11:14.750Z09/30 Flush
88992024-04-20T01:11:14.754Z10/30 Write block 141 len 28672 data: 10 10 10 10 10 9 8
89002024-04-20T01:11:14.754Z11/30 Write block 108 len 36864 data: 6 5 6 6 7 7 7 7 6
89012024-04-20T01:11:14.754Z12/30 Read block 97 len 32768
89022024-04-20T01:11:14.754Z13/30 Read block 99 len 36864
89032024-04-20T01:11:14.754Z14/30 Flush
89042024-04-20T01:11:14.758Z15/30 Write block 401 len 36864 data: 6 6 6 5 6 7 6 6 6
89052024-04-20T01:11:14.759Z16/30 Read block 152 len 8192
89062024-04-20T01:11:14.763Z17/30 Write block 354 len 16384 data: 5 6 8 7
89072024-04-20T01:11:14.763Z18/30 Read block 464 len 28672
89082024-04-20T01:11:14.763Z19/30 Read block 532 len 40960
89092024-04-20T01:11:14.764Z20/30 Write block 55 len 8192 data: 8 7
89102024-04-20T01:11:14.764Z21/30 Read block 174 len 4096
89112024-04-20T01:11:14.764Z22/30 Read block 251 len 28672
89122024-04-20T01:11:14.764Z23/30 Read block 30 len 24576
89132024-04-20T01:11:14.767Z24/30 Write block 40 len 8192 data: 9 8
89142024-04-20T01:11:14.767Z25/30 Read block 191 len 20480
89152024-04-20T01:11:14.767Z26/30 Write block 336 len 20480 data: 3 4 6 6 8
89162024-04-20T01:11:14.767Z27/30 Write block 117 len 4096 data: 5
89172024-04-20T01:11:14.767Z28/30 Write block 34 len 8192 data: 8 8
89182024-04-20T01:11:14.767Z29/30 Read block 211 len 32768
89192024-04-20T01:11:14.767Z30/30 Read block 56 len 24576
89202024-04-20T01:11:14.771Z----------------------------------------------------------------
89212024-04-20T01:11:14.771Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
89222024-04-20T01:11:14.771ZStates: Active Active Active
89232024-04-20T01:11:14.771ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89242024-04-20T01:11:14.772Z 1 Acked 1000 Write 7 Done Done Done false
89252024-04-20T01:11:14.772Z 2 Acked 1001 Read 1 Done Done Done false
89262024-04-20T01:11:14.772Z 3 Acked 1002 Write 3 Done Done Done false
89272024-04-20T01:11:14.772Z 4 Acked 1003 Write 10 Done Done Done false
89282024-04-20T01:11:14.772Z 5 Acked 1004 Read 9 Done Done Done false
89292024-04-20T01:11:14.772Z 6 Acked 1005 Write 9 Done Done Sent false
89302024-04-20T01:11:14.772Z 7 Acked 1006 Write 4 Done Done Sent false
89312024-04-20T01:11:14.772Z 8 Acked 1007 Flush 0 Done Done Sent false
89322024-04-20T01:11:14.772Z 9 Acked 1008 Flush 0 Done Done Sent false
89332024-04-20T01:11:14.772Z 10 Acked 1009 Write 7 Done Done Sent false
89342024-04-20T01:11:14.772Z 11 Acked 1010 Write 9 Done Done Sent false
89352024-04-20T01:11:14.772Z 12 Acked 1011 Read 8 Done Done Sent false
89362024-04-20T01:11:14.772Z 13 Acked 1012 Read 9 Done Done Sent false
89372024-04-20T01:11:14.772Z 14 Acked 1013 Flush 0 Done Done Sent false
89382024-04-20T01:11:14.772Z 15 Acked 1014 Write 9 Done Done Sent false
89392024-04-20T01:11:14.772Z 16 Acked 1015 Read 2 Done Done Sent false
89402024-04-20T01:11:14.772Z 17 Acked 1016 Write 4 Done Done Sent false
89412024-04-20T01:11:14.772Z 18 Acked 1017 Read 7 Done Done Sent false
89422024-04-20T01:11:14.772Z 19 Acked 1018 Read 10 Done Done Sent false
89432024-04-20T01:11:14.772Z 20 Acked 1019 Write 2 Done Done Sent false
89442024-04-20T01:11:14.772Z 21 Acked 1020 Read 1 Done Done Sent false
89452024-04-20T01:11:14.772Z 22 Acked 1021 Read 7 Done Done Sent false
89462024-04-20T01:11:14.772Z 23 Acked 1022 Read 6 Done Done Sent false
89472024-04-20T01:11:14.773Z 24 Acked 1023 Write 2 Done Done Sent false
89482024-04-20T01:11:14.773Z 25 Acked 1024 Read 5 Done Done Sent false
89492024-04-20T01:11:14.773Z 26 Acked 1025 Write 5 Done Done Sent false
89502024-04-20T01:11:14.773Z 27 Acked 1026 Write 1 Done Done Sent false
89512024-04-20T01:11:14.773Z 28 Acked 1027 Write 2 Done Done Sent false
89522024-04-20T01:11:14.773Z 29 Acked 1028 Read 8 Done Done Sent false
89532024-04-20T01:11:14.773Z 30 Acked 1029 Read 6 Done Done Sent false
89542024-04-20T01:11:14.773Z STATES DS:0 DS:1 DS:2 TOTAL
89552024-04-20T01:11:14.773Z New 0 0 0 0
89562024-04-20T01:11:14.773Z Sent 0 0 25 25
89572024-04-20T01:11:14.773Z Done 30 30 5 65
89582024-04-20T01:11:14.773Z Skipped 0 0 0 0
89592024-04-20T01:11:14.773Z Error 0 0 0 0
89602024-04-20T01:11:14.773ZLast Flush: 1013 1013 0
89612024-04-20T01:11:14.773ZDownstairs last five completed:
89622024-04-20T01:11:14.773ZUpstairs last five completed: 30 29 28 27 26
89632024-04-20T01:11:14.773Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:11:13.272529742Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1499}
89642024-04-20T01:11:14.773ZWrote out file "/tmp/test_repair_verify.data"
89652024-04-20T01:11:14.778Z
89662024-04-20T01:11:14.778ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89672024-04-20T01:11:14.792ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89682024-04-20T01:11:14.792Z 0 000-019 76 76 76 127 127 127 F F F
89692024-04-20T01:11:14.792Z 1 020-039 76 76 76 127 127 127 T T F <---
89702024-04-20T01:11:14.792Z 2 040-059 79 79 79 132 132 132 T T F <---
89712024-04-20T01:11:14.792Z 3 060-079 82 82 79 136 136 132 F F T <---
89722024-04-20T01:11:14.792Z 4 080-099 81 81 81 135 135 135 F F F
89732024-04-20T01:11:14.792Z 5 100-119 82 82 74 138 138 125 T T F <---
89742024-04-20T01:11:14.792Z 6 120-139 81 81 81 135 135 135 F F F
89752024-04-20T01:11:14.792Z 7 140-159 82 82 81 138 138 135 F F F <---
89762024-04-20T01:11:14.796Z 8 160-179 78 78 78 131 131 131 F F F
89772024-04-20T01:11:14.796Z 9 180-199 78 78 78 131 131 131 F F F
89782024-04-20T01:11:14.796Z 10 200-219 74 74 74 125 125 125 F F F
89792024-04-20T01:11:14.796Z 11 220-239 82 82 77 136 136 129 F F T <---
89802024-04-20T01:11:14.796Z 12 240-259 78 78 78 131 131 131 F F F
89812024-04-20T01:11:14.796Z 13 260-279 78 78 78 131 131 131 F F F
89822024-04-20T01:11:14.796Z 14 280-299 78 78 78 131 131 131 F F F
89832024-04-20T01:11:14.796Z 15 300-319 81 81 81 135 135 135 F F F
89842024-04-20T01:11:14.796Z 16 320-339 81 81 81 135 135 135 T T F <---
89852024-04-20T01:11:14.796Z 17 340-359 82 82 79 136 136 132 T T F <---
89862024-04-20T01:11:14.796Z 18 360-379 79 79 79 132 132 132 F F F
89872024-04-20T01:11:14.796Z 19 380-399 79 79 79 132 132 132 F F F
89882024-04-20T01:11:14.796Z 20 400-419 77 77 77 129 129 129 T T F <---
89892024-04-20T01:11:14.797Z 21 420-439 78 78 78 130 130 130 F F F
89902024-04-20T01:11:14.797Z 22 440-459 82 82 78 136 136 130 F F T <---
89912024-04-20T01:11:14.797Z 23 460-479 82 82 80 136 136 133 F F T <---
89922024-04-20T01:11:14.797Z 24 480-499 76 76 76 128 128 128 F F F
89932024-04-20T01:11:14.797Z 25 500-519 81 81 81 135 135 135 F F F
89942024-04-20T01:11:14.797Z 26 520-539 81 81 81 135 135 135 F F F
89952024-04-20T01:11:14.797Z 27 540-559 72 72 72 123 123 123 F F F
89962024-04-20T01:11:14.797Z 28 560-579 80 80 80 134 134 134 F F F
89972024-04-20T01:11:14.797Z 29 580-599 80 80 80 134 134 134 F F F
89982024-04-20T01:11:14.797ZMax gen: 82, Max flush: 138
89992024-04-20T01:11:14.797ZError: Difference in extent metadata found!
90002024-04-20T01:11:14.797ZA Difference in extent metadata is expected here
90012024-04-20T01:11:14.797ZOn loop 41
90022024-04-20T01:11:14.797Z
90032024-04-20T01:11:14.801ZVerifying data now
90042024-04-20T01:11:29.319ZNo change after verify
90052024-04-20T01:11:29.319ZLoop: 41 Downstairs dump after verify (and repair):
90062024-04-20T01:11:29.333ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90072024-04-20T01:11:29.333Z 0 000-019 76 76 76 127 127 127 F F F
90082024-04-20T01:11:29.333Z 1 020-039 83 83 83 139 139 139 F F F
90092024-04-20T01:11:29.333Z 2 040-059 83 83 83 139 139 139 F F F
90102024-04-20T01:11:29.333Z 3 060-079 82 82 82 136 136 136 F F F
90112024-04-20T01:11:29.333Z 4 080-099 81 81 81 135 135 135 F F F
90122024-04-20T01:11:29.333Z 5 100-119 83 83 83 139 139 139 F F F
90132024-04-20T01:11:29.333Z 6 120-139 81 81 81 135 135 135 F F F
90142024-04-20T01:11:29.333Z 7 140-159 82 82 82 138 138 138 F F F
90152024-04-20T01:11:29.333Z 8 160-179 78 78 78 131 131 131 F F F
90162024-04-20T01:11:29.333Z 9 180-199 78 78 78 131 131 131 F F F
90172024-04-20T01:11:29.334Z 10 200-219 74 74 74 125 125 125 F F F
90182024-04-20T01:11:29.334Z 11 220-239 82 82 82 136 136 136 F F F
90192024-04-20T01:11:29.337Z 12 240-259 78 78 78 131 131 131 F F F
90202024-04-20T01:11:29.337Z 13 260-279 78 78 78 131 131 131 F F F
90212024-04-20T01:11:29.337Z 14 280-299 78 78 78 131 131 131 F F F
90222024-04-20T01:11:29.337Z 15 300-319 81 81 81 135 135 135 F F F
90232024-04-20T01:11:29.337Z 16 320-339 83 83 83 139 139 139 F F F
90242024-04-20T01:11:29.337Z 17 340-359 83 83 83 139 139 139 F F F
90252024-04-20T01:11:29.337Z 18 360-379 79 79 79 132 132 132 F F F
90262024-04-20T01:11:29.338Z 19 380-399 79 79 79 132 132 132 F F F
90272024-04-20T01:11:29.338Z 20 400-419 83 83 83 139 139 139 F F F
90282024-04-20T01:11:29.338Z 21 420-439 78 78 78 130 130 130 F F F
90292024-04-20T01:11:29.338Z 22 440-459 82 82 82 136 136 136 F F F
90302024-04-20T01:11:29.338Z 23 460-479 82 82 82 136 136 136 F F F
90312024-04-20T01:11:29.338Z 24 480-499 76 76 76 128 128 128 F F F
90322024-04-20T01:11:29.338Z 25 500-519 81 81 81 135 135 135 F F F
90332024-04-20T01:11:29.338Z 26 520-539 81 81 81 135 135 135 F F F
90342024-04-20T01:11:29.338Z 27 540-559 72 72 72 123 123 123 F F F
90352024-04-20T01:11:29.338Z 28 560-579 80 80 80 134 134 134 F F F
90362024-04-20T01:11:29.338Z 29 580-599 80 80 80 134 134 134 F F F
90372024-04-20T01:11:29.338ZMax gen: 83, Max flush: 139
90382024-04-20T01:11:29.338Z
90392024-04-20T01:11:29.338ZBegin loop 42
90402024-04-20T01:11:29.338ZDownstairs to restart: 0
90412024-04-20T01:11:29.355Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.858939682Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507}
90422024-04-20T01:11:29.355Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859163082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507}
90432024-04-20T01:11:29.363Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859189762Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507}
90442024-04-20T01:11:29.363Z{"msg":"Crucible 77178183-146d-44fc-82ac-3017c16fbe90 has session id: e6f7c95f-cc77-495a-b384-ec8fc906413e","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859223402Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90452024-04-20T01:11:29.363Z{"msg":"Upstairs opts: Upstairs UUID: 77178183-146d-44fc-82ac-3017c16fbe90, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859262552Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90462024-04-20T01:11:29.363Z{"msg":"Crucible stats registered with UUID: 77178183-146d-44fc-82ac-3017c16fbe90","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859299203Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90472024-04-20T01:11:29.363ZCrucible runtime is spawned
90482024-04-20T01:11:29.363Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859438683Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507}
90492024-04-20T01:11:29.363Z{"msg":"77178183-146d-44fc-82ac-3017c16fbe90 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859510513Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90502024-04-20T01:11:29.363Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859544263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"0","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90512024-04-20T01:11:29.363Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859560603Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"0","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90522024-04-20T01:11:29.363Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859582933Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"1","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90532024-04-20T01:11:29.363Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859597333Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"1","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90542024-04-20T01:11:29.363Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859626983Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"2","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90552024-04-20T01:11:29.363Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859645633Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"2","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90562024-04-20T01:11:29.363Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859667223Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90572024-04-20T01:11:29.363Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859681473Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90582024-04-20T01:11:29.363Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859701833Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90592024-04-20T01:11:29.363Z{"msg":"ds_connection connected from Ok(127.0.0.1:60856)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859892544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90602024-04-20T01:11:29.363Z{"msg":"ds_connection connected from Ok(127.0.0.1:35172)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859952654Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90612024-04-20T01:11:29.363Z{"msg":"ds_connection connected from Ok(127.0.0.1:49915)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.859984064Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90622024-04-20T01:11:29.363Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.860720206Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"2","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90632024-04-20T01:11:29.364Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.860768286Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"2","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90642024-04-20T01:11:29.364Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.860798836Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"1","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90652024-04-20T01:11:29.364Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.860841786Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"1","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90662024-04-20T01:11:29.364Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.861205027Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"0","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90672024-04-20T01:11:29.364Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.861243397Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"0","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90682024-04-20T01:11:29.364Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.861295537Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"2","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90692024-04-20T01:11:29.364Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.861342967Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"1","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90702024-04-20T01:11:29.364Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.861771728Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"2","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90712024-04-20T01:11:29.364Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.861808398Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"2","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90722024-04-20T01:11:29.364Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.861824318Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90732024-04-20T01:11:29.364Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.861925188Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"0","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90742024-04-20T01:11:29.364Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.861961128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"1","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90752024-04-20T01:11:29.364Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.861978058Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"1","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90762024-04-20T01:11:29.364Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.861991889Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90772024-04-20T01:11:29.364Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.862352209Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"0","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90782024-04-20T01:11:29.364Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.862388539Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"0","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90792024-04-20T01:11:29.364Z{"msg":"[0]R flush_numbers[0..12]: [127, 139, 139, 136, 135, 139, 135, 138, 131, 131, 125, 136]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.86243465Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90802024-04-20T01:11:29.364Z{"msg":"[0]R generation[0..12]: [76, 83, 83, 82, 81, 83, 81, 82, 78, 78, 74, 82]","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO
90812024-04-20T01:11:29.364Z:"2024-04-20T01:11:27.862490919Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90822024-04-20T01:11:29.364Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.86253392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90832024-04-20T01:11:29.364Z{"msg":"[1]R flush_numbers[0..12]: [127, 139, 139, 136, 135, 139, 135, 138, 131, 131, 125, 136]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.86256935Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90842024-04-20T01:11:29.364Z{"msg":"[1]R generation[0..12]: [76, 83, 83, 82, 81, 83, 81, 82, 78, 78, 74, 82]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.86259567Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90852024-04-20T01:11:29.364Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.862629Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90862024-04-20T01:11:29.365Z{"msg":"[2]R flush_numbers[0..12]: [127, 139, 139, 136, 135, 139, 135, 138, 131, 131, 125, 136]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.86266925Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90872024-04-20T01:11:29.365Z{"msg":"[2]R generation[0..12]: [76, 83, 83, 82, 81, 83, 81, 82, 78, 78, 74, 82]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.86270335Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90882024-04-20T01:11:29.365ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
90892024-04-20T01:11:29.365Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.86274246Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90902024-04-20T01:11:29.365Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.86276453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90912024-04-20T01:11:29.365Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.86280071Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90922024-04-20T01:11:29.365Z{"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.86282525Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90932024-04-20T01:11:29.365Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.86284744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"":"downstairs","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90942024-04-20T01:11:29.365Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.86287758Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90952024-04-20T01:11:29.365Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.8628991Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"0","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90962024-04-20T01:11:29.365Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.862926971Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"1","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90972024-04-20T01:11:29.365Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.862950451Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"client":"2","":"downstairs","session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90982024-04-20T01:11:29.365Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.862978051Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
90992024-04-20T01:11:29.365Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.863012371Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
91002024-04-20T01:11:29.365Z{"msg":"77178183-146d-44fc-82ac-3017c16fbe90 is now active with session: e6f7c95f-cc77-495a-b384-ec8fc906413e","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.863037691Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
91012024-04-20T01:11:29.365Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.863050541Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507,"session_id":"e6f7c95f-cc77-495a-b384-ec8fc906413e"}
91022024-04-20T01:11:29.365Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:27.863062761Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507}
91032024-04-20T01:11:29.369ZLoading write count information from file "/tmp/test_repair_verify.data"
91042024-04-20T01:11:29.370ZRun Repair workload
91052024-04-20T01:11:29.370Z01/30 Write block 460 len 32768 data: 10 13 10 10 7 9 11 10
91062024-04-20T01:11:29.370Z02/30 Flush
91072024-04-20T01:11:29.370Z03/30 Write block 594 len 12288 data: 9 8 8
91082024-04-20T01:11:29.370Z04/30 Read block 575 len 40960
91092024-04-20T01:11:29.370Z05/30 Write block 271 len 12288 data: 7 8 9
91102024-04-20T01:11:29.370Z06/30 Write block 342 len 20480 data: 6 7 6 7 5
91112024-04-20T01:11:29.370Z07/30 Read block 218 len 24576
91122024-04-20T01:11:29.370Z08/30 Read block 161 len 8192
91132024-04-20T01:11:29.370Z09/30 Flush
91142024-04-20T01:11:29.374Z10/30 Write block 124 len 40960 data: 10 11 10 9 10 9 10 10 11 9
91152024-04-20T01:11:29.374Z11/30 Read block 504 len 16384
91162024-04-20T01:11:29.374Z12/30 Read block 402 len 40960
91172024-04-20T01:11:29.374Z13/30 Write block 429 len 16384 data: 8 8 7 6
91182024-04-20T01:11:29.377Z14/30 Write block 231 len 28672 data: 10 11 13 14 12 12 13
91192024-04-20T01:11:29.377Z15/30 Write block 554 len 8192 data: 6 6
91202024-04-20T01:11:29.377Z16/30 Read block 217 len 36864
91212024-04-20T01:11:29.377Z17/30 Write block 365 len 28672 data: 6 8 9 7 8 10 13
91222024-04-20T01:11:29.381Z18/30 Write block 436 len 8192 data: 9 8
91232024-04-20T01:11:29.381Z19/30 Read block 214 len 8192
91242024-04-20T01:11:29.381Z20/30 Read block 110 len 20480
91252024-04-20T01:11:29.381Z21/30 Read block 47 len 16384
91262024-04-20T01:11:29.382Z22/30 Read block 258 len 16384
91272024-04-20T01:11:29.382Z23/30 Read block 558 len 20480
91282024-04-20T01:11:29.385Z24/30 Write block 353 len 40960 data: 5 6 7 9 8 8 9 7 8 9
91292024-04-20T01:11:29.385Z25/30 Read block 416 len 4096
91302024-04-20T01:11:29.385Z26/30 Flush
91312024-04-20T01:11:29.392Z27/30 Write block 173 len 40960 data: 6 5 5 6 7 8 10 10 11 8
91322024-04-20T01:11:29.393Z28/30 Read block 190 len 4096
91332024-04-20T01:11:29.397Z29/30 Write block 409 len 4096 data: 7
91342024-04-20T01:11:29.397Z30/30 Write block 286 len 28672 data: 4 4 4 5 4 3 4
91352024-04-20T01:11:29.397Z----------------------------------------------------------------
91362024-04-20T01:11:29.397Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:28
91372024-04-20T01:11:29.397ZStates: Active Active Active
91382024-04-20T01:11:29.397ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91392024-04-20T01:11:29.397Z 3 Acked 1002 Write 3 Done Done Done false
91402024-04-20T01:11:29.397Z 4 Acked 1003 Read 10 Done Done Done false
91412024-04-20T01:11:29.397Z 5 Acked 1004 Write 3 Done Done Done false
91422024-04-20T01:11:29.397Z 6 Acked 1005 Write 5 Done Done Done false
91432024-04-20T01:11:29.397Z 7 Acked 1006 Read 6 Done Done Done false
91442024-04-20T01:11:29.397Z 8 Acked 1007 Read 2 Sent Done Done false
91452024-04-20T01:11:29.397Z 9 Acked 1008 Flush 0 Sent Done Done false
91462024-04-20T01:11:29.397Z 10 Acked 1009 Write 10 Sent Done Done false
91472024-04-20T01:11:29.397Z 11 Acked 1010 Read 4 Sent Done Done false
91482024-04-20T01:11:29.398Z 12 Acked 1011 Read 10 Sent Done Done false
91492024-04-20T01:11:29.398Z 13 Acked 1012 Write 4 Sent Done Done false
91502024-04-20T01:11:29.398Z 14 Acked 1013 Write 7 Sent Done Done false
91512024-04-20T01:11:29.398Z 15 Acked 1014 Write 2 Sent Done Done false
91522024-04-20T01:11:29.398Z 16 Acked 1015 Read 9 Sent Done Done false
91532024-04-20T01:11:29.398Z 17 Acked 1016 Write 7 Sent Done Done false
91542024-04-20T01:11:29.398Z 18 Acked 1017 Write 2 Sent Done Done false
91552024-04-20T01:11:29.398Z 19 Acked 1018 Read 2 Sent Done Done false
91562024-04-20T01:11:29.398Z 20 Acked 1019 Read 5 Sent Done Done false
91572024-04-20T01:11:29.398Z 21 Acked 1020 Read 4 Sent Done Done false
91582024-04-20T01:11:29.398Z 22 Acked 1021 Read 4 Sent Done Done false
91592024-04-20T01:11:29.398Z 23 Acked 1022 Read 5 Sent Done Done false
91602024-04-20T01:11:29.398Z 24 Acked 1023 Write 10 Sent Done Done false
91612024-04-20T01:11:29.398Z 25 Acked 1024 Read 1 Sent Done Done false
91622024-04-20T01:11:29.398Z 26 Acked 1025 Flush 0 Sent Done Done false
91632024-04-20T01:11:29.398Z 27 Acked 1026 Write 10 Sent Done Done false
91642024-04-20T01:11:29.398Z 28 Acked 1027 Read 1 Sent Done Done false
91652024-04-20T01:11:29.398Z 29 Acked 1028 Write 1 Sent Done Done false
91662024-04-20T01:11:29.398Z 30 Acked 1029 Write 7 Sent Sent Sent false
91672024-04-20T01:11:29.398Z STATES DS:0 DS:1 DS:2 TOTAL
91682024-04-20T01:11:29.398Z New 0 0 0 0
91692024-04-20T01:11:29.398Z Sent 23 1 1 25
91702024-04-20T01:11:29.398Z Done 5 27 27 59
91712024-04-20T01:11:29.398Z Skipped 0 0 0 0
91722024-04-20T01:11:29.399Z Error 0 0 0 0
91732024-04-20T01:11:29.399ZLast Flush: 1001 1025 1025
91742024-04-20T01:11:29.399ZDownstairs last five completed: 1001 1000
91752024-04-20T01:11:29.399ZUpstairs last five completed: 30 29 28 27 26
91762024-04-20T01:11:29.399Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:11:27.895500583Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1507}
91772024-04-20T01:11:29.399ZWrote out file "/tmp/test_repair_verify.data"
91782024-04-20T01:11:29.399Z
91792024-04-20T01:11:29.402ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91802024-04-20T01:11:29.415ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91812024-04-20T01:11:29.415Z 0 000-019 76 76 76 127 127 127 F F F
91822024-04-20T01:11:29.415Z 1 020-039 83 83 83 139 139 139 F F F
91832024-04-20T01:11:29.415Z 2 040-059 83 83 83 139 139 139 F F F
91842024-04-20T01:11:29.415Z 3 060-079 82 82 82 136 136 136 F F F
91852024-04-20T01:11:29.415Z 4 080-099 81 81 81 135 135 135 F F F
91862024-04-20T01:11:29.415Z 5 100-119 83 83 83 139 139 139 F F F
91872024-04-20T01:11:29.419Z 6 120-139 81 84 84 135 142 142 F F F <---
91882024-04-20T01:11:29.419Z 7 140-159 82 82 82 138 138 138 F F F
91892024-04-20T01:11:29.419Z 8 160-179 78 78 78 131 131 131 F T T <---
91902024-04-20T01:11:29.419Z 9 180-199 78 78 78 131 131 131 F T T <---
91912024-04-20T01:11:29.419Z 10 200-219 74 74 74 125 125 125 F F F
91922024-04-20T01:11:29.419Z 11 220-239 82 84 84 136 142 142 F F F <---
91932024-04-20T01:11:29.419Z 12 240-259 78 78 78 131 131 131 F F F
91942024-04-20T01:11:29.419Z 13 260-279 78 84 84 131 141 141 T F F <---
91952024-04-20T01:11:29.419Z 14 280-299 78 78 78 131 131 131 F T T <---
91962024-04-20T01:11:29.419Z 15 300-319 81 81 81 135 135 135 F F F
91972024-04-20T01:11:29.419Z 16 320-339 83 83 83 139 139 139 F F F
91982024-04-20T01:11:29.419Z 17 340-359 83 84 84 139 142 142 T F F <---
91992024-04-20T01:11:29.420Z 18 360-379 79 84 84 132 142 142 F F F <---
92002024-04-20T01:11:29.420Z 19 380-399 79 79 79 132 132 132 F F F
92012024-04-20T01:11:29.420Z 20 400-419 83 83 83 139 139 139 F T T <---
92022024-04-20T01:11:29.420Z 21 420-439 78 84 84 130 142 142 F F F <---
92032024-04-20T01:11:29.420Z 22 440-459 82 82 82 136 136 136 F F F
92042024-04-20T01:11:29.420Z 23 460-479 84 84 84 140 140 140 F F F
92052024-04-20T01:11:29.420Z 24 480-499 76 76 76 128 128 128 F F F
92062024-04-20T01:11:29.420Z 25 500-519 81 81 81 135 135 135 F F F
92072024-04-20T01:11:29.420Z 26 520-539 81 81 81 135 135 135 F F F
92082024-04-20T01:11:29.420Z 27 540-559 72 84 84 123 142 142 F F F <---
92092024-04-20T01:11:29.420Z 28 560-579 80 80 80 134 134 134 F F F
92102024-04-20T01:11:29.420Z 29 580-599 80 84 84 134 141 141 T F F <---
92112024-04-20T01:11:29.420ZMax gen: 84, Max flush: 142
92122024-04-20T01:11:29.420ZError: Difference in extent metadata found!
92132024-04-20T01:11:29.420ZA Difference in extent metadata is expected here
92142024-04-20T01:11:29.420ZOn loop 42
92152024-04-20T01:11:29.420Z
92162024-04-20T01:11:29.424ZVerifying data now
92172024-04-20T01:11:43.967ZNo change after verify
92182024-04-20T01:11:43.967ZLoop: 42 Downstairs dump after verify (and repair):
92192024-04-20T01:11:43.981ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92202024-04-20T01:11:43.981Z 0 000-019 76 76 76 127 127 127 F F F
92212024-04-20T01:11:43.981Z 1 020-039 83 83 83 139 139 139 F F F
92222024-04-20T01:11:43.981Z 2 040-059 83 83 83 139 139 139 F F F
92232024-04-20T01:11:43.981Z 3 060-079 82 82 82 136 136 136 F F F
92242024-04-20T01:11:43.981Z 4 080-099 81 81 81 135 135 135 F F F
92252024-04-20T01:11:43.981Z 5 100-119 83 83 83 139 139 139 F F F
92262024-04-20T01:11:43.981Z 6 120-139 84 84 84 142 142 142 F F F
92272024-04-20T01:11:43.981Z 7 140-159 82 82 82 138 138 138 F F F
92282024-04-20T01:11:43.981Z 8 160-179 85 85 85 143 143 143 F F F
92292024-04-20T01:11:43.985Z 9 180-199 85 85 85 143 143 143 F F F
92302024-04-20T01:11:43.985Z 10 200-219 74 74 74 125 125 125 F F F
92312024-04-20T01:11:43.985Z 11 220-239 84 84 84 142 142 142 F F F
92322024-04-20T01:11:43.985Z 12 240-259 78 78 78 131 131 131 F F F
92332024-04-20T01:11:43.985Z 13 260-279 84 84 84 141 141 141 F F F
92342024-04-20T01:11:43.985Z 14 280-299 85 85 85 143 143 143 F F F
92352024-04-20T01:11:43.985Z 15 300-319 81 81 81 135 135 135 F F F
92362024-04-20T01:11:43.985Z 16 320-339 83 83 83 139 139 139 F F F
92372024-04-20T01:11:43.986Z 17 340-359 84 84 84 142 142 142 F F F
92382024-04-20T01:11:43.986Z 18 360-379 84 84 84 142 142 142 F F F
92392024-04-20T01:11:43.986Z 19 380-399 79 79 79 132 132 132 F F F
92402024-04-20T01:11:43.986Z 20 400-419 85 85 85 143 143 143 F F F
92412024-04-20T01:11:43.986Z 21 420-439 84 84 84 142 142 142 F F F
92422024-04-20T01:11:43.986Z 22 440-459 82 82 82 136 136 136 F F F
92432024-04-20T01:11:43.986Z 23 460-479 84 84 84 140 140 140 F F F
92442024-04-20T01:11:43.986Z 24 480-499 76 76 76 128 128 128 F F F
92452024-04-20T01:11:43.986Z 25 500-519 81 81 81 135 135 135 F F F
92462024-04-20T01:11:43.986Z 26 520-539 81 81 81 135 135 135 F F F
92472024-04-20T01:11:43.986Z 27 540-559 84 84 84 142 142 142 F F F
92482024-04-20T01:11:43.986Z 28 560-579 80 80 80 134 134 134 F F F
92492024-04-20T01:11:43.986Z 29 580-599 84 84 84 141 141 141 F F F
92502024-04-20T01:11:43.986ZMax gen: 85, Max flush: 143
92512024-04-20T01:11:43.986Z
92522024-04-20T01:11:43.986ZBegin loop 43
92532024-04-20T01:11:43.986ZDownstairs to restart: 1
92542024-04-20T01:11:44.004Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.507387229Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515}
92552024-04-20T01:11:44.004Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.50762071Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515}
92562024-04-20T01:11:44.012Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.50767626Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515}
92572024-04-20T01:11:44.012Z{"msg":"Crucible e846a41b-0bfe-4e1d-b4ca-9cd5a20e5246 has session id: decbf83a-c7e1-4b96-ae60-1e30fa86cdbc","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.50770169Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92582024-04-20T01:11:44.012Z{"msg":"Upstairs opts: Upstairs UUID: e846a41b-0bfe-4e1d-b4ca-9cd5a20e5246, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
92592024-04-20T01:11:44.012Z,"time":"2024-04-20T01:11:42.50775034Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92602024-04-20T01:11:44.012Z{"msg":"Crucible stats registered with UUID: e846a41b-0bfe-4e1d-b4ca-9cd5a20e5246","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.50784569Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92612024-04-20T01:11:44.012Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.50786693Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515}
92622024-04-20T01:11:44.012Z{"msg":"e846a41b-0bfe-4e1d-b4ca-9cd5a20e5246 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.50789521Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92632024-04-20T01:11:44.012Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.50791234Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"0","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92642024-04-20T01:11:44.012Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.50794736Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"0","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92652024-04-20T01:11:44.012Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.507971Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"1","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92662024-04-20T01:11:44.013Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.508013981Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"1","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92672024-04-20T01:11:44.013Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.508030451Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"2","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92682024-04-20T01:11:44.013Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.508044201Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"2","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92692024-04-20T01:11:44.013Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.508060061Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92702024-04-20T01:11:44.013Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.508091851Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92712024-04-20T01:11:44.013Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.508109891Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92722024-04-20T01:11:44.013Z{"msg":"ds_connection connected from Ok(127.0.0.1:49544)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.508272581Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92732024-04-20T01:11:44.013Z{"msg":"ds_connection connected from Ok(127.0.0.1:46687)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.508305031Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92742024-04-20T01:11:44.013Z{"msg":"ds_connection connected from Ok(127.0.0.1:54996)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.508331241Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92752024-04-20T01:11:44.013Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.508984543Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"2","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92762024-04-20T01:11:44.013Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.509038483Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"2","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92772024-04-20T01:11:44.013Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.509081383Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"0","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92782024-04-20T01:11:44.013Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.509104963Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"0","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92792024-04-20T01:11:44.013Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.509471544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"1","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92802024-04-20T01:11:44.013Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.509514904Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"1","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92812024-04-20T01:11:44.013Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.509537904Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"0","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92822024-04-20T01:11:44.013Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.509657994Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"2","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92832024-04-20T01:11:44.013Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510029345Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"0","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92842024-04-20T01:11:44.013Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510081715Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"0","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92852024-04-20T01:11:44.013Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510126425Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92862024-04-20T01:11:44.013Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510167396Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"2","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92872024-04-20T01:11:44.013Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510207056Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"2","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92882024-04-20T01:11:44.013Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510244916Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92892024-04-20T01:11:44.014Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510267046Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"1","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92902024-04-20T01:11:44.014Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510600196Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"1","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92912024-04-20T01:11:44.014Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510637646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"1","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92922024-04-20T01:11:44.014Z{"msg":"[0]R flush_numbers[0..12]: [127, 139, 139, 136, 135, 139, 142, 138, 143, 143, 125, 142]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510653877Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92932024-04-20T01:11:44.014Z{"msg":"[0]R generation[0..12]: [76, 83, 83, 82, 81, 83, 84, 82, 85, 85, 74, 84]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510675697Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92942024-04-20T01:11:44.014Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510690267Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92952024-04-20T01:11:44.014Z{"msg":"[1]R flush_numbers[0..12]: [127, 139, 139, 136, 135, 139, 142, 138, 143, 143, 125, 142]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510704517Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92962024-04-20T01:11:44.014Z{"msg":"[1]R generation[0..12]: [76, 83, 83, 82, 81, 83, 84, 82, 85, 85, 74, 84]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510725487Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92972024-04-20T01:11:44.014Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510739667Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
92982024-04-20T01:11:44.014Z{"msg":"[2]R flush_numbers[0..12]: [127, 139, 139, 136, 135, 139, 142, 138, 143, 143, 125, 142]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
92992024-04-20T01:11:44.014Z,"time":"2024-04-20T01:11:42.510760487Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93002024-04-20T01:11:44.014Z{"msg":"[2]R generation[0..12]: [76, 83, 83, 82, 81, 83, 84, 82, 85, 85, 74, 84]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510776207Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93012024-04-20T01:11:44.014Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510797137Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93022024-04-20T01:11:44.014Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510810327Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93032024-04-20T01:11:44.014Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510830447Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93042024-04-20T01:11:44.014Z{"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510843767Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93052024-04-20T01:11:44.014Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510863717Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"":"downstairs","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93062024-04-20T01:11:44.014Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510876927Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93072024-04-20T01:11:44.014Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510896477Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"0","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93082024-04-20T01:11:44.014Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510916717Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"1","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93092024-04-20T01:11:44.014Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510930597Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"client":"2","":"downstairs","session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93102024-04-20T01:11:44.014Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510944507Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93112024-04-20T01:11:44.014Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510963297Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93122024-04-20T01:11:44.015Z{"msg":"e846a41b-0bfe-4e1d-b4ca-9cd5a20e5246 is now active with session: decbf83a-c7e1-4b96-ae60-1e30fa86cdbc","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510976147Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93132024-04-20T01:11:44.015Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.510987937Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515,"session_id":"decbf83a-c7e1-4b96-ae60-1e30fa86cdbc"}
93142024-04-20T01:11:44.015Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:42.511007097Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515}
93152024-04-20T01:11:44.015ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
93162024-04-20T01:11:44.015ZLoading write count information from file "/tmp/test_repair_verify.data"
93172024-04-20T01:11:44.015ZRun Repair workload
93182024-04-20T01:11:44.015Z01/30 Flush
93192024-04-20T01:11:44.019Z02/30 Write block 291 len 40960 data: 4 5 5 7 4 4 5 3 3 4
93202024-04-20T01:11:44.019Z03/30 Write block 450 len 12288 data: 7 7 8
93212024-04-20T01:11:44.019Z04/30 Write block 477 len 36864 data: 10 8 5 5 8 8 8 7 8
93222024-04-20T01:11:44.020Z05/30 Read block 137 len 32768
93232024-04-20T01:11:44.020Z06/30 Write block 4 len 28672 data: 5 4 4 4 4 4 5
93242024-04-20T01:11:44.020Z07/30 Read block 83 len 20480
93252024-04-20T01:11:44.020Z08/30 Write block 115 len 16384 data: 8 7 6 6
93262024-04-20T01:11:44.020Z09/30 Write block 178 len 12288 data: 9 11 11
93272024-04-20T01:11:44.020Z10/30 Read block 378 len 28672
93282024-04-20T01:11:44.020Z11/30 Read block 327 len 40960
93292024-04-20T01:11:44.020Z12/30 Write block 397 len 8192 data: 5 6
93302024-04-20T01:11:44.024Z13/30 Write block 55 len 16384 data: 9 8 6 8
93312024-04-20T01:11:44.024Z14/30 Write block 416 len 24576 data: 11 9 10 10 8 8
93322024-04-20T01:11:44.024Z15/30 Write block 449 len 12288 data: 8 8 8
93332024-04-20T01:11:44.024Z16/30 Read block 373 len 20480
93342024-04-20T01:11:44.024Z17/30 Flush
93352024-04-20T01:11:44.028Z18/30 Write block 345 len 12288 data: 8 6 7
93362024-04-20T01:11:44.032Z19/30 Write block 30 len 40960 data: 7 6 6 6 9 9 7 8 9 9
93372024-04-20T01:11:44.032Z20/30 Read block 528 len 40960
93382024-04-20T01:11:44.032Z21/30 Read block 5 len 40960
93392024-04-20T01:11:44.032Z22/30 Read block 178 len 40960
93402024-04-20T01:11:44.035Z23/30 Write block 154 len 8192 data: 9 11
93412024-04-20T01:11:44.035Z24/30 Write block 332 len 32768 data: 10 8 8 6 4 5 7 7
93422024-04-20T01:11:44.035Z25/30 Write block 341 len 24576 data: 8 7 8 7 9 7
93432024-04-20T01:11:44.035Z26/30 Read block 524 len 32768
93442024-04-20T01:11:44.041Z27/30 Read block 422 len 12288
93452024-04-20T01:11:44.041Z28/30 Write block 127 len 16384 data: 10 11 10 11
93462024-04-20T01:11:44.041Z29/30 Read block 70 len 8192
93472024-04-20T01:11:44.041Z30/30 Read block 26 len 24576
93482024-04-20T01:11:44.041Z----------------------------------------------------------------
93492024-04-20T01:11:44.041Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:29
93502024-04-20T01:11:44.041ZStates: Active Active Active
93512024-04-20T01:11:44.041ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93522024-04-20T01:11:44.041Z 2 Acked 1001 Write 10 Done Sent Done false
93532024-04-20T01:11:44.041Z 3 Acked 1002 Write 3 Done Sent Done false
93542024-04-20T01:11:44.041Z 4 Acked 1003 Write 9 Done Sent Done false
93552024-04-20T01:11:44.041Z 5 Acked 1004 Read 8 Done Sent Done false
93562024-04-20T01:11:44.041Z 6 Acked 1005 Write 7 Done Sent Done false
93572024-04-20T01:11:44.041Z 7 Acked 1006 Read 5 Done Sent Done false
93582024-04-20T01:11:44.041Z 8 Acked 1007 Write 4 Done Sent Done false
93592024-04-20T01:11:44.041Z 9 Acked 1008 Write 3 Done Sent Done false
93602024-04-20T01:11:44.041Z 10 Acked 1009 Read 7 Done Sent Done false
93612024-04-20T01:11:44.041Z 11 Acked 1010 Read 10 Done Sent Done false
93622024-04-20T01:11:44.042Z 12 Acked 1011 Write 2 Done Sent Done false
93632024-04-20T01:11:44.042Z 13 Acked 1012 Write 4 Done Sent Done false
93642024-04-20T01:11:44.042Z 14 Acked 1013 Write 6 Done Sent Done false
93652024-04-20T01:11:44.042Z 15 Acked 1014 Write 3 Done Sent Done false
93662024-04-20T01:11:44.042Z 16 Acked 1015 Read 5 Done Sent Done false
93672024-04-20T01:11:44.042Z 17 Acked 1016 Flush 0 Done Sent Done false
93682024-04-20T01:11:44.042Z 18 Acked 1017 Write 3 Done Sent Done false
93692024-04-20T01:11:44.042Z 19 Acked 1018 Write 10 Done Sent Done false
93702024-04-20T01:11:44.042Z 20 Acked 1019 Read 10 Done Sent Done false
93712024-04-20T01:11:44.042Z 21 Acked 1020 Read 10 Done Sent Done false
93722024-04-20T01:11:44.042Z 22 Acked 1021 Read 10 Done Sent Done false
93732024-04-20T01:11:44.042Z 23 Acked 1022 Write 2 Done Sent Done false
93742024-04-20T01:11:44.042Z 24 Acked 1023 Write 8 Done Sent Done false
93752024-04-20T01:11:44.042Z 25 Acked 1024 Write 6 Done Sent Done false
93762024-04-20T01:11:44.042Z 26 Acked 1025 Read 8 Done Sent Done false
93772024-04-20T01:11:44.042Z 27 Acked 1026 Read 3 Done Sent Done false
93782024-04-20T01:11:44.042Z 28 Acked 1027 Write 4 Done Sent Done false
93792024-04-20T01:11:44.042Z 29 Acked 1028 Read 2 Done Sent Done false
93802024-04-20T01:11:44.042Z 30 Acked 1029 Read 6 Done Sent Done false
93812024-04-20T01:11:44.042Z STATES DS:0 DS:1 DS:2 TOTAL
93822024-04-20T01:11:44.042Z New 0 0 0 0
93832024-04-20T01:11:44.042Z Sent 0 29 0 29
93842024-04-20T01:11:44.042Z Done 29 0 29 58
93852024-04-20T01:11:44.043Z Skipped 0 0 0 0
93862024-04-20T01:11:44.043Z Error 0 0 0 0
93872024-04-20T01:11:44.043ZLast Flush: 1016 1000 1016
93882024-04-20T01:11:44.043ZDownstairs last five completed: 1000
93892024-04-20T01:11:44.043ZUpstairs last five completed: 30 29 28 27 26
93902024-04-20T01:11:44.043Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:11:42.543015138Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1515}
93912024-04-20T01:11:44.043ZWrote out file "/tmp/test_repair_verify.data"
93922024-04-20T01:11:44.047Z
93932024-04-20T01:11:44.047ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
93942024-04-20T01:11:44.061ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93952024-04-20T01:11:44.061Z 0 000-019 86 76 86 145 127 145 F F F <---
93962024-04-20T01:11:44.061Z 1 020-039 83 83 83 139 139 139 T F T <---
93972024-04-20T01:11:44.061Z 2 040-059 86 83 86 145 139 145 F F F <---
93982024-04-20T01:11:44.062Z 3 060-079 82 82 82 136 136 136 F F F
93992024-04-20T01:11:44.062Z 4 080-099 81 81 81 135 135 135 F F F
94002024-04-20T01:11:44.062Z 5 100-119 86 83 86 145 139 145 F F F <---
94012024-04-20T01:11:44.062Z 6 120-139 84 84 84 142 142 142 T F T <---
94022024-04-20T01:11:44.062Z 7 140-159 82 82 82 138 138 138 T F T <---
94032024-04-20T01:11:44.062Z 8 160-179 86 85 86 145 143 145 F F F <---
94042024-04-20T01:11:44.062Z 9 180-199 86 85 86 145 143 145 F F F <---
94052024-04-20T01:11:44.062Z 10 200-219 74 74 74 125 125 125 F F F
94062024-04-20T01:11:44.062Z 11 220-239 84 84 84 142 142 142 F F F
94072024-04-20T01:11:44.062Z 12 240-259 78 78 78 131 131 131 F F F
94082024-04-20T01:11:44.062Z 13 260-279 84 84 84 141 141 141 F F F
94092024-04-20T01:11:44.062Z 14 280-299 86 85 86 145 143 145 F F F <---
94102024-04-20T01:11:44.062Z 15 300-319 86 81 86 145 135 145 F F F <---
94112024-04-20T01:11:44.066Z 16 320-339 83 83 83 139 139 139 T F T <---
94122024-04-20T01:11:44.066Z 17 340-359 84 84 84 142 142 142 T F T <---
94132024-04-20T01:11:44.066Z 18 360-379 84 84 84 142 142 142 F F F
94142024-04-20T01:11:44.066Z 19 380-399 86 79 86 145 132 145 F F F <---
94152024-04-20T01:11:44.066Z 20 400-419 86 85 86 145 143 145 F F F <---
94162024-04-20T01:11:44.066Z 21 420-439 86 84 86 145 142 145 F F F <---
94172024-04-20T01:11:44.066Z 22 440-459 86 82 86 145 136 145 F F F <---
94182024-04-20T01:11:44.066Z 23 460-479 86 84 86 145 140 145 F F F <---
94192024-04-20T01:11:44.066Z 24 480-499 86 76 86 145 128 145 F F F <---
94202024-04-20T01:11:44.066Z 25 500-519 81 81 81 135 135 135 F F F
94212024-04-20T01:11:44.066Z 26 520-539 81 81 81 135 135 135 F F F
94222024-04-20T01:11:44.066Z 27 540-559 84 84 84 142 142 142 F F F
94232024-04-20T01:11:44.066Z 28 560-579 80 80 80 134 134 134 F F F
94242024-04-20T01:11:44.066Z 29 580-599 84 84 84 141 141 141 F F F
94252024-04-20T01:11:44.066ZMax gen: 86, Max flush: 145
94262024-04-20T01:11:44.066ZError: Difference in extent metadata found!
94272024-04-20T01:11:44.067ZA Difference in extent metadata is expected here
94282024-04-20T01:11:44.067ZOn loop 43
94292024-04-20T01:11:44.067Z
94302024-04-20T01:11:44.070ZVerifying data now
94312024-04-20T01:11:48.860ZNo change after verify
94322024-04-20T01:11:48.860ZLoop: 43 Downstairs dump after verify (and repair):
94332024-04-20T01:11:48.874ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94342024-04-20T01:11:48.874Z 0 000-019 86 86 86 145 145 145 F F F
94352024-04-20T01:11:48.874Z 1 020-039 87 87 87 146 146 146 F F F
94362024-04-20T01:11:48.874Z 2 040-059 86 86 86 145 145 145 F F F
94372024-04-20T01:11:48.874Z 3 060-079 82 82 82 136 136 136 F F F
94382024-04-20T01:11:48.874Z 4 080-099 81 81 81 135 135 135 F F F
94392024-04-20T01:11:48.879Z 5 100-119 86 86 86 145 145 145 F F F
94402024-04-20T01:11:48.879Z 6 120-139 87 87 87 146 146 146 F F F
94412024-04-20T01:11:48.879Z 7 140-159 87 87 87 146 146 146 F F F
94422024-04-20T01:11:48.879Z 8 160-179 86 86 86 145 145 145 F F F
94432024-04-20T01:11:48.879Z 9 180-199 86 86 86 145 145 145 F F F
94442024-04-20T01:11:48.879Z 10 200-219 74 74 74 125 125 125 F F F
94452024-04-20T01:11:48.879Z 11 220-239 84 84 84 142 142 142 F F F
94462024-04-20T01:11:48.879Z 12 240-259 78 78 78 131 131 131 F F F
94472024-04-20T01:11:48.879Z 13 260-279 84 84 84 141 141 141 F F F
94482024-04-20T01:11:48.879Z 14 280-299 86 86 86 145 145 145 F F F
94492024-04-20T01:11:48.879Z 15 300-319 86 86 86 145 145 145 F F F
94502024-04-20T01:11:48.879Z 16 320-339 87 87 87 146 146 146 F F F
94512024-04-20T01:11:48.880Z 17 340-359 87 87 87 146 146 146 F F F
94522024-04-20T01:11:48.880Z 18 360-379 84 84 84 142 142 142 F F F
94532024-04-20T01:11:48.880Z 19 380-399 86 86 86 145 145 145 F F F
94542024-04-20T01:11:48.880Z 20 400-419 86 86 86 145 145 145 F F F
94552024-04-20T01:11:48.880Z 21 420-439 86 86 86 145 145 145 F F F
94562024-04-20T01:11:48.880Z 22 440-459 86 86 86 145 145 145 F F F
94572024-04-20T01:11:48.880Z 23 460-479 86 86 86 145 145 145 F F F
94582024-04-20T01:11:48.880Z 24 480-499 86 86 86 145 145 145 F F F
94592024-04-20T01:11:48.880Z 25 500-519 81 81 81 135 135 135 F F F
94602024-04-20T01:11:48.880Z 26 520-539 81 81 81 135 135 135 F F F
94612024-04-20T01:11:48.880Z 27 540-559 84 84 84 142 142 142 F F F
94622024-04-20T01:11:48.880Z 28 560-579 80 80 80 134 134 134 F F F
94632024-04-20T01:11:48.881Z 29 580-599 84 84 84 141 141 141 F F F
94642024-04-20T01:11:48.881ZMax gen: 87, Max flush: 146
94652024-04-20T01:11:48.881Z
94662024-04-20T01:11:48.881ZBegin loop 44
94672024-04-20T01:11:48.881ZDownstairs to restart: 1
94682024-04-20T01:11:48.897Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.398897258Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523}
94692024-04-20T01:11:48.897Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399101938Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523}
94702024-04-20T01:11:48.897Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399122628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523}
94712024-04-20T01:11:48.906Z{"msg":"Crucible 465ae41e-947b-4b0b-a882-7fffcf25a6fd has session id: 6d9b9495-4e27-47df-89e6-e684cb1ac2e1","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399158388Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94722024-04-20T01:11:48.906Z{Crucible runtime is spawned
94732024-04-20T01:11:48.906Z"msg":"Upstairs opts: Upstairs UUID: 465ae41e-947b-4b0b-a882-7fffcf25a6fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399246818Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94742024-04-20T01:11:48.906Z{"msg":"Crucible stats registered with UUID: 465ae41e-947b-4b0b-a882-7fffcf25a6fd","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399277658Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94752024-04-20T01:11:48.906Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399304848Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523}
94762024-04-20T01:11:48.906Z{"msg":"465ae41e-947b-4b0b-a882-7fffcf25a6fd active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399365588Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94772024-04-20T01:11:48.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399391019Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"0","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94782024-04-20T01:11:48.906Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399409919Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"0","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94792024-04-20T01:11:48.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399442769Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"1","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94802024-04-20T01:11:48.906Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399489409Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"1","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94812024-04-20T01:11:48.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399517309Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"2","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94822024-04-20T01:11:48.906Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399549499Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"2","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94832024-04-20T01:11:48.906Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399576169Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94842024-04-20T01:11:48.906Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399604049Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94852024-04-20T01:11:48.906Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.399631919Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94862024-04-20T01:11:48.907Z{"msg":"ds_connection connected from Ok(127.0.0.1:55007)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.39975096Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94872024-04-20T01:11:48.907Z{"msg":"ds_connection connected from Ok(127.0.0.1:49737)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.39980022Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94882024-04-20T01:11:48.907Z{"msg":"ds_connection connected from Ok(127.0.0.1:58967)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.39982874Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94892024-04-20T01:11:48.907Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.400498271Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"0","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94902024-04-20T01:11:48.907Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.400534851Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"0","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94912024-04-20T01:11:48.907Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.400562621Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"2","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94922024-04-20T01:11:48.907Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.400601121Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"2","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94932024-04-20T01:11:48.907Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.401051172Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"1","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94942024-04-20T01:11:48.907Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.401078112Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"1","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94952024-04-20T01:11:48.907Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.401098123Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"2","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94962024-04-20T01:11:48.907Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.401138552Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"0","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94972024-04-20T01:11:48.907Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.401589464Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"2","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94982024-04-20T01:11:48.907Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.401624804Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"2","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
94992024-04-20T01:11:48.907Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.401640964Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95002024-04-20T01:11:48.907Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.401703044Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"1","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95012024-04-20T01:11:48.907Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.401806064Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"0","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95022024-04-20T01:11:48.907Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.401832364Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"0","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95032024-04-20T01:11:48.907Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.401846454Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95042024-04-20T01:11:48.907Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402240455Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"1","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95052024-04-20T01:11:48.907Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402273345Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"1","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95062024-04-20T01:11:48.908Z{"msg":"[0]R flush_numbers[0..12]: [145, 146, 145, 136, 135, 145, 146, 146, 145, 145, 125, 142]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402288925Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95072024-04-20T01:11:48.908Z{"msg":"[0]R generation[0..12]: [86, 87, 86, 82, 81, 86, 87, 87, 86, 86, 74, 84]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402310055Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95082024-04-20T01:11:48.908Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402325635Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95092024-04-20T01:11:48.908Z{"msg":"[1]R flush_numbers[0..12]: [145, 146, 145, 136, 135, 145, 146, 146, 145, 145, 125, 142]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402346865Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95102024-04-20T01:11:48.908Z{"msg":"[1]R generation[0..12]: [86, 87, 86, 82, 81, 86, 87, 87, 86, 86, 74, 84]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402361265Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95112024-04-20T01:11:48.908Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402376135Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95122024-04-20T01:11:48.908Z{"msg":"[2]R flush_numbers[0..12]: [145, 146, 145, 136, 135, 145, 146, 146, 145, 145, 125, 142]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402396895Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95132024-04-20T01:11:48.908ZWait for a query_work_queue command to finish before sending IO
95142024-04-20T01:11:48.908Z{"msg":"[2]R generation[0..12]: [86, 87, 86, 82, 81, 86, 87, 87, 86, 86, 74, 84]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402449235Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95152024-04-20T01:11:48.908Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402486675Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95162024-04-20T01:11:48.908Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402521956Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95172024-04-20T01:11:48.908Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402565146Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95182024-04-20T01:11:48.908Z{"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402596146Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95192024-04-20T01:11:48.908Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402610846Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"":"downstairs","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95202024-04-20T01:11:48.908Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402642666Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95212024-04-20T01:11:48.908Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402656566Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"0","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95222024-04-20T01:11:48.908Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402686596Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"1","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95232024-04-20T01:11:48.908Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402702386Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"client":"2","":"downstairs","session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95242024-04-20T01:11:48.908Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402733196Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95252024-04-20T01:11:48.908Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402747286Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95262024-04-20T01:11:48.909Z{"msg":"465ae41e-947b-4b0b-a882-7fffcf25a6fd is now active with session: 6d9b9495-4e27-47df-89e6-e684cb1ac2e1","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402761436Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
95272024-04-20T01:11:48.909Z6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95282024-04-20T01:11:48.909Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402791816Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523,"session_id":"6d9b9495-4e27-47df-89e6-e684cb1ac2e1"}
95292024-04-20T01:11:48.909Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:11:47.402805946Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523}
95302024-04-20T01:11:48.909ZLoading write count information from file "/tmp/test_repair_verify.data"
95312024-04-20T01:11:48.909ZRun Repair workload
95322024-04-20T01:11:48.909Z01/30 Write block 217 len 32768 data: 10 10 8 10 8 10 10 10
95332024-04-20T01:11:48.913Z02/30 Write block 433 len 20480 data: 6 8 9 10 9
95342024-04-20T01:11:48.913Z03/30 Flush
95352024-04-20T01:11:48.913Z04/30 Write block 377 len 32768 data: 8 8 9 9 7 6 5 3
95362024-04-20T01:11:48.913Z05/30 Write block 516 len 8192 data: 11 12
95372024-04-20T01:11:48.913Z06/30 Write block 519 len 4096 data: 10
95382024-04-20T01:11:48.913Z07/30 Write block 367 len 8192 data: 10 8
95392024-04-20T01:11:48.914Z08/30 Read block 491 len 20480
95402024-04-20T01:11:48.914Z09/30 Write block 274 len 36864 data: 8 7 4 6 6 5 7 6 4
95412024-04-20T01:11:48.914Z10/30 Read block 195 len 8192
95422024-04-20T01:11:48.914Z11/30 Flush
95432024-04-20T01:11:48.918Z12/30 Write block 178 len 20480 data: 10 12 12 12 9
95442024-04-20T01:11:48.918Z13/30 Write block 254 len 12288 data: 7 6 6
95452024-04-20T01:11:48.918Z14/30 Read block 351 len 28672
95462024-04-20T01:11:48.918Z15/30 Write block 457 len 8192 data: 14 12
95472024-04-20T01:11:48.918Z16/30 Write block 118 len 4096 data: 7
95482024-04-20T01:11:48.919Z17/30 Read block 179 len 40960
95492024-04-20T01:11:48.923Z18/30 Read block 231 len 24576
95502024-04-20T01:11:48.923Z19/30 Flush
95512024-04-20T01:11:48.923Z20/30 Write block 518 len 36864 data: 9 11 11 11 10 11 10 9 9
95522024-04-20T01:11:48.927Z21/30 Read block 344 len 28672
95532024-04-20T01:11:48.927Z22/30 Write block 533 len 24576 data: 6 7 6 5 5 6
95542024-04-20T01:11:48.927Z23/30 Read block 54 len 12288
95552024-04-20T01:11:48.927Z24/30 Write block 523 len 12288 data: 12 11 10
95562024-04-20T01:11:48.931Z25/30 Write block 53 len 36864 data: 13 10 10 9 7 9 9 9 8
95572024-04-20T01:11:48.931Z26/30 Read block 534 len 12288
95582024-04-20T01:11:48.931Z27/30 Write block 352 len 8192 data: 6 6
95592024-04-20T01:11:48.931Z28/30 Read block 350 len 12288
95602024-04-20T01:11:48.931Z29/30 Write block 312 len 4096 data: 2
95612024-04-20T01:11:48.936Z30/30 Write block 434 len 36864 data: 9 10 11 10 6 7 7 8 8
95622024-04-20T01:11:48.936Z----------------------------------------------------------------
95632024-04-20T01:11:48.936Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
95642024-04-20T01:11:48.936ZStates: Active Active Active
95652024-04-20T01:11:48.936ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95662024-04-20T01:11:48.936Z 1 Acked 1000 Write 8 Done Done Done false
95672024-04-20T01:11:48.937Z 2 Acked 1001 Write 5 Done Sent Done false
95682024-04-20T01:11:48.937Z 3 Acked 1002 Flush 0 Done Sent Done false
95692024-04-20T01:11:48.937Z 4 Acked 1003 Write 8 Done Sent Done false
95702024-04-20T01:11:48.937Z 5 Acked 1004 Write 2 Done Sent Done false
95712024-04-20T01:11:48.937Z 6 Acked 1005 Write 1 Done Sent Done false
95722024-04-20T01:11:48.937Z 7 Acked 1006 Write 2 Done Sent Done false
95732024-04-20T01:11:48.937Z 8 Acked 1007 Read 5 Done Sent Done false
95742024-04-20T01:11:48.937Z 9 Acked 1008 Write 9 Done Sent Done false
95752024-04-20T01:11:48.937Z 10 Acked 1009 Read 2 Done Sent Done false
95762024-04-20T01:11:48.937Z 11 Acked 1010 Flush 0 Done Sent Done false
95772024-04-20T01:11:48.937Z 12 Acked 1011 Write 5 Done Sent Done false
95782024-04-20T01:11:48.937Z 13 Acked 1012 Write 3 Done Sent Done false
95792024-04-20T01:11:48.937Z 14 Acked 1013 Read 7 Done Sent Done false
95802024-04-20T01:11:48.937Z 15 Acked 1014 Write 2 Done Sent Done false
95812024-04-20T01:11:48.937Z 16 Acked 1015 Write 1 Done Sent Done false
95822024-04-20T01:11:48.937Z 17 Acked 1016 Read 10 Done Sent Done false
95832024-04-20T01:11:48.937Z 18 Acked 1017 Read 6 Done Sent Done false
95842024-04-20T01:11:48.937Z 19 Acked 1018 Flush 0 Done Sent Done false
95852024-04-20T01:11:48.937Z 20 Acked 1019 Write 9 Done Sent Done false
95862024-04-20T01:11:48.938Z 21 Acked 1020 Read 7 Done Sent Done false
95872024-04-20T01:11:48.938Z 22 Acked 1021 Write 6 Done Sent Done false
95882024-04-20T01:11:48.938Z 23 Acked 1022 Read 3 Done Sent Done false
95892024-04-20T01:11:48.938Z 24 Acked 1023 Write 3 Done Sent Done false
95902024-04-20T01:11:48.938Z 25 Acked 1024 Write 9 Done Sent Done false
95912024-04-20T01:11:48.938Z 26 Acked 1025 Read 3 Done Sent Done false
95922024-04-20T01:11:48.938Z 27 Acked 1026 Write 2 Done Sent Done false
95932024-04-20T01:11:48.938Z 28 Acked 1027 Read 3 Done Sent Done false
95942024-04-20T01:11:48.938Z 29 Acked 1028 Write 1 Done Sent Done false
95952024-04-20T01:11:48.938Z 30 Acked 1029 Write 9 Sent Sent Sent false
95962024-04-20T01:11:48.938Z STATES DS:0 DS:1 DS:2 TOTAL
95972024-04-20T01:11:48.938Z New 0 0 0 0
95982024-04-20T01:11:48.938Z Sent 1 29 1 31
95992024-04-20T01:11:48.938Z Done 29 1 29 59
96002024-04-20T01:11:48.938Z Skipped 0 0 0 0
96012024-04-20T01:11:48.938Z Error 0 0 0 0
96022024-04-20T01:11:48.938ZLast Flush: 1018 0 1018
96032024-04-20T01:11:48.938ZDownstairs last five completed:
96042024-04-20T01:11:48.938ZUpstairs last five completed: 30 29 28 27 26
96052024-04-20T01:11:48.938Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:11:47.43613817Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1523}
96062024-04-20T01:11:48.938ZWrote out file "/tmp/test_repair_verify.data"
96072024-04-20T01:11:48.943Z
96082024-04-20T01:11:48.943ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
96092024-04-20T01:11:48.968ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96102024-04-20T01:11:48.968Z 0 000-019 86 86 86 145 145 145 F F F
96112024-04-20T01:11:48.968Z 1 020-039 87 87 87 146 146 146 F F F
96122024-04-20T01:11:48.968Z 2 040-059 86 86 86 145 145 145 T F T <---
96132024-04-20T01:11:48.968Z 3 060-079 82 82 82 136 136 136 T F T <---
96142024-04-20T01:11:48.968Z 4 080-099 81 81 81 135 135 135 F F F
96152024-04-20T01:11:48.968Z 5 100-119 88 86 88 149 145 149 F F F <---
96162024-04-20T01:11:48.968Z 6 120-139 87 87 87 146 146 146 F F F
96172024-04-20T01:11:48.968Z 7 140-159 87 87 87 146 146 146 F F F
96182024-04-20T01:11:48.968Z 8 160-179 88 86 88 149 145 149 F F F <---
96192024-04-20T01:11:48.972Z 9 180-199 88 86 88 149 145 149 F F F <---
96202024-04-20T01:11:48.972Z 10 200-219 88 74 88 147 125 147 F T F <---
96212024-04-20T01:11:48.972Z 11 220-239 88 84 88 147 142 147 F T F <---
96222024-04-20T01:11:48.973Z 12 240-259 88 78 88 149 131 149 F F F <---
96232024-04-20T01:11:48.973Z 13 260-279 88 84 88 148 141 148 F F F <---
96242024-04-20T01:11:48.973Z 14 280-299 88 86 88 148 145 148 F F F <---
96252024-04-20T01:11:48.973Z 15 300-319 86 86 86 145 145 145 T F T <---
96262024-04-20T01:11:48.973Z 16 320-339 87 87 87 146 146 146 F F F
96272024-04-20T01:11:48.973Z 17 340-359 87 87 87 146 146 146 T F T <---
96282024-04-20T01:11:48.973Z 18 360-379 88 84 88 148 142 148 F F F <---
96292024-04-20T01:11:48.973Z 19 380-399 88 86 88 148 145 148 F F F <---
96302024-04-20T01:11:48.973Z 20 400-419 86 86 86 145 145 145 F F F
96312024-04-20T01:11:48.973Z 21 420-439 88 86 88 147 145 147 T F T <---
96322024-04-20T01:11:48.973Z 22 440-459 88 86 88 149 145 149 T F T <---
96332024-04-20T01:11:48.973Z 23 460-479 86 86 86 145 145 145 F F F
96342024-04-20T01:11:48.973Z 24 480-499 86 86 86 145 145 145 F F F
96352024-04-20T01:11:48.973Z 25 500-519 88 81 88 148 135 148 T F T <---
96362024-04-20T01:11:48.973Z 26 520-539 81 81 81 135 135 135 T F T <---
96372024-04-20T01:11:48.973Z 27 540-559 84 84 84 142 142 142 F F F
96382024-04-20T01:11:48.973Z 28 560-579 80 80 80 134 134 134 F F F
96392024-04-20T01:11:48.973Z 29 580-599 84 84 84 141 141 141 F F F
96402024-04-20T01:11:48.973ZMax gen: 88, Max flush: 149
96412024-04-20T01:11:48.973ZError: Difference in extent metadata found!
96422024-04-20T01:11:48.973ZA Difference in extent metadata is expected here
96432024-04-20T01:11:48.973ZOn loop 44
96442024-04-20T01:11:48.974Z
96452024-04-20T01:11:48.974ZVerifying data now
96462024-04-20T01:12:03.775ZNo change after verify
96472024-04-20T01:12:03.775ZLoop: 44 Downstairs dump after verify (and repair):
96482024-04-20T01:12:03.788ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96492024-04-20T01:12:03.789Z 0 000-019 86 86 86 145 145 145 F F F
96502024-04-20T01:12:03.789Z 1 020-039 87 87 87 146 146 146 F F F
96512024-04-20T01:12:03.789Z 2 040-059 89 89 89 150 150 150 F F F
96522024-04-20T01:12:03.789Z 3 060-079 89 89 89 150 150 150 F F F
96532024-04-20T01:12:03.789Z 4 080-099 81 81 81 135 135 135 F F F
96542024-04-20T01:12:03.789Z 5 100-119 88 88 88 149 149 149 F F F
96552024-04-20T01:12:03.789Z 6 120-139 87 87 87 146 146 146 F F F
96562024-04-20T01:12:03.789Z 7 140-159 87 87 87 146 146 146 F F F
96572024-04-20T01:12:03.789Z 8 160-179 88 88 88 149 149 149 F F F
96582024-04-20T01:12:03.793Z 9 180-199 88 88 88 149 149 149 F F F
96592024-04-20T01:12:03.793Z 10 200-219 88 88 88 147 147 147 F F F
96602024-04-20T01:12:03.793Z 11 220-239 88 88 88 147 147 147 F F F
96612024-04-20T01:12:03.793Z 12 240-259 88 88 88 149 149 149 F F F
96622024-04-20T01:12:03.793Z 13 260-279 88 88 88 148 148 148 F F F
96632024-04-20T01:12:03.793Z 14 280-299 88 88 88 148 148 148 F F F
96642024-04-20T01:12:03.793Z 15 300-319 89 89 89 150 150 150 F F F
96652024-04-20T01:12:03.793Z 16 320-339 87 87 87 146 146 146 F F F
96662024-04-20T01:12:03.793Z 17 340-359 89 89 89 150 150 150 F F F
96672024-04-20T01:12:03.793Z 18 360-379 88 88 88 148 148 148 F F F
96682024-04-20T01:12:03.793Z 19 380-399 88 88 88 148 148 148 F F F
96692024-04-20T01:12:03.793Z 20 400-419 86 86 86 145 145 145 F F F
96702024-04-20T01:12:03.793Z 21 420-439 89 89 89 150 150 150 F F F
96712024-04-20T01:12:03.793Z 22 440-459 89 89 89 150 150 150 F F F
96722024-04-20T01:12:03.793Z 23 460-479 86 86 86 145 145 145 F F F
96732024-04-20T01:12:03.794Z 24 480-499 86 86 86 145 145 145 F F F
96742024-04-20T01:12:03.794Z 25 500-519 89 89 89 150 150 150 F F F
96752024-04-20T01:12:03.794Z 26 520-539 89 89 89 150 150 150 F F F
96762024-04-20T01:12:03.794Z 27 540-559 84 84 84 142 142 142 F F F
96772024-04-20T01:12:03.794Z 28 560-579 80 80 80 134 134 134 F F F
96782024-04-20T01:12:03.794Z 29 580-599 84 84 84 141 141 141 F F F
96792024-04-20T01:12:03.794ZMax gen: 89, Max flush: 150
96802024-04-20T01:12:03.794Z
96812024-04-20T01:12:03.794ZBegin loop 45
96822024-04-20T01:12:03.794ZDownstairs to restart: 1
96832024-04-20T01:12:03.810Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.313556238Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531}
96842024-04-20T01:12:03.810Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.313781608Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531}
96852024-04-20T01:12:03.810Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.313800218Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531}
96862024-04-20T01:12:03.810Z{"msg":"Crucible a6b11de0-e565-40e7-8bbd-af36865e7249 has session id: b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.313814358Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
96872024-04-20T01:12:03.820Z{"msg":"Upstairs opts: Upstairs UUID: a6b11de0-e565-40e7-8bbd-af36865e7249, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.313842098Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
96882024-04-20T01:12:03.820Z{"msg":"Crucible stats registered with UUID: a6b11de0-e565-40e7-8bbd-af36865e7249","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.313876658Z","hostnameCrucible runtime is spawned
96892024-04-20T01:12:03.820Z":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
96902024-04-20T01:12:03.820Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.313973648Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531}
96912024-04-20T01:12:03.820Z{"msg":"a6b11de0-e565-40e7-8bbd-af36865e7249 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.314002748Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
96922024-04-20T01:12:03.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.314016828Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"0","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
96932024-04-20T01:12:03.821Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.314031559Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"0","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
96942024-04-20T01:12:03.821Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.314065559Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"1","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
96952024-04-20T01:12:03.821Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.314088729Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"1","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
96962024-04-20T01:12:03.821Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.314143149Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"2","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
96972024-04-20T01:12:03.821Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.314182439Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"2","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
96982024-04-20T01:12:03.821Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.314216089Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
96992024-04-20T01:12:03.821Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.314253339Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97002024-04-20T01:12:03.821Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.314293939Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97012024-04-20T01:12:03.821Z{"msg":"ds_connection connected from Ok(127.0.0.1:63120)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.314330609Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97022024-04-20T01:12:03.821Z{"msg":"ds_connection connected from Ok(127.0.0.1:64519)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.314419489Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97032024-04-20T01:12:03.821Z{"msg":"ds_connection connected from Ok(127.0.0.1:60685)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.31447146Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97042024-04-20T01:12:03.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.315065471Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"2","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97052024-04-20T01:12:03.821Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.315104051Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"2","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97062024-04-20T01:12:03.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.315130111Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"0","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97072024-04-20T01:12:03.821Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.315155301Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"0","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97082024-04-20T01:12:03.821Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.315670352Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"2","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97092024-04-20T01:12:03.821Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.315714402Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"0","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97102024-04-20T01:12:03.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.315856123Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"1","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97112024-04-20T01:12:03.821Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.315928843Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"1","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97122024-04-20T01:12:03.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.316167283Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"0","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97132024-04-20T01:12:03.821Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.316218384Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"0","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97142024-04-20T01:12:03.821Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.316263964Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97152024-04-20T01:12:03.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.316344714Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"2","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97162024-04-20T01:12:03.822Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.316373204Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"2","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97172024-04-20T01:12:03.822Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.316389044Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97182024-04-20T01:12:03.822Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.316558174Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"1","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97192024-04-20T01:12:03.822Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.316966685Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"1","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97202024-04-20T01:12:03.822Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317006945Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"1","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97212024-04-20T01:12:03.822Z{"msg":"[0]R flush_numbers[0..12]: [145, 146, 150, 150, 135, 149, 146, 146, 149, 149, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317025385Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97222024-04-20T01:12:03.822Z{"msg":"[0]R generation[0..12]: [86, 87, 89, 89, 81, 88, 87, 87, 88, 88, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317047325Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97232024-04-20T01:12:03.822Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317062695Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97242024-04-20T01:12:03.822Z{"msg":"[1]R flush_numbers[0..12]: [145, 146, 150, 150, 135, 149, 146, 146, 149, 149, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317084345Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97252024-04-20T01:12:03.822Z{"msg":"[1]R generation[0..12]: [86, 87, 89, 89, 81, 88, 87, 87, 88, 88, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317099295Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97262024-04-20T01:12:03.822Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317121926Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97272024-04-20T01:12:03.822Z{"msg":"[2]R flush_numbers[0..12]: [145, 146, 150, 150, 135, 149, 146, 146, 149, 149, 147, 147]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
97282024-04-20T01:12:03.822Z,"time":"2024-04-20T01:12:02.317137086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97292024-04-20T01:12:03.822Z{"msg":"[2]R generation[0..12]: [86, 87, 89, 89, 81, 88, 87, 87, 88, 88, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317159226Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97302024-04-20T01:12:03.822Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317174146Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97312024-04-20T01:12:03.822Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317194956Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97322024-04-20T01:12:03.822Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317209706Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97332024-04-20T01:12:03.822Z{"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317229106Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97342024-04-20T01:12:03.822Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317243156Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"":"downstairs","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97352024-04-20T01:12:03.822Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317263586Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97362024-04-20T01:12:03.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317276836Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"0","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97372024-04-20T01:12:03.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317290636Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"1","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97382024-04-20T01:12:03.823Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317304216Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"client":"2","":"downstairs","session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97392024-04-20T01:12:03.823Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317317426Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97402024-04-20T01:12:03.823Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317336856Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97412024-04-20T01:12:03.823Z{"msg":"a6b11de0-e565-40e7-8bbd-af36865e7249 is now active with session: b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317350536Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97422024-04-20T01:12:03.823Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317362556Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531,"session_id":"b2ef96c1-0a7b-4ab0-bd1c-d01dda9678d0"}
97432024-04-20T01:12:03.823Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:02.317381846Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531}
97442024-04-20T01:12:03.823ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97452024-04-20T01:12:03.823ZLoading write count information from file "/tmp/test_repair_verify.data"
97462024-04-20T01:12:03.823ZRun Repair workload
97472024-04-20T01:12:03.823Z01/30 Write block 423 len 20480 data: 5 4 6 5 5
97482024-04-20T01:12:03.838Z02/30 Read block 84 len 28672
97492024-04-20T01:12:03.838Z03/30 Write block 514 len 12288 data: 10 9 12
97502024-04-20T01:12:03.838Z04/30 Flush
97512024-04-20T01:12:03.838Z05/30 Write block 462 len 12288 data: 11 11 8
97522024-04-20T01:12:03.838Z06/30 Write block 316 len 8192 data: 4 4
97532024-04-20T01:12:03.838Z07/30 Read block 118 len 20480
97542024-04-20T01:12:03.838Z08/30 Read block 232 len 16384
97552024-04-20T01:12:03.838Z09/30 Write block 181 len 8192 data: 13 10
97562024-04-20T01:12:03.838Z10/30 Write block 367 len 8192 data: 11 9
97572024-04-20T01:12:03.838Z11/30 Flush
97582024-04-20T01:12:03.838Z12/30 Flush
97592024-04-20T01:12:03.838Z13/30 Write block 160 len 24576 data: 8 8 7 6 5 5
97602024-04-20T01:12:03.838Z14/30 Write block 318 len 16384 data: 10 10 10 10
97612024-04-20T01:12:03.838Z15/30 Read block 10 len 8192
97622024-04-20T01:12:03.838Z16/30 Read block 413 len 24576
97632024-04-20T01:12:03.838Z17/30 Read block 280 len 20480
97642024-04-20T01:12:03.838Z18/30 Write block 554 len 24576 data: 7 7 5 6 5 5
97652024-04-20T01:12:03.838Z19/30 Write block 16 len 20480 data: 8 7 6 6 6
97662024-04-20T01:12:03.838Z20/30 Read block 181 len 12288
97672024-04-20T01:12:03.838Z21/30 Write block 414 len 8192 data: 11 10
97682024-04-20T01:12:03.838Z22/30 Write block 201 len 40960 data: 6 8 6 5 6 7 6 6 6 5
97692024-04-20T01:12:03.839Z23/30 Read block 154 len 4096
97702024-04-20T01:12:03.839Z24/30 Read block 375 len 36864
97712024-04-20T01:12:03.839Z25/30 Write block 243 len 12288 data: 3 8 9
97722024-04-20T01:12:03.847Z26/30 Read block 209 len 28672
97732024-04-20T01:12:03.847Z27/30 Read block 43 len 12288
97742024-04-20T01:12:03.847Z28/30 Write block 252 len 20480 data: 7 5 8 7 7
97752024-04-20T01:12:03.847Z29/30 Read block 221 len 4096
97762024-04-20T01:12:03.847Z30/30 Read block 215 len 28672
97772024-04-20T01:12:03.847Z----------------------------------------------------------------
97782024-04-20T01:12:03.847Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
97792024-04-20T01:12:03.847ZStates: Active Active Active
97802024-04-20T01:12:03.847ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
97812024-04-20T01:12:03.848Z 1 Acked 1000 Write 5 Done Done Done false
97822024-04-20T01:12:03.848Z 2 Acked 1001 Read 7 Done Sent Done false
97832024-04-20T01:12:03.848Z 3 Acked 1002 Write 3 Done Sent Done false
97842024-04-20T01:12:03.848Z 4 Acked 1003 Flush 0 Done Sent Done false
97852024-04-20T01:12:03.848Z 5 Acked 1004 Write 3 Done Sent Done false
97862024-04-20T01:12:03.848Z 6 Acked 1005 Write 2 Done Sent Done false
97872024-04-20T01:12:03.848Z 7 Acked 1006 Read 5 Done Sent Done false
97882024-04-20T01:12:03.848Z 8 Acked 1007 Read 4 Done Sent Done false
97892024-04-20T01:12:03.848Z 9 Acked 1008 Write 2 Done Sent Done false
97902024-04-20T01:12:03.848Z 10 Acked 1009 Write 2 Done Sent Done false
97912024-04-20T01:12:03.848Z 11 Acked 1010 Flush 0 Done Sent Done false
97922024-04-20T01:12:03.848Z 12 Acked 1011 Flush 0 Done Sent Done false
97932024-04-20T01:12:03.848Z 13 Acked 1012 Write 6 Done Sent Done false
97942024-04-20T01:12:03.848Z 14 Acked 1013 Write 4 Done Sent Done false
97952024-04-20T01:12:03.848Z 15 Acked 1014 Read 2 Done Sent Done false
97962024-04-20T01:12:03.848Z 16 Acked 1015 Read 6 Done Sent Done false
97972024-04-20T01:12:03.848Z 17 Acked 1016 Read 5 Done Sent Done false
97982024-04-20T01:12:03.848Z 18 Acked 1017 Write 6 Done Sent Done false
97992024-04-20T01:12:03.848Z 19 Acked 1018 Write 5 Done Sent Done false
98002024-04-20T01:12:03.848Z 20 Acked 1019 Read 3 Done Sent Done false
98012024-04-20T01:12:03.848Z 21 Acked 1020 Write 2 Done Sent Done false
98022024-04-20T01:12:03.848Z 22 Acked 1021 Write 10 Done Sent Done false
98032024-04-20T01:12:03.848Z 23 Acked 1022 Read 1 Done Sent Done false
98042024-04-20T01:12:03.849Z 24 Acked 1023 Read 9 Done Sent Done false
98052024-04-20T01:12:03.849Z 25 Acked 1024 Write 3 Done Sent Done false
98062024-04-20T01:12:03.849Z 26 Acked 1025 Read 7 Done Sent Done false
98072024-04-20T01:12:03.849Z 27 Acked 1026 Read 3 Done Sent Done false
98082024-04-20T01:12:03.849Z 28 Acked 1027 Write 5 Done Sent Done false
98092024-04-20T01:12:03.849Z 29 Acked 1028 Read 1 Done Sent Done false
98102024-04-20T01:12:03.849Z 30 Acked 1029 Read 7 Done Sent Done false
98112024-04-20T01:12:03.849Z STATES DS:0 DS:1 DS:2 TOTAL
98122024-04-20T01:12:03.849Z New 0 0 0 0
98132024-04-20T01:12:03.849Z Sent 0 29 0 29
98142024-04-20T01:12:03.849Z Done 30 1 30 61
98152024-04-20T01:12:03.849Z Skipped 0 0 0 0
98162024-04-20T01:12:03.849Z Error 0 0 0 0
98172024-04-20T01:12:03.849ZLast Flush: 1011 0 1011
98182024-04-20T01:12:03.849ZDownstairs last five completed:
98192024-04-20T01:12:03.849ZUpstairs last five completed: 30 29 28 27 26
98202024-04-20T01:12:03.849Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:12:02.345596909Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1531}
98212024-04-20T01:12:03.849ZWrote out file "/tmp/test_repair_verify.data"
98222024-04-20T01:12:03.856Z
98232024-04-20T01:12:03.856ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
98242024-04-20T01:12:03.866ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98252024-04-20T01:12:03.866Z 0 000-019 86 86 86 145 145 145 T F T <---
98262024-04-20T01:12:03.866Z 1 020-039 87 87 87 146 146 146 T F T <---
98272024-04-20T01:12:03.866Z 2 040-059 89 89 89 150 150 150 F F F
98282024-04-20T01:12:03.866Z 3 060-079 89 89 89 150 150 150 F F F
98292024-04-20T01:12:03.866Z 4 080-099 81 81 81 135 135 135 F F F
98302024-04-20T01:12:03.866Z 5 100-119 88 88 88 149 149 149 F F F
98312024-04-20T01:12:03.866Z 6 120-139 87 87 87 146 146 146 F F F
98322024-04-20T01:12:03.866Z 7 140-159 87 87 87 146 146 146 F F F
98332024-04-20T01:12:03.866Z 8 160-179 88 88 88 149 149 149 T F T <---
98342024-04-20T01:12:03.870Z 9 180-199 90 88 90 152 149 152 F F F <---
98352024-04-20T01:12:03.870Z 10 200-219 88 88 88 147 147 147 T F T <---
98362024-04-20T01:12:03.870Z 11 220-239 88 88 88 147 147 147 F F F
98372024-04-20T01:12:03.871Z 12 240-259 88 88 88 149 149 149 T F T <---
98382024-04-20T01:12:03.871Z 13 260-279 88 88 88 148 148 148 F F F
98392024-04-20T01:12:03.871Z 14 280-299 88 88 88 148 148 148 F F F
98402024-04-20T01:12:03.871Z 15 300-319 90 89 90 152 150 152 T F T <---
98412024-04-20T01:12:03.871Z 16 320-339 87 87 87 146 146 146 T F T <---
98422024-04-20T01:12:03.871Z 17 340-359 89 89 89 150 150 150 F F F
98432024-04-20T01:12:03.871Z 18 360-379 90 88 90 152 148 152 F F F <---
98442024-04-20T01:12:03.871Z 19 380-399 88 88 88 148 148 148 F F F
98452024-04-20T01:12:03.871Z 20 400-419 86 86 86 145 145 145 T F T <---
98462024-04-20T01:12:03.871Z 21 420-439 90 89 90 151 150 151 F T F <---
98472024-04-20T01:12:03.871Z 22 440-459 89 89 89 150 150 150 F F F
98482024-04-20T01:12:03.871Z 23 460-479 90 86 90 152 145 152 F F F <---
98492024-04-20T01:12:03.871Z 24 480-499 86 86 86 145 145 145 F F F
98502024-04-20T01:12:03.871Z 25 500-519 90 89 90 151 150 151 F F F <---
98512024-04-20T01:12:03.871Z 26 520-539 89 89 89 150 150 150 F F F
98522024-04-20T01:12:03.871Z 27 540-559 84 84 84 142 142 142 T F T <---
98532024-04-20T01:12:03.871Z 28 560-579 80 80 80 134 134 134 F F F
98542024-04-20T01:12:03.871Z 29 580-599 84 84 84 141 141 141 F F F
98552024-04-20T01:12:03.871ZMax gen: 90, Max flush: 152
98562024-04-20T01:12:03.871ZError: Difference in extent metadata found!
98572024-04-20T01:12:03.871ZA Difference in extent metadata is expected here
98582024-04-20T01:12:03.871ZOn loop 45
98592024-04-20T01:12:03.871Z
98602024-04-20T01:12:03.876ZVerifying data now
98612024-04-20T01:12:08.517ZNo change after verify
98622024-04-20T01:12:08.517ZLoop: 45 Downstairs dump after verify (and repair):
98632024-04-20T01:12:08.531ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98642024-04-20T01:12:08.531Z 0 000-019 91 91 91 153 153 153 F F F
98652024-04-20T01:12:08.531Z 1 020-039 91 91 91 153 153 153 F F F
98662024-04-20T01:12:08.531Z 2 040-059 89 89 89 150 150 150 F F F
98672024-04-20T01:12:08.531Z 3 060-079 89 89 89 150 150 150 F F F
98682024-04-20T01:12:08.531Z 4 080-099 81 81 81 135 135 135 F F F
98692024-04-20T01:12:08.531Z 5 100-119 88 88 88 149 149 149 F F F
98702024-04-20T01:12:08.535Z 6 120-139 87 87 87 146 146 146 F F F
98712024-04-20T01:12:08.535Z 7 140-159 87 87 87 146 146 146 F F F
98722024-04-20T01:12:08.535Z 8 160-179 91 91 91 153 153 153 F F F
98732024-04-20T01:12:08.536Z 9 180-199 90 90 90 152 152 152 F F F
98742024-04-20T01:12:08.536Z 10 200-219 91 91 91 153 153 153 F F F
98752024-04-20T01:12:08.536Z 11 220-239 88 88 88 147 147 147 F F F
98762024-04-20T01:12:08.536Z 12 240-259 91 91 91 153 153 153 F F F
98772024-04-20T01:12:08.536Z 13 260-279 88 88 88 148 148 148 F F F
98782024-04-20T01:12:08.536Z 14 280-299 88 88 88 148 148 148 F F F
98792024-04-20T01:12:08.536Z 15 300-319 91 91 91 153 153 153 F F F
98802024-04-20T01:12:08.536Z 16 320-339 91 91 91 153 153 153 F F F
98812024-04-20T01:12:08.536Z 17 340-359 89 89 89 150 150 150 F F F
98822024-04-20T01:12:08.536Z 18 360-379 90 90 90 152 152 152 F F F
98832024-04-20T01:12:08.536Z 19 380-399 88 88 88 148 148 148 F F F
98842024-04-20T01:12:08.536Z 20 400-419 91 91 91 153 153 153 F F F
98852024-04-20T01:12:08.536Z 21 420-439 90 90 90 151 151 151 F F F
98862024-04-20T01:12:08.536Z 22 440-459 89 89 89 150 150 150 F F F
98872024-04-20T01:12:08.536Z 23 460-479 90 90 90 152 152 152 F F F
98882024-04-20T01:12:08.536Z 24 480-499 86 86 86 145 145 145 F F F
98892024-04-20T01:12:08.536Z 25 500-519 90 90 90 151 151 151 F F F
98902024-04-20T01:12:08.536Z 26 520-539 89 89 89 150 150 150 F F F
98912024-04-20T01:12:08.536Z 27 540-559 91 91 91 153 153 153 F F F
98922024-04-20T01:12:08.536Z 28 560-579 80 80 80 134 134 134 F F F
98932024-04-20T01:12:08.536Z 29 580-599 84 84 84 141 141 141 F F F
98942024-04-20T01:12:08.536ZMax gen: 91, Max flush: 153
98952024-04-20T01:12:08.536Z
98962024-04-20T01:12:08.537ZBegin loop 46
98972024-04-20T01:12:08.537ZDownstairs to restart: 2
98982024-04-20T01:12:08.554Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.057665318Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539}
98992024-04-20T01:12:08.554Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.057887999Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539}
99002024-04-20T01:12:08.554Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.057904819Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539}
99012024-04-20T01:12:08.554Z{"msg":"Crucible f994b058-9e72-4146-956c-3eb36f968028 has session id: 4cc80cb9-9572-4190-b476-683bd1c829e7","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.057919279Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99022024-04-20T01:12:08.561Z{"msg":"Upstairs opts: Upstairs UUID: f994b058-9e72-4146-956c-3eb36f968028, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.057953629Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99032024-04-20T01:12:08.561Z{"msg":"Crucible stats registered with UUID: f994b058-9e72-4146-956c-3eb36f968028","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.057987719Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99042024-04-20T01:12:08.561ZCrucible runtime is spawned
99052024-04-20T01:12:08.561Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.058119869Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539}
99062024-04-20T01:12:08.562Z{"msg":"f994b058-9e72-4146-956c-3eb36f968028 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.058227189Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99072024-04-20T01:12:08.562Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.058259469Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"0","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99082024-04-20T01:12:08.562Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.058274829Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"0","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99092024-04-20T01:12:08.562Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.058364Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"1","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99102024-04-20T01:12:08.562Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.05847857Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"1","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99112024-04-20T01:12:08.562Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.05852009Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"2","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99122024-04-20T01:12:08.562Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.0586348Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"2","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99132024-04-20T01:12:08.562Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.05868576Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99142024-04-20T01:12:08.562Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.05872433Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99152024-04-20T01:12:08.562Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.058741981Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99162024-04-20T01:12:08.562Z{"msg":"ds_connection connected from Ok(127.0.0.1:59267)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.058766901Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99172024-04-20T01:12:08.562Z{"msg":"ds_connection connected from Ok(127.0.0.1:37730)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.058801681Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99182024-04-20T01:12:08.562Z{"msg":"ds_connection connected from Ok(127.0.0.1:45897)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.058834801Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99192024-04-20T01:12:08.562Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.059289142Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"1","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99202024-04-20T01:12:08.562Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.059406832Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"1","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99212024-04-20T01:12:08.562Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.059461532Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"0","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99222024-04-20T01:12:08.562Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.059494442Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"0","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99232024-04-20T01:12:08.562Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.059538392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"2","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99242024-04-20T01:12:08.562Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.059583312Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"2","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99252024-04-20T01:12:08.562Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.059879033Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"0","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99262024-04-20T01:12:08.562Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.060014263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"1","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99272024-04-20T01:12:08.562Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.060315884Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"2","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99282024-04-20T01:12:08.563Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.060354024Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"0","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99292024-04-20T01:12:08.563Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.060371414Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"0","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99302024-04-20T01:12:08.563Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.060385844Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99312024-04-20T01:12:08.563Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.060623265Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"1","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99322024-04-20T01:12:08.563Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.060707945Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"1","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99332024-04-20T01:12:08.563Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.060737075Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99342024-04-20T01:12:08.563Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.060891105Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"2","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99352024-04-20T01:12:08.563Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.060931605Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"2","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99362024-04-20T01:12:08.563Z{"msg":"[0]R flush_numbers[0..12]: [153, 153, 150, 150, 135, 149, 146, 146, 153, 152, 153, 147]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.060950355Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99372024-04-20T01:12:08.563Z{"msg":"[0]R generation[0..12]: [91, 91, 89, 89, 81, 88, 87, 87, 91, 90, 91, 88]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.060982726Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99382024-04-20T01:12:08.563Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061005826Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99392024-04-20T01:12:08.563Z{"msg":"[1]R flush_numbers[0..12]: [153, 153, 150, 150, 135, 149, 146, 146, 153, 152, 153, 147]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061034486Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99402024-04-20T01:12:08.563Z{"msg":"[1]R generation[0..12]: [91, 91, 89, 89, 81, 88, 87, 87, 91, 90, 91, 88]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061053786Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99412024-04-20T01:12:08.563Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
99422024-04-20T01:12:08.563Z,"time":"2024-04-20T01:12:07.061093096Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99432024-04-20T01:12:08.563Z{"msg":"[2]R flush_numbers[0..12]: [153, 153, 150, 150, 135, 149, 146, 146, 153, 152, 153, 147]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061120366Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99442024-04-20T01:12:08.563Z{"msg":"[2]R generation[0..12]: [91, 91, 89, 89, 81, 88, 87, 87, 91, 90, 91, 88]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061153256Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99452024-04-20T01:12:08.563Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061187416Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99462024-04-20T01:12:08.563Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061223256Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99472024-04-20T01:12:08.563Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061249926Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99482024-04-20T01:12:08.563Z{"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061274036Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99492024-04-20T01:12:08.563Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061299066Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"":"downstairs","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99502024-04-20T01:12:08.563Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061336266Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99512024-04-20T01:12:08.563Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061361426Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"0","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99522024-04-20T01:12:08.564Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061397876Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"1","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99532024-04-20T01:12:08.564Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061426056Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"client":"2","":"downstairs","session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99542024-04-20T01:12:08.564Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061462187Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99552024-04-20T01:12:08.564Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061487447Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99562024-04-20T01:12:08.564Z{"msg":"f994b058-9e72-4146-956c-3eb36f968028 is now active with session: 4cc80cb9-9572-4190-b476-683bd1c829e7","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061522257Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99572024-04-20T01:12:08.564Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061547557Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539,"session_id":"4cc80cb9-9572-4190-b476-683bd1c829e7"}
99582024-04-20T01:12:08.564Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:07.061571577Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539}
99592024-04-20T01:12:08.564ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
99602024-04-20T01:12:08.568ZLoading write count information from file "/tmp/test_repair_verify.data"
99612024-04-20T01:12:08.568ZRun Repair workload
99622024-04-20T01:12:08.568Z01/30 Write block 520 len 12288 data: 12 12 11
99632024-04-20T01:12:08.568Z02/30 Write block 226 len 12288 data: 12 7 9
99642024-04-20T01:12:08.568Z03/30 Write block 54 len 20480 data: 11 11 10 8 10
99652024-04-20T01:12:08.568Z04/30 Read block 67 len 36864
99662024-04-20T01:12:08.568Z05/30 Read block 468 len 36864
99672024-04-20T01:12:08.568Z06/30 Read block 350 len 28672
99682024-04-20T01:12:08.568Z07/30 Read block 76 len 24576
99692024-04-20T01:12:08.568Z08/30 Read block 269 len 16384
99702024-04-20T01:12:08.568Z09/30 Read block 131 len 36864
99712024-04-20T01:12:08.568Z10/30 Write block 399 len 16384 data: 8 7 7 7
99722024-04-20T01:12:08.568Z11/30 Read block 164 len 32768
99732024-04-20T01:12:08.572Z12/30 Write block 250 len 32768 data: 10 8 8 6 9 8 8 6
99742024-04-20T01:12:08.572Z13/30 Write block 202 len 40960 data: 9 7 6 7 8 7 7 7 6 5
99752024-04-20T01:12:08.572Z14/30 Write block 383 len 8192 data: 6 4
99762024-04-20T01:12:08.572Z15/30 Read block 147 len 12288
99772024-04-20T01:12:08.572Z16/30 Flush
99782024-04-20T01:12:08.575Z17/30 Write block 258 len 8192 data: 6 6
99792024-04-20T01:12:08.575Z18/30 Read block 570 len 36864
99802024-04-20T01:12:08.579Z19/30 Write block 269 len 20480 data: 5 6 8 9 10
99812024-04-20T01:12:08.579Z20/30 Read block 364 len 32768
99822024-04-20T01:12:08.579Z21/30 Read block 116 len 40960
99832024-04-20T01:12:08.579Z22/30 Read block 73 len 20480
99842024-04-20T01:12:08.582Z23/30 Flush
99852024-04-20T01:12:08.582Z24/30 Write block 210 len 32768 data: 7 6 6 7 7 6 7 11
99862024-04-20T01:12:08.588Z25/30 Write block 242 len 40960 data: 5 4 9 10 9 8 12 12 11 9
99872024-04-20T01:12:08.588Z26/30 Read block 434 len 28672
99882024-04-20T01:12:08.588Z27/30 Read block 228 len 32768
99892024-04-20T01:12:08.593Z28/30 Read block 74 len 24576
99902024-04-20T01:12:08.593Z29/30 Write block 81 len 36864 data: 7 6 8 7 7 9 9 9 10
99912024-04-20T01:12:08.593Z30/30 Write block 384 len 28672 data: 5 2 4 3 4 5 5
99922024-04-20T01:12:08.593Z----------------------------------------------------------------
99932024-04-20T01:12:08.593Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
99942024-04-20T01:12:08.593ZStates: Active Active Active
99952024-04-20T01:12:08.594ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
99962024-04-20T01:12:08.594Z 1 Acked 1000 Write 3 Done Done Sent false
99972024-04-20T01:12:08.594Z 2 Acked 1001 Write 3 Done Done Sent false
99982024-04-20T01:12:08.594Z 3 Acked 1002 Write 5 Done Done Sent false
99992024-04-20T01:12:08.594Z 4 Acked 1003 Read 9 Done Done Sent false
100002024-04-20T01:12:08.594Z 5 Acked 1004 Read 9 Done Done Sent false
100012024-04-20T01:12:08.594Z 6 Acked 1005 Read 7 Done Done Sent false
100022024-04-20T01:12:08.594Z 7 Acked 1006 Read 6 Done Done Sent false
100032024-04-20T01:12:08.594Z 8 Acked 1007 Read 4 Done Done Sent false
100042024-04-20T01:12:08.594Z 9 Acked 1008 Read 9 Done Done Sent false
100052024-04-20T01:12:08.594Z 10 Acked 1009 Write 4 Done Done Sent false
100062024-04-20T01:12:08.594Z 11 Acked 1010 Read 8 Done Done Sent false
100072024-04-20T01:12:08.594Z 12 Acked 1011 Write 8 Done Done Sent false
100082024-04-20T01:12:08.594Z 13 Acked 1012 Write 10 Done Done Sent false
100092024-04-20T01:12:08.594Z 14 Acked 1013 Write 2 Done Done Sent false
100102024-04-20T01:12:08.594Z 15 Acked 1014 Read 3 Done Done Sent false
100112024-04-20T01:12:08.594Z 16 Acked 1015 Flush 0 Done Done Sent false
100122024-04-20T01:12:08.594Z 17 Acked 1016 Write 2 Done Done Sent false
100132024-04-20T01:12:08.594Z 18 Acked 1017 Read 9 Done Done Sent false
100142024-04-20T01:12:08.594Z 19 Acked 1018 Write 5 Done Done Sent false
100152024-04-20T01:12:08.594Z 20 Acked 1019 Read 8 Done Done Sent false
100162024-04-20T01:12:08.594Z 21 Acked 1020 Read 10 Done Done Sent false
100172024-04-20T01:12:08.594Z 22 Acked 1021 Read 5 Done Done Sent false
100182024-04-20T01:12:08.595Z 23 Acked 1022 Flush 0 Done Done Sent false
100192024-04-20T01:12:08.595Z 24 Acked 1023 Write 8 Done Done Sent false
100202024-04-20T01:12:08.595Z 25 Acked 1024 Write 10 Done Done Sent false
100212024-04-20T01:12:08.595Z 26 Acked 1025 Read 7 Done Done Sent false
100222024-04-20T01:12:08.595Z 27 Acked 1026 Read 8 Done Done Sent false
100232024-04-20T01:12:08.595Z 28 Acked 1027 Read 6 Done Done Sent false
100242024-04-20T01:12:08.595Z 29 Acked 1028 Write 9 Done Done Sent false
100252024-04-20T01:12:08.595Z 30 Acked 1029 Write 7 Sent Sent Sent false
100262024-04-20T01:12:08.595Z STATES DS:0 DS:1 DS:2 TOTAL
100272024-04-20T01:12:08.595Z New 0 0 0 0
100282024-04-20T01:12:08.595Z Sent 1 1 30 32
100292024-04-20T01:12:08.595Z Done 29 29 0 58
100302024-04-20T01:12:08.595Z Skipped 0 0 0 0
100312024-04-20T01:12:08.595Z Error 0 0 0 0
100322024-04-20T01:12:08.595ZLast Flush: 1022 1022 0
100332024-04-20T01:12:08.595ZDownstairs last five completed:
100342024-04-20T01:12:08.595ZUpstairs last five completed: 30 29 28 27 26
100352024-04-20T01:12:08.595Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:12:07.093221307Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1539}
100362024-04-20T01:12:08.595ZWrote out file "/tmp/test_repair_verify.data"
100372024-04-20T01:12:08.599Z
100382024-04-20T01:12:08.599ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100392024-04-20T01:12:08.612ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100402024-04-20T01:12:08.612Z 0 000-019 91 91 91 153 153 153 F F F
100412024-04-20T01:12:08.612Z 1 020-039 91 91 91 153 153 153 F F F
100422024-04-20T01:12:08.612Z 2 040-059 92 92 89 154 154 150 F F F <---
100432024-04-20T01:12:08.613Z 3 060-079 89 89 89 150 150 150 F F F
100442024-04-20T01:12:08.613Z 4 080-099 81 81 81 135 135 135 T T F <---
100452024-04-20T01:12:08.613Z 5 100-119 88 88 88 149 149 149 F F F
100462024-04-20T01:12:08.616Z 6 120-139 87 87 87 146 146 146 F F F
100472024-04-20T01:12:08.616Z 7 140-159 87 87 87 146 146 146 F F F
100482024-04-20T01:12:08.616Z 8 160-179 91 91 91 153 153 153 F F F
100492024-04-20T01:12:08.616Z 9 180-199 90 90 90 152 152 152 F F F
100502024-04-20T01:12:08.616Z 10 200-219 92 92 91 154 154 153 T T F <---
100512024-04-20T01:12:08.616Z 11 220-239 92 92 88 154 154 147 F F F <---
100522024-04-20T01:12:08.617Z 12 240-259 92 92 91 155 155 153 T T F <---
100532024-04-20T01:12:08.617Z 13 260-279 92 92 88 155 155 148 F F F <---
100542024-04-20T01:12:08.617Z 14 280-299 88 88 88 148 148 148 F F F
100552024-04-20T01:12:08.617Z 15 300-319 91 91 91 153 153 153 F F F
100562024-04-20T01:12:08.617Z 16 320-339 91 91 91 153 153 153 F F F
100572024-04-20T01:12:08.617Z 17 340-359 89 89 89 150 150 150 F F F
100582024-04-20T01:12:08.617Z 18 360-379 90 90 90 152 152 152 F F F
100592024-04-20T01:12:08.617Z 19 380-399 92 92 88 154 154 148 T T F <---
100602024-04-20T01:12:08.617Z 20 400-419 92 92 91 154 154 153 F F F <---
100612024-04-20T01:12:08.617Z 21 420-439 90 90 90 151 151 151 F F F
100622024-04-20T01:12:08.617Z 22 440-459 89 89 89 150 150 150 F F F
100632024-04-20T01:12:08.617Z 23 460-479 90 90 90 152 152 152 F F F
100642024-04-20T01:12:08.617Z 24 480-499 86 86 86 145 145 145 F F F
100652024-04-20T01:12:08.617Z 25 500-519 90 90 90 151 151 151 F F F
100662024-04-20T01:12:08.617Z 26 520-539 92 92 89 154 154 150 F F F <---
100672024-04-20T01:12:08.617Z 27 540-559 91 91 91 153 153 153 F F F
100682024-04-20T01:12:08.617Z 28 560-579 80 80 80 134 134 134 F F F
100692024-04-20T01:12:08.617Z 29 580-599 84 84 84 141 141 141 F F F
100702024-04-20T01:12:08.617ZMax gen: 92, Max flush: 155
100712024-04-20T01:12:08.617ZError: Difference in extent metadata found!
100722024-04-20T01:12:08.617ZA Difference in extent metadata is expected here
100732024-04-20T01:12:08.617ZOn loop 46
100742024-04-20T01:12:08.617Z
100752024-04-20T01:12:08.622ZVerifying data now
100762024-04-20T01:12:13.045ZNo change after verify
100772024-04-20T01:12:13.045ZLoop: 46 Downstairs dump after verify (and repair):
100782024-04-20T01:12:13.060ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100792024-04-20T01:12:13.060Z 0 000-019 91 91 91 153 153 153 F F F
100802024-04-20T01:12:13.060Z 1 020-039 91 91 91 153 153 153 F F F
100812024-04-20T01:12:13.060Z 2 040-059 92 92 92 154 154 154 F F F
100822024-04-20T01:12:13.060Z 3 060-079 89 89 89 150 150 150 F F F
100832024-04-20T01:12:13.060Z 4 080-099 93 93 93 156 156 156 F F F
100842024-04-20T01:12:13.060Z 5 100-119 88 88 88 149 149 149 F F F
100852024-04-20T01:12:13.064Z 6 120-139 87 87 87 146 146 146 F F F
100862024-04-20T01:12:13.064Z 7 140-159 87 87 87 146 146 146 F F F
100872024-04-20T01:12:13.064Z 8 160-179 91 91 91 153 153 153 F F F
100882024-04-20T01:12:13.064Z 9 180-199 90 90 90 152 152 152 F F F
100892024-04-20T01:12:13.064Z 10 200-219 93 93 93 156 156 156 F F F
100902024-04-20T01:12:13.064Z 11 220-239 92 92 92 154 154 154 F F F
100912024-04-20T01:12:13.064Z 12 240-259 93 93 93 156 156 156 F F F
100922024-04-20T01:12:13.064Z 13 260-279 92 92 92 155 155 155 F F F
100932024-04-20T01:12:13.064Z 14 280-299 88 88 88 148 148 148 F F F
100942024-04-20T01:12:13.064Z 15 300-319 91 91 91 153 153 153 F F F
100952024-04-20T01:12:13.064Z 16 320-339 91 91 91 153 153 153 F F F
100962024-04-20T01:12:13.064Z 17 340-359 89 89 89 150 150 150 F F F
100972024-04-20T01:12:13.064Z 18 360-379 90 90 90 152 152 152 F F F
100982024-04-20T01:12:13.064Z 19 380-399 93 93 93 156 156 156 F F F
100992024-04-20T01:12:13.064Z 20 400-419 92 92 92 154 154 154 F F F
101002024-04-20T01:12:13.064Z 21 420-439 90 90 90 151 151 151 F F F
101012024-04-20T01:12:13.064Z 22 440-459 89 89 89 150 150 150 F F F
101022024-04-20T01:12:13.065Z 23 460-479 90 90 90 152 152 152 F F F
101032024-04-20T01:12:13.065Z 24 480-499 86 86 86 145 145 145 F F F
101042024-04-20T01:12:13.065Z 25 500-519 90 90 90 151 151 151 F F F
101052024-04-20T01:12:13.065Z 26 520-539 92 92 92 154 154 154 F F F
101062024-04-20T01:12:13.065Z 27 540-559 91 91 91 153 153 153 F F F
101072024-04-20T01:12:13.065Z 28 560-579 80 80 80 134 134 134 F F F
101082024-04-20T01:12:13.065Z 29 580-599 84 84 84 141 141 141 F F F
101092024-04-20T01:12:13.065ZMax gen: 93, Max flush: 156
101102024-04-20T01:12:13.065Z
101112024-04-20T01:12:13.065ZBegin loop 47
101122024-04-20T01:12:13.065ZDownstairs to restart: 1
101132024-04-20T01:12:13.081Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.584479995Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547}
101142024-04-20T01:12:13.081Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.584703995Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547}
101152024-04-20T01:12:13.089Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.584722345Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547}
101162024-04-20T01:12:13.089Z{"msg":"Crucible e2b06e47-d4e2-42ff-85ad-44b5e65f46dd has session id: b255690b-90ec-4d60-9fcc-a1c309f25768","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.584755735Z","Crucible runtime is spawned
101172024-04-20T01:12:13.089Zhostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101182024-04-20T01:12:13.089Z{"msg":"Upstairs opts: Upstairs UUID: e2b06e47-d4e2-42ff-85ad-44b5e65f46dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.584856575Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101192024-04-20T01:12:13.089Z{"msg":"Crucible stats registered with UUID: e2b06e47-d4e2-42ff-85ad-44b5e65f46dd","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.584903685Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101202024-04-20T01:12:13.089Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.584922365Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547}
101212024-04-20T01:12:13.089Z{"msg":"e2b06e47-d4e2-42ff-85ad-44b5e65f46dd active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.584960356Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101222024-04-20T01:12:13.089Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585009696Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"0","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101232024-04-20T01:12:13.089Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585054326Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"0","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101242024-04-20T01:12:13.089Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585099946Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"1","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101252024-04-20T01:12:13.089Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585133636Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"1","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101262024-04-20T01:12:13.089Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585179536Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"2","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101272024-04-20T01:12:13.089Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585208766Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"2","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101282024-04-20T01:12:13.090Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585247186Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101292024-04-20T01:12:13.090Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585273766Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101302024-04-20T01:12:13.090Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585303106Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101312024-04-20T01:12:13.090Z{"msg":"ds_connection connected from Ok(127.0.0.1:50717)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585335326Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101322024-04-20T01:12:13.090Z{"msg":"ds_connection connected from Ok(127.0.0.1:59958)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585378466Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101332024-04-20T01:12:13.090Z{"msg":"ds_connection connected from Ok(127.0.0.1:63593)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585432217Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101342024-04-20T01:12:13.090Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585933878Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"2","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101352024-04-20T01:12:13.090Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.585997048Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"2","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101362024-04-20T01:12:13.090Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.586021558Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"0","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101372024-04-20T01:12:13.090Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.586043028Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"0","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101382024-04-20T01:12:13.090Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.586576009Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"2","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101392024-04-20T01:12:13.090Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.586632569Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"0","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101402024-04-20T01:12:13.090Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.58680252Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"1","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101412024-04-20T01:12:13.090Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.58684932Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"1","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101422024-04-20T01:12:13.090Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.58707903Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"2","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101432024-04-20T01:12:13.090Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.5871215Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"2","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101442024-04-20T01:12:13.090Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.58714902Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101452024-04-20T01:12:13.090Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587223261Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"0","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101462024-04-20T01:12:13.090Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587259501Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"0","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101472024-04-20T01:12:13.090Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587275561Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101482024-04-20T01:12:13.090Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587323451Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"1","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101492024-04-20T01:12:13.090Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587721122Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"1","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101502024-04-20T01:12:13.090Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587757452Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"1","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101512024-04-20T01:12:13.091Z{"msg":"[0]R flush_numbers[0..12]: [153, 153, 154, 150, 156, 149, 146, 146, 153, 152, 156, 154]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587773572Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101522024-04-20T01:12:13.091Z{"msg":"[0]R generation[0..12]: [91, 91, 92, 89, 93, 88, 87, 87, 91, 90, 93, 92]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587788162Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101532024-04-20T01:12:13.091Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587810982Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101542024-04-20T01:12:13.091Z{"msg":"[1]R flush_numbers[0..12]: [153, 153, 154, 150, 156, 149, 146, 146, 153, 152, 156, 154]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587825362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101552024-04-20T01:12:13.091Z{"msg":"[1]R generation[0..12]: [91, 91, 92, 89, 93, 88, 87, 87, 91, 90, 93, 92]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587840332Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101562024-04-20T01:12:13.091Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587855092Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101572024-04-20T01:12:13.091Z{"msg":"[2]R flush_numbers[0..12]: [153, 153, 154, 150, 156, 149, 146, 146, 153, 152, 156, 154]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587877112Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101582024-04-20T01:12:13.091Z{"msg":"[2]R generation[0..12]: [91, 91, 92, 89, 93, 88, 87, 87, 91, 90, 93, 92]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587891952Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101592024-04-20T01:12:13.091Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587906672Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101602024-04-20T01:12:13.091Z{"msg":"Max found gen is 94","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
101612024-04-20T01:12:13.091Zcrucible","level":30,"time":"2024-04-20T01:12:11.587929862Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101622024-04-20T01:12:13.091Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587943452Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101632024-04-20T01:12:13.091Z{"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587956062Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101642024-04-20T01:12:13.091Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587969092Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"":"downstairs","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101652024-04-20T01:12:13.091Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.587990932Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101662024-04-20T01:12:13.091Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.588004482Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"0","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101672024-04-20T01:12:13.091Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.588021582Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"1","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101682024-04-20T01:12:13.091Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.588045172Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"client":"2","":"downstairs","session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101692024-04-20T01:12:13.091Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.588079172Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101702024-04-20T01:12:13.091Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.588100203Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101712024-04-20T01:12:13.091Z{"msg":"e2b06e47-d4e2-42ff-85ad-44b5e65f46dd is now active with session: b255690b-90ec-4d60-9fcc-a1c309f25768","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.588127783Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101722024-04-20T01:12:13.091Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.588150293Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547,"session_id":"b255690b-90ec-4d60-9fcc-a1c309f25768"}
101732024-04-20T01:12:13.091Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:11.588165913Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547}
101742024-04-20T01:12:13.092ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101752024-04-20T01:12:13.092ZLoading write count information from file "/tmp/test_repair_verify.data"
101762024-04-20T01:12:13.092ZRun Repair workload
101772024-04-20T01:12:13.095Z01/30 Write block 380 len 16384 data: 10 8 7 7
101782024-04-20T01:12:13.096Z02/30 Read block 159 len 20480
101792024-04-20T01:12:13.096Z03/30 Read block 257 len 40960
101802024-04-20T01:12:13.096Z04/30 Read block 350 len 36864
101812024-04-20T01:12:13.096Z05/30 Read block 309 len 16384
101822024-04-20T01:12:13.096Z06/30 Read block 380 len 8192
101832024-04-20T01:12:13.096Z07/30 Write block 518 len 24576 data: 10 12 13 13 12 13
101842024-04-20T01:12:13.096Z08/30 Read block 39 len 32768
101852024-04-20T01:12:13.096Z09/30 Read block 268 len 16384
101862024-04-20T01:12:13.096Z10/30 Write block 404 len 40960 data: 6 7 8 7 7 8 8 10 11 12
101872024-04-20T01:12:13.096Z11/30 Read block 501 len 28672
101882024-04-20T01:12:13.100Z12/30 Write block 2 len 24576 data: 5 5 6 5 5 5
101892024-04-20T01:12:13.100Z13/30 Read block 136 len 36864
101902024-04-20T01:12:13.100Z14/30 Write block 398 len 20480 data: 7 9 8 8 8
101912024-04-20T01:12:13.100Z15/30 Write block 90 len 12288 data: 11 9 8
101922024-04-20T01:12:13.100Z16/30 Read block 218 len 4096
101932024-04-20T01:12:13.100Z17/30 Read block 525 len 12288
101942024-04-20T01:12:13.104Z18/30 Read block 21 len 16384
101952024-04-20T01:12:13.104Z19/30 Write block 221 len 16384 data: 9 11 11 11
101962024-04-20T01:12:13.104Z20/30 Read block 59 len 20480
101972024-04-20T01:12:13.104Z21/30 Write block 444 len 20480 data: 8 7 6 6 6
101982024-04-20T01:12:13.104Z22/30 Read block 476 len 8192
101992024-04-20T01:12:13.108Z23/30 Read block 270 len 16384
102002024-04-20T01:12:13.108Z24/30 Write block 586 len 16384 data: 14 12 12 10
102012024-04-20T01:12:13.108Z25/30 Read block 75 len 32768
102022024-04-20T01:12:13.108Z26/30 Read block 42 len 20480
102032024-04-20T01:12:13.108Z27/30 Read block 157 len 20480
102042024-04-20T01:12:13.108Z28/30 Read block 103 len 12288
102052024-04-20T01:12:13.126Z29/30 Write block 152 len 32768 data: 6 7 10 12 9 9 9 7
102062024-04-20T01:12:13.126Z30/30 Write block 254 len 12288 data: 10 9 9
102072024-04-20T01:12:13.126Z----------------------------------------------------------------
102082024-04-20T01:12:13.126Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30
102092024-04-20T01:12:13.126ZStates: Active Active Active
102102024-04-20T01:12:13.126ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
102112024-04-20T01:12:13.126Z 1 Acked 1000 Write 4 Done Done Done false
102122024-04-20T01:12:13.126Z 2 Acked 1001 Read 5 Done Done Done false
102132024-04-20T01:12:13.127Z 3 Acked 1002 Read 10 Done Done Done false
102142024-04-20T01:12:13.127Z 4 Acked 1003 Read 9 Done Sent Done false
102152024-04-20T01:12:13.127Z 5 Acked 1004 Read 4 Done Sent Done false
102162024-04-20T01:12:13.127Z 6 Acked 1005 Read 2 Done Sent Done false
102172024-04-20T01:12:13.127Z 7 Acked 1006 Write 6 Done Sent Done false
102182024-04-20T01:12:13.127Z 8 Acked 1007 Read 8 Done Sent Done false
102192024-04-20T01:12:13.127Z 9 Acked 1008 Read 4 Done Sent Done false
102202024-04-20T01:12:13.127Z 10 Acked 1009 Write 10 Done Sent Done false
102212024-04-20T01:12:13.127Z 11 Acked 1010 Read 7 Done Sent Done false
102222024-04-20T01:12:13.127Z 12 Acked 1011 Write 6 Done Sent Done false
102232024-04-20T01:12:13.127Z 13 Acked 1012 Read 9 Done Sent Done false
102242024-04-20T01:12:13.127Z 14 Acked 1013 Write 5 Done Sent Done false
102252024-04-20T01:12:13.128Z 15 Acked 1014 Write 3 Done Sent Done false
102262024-04-20T01:12:13.128Z 16 Acked 1015 Read 1 Done Sent Done false
102272024-04-20T01:12:13.128Z 17 Acked 1016 Read 3 Done Sent Done false
102282024-04-20T01:12:13.128Z 18 Acked 1017 Read 4 Done Sent Done false
102292024-04-20T01:12:13.128Z 19 Acked 1018 Write 4 Done Sent Done false
102302024-04-20T01:12:13.128Z 20 Acked 1019 Read 5 Done Sent Done false
102312024-04-20T01:12:13.128Z 21 Acked 1020 Write 5 Done Sent Done false
102322024-04-20T01:12:13.128Z 22 Acked 1021 Read 2 Done Sent Done false
102332024-04-20T01:12:13.128Z 23 Acked 1022 Read 4 Done Sent Done false
102342024-04-20T01:12:13.128Z 24 Acked 1023 Write 4 Done Sent Done false
102352024-04-20T01:12:13.128Z 25 Acked 1024 Read 8 Done Sent Done false
102362024-04-20T01:12:13.128Z 26 Acked 1025 Read 5 Done Sent Done false
102372024-04-20T01:12:13.129Z 27 Acked 1026 Read 5 Done Sent Done false
102382024-04-20T01:12:13.129Z 28 Acked 1027 Read 3 Done Sent Done false
102392024-04-20T01:12:13.129Z 29 Acked 1028 Write 8 Sent Sent Done false
102402024-04-20T01:12:13.129Z 30 Acked 1029 Write 3 Sent Sent Sent false
102412024-04-20T01:12:13.129Z STATES DS:0 DS:1 DS:2 TOTAL
102422024-04-20T01:12:13.129Z New 0 0 0 0
102432024-04-20T01:12:13.129Z Sent 2 27 1 30
102442024-04-20T01:12:13.129Z Done 28 3 29 60
102452024-04-20T01:12:13.129Z Skipped 0 0 0 0
102462024-04-20T01:12:13.129Z Error 0 0 0 0
102472024-04-20T01:12:13.129ZLast Flush: 0 0 0
102482024-04-20T01:12:13.129ZDownstairs last five completed:
102492024-04-20T01:12:13.130ZUpstairs last five completed: 30 29 28 27 26
102502024-04-20T01:12:13.130Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:12:11.614114061Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1547}
102512024-04-20T01:12:13.130ZWrote out file "/tmp/test_repair_verify.data"
102522024-04-20T01:12:13.134Z
102532024-04-20T01:12:13.134ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102542024-04-20T01:12:13.138ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102552024-04-20T01:12:13.138Z 0 000-019 91 91 91 153 153 153 T F T <---
102562024-04-20T01:12:13.139Z 1 020-039 91 91 91 153 153 153 F F F
102572024-04-20T01:12:13.139Z 2 040-059 92 92 92 154 154 154 F F F
102582024-04-20T01:12:13.139Z 3 060-079 89 89 89 150 150 150 F F F
102592024-04-20T01:12:13.139Z 4 080-099 93 93 93 156 156 156 T F T <---
102602024-04-20T01:12:13.139Z 5 100-119 88 88 88 149 149 149 F F F
102612024-04-20T01:12:13.139Z 6 120-139 87 87 87 146 146 146 F F F
102622024-04-20T01:12:13.139Z 7 140-159 87 87 87 146 146 146 T F T <---
102632024-04-20T01:12:13.139Z 8 160-179 91 91 91 153 153 153 F F F
102642024-04-20T01:12:13.139Z 9 180-199 90 90 90 152 152 152 F F F
102652024-04-20T01:12:13.139Z 10 200-219 93 93 93 156 156 156 F F F
102662024-04-20T01:12:13.139Z 11 220-239 92 92 92 154 154 154 T F T <---
102672024-04-20T01:12:13.139Z 12 240-259 93 93 93 156 156 156 T F T <---
102682024-04-20T01:12:13.139Z 13 260-279 92 92 92 155 155 155 F F F
102692024-04-20T01:12:13.139Z 14 280-299 88 88 88 148 148 148 F F F
102702024-04-20T01:12:13.140Z 15 300-319 91 91 91 153 153 153 F F F
102712024-04-20T01:12:13.140Z 16 320-339 91 91 91 153 153 153 F F F
102722024-04-20T01:12:13.140Z 17 340-359 89 89 89 150 150 150 F F F
102732024-04-20T01:12:13.140Z 18 360-379 90 90 90 152 152 152 F F F
102742024-04-20T01:12:13.140Z 19 380-399 93 93 93 156 156 156 T T T
102752024-04-20T01:12:13.140Z 20 400-419 92 92 92 154 154 154 T F T <---
102762024-04-20T01:12:13.140Z 21 420-439 90 90 90 151 151 151 F F F
102772024-04-20T01:12:13.140Z 22 440-459 89 89 89 150 150 150 T F T <---
102782024-04-20T01:12:13.140Z 23 460-479 90 90 90 152 152 152 F F F
102792024-04-20T01:12:13.140Z 24 480-499 86 86 86 145 145 145 F F F
102802024-04-20T01:12:13.140Z 25 500-519 90 90 90 151 151 151 T F T <---
102812024-04-20T01:12:13.140Z 26 520-539 92 92 92 154 154 154 T F T <---
102822024-04-20T01:12:13.140Z 27 540-559 91 91 91 153 153 153 F F F
102832024-04-20T01:12:13.140Z 28 560-579 80 80 80 134 134 134 F F F
102842024-04-20T01:12:13.140Z 29 580-599 84 84 84 141 141 141 T F T <---
102852024-04-20T01:12:13.140ZMax gen: 93, Max flush: 156
102862024-04-20T01:12:13.140ZError: Difference in extent metadata found!
102872024-04-20T01:12:13.144ZA Difference in extent metadata is expected here
102882024-04-20T01:12:13.144ZOn loop 47
102892024-04-20T01:12:13.144Z
102902024-04-20T01:12:13.144ZVerifying data now
102912024-04-20T01:12:13.666ZNo change after verify
102922024-04-20T01:12:13.666ZLoop: 47 Downstairs dump after verify (and repair):
102932024-04-20T01:12:13.679ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102942024-04-20T01:12:13.679Z 0 000-019 94 94 94 157 157 157 F F F
102952024-04-20T01:12:13.679Z 1 020-039 91 91 91 153 153 153 F F F
102962024-04-20T01:12:13.679Z 2 040-059 92 92 92 154 154 154 F F F
102972024-04-20T01:12:13.679Z 3 060-079 89 89 89 150 150 150 F F F
102982024-04-20T01:12:13.684Z 4 080-099 94 94 94 157 157 157 F F F
102992024-04-20T01:12:13.685Z 5 100-119 88 88 88 149 149 149 F F F
103002024-04-20T01:12:13.685Z 6 120-139 87 87 87 146 146 146 F F F
103012024-04-20T01:12:13.685Z 7 140-159 94 94 94 157 157 157 F F F
103022024-04-20T01:12:13.685Z 8 160-179 91 91 91 153 153 153 F F F
103032024-04-20T01:12:13.685Z 9 180-199 90 90 90 152 152 152 F F F
103042024-04-20T01:12:13.685Z 10 200-219 93 93 93 156 156 156 F F F
103052024-04-20T01:12:13.685Z 11 220-239 94 94 94 157 157 157 F F F
103062024-04-20T01:12:13.685Z 12 240-259 94 94 94 157 157 157 F F F
103072024-04-20T01:12:13.685Z 13 260-279 92 92 92 155 155 155 F F F
103082024-04-20T01:12:13.685Z 14 280-299 88 88 88 148 148 148 F F F
103092024-04-20T01:12:13.685Z 15 300-319 91 91 91 153 153 153 F F F
103102024-04-20T01:12:13.685Z 16 320-339 91 91 91 153 153 153 F F F
103112024-04-20T01:12:13.685Z 17 340-359 89 89 89 150 150 150 F F F
103122024-04-20T01:12:13.686Z 18 360-379 90 90 90 152 152 152 F F F
103132024-04-20T01:12:13.686Z 19 380-399 94 94 94 157 157 157 F F F
103142024-04-20T01:12:13.686Z 20 400-419 94 94 94 157 157 157 F F F
103152024-04-20T01:12:13.686Z 21 420-439 90 90 90 151 151 151 F F F
103162024-04-20T01:12:13.686Z 22 440-459 94 94 94 157 157 157 F F F
103172024-04-20T01:12:13.686Z 23 460-479 90 90 90 152 152 152 F F F
103182024-04-20T01:12:13.686Z 24 480-499 86 86 86 145 145 145 F F F
103192024-04-20T01:12:13.686Z 25 500-519 94 94 94 157 157 157 F F F
103202024-04-20T01:12:13.686Z 26 520-539 94 94 94 157 157 157 F F F
103212024-04-20T01:12:13.686Z 27 540-559 91 91 91 153 153 153 F F F
103222024-04-20T01:12:13.686Z 28 560-579 80 80 80 134 134 134 F F F
103232024-04-20T01:12:13.686Z 29 580-599 94 94 94 157 157 157 F F F
103242024-04-20T01:12:13.686ZMax gen: 94, Max flush: 157
103252024-04-20T01:12:13.686Z
103262024-04-20T01:12:13.686ZBegin loop 48
103272024-04-20T01:12:13.686ZDownstairs to restart: 0
103282024-04-20T01:12:13.702Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.204965166Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555}
103292024-04-20T01:12:13.702Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205193756Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555}
103302024-04-20T01:12:13.702Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205210836Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555}
103312024-04-20T01:12:13.702Z{"msg":"Crucible 74b709e2-8ed5-4da0-b643-cb321da225e1 has session id: 3e479a3c-7c23-4503-8bb4-0ca217b7979d","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205242777Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103322024-04-20T01:12:13.710Z{"msg":"Upstairs opts: Upstairs UUID: 74b709e2-8ed5-4da0-b643-cb321da225e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205289177Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103332024-04-20T01:12:13.710Z{"msg":"Crucible stats registered with UUID: 74b709e2-8ed5-4da0-b643-cb321da225e1","v":0,"name":"crucible","level":30Crucible runtime is spawned
103342024-04-20T01:12:13.710Z,"time":"2024-04-20T01:12:12.205318046Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103352024-04-20T01:12:13.710Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205411157Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555}
103362024-04-20T01:12:13.710Z{"msg":"74b709e2-8ed5-4da0-b643-cb321da225e1 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205463797Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103372024-04-20T01:12:13.711Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205495597Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"0","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103382024-04-20T01:12:13.711Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205520567Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"0","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103392024-04-20T01:12:13.711Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205549537Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"1","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103402024-04-20T01:12:13.711Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205593117Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"1","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103412024-04-20T01:12:13.711Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205615747Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"2","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103422024-04-20T01:12:13.711Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205641227Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"2","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103432024-04-20T01:12:13.711Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205655167Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103442024-04-20T01:12:13.711Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205672857Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103452024-04-20T01:12:13.711Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205706158Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103462024-04-20T01:12:13.711Z{"msg":"ds_connection connected from Ok(127.0.0.1:63253)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205795928Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103472024-04-20T01:12:13.711Z{"msg":"ds_connection connected from Ok(127.0.0.1:48984)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205843888Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103482024-04-20T01:12:13.711Z{"msg":"ds_connection connected from Ok(127.0.0.1:55735)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.205877548Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103492024-04-20T01:12:13.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.2066431Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"1","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103502024-04-20T01:12:13.711Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.206670649Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"1","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103512024-04-20T01:12:13.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.20668608Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"2","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103522024-04-20T01:12:13.711Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.20671631Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"2","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103532024-04-20T01:12:13.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.20673077Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"0","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103542024-04-20T01:12:13.711Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.20674589Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"0","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103552024-04-20T01:12:13.711Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.207195811Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"2","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103562024-04-20T01:12:13.711Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.207229381Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"1","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103572024-04-20T01:12:13.711Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.207316841Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"0","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103582024-04-20T01:12:13.711Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.207707882Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"2","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103592024-04-20T01:12:13.711Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.207754612Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"2","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103602024-04-20T01:12:13.712Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.207794782Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103612024-04-20T01:12:13.712Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.207851162Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"1","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103622024-04-20T01:12:13.712Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.207919632Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"1","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103632024-04-20T01:12:13.712Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.207985223Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103642024-04-20T01:12:13.712Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208028633Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"0","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103652024-04-20T01:12:13.712Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208045133Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"0","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103662024-04-20T01:12:13.712Z{"msg":"[0]R flush_numbers[0..12]: [157, 153, 154, 150, 157, 149, 146, 157, 153, 152, 156, 157]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208060693Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103672024-04-20T01:12:13.712Z{"msg":"[0]R generation[0..12]: [94, 91, 92, 89, 94, 88, 87, 94, 91, 90, 93, 94]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208075243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103682024-04-20T01:12:13.712Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208097513Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103692024-04-20T01:12:13.712Z{"msg":"[1]R flush_numbers[0..12]: [157, 153, 154, 150, 157, 149, 146, 157, 153, 152, 156, 157]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208111873Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103702024-04-20T01:12:13.712Z{"msg":"[1]R generation[0..12]: [94, 91, 92, 89, 94, 88, 87, 94, 91, 90, 93, 94]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
103712024-04-20T01:12:13.712Z,"time":"2024-04-20T01:12:12.208126903Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103722024-04-20T01:12:13.712Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208143833Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103732024-04-20T01:12:13.712Z{"msg":"[2]R flush_numbers[0..12]: [157, 153, 154, 150, 157, 149, 146, 157, 153, 152, 156, 157]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208165243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103742024-04-20T01:12:13.712Z{"msg":"[2]R generation[0..12]: [94, 91, 92, 89, 94, 88, 87, 94, 91, 90, 93, 94]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208179233Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103752024-04-20T01:12:13.712Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208193053Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103762024-04-20T01:12:13.712Z{"msg":"Max found gen is 95","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208213183Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103772024-04-20T01:12:13.712Z{"msg":"Generation requested: 96 >= found:95","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208226953Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103782024-04-20T01:12:13.712Z{"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208246923Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103792024-04-20T01:12:13.712Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208260303Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"":"downstairs","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103802024-04-20T01:12:13.712Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208273263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103812024-04-20T01:12:13.712Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208292063Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"0","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103822024-04-20T01:12:13.712Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208305623Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"1","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103832024-04-20T01:12:13.713Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208318423Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"client":"2","":"downstairs","session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103842024-04-20T01:12:13.713Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208332013Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103852024-04-20T01:12:13.713Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208344133Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103862024-04-20T01:12:13.713Z{"msg":"74b709e2-8ed5-4da0-b643-cb321da225e1 is now active with session: 3e479a3c-7c23-4503-8bb4-0ca217b7979d","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208356723Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103872024-04-20T01:12:13.713Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208368893Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555,"session_id":"3e479a3c-7c23-4503-8bb4-0ca217b7979d"}
103882024-04-20T01:12:13.713Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:12.208380963Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555}
103892024-04-20T01:12:13.713ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103902024-04-20T01:12:13.713ZLoading write count information from file "/tmp/test_repair_verify.data"
103912024-04-20T01:12:13.713ZRun Repair workload
103922024-04-20T01:12:13.713Z01/30 Flush
103932024-04-20T01:12:13.717Z02/30 Write block 336 len 12288 data: 5 6 8
103942024-04-20T01:12:13.717Z03/30 Read block 175 len 8192
103952024-04-20T01:12:13.717Z04/30 Write block 496 len 8192 data: 6 5
103962024-04-20T01:12:13.717Z05/30 Write block 195 len 32768 data: 7 5 8 9 9 6 7 10
103972024-04-20T01:12:13.717Z06/30 Write block 331 len 36864 data: 10 11 9 9 7 6 7 9 8
103982024-04-20T01:12:13.717Z07/30 Write block 302 len 32768 data: 7 7 7 6 5 4 4 3
103992024-04-20T01:12:13.717Z08/30 Write block 173 len 28672 data: 7 6 6 7 8 11 13
104002024-04-20T01:12:13.717Z09/30 Write block 66 len 16384 data: 11 9 10 11
104012024-04-20T01:12:13.725Z10/30 Read block 448 len 20480
104022024-04-20T01:12:13.725Z11/30 Write block 386 len 4096 data: 5
104032024-04-20T01:12:13.725Z12/30 Read block 31 len 12288
104042024-04-20T01:12:13.725Z13/30 Write block 108 len 8192 data: 7 6
104052024-04-20T01:12:13.725Z14/30 Write block 444 len 32768 data: 9 8 7 7 7 9 9 9
104062024-04-20T01:12:13.725Z15/30 Read block 460 len 16384
104072024-04-20T01:12:13.729Z16/30 Read block 222 len 28672
104082024-04-20T01:12:13.729Z17/30 Write block 266 len 12288 data: 3 3 2
104092024-04-20T01:12:13.729Z18/30 Write block 371 len 40960 data: 14 13 14 13 12 8 9 9 10 11
104102024-04-20T01:12:13.729Z19/30 Read block 513 len 16384
104112024-04-20T01:12:13.729Z20/30 Write block 467 len 36864 data: 11 9 11 11 10 10 11 11 12
104122024-04-20T01:12:13.729Z21/30 Write block 282 len 32768 data: 5 4 5 5 5 5 5 6
104132024-04-20T01:12:13.733Z22/30 Write block 287 len 40960 data: 6 6 7 5 5 6 6 8 5 5
104142024-04-20T01:12:13.733Z23/30 Read block 145 len 24576
104152024-04-20T01:12:13.733Z24/30 Write block 295 len 20480 data: 6 6 6 4 4
104162024-04-20T01:12:13.733Z25/30 Read block 80 len 24576
104172024-04-20T01:12:13.736Z26/30 Flush
104182024-04-20T01:12:13.739Z27/30 Write block 161 len 4096 data: 9
104192024-04-20T01:12:13.739Z28/30 Read block 127 len 36864
104202024-04-20T01:12:13.739Z29/30 Read block 117 len 12288
104212024-04-20T01:12:13.743Z30/30 Write block 408 len 16384 data: 8 9 9 11
104222024-04-20T01:12:13.743Z----------------------------------------------------------------
104232024-04-20T01:12:13.743Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:29
104242024-04-20T01:12:13.744ZStates: Active Active Active
104252024-04-20T01:12:13.744ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
104262024-04-20T01:12:13.744Z 2 Acked 1001 Write 3 Done Done Done false
104272024-04-20T01:12:13.744Z 3 Acked 1002 Read 2 Done Done Done false
104282024-04-20T01:12:13.744Z 4 Acked 1003 Write 2 Done Done Done false
104292024-04-20T01:12:13.744Z 5 Acked 1004 Write 8 Done Done Done false
104302024-04-20T01:12:13.744Z 6 Acked 1005 Write 9 Sent Done Done false
104312024-04-20T01:12:13.744Z 7 Acked 1006 Write 8 Sent Done Done false
104322024-04-20T01:12:13.744Z 8 Acked 1007 Write 7 Sent Done Done false
104332024-04-20T01:12:13.744Z 9 Acked 1008 Write 4 Sent Done Done false
104342024-04-20T01:12:13.744Z 10 Acked 1009 Read 5 Sent Done Done false
104352024-04-20T01:12:13.744Z 11 Acked 1010 Write 1 Sent Done Done false
104362024-04-20T01:12:13.744Z 12 Acked 1011 Read 3 Sent Done Done false
104372024-04-20T01:12:13.744Z 13 Acked 1012 Write 2 Sent Done Done false
104382024-04-20T01:12:13.744Z 14 Acked 1013 Write 8 Sent Done Done false
104392024-04-20T01:12:13.744Z 15 Acked 1014 Read 4 Sent Done Done false
104402024-04-20T01:12:13.744Z 16 Acked 1015 Read 7 Sent Done Done false
104412024-04-20T01:12:13.744Z 17 Acked 1016 Write 3 Sent Done Done false
104422024-04-20T01:12:13.744Z 18 Acked 1017 Write 10 Sent Done Done false
104432024-04-20T01:12:13.744Z 19 Acked 1018 Read 4 Sent Done Done false
104442024-04-20T01:12:13.744Z 20 Acked 1019 Write 9 Sent Done Done false
104452024-04-20T01:12:13.744Z 21 Acked 1020 Write 8 Sent Done Done false
104462024-04-20T01:12:13.744Z 22 Acked 1021 Write 10 Sent Done Done false
104472024-04-20T01:12:13.744Z 23 Acked 1022 Read 6 Sent Done Done false
104482024-04-20T01:12:13.745Z 24 Acked 1023 Write 5 Sent Done Done false
104492024-04-20T01:12:13.745Z 25 Acked 1024 Read 6 Sent Done Done false
104502024-04-20T01:12:13.745Z 26 Acked 1025 Flush 0 Sent Done Done false
104512024-04-20T01:12:13.745Z 27 Acked 1026 Write 1 Sent Done Done false
104522024-04-20T01:12:13.745Z 28 Acked 1027 Read 9 Sent Done Done false
104532024-04-20T01:12:13.745Z 29 Acked 1028 Read 3 Sent Done Done false
104542024-04-20T01:12:13.745Z 30 Acked 1029 Write 4 Sent Sent Sent false
104552024-04-20T01:12:13.745Z STATES DS:0 DS:1 DS:2 TOTAL
104562024-04-20T01:12:13.745Z New 0 0 0 0
104572024-04-20T01:12:13.745Z Sent 25 1 1 27
104582024-04-20T01:12:13.745Z Done 4 28 28 60
104592024-04-20T01:12:13.745Z Skipped 0 0 0 0
104602024-04-20T01:12:13.745Z Error 0 0 0 0
104612024-04-20T01:12:13.745ZLast Flush: 1000 1025 1025
104622024-04-20T01:12:13.745ZDownstairs last five completed: 1000
104632024-04-20T01:12:13.745ZUpstairs last five completed: 30 29 28 27 26
104642024-04-20T01:12:13.745Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:12:12.243997183Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1555}
104652024-04-20T01:12:13.745ZWrote out file "/tmp/test_repair_verify.data"
104662024-04-20T01:12:13.749Z
104672024-04-20T01:12:13.749ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104682024-04-20T01:12:13.763ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104692024-04-20T01:12:13.763Z 0 000-019 94 94 94 157 157 157 F F F
104702024-04-20T01:12:13.763Z 1 020-039 91 91 91 153 153 153 F F F
104712024-04-20T01:12:13.763Z 2 040-059 92 92 92 154 154 154 F F F
104722024-04-20T01:12:13.763Z 3 060-079 89 96 96 150 159 159 F F F <---
104732024-04-20T01:12:13.763Z 4 080-099 94 94 94 157 157 157 F F F
104742024-04-20T01:12:13.767Z 5 100-119 88 96 96 149 159 159 F F F <---
104752024-04-20T01:12:13.767Z 6 120-139 87 87 87 146 146 146 F F F
104762024-04-20T01:12:13.767Z 7 140-159 94 94 94 157 157 157 F F F
104772024-04-20T01:12:13.767Z 8 160-179 91 96 96 153 159 159 F T T <---
104782024-04-20T01:12:13.767Z 9 180-199 90 96 96 152 159 159 T F F <---
104792024-04-20T01:12:13.767Z 10 200-219 93 96 96 156 159 159 T F F <---
104802024-04-20T01:12:13.767Z 11 220-239 94 94 94 157 157 157 F F F
104812024-04-20T01:12:13.767Z 12 240-259 94 94 94 157 157 157 F F F
104822024-04-20T01:12:13.767Z 13 260-279 92 96 96 155 159 159 F F F <---
104832024-04-20T01:12:13.767Z 14 280-299 88 96 96 148 159 159 F F F <---
104842024-04-20T01:12:13.768Z 15 300-319 91 96 96 153 159 159 F F F <---
104852024-04-20T01:12:13.768Z 16 320-339 91 96 96 153 159 159 T F F <---
104862024-04-20T01:12:13.768Z 17 340-359 89 89 89 150 150 150 F F F
104872024-04-20T01:12:13.768Z 18 360-379 90 96 96 152 159 159 F F F <---
104882024-04-20T01:12:13.768Z 19 380-399 94 96 96 157 159 159 F F F <---
104892024-04-20T01:12:13.768Z 20 400-419 94 94 94 157 157 157 F T T <---
104902024-04-20T01:12:13.768Z 21 420-439 90 90 90 151 151 151 F F F
104912024-04-20T01:12:13.768Z 22 440-459 94 96 96 157 159 159 F F F <---
104922024-04-20T01:12:13.768Z 23 460-479 90 96 96 152 159 159 F F F <---
104932024-04-20T01:12:13.768Z 24 480-499 86 96 96 145 159 159 T F F <---
104942024-04-20T01:12:13.768Z 25 500-519 94 94 94 157 157 157 F F F
104952024-04-20T01:12:13.768Z 26 520-539 94 94 94 157 157 157 F F F
104962024-04-20T01:12:13.768Z 27 540-559 91 91 91 153 153 153 F F F
104972024-04-20T01:12:13.768Z 28 560-579 80 80 80 134 134 134 F F F
104982024-04-20T01:12:13.768Z 29 580-599 94 94 94 157 157 157 F F F
104992024-04-20T01:12:13.768ZMax gen: 96, Max flush: 159
105002024-04-20T01:12:13.768ZError: Difference in extent metadata found!
105012024-04-20T01:12:13.768ZA Difference in extent metadata is expected here
105022024-04-20T01:12:13.768ZOn loop 48
105032024-04-20T01:12:13.768Z
105042024-04-20T01:12:13.772ZVerifying data now
105052024-04-20T01:12:28.435ZNo change after verify
105062024-04-20T01:12:28.435ZLoop: 48 Downstairs dump after verify (and repair):
105072024-04-20T01:12:28.449ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105082024-04-20T01:12:28.449Z 0 000-019 94 94 94 157 157 157 F F F
105092024-04-20T01:12:28.449Z 1 020-039 91 91 91 153 153 153 F F F
105102024-04-20T01:12:28.449Z 2 040-059 92 92 92 154 154 154 F F F
105112024-04-20T01:12:28.449Z 3 060-079 96 96 96 159 159 159 F F F
105122024-04-20T01:12:28.449Z 4 080-099 94 94 94 157 157 157 F F F
105132024-04-20T01:12:28.449Z 5 100-119 96 96 96 159 159 159 F F F
105142024-04-20T01:12:28.449Z 6 120-139 87 87 87 146 146 146 F F F
105152024-04-20T01:12:28.450Z 7 140-159 94 94 94 157 157 157 F F F
105162024-04-20T01:12:28.450Z 8 160-179 97 97 97 160 160 160 F F F
105172024-04-20T01:12:28.450Z 9 180-199 96 96 96 159 159 159 F F F
105182024-04-20T01:12:28.450Z 10 200-219 96 96 96 159 159 159 F F F
105192024-04-20T01:12:28.453Z 11 220-239 94 94 94 157 157 157 F F F
105202024-04-20T01:12:28.454Z 12 240-259 94 94 94 157 157 157 F F F
105212024-04-20T01:12:28.454Z 13 260-279 96 96 96 159 159 159 F F F
105222024-04-20T01:12:28.454Z 14 280-299 96 96 96 159 159 159 F F F
105232024-04-20T01:12:28.454Z 15 300-319 96 96 96 159 159 159 F F F
105242024-04-20T01:12:28.454Z 16 320-339 96 96 96 159 159 159 F F F
105252024-04-20T01:12:28.454Z 17 340-359 89 89 89 150 150 150 F F F
105262024-04-20T01:12:28.454Z 18 360-379 96 96 96 159 159 159 F F F
105272024-04-20T01:12:28.454Z 19 380-399 96 96 96 159 159 159 F F F
105282024-04-20T01:12:28.454Z 20 400-419 97 97 97 160 160 160 F F F
105292024-04-20T01:12:28.454Z 21 420-439 90 90 90 151 151 151 F F F
105302024-04-20T01:12:28.454Z 22 440-459 96 96 96 159 159 159 F F F
105312024-04-20T01:12:28.454Z 23 460-479 96 96 96 159 159 159 F F F
105322024-04-20T01:12:28.454Z 24 480-499 96 96 96 159 159 159 F F F
105332024-04-20T01:12:28.454Z 25 500-519 94 94 94 157 157 157 F F F
105342024-04-20T01:12:28.454Z 26 520-539 94 94 94 157 157 157 F F F
105352024-04-20T01:12:28.454Z 27 540-559 91 91 91 153 153 153 F F F
105362024-04-20T01:12:28.454Z 28 560-579 80 80 80 134 134 134 F F F
105372024-04-20T01:12:28.454Z 29 580-599 94 94 94 157 157 157 F F F
105382024-04-20T01:12:28.454ZMax gen: 97, Max flush: 160
105392024-04-20T01:12:28.454Z
105402024-04-20T01:12:28.454ZBegin loop 49
105412024-04-20T01:12:28.454ZDownstairs to restart: 1
105422024-04-20T01:12:28.471Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.974404662Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563}
105432024-04-20T01:12:28.478Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.974662473Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563}
105442024-04-20T01:12:28.478Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.974693123Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563}
105452024-04-20T01:12:28.478Z{"msg":"Crucible 5ac613f3-add5-458c-a6ae-9a6b1e3c104e has session id: aa3aa132-c2f2-42c5-9b12-1d68d6a5412c","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.974714843Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105462024-04-20T01:12:28.478Z{"msg":"Upstairs opts: Upstairs UUID: 5ac613f3-add5-458c-a6ae-9a6b1e3c104e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false"Crucible runtime is spawned
105472024-04-20T01:12:28.478Z,"v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.974830103Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105482024-04-20T01:12:28.478Z{"msg":"Crucible stats registered with UUID: 5ac613f3-add5-458c-a6ae-9a6b1e3c104e","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.974863674Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105492024-04-20T01:12:28.478Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.974887814Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563}
105502024-04-20T01:12:28.478Z{"msg":"5ac613f3-add5-458c-a6ae-9a6b1e3c104e active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975017494Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105512024-04-20T01:12:28.478Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975053424Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"0","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105522024-04-20T01:12:28.479Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975081464Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"0","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105532024-04-20T01:12:28.479Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975118744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"1","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105542024-04-20T01:12:28.479Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975165444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"1","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105552024-04-20T01:12:28.479Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975191794Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"2","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105562024-04-20T01:12:28.479Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975214754Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"2","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105572024-04-20T01:12:28.479Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975239414Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105582024-04-20T01:12:28.479Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975276544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105592024-04-20T01:12:28.479Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975302875Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105602024-04-20T01:12:28.479Z{"msg":"ds_connection connected from Ok(127.0.0.1:46315)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975459845Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105612024-04-20T01:12:28.479Z{"msg":"ds_connection connected from Ok(127.0.0.1:51321)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975505225Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105622024-04-20T01:12:28.479Z{"msg":"ds_connection connected from Ok(127.0.0.1:39696)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.975533815Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105632024-04-20T01:12:28.479Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.976219546Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"0","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105642024-04-20T01:12:28.479Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.976255667Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"0","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105652024-04-20T01:12:28.479Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.976270877Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"1","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105662024-04-20T01:12:28.479Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.976297897Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"1","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105672024-04-20T01:12:28.479Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.976313857Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"2","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105682024-04-20T01:12:28.479Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.976335707Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"2","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105692024-04-20T01:12:28.479Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.976828228Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"0","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105702024-04-20T01:12:28.479Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.976866698Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"1","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105712024-04-20T01:12:28.479Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.976962328Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"2","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105722024-04-20T01:12:28.479Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.977427929Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"0","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105732024-04-20T01:12:28.479Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.977463349Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"0","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105742024-04-20T01:12:28.479Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.977478059Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105752024-04-20T01:12:28.480Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97750902Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"1","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105762024-04-20T01:12:28.480Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.977541229Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"1","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105772024-04-20T01:12:28.480Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97756032Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105782024-04-20T01:12:28.480Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97758808Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"2","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105792024-04-20T01:12:28.480Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.9776086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"2","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105802024-04-20T01:12:28.480Z{"msg":"[0]R flush_numbers[0..12]: [157, 153, 154, 159, 157, 159, 146, 157, 160, 159, 159, 157]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97765228Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105812024-04-20T01:12:28.480Z{"msg":"[0]R generation[0..12]: [94, 91, 92, 96, 94, 96, 87, 94, 97, 96, 96, 94]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97768642Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105822024-04-20T01:12:28.480ZWait for a query_work_queue command to finish before sending IO
105832024-04-20T01:12:28.480Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97772393Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105842024-04-20T01:12:28.480Z{"msg":"[1]R flush_numbers[0..12]: [157, 153, 154, 159, 157, 159, 146, 157, 160, 159, 159, 157]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.9777581Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105852024-04-20T01:12:28.480Z{"msg":"[1]R generation[0..12]: [94, 91, 92, 96, 94, 96, 87, 94, 97, 96, 96, 94]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97779107Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105862024-04-20T01:12:28.480Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97782339Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105872024-04-20T01:12:28.480Z{"msg":"[2]R flush_numbers[0..12]: [157, 153, 154, 159, 157, 159, 146, 157, 160, 159, 159, 157]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97785513Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105882024-04-20T01:12:28.480Z{"msg":"[2]R generation[0..12]: [94, 91, 92, 96, 94, 96, 87, 94, 97, 96, 96, 94]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97788787Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105892024-04-20T01:12:28.480Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97791977Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105902024-04-20T01:12:28.480Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97795117Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105912024-04-20T01:12:28.485Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.97798085Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105922024-04-20T01:12:28.485Z{"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.978037661Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105932024-04-20T01:12:28.485Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.978054771Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"":"downstairs","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105942024-04-20T01:12:28.485Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.978071831Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105952024-04-20T01:12:28.485ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
105962024-04-20T01:12:28.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.978104031Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"0","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105972024-04-20T01:12:28.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.978120251Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"1","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105982024-04-20T01:12:28.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.978151111Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"client":"2","":"downstairs","session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
105992024-04-20T01:12:28.485Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.978171621Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
106002024-04-20T01:12:28.485Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.978198531Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
106012024-04-20T01:12:28.485Z{"msg":"5ac613f3-add5-458c-a6ae-9a6b1e3c104e is now active with session: aa3aa132-c2f2-42c5-9b12-1d68d6a5412c","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.978216661Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
106022024-04-20T01:12:28.485Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.978243871Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563,"session_id":"aa3aa132-c2f2-42c5-9b12-1d68d6a5412c"}
106032024-04-20T01:12:28.485Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:26.978257851Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563}
106042024-04-20T01:12:28.485ZLoading write count information from file "/tmp/test_repair_verify.data"
106052024-04-20T01:12:28.485ZRun Repair workload
106062024-04-20T01:12:28.485Z01/30 Write block 32 len 32768 data: 7 7 10 10 8 9 10 10
106072024-04-20T01:12:28.485Z02/30 Read block 414 len 24576
106082024-04-20T01:12:28.485Z03/30 Write block 140 len 40960 data: 11 11 11 11 11 11 10 9 8 6
106092024-04-20T01:12:28.486Z04/30 Write block 60 len 40960 data: 10 9 9 7 9 9 12 10 11 12
106102024-04-20T01:12:28.486Z05/30 Read block 15 len 28672
106112024-04-20T01:12:28.486Z06/30 Write block 245 len 20480 data: 11 10 9 13 13
106122024-04-20T01:12:28.486Z07/30 Write block 510 len 8192 data: 11 11
106132024-04-20T01:12:28.486Z08/30 Read block 331 len 32768
106142024-04-20T01:12:28.489Z09/30 Write block 575 len 40960 data: 3 5 7 7 10 10 10 10 7 9
106152024-04-20T01:12:28.489Z10/30 Read block 217 len 16384
106162024-04-20T01:12:28.489Z11/30 Write block 172 len 40960 data: 5 8 7 7 8 9 12 14 13 14
106172024-04-20T01:12:28.489Z12/30 Read block 185 len 4096
106182024-04-20T01:12:28.489Z13/30 Write block 170 len 8192 data: 3 4
106192024-04-20T01:12:28.493Z14/30 Read block 431 len 4096
106202024-04-20T01:12:28.493Z15/30 Flush
106212024-04-20T01:12:28.496Z16/30 Flush
106222024-04-20T01:12:28.496Z17/30 Write block 542 len 20480 data: 10 11 9 7 7
106232024-04-20T01:12:28.497Z18/30 Read block 557 len 12288
106242024-04-20T01:12:28.497Z19/30 Write block 345 len 4096 data: 10
106252024-04-20T01:12:28.500Z20/30 Write block 541 len 40960 data: 8 11 12 10 8 8 5 6 8 10
106262024-04-20T01:12:28.500Z21/30 Flush
106272024-04-20T01:12:28.503Z22/30 Write block 31 len 12288 data: 7 8 8
106282024-04-20T01:12:28.504Z23/30 Read block 497 len 40960
106292024-04-20T01:12:28.504Z24/30 Flush
106302024-04-20T01:12:28.507Z25/30 Write block 225 len 28672 data: 9 13 8 10 7 8 11
106312024-04-20T01:12:28.507Z26/30 Flush
106322024-04-20T01:12:28.510Z27/30 Write block 299 len 40960 data: 5 5 7 8 8 8 7 6 5 5
106332024-04-20T01:12:28.510Z28/30 Read block 403 len 32768
106342024-04-20T01:12:28.514Z29/30 Write block 548 len 40960 data: 7 9 11 10 8 6 8 8 6 7
106352024-04-20T01:12:28.514Z30/30 Read block 254 len 24576
106362024-04-20T01:12:28.514Z----------------------------------------------------------------
106372024-04-20T01:12:28.515Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
106382024-04-20T01:12:28.515ZStates: Active Active Active
106392024-04-20T01:12:28.515ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106402024-04-20T01:12:28.515Z 1 Acked 1000 Write 8 Done Done Done false
106412024-04-20T01:12:28.515Z 2 Acked 1001 Read 6 Done Done Done false
106422024-04-20T01:12:28.515Z 3 Acked 1002 Write 10 Done Done Done false
106432024-04-20T01:12:28.515Z 4 Acked 1003 Write 10 Done Sent Done false
106442024-04-20T01:12:28.515Z 5 Acked 1004 Read 7 Done Sent Done false
106452024-04-20T01:12:28.515Z 6 Acked 1005 Write 5 Done Sent Done false
106462024-04-20T01:12:28.515Z 7 Acked 1006 Write 2 Done Sent Done false
106472024-04-20T01:12:28.515Z 8 Acked 1007 Read 8 Done Sent Done false
106482024-04-20T01:12:28.515Z 9 Acked 1008 Write 10 Done Sent Done false
106492024-04-20T01:12:28.515Z 10 Acked 1009 Read 4 Done Sent Done false
106502024-04-20T01:12:28.515Z 11 Acked 1010 Write 10 Done Sent Done false
106512024-04-20T01:12:28.515Z 12 Acked 1011 Read 1 Done Sent Done false
106522024-04-20T01:12:28.515Z 13 Acked 1012 Write 2 Done Sent Done false
106532024-04-20T01:12:28.515Z 14 Acked 1013 Read 1 Done Sent Done false
106542024-04-20T01:12:28.515Z 15 Acked 1014 Flush 0 Done Sent Done false
106552024-04-20T01:12:28.515Z 16 Acked 1015 Flush 0 Done Sent Done false
106562024-04-20T01:12:28.515Z 17 Acked 1016 Write 5 Done Sent Done false
106572024-04-20T01:12:28.515Z 18 Acked 1017 Read 3 Done Sent Done false
106582024-04-20T01:12:28.515Z 19 Acked 1018 Write 1 Done Sent Done false
106592024-04-20T01:12:28.515Z 20 Acked 1019 Write 10 Done Sent Done false
106602024-04-20T01:12:28.515Z 21 Acked 1020 Flush 0 Done Sent Done false
106612024-04-20T01:12:28.516Z 22 Acked 1021 Write 3 Done Sent Done false
106622024-04-20T01:12:28.516Z 23 Acked 1022 Read 10 Done Sent Done false
106632024-04-20T01:12:28.516Z 24 Acked 1023 Flush 0 Done Sent Done false
106642024-04-20T01:12:28.516Z 25 Acked 1024 Write 7 Done Sent Done false
106652024-04-20T01:12:28.516Z 26 Acked 1025 Flush 0 Done Sent Done false
106662024-04-20T01:12:28.516Z 27 Acked 1026 Write 10 Done Sent Done false
106672024-04-20T01:12:28.516Z 28 Acked 1027 Read 8 Done Sent Done false
106682024-04-20T01:12:28.516Z 29 Acked 1028 Write 10 Done Sent Done false
106692024-04-20T01:12:28.516Z 30 Acked 1029 Read 6 Done Sent Sent false
106702024-04-20T01:12:28.516Z STATES DS:0 DS:1 DS:2 TOTAL
106712024-04-20T01:12:28.516Z New 0 0 0 0
106722024-04-20T01:12:28.516Z Sent 0 27 1 28
106732024-04-20T01:12:28.516Z Done 30 3 29 62
106742024-04-20T01:12:28.516Z Skipped 0 0 0 0
106752024-04-20T01:12:28.516Z Error 0 0 0 0
106762024-04-20T01:12:28.516ZLast Flush: 1025 0 1025
106772024-04-20T01:12:28.516ZDownstairs last five completed:
106782024-04-20T01:12:28.516ZUpstairs last five completed: 30 29 28 27 26
106792024-04-20T01:12:28.516Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:12:27.016262776Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1563}
106802024-04-20T01:12:28.516ZWrote out file "/tmp/test_repair_verify.data"
106812024-04-20T01:12:28.520Z
106822024-04-20T01:12:28.520ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106832024-04-20T01:12:28.537ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106842024-04-20T01:12:28.537Z 0 000-019 94 94 94 157 157 157 F F F
106852024-04-20T01:12:28.537Z 1 020-039 98 91 98 164 153 164 F T F <---
106862024-04-20T01:12:28.537Z 2 040-059 92 92 92 154 154 154 F F F
106872024-04-20T01:12:28.537Z 3 060-079 98 96 98 161 159 161 F F F <---
106882024-04-20T01:12:28.537Z 4 080-099 94 94 94 157 157 157 F F F
106892024-04-20T01:12:28.541Z 5 100-119 96 96 96 159 159 159 F F F
106902024-04-20T01:12:28.541Z 6 120-139 87 87 87 146 146 146 F F F
106912024-04-20T01:12:28.541Z 7 140-159 98 94 98 161 157 161 F T F <---
106922024-04-20T01:12:28.541Z 8 160-179 98 97 98 161 160 161 F F F <---
106932024-04-20T01:12:28.541Z 9 180-199 98 96 98 161 159 161 F F F <---
106942024-04-20T01:12:28.541Z 10 200-219 96 96 96 159 159 159 F F F
106952024-04-20T01:12:28.541Z 11 220-239 98 94 98 165 157 165 F F F <---
106962024-04-20T01:12:28.541Z 12 240-259 98 94 98 161 157 161 F F F <---
106972024-04-20T01:12:28.541Z 13 260-279 96 96 96 159 159 159 F F F
106982024-04-20T01:12:28.541Z 14 280-299 96 96 96 159 159 159 T F T <---
106992024-04-20T01:12:28.541Z 15 300-319 96 96 96 159 159 159 T F T <---
107002024-04-20T01:12:28.541Z 16 320-339 96 96 96 159 159 159 F F F
107012024-04-20T01:12:28.541Z 17 340-359 98 89 98 163 150 163 F F F <---
107022024-04-20T01:12:28.541Z 18 360-379 96 96 96 159 159 159 F F F
107032024-04-20T01:12:28.541Z 19 380-399 96 96 96 159 159 159 F F F
107042024-04-20T01:12:28.541Z 20 400-419 97 97 97 160 160 160 F F F
107052024-04-20T01:12:28.541Z 21 420-439 90 90 90 151 151 151 F F F
107062024-04-20T01:12:28.541Z 22 440-459 96 96 96 159 159 159 F F F
107072024-04-20T01:12:28.541Z 23 460-479 96 96 96 159 159 159 F F F
107082024-04-20T01:12:28.541Z 24 480-499 96 96 96 159 159 159 F F F
107092024-04-20T01:12:28.541Z 25 500-519 98 94 98 161 157 161 F F F <---
107102024-04-20T01:12:28.541Z 26 520-539 94 94 94 157 157 157 F F F
107112024-04-20T01:12:28.542Z 27 540-559 98 91 98 163 153 163 T F T <---
107122024-04-20T01:12:28.542Z 28 560-579 98 80 98 161 134 161 F F F <---
107132024-04-20T01:12:28.542Z 29 580-599 98 94 98 161 157 161 F F F <---
107142024-04-20T01:12:28.542ZMax gen: 98, Max flush: 165
107152024-04-20T01:12:28.542ZError: Difference in extent metadata found!
107162024-04-20T01:12:28.542ZA Difference in extent metadata is expected here
107172024-04-20T01:12:28.542ZOn loop 49
107182024-04-20T01:12:28.542Z
107192024-04-20T01:12:28.545ZVerifying data now
107202024-04-20T01:12:43.166ZNo change after verify
107212024-04-20T01:12:43.166ZLoop: 49 Downstairs dump after verify (and repair):
107222024-04-20T01:12:43.180ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107232024-04-20T01:12:43.180Z 0 000-019 94 94 94 157 157 157 F F F
107242024-04-20T01:12:43.180Z 1 020-039 98 98 98 164 164 164 F F F
107252024-04-20T01:12:43.180Z 2 040-059 92 92 92 154 154 154 F F F
107262024-04-20T01:12:43.180Z 3 060-079 98 98 98 161 161 161 F F F
107272024-04-20T01:12:43.180Z 4 080-099 94 94 94 157 157 157 F F F
107282024-04-20T01:12:43.180Z 5 100-119 96 96 96 159 159 159 F F F
107292024-04-20T01:12:43.180Z 6 120-139 87 87 87 146 146 146 F F F
107302024-04-20T01:12:43.184Z 7 140-159 98 98 98 161 161 161 F F F
107312024-04-20T01:12:43.184Z 8 160-179 98 98 98 161 161 161 F F F
107322024-04-20T01:12:43.184Z 9 180-199 98 98 98 161 161 161 F F F
107332024-04-20T01:12:43.184Z 10 200-219 96 96 96 159 159 159 F F F
107342024-04-20T01:12:43.184Z 11 220-239 98 98 98 165 165 165 F F F
107352024-04-20T01:12:43.184Z 12 240-259 98 98 98 161 161 161 F F F
107362024-04-20T01:12:43.184Z 13 260-279 96 96 96 159 159 159 F F F
107372024-04-20T01:12:43.184Z 14 280-299 99 99 99 166 166 166 F F F
107382024-04-20T01:12:43.184Z 15 300-319 99 99 99 166 166 166 F F F
107392024-04-20T01:12:43.184Z 16 320-339 96 96 96 159 159 159 F F F
107402024-04-20T01:12:43.184Z 17 340-359 98 98 98 163 163 163 F F F
107412024-04-20T01:12:43.184Z 18 360-379 96 96 96 159 159 159 F F F
107422024-04-20T01:12:43.185Z 19 380-399 96 96 96 159 159 159 F F F
107432024-04-20T01:12:43.185Z 20 400-419 97 97 97 160 160 160 F F F
107442024-04-20T01:12:43.185Z 21 420-439 90 90 90 151 151 151 F F F
107452024-04-20T01:12:43.185Z 22 440-459 96 96 96 159 159 159 F F F
107462024-04-20T01:12:43.185Z 23 460-479 96 96 96 159 159 159 F F F
107472024-04-20T01:12:43.185Z 24 480-499 96 96 96 159 159 159 F F F
107482024-04-20T01:12:43.185Z 25 500-519 98 98 98 161 161 161 F F F
107492024-04-20T01:12:43.185Z 26 520-539 94 94 94 157 157 157 F F F
107502024-04-20T01:12:43.185Z 27 540-559 99 99 99 166 166 166 F F F
107512024-04-20T01:12:43.185Z 28 560-579 98 98 98 161 161 161 F F F
107522024-04-20T01:12:43.185Z 29 580-599 98 98 98 161 161 161 F F F
107532024-04-20T01:12:43.185ZMax gen: 99, Max flush: 166
107542024-04-20T01:12:43.185Z
107552024-04-20T01:12:43.185ZBegin loop 50
107562024-04-20T01:12:43.185ZDownstairs to restart: 1
107572024-04-20T01:12:43.202Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.704702212Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571}
107582024-04-20T01:12:43.202Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.704916483Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571}
107592024-04-20T01:12:43.202Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.704933263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571}
107602024-04-20T01:12:43.202Z{"msg":"Crucible e2fa5edc-de2e-470a-ab5f-bee80b23ad75 has session id: 01fe423d-dac8-4c8f-aed6-e0e3ca5fc256","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.704947613Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107612024-04-20T01:12:43.210Z{"msg":"Upstairs opts: Upstairs UUID: e2fa5edc-de2e-470a-ab5f-bee80b23ad75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.704984853Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107622024-04-20T01:12:43.210Z{"msg":"Crucible stats registered with UUID: e2fa5edc-de2e-470a-ab5f-bee80b23ad75","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705011713Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107632024-04-20T01:12:43.211ZCrucible runtime is spawned
107642024-04-20T01:12:43.211Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705119773Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571}
107652024-04-20T01:12:43.211Z{"msg":"e2fa5edc-de2e-470a-ab5f-bee80b23ad75 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705163243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107662024-04-20T01:12:43.211Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705178793Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"0","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107672024-04-20T01:12:43.211Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705204003Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"0","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107682024-04-20T01:12:43.211Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705239913Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"1","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107692024-04-20T01:12:43.211Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705264663Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"1","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107702024-04-20T01:12:43.211Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705288803Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"2","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107712024-04-20T01:12:43.211Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705313724Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"2","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107722024-04-20T01:12:43.211Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705360493Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107732024-04-20T01:12:43.211Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705386354Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107742024-04-20T01:12:43.211Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705411954Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107752024-04-20T01:12:43.211Z{"msg":"ds_connection connected from Ok(127.0.0.1:33761)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705661544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107762024-04-20T01:12:43.211Z{"msg":"ds_connection connected from Ok(127.0.0.1:62745)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705695624Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107772024-04-20T01:12:43.211Z{"msg":"ds_connection connected from Ok(127.0.0.1:48479)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.705711974Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107782024-04-20T01:12:43.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.706455556Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"2","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107792024-04-20T01:12:43.211Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.706504316Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"2","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107802024-04-20T01:12:43.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.706531936Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"0","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107812024-04-20T01:12:43.211Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.706558246Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"0","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107822024-04-20T01:12:43.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.706625346Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"1","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107832024-04-20T01:12:43.211Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.706717036Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"1","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107842024-04-20T01:12:43.211Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.706955587Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"0","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107852024-04-20T01:12:43.211Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707008237Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"2","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107862024-04-20T01:12:43.212Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707235838Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"1","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107872024-04-20T01:12:43.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707464138Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"0","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107882024-04-20T01:12:43.212Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707506288Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"0","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107892024-04-20T01:12:43.212Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707531678Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107902024-04-20T01:12:43.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707617318Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"2","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107912024-04-20T01:12:43.212Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707656909Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"2","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107922024-04-20T01:12:43.212Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707683909Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107932024-04-20T01:12:43.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707707639Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"1","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107942024-04-20T01:12:43.212Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707739239Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"1","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107952024-04-20T01:12:43.212Z{"msg":"[0]R flush_numbers[0..12]: [157, 164, 154, 161, 157, 159, 146, 161, 161, 161, 159, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707770169Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107962024-04-20T01:12:43.212Z{"msg":"[0]R generation[0..12]: [94, 98, 92, 98, 94, 96, 87, 98, 98, 98, 96, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707791189Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107972024-04-20T01:12:43.212Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707822219Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107982024-04-20T01:12:43.212Z{"msg":"[1]R flush_numbers[0..12]: [157, 164, 154, 161, 157, 159, 146, 161, 161, 161, 159, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707841729Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
107992024-04-20T01:12:43.212Z{"msg":"[1]R generation[0..12]: [94, 98, 92, 98, 94, 96, 87, 98, 98, 98, 96, 98]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
108002024-04-20T01:12:43.212Z,"time":"2024-04-20T01:12:41.707871199Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108012024-04-20T01:12:43.212Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707904689Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108022024-04-20T01:12:43.212Z{"msg":"[2]R flush_numbers[0..12]: [157, 164, 154, 161, 157, 159, 146, 161, 161, 161, 159, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707937279Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108032024-04-20T01:12:43.212Z{"msg":"[2]R generation[0..12]: [94, 98, 92, 98, 94, 96, 87, 98, 98, 98, 96, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.707969479Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108042024-04-20T01:12:43.212Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.70800283Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108052024-04-20T01:12:43.212Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.70803299Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108062024-04-20T01:12:43.212Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.70806487Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108072024-04-20T01:12:43.212Z{"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.70809575Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108082024-04-20T01:12:43.212Z{"msg":"All extents match","v":0,"name":"crucible","level":30,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108092024-04-20T01:12:43.213Z"time":"2024-04-20T01:12:41.70812667Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"":"downstairs","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108102024-04-20T01:12:43.213Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.70816391Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108112024-04-20T01:12:43.213Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.70819362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"0","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108122024-04-20T01:12:43.213Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.70822181Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"1","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108132024-04-20T01:12:43.213Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.70825459Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"client":"2","":"downstairs","session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108142024-04-20T01:12:43.213Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.7082835Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108152024-04-20T01:12:43.213Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.70831445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108162024-04-20T01:12:43.213Z{"msg":"e2fa5edc-de2e-470a-ab5f-bee80b23ad75 is now active with session: 01fe423d-dac8-4c8f-aed6-e0e3ca5fc256","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.7083457Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108172024-04-20T01:12:43.213Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.70837515Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571,"session_id":"01fe423d-dac8-4c8f-aed6-e0e3ca5fc256"}
108182024-04-20T01:12:43.213Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:41.70838982Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571}
108192024-04-20T01:12:43.213ZLoading write count information from file "/tmp/test_repair_verify.data"
108202024-04-20T01:12:43.213ZRun Repair workload
108212024-04-20T01:12:43.220Z01/30 Write block 120 len 20480 data: 6 6 7 8 11
108222024-04-20T01:12:43.220Z02/30 Write block 463 len 32768 data: 12 9 10 12 12 10 12 12
108232024-04-20T01:12:43.220Z03/30 Write block 199 len 12288 data: 10 7 8
108242024-04-20T01:12:43.220Z04/30 Read block 42 len 8192
108252024-04-20T01:12:43.220Z05/30 Read block 506 len 4096
108262024-04-20T01:12:43.220Z06/30 Flush
108272024-04-20T01:12:43.220Z07/30 Write block 495 len 12288 data: 7 7 6
108282024-04-20T01:12:43.220Z08/30 Read block 301 len 28672
108292024-04-20T01:12:43.220Z09/30 Read block 315 len 12288
108302024-04-20T01:12:43.220Z10/30 Write block 73 len 12288 data: 7 8 5
108312024-04-20T01:12:43.220Z11/30 Write block 266 len 12288 data: 4 4 3
108322024-04-20T01:12:43.220Z12/30 Write block 546 len 4096 data: 9
108332024-04-20T01:12:43.220Z13/30 Read block 565 len 16384
108342024-04-20T01:12:43.225Z14/30 Flush
108352024-04-20T01:12:43.225Z15/30 Write block 377 len 8192 data: 10 10
108362024-04-20T01:12:43.225Z16/30 Read block 570 len 28672
108372024-04-20T01:12:43.225Z17/30 Write block 199 len 4096 data: 11
108382024-04-20T01:12:43.225Z18/30 Read block 110 len 12288
108392024-04-20T01:12:43.225Z19/30 Read block 427 len 40960
108402024-04-20T01:12:43.225Z20/30 Read block 546 len 20480
108412024-04-20T01:12:43.225Z21/30 Read block 487 len 16384
108422024-04-20T01:12:43.225Z22/30 Read block 153 len 4096
108432024-04-20T01:12:43.228Z23/30 Flush
108442024-04-20T01:12:43.229Z24/30 Write block 181 len 4096 data: 15
108452024-04-20T01:12:43.229Z25/30 Read block 113 len 36864
108462024-04-20T01:12:43.232Z26/30 Write block 527 len 40960 data: 8 6 6 6 8 8 7 8 7 6
108472024-04-20T01:12:43.232Z27/30 Read block 486 len 24576
108482024-04-20T01:12:43.232Z28/30 Read block 14 len 16384
108492024-04-20T01:12:43.233Z29/30 Write block 53 len 16384 data: 14 12 12 11
108502024-04-20T01:12:43.233Z30/30 Read block 183 len 12288
108512024-04-20T01:12:43.233Z----------------------------------------------------------------
108522024-04-20T01:12:43.233Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
108532024-04-20T01:12:43.233ZStates: Active Active Active
108542024-04-20T01:12:43.233ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
108552024-04-20T01:12:43.233Z 1 Acked 1000 Write 5 Done Done Done false
108562024-04-20T01:12:43.233Z 2 Acked 1001 Write 8 Done Sent Done false
108572024-04-20T01:12:43.233Z 3 Acked 1002 Write 3 Done Sent Done false
108582024-04-20T01:12:43.237Z 4 Acked 1003 Read 2 Done Sent Done false
108592024-04-20T01:12:43.237Z 5 Acked 1004 Read 1 Done Sent Done false
108602024-04-20T01:12:43.237Z 6 Acked 1005 Flush 0 Done Sent Done false
108612024-04-20T01:12:43.237Z 7 Acked 1006 Write 3 Done Sent Done false
108622024-04-20T01:12:43.237Z 8 Acked 1007 Read 7 Done Sent Done false
108632024-04-20T01:12:43.237Z 9 Acked 1008 Read 3 Done Sent Done false
108642024-04-20T01:12:43.237Z 10 Acked 1009 Write 3 Done Sent Done false
108652024-04-20T01:12:43.237Z 11 Acked 1010 Write 3 Done Sent Done false
108662024-04-20T01:12:43.237Z 12 Acked 1011 Write 1 Done Sent Done false
108672024-04-20T01:12:43.237Z 13 Acked 1012 Read 4 Done Sent Done false
108682024-04-20T01:12:43.237Z 14 Acked 1013 Flush 0 Done Sent Done false
108692024-04-20T01:12:43.238Z 15 Acked 1014 Write 2 Done Sent Done false
108702024-04-20T01:12:43.238Z 16 Acked 1015 Read 7 Done Sent Done false
108712024-04-20T01:12:43.238Z 17 Acked 1016 Write 1 Done Sent Done false
108722024-04-20T01:12:43.238Z 18 Acked 1017 Read 3 Done Sent Done false
108732024-04-20T01:12:43.238Z 19 Acked 1018 Read 10 Done Sent Done false
108742024-04-20T01:12:43.238Z 20 Acked 1019 Read 5 Done Sent Done false
108752024-04-20T01:12:43.238Z 21 Acked 1020 Read 4 Done Sent Done false
108762024-04-20T01:12:43.238Z 22 Acked 1021 Read 1 Done Sent Done false
108772024-04-20T01:12:43.238Z 23 Acked 1022 Flush 0 Done Sent Done false
108782024-04-20T01:12:43.238Z 24 Acked 1023 Write 1 Done Sent Done false
108792024-04-20T01:12:43.238Z 25 Acked 1024 Read 9 Done Sent Done false
108802024-04-20T01:12:43.238Z 26 Acked 1025 Write 10 Done Sent Done false
108812024-04-20T01:12:43.238Z 27 Acked 1026 Read 6 Done Sent Done false
108822024-04-20T01:12:43.238Z 28 Acked 1027 Read 4 Done Sent Done false
108832024-04-20T01:12:43.238Z 29 Acked 1028 Write 4 Done Sent Done false
108842024-04-20T01:12:43.238Z 30 Acked 1029 Read 3 Done Sent Sent false
108852024-04-20T01:12:43.238Z STATES DS:0 DS:1 DS:2 TOTAL
108862024-04-20T01:12:43.238Z New 0 0 0 0
108872024-04-20T01:12:43.238Z Sent 0 29 1 30
108882024-04-20T01:12:43.238Z Done 30 1 29 60
108892024-04-20T01:12:43.238Z Skipped 0 0 0 0
108902024-04-20T01:12:43.238Z Error 0 0 0 0
108912024-04-20T01:12:43.238ZLast Flush: 1022 0 1022
108922024-04-20T01:12:43.238ZDownstairs last five completed:
108932024-04-20T01:12:43.239ZUpstairs last five completed: 30 29 28 27 26
108942024-04-20T01:12:43.239Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:12:41.735829031Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1571}
108952024-04-20T01:12:43.239ZWrote out file "/tmp/test_repair_verify.data"
108962024-04-20T01:12:43.243Z
108972024-04-20T01:12:43.243ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
108982024-04-20T01:12:43.256ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108992024-04-20T01:12:43.256Z 0 000-019 94 94 94 157 157 157 F F F
109002024-04-20T01:12:43.256Z 1 020-039 98 98 98 164 164 164 F F F
109012024-04-20T01:12:43.256Z 2 040-059 92 92 92 154 154 154 T F T <---
109022024-04-20T01:12:43.256Z 3 060-079 100 98 100 168 161 168 F F F <---
109032024-04-20T01:12:43.256Z 4 080-099 94 94 94 157 157 157 F F F
109042024-04-20T01:12:43.259Z 5 100-119 96 96 96 159 159 159 F F F
109052024-04-20T01:12:43.259Z 6 120-139 100 87 100 167 146 167 F T F <---
109062024-04-20T01:12:43.259Z 7 140-159 98 98 98 161 161 161 F F F
109072024-04-20T01:12:43.260Z 8 160-179 98 98 98 161 161 161 F F F
109082024-04-20T01:12:43.260Z 9 180-199 100 98 100 169 161 169 T F T <---
109092024-04-20T01:12:43.260Z 10 200-219 100 96 100 167 159 167 F F F <---
109102024-04-20T01:12:43.260Z 11 220-239 98 98 98 165 165 165 F F F
109112024-04-20T01:12:43.260Z 12 240-259 98 98 98 161 161 161 F F F
109122024-04-20T01:12:43.260Z 13 260-279 100 96 100 168 159 168 F F F <---
109132024-04-20T01:12:43.260Z 14 280-299 99 99 99 166 166 166 F F F
109142024-04-20T01:12:43.260Z 15 300-319 99 99 99 166 166 166 F F F
109152024-04-20T01:12:43.260Z 16 320-339 96 96 96 159 159 159 F F F
109162024-04-20T01:12:43.260Z 17 340-359 98 98 98 163 163 163 F F F
109172024-04-20T01:12:43.260Z 18 360-379 100 96 100 169 159 169 F F F <---
109182024-04-20T01:12:43.260Z 19 380-399 96 96 96 159 159 159 F F F
109192024-04-20T01:12:43.260Z 20 400-419 97 97 97 160 160 160 F F F
109202024-04-20T01:12:43.260Z 21 420-439 90 90 90 151 151 151 F F F
109212024-04-20T01:12:43.260Z 22 440-459 96 96 96 159 159 159 F F F
109222024-04-20T01:12:43.260Z 23 460-479 100 96 100 167 159 167 F F F <---
109232024-04-20T01:12:43.260Z 24 480-499 100 96 100 168 159 168 F F F <---
109242024-04-20T01:12:43.260Z 25 500-519 98 98 98 161 161 161 F F F
109252024-04-20T01:12:43.260Z 26 520-539 94 94 94 157 157 157 T F T <---
109262024-04-20T01:12:43.260Z 27 540-559 100 99 100 168 166 168 F F F <---
109272024-04-20T01:12:43.260Z 28 560-579 98 98 98 161 161 161 F F F
109282024-04-20T01:12:43.260Z 29 580-599 98 98 98 161 161 161 F F F
109292024-04-20T01:12:43.260ZMax gen: 100, Max flush: 169
109302024-04-20T01:12:43.261ZError: Difference in extent metadata found!
109312024-04-20T01:12:43.261ZA Difference in extent metadata is expected here
109322024-04-20T01:12:43.261ZOn loop 50
109332024-04-20T01:12:43.261Z
109342024-04-20T01:12:43.264ZVerifying data now
109352024-04-20T01:12:47.777ZNo change after verify
109362024-04-20T01:12:47.777ZLoop: 50 Downstairs dump after verify (and repair):
109372024-04-20T01:12:47.794ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109382024-04-20T01:12:47.794Z 0 000-019 94 94 94 157 157 157 F F F
109392024-04-20T01:12:47.794Z 1 020-039 98 98 98 164 164 164 F F F
109402024-04-20T01:12:47.794Z 2 040-059 101 101 101 170 170 170 F F F
109412024-04-20T01:12:47.794Z 3 060-079 100 100 100 168 168 168 F F F
109422024-04-20T01:12:47.794Z 4 080-099 94 94 94 157 157 157 F F F
109432024-04-20T01:12:47.794Z 5 100-119 96 96 96 159 159 159 F F F
109442024-04-20T01:12:47.794Z 6 120-139 100 100 100 167 167 167 F F F
109452024-04-20T01:12:47.794Z 7 140-159 98 98 98 161 161 161 F F F
109462024-04-20T01:12:47.798Z 8 160-179 98 98 98 161 161 161 F F F
109472024-04-20T01:12:47.798Z 9 180-199 101 101 101 170 170 170 F F F
109482024-04-20T01:12:47.798Z 10 200-219 100 100 100 167 167 167 F F F
109492024-04-20T01:12:47.798Z 11 220-239 98 98 98 165 165 165 F F F
109502024-04-20T01:12:47.798Z 12 240-259 98 98 98 161 161 161 F F F
109512024-04-20T01:12:47.798Z 13 260-279 100 100 100 168 168 168 F F F
109522024-04-20T01:12:47.798Z 14 280-299 99 99 99 166 166 166 F F F
109532024-04-20T01:12:47.798Z 15 300-319 99 99 99 166 166 166 F F F
109542024-04-20T01:12:47.798Z 16 320-339 96 96 96 159 159 159 F F F
109552024-04-20T01:12:47.798Z 17 340-359 98 98 98 163 163 163 F F F
109562024-04-20T01:12:47.798Z 18 360-379 100 100 100 169 169 169 F F F
109572024-04-20T01:12:47.798Z 19 380-399 96 96 96 159 159 159 F F F
109582024-04-20T01:12:47.798Z 20 400-419 97 97 97 160 160 160 F F F
109592024-04-20T01:12:47.798Z 21 420-439 90 90 90 151 151 151 F F F
109602024-04-20T01:12:47.799Z 22 440-459 96 96 96 159 159 159 F F F
109612024-04-20T01:12:47.799Z 23 460-479 100 100 100 167 167 167 F F F
109622024-04-20T01:12:47.799Z 24 480-499 100 100 100 168 168 168 F F F
109632024-04-20T01:12:47.799Z 25 500-519 98 98 98 161 161 161 F F F
109642024-04-20T01:12:47.799Z 26 520-539 101 101 101 170 170 170 F F F
109652024-04-20T01:12:47.799Z 27 540-559 100 100 100 168 168 168 F F F
109662024-04-20T01:12:47.799Z 28 560-579 98 98 98 161 161 161 F F F
109672024-04-20T01:12:47.799Z 29 580-599 98 98 98 161 161 161 F F F
109682024-04-20T01:12:47.799ZMax gen: 101, Max flush: 170
109692024-04-20T01:12:47.799Z
109702024-04-20T01:12:47.799ZBegin loop 51
109712024-04-20T01:12:47.799ZDownstairs to restart: 2
109722024-04-20T01:12:47.812Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315395306Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579}
109732024-04-20T01:12:47.812Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315618646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579}
109742024-04-20T01:12:47.813Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315641756Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579}
109752024-04-20T01:12:47.813Z{"msg":"Crucible 7fe1de27-5aae-40bc-b752-17d83a1c66fd has session id: e067e39b-7916-4f9a-939a-7e2db762e06a","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315673746Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109762024-04-20T01:12:47.820Z{"msg":"Upstairs opts: Upstairs UUID: 7fe1de27-5aae-40bc-b752-17d83a1c66fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315720986Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109772024-04-20T01:12:47.820ZCrucible runtime is spawned
109782024-04-20T01:12:47.820Z{"msg":"Crucible stats registered with UUID: 7fe1de27-5aae-40bc-b752-17d83a1c66fd","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315759576Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109792024-04-20T01:12:47.820Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315832877Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579}
109802024-04-20T01:12:47.820Z{"msg":"7fe1de27-5aae-40bc-b752-17d83a1c66fd active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315856987Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109812024-04-20T01:12:47.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315870677Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"0","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109822024-04-20T01:12:47.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315885957Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"0","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109832024-04-20T01:12:47.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315899567Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"1","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109842024-04-20T01:12:47.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315924927Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"1","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109852024-04-20T01:12:47.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315950177Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"2","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109862024-04-20T01:12:47.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.315997187Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"2","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109872024-04-20T01:12:47.820Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.316036847Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109882024-04-20T01:12:47.820Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.316111867Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109892024-04-20T01:12:47.820Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.316160967Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109902024-04-20T01:12:47.820Z{"msg":"ds_connection connected from Ok(127.0.0.1:50513)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.316270597Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109912024-04-20T01:12:47.821Z{"msg":"ds_connection connected from Ok(127.0.0.1:33944)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.316329878Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109922024-04-20T01:12:47.821Z{"msg":"ds_connection connected from Ok(127.0.0.1:54162)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.316370198Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109932024-04-20T01:12:47.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.31709849Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"0","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109942024-04-20T01:12:47.821Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.31713481Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"0","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109952024-04-20T01:12:47.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.31715113Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"1","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109962024-04-20T01:12:47.821Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.31718778Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"1","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109972024-04-20T01:12:47.821Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.317685781Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"1","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109982024-04-20T01:12:47.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.317735521Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"2","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
109992024-04-20T01:12:47.821Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.317766511Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"2","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110002024-04-20T01:12:47.821Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.317810781Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"0","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110012024-04-20T01:12:47.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.318200772Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"1","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110022024-04-20T01:12:47.821Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.318247362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"1","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110032024-04-20T01:12:47.821Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.318263512Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110042024-04-20T01:12:47.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.318276022Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"0","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110052024-04-20T01:12:47.821Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.318297302Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"0","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110062024-04-20T01:12:47.821Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.318311162Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110072024-04-20T01:12:47.821Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.318540343Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"2","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110082024-04-20T01:12:47.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.318947923Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"2","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110092024-04-20T01:12:47.821Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.318982724Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"2","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110102024-04-20T01:12:47.821Z{"msg":"[0]R flush_numbers[0..12]: [157, 164, 170, 168, 157, 159, 167, 161, 161, 170, 167, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.318998764Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110112024-04-20T01:12:47.821Z{"msg":"[0]R generation[0..12]: [94, 98, 101, 100, 94, 96, 100, 98, 98, 101, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319020754Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110122024-04-20T01:12:47.821Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319035714Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110132024-04-20T01:12:47.821Z{"msg":"[1]R flush_numbers[0..12]: [157, 164, 170, 168, 157, 159, 167, 161, 161, 170, 167, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319056654Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110142024-04-20T01:12:47.822Z{"msg":"[1]R generation[0..12]: [94, 98, 101, 100, 94, 96, 100, 98, 98, 101, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319070874Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110152024-04-20T01:12:47.822Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319085154Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110162024-04-20T01:12:47.822Z{"msg":"[2]R flush_numbers[0..12]: [157, 164, 170, 168, 157, 159, 167, 161, 161, 170, 167, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319098864Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110172024-04-20T01:12:47.822Z{"msg":"[2]R generation[0..12]: [94, 98, 101, 100, 94, 96, 100, 98, 98, 101, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319113194Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110182024-04-20T01:12:47.822Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
110192024-04-20T01:12:47.822Z,"time":"2024-04-20T01:12:46.319127634Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110202024-04-20T01:12:47.822Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319151084Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110212024-04-20T01:12:47.822Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319165204Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110222024-04-20T01:12:47.822Z{"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319190904Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110232024-04-20T01:12:47.822Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319204184Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"":"downstairs","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110242024-04-20T01:12:47.822Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319217284Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110252024-04-20T01:12:47.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319236494Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"0","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110262024-04-20T01:12:47.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319250424Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"1","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110272024-04-20T01:12:47.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319270124Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"client":"2","":"downstairs","session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110282024-04-20T01:12:47.822Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319283524Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110292024-04-20T01:12:47.822Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319296254Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110302024-04-20T01:12:47.822Z{"msg":"7fe1de27-5aae-40bc-b752-17d83a1c66fd is now active with session: e067e39b-7916-4f9a-939a-7e2db762e06a","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319319414Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110312024-04-20T01:12:47.822Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319346484Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579,"session_id":"e067e39b-7916-4f9a-939a-7e2db762e06a"}
110322024-04-20T01:12:47.826Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:46.319359884Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579}
110332024-04-20T01:12:47.827ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
110342024-04-20T01:12:47.827ZLoading write count information from file "/tmp/test_repair_verify.data"
110352024-04-20T01:12:47.827ZRun Repair workload
110362024-04-20T01:12:47.827Z01/30 Flush
110372024-04-20T01:12:47.827Z02/30 Write block 593 len 28672 data: 8 10 9 9 6 6 2
110382024-04-20T01:12:47.827Z03/30 Read block 53 len 16384
110392024-04-20T01:12:47.827Z04/30 Write block 119 len 16384 data: 7 7 7 8
110402024-04-20T01:12:47.827Z05/30 Read block 135 len 36864
110412024-04-20T01:12:47.827Z06/30 Write block 498 len 32768 data: 6 6 7 6 6 9 9 9
110422024-04-20T01:12:47.827Z07/30 Flush
110432024-04-20T01:12:47.831Z08/30 Flush
110442024-04-20T01:12:47.831Z09/30 Write block 534 len 16384 data: 9 8 7 6
110452024-04-20T01:12:47.831Z10/30 Write block 501 len 16384 data: 7 7 10 10
110462024-04-20T01:12:47.831Z11/30 Read block 150 len 16384
110472024-04-20T01:12:47.831Z12/30 Write block 585 len 28672 data: 12 15 13 13 11 8 8
110482024-04-20T01:12:47.831Z13/30 Flush
110492024-04-20T01:12:47.834Z14/30 Write block 447 len 40960 data: 8 8 10 10 10 9 7 8 12 12
110502024-04-20T01:12:47.838Z15/30 Read block 497 len 16384
110512024-04-20T01:12:47.838Z16/30 Read block 140 len 28672
110522024-04-20T01:12:47.838Z17/30 Read block 197 len 4096
110532024-04-20T01:12:47.838Z18/30 Flush
110542024-04-20T01:12:47.841Z19/30 Write block 442 len 8192 data: 9 8
110552024-04-20T01:12:47.841Z20/30 Write block 410 len 16384 data: 10 12 12 13
110562024-04-20T01:12:47.841Z21/30 Write block 482 len 4096 data: 9
110572024-04-20T01:12:47.841Z22/30 Read block 360 len 16384
110582024-04-20T01:12:47.845Z23/30 Write block 481 len 4096 data: 9
110592024-04-20T01:12:47.845Z24/30 Read block 41 len 28672
110602024-04-20T01:12:47.845Z25/30 Write block 182 len 8192 data: 11 7
110612024-04-20T01:12:47.845Z26/30 Write block 68 len 32768 data: 12 13 8 8 6 8 9 6
110622024-04-20T01:12:47.848Z27/30 Read block 457 len 36864
110632024-04-20T01:12:47.848Z28/30 Write block 526 len 36864 data: 10 9 7 7 7 9 9 8 10
110642024-04-20T01:12:47.848Z29/30 Read block 49 len 8192
110652024-04-20T01:12:47.848Z30/30 Read block 515 len 12288
110662024-04-20T01:12:47.853Z----------------------------------------------------------------
110672024-04-20T01:12:47.853Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
110682024-04-20T01:12:47.853ZStates: Active Active Active
110692024-04-20T01:12:47.853ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
110702024-04-20T01:12:47.853Z 1 Acked 1000 Flush 0 Done Done Sent false
110712024-04-20T01:12:47.853Z 2 Acked 1001 Write 7 Done Done Sent false
110722024-04-20T01:12:47.853Z 3 Acked 1002 Read 4 Done Done Sent false
110732024-04-20T01:12:47.853Z 4 Acked 1003 Write 4 Done Done Sent false
110742024-04-20T01:12:47.853Z 5 Acked 1004 Read 9 Done Done Sent false
110752024-04-20T01:12:47.853Z 6 Acked 1005 Write 8 Done Done Sent false
110762024-04-20T01:12:47.853Z 7 Acked 1006 Flush 0 Done Done Sent false
110772024-04-20T01:12:47.853Z 8 Acked 1007 Flush 0 Done Done Sent false
110782024-04-20T01:12:47.853Z 9 Acked 1008 Write 4 Done Done Sent false
110792024-04-20T01:12:47.853Z 10 Acked 1009 Write 4 Done Done Sent false
110802024-04-20T01:12:47.853Z 11 Acked 1010 Read 4 Done Done Sent false
110812024-04-20T01:12:47.853Z 12 Acked 1011 Write 7 Done Done Sent false
110822024-04-20T01:12:47.853Z 13 Acked 1012 Flush 0 Done Done Sent false
110832024-04-20T01:12:47.854Z 14 Acked 1013 Write 10 Done Done Sent false
110842024-04-20T01:12:47.854Z 15 Acked 1014 Read 4 Done Done Sent false
110852024-04-20T01:12:47.854Z 16 Acked 1015 Read 7 Done Done Sent false
110862024-04-20T01:12:47.854Z 17 Acked 1016 Read 1 Done Done Sent false
110872024-04-20T01:12:47.854Z 18 Acked 1017 Flush 0 Done Done Sent false
110882024-04-20T01:12:47.854Z 19 Acked 1018 Write 2 Done Done Sent false
110892024-04-20T01:12:47.854Z 20 Acked 1019 Write 4 Done Done Sent false
110902024-04-20T01:12:47.854Z 21 Acked 1020 Write 1 Done Done Sent false
110912024-04-20T01:12:47.854Z 22 Acked 1021 Read 4 Done Done Sent false
110922024-04-20T01:12:47.854Z 23 Acked 1022 Write 1 Done Done Sent false
110932024-04-20T01:12:47.854Z 24 Acked 1023 Read 7 Done Done Sent false
110942024-04-20T01:12:47.854Z 25 Acked 1024 Write 2 Done Done Sent false
110952024-04-20T01:12:47.854Z 26 Acked 1025 Write 8 Done Done Sent false
110962024-04-20T01:12:47.854Z 27 Acked 1026 Read 9 Done Done Sent false
110972024-04-20T01:12:47.854Z 28 Acked 1027 Write 9 Done Done Sent false
110982024-04-20T01:12:47.854Z 29 Acked 1028 Read 2 Done Done Sent false
110992024-04-20T01:12:47.854Z 30 Acked 1029 Read 3 Done Done Sent false
111002024-04-20T01:12:47.854Z STATES DS:0 DS:1 DS:2 TOTAL
111012024-04-20T01:12:47.854Z New 0 0 0 0
111022024-04-20T01:12:47.854Z Sent 0 0 30 30
111032024-04-20T01:12:47.854Z Done 30 30 0 60
111042024-04-20T01:12:47.854Z Skipped 0 0 0 0
111052024-04-20T01:12:47.854Z Error 0 0 0 0
111062024-04-20T01:12:47.855ZLast Flush: 1017 1017 0
111072024-04-20T01:12:47.855ZDownstairs last five completed:
111082024-04-20T01:12:47.855ZUpstairs last five completed: 30 29 28 27 26
111092024-04-20T01:12:47.855Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:12:46.352272968Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1579}
111102024-04-20T01:12:47.855ZWrote out file "/tmp/test_repair_verify.data"
111112024-04-20T01:12:47.858Z
111122024-04-20T01:12:47.858ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
111132024-04-20T01:12:47.872ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111142024-04-20T01:12:47.873Z 0 000-019 94 94 94 157 157 157 F F F
111152024-04-20T01:12:47.873Z 1 020-039 98 98 98 164 164 164 F F F
111162024-04-20T01:12:47.873Z 2 040-059 101 101 101 170 170 170 F F F
111172024-04-20T01:12:47.873Z 3 060-079 100 100 100 168 168 168 T T F <---
111182024-04-20T01:12:47.873Z 4 080-099 94 94 94 157 157 157 F F F
111192024-04-20T01:12:47.873Z 5 100-119 102 102 96 172 172 159 F F F <---
111202024-04-20T01:12:47.873Z 6 120-139 102 102 100 172 172 167 F F F <---
111212024-04-20T01:12:47.873Z 7 140-159 98 98 98 161 161 161 F F F
111222024-04-20T01:12:47.873Z 8 160-179 98 98 98 161 161 161 F F F
111232024-04-20T01:12:47.886Z 9 180-199 101 101 101 170 170 170 T T F <---
111242024-04-20T01:12:47.886Z 10 200-219 100 100 100 167 167 167 F F F
111252024-04-20T01:12:47.886Z 11 220-239 98 98 98 165 165 165 F F F
111262024-04-20T01:12:47.886Z 12 240-259 98 98 98 161 161 161 F F F
111272024-04-20T01:12:47.886Z 13 260-279 100 100 100 168 168 168 F F F
111282024-04-20T01:12:47.886Z 14 280-299 99 99 99 166 166 166 F F F
111292024-04-20T01:12:47.886Z 15 300-319 99 99 99 166 166 166 F F F
111302024-04-20T01:12:47.886Z 16 320-339 96 96 96 159 159 159 F F F
111312024-04-20T01:12:47.886Z 17 340-359 98 98 98 163 163 163 F F F
111322024-04-20T01:12:47.886Z 18 360-379 100 100 100 169 169 169 F F F
111332024-04-20T01:12:47.887Z 19 380-399 96 96 96 159 159 159 F F F
111342024-04-20T01:12:47.887Z 20 400-419 97 97 97 160 160 160 T T F <---
111352024-04-20T01:12:47.887Z 21 420-439 90 90 90 151 151 151 F F F
111362024-04-20T01:12:47.887Z 22 440-459 102 102 96 175 175 159 T T F <---
111372024-04-20T01:12:47.887Z 23 460-479 100 100 100 167 167 167 F F F
111382024-04-20T01:12:47.887Z 24 480-499 102 102 100 172 172 168 T T F <---
111392024-04-20T01:12:47.887Z 25 500-519 102 102 98 174 174 161 F F F <---
111402024-04-20T01:12:47.887Z 26 520-539 102 102 101 174 174 170 T T F <---
111412024-04-20T01:12:47.887Z 27 540-559 100 100 100 168 168 168 F F F
111422024-04-20T01:12:47.887Z 28 560-579 98 98 98 161 161 161 F F F
111432024-04-20T01:12:47.887Z 29 580-599 102 102 98 174 174 161 F F F <---
111442024-04-20T01:12:47.887ZMax gen: 102, Max flush: 175
111452024-04-20T01:12:47.887ZError: Difference in extent metadata found!
111462024-04-20T01:12:47.887ZA Difference in extent metadata is expected here
111472024-04-20T01:12:47.887ZOn loop 51
111482024-04-20T01:12:47.887Z
111492024-04-20T01:12:47.887ZVerifying data now
111502024-04-20T01:12:52.359ZNo change after verify
111512024-04-20T01:12:52.359ZLoop: 51 Downstairs dump after verify (and repair):
111522024-04-20T01:12:52.373ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111532024-04-20T01:12:52.373Z 0 000-019 94 94 94 157 157 157 F F F
111542024-04-20T01:12:52.374Z 1 020-039 98 98 98 164 164 164 F F F
111552024-04-20T01:12:52.378Z 2 040-059 101 101 101 170 170 170 F F F
111562024-04-20T01:12:52.378Z 3 060-079 103 103 103 176 176 176 F F F
111572024-04-20T01:12:52.378Z 4 080-099 94 94 94 157 157 157 F F F
111582024-04-20T01:12:52.378Z 5 100-119 102 102 102 172 172 172 F F F
111592024-04-20T01:12:52.378Z 6 120-139 102 102 102 172 172 172 F F F
111602024-04-20T01:12:52.378Z 7 140-159 98 98 98 161 161 161 F F F
111612024-04-20T01:12:52.378Z 8 160-179 98 98 98 161 161 161 F F F
111622024-04-20T01:12:52.378Z 9 180-199 103 103 103 176 176 176 F F F
111632024-04-20T01:12:52.378Z 10 200-219 100 100 100 167 167 167 F F F
111642024-04-20T01:12:52.378Z 11 220-239 98 98 98 165 165 165 F F F
111652024-04-20T01:12:52.378Z 12 240-259 98 98 98 161 161 161 F F F
111662024-04-20T01:12:52.378Z 13 260-279 100 100 100 168 168 168 F F F
111672024-04-20T01:12:52.378Z 14 280-299 99 99 99 166 166 166 F F F
111682024-04-20T01:12:52.378Z 15 300-319 99 99 99 166 166 166 F F F
111692024-04-20T01:12:52.378Z 16 320-339 96 96 96 159 159 159 F F F
111702024-04-20T01:12:52.378Z 17 340-359 98 98 98 163 163 163 F F F
111712024-04-20T01:12:52.378Z 18 360-379 100 100 100 169 169 169 F F F
111722024-04-20T01:12:52.378Z 19 380-399 96 96 96 159 159 159 F F F
111732024-04-20T01:12:52.378Z 20 400-419 103 103 103 176 176 176 F F F
111742024-04-20T01:12:52.378Z 21 420-439 90 90 90 151 151 151 F F F
111752024-04-20T01:12:52.378Z 22 440-459 103 103 103 176 176 176 F F F
111762024-04-20T01:12:52.378Z 23 460-479 100 100 100 167 167 167 F F F
111772024-04-20T01:12:52.379Z 24 480-499 103 103 103 176 176 176 F F F
111782024-04-20T01:12:52.379Z 25 500-519 102 102 102 174 174 174 F F F
111792024-04-20T01:12:52.379Z 26 520-539 103 103 103 176 176 176 F F F
111802024-04-20T01:12:52.379Z 27 540-559 100 100 100 168 168 168 F F F
111812024-04-20T01:12:52.379Z 28 560-579 98 98 98 161 161 161 F F F
111822024-04-20T01:12:52.379Z 29 580-599 102 102 102 174 174 174 F F F
111832024-04-20T01:12:52.379ZMax gen: 103, Max flush: 176
111842024-04-20T01:12:52.379Z
111852024-04-20T01:12:52.379ZBegin loop 52
111862024-04-20T01:12:52.379ZDownstairs to restart: 1
111872024-04-20T01:12:52.395Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.897689576Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587}
111882024-04-20T01:12:52.399Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.897969396Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587}
111892024-04-20T01:12:52.399Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.898011197Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587}
111902024-04-20T01:12:52.399Z{"msg":"Crucible 66f475dd-7a04-4c8d-882b-9a2708219088 has session id: 519a60bf-c97b-4b32-abce-ddbb3c512355","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.900925623Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
111912024-04-20T01:12:52.399Z{"msg":"Upstairs opts: Upstairs UUID: 66f475dd-7a04-4c8d-882b-9a2708219088, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.900971103Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
111922024-04-20T01:12:52.399Z{"msg":"Crucible stats registered with UUID: 66f475dd-7a04-4c8d-882b-9a2708219088","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.900987863Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
111932024-04-20T01:12:52.399ZCrucible runtime is spawned
111942024-04-20T01:12:52.399Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901241354Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587}
111952024-04-20T01:12:52.399Z{"msg":"66f475dd-7a04-4c8d-882b-9a2708219088 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901354304Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
111962024-04-20T01:12:52.399Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901391204Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"0","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
111972024-04-20T01:12:52.399Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901429414Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"0","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
111982024-04-20T01:12:52.399Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901463724Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"1","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
111992024-04-20T01:12:52.399Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901489404Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"1","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112002024-04-20T01:12:52.400Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901523715Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"2","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112012024-04-20T01:12:52.400Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901547104Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"2","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112022024-04-20T01:12:52.400Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901584164Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112032024-04-20T01:12:52.400Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901607515Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112042024-04-20T01:12:52.407Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901633535Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112052024-04-20T01:12:52.407Z{"msg":"ds_connection connected from Ok(127.0.0.1:50147)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901672965Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112062024-04-20T01:12:52.407Z{"msg":"ds_connection connected from Ok(127.0.0.1:46855)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901773075Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112072024-04-20T01:12:52.407Z{"msg":"ds_connection connected from Ok(127.0.0.1:48453)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.901818115Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112082024-04-20T01:12:52.407Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.902413817Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"2","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112092024-04-20T01:12:52.407Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.902458537Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"2","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112102024-04-20T01:12:52.407Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.902486477Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"1","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112112024-04-20T01:12:52.407Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.902553007Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"1","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112122024-04-20T01:12:52.407Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.902582087Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"0","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112132024-04-20T01:12:52.407Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.902644927Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"0","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112142024-04-20T01:12:52.407Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.903090658Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"0","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112152024-04-20T01:12:52.407Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.903128358Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"2","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112162024-04-20T01:12:52.407Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.903213848Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"1","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112172024-04-20T01:12:52.407Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.903638169Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"2","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112182024-04-20T01:12:52.407Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.903688999Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"2","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112192024-04-20T01:12:52.407Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.903709809Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112202024-04-20T01:12:52.407Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.903735329Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"0","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112212024-04-20T01:12:52.407Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90375506Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"0","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112222024-04-20T01:12:52.408Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.903778069Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112232024-04-20T01:12:52.408Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.903790929Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"1","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112242024-04-20T01:12:52.408Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.903804949Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"1","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112252024-04-20T01:12:52.408Z{"msg":"[0]R flush_numbers[0..12]: [157, 164, 170, 176, 157, 172, 172, 161, 161, 176, 167, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.903826489Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112262024-04-20T01:12:52.408Z{"msg":"[0]R generation[0..12]: [94, 98, 101, 103, 94, 102, 102, 98, 98, 103, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90384132Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112272024-04-20T01:12:52.408Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90386468Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112282024-04-20T01:12:52.408Z{"msg":"[1]R flush_numbers[0..12]: [157, 164, 170, 176, 157, 172, 172, 161, 161, 176, 167, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.903879Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112292024-04-20T01:12:52.408Z{"msg":"[1]R generation[0..12]: [94, 98, 101, 103, 94, 102, 102, 98, 98, 103, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90389867Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112302024-04-20T01:12:52.408Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90391387Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112312024-04-20T01:12:52.408Z{"msg":"[2]R flush_numbers[0..12]: [157, 164, 170, 176, 157, 172, 172, 161, 161, 176, 167, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90394064Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112322024-04-20T01:12:52.408Z{"msg":"[2]R generation[0..12]: [94, 98, 101, 103, 94, 102, 102, 98, 98, 103, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90395469Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112332024-04-20T01:12:52.408ZWait for a query_work_queue command to finish before sending IO
112342024-04-20T01:12:52.408Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90398751Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112352024-04-20T01:12:52.408Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90401079Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112362024-04-20T01:12:52.408Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90404275Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112372024-04-20T01:12:52.408Z{"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90405774Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112382024-04-20T01:12:52.408Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90407093Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"":"downstairs","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112392024-04-20T01:12:52.408Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90409283Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112402024-04-20T01:12:52.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90411454Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"0","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112412024-04-20T01:12:52.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.9041417Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"1","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112422024-04-20T01:12:52.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90415611Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"client":"2","":"downstairs","session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112432024-04-20T01:12:52.408Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90418106Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112442024-04-20T01:12:52.409Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90420969Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112452024-04-20T01:12:52.409Z{"msg":"66f475dd-7a04-4c8d-882b-9a2708219088 is now active with session: 519a60bf-c97b-4b32-abce-ddbb3c512355","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.90423417Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
112462024-04-20T01:12:52.409Zpid":1587,"session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112472024-04-20T01:12:52.409Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.904267081Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587,"session_id":"519a60bf-c97b-4b32-abce-ddbb3c512355"}
112482024-04-20T01:12:52.409Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:12:50.904279841Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587}
112492024-04-20T01:12:52.409ZLoading write count information from file "/tmp/test_repair_verify.data"
112502024-04-20T01:12:52.409ZRun Repair workload
112512024-04-20T01:12:52.409Z01/30 Write block 328 len 4096 data: 9
112522024-04-20T01:12:52.409Z02/30 Write block 481 len 12288 data: 10 10 9
112532024-04-20T01:12:52.413Z03/30 Read block 386 len 4096
112542024-04-20T01:12:52.413Z04/30 Read block 12 len 20480
112552024-04-20T01:12:52.413Z05/30 Flush
112562024-04-20T01:12:52.413Z06/30 Write block 77 len 16384 data: 8 8 8 7
112572024-04-20T01:12:52.413Z07/30 Read block 525 len 28672
112582024-04-20T01:12:52.413Z08/30 Write block 374 len 20480 data: 14 13 9 11 11
112592024-04-20T01:12:52.413Z09/30 Read block 11 len 36864
112602024-04-20T01:12:52.413Z10/30 Read block 209 len 40960
112612024-04-20T01:12:52.413Z11/30 Write block 171 len 24576 data: 5 6 9 8 8 9
112622024-04-20T01:12:52.417Z12/30 Write block 170 len 32768 data: 4 6 7 10 9 9 10 10
112632024-04-20T01:12:52.417Z13/30 Write block 334 len 12288 data: 10 8 7
112642024-04-20T01:12:52.417Z14/30 Write block 113 len 8192 data: 8 8
112652024-04-20T01:12:52.418Z15/30 Write block 329 len 40960 data: 8 13 11 12 10 11 9 8 8 10
112662024-04-20T01:12:52.421Z16/30 Read block 339 len 36864
112672024-04-20T01:12:52.421Z17/30 Write block 447 len 32768 data: 9 9 11 11 11 10 8 9
112682024-04-20T01:12:52.421Z18/30 Read block 105 len 12288
112692024-04-20T01:12:52.421Z19/30 Read block 582 len 28672
112702024-04-20T01:12:52.428Z20/30 Write block 460 len 40960 data: 11 14 12 13 10 11 13 13 11 13
112712024-04-20T01:12:52.428Z21/30 Read block 169 len 36864
112722024-04-20T01:12:52.428Z22/30 Read block 313 len 4096
112732024-04-20T01:12:52.428Z23/30 Read block 187 len 16384
112742024-04-20T01:12:52.432Z24/30 Write block 18 len 28672 data: 7 7 7 6 6 4 5
112752024-04-20T01:12:52.432Z25/30 Read block 126 len 12288
112762024-04-20T01:12:52.432Z26/30 Read block 467 len 36864
112772024-04-20T01:12:52.432Z27/30 Read block 521 len 12288
112782024-04-20T01:12:52.432Z28/30 Write block 499 len 24576 data: 7 8 8 8 11 11
112792024-04-20T01:12:52.432Z29/30 Write block 363 len 20480 data: 9 7 7 9 12
112802024-04-20T01:12:52.432Z30/30 Write block 259 len 32768 data: 7 5 5 5 3 3 3 5
112812024-04-20T01:12:52.432Z----------------------------------------------------------------
112822024-04-20T01:12:52.432Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
112832024-04-20T01:12:52.432ZStates: Active Active Active
112842024-04-20T01:12:52.432ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112852024-04-20T01:12:52.432Z 1 Acked 1000 Write 1 Done Done Done false
112862024-04-20T01:12:52.432Z 2 Acked 1001 Write 3 Done Done Done false
112872024-04-20T01:12:52.432Z 3 Acked 1002 Read 1 Done Sent Done false
112882024-04-20T01:12:52.433Z 4 Acked 1003 Read 5 Done Sent Done false
112892024-04-20T01:12:52.433Z 5 Acked 1004 Flush 0 Done Sent Done false
112902024-04-20T01:12:52.433Z 6 Acked 1005 Write 4 Done Sent Done false
112912024-04-20T01:12:52.433Z 7 Acked 1006 Read 7 Done Sent Done false
112922024-04-20T01:12:52.433Z 8 Acked 1007 Write 5 Done Sent Done false
112932024-04-20T01:12:52.433Z 9 Acked 1008 Read 9 Done Sent Done false
112942024-04-20T01:12:52.433Z 10 Acked 1009 Read 10 Done Sent Done false
112952024-04-20T01:12:52.433Z 11 Acked 1010 Write 6 Done Sent Done false
112962024-04-20T01:12:52.433Z 12 Acked 1011 Write 8 Done Sent Done false
112972024-04-20T01:12:52.433Z 13 Acked 1012 Write 3 Done Sent Done false
112982024-04-20T01:12:52.433Z 14 Acked 1013 Write 2 Done Sent Done false
112992024-04-20T01:12:52.433Z 15 Acked 1014 Write 10 Done Sent Done false
113002024-04-20T01:12:52.433Z 16 Acked 1015 Read 9 Done Sent Done false
113012024-04-20T01:12:52.433Z 17 Acked 1016 Write 8 Done Sent Done false
113022024-04-20T01:12:52.433Z 18 Acked 1017 Read 3 Done Sent Done false
113032024-04-20T01:12:52.433Z 19 Acked 1018 Read 7 Done Sent Done false
113042024-04-20T01:12:52.433Z 20 Acked 1019 Write 10 Done Sent Done false
113052024-04-20T01:12:52.433Z 21 Acked 1020 Read 9 Done Sent Done false
113062024-04-20T01:12:52.433Z 22 Acked 1021 Read 1 Done Sent Done false
113072024-04-20T01:12:52.433Z 23 Acked 1022 Read 4 Done Sent Done false
113082024-04-20T01:12:52.433Z 24 Acked 1023 Write 7 Done Sent Done false
113092024-04-20T01:12:52.433Z 25 Acked 1024 Read 3 Done Sent Done false
113102024-04-20T01:12:52.433Z 26 Acked 1025 Read 9 Done Sent Done false
113112024-04-20T01:12:52.434Z 27 Acked 1026 Read 3 Done Sent Done false
113122024-04-20T01:12:52.434Z 28 Acked 1027 Write 6 Done Sent Done false
113132024-04-20T01:12:52.434Z 29 Acked 1028 Write 5 Done Sent Done false
113142024-04-20T01:12:52.434Z 30 Acked 1029 Write 8 Sent Sent Sent false
113152024-04-20T01:12:52.434Z STATES DS:0 DS:1 DS:2 TOTAL
113162024-04-20T01:12:52.434Z New 0 0 0 0
113172024-04-20T01:12:52.434Z Sent 1 28 1 30
113182024-04-20T01:12:52.434Z Done 29 2 29 60
113192024-04-20T01:12:52.434Z Skipped 0 0 0 0
113202024-04-20T01:12:52.434Z Error 0 0 0 0
113212024-04-20T01:12:52.434ZLast Flush: 1004 0 1004
113222024-04-20T01:12:52.434ZDownstairs last five completed:
113232024-04-20T01:12:52.434ZUpstairs last five completed: 30 29 28 27 26
113242024-04-20T01:12:52.434Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:12:50.934295368Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1587}
113252024-04-20T01:12:52.438ZWrote out file "/tmp/test_repair_verify.data"
113262024-04-20T01:12:52.438Z
113272024-04-20T01:12:52.441ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
113282024-04-20T01:12:52.458ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113292024-04-20T01:12:52.458Z 0 000-019 94 94 94 157 157 157 T F T <---
113302024-04-20T01:12:52.459Z 1 020-039 98 98 98 164 164 164 T F T <---
113312024-04-20T01:12:52.459Z 2 040-059 101 101 101 170 170 170 F F F
113322024-04-20T01:12:52.459Z 3 060-079 103 103 103 176 176 176 T F T <---
113332024-04-20T01:12:52.459Z 4 080-099 94 94 94 157 157 157 T F T <---
113342024-04-20T01:12:52.459Z 5 100-119 102 102 102 172 172 172 T F T <---
113352024-04-20T01:12:52.459Z 6 120-139 102 102 102 172 172 172 F F F
113362024-04-20T01:12:52.459Z 7 140-159 98 98 98 161 161 161 F F F
113372024-04-20T01:12:52.459Z 8 160-179 98 98 98 161 161 161 T F T <---
113382024-04-20T01:12:52.459Z 9 180-199 103 103 103 176 176 176 F F F
113392024-04-20T01:12:52.459Z 10 200-219 100 100 100 167 167 167 F F F
113402024-04-20T01:12:52.459Z 11 220-239 98 98 98 165 165 165 F F F
113412024-04-20T01:12:52.463Z 12 240-259 98 98 98 161 161 161 T F T <---
113422024-04-20T01:12:52.463Z 13 260-279 100 100 100 168 168 168 T F T <---
113432024-04-20T01:12:52.463Z 14 280-299 99 99 99 166 166 166 F F F
113442024-04-20T01:12:52.463Z 15 300-319 99 99 99 166 166 166 F F F
113452024-04-20T01:12:52.463Z 16 320-339 104 96 104 177 159 177 T T T <---
113462024-04-20T01:12:52.463Z 17 340-359 98 98 98 163 163 163 F F F
113472024-04-20T01:12:52.463Z 18 360-379 100 100 100 169 169 169 T F T <---
113482024-04-20T01:12:52.463Z 19 380-399 96 96 96 159 159 159 F F F
113492024-04-20T01:12:52.463Z 20 400-419 103 103 103 176 176 176 F F F
113502024-04-20T01:12:52.463Z 21 420-439 90 90 90 151 151 151 F F F
113512024-04-20T01:12:52.463Z 22 440-459 103 103 103 176 176 176 T F T <---
113522024-04-20T01:12:52.463Z 23 460-479 100 100 100 167 167 167 T F T <---
113532024-04-20T01:12:52.463Z 24 480-499 104 103 104 177 176 177 T T T <---
113542024-04-20T01:12:52.463Z 25 500-519 102 102 102 174 174 174 T F T <---
113552024-04-20T01:12:52.464Z 26 520-539 103 103 103 176 176 176 F F F
113562024-04-20T01:12:52.464Z 27 540-559 100 100 100 168 168 168 F F F
113572024-04-20T01:12:52.464Z 28 560-579 98 98 98 161 161 161 F F F
113582024-04-20T01:12:52.464Z 29 580-599 102 102 102 174 174 174 F F F
113592024-04-20T01:12:52.464ZMax gen: 104, Max flush: 177
113602024-04-20T01:12:52.464ZError: Difference in extent metadata found!
113612024-04-20T01:12:52.464ZA Difference in extent metadata is expected here
113622024-04-20T01:12:52.464ZOn loop 52
113632024-04-20T01:12:52.464Z
113642024-04-20T01:12:52.467ZVerifying data now
113652024-04-20T01:13:07.115ZNo change after verify
113662024-04-20T01:13:07.115ZLoop: 52 Downstairs dump after verify (and repair):
113672024-04-20T01:13:07.129ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113682024-04-20T01:13:07.129Z 0 000-019 105 105 105 178 178 178 F F F
113692024-04-20T01:13:07.133Z 1 020-039 105 105 105 178 178 178 F F F
113702024-04-20T01:13:07.133Z 2 040-059 101 101 101 170 170 170 F F F
113712024-04-20T01:13:07.133Z 3 060-079 105 105 105 178 178 178 F F F
113722024-04-20T01:13:07.133Z 4 080-099 105 105 105 178 178 178 F F F
113732024-04-20T01:13:07.133Z 5 100-119 105 105 105 178 178 178 F F F
113742024-04-20T01:13:07.133Z 6 120-139 102 102 102 172 172 172 F F F
113752024-04-20T01:13:07.134Z 7 140-159 98 98 98 161 161 161 F F F
113762024-04-20T01:13:07.134Z 8 160-179 105 105 105 178 178 178 F F F
113772024-04-20T01:13:07.134Z 9 180-199 103 103 103 176 176 176 F F F
113782024-04-20T01:13:07.134Z 10 200-219 100 100 100 167 167 167 F F F
113792024-04-20T01:13:07.134Z 11 220-239 98 98 98 165 165 165 F F F
113802024-04-20T01:13:07.134Z 12 240-259 105 105 105 178 178 178 F F F
113812024-04-20T01:13:07.134Z 13 260-279 105 105 105 178 178 178 F F F
113822024-04-20T01:13:07.134Z 14 280-299 99 99 99 166 166 166 F F F
113832024-04-20T01:13:07.134Z 15 300-319 99 99 99 166 166 166 F F F
113842024-04-20T01:13:07.134Z 16 320-339 105 105 105 178 178 178 F F F
113852024-04-20T01:13:07.134Z 17 340-359 98 98 98 163 163 163 F F F
113862024-04-20T01:13:07.134Z 18 360-379 105 105 105 178 178 178 F F F
113872024-04-20T01:13:07.134Z 19 380-399 96 96 96 159 159 159 F F F
113882024-04-20T01:13:07.134Z 20 400-419 103 103 103 176 176 176 F F F
113892024-04-20T01:13:07.134Z 21 420-439 90 90 90 151 151 151 F F F
113902024-04-20T01:13:07.134Z 22 440-459 105 105 105 178 178 178 F F F
113912024-04-20T01:13:07.134Z 23 460-479 105 105 105 178 178 178 F F F
113922024-04-20T01:13:07.134Z 24 480-499 105 105 105 178 178 178 F F F
113932024-04-20T01:13:07.134Z 25 500-519 105 105 105 178 178 178 F F F
113942024-04-20T01:13:07.134Z 26 520-539 103 103 103 176 176 176 F F F
113952024-04-20T01:13:07.134Z 27 540-559 100 100 100 168 168 168 F F F
113962024-04-20T01:13:07.134Z 28 560-579 98 98 98 161 161 161 F F F
113972024-04-20T01:13:07.134Z 29 580-599 102 102 102 174 174 174 F F F
113982024-04-20T01:13:07.135ZMax gen: 105, Max flush: 178
113992024-04-20T01:13:07.135Z
114002024-04-20T01:13:07.135ZBegin loop 53
114012024-04-20T01:13:07.135ZDownstairs to restart: 2
114022024-04-20T01:13:07.151Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653150511Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595}
114032024-04-20T01:13:07.151Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653427882Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595}
114042024-04-20T01:13:07.157Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653466512Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595}
114052024-04-20T01:13:07.157ZCrucible runtime is spawned
114062024-04-20T01:13:07.157Z{"msg":"Crucible 4490d985-8bb0-413c-9bbc-3bc5e83996e8 has session id: d90c07e0-9245-4c4d-a566-c752002226b2","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653519932Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114072024-04-20T01:13:07.157Z{"msg":"Upstairs opts: Upstairs UUID: 4490d985-8bb0-413c-9bbc-3bc5e83996e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653617702Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114082024-04-20T01:13:07.157Z{"msg":"Crucible stats registered with UUID: 4490d985-8bb0-413c-9bbc-3bc5e83996e8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653651273Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114092024-04-20T01:13:07.157Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653668422Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595}
114102024-04-20T01:13:07.157Z{"msg":"4490d985-8bb0-413c-9bbc-3bc5e83996e8 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653683533Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114112024-04-20T01:13:07.157Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653699782Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"0","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114122024-04-20T01:13:07.157Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653719082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"0","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114132024-04-20T01:13:07.157Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653760903Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"1","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114142024-04-20T01:13:07.158Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653818433Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"1","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114152024-04-20T01:13:07.158Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653851983Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"2","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114162024-04-20T01:13:07.158Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653877933Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"2","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114172024-04-20T01:13:07.158Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653905203Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114182024-04-20T01:13:07.158Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.653945133Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114192024-04-20T01:13:07.158Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.654071033Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114202024-04-20T01:13:07.158Z{"msg":"ds_connection connected from Ok(127.0.0.1:63380)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.654123203Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114212024-04-20T01:13:07.158Z{"msg":"ds_connection connected from Ok(127.0.0.1:43355)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.654196544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114222024-04-20T01:13:07.158Z{"msg":"ds_connection connected from Ok(127.0.0.1:45686)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.654242254Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114232024-04-20T01:13:07.158Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.654817405Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"0","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114242024-04-20T01:13:07.158Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.654890215Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"0","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114252024-04-20T01:13:07.158Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.654941425Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"1","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114262024-04-20T01:13:07.158Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.654972325Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"1","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114272024-04-20T01:13:07.158Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.655409666Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"0","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114282024-04-20T01:13:07.158Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.655560607Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"1","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114292024-04-20T01:13:07.158Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.655597797Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"2","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114302024-04-20T01:13:07.158Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.655622927Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"2","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114312024-04-20T01:13:07.158Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.655854697Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"0","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114322024-04-20T01:13:07.158Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.655906158Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"0","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114332024-04-20T01:13:07.158Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.655935078Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114342024-04-20T01:13:07.158Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.655976178Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"1","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114352024-04-20T01:13:07.158Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.656013498Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"1","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114362024-04-20T01:13:07.159Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.656047068Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114372024-04-20T01:13:07.159Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.656204788Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"2","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114382024-04-20T01:13:07.159Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.656584599Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"2","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114392024-04-20T01:13:07.159Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.656629459Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"2","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114402024-04-20T01:13:07.159Z{"msg":"[0]R flush_numbers[0..12]: [178, 178, 170, 178, 178, 178, 172, 161, 178, 176, 167, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.656667319Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114412024-04-20T01:13:07.159Z{"msg":"[0]R generation[0..12]: [105, 105, 101, 105, 105, 105, 102, 98, 105, 103, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.656702569Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114422024-04-20T01:13:07.159Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.656738649Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114432024-04-20T01:13:07.159Z{"msg":"[1]R flush_numbers[0..12]: [178, 178, 170, 178, 178, 178, 172, 161, 178, 176, 167, 165]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
114442024-04-20T01:13:07.159Z,"time":"2024-04-20T01:13:05.656774759Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114452024-04-20T01:13:07.159Z{"msg":"[1]R generation[0..12]: [105, 105, 101, 105, 105, 105, 102, 98, 105, 103, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.656811529Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114462024-04-20T01:13:07.159Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.65685044Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114472024-04-20T01:13:07.159Z{"msg":"[2]R flush_numbers[0..12]: [178, 178, 170, 178, 178, 178, 172, 161, 178, 176, 167, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.65690854Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114482024-04-20T01:13:07.159Z{"msg":"[2]R generation[0..12]: [105, 105, 101, 105, 105, 105, 102, 98, 105, 103, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.65694745Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114492024-04-20T01:13:07.159Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.65698641Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114502024-04-20T01:13:07.159Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.6570243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114512024-04-20T01:13:07.164Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.65706141Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114522024-04-20T01:13:07.164Z{"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.6570965Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114532024-04-20T01:13:07.164Z{"msg":"All extents match","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
114542024-04-20T01:13:07.164Z,"time":"2024-04-20T01:13:05.65713377Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"":"downstairs","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114552024-04-20T01:13:07.164Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.65717236Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114562024-04-20T01:13:07.164Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.65719573Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"0","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114572024-04-20T01:13:07.164Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.6572112Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"1","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114582024-04-20T01:13:07.164Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.65723163Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"client":"2","":"downstairs","session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114592024-04-20T01:13:07.164Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.657245321Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114602024-04-20T01:13:07.164Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.65726429Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114612024-04-20T01:13:07.164Z{"msg":"4490d985-8bb0-413c-9bbc-3bc5e83996e8 is now active with session: d90c07e0-9245-4c4d-a566-c752002226b2","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.65727755Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114622024-04-20T01:13:07.164Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.657296381Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595,"session_id":"d90c07e0-9245-4c4d-a566-c752002226b2"}
114632024-04-20T01:13:07.164Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:05.657308981Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595}
114642024-04-20T01:13:07.164ZLoading write count information from file "/tmp/test_repair_verify.data"
114652024-04-20T01:13:07.164ZRun Repair workload
114662024-04-20T01:13:07.164Z01/30 Write block 161 len 40960 data: 10 8 7 6 6 3 2 4 3 5
114672024-04-20T01:13:07.164Z02/30 Read block 195 len 40960
114682024-04-20T01:13:07.165Z03/30 Write block 321 len 28672 data: 11 11 11 10 9 10 8
114692024-04-20T01:13:07.165Z04/30 Write block 3 len 16384 data: 6 7 6 6
114702024-04-20T01:13:07.165Z05/30 Write block 475 len 16384 data: 13 10 11 9
114712024-04-20T01:13:07.165Z06/30 Read block 537 len 36864
114722024-04-20T01:13:07.171Z07/30 Write block 422 len 36864 data: 7 6 5 7 6 6 6 9 9
114732024-04-20T01:13:07.171Z08/30 Write block 287 len 20480 data: 7 7 8 6 6
114742024-04-20T01:13:07.171Z09/30 Write block 414 len 12288 data: 12 11 12
114752024-04-20T01:13:07.171Z10/30 Write block 57 len 16384 data: 9 11 10 11
114762024-04-20T01:13:07.171Z11/30 Write block 130 len 20480 data: 12 11 12 10 9
114772024-04-20T01:13:07.171Z12/30 Read block 578 len 20480
114782024-04-20T01:13:07.175Z13/30 Write block 307 len 32768 data: 6 6 4 3 3 3 4 5
114792024-04-20T01:13:07.175Z14/30 Read block 586 len 24576
114802024-04-20T01:13:07.175Z15/30 Read block 469 len 24576
114812024-04-20T01:13:07.175Z16/30 Read block 368 len 32768
114822024-04-20T01:13:07.175Z17/30 Read block 432 len 24576
114832024-04-20T01:13:07.175Z18/30 Read block 515 len 24576
114842024-04-20T01:13:07.175Z19/30 Flush
114852024-04-20T01:13:07.179Z20/30 Write block 332 len 8192 data: 13 11
114862024-04-20T01:13:07.179Z21/30 Read block 109 len 8192
114872024-04-20T01:13:07.179Z22/30 Read block 176 len 4096
114882024-04-20T01:13:07.182Z23/30 Flush
114892024-04-20T01:13:07.182Z24/30 Write block 506 len 4096 data: 10
114902024-04-20T01:13:07.182Z25/30 Read block 244 len 40960
114912024-04-20T01:13:07.182Z26/30 Read block 510 len 36864
114922024-04-20T01:13:07.186Z27/30 Read block 185 len 36864
114932024-04-20T01:13:07.186Z28/30 Read block 453 len 16384
114942024-04-20T01:13:07.186Z29/30 Write block 54 len 12288 data: 13 13 12
114952024-04-20T01:13:07.186Z30/30 Write block 165 len 24576 data: 7 4 3 5 4 6
114962024-04-20T01:13:07.191Z----------------------------------------------------------------
114972024-04-20T01:13:07.191Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
114982024-04-20T01:13:07.191ZStates: Active Active Active
114992024-04-20T01:13:07.191ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
115002024-04-20T01:13:07.191Z 1 Acked 1000 Write 10 Done Done Done false
115012024-04-20T01:13:07.191Z 2 Acked 1001 Read 10 Done Done Done false
115022024-04-20T01:13:07.191Z 3 Acked 1002 Write 7 Done Done Done false
115032024-04-20T01:13:07.191Z 4 Acked 1003 Write 4 Done Done Done false
115042024-04-20T01:13:07.191Z 5 Acked 1004 Write 4 Done Done Done false
115052024-04-20T01:13:07.191Z 6 Acked 1005 Read 9 Done Done Done false
115062024-04-20T01:13:07.191Z 7 Acked 1006 Write 9 Done Done Done false
115072024-04-20T01:13:07.191Z 8 Acked 1007 Write 5 Done Done Done false
115082024-04-20T01:13:07.191Z 9 Acked 1008 Write 3 Done Done Sent false
115092024-04-20T01:13:07.191Z 10 Acked 1009 Write 4 Done Done Sent false
115102024-04-20T01:13:07.191Z 11 Acked 1010 Write 5 Done Done Sent false
115112024-04-20T01:13:07.191Z 12 Acked 1011 Read 5 Done Done Sent false
115122024-04-20T01:13:07.191Z 13 Acked 1012 Write 8 Done Done Sent false
115132024-04-20T01:13:07.191Z 14 Acked 1013 Read 6 Done Done Sent false
115142024-04-20T01:13:07.191Z 15 Acked 1014 Read 6 Done Done Sent false
115152024-04-20T01:13:07.192Z 16 Acked 1015 Read 8 Done Done Sent false
115162024-04-20T01:13:07.192Z 17 Acked 1016 Read 6 Done Done Sent false
115172024-04-20T01:13:07.192Z 18 Acked 1017 Read 6 Done Done Sent false
115182024-04-20T01:13:07.192Z 19 Acked 1018 Flush 0 Done Done Sent false
115192024-04-20T01:13:07.192Z 20 Acked 1019 Write 2 Done Done Sent false
115202024-04-20T01:13:07.192Z 21 Acked 1020 Read 2 Done Done Sent false
115212024-04-20T01:13:07.192Z 22 Acked 1021 Read 1 Done Done Sent false
115222024-04-20T01:13:07.192Z 23 Acked 1022 Flush 0 Done Done Sent false
115232024-04-20T01:13:07.192Z 24 Acked 1023 Write 1 Done Done Sent false
115242024-04-20T01:13:07.192Z 25 Acked 1024 Read 10 Done Done Sent false
115252024-04-20T01:13:07.192Z 26 Acked 1025 Read 9 Done Done Sent false
115262024-04-20T01:13:07.192Z 27 Acked 1026 Read 9 Done Done Sent false
115272024-04-20T01:13:07.192Z 28 Acked 1027 Read 4 Done Done Sent false
115282024-04-20T01:13:07.192Z 29 Acked 1028 Write 3 Done Done Sent false
115292024-04-20T01:13:07.192Z 30 Acked 1029 Write 6 Sent Sent Sent false
115302024-04-20T01:13:07.192Z STATES DS:0 DS:1 DS:2 TOTAL
115312024-04-20T01:13:07.192Z New 0 0 0 0
115322024-04-20T01:13:07.192Z Sent 1 1 22 24
115332024-04-20T01:13:07.192Z Done 29 29 8 66
115342024-04-20T01:13:07.192Z Skipped 0 0 0 0
115352024-04-20T01:13:07.192Z Error 0 0 0 0
115362024-04-20T01:13:07.192ZLast Flush: 1022 1022 0
115372024-04-20T01:13:07.192ZDownstairs last five completed:
115382024-04-20T01:13:07.193ZUpstairs last five completed: 30 29 28 27 26
115392024-04-20T01:13:07.193Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:13:05.688974271Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1595}
115402024-04-20T01:13:07.193ZWrote out file "/tmp/test_repair_verify.data"
115412024-04-20T01:13:07.196Z
115422024-04-20T01:13:07.196ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
115432024-04-20T01:13:07.213ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115442024-04-20T01:13:07.213Z 0 000-019 106 106 105 179 179 178 F F T <---
115452024-04-20T01:13:07.213Z 1 020-039 105 105 105 178 178 178 F F F
115462024-04-20T01:13:07.213Z 2 040-059 106 106 101 179 179 170 T T F <---
115472024-04-20T01:13:07.213Z 3 060-079 106 106 105 179 179 178 F F F <---
115482024-04-20T01:13:07.213Z 4 080-099 105 105 105 178 178 178 F F F
115492024-04-20T01:13:07.213Z 5 100-119 105 105 105 178 178 178 F F F
115502024-04-20T01:13:07.213Z 6 120-139 106 106 102 179 179 172 F F F <---
115512024-04-20T01:13:07.213Z 7 140-159 98 98 98 161 161 161 F F F
115522024-04-20T01:13:07.213Z 8 160-179 106 106 105 179 179 178 T T T <---
115532024-04-20T01:13:07.213Z 9 180-199 103 103 103 176 176 176 F F F
115542024-04-20T01:13:07.213Z 10 200-219 100 100 100 167 167 167 F F F
115552024-04-20T01:13:07.214Z 11 220-239 98 98 98 165 165 165 F F F
115562024-04-20T01:13:07.214Z 12 240-259 105 105 105 178 178 178 F F F
115572024-04-20T01:13:07.214Z 13 260-279 105 105 105 178 178 178 F F F
115582024-04-20T01:13:07.214Z 14 280-299 106 106 99 179 179 166 F F T <---
115592024-04-20T01:13:07.214Z 15 300-319 106 106 99 179 179 166 F F F <---
115602024-04-20T01:13:07.214Z 16 320-339 106 106 105 180 180 178 F F T <---
115612024-04-20T01:13:07.214Z 17 340-359 98 98 98 163 163 163 F F F
115622024-04-20T01:13:07.214Z 18 360-379 105 105 105 178 178 178 F F F
115632024-04-20T01:13:07.214Z 19 380-399 96 96 96 159 159 159 F F F
115642024-04-20T01:13:07.214Z 20 400-419 106 106 103 179 179 176 F F F <---
115652024-04-20T01:13:07.214Z 21 420-439 106 106 90 179 179 151 F F T <---
115662024-04-20T01:13:07.214Z 22 440-459 105 105 105 178 178 178 F F F
115672024-04-20T01:13:07.214Z 23 460-479 106 106 105 179 179 178 F F T <---
115682024-04-20T01:13:07.214Z 24 480-499 105 105 105 178 178 178 F F F
115692024-04-20T01:13:07.214Z 25 500-519 105 105 105 178 178 178 T T F <---
115702024-04-20T01:13:07.214Z 26 520-539 103 103 103 176 176 176 F F F
115712024-04-20T01:13:07.214Z 27 540-559 100 100 100 168 168 168 F F F
115722024-04-20T01:13:07.214Z 28 560-579 98 98 98 161 161 161 F F F
115732024-04-20T01:13:07.214Z 29 580-599 102 102 102 174 174 174 F F F
115742024-04-20T01:13:07.214ZMax gen: 106, Max flush: 180
115752024-04-20T01:13:07.214ZError: Difference in extent metadata found!
115762024-04-20T01:13:07.214ZA Difference in extent metadata is expected here
115772024-04-20T01:13:07.214ZOn loop 53
115782024-04-20T01:13:07.214Z
115792024-04-20T01:13:07.218ZVerifying data now
115802024-04-20T01:13:17.766ZNo change after verify
115812024-04-20T01:13:17.766ZLoop: 53 Downstairs dump after verify (and repair):
115822024-04-20T01:13:17.780ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115832024-04-20T01:13:17.780Z 0 000-019 106 106 106 179 179 179 F F F
115842024-04-20T01:13:17.780Z 1 020-039 105 105 105 178 178 178 F F F
115852024-04-20T01:13:17.780Z 2 040-059 107 107 107 181 181 181 F F F
115862024-04-20T01:13:17.780Z 3 060-079 106 106 106 179 179 179 F F F
115872024-04-20T01:13:17.780Z 4 080-099 105 105 105 178 178 178 F F F
115882024-04-20T01:13:17.780Z 5 100-119 105 105 105 178 178 178 F F F
115892024-04-20T01:13:17.780Z 6 120-139 106 106 106 179 179 179 F F F
115902024-04-20T01:13:17.784Z 7 140-159 98 98 98 161 161 161 F F F
115912024-04-20T01:13:17.784Z 8 160-179 107 107 107 181 181 181 F F F
115922024-04-20T01:13:17.784Z 9 180-199 103 103 103 176 176 176 F F F
115932024-04-20T01:13:17.784Z 10 200-219 100 100 100 167 167 167 F F F
115942024-04-20T01:13:17.784Z 11 220-239 98 98 98 165 165 165 F F F
115952024-04-20T01:13:17.784Z 12 240-259 105 105 105 178 178 178 F F F
115962024-04-20T01:13:17.784Z 13 260-279 105 105 105 178 178 178 F F F
115972024-04-20T01:13:17.784Z 14 280-299 106 106 106 179 179 179 F F F
115982024-04-20T01:13:17.784Z 15 300-319 106 106 106 179 179 179 F F F
115992024-04-20T01:13:17.784Z 16 320-339 106 106 106 180 180 180 F F F
116002024-04-20T01:13:17.784Z 17 340-359 98 98 98 163 163 163 F F F
116012024-04-20T01:13:17.784Z 18 360-379 105 105 105 178 178 178 F F F
116022024-04-20T01:13:17.784Z 19 380-399 96 96 96 159 159 159 F F F
116032024-04-20T01:13:17.784Z 20 400-419 106 106 106 179 179 179 F F F
116042024-04-20T01:13:17.784Z 21 420-439 106 106 106 179 179 179 F F F
116052024-04-20T01:13:17.784Z 22 440-459 105 105 105 178 178 178 F F F
116062024-04-20T01:13:17.785Z 23 460-479 106 106 106 179 179 179 F F F
116072024-04-20T01:13:17.785Z 24 480-499 105 105 105 178 178 178 F F F
116082024-04-20T01:13:17.785Z 25 500-519 107 107 107 181 181 181 F F F
116092024-04-20T01:13:17.785Z 26 520-539 103 103 103 176 176 176 F F F
116102024-04-20T01:13:17.785Z 27 540-559 100 100 100 168 168 168 F F F
116112024-04-20T01:13:17.785Z 28 560-579 98 98 98 161 161 161 F F F
116122024-04-20T01:13:17.785Z 29 580-599 102 102 102 174 174 174 F F F
116132024-04-20T01:13:17.785ZMax gen: 107, Max flush: 181
116142024-04-20T01:13:17.785Z
116152024-04-20T01:13:17.785ZBegin loop 54
116162024-04-20T01:13:17.785ZDownstairs to restart: 2
116172024-04-20T01:13:17.803Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.305406733Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603}
116182024-04-20T01:13:17.803Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.305688054Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603}
116192024-04-20T01:13:17.811Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.305724954Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603}
116202024-04-20T01:13:17.811Z{"msg":"Crucible 069d63a3-a8e6-4a0b-b4a4-1ff006b1d56e has session id: dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.305747124Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116212024-04-20T01:13:17.811Z{"msg":"Crucible runtime is spawned
116222024-04-20T01:13:17.811ZUpstairs opts: Upstairs UUID: 069d63a3-a8e6-4a0b-b4a4-1ff006b1d56e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.305795394Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116232024-04-20T01:13:17.811Z{"msg":"Crucible stats registered with UUID: 069d63a3-a8e6-4a0b-b4a4-1ff006b1d56e","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.305819664Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116242024-04-20T01:13:17.811Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.305843144Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603}
116252024-04-20T01:13:17.812Z{"msg":"069d63a3-a8e6-4a0b-b4a4-1ff006b1d56e active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.305968065Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116262024-04-20T01:13:17.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.306005965Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"0","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116272024-04-20T01:13:17.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.306033435Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"0","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116282024-04-20T01:13:17.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.306078875Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"1","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116292024-04-20T01:13:17.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.306105025Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"1","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116302024-04-20T01:13:17.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.306136495Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"2","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116312024-04-20T01:13:17.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.306163155Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"2","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116322024-04-20T01:13:17.812Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.306204645Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116332024-04-20T01:13:17.812Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.306254145Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116342024-04-20T01:13:17.812Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.306320575Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116352024-04-20T01:13:17.812Z{"msg":"ds_connection connected from Ok(127.0.0.1:45101)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.306373786Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116362024-04-20T01:13:17.812Z{"msg":"ds_connection connected from Ok(127.0.0.1:38516)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.306422876Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116372024-04-20T01:13:17.812Z{"msg":"ds_connection connected from Ok(127.0.0.1:39787)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.306449776Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116382024-04-20T01:13:17.812Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.307015407Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"1","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116392024-04-20T01:13:17.812Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.307045737Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"1","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116402024-04-20T01:13:17.812Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.307061377Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"0","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116412024-04-20T01:13:17.812Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.307093937Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"0","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116422024-04-20T01:13:17.812Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.307504978Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"2","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116432024-04-20T01:13:17.812Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.307572608Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"2","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116442024-04-20T01:13:17.812Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.307603718Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"0","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116452024-04-20T01:13:17.812Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.307694029Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"1","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116462024-04-20T01:13:17.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308133459Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"1","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116472024-04-20T01:13:17.812Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.30817085Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"1","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116482024-04-20T01:13:17.813Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.30818753Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116492024-04-20T01:13:17.813Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.30821931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"0","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116502024-04-20T01:13:17.813Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.30824019Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"0","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116512024-04-20T01:13:17.813Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.30826882Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116522024-04-20T01:13:17.813Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.30828683Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"2","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116532024-04-20T01:13:17.813Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308696681Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"2","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116542024-04-20T01:13:17.813Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308727191Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"2","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116552024-04-20T01:13:17.813Z{"msg":"[0]R flush_numbers[0..12]: [179, 178, 181, 179, 178, 178, 179, 161, 181, 176, 167, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308744481Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116562024-04-20T01:13:17.813Z{"msg":"[0]R generation[0..12]: [106, 105, 107, 106, 105, 105, 106, 98, 107, 103, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308776621Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116572024-04-20T01:13:17.813Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308807981Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116582024-04-20T01:13:17.813Z{"msg":"[1]R flush_numbers[0..12]: [179, 178, 181, 179, 178, 178, 179, 161, 181, 176, 167, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308843051Z","hostname":"Wait for a query_work_queue command to finish before sending IO
116592024-04-20T01:13:17.813Zw-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116602024-04-20T01:13:17.813Z{"msg":"[1]R generation[0..12]: [106, 105, 107, 106, 105, 105, 106, 98, 107, 103, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308881271Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116612024-04-20T01:13:17.813Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308915261Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116622024-04-20T01:13:17.813Z{"msg":"[2]R flush_numbers[0..12]: [179, 178, 181, 179, 178, 178, 179, 161, 181, 176, 167, 165]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308941431Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116632024-04-20T01:13:17.813Z{"msg":"[2]R generation[0..12]: [106, 105, 107, 106, 105, 105, 106, 98, 107, 103, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308958791Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116642024-04-20T01:13:17.813Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308973641Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116652024-04-20T01:13:17.813Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.308987061Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116662024-04-20T01:13:17.813Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.309000331Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116672024-04-20T01:13:17.813Z{"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.309020692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116682024-04-20T01:13:17.813Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.309033732Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"":"downstairs","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116692024-04-20T01:13:17.813Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.309054321Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116702024-04-20T01:13:17.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.309066661Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"0","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116712024-04-20T01:13:17.814Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.309087002Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"1","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116722024-04-20T01:13:17.814Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.309100402Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"client":"2","":"downstairs","session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116732024-04-20T01:13:17.814Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.309114112Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116742024-04-20T01:13:17.814Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.309137332Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116752024-04-20T01:13:17.814Z{"msg":"069d63a3-a8e6-4a0b-b4a4-1ff006b1d56e is now active with session: dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.309151122Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116762024-04-20T01:13:17.814Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.309170142Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603,"session_id":"dbe8e2a6-0e8e-4a8e-b3b8-344f9737c203"}
116772024-04-20T01:13:17.814Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.309182402Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603}
116782024-04-20T01:13:17.814ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
116792024-04-20T01:13:17.818ZLoading write count information from file "/tmp/test_repair_verify.data"
116802024-04-20T01:13:17.818ZRun Repair workload
116812024-04-20T01:13:17.818Z01/30 Write block 431 len 8192 data: 8 7
116822024-04-20T01:13:17.818Z02/30 Read block 311 len 28672
116832024-04-20T01:13:17.818Z03/30 Read block 422 len 20480
116842024-04-20T01:13:17.818Z04/30 Write block 212 len 36864 data: 7 8 8 7 8 12 11 9 11
116852024-04-20T01:13:17.818Z05/30 Read block 62 len 12288
116862024-04-20T01:13:17.818Z06/30 Write block 14 len 12288 data: 5 8 9
116872024-04-20T01:13:17.818Z07/30 Read block 267 len 24576
116882024-04-20T01:13:17.818Z08/30 Write block 423 len 4096 data: 7
116892024-04-20T01:13:17.818Z09/30 Read block 360 len 16384
116902024-04-20T01:13:17.818Z10/30 Read block 97 len 16384
116912024-04-20T01:13:17.818Z11/30 Read block 549 len 16384
116922024-04-20T01:13:17.818Z12/30 Read block 5 len 36864
116932024-04-20T01:13:17.818Z13/30 Read block 25 len 24576
116942024-04-20T01:13:17.822Z14/30 Read block 533 len 40960
116952024-04-20T01:13:17.822Z15/30 Read block 249 len 28672
116962024-04-20T01:13:17.822Z16/30 Read block 532 len 12288
116972024-04-20T01:13:17.822Z17/30 Read block 316 len 16384
116982024-04-20T01:13:17.822Z18/30 Read block 467 len 24576
116992024-04-20T01:13:17.826Z19/30 Write block 406 len 24576 data: 9 8 9 10 11 13
117002024-04-20T01:13:17.826Z20/30 Write block 253 len 32768 data: 7 11 10 10 7 7 8 6
117012024-04-20T01:13:17.826Z21/30 Read block 335 len 20480
117022024-04-20T01:13:17.832Z22/30 Write block 433 len 40960 data: 7 10 11 12 11 7 8 8 9 10
117032024-04-20T01:13:17.832Z23/30 Write block 316 len 32768 data: 5 5 11 11 11 12 12 12
117042024-04-20T01:13:17.832Z24/30 Write block 44 len 24576 data: 4 4 4 5 5 7
117052024-04-20T01:13:17.832Z25/30 Read block 417 len 28672
117062024-04-20T01:13:17.836Z26/30 Read block 13 len 12288
117072024-04-20T01:13:17.836Z27/30 Write block 233 len 36864 data: 14 15 13 13 14 11 11 8 7
117082024-04-20T01:13:17.836Z28/30 Write block 403 len 16384 data: 7 7 8 10
117092024-04-20T01:13:17.837Z29/30 Write block 273 len 4096 data: 11
117102024-04-20T01:13:17.837Z30/30 Read block 313 len 32768
117112024-04-20T01:13:17.837Z----------------------------------------------------------------
117122024-04-20T01:13:17.837Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
117132024-04-20T01:13:17.837ZStates: Active Active Active
117142024-04-20T01:13:17.837ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
117152024-04-20T01:13:17.837Z 1 Acked 1000 Write 2 Done Done Sent false
117162024-04-20T01:13:17.837Z 2 Acked 1001 Read 7 Done Done Sent false
117172024-04-20T01:13:17.837Z 3 Acked 1002 Read 5 Done Done Sent false
117182024-04-20T01:13:17.837Z 4 Acked 1003 Write 9 Done Done Sent false
117192024-04-20T01:13:17.837Z 5 Acked 1004 Read 3 Done Done Sent false
117202024-04-20T01:13:17.837Z 6 Acked 1005 Write 3 Done Done Sent false
117212024-04-20T01:13:17.837Z 7 Acked 1006 Read 6 Done Done Sent false
117222024-04-20T01:13:17.837Z 8 Acked 1007 Write 1 Done Done Sent false
117232024-04-20T01:13:17.837Z 9 Acked 1008 Read 4 Done Done Sent false
117242024-04-20T01:13:17.837Z 10 Acked 1009 Read 4 Done Done Sent false
117252024-04-20T01:13:17.837Z 11 Acked 1010 Read 4 Done Done Sent false
117262024-04-20T01:13:17.837Z 12 Acked 1011 Read 9 Done Done Sent false
117272024-04-20T01:13:17.837Z 13 Acked 1012 Read 6 Done Done Sent false
117282024-04-20T01:13:17.837Z 14 Acked 1013 Read 10 Done Done Sent false
117292024-04-20T01:13:17.837Z 15 Acked 1014 Read 7 Done Done Sent false
117302024-04-20T01:13:17.837Z 16 Acked 1015 Read 3 Done Done Sent false
117312024-04-20T01:13:17.837Z 17 Acked 1016 Read 4 Done Done Sent false
117322024-04-20T01:13:17.838Z 18 Acked 1017 Read 6 Done Done Sent false
117332024-04-20T01:13:17.838Z 19 Acked 1018 Write 6 Done Done Sent false
117342024-04-20T01:13:17.838Z 20 Acked 1019 Write 8 Done Done Sent false
117352024-04-20T01:13:17.838Z 21 Acked 1020 Read 5 Done Done Sent false
117362024-04-20T01:13:17.838Z 22 Acked 1021 Write 10 Done Done Sent false
117372024-04-20T01:13:17.838Z 23 Acked 1022 Write 8 Done Done Sent false
117382024-04-20T01:13:17.838Z 24 Acked 1023 Write 6 Done Done Sent false
117392024-04-20T01:13:17.838Z 25 Acked 1024 Read 7 Done Done Sent false
117402024-04-20T01:13:17.838Z 26 Acked 1025 Read 3 Done Done Sent false
117412024-04-20T01:13:17.838Z 27 Acked 1026 Write 9 Done Done Sent false
117422024-04-20T01:13:17.838Z 28 Acked 1027 Write 4 Done Done Sent false
117432024-04-20T01:13:17.838Z 29 Acked 1028 Write 1 Done Done Sent false
117442024-04-20T01:13:17.838Z 30 Acked 1029 Read 8 Done Done Sent false
117452024-04-20T01:13:17.838Z STATES DS:0 DS:1 DS:2 TOTAL
117462024-04-20T01:13:17.838Z New 0 0 0 0
117472024-04-20T01:13:17.838Z Sent 0 0 30 30
117482024-04-20T01:13:17.838Z Done 30 30 0 60
117492024-04-20T01:13:17.838Z Skipped 0 0 0 0
117502024-04-20T01:13:17.838Z Error 0 0 0 0
117512024-04-20T01:13:17.838ZLast Flush: 0 0 0
117522024-04-20T01:13:17.838ZDownstairs last five completed:
117532024-04-20T01:13:17.838ZUpstairs last five completed: 30 29 28 27 26
117542024-04-20T01:13:17.838Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:13:16.336711793Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1603}
117552024-04-20T01:13:17.838ZWrote out file "/tmp/test_repair_verify.data"
117562024-04-20T01:13:17.842Z
117572024-04-20T01:13:17.842ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
117582024-04-20T01:13:17.860ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117592024-04-20T01:13:17.860Z 0 000-019 106 106 106 179 179 179 T T F <---
117602024-04-20T01:13:17.860Z 1 020-039 105 105 105 178 178 178 F F F
117612024-04-20T01:13:17.860Z 2 040-059 107 107 107 181 181 181 T T F <---
117622024-04-20T01:13:17.860Z 3 060-079 106 106 106 179 179 179 F F F
117632024-04-20T01:13:17.860Z 4 080-099 105 105 105 178 178 178 F F F
117642024-04-20T01:13:17.860Z 5 100-119 105 105 105 178 178 178 F F F
117652024-04-20T01:13:17.864Z 6 120-139 106 106 106 179 179 179 F F F
117662024-04-20T01:13:17.864Z 7 140-159 98 98 98 161 161 161 F F F
117672024-04-20T01:13:17.864Z 8 160-179 107 107 107 181 181 181 F F F
117682024-04-20T01:13:17.864Z 9 180-199 103 103 103 176 176 176 F F F
117692024-04-20T01:13:17.864Z 10 200-219 100 100 100 167 167 167 T T F <---
117702024-04-20T01:13:17.864Z 11 220-239 98 98 98 165 165 165 T T F <---
117712024-04-20T01:13:17.864Z 12 240-259 105 105 105 178 178 178 T T F <---
117722024-04-20T01:13:17.864Z 13 260-279 105 105 105 178 178 178 T T F <---
117732024-04-20T01:13:17.864Z 14 280-299 106 106 106 179 179 179 F F F
117742024-04-20T01:13:17.865Z 15 300-319 106 106 106 179 179 179 T T F <---
117752024-04-20T01:13:17.865Z 16 320-339 106 106 106 180 180 180 T T F <---
117762024-04-20T01:13:17.865Z 17 340-359 98 98 98 163 163 163 F F F
117772024-04-20T01:13:17.865Z 18 360-379 105 105 105 178 178 178 F F F
117782024-04-20T01:13:17.865Z 19 380-399 96 96 96 159 159 159 F F F
117792024-04-20T01:13:17.865Z 20 400-419 106 106 106 179 179 179 T T F <---
117802024-04-20T01:13:17.865Z 21 420-439 106 106 106 179 179 179 T T F <---
117812024-04-20T01:13:17.865Z 22 440-459 105 105 105 178 178 178 T T F <---
117822024-04-20T01:13:17.865Z 23 460-479 106 106 106 179 179 179 F F F
117832024-04-20T01:13:17.865Z 24 480-499 105 105 105 178 178 178 F F F
117842024-04-20T01:13:17.865Z 25 500-519 107 107 107 181 181 181 F F F
117852024-04-20T01:13:17.865Z 26 520-539 103 103 103 176 176 176 F F F
117862024-04-20T01:13:17.865Z 27 540-559 100 100 100 168 168 168 F F F
117872024-04-20T01:13:17.865Z 28 560-579 98 98 98 161 161 161 F F F
117882024-04-20T01:13:17.865Z 29 580-599 102 102 102 174 174 174 F F F
117892024-04-20T01:13:17.865ZMax gen: 107, Max flush: 181
117902024-04-20T01:13:17.865ZError: Difference in extent metadata found!
117912024-04-20T01:13:17.865ZA Difference in extent metadata is expected here
117922024-04-20T01:13:17.865ZOn loop 54
117932024-04-20T01:13:17.865Z
117942024-04-20T01:13:17.869ZVerifying data now
117952024-04-20T01:13:18.397ZNo change after verify
117962024-04-20T01:13:18.397ZLoop: 54 Downstairs dump after verify (and repair):
117972024-04-20T01:13:18.411ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117982024-04-20T01:13:18.411Z 0 000-019 108 108 108 182 182 182 F F F
117992024-04-20T01:13:18.411Z 1 020-039 105 105 105 178 178 178 F F F
118002024-04-20T01:13:18.411Z 2 040-059 108 108 108 182 182 182 F F F
118012024-04-20T01:13:18.411Z 3 060-079 106 106 106 179 179 179 F F F
118022024-04-20T01:13:18.411Z 4 080-099 105 105 105 178 178 178 F F F
118032024-04-20T01:13:18.415Z 5 100-119 105 105 105 178 178 178 F F F
118042024-04-20T01:13:18.415Z 6 120-139 106 106 106 179 179 179 F F F
118052024-04-20T01:13:18.415Z 7 140-159 98 98 98 161 161 161 F F F
118062024-04-20T01:13:18.415Z 8 160-179 107 107 107 181 181 181 F F F
118072024-04-20T01:13:18.415Z 9 180-199 103 103 103 176 176 176 F F F
118082024-04-20T01:13:18.415Z 10 200-219 108 108 108 182 182 182 F F F
118092024-04-20T01:13:18.415Z 11 220-239 108 108 108 182 182 182 F F F
118102024-04-20T01:13:18.415Z 12 240-259 108 108 108 182 182 182 F F F
118112024-04-20T01:13:18.415Z 13 260-279 108 108 108 182 182 182 F F F
118122024-04-20T01:13:18.415Z 14 280-299 106 106 106 179 179 179 F F F
118132024-04-20T01:13:18.416Z 15 300-319 108 108 108 182 182 182 F F F
118142024-04-20T01:13:18.416Z 16 320-339 108 108 108 182 182 182 F F F
118152024-04-20T01:13:18.416Z 17 340-359 98 98 98 163 163 163 F F F
118162024-04-20T01:13:18.416Z 18 360-379 105 105 105 178 178 178 F F F
118172024-04-20T01:13:18.416Z 19 380-399 96 96 96 159 159 159 F F F
118182024-04-20T01:13:18.416Z 20 400-419 108 108 108 182 182 182 F F F
118192024-04-20T01:13:18.416Z 21 420-439 108 108 108 182 182 182 F F F
118202024-04-20T01:13:18.416Z 22 440-459 108 108 108 182 182 182 F F F
118212024-04-20T01:13:18.416Z 23 460-479 106 106 106 179 179 179 F F F
118222024-04-20T01:13:18.416Z 24 480-499 105 105 105 178 178 178 F F F
118232024-04-20T01:13:18.416Z 25 500-519 107 107 107 181 181 181 F F F
118242024-04-20T01:13:18.416Z 26 520-539 103 103 103 176 176 176 F F F
118252024-04-20T01:13:18.416Z 27 540-559 100 100 100 168 168 168 F F F
118262024-04-20T01:13:18.416Z 28 560-579 98 98 98 161 161 161 F F F
118272024-04-20T01:13:18.416Z 29 580-599 102 102 102 174 174 174 F F F
118282024-04-20T01:13:18.416ZMax gen: 108, Max flush: 182
118292024-04-20T01:13:18.416Z
118302024-04-20T01:13:18.416ZBegin loop 55
118312024-04-20T01:13:18.416ZDownstairs to restart: 0
118322024-04-20T01:13:18.433Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.935629366Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611}
118332024-04-20T01:13:18.434Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.935896027Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611}
118342024-04-20T01:13:18.434ZCrucible runtime is spawned
118352024-04-20T01:13:18.441Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.935950297Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611}
118362024-04-20T01:13:18.441Z{"msg":"Crucible 381f8a76-ea17-42b5-b57d-4401e907ee94 has session id: 69aa9d44-6944-44f4-8809-258715a486da","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.935980037Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118372024-04-20T01:13:18.441Z{"msg":"Upstairs opts: Upstairs UUID: 381f8a76-ea17-42b5-b57d-4401e907ee94, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936014927Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118382024-04-20T01:13:18.441Z{"msg":"Crucible stats registered with UUID: 381f8a76-ea17-42b5-b57d-4401e907ee94","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936056797Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118392024-04-20T01:13:18.441Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936078597Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611}
118402024-04-20T01:13:18.441Z{"msg":"381f8a76-ea17-42b5-b57d-4401e907ee94 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936097457Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118412024-04-20T01:13:18.442Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936122657Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"0","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118422024-04-20T01:13:18.442Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936138737Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"0","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118432024-04-20T01:13:18.442Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936153947Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"1","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118442024-04-20T01:13:18.442Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936188628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"1","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118452024-04-20T01:13:18.442Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936211377Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"2","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118462024-04-20T01:13:18.442Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936254848Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"2","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118472024-04-20T01:13:18.442Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936295078Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118482024-04-20T01:13:18.442Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936336198Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118492024-04-20T01:13:18.442Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936373898Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118502024-04-20T01:13:18.442Z{"msg":"ds_connection connected from Ok(127.0.0.1:34191)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936466068Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118512024-04-20T01:13:18.442Z{"msg":"ds_connection connected from Ok(127.0.0.1:40851)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936519308Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118522024-04-20T01:13:18.442Z{"msg":"ds_connection connected from Ok(127.0.0.1:47914)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.936544228Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118532024-04-20T01:13:18.442Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.93728997Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"2","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118542024-04-20T01:13:18.442Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.93732204Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"2","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118552024-04-20T01:13:18.442Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.93734784Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"1","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118562024-04-20T01:13:18.442Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.9373881Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"1","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118572024-04-20T01:13:18.442Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.937748831Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"0","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118582024-04-20T01:13:18.442Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.937795301Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"0","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118592024-04-20T01:13:18.442Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.937896071Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"1","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118602024-04-20T01:13:18.442Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.937938151Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"2","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118612024-04-20T01:13:18.442Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.938384752Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"0","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118622024-04-20T01:13:18.442Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.938515633Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"2","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118632024-04-20T01:13:18.442Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.938549143Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"2","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118642024-04-20T01:13:18.443Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.938564563Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118652024-04-20T01:13:18.443Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.938578263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"1","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118662024-04-20T01:13:18.443Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.938591783Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"1","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118672024-04-20T01:13:18.443Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.938631173Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118682024-04-20T01:13:18.443Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939028764Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"0","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118692024-04-20T01:13:18.443Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939059774Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"0","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118702024-04-20T01:13:18.443Z{"msg":"[0]R flush_numbers[0..12]: [182, 178, 182, 179, 178, 178, 179, 161, 181, 176, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939075354Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118712024-04-20T01:13:18.443Z{"msg":"[0]R generation[0..12]: [108, 105, 108, 106, 105, 105, 106, 98, 107, 103, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939097344Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118722024-04-20T01:13:18.443Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939113314Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118732024-04-20T01:13:18.443Z{"msg":"[1]R flush_numbers[0..12]: [182, 178, 182, 179, 178, 178, 179, 161, 181, 176, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939134804Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118742024-04-20T01:13:18.443Z{"msg":"[1]R generation[0..12]: [108, 105, 108, 106, 105, 105, 106, 98, 107, 103, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939150484Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118752024-04-20T01:13:18.443Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939179004Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118762024-04-20T01:13:18.443ZWait for a query_work_queue command to finish before sending IO
118772024-04-20T01:13:18.443Z{"msg":"[2]R flush_numbers[0..12]: [182, 178, 182, 179, 178, 178, 179, 161, 181, 176, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939216404Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118782024-04-20T01:13:18.443Z{"msg":"[2]R generation[0..12]: [108, 105, 108, 106, 105, 105, 106, 98, 107, 103, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939254724Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118792024-04-20T01:13:18.443Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939294164Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118802024-04-20T01:13:18.443Z{"msg":"Max found gen is 109","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939325724Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118812024-04-20T01:13:18.443Z{"msg":"Generation requested: 110 >= found:109","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939360824Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118822024-04-20T01:13:18.443Z{"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939396975Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118832024-04-20T01:13:18.443Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939422045Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"":"downstairs","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118842024-04-20T01:13:18.443Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939442415Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"session_id":"69aa9d44-6944-44f4-8809-258715a486da"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118852024-04-20T01:13:18.443Z}
118862024-04-20T01:13:18.443Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939479185Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"0","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118872024-04-20T01:13:18.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939512305Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"1","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118882024-04-20T01:13:18.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939546025Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"client":"2","":"downstairs","session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118892024-04-20T01:13:18.444Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939579075Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118902024-04-20T01:13:18.444Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939609595Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118912024-04-20T01:13:18.444Z{"msg":"381f8a76-ea17-42b5-b57d-4401e907ee94 is now active with session: 69aa9d44-6944-44f4-8809-258715a486da","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939639855Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118922024-04-20T01:13:18.444Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939668915Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611,"session_id":"69aa9d44-6944-44f4-8809-258715a486da"}
118932024-04-20T01:13:18.444Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:16.939698285Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611}
118942024-04-20T01:13:18.448ZLoading write count information from file "/tmp/test_repair_verify.data"
118952024-04-20T01:13:18.448ZRun Repair workload
118962024-04-20T01:13:18.448Z01/30 Write block 316 len 20480 data: 6 6 12 12 12
118972024-04-20T01:13:18.448Z02/30 Write block 486 len 8192 data: 8 6
118982024-04-20T01:13:18.448Z03/30 Flush
118992024-04-20T01:13:18.448Z04/30 Write block 446 len 12288 data: 8 10 10
119002024-04-20T01:13:18.448Z05/30 Read block 377 len 28672
119012024-04-20T01:13:18.448Z06/30 Write block 429 len 20480 data: 10 10 9 8 8
119022024-04-20T01:13:18.448Z07/30 Flush
119032024-04-20T01:13:18.451Z08/30 Write block 135 len 32768 data: 9 9 9 11 11 12 12 12
119042024-04-20T01:13:18.451Z09/30 Read block 575 len 28672
119052024-04-20T01:13:18.452Z10/30 Read block 231 len 32768
119062024-04-20T01:13:18.452Z11/30 Read block 295 len 8192
119072024-04-20T01:13:18.452Z12/30 Read block 427 len 40960
119082024-04-20T01:13:18.455Z13/30 Write block 170 len 20480 data: 7 7 8 11 10
119092024-04-20T01:13:18.455Z14/30 Read block 468 len 4096
119102024-04-20T01:13:18.455Z15/30 Read block 492 len 12288
119112024-04-20T01:13:18.455Z16/30 Read block 588 len 8192
119122024-04-20T01:13:18.455Z17/30 Read block 126 len 4096
119132024-04-20T01:13:18.461Z18/30 Read block 240 len 16384
119142024-04-20T01:13:18.461Z19/30 Flush
119152024-04-20T01:13:18.461Z20/30 Flush
119162024-04-20T01:13:18.465Z21/30 Flush
119172024-04-20T01:13:18.465Z22/30 Write block 196 len 40960 data: 6 9 10 12 8 9 11 8 7 8
119182024-04-20T01:13:18.465Z23/30 Write block 49 len 36864 data: 8 10 12 13 15 14 14 13 10
119192024-04-20T01:13:18.465Z24/30 Read block 252 len 20480
119202024-04-20T01:13:18.465Z25/30 Read block 74 len 12288
119212024-04-20T01:13:18.465Z26/30 Write block 378 len 20480 data: 12 11 12 9 8
119222024-04-20T01:13:18.465Z27/30 Read block 504 len 32768
119232024-04-20T01:13:18.469Z28/30 Read block 19 len 4096
119242024-04-20T01:13:18.469Z29/30 Read block 233 len 12288
119252024-04-20T01:13:18.469Z30/30 Write block 46 len 4096 data: 5
119262024-04-20T01:13:18.469Z----------------------------------------------------------------
119272024-04-20T01:13:18.470Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
119282024-04-20T01:13:18.470ZStates: Active Active Active
119292024-04-20T01:13:18.470ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
119302024-04-20T01:13:18.470Z 1 Acked 1000 Write 5 Done Done Done false
119312024-04-20T01:13:18.470Z 2 Acked 1001 Write 2 Done Done Done false
119322024-04-20T01:13:18.470Z 3 Acked 1002 Flush 0 Sent Done Done false
119332024-04-20T01:13:18.470Z 4 Acked 1003 Write 3 Sent Done Done false
119342024-04-20T01:13:18.470Z 5 Acked 1004 Read 7 Sent Done Done false
119352024-04-20T01:13:18.470Z 6 Acked 1005 Write 5 Sent Done Done false
119362024-04-20T01:13:18.470Z 7 Acked 1006 Flush 0 Sent Done Done false
119372024-04-20T01:13:18.470Z 8 Acked 1007 Write 8 Sent Done Done false
119382024-04-20T01:13:18.470Z 9 Acked 1008 Read 7 Sent Done Done false
119392024-04-20T01:13:18.470Z 10 Acked 1009 Read 8 Sent Done Done false
119402024-04-20T01:13:18.470Z 11 Acked 1010 Read 2 Sent Done Done false
119412024-04-20T01:13:18.470Z 12 Acked 1011 Read 10 Sent Done Done false
119422024-04-20T01:13:18.470Z 13 Acked 1012 Write 5 Sent Done Done false
119432024-04-20T01:13:18.470Z 14 Acked 1013 Read 1 Sent Done Done false
119442024-04-20T01:13:18.470Z 15 Acked 1014 Read 3 Sent Done Done false
119452024-04-20T01:13:18.470Z 16 Acked 1015 Read 2 Sent Done Done false
119462024-04-20T01:13:18.470Z 17 Acked 1016 Read 1 Sent Done Done false
119472024-04-20T01:13:18.470Z 18 Acked 1017 Read 4 Sent Done Done false
119482024-04-20T01:13:18.470Z 19 Acked 1018 Flush 0 Sent Done Done false
119492024-04-20T01:13:18.470Z 20 Acked 1019 Flush 0 Sent Done Done false
119502024-04-20T01:13:18.470Z 21 Acked 1020 Flush 0 Sent Done Done false
119512024-04-20T01:13:18.471Z 22 Acked 1021 Write 10 Sent Done Done false
119522024-04-20T01:13:18.471Z 23 Acked 1022 Write 9 Sent Done Done false
119532024-04-20T01:13:18.471Z 24 Acked 1023 Read 5 Sent Done Done false
119542024-04-20T01:13:18.471Z 25 Acked 1024 Read 3 Sent Done Done false
119552024-04-20T01:13:18.471Z 26 Acked 1025 Write 5 Sent Done Done false
119562024-04-20T01:13:18.471Z 27 Acked 1026 Read 8 Sent Done Done false
119572024-04-20T01:13:18.471Z 28 Acked 1027 Read 1 Sent Done Done false
119582024-04-20T01:13:18.471Z 29 Acked 1028 Read 3 Sent Done Done false
119592024-04-20T01:13:18.471Z 30 Acked 1029 Write 1 Sent Sent Sent false
119602024-04-20T01:13:18.471Z STATES DS:0 DS:1 DS:2 TOTAL
119612024-04-20T01:13:18.471Z New 0 0 0 0
119622024-04-20T01:13:18.471Z Sent 28 1 1 30
119632024-04-20T01:13:18.471Z Done 2 29 29 60
119642024-04-20T01:13:18.471Z Skipped 0 0 0 0
119652024-04-20T01:13:18.471Z Error 0 0 0 0
119662024-04-20T01:13:18.471ZLast Flush: 0 1020 1020
119672024-04-20T01:13:18.471ZDownstairs last five completed:
119682024-04-20T01:13:18.471ZUpstairs last five completed: 30 29 28 27 26
119692024-04-20T01:13:18.471Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:13:16.969723892Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1611}
119702024-04-20T01:13:18.471ZWrote out file "/tmp/test_repair_verify.data"
119712024-04-20T01:13:18.475Z
119722024-04-20T01:13:18.475ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119732024-04-20T01:13:18.491ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119742024-04-20T01:13:18.491Z 0 000-019 108 108 108 182 182 182 F F F
119752024-04-20T01:13:18.491Z 1 020-039 105 105 105 178 178 178 F F F
119762024-04-20T01:13:18.491Z 2 040-059 108 108 108 182 182 182 F T T <---
119772024-04-20T01:13:18.491Z 3 060-079 106 106 106 179 179 179 F F F
119782024-04-20T01:13:18.491Z 4 080-099 105 105 105 178 178 178 F F F
119792024-04-20T01:13:18.495Z 5 100-119 105 105 105 178 178 178 F F F
119802024-04-20T01:13:18.495Z 6 120-139 106 110 110 179 185 185 F F F <---
119812024-04-20T01:13:18.495Z 7 140-159 98 110 110 161 185 185 F F F <---
119822024-04-20T01:13:18.495Z 8 160-179 107 110 110 181 185 185 F F F <---
119832024-04-20T01:13:18.495Z 9 180-199 103 103 103 176 176 176 F T T <---
119842024-04-20T01:13:18.495Z 10 200-219 108 108 108 182 182 182 F T T <---
119852024-04-20T01:13:18.495Z 11 220-239 108 108 108 182 182 182 F F F
119862024-04-20T01:13:18.495Z 12 240-259 108 108 108 182 182 182 F F F
119872024-04-20T01:13:18.495Z 13 260-279 108 108 108 182 182 182 F F F
119882024-04-20T01:13:18.495Z 14 280-299 106 106 106 179 179 179 F F F
119892024-04-20T01:13:18.495Z 15 300-319 108 110 110 182 183 183 T F F <---
119902024-04-20T01:13:18.496Z 16 320-339 108 110 110 182 183 183 T F F <---
119912024-04-20T01:13:18.496Z 17 340-359 98 98 98 163 163 163 F F F
119922024-04-20T01:13:18.496Z 18 360-379 105 105 105 178 178 178 F T T <---
119932024-04-20T01:13:18.496Z 19 380-399 96 96 96 159 159 159 F T T <---
119942024-04-20T01:13:18.496Z 20 400-419 108 108 108 182 182 182 F F F
119952024-04-20T01:13:18.496Z 21 420-439 108 110 110 182 184 184 F F F <---
119962024-04-20T01:13:18.496Z 22 440-459 108 110 110 182 184 184 F F F <---
119972024-04-20T01:13:18.496Z 23 460-479 106 106 106 179 179 179 F F F
119982024-04-20T01:13:18.496Z 24 480-499 105 110 110 178 183 183 T F F <---
119992024-04-20T01:13:18.496Z 25 500-519 107 107 107 181 181 181 F F F
120002024-04-20T01:13:18.496Z 26 520-539 103 103 103 176 176 176 F F F
120012024-04-20T01:13:18.496Z 27 540-559 100 100 100 168 168 168 F F F
120022024-04-20T01:13:18.496Z 28 560-579 98 98 98 161 161 161 F F F
120032024-04-20T01:13:18.496Z 29 580-599 102 102 102 174 174 174 F F F
120042024-04-20T01:13:18.496ZMax gen: 110, Max flush: 185
120052024-04-20T01:13:18.496ZError: Difference in extent metadata found!
120062024-04-20T01:13:18.496ZA Difference in extent metadata is expected here
120072024-04-20T01:13:18.496ZOn loop 55
120082024-04-20T01:13:18.496Z
120092024-04-20T01:13:18.501ZVerifying data now
120102024-04-20T01:13:33.086ZNo change after verify
120112024-04-20T01:13:33.086ZLoop: 55 Downstairs dump after verify (and repair):
120122024-04-20T01:13:33.102ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120132024-04-20T01:13:33.102Z 0 000-019 108 108 108 182 182 182 F F F
120142024-04-20T01:13:33.102Z 1 020-039 105 105 105 178 178 178 F F F
120152024-04-20T01:13:33.102Z 2 040-059 111 111 111 186 186 186 F F F
120162024-04-20T01:13:33.102Z 3 060-079 106 106 106 179 179 179 F F F
120172024-04-20T01:13:33.102Z 4 080-099 105 105 105 178 178 178 F F F
120182024-04-20T01:13:33.102Z 5 100-119 105 105 105 178 178 178 F F F
120192024-04-20T01:13:33.102Z 6 120-139 110 110 110 185 185 185 F F F
120202024-04-20T01:13:33.102Z 7 140-159 110 110 110 185 185 185 F F F
120212024-04-20T01:13:33.102Z 8 160-179 110 110 110 185 185 185 F F F
120222024-04-20T01:13:33.102Z 9 180-199 111 111 111 186 186 186 F F F
120232024-04-20T01:13:33.102Z 10 200-219 111 111 111 186 186 186 F F F
120242024-04-20T01:13:33.106Z 11 220-239 108 108 108 182 182 182 F F F
120252024-04-20T01:13:33.106Z 12 240-259 108 108 108 182 182 182 F F F
120262024-04-20T01:13:33.106Z 13 260-279 108 108 108 182 182 182 F F F
120272024-04-20T01:13:33.106Z 14 280-299 106 106 106 179 179 179 F F F
120282024-04-20T01:13:33.106Z 15 300-319 110 110 110 183 183 183 F F F
120292024-04-20T01:13:33.106Z 16 320-339 110 110 110 183 183 183 F F F
120302024-04-20T01:13:33.106Z 17 340-359 98 98 98 163 163 163 F F F
120312024-04-20T01:13:33.106Z 18 360-379 111 111 111 186 186 186 F F F
120322024-04-20T01:13:33.106Z 19 380-399 111 111 111 186 186 186 F F F
120332024-04-20T01:13:33.106Z 20 400-419 108 108 108 182 182 182 F F F
120342024-04-20T01:13:33.106Z 21 420-439 110 110 110 184 184 184 F F F
120352024-04-20T01:13:33.106Z 22 440-459 110 110 110 184 184 184 F F F
120362024-04-20T01:13:33.106Z 23 460-479 106 106 106 179 179 179 F F F
120372024-04-20T01:13:33.106Z 24 480-499 110 110 110 183 183 183 F F F
120382024-04-20T01:13:33.106Z 25 500-519 107 107 107 181 181 181 F F F
120392024-04-20T01:13:33.106Z 26 520-539 103 103 103 176 176 176 F F F
120402024-04-20T01:13:33.107Z 27 540-559 100 100 100 168 168 168 F F F
120412024-04-20T01:13:33.107Z 28 560-579 98 98 98 161 161 161 F F F
120422024-04-20T01:13:33.107Z 29 580-599 102 102 102 174 174 174 F F F
120432024-04-20T01:13:33.107ZMax gen: 111, Max flush: 186
120442024-04-20T01:13:33.107Z
120452024-04-20T01:13:33.107ZBegin loop 56
120462024-04-20T01:13:33.107ZDownstairs to restart: 0
120472024-04-20T01:13:33.121Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.623225461Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619}
120482024-04-20T01:13:33.121Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.623497261Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619}
120492024-04-20T01:13:33.128Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.623538821Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619}
120502024-04-20T01:13:33.128Z{"msg":"Crucible fbb4fa37-344f-4f3d-9ee2-0bf63bbaad64 has session id: a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.623577651Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120512024-04-20T01:13:33.128Z{"msg":"Upstairs opts: Upstairs UUID: fbb4fa37-344f-4f3d-9ee2-0bf63bbaad64, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
120522024-04-20T01:13:33.128Z,"time":"2024-04-20T01:13:31.623670812Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120532024-04-20T01:13:33.128Z{"msg":"Crucible stats registered with UUID: fbb4fa37-344f-4f3d-9ee2-0bf63bbaad64","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.623733522Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120542024-04-20T01:13:33.128Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.623776342Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619}
120552024-04-20T01:13:33.128Z{"msg":"fbb4fa37-344f-4f3d-9ee2-0bf63bbaad64 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.623812262Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120562024-04-20T01:13:33.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.623857012Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"0","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120572024-04-20T01:13:33.128Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.623886392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"0","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120582024-04-20T01:13:33.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.623935172Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"1","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120592024-04-20T01:13:33.128Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.623964372Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"1","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120602024-04-20T01:13:33.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.624005692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"2","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120612024-04-20T01:13:33.128Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.624055183Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"2","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120622024-04-20T01:13:33.128Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.624086023Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120632024-04-20T01:13:33.128Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.624138953Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120642024-04-20T01:13:33.129Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.624169573Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120652024-04-20T01:13:33.129Z{"msg":"ds_connection connected from Ok(127.0.0.1:42601)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.624199413Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120662024-04-20T01:13:33.129Z{"msg":"ds_connection connected from Ok(127.0.0.1:38262)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.624253443Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120672024-04-20T01:13:33.129Z{"msg":"ds_connection connected from Ok(127.0.0.1:39488)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.624289823Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120682024-04-20T01:13:33.129Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.624984275Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"2","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120692024-04-20T01:13:33.129Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.625020845Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"2","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120702024-04-20T01:13:33.129Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.625036045Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"1","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120712024-04-20T01:13:33.129Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.625049725Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"1","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120722024-04-20T01:13:33.129Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.625601866Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"1","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120732024-04-20T01:13:33.129Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.625657916Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"2","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120742024-04-20T01:13:33.129Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.626158487Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"2","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120752024-04-20T01:13:33.129Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.626193577Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"2","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120762024-04-20T01:13:33.129Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.626208727Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120772024-04-20T01:13:33.129Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.626228107Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"0","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120782024-04-20T01:13:33.129Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.626241708Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"0","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120792024-04-20T01:13:33.129Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.626261437Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"1","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120802024-04-20T01:13:33.129Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.626274928Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"1","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120812024-04-20T01:13:33.129Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.626294688Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120822024-04-20T01:13:33.129Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.626731318Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"0","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120832024-04-20T01:13:33.129Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.627153799Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"0","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120842024-04-20T01:13:33.129Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.627179899Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"0","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120852024-04-20T01:13:33.130Z{"msg":"[0]R flush_numbers[0..12]: [182, 178, 186, 179, 178, 178, 185, 185, 185, 186, 186, 182]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.627195279Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120862024-04-20T01:13:33.130Z{"msg":"[0]R generation[0..12]: [108, 105, 111, 106, 105, 105, 110, 110, 110, 111, 111, 108]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62720969Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120872024-04-20T01:13:33.130Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.6272319Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120882024-04-20T01:13:33.130Z{"msg":"[1]R flush_numbers[0..12]: [182, 178, 186, 179, 178, 178, 185, 185, 185, 186, 186, 182]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62724657Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120892024-04-20T01:13:33.130Z{"msg":"[1]R generation[0..12]: [108, 105, 111, 106, 105, 105, 110, 110, 110, 111, 111, 108]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62726054Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120902024-04-20T01:13:33.130Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.6272819Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120912024-04-20T01:13:33.130Z{"msg":"[2]R flush_numbers[0..12]: [182, 178, 186, 179, 178, 178, 185, 185, 185, 186, 186, 182]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62729619Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120922024-04-20T01:13:33.130Z{"msg":"[2]R generation[0..12]: [108, 105, 111, 106, 105, 105, 110, 110, 110, 111, 111, 108]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62730979Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120932024-04-20T01:13:33.130Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62733107Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120942024-04-20T01:13:33.130Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62734412Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid"Wait for a query_work_queue command to finish before sending IO
120952024-04-20T01:13:33.135Z:1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120962024-04-20T01:13:33.135Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62737458Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120972024-04-20T01:13:33.135Z{"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62739827Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120982024-04-20T01:13:33.135Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62742601Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"":"downstairs","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
120992024-04-20T01:13:33.135Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62744931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
121002024-04-20T01:13:33.135Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62747716Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"0","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
121012024-04-20T01:13:33.135Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62750128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"1","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
121022024-04-20T01:13:33.135Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.6275292Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"client":"2","":"downstairs","session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
121032024-04-20T01:13:33.135Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62755319Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
121042024-04-20T01:13:33.135Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62758034Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
121052024-04-20T01:13:33.135Z{"msg":"fbb4fa37-344f-4f3d-9ee2-0bf63bbaad64 is now active with session: a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.62760384Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
121062024-04-20T01:13:33.135Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:31.627630251Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619,"session_id":"a81d34e0-b58a-41ff-8ab3-bccaf4bf90d1"}
121072024-04-20T01:13:33.135Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
121082024-04-20T01:13:33.135Z,"time":"2024-04-20T01:13:31.627651531Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619}
121092024-04-20T01:13:33.135ZLoading write count information from file "/tmp/test_repair_verify.data"
121102024-04-20T01:13:33.135ZRun Repair workload
121112024-04-20T01:13:33.135Z01/30 Write block 480 len 36864 data: 6 11 11 10 8 9 9 7 6
121122024-04-20T01:13:33.135Z02/30 Write block 436 len 28672 data: 13 12 8 9 9 10 11
121132024-04-20T01:13:33.136Z03/30 Read block 445 len 36864
121142024-04-20T01:13:33.136Z04/30 Write block 48 len 40960 data: 6 9 11 13 14 16 15 15 14 11
121152024-04-20T01:13:33.136Z05/30 Flush
121162024-04-20T01:13:33.142Z06/30 Write block 102 len 20480 data: 9 9 7 6 7
121172024-04-20T01:13:33.142Z07/30 Read block 172 len 24576
121182024-04-20T01:13:33.142Z08/30 Write block 94 len 4096 data: 6
121192024-04-20T01:13:33.142Z09/30 Read block 178 len 40960
121202024-04-20T01:13:33.142Z10/30 Write block 425 len 12288 data: 8 7 7
121212024-04-20T01:13:33.142Z11/30 Flush
121222024-04-20T01:13:33.146Z12/30 Write block 206 len 24576 data: 9 8 8 8 8 7
121232024-04-20T01:13:33.146Z13/30 Write block 224 len 12288 data: 12 10 14
121242024-04-20T01:13:33.146Z14/30 Read block 412 len 28672
121252024-04-20T01:13:33.146Z15/30 Read block 379 len 24576
121262024-04-20T01:13:33.146Z16/30 Read block 557 len 24576
121272024-04-20T01:13:33.146Z17/30 Write block 566 len 40960 data: 9 8 9 11 10 10 8 6 5 4
121282024-04-20T01:13:33.149Z18/30 Read block 416 len 28672
121292024-04-20T01:13:33.149Z19/30 Read block 503 len 16384
121302024-04-20T01:13:33.149Z20/30 Read block 10 len 40960
121312024-04-20T01:13:33.149Z21/30 Read block 518 len 4096
121322024-04-20T01:13:33.149Z22/30 Read block 404 len 20480
121332024-04-20T01:13:33.153Z23/30 Read block 45 len 16384
121342024-04-20T01:13:33.153Z24/30 Write block 141 len 24576 data: 13 13 12 12 12 11
121352024-04-20T01:13:33.153Z25/30 Write block 591 len 4096 data: 9
121362024-04-20T01:13:33.153Z26/30 Read block 517 len 4096
121372024-04-20T01:13:33.157Z27/30 Write block 151 len 12288 data: 7 7 8
121382024-04-20T01:13:33.157Z28/30 Write block 420 len 4096 data: 9
121392024-04-20T01:13:33.157Z29/30 Read block 502 len 8192
121402024-04-20T01:13:33.157Z30/30 Read block 193 len 12288
121412024-04-20T01:13:33.157Z----------------------------------------------------------------
121422024-04-20T01:13:33.157Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
121432024-04-20T01:13:33.157ZStates: Active Active Active
121442024-04-20T01:13:33.157ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
121452024-04-20T01:13:33.157Z 1 Acked 1000 Write 9 Sent Done Done false
121462024-04-20T01:13:33.157Z 2 Acked 1001 Write 7 Sent Done Done false
121472024-04-20T01:13:33.157Z 3 Acked 1002 Read 9 Sent Done Done false
121482024-04-20T01:13:33.157Z 4 Acked 1003 Write 10 Sent Done Done false
121492024-04-20T01:13:33.158Z 5 Acked 1004 Flush 0 Sent Done Done false
121502024-04-20T01:13:33.158Z 6 Acked 1005 Write 5 Sent Done Done false
121512024-04-20T01:13:33.158Z 7 Acked 1006 Read 6 Sent Done Done false
121522024-04-20T01:13:33.158Z 8 Acked 1007 Write 1 Sent Done Done false
121532024-04-20T01:13:33.158Z 9 Acked 1008 Read 10 Sent Done Done false
121542024-04-20T01:13:33.158Z 10 Acked 1009 Write 3 Sent Done Done false
121552024-04-20T01:13:33.158Z 11 Acked 1010 Flush 0 Sent Done Done false
121562024-04-20T01:13:33.158Z 12 Acked 1011 Write 6 Sent Done Done false
121572024-04-20T01:13:33.158Z 13 Acked 1012 Write 3 Sent Done Done false
121582024-04-20T01:13:33.158Z 14 Acked 1013 Read 7 Sent Done Done false
121592024-04-20T01:13:33.158Z 15 Acked 1014 Read 6 Sent Done Done false
121602024-04-20T01:13:33.158Z 16 Acked 1015 Read 6 Sent Done Done false
121612024-04-20T01:13:33.158Z 17 Acked 1016 Write 10 Sent Done Done false
121622024-04-20T01:13:33.158Z 18 Acked 1017 Read 7 Sent Done Done false
121632024-04-20T01:13:33.158Z 19 Acked 1018 Read 4 Sent Done Done false
121642024-04-20T01:13:33.158Z 20 Acked 1019 Read 10 Sent Done Done false
121652024-04-20T01:13:33.158Z 21 Acked 1020 Read 1 Sent Done Done false
121662024-04-20T01:13:33.158Z 22 Acked 1021 Read 5 Sent Done Done false
121672024-04-20T01:13:33.158Z 23 Acked 1022 Read 4 Sent Done Done false
121682024-04-20T01:13:33.158Z 24 Acked 1023 Write 6 Sent Done Done false
121692024-04-20T01:13:33.158Z 25 Acked 1024 Write 1 Sent Done Done false
121702024-04-20T01:13:33.158Z 26 Acked 1025 Read 1 Sent Done Done false
121712024-04-20T01:13:33.158Z 27 Acked 1026 Write 3 Sent Done Done false
121722024-04-20T01:13:33.158Z 28 Acked 1027 Write 1 Sent Done Done false
121732024-04-20T01:13:33.159Z 29 Acked 1028 Read 2 Sent Done Done false
121742024-04-20T01:13:33.159Z 30 Acked 1029 Read 3 Sent Done Done false
121752024-04-20T01:13:33.159Z STATES DS:0 DS:1 DS:2 TOTAL
121762024-04-20T01:13:33.159Z New 0 0 0 0
121772024-04-20T01:13:33.159Z Sent 30 0 0 30
121782024-04-20T01:13:33.159Z Done 0 30 30 60
121792024-04-20T01:13:33.159Z Skipped 0 0 0 0
121802024-04-20T01:13:33.159Z Error 0 0 0 0
121812024-04-20T01:13:33.159ZLast Flush: 0 1010 1010
121822024-04-20T01:13:33.159ZDownstairs last five completed:
121832024-04-20T01:13:33.159ZUpstairs last five completed: 30 29 28 27 26
121842024-04-20T01:13:33.159Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:13:31.657889148Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1619}
121852024-04-20T01:13:33.159ZWrote out file "/tmp/test_repair_verify.data"
121862024-04-20T01:13:33.163Z
121872024-04-20T01:13:33.167ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121882024-04-20T01:13:33.180ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121892024-04-20T01:13:33.180Z 0 000-019 108 108 108 182 182 182 F F F
121902024-04-20T01:13:33.180Z 1 020-039 105 105 105 178 178 178 F F F
121912024-04-20T01:13:33.180Z 2 040-059 111 112 112 186 187 187 F F F <---
121922024-04-20T01:13:33.180Z 3 060-079 106 106 106 179 179 179 F F F
121932024-04-20T01:13:33.180Z 4 080-099 105 112 112 178 188 188 F F F <---
121942024-04-20T01:13:33.180Z 5 100-119 105 112 112 178 188 188 F F F <---
121952024-04-20T01:13:33.180Z 6 120-139 110 110 110 185 185 185 F F F
121962024-04-20T01:13:33.180Z 7 140-159 110 110 110 185 185 185 F T T <---
121972024-04-20T01:13:33.184Z 8 160-179 110 110 110 185 185 185 F F F
121982024-04-20T01:13:33.184Z 9 180-199 111 111 111 186 186 186 F F F
121992024-04-20T01:13:33.184Z 10 200-219 111 111 111 186 186 186 F T T <---
122002024-04-20T01:13:33.185Z 11 220-239 108 108 108 182 182 182 F T T <---
122012024-04-20T01:13:33.185Z 12 240-259 108 108 108 182 182 182 F F F
122022024-04-20T01:13:33.185Z 13 260-279 108 108 108 182 182 182 F F F
122032024-04-20T01:13:33.185Z 14 280-299 106 106 106 179 179 179 F F F
122042024-04-20T01:13:33.185Z 15 300-319 110 110 110 183 183 183 F F F
122052024-04-20T01:13:33.185Z 16 320-339 110 110 110 183 183 183 F F F
122062024-04-20T01:13:33.185Z 17 340-359 98 98 98 163 163 163 F F F
122072024-04-20T01:13:33.185Z 18 360-379 111 111 111 186 186 186 F F F
122082024-04-20T01:13:33.185Z 19 380-399 111 111 111 186 186 186 F F F
122092024-04-20T01:13:33.185Z 20 400-419 108 108 108 182 182 182 F F F
122102024-04-20T01:13:33.185Z 21 420-439 110 112 112 184 188 188 F T T <---
122112024-04-20T01:13:33.185Z 22 440-459 110 112 112 184 187 187 F F F <---
122122024-04-20T01:13:33.185Z 23 460-479 106 106 106 179 179 179 F F F
122132024-04-20T01:13:33.185Z 24 480-499 110 112 112 183 187 187 F F F <---
122142024-04-20T01:13:33.185Z 25 500-519 107 107 107 181 181 181 F F F
122152024-04-20T01:13:33.185Z 26 520-539 103 103 103 176 176 176 F F F
122162024-04-20T01:13:33.185Z 27 540-559 100 100 100 168 168 168 F F F
122172024-04-20T01:13:33.185Z 28 560-579 98 98 98 161 161 161 F T T <---
122182024-04-20T01:13:33.185Z 29 580-599 102 102 102 174 174 174 F T T <---
122192024-04-20T01:13:33.185ZMax gen: 112, Max flush: 188
122202024-04-20T01:13:33.185ZError: Difference in extent metadata found!
122212024-04-20T01:13:33.185ZA Difference in extent metadata is expected here
122222024-04-20T01:13:33.186ZOn loop 56
122232024-04-20T01:13:33.186Z
122242024-04-20T01:13:33.189ZVerifying data now
122252024-04-20T01:13:37.701ZNo change after verify
122262024-04-20T01:13:37.702ZLoop: 56 Downstairs dump after verify (and repair):
122272024-04-20T01:13:37.715ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122282024-04-20T01:13:37.715Z 0 000-019 108 108 108 182 182 182 F F F
122292024-04-20T01:13:37.720Z 1 020-039 105 105 105 178 178 178 F F F
122302024-04-20T01:13:37.720Z 2 040-059 112 112 112 187 187 187 F F F
122312024-04-20T01:13:37.720Z 3 060-079 106 106 106 179 179 179 F F F
122322024-04-20T01:13:37.720Z 4 080-099 112 112 112 188 188 188 F F F
122332024-04-20T01:13:37.720Z 5 100-119 112 112 112 188 188 188 F F F
122342024-04-20T01:13:37.720Z 6 120-139 110 110 110 185 185 185 F F F
122352024-04-20T01:13:37.720Z 7 140-159 113 113 113 189 189 189 F F F
122362024-04-20T01:13:37.720Z 8 160-179 110 110 110 185 185 185 F F F
122372024-04-20T01:13:37.720Z 9 180-199 111 111 111 186 186 186 F F F
122382024-04-20T01:13:37.720Z 10 200-219 113 113 113 189 189 189 F F F
122392024-04-20T01:13:37.720Z 11 220-239 113 113 113 189 189 189 F F F
122402024-04-20T01:13:37.720Z 12 240-259 108 108 108 182 182 182 F F F
122412024-04-20T01:13:37.720Z 13 260-279 108 108 108 182 182 182 F F F
122422024-04-20T01:13:37.720Z 14 280-299 106 106 106 179 179 179 F F F
122432024-04-20T01:13:37.720Z 15 300-319 110 110 110 183 183 183 F F F
122442024-04-20T01:13:37.720Z 16 320-339 110 110 110 183 183 183 F F F
122452024-04-20T01:13:37.720Z 17 340-359 98 98 98 163 163 163 F F F
122462024-04-20T01:13:37.720Z 18 360-379 111 111 111 186 186 186 F F F
122472024-04-20T01:13:37.720Z 19 380-399 111 111 111 186 186 186 F F F
122482024-04-20T01:13:37.720Z 20 400-419 108 108 108 182 182 182 F F F
122492024-04-20T01:13:37.720Z 21 420-439 113 113 113 189 189 189 F F F
122502024-04-20T01:13:37.720Z 22 440-459 112 112 112 187 187 187 F F F
122512024-04-20T01:13:37.721Z 23 460-479 106 106 106 179 179 179 F F F
122522024-04-20T01:13:37.721Z 24 480-499 112 112 112 187 187 187 F F F
122532024-04-20T01:13:37.721Z 25 500-519 107 107 107 181 181 181 F F F
122542024-04-20T01:13:37.721Z 26 520-539 103 103 103 176 176 176 F F F
122552024-04-20T01:13:37.721Z 27 540-559 100 100 100 168 168 168 F F F
122562024-04-20T01:13:37.721Z 28 560-579 113 113 113 189 189 189 F F F
122572024-04-20T01:13:37.721Z 29 580-599 113 113 113 189 189 189 F F F
122582024-04-20T01:13:37.721ZMax gen: 113, Max flush: 189
122592024-04-20T01:13:37.721Z
122602024-04-20T01:13:37.721ZBegin loop 57
122612024-04-20T01:13:37.721ZDownstairs to restart: 0
122622024-04-20T01:13:37.736Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.238297114Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627}
122632024-04-20T01:13:37.744Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.238537324Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627}
122642024-04-20T01:13:37.744Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.238574965Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627}
122652024-04-20T01:13:37.745Z{"msg":"Crucible a5ef1128-312d-42e5-971b-f6d9eccc6276 has session id: 07fa1e48-af60-44b9-9de6-29cfd641680d","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.238596795Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122662024-04-20T01:13:37.745Z{"msg":"Upstairs opts: Upstairs UUID: a5ef1128-312d-42e5-971b-f6d9eccc6276, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
122672024-04-20T01:13:37.745Z,"time":"2024-04-20T01:13:36.238680985Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122682024-04-20T01:13:37.745Z{"msg":"Crucible stats registered with UUID: a5ef1128-312d-42e5-971b-f6d9eccc6276","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.238727965Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122692024-04-20T01:13:37.745Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.238816195Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627}
122702024-04-20T01:13:37.745Z{"msg":"a5ef1128-312d-42e5-971b-f6d9eccc6276 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.238873215Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122712024-04-20T01:13:37.745Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.238897575Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"0","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122722024-04-20T01:13:37.745Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.238913395Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"0","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122732024-04-20T01:13:37.745Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.238945976Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"1","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122742024-04-20T01:13:37.745Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.238991876Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"1","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122752024-04-20T01:13:37.745Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.239041786Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"2","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122762024-04-20T01:13:37.745Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.239067586Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"2","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122772024-04-20T01:13:37.745Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.239100646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122782024-04-20T01:13:37.745Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.239146806Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122792024-04-20T01:13:37.745Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.239175606Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122802024-04-20T01:13:37.745Z{"msg":"ds_connection connected from Ok(127.0.0.1:62508)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.239203936Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122812024-04-20T01:13:37.745Z{"msg":"ds_connection connected from Ok(127.0.0.1:60106)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.239239366Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122822024-04-20T01:13:37.745Z{"msg":"ds_connection connected from Ok(127.0.0.1:64628)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.239264136Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122832024-04-20T01:13:37.745Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.239896427Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"1","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122842024-04-20T01:13:37.745Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.239933968Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"1","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122852024-04-20T01:13:37.745Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.239985198Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"2","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122862024-04-20T01:13:37.746Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.240044128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"2","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122872024-04-20T01:13:37.746Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.240500129Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"2","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122882024-04-20T01:13:37.746Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.240606179Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"1","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122892024-04-20T01:13:37.746Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.240715769Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"0","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122902024-04-20T01:13:37.746Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.24074722Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"0","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122912024-04-20T01:13:37.746Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.24095999Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"2","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122922024-04-20T01:13:37.746Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.2409991Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"2","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122932024-04-20T01:13:37.746Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.24102713Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122942024-04-20T01:13:37.746Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.24106703Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"1","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122952024-04-20T01:13:37.746Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.24109651Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"1","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122962024-04-20T01:13:37.746Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.24111078Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122972024-04-20T01:13:37.746Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241319821Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"0","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122982024-04-20T01:13:37.746Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241737662Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"0","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
122992024-04-20T01:13:37.746Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241773032Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"0","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123002024-04-20T01:13:37.746Z{"msg":"[0]R flush_numbers[0..12]: [182, 178, 187, 179, 188, 188, 185, 189, 185, 186, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241789732Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123012024-04-20T01:13:37.746Z{"msg":"[0]R generation[0..12]: [108, 105, 112, 106, 112, 112, 110, 113, 110, 111, 113, 113]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241805742Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123022024-04-20T01:13:37.746Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241839662Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123032024-04-20T01:13:37.746Z{"msg":"[1]R flush_numbers[0..12]: [182, 178, 187, 179, 188, 188, 185, 189, 185, 186, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241855232Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123042024-04-20T01:13:37.746Z{"msg":"[1]R generation[0..12]: [108, 105, 112, 106, 112, 112, 110, 113, 110, 111, 113, 113]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241870092Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123052024-04-20T01:13:37.746Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241891242Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123062024-04-20T01:13:37.746Z{"msg":"[2]R flush_numbers[0..12]: [182, 178, 187, 179, 188, 188, 185, 189, 185, 186, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241905252Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123072024-04-20T01:13:37.746Z{"msg":"[2]R generation[0..12]: [108, 105, 112, 106, 112, 112, 110, 113, 110, 111, 113, 113]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241925952Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123082024-04-20T01:13:37.746Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241940202Z",Wait for a query_work_queue command to finish before sending IO
123092024-04-20T01:13:37.746Z"hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123102024-04-20T01:13:37.747Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241965132Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123112024-04-20T01:13:37.747Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241978062Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123122024-04-20T01:13:37.747Z{"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.241990902Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123132024-04-20T01:13:37.747Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.242004872Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"":"downstairs","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123142024-04-20T01:13:37.747Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.242018052Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123152024-04-20T01:13:37.747Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.242037032Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"0","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123162024-04-20T01:13:37.747Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.242050442Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"1","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123172024-04-20T01:13:37.747Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.242081643Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"client":"2","":"downstairs","session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123182024-04-20T01:13:37.747Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.242116883Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123192024-04-20T01:13:37.747Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
123202024-04-20T01:13:37.747Z,"time":"2024-04-20T01:13:36.242146403Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123212024-04-20T01:13:37.747Z{"msg":"a5ef1128-312d-42e5-971b-f6d9eccc6276 is now active with session: 07fa1e48-af60-44b9-9de6-29cfd641680d","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.242180823Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123222024-04-20T01:13:37.747Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.242196783Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627,"session_id":"07fa1e48-af60-44b9-9de6-29cfd641680d"}
123232024-04-20T01:13:37.747Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:36.242225613Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627}
123242024-04-20T01:13:37.751ZLoading write count information from file "/tmp/test_repair_verify.data"
123252024-04-20T01:13:37.751ZRun Repair workload
123262024-04-20T01:13:37.751Z01/30 Write block 279 len 12288 data: 6 8 7
123272024-04-20T01:13:37.751Z02/30 Write block 317 len 16384 data: 7 13 13 13
123282024-04-20T01:13:37.751Z03/30 Read block 105 len 32768
123292024-04-20T01:13:37.751Z04/30 Read block 410 len 36864
123302024-04-20T01:13:37.751Z05/30 Write block 415 len 8192 data: 12 13
123312024-04-20T01:13:37.751Z06/30 Read block 326 len 12288
123322024-04-20T01:13:37.751Z07/30 Read block 142 len 36864
123332024-04-20T01:13:37.751Z08/30 Write block 57 len 16384 data: 12 12 11 12
123342024-04-20T01:13:37.751Z09/30 Read block 166 len 20480
123352024-04-20T01:13:37.751Z10/30 Write block 210 len 20480 data: 9 8 8 9 9
123362024-04-20T01:13:37.751Z11/30 Read block 207 len 8192
123372024-04-20T01:13:37.751Z12/30 Flush
123382024-04-20T01:13:37.755Z13/30 Write block 430 len 4096 data: 11
123392024-04-20T01:13:37.755Z14/30 Write block 269 len 8192 data: 6 7
123402024-04-20T01:13:37.755Z15/30 Write block 164 len 20480 data: 7 8 5 4 6
123412024-04-20T01:13:37.759Z16/30 Write block 564 len 32768 data: 6 8 10 9 10 12 11 11
123422024-04-20T01:13:37.759Z17/30 Write block 211 len 32768 data: 9 9 10 10 8 9 13 12
123432024-04-20T01:13:37.759Z18/30 Write block 8 len 4096 data: 5
123442024-04-20T01:13:37.759Z19/30 Read block 257 len 28672
123452024-04-20T01:13:37.763Z20/30 Write block 545 len 28672 data: 9 10 6 8 10 12 11
123462024-04-20T01:13:37.763Z21/30 Read block 15 len 12288
123472024-04-20T01:13:37.763Z22/30 Read block 163 len 16384
123482024-04-20T01:13:37.763Z23/30 Read block 281 len 36864
123492024-04-20T01:13:37.767Z24/30 Write block 466 len 16384 data: 14 14 12 14
123502024-04-20T01:13:37.767Z25/30 Write block 355 len 24576 data: 8 10 9 9 10 8
123512024-04-20T01:13:37.767Z26/30 Read block 373 len 40960
123522024-04-20T01:13:37.767Z27/30 Write block 584 len 16384 data: 10 13 16 14
123532024-04-20T01:13:37.772Z28/30 Write block 225 len 28672 data: 11 15 9 11 8 9 12
123542024-04-20T01:13:37.772Z29/30 Read block 89 len 24576
123552024-04-20T01:13:37.772Z30/30 Read block 581 len 28672
123562024-04-20T01:13:37.772Z----------------------------------------------------------------
123572024-04-20T01:13:37.772Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
123582024-04-20T01:13:37.772ZStates: Active Active Active
123592024-04-20T01:13:37.772ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
123602024-04-20T01:13:37.772Z 1 Acked 1000 Write 3 Sent Done Done false
123612024-04-20T01:13:37.772Z 2 Acked 1001 Write 4 Sent Done Done false
123622024-04-20T01:13:37.772Z 3 Acked 1002 Read 8 Sent Done Done false
123632024-04-20T01:13:37.772Z 4 Acked 1003 Read 9 Sent Done Done false
123642024-04-20T01:13:37.772Z 5 Acked 1004 Write 2 Sent Done Done false
123652024-04-20T01:13:37.772Z 6 Acked 1005 Read 3 Sent Done Done false
123662024-04-20T01:13:37.772Z 7 Acked 1006 Read 9 Sent Done Done false
123672024-04-20T01:13:37.773Z 8 Acked 1007 Write 4 Sent Done Done false
123682024-04-20T01:13:37.773Z 9 Acked 1008 Read 5 Sent Done Done false
123692024-04-20T01:13:37.773Z 10 Acked 1009 Write 5 Sent Done Done false
123702024-04-20T01:13:37.773Z 11 Acked 1010 Read 2 Sent Done Done false
123712024-04-20T01:13:37.773Z 12 Acked 1011 Flush 0 Sent Done Done false
123722024-04-20T01:13:37.773Z 13 Acked 1012 Write 1 Sent Done Done false
123732024-04-20T01:13:37.773Z 14 Acked 1013 Write 2 Sent Done Done false
123742024-04-20T01:13:37.773Z 15 Acked 1014 Write 5 Sent Done Done false
123752024-04-20T01:13:37.773Z 16 Acked 1015 Write 8 Sent Done Done false
123762024-04-20T01:13:37.773Z 17 Acked 1016 Write 8 Sent Done Done false
123772024-04-20T01:13:37.773Z 18 Acked 1017 Write 1 Sent Done Done false
123782024-04-20T01:13:37.773Z 19 Acked 1018 Read 7 Sent Done Done false
123792024-04-20T01:13:37.773Z 20 Acked 1019 Write 7 Sent Done Done false
123802024-04-20T01:13:37.773Z 21 Acked 1020 Read 3 Sent Done Done false
123812024-04-20T01:13:37.773Z 22 Acked 1021 Read 4 Sent Done Done false
123822024-04-20T01:13:37.773Z 23 Acked 1022 Read 9 Sent Done Done false
123832024-04-20T01:13:37.773Z 24 Acked 1023 Write 4 Sent Done Done false
123842024-04-20T01:13:37.773Z 25 Acked 1024 Write 6 Sent Done Done false
123852024-04-20T01:13:37.773Z 26 Acked 1025 Read 10 Sent Done Done false
123862024-04-20T01:13:37.773Z 27 Acked 1026 Write 4 Sent Done Done false
123872024-04-20T01:13:37.773Z 28 Acked 1027 Write 7 Sent Done Done false
123882024-04-20T01:13:37.773Z 29 Acked 1028 Read 6 Sent Done Done false
123892024-04-20T01:13:37.774Z 30 Acked 1029 Read 7 Sent Done Done false
123902024-04-20T01:13:37.774Z STATES DS:0 DS:1 DS:2 TOTAL
123912024-04-20T01:13:37.774Z New 0 0 0 0
123922024-04-20T01:13:37.774Z Sent 30 0 0 30
123932024-04-20T01:13:37.774Z Done 0 30 30 60
123942024-04-20T01:13:37.774Z Skipped 0 0 0 0
123952024-04-20T01:13:37.774Z Error 0 0 0 0
123962024-04-20T01:13:37.774ZLast Flush: 0 1011 1011
123972024-04-20T01:13:37.774ZDownstairs last five completed:
123982024-04-20T01:13:37.774ZUpstairs last five completed: 30 29 28 27 26
123992024-04-20T01:13:37.774Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:13:36.271171717Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1627}
124002024-04-20T01:13:37.774ZWrote out file "/tmp/test_repair_verify.data"
124012024-04-20T01:13:37.787Z
124022024-04-20T01:13:37.787ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
124032024-04-20T01:13:37.794ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124042024-04-20T01:13:37.794Z 0 000-019 108 108 108 182 182 182 F T T <---
124052024-04-20T01:13:37.794Z 1 020-039 105 105 105 178 178 178 F F F
124062024-04-20T01:13:37.794Z 2 040-059 112 114 114 187 190 190 F F F <---
124072024-04-20T01:13:37.794Z 3 060-079 106 114 114 179 190 190 F F F <---
124082024-04-20T01:13:37.794Z 4 080-099 112 112 112 188 188 188 F F F
124092024-04-20T01:13:37.795Z 5 100-119 112 112 112 188 188 188 F F F
124102024-04-20T01:13:37.795Z 6 120-139 110 110 110 185 185 185 F F F
124112024-04-20T01:13:37.795Z 7 140-159 113 113 113 189 189 189 F F F
124122024-04-20T01:13:37.795Z 8 160-179 110 110 110 185 185 185 F T T <---
124132024-04-20T01:13:37.795Z 9 180-199 111 111 111 186 186 186 F F F
124142024-04-20T01:13:37.795Z 10 200-219 113 114 114 189 190 190 F T T <---
124152024-04-20T01:13:37.795Z 11 220-239 113 113 113 189 189 189 F T T <---
124162024-04-20T01:13:37.799Z 12 240-259 108 108 108 182 182 182 F F F
124172024-04-20T01:13:37.799Z 13 260-279 108 114 114 182 190 190 F T T <---
124182024-04-20T01:13:37.799Z 14 280-299 106 114 114 179 190 190 F F F <---
124192024-04-20T01:13:37.799Z 15 300-319 110 114 114 183 190 190 F F F <---
124202024-04-20T01:13:37.799Z 16 320-339 110 114 114 183 190 190 F F F <---
124212024-04-20T01:13:37.799Z 17 340-359 98 98 98 163 163 163 F T T <---
124222024-04-20T01:13:37.799Z 18 360-379 111 111 111 186 186 186 F T T <---
124232024-04-20T01:13:37.799Z 19 380-399 111 111 111 186 186 186 F F F
124242024-04-20T01:13:37.799Z 20 400-419 108 114 114 182 190 190 F F F <---
124252024-04-20T01:13:37.799Z 21 420-439 113 113 113 189 189 189 F T T <---
124262024-04-20T01:13:37.799Z 22 440-459 112 112 112 187 187 187 F F F
124272024-04-20T01:13:37.799Z 23 460-479 106 106 106 179 179 179 F T T <---
124282024-04-20T01:13:37.799Z 24 480-499 112 112 112 187 187 187 F F F
124292024-04-20T01:13:37.799Z 25 500-519 107 107 107 181 181 181 F F F
124302024-04-20T01:13:37.799Z 26 520-539 103 103 103 176 176 176 F F F
124312024-04-20T01:13:37.799Z 27 540-559 100 100 100 168 168 168 F T T <---
124322024-04-20T01:13:37.799Z 28 560-579 113 113 113 189 189 189 F T T <---
124332024-04-20T01:13:37.799Z 29 580-599 113 113 113 189 189 189 F T T <---
124342024-04-20T01:13:37.799ZMax gen: 114, Max flush: 190
124352024-04-20T01:13:37.799ZError: Difference in extent metadata found!
124362024-04-20T01:13:37.799ZA Difference in extent metadata is expected here
124372024-04-20T01:13:37.799ZOn loop 57
124382024-04-20T01:13:37.800Z
124392024-04-20T01:13:37.803ZVerifying data now
124402024-04-20T01:13:42.631ZNo change after verify
124412024-04-20T01:13:42.631ZLoop: 57 Downstairs dump after verify (and repair):
124422024-04-20T01:13:42.646ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124432024-04-20T01:13:42.646Z 0 000-019 115 115 115 191 191 191 F F F
124442024-04-20T01:13:42.646Z 1 020-039 105 105 105 178 178 178 F F F
124452024-04-20T01:13:42.646Z 2 040-059 114 114 114 190 190 190 F F F
124462024-04-20T01:13:42.646Z 3 060-079 114 114 114 190 190 190 F F F
124472024-04-20T01:13:42.646Z 4 080-099 112 112 112 188 188 188 F F F
124482024-04-20T01:13:42.646Z 5 100-119 112 112 112 188 188 188 F F F
124492024-04-20T01:13:42.646Z 6 120-139 110 110 110 185 185 185 F F F
124502024-04-20T01:13:42.646Z 7 140-159 113 113 113 189 189 189 F F F
124512024-04-20T01:13:42.650Z 8 160-179 115 115 115 191 191 191 F F F
124522024-04-20T01:13:42.650Z 9 180-199 111 111 111 186 186 186 F F F
124532024-04-20T01:13:42.650Z 10 200-219 115 115 115 191 191 191 F F F
124542024-04-20T01:13:42.650Z 11 220-239 115 115 115 191 191 191 F F F
124552024-04-20T01:13:42.650Z 12 240-259 108 108 108 182 182 182 F F F
124562024-04-20T01:13:42.650Z 13 260-279 115 115 115 191 191 191 F F F
124572024-04-20T01:13:42.650Z 14 280-299 114 114 114 190 190 190 F F F
124582024-04-20T01:13:42.650Z 15 300-319 114 114 114 190 190 190 F F F
124592024-04-20T01:13:42.650Z 16 320-339 114 114 114 190 190 190 F F F
124602024-04-20T01:13:42.650Z 17 340-359 115 115 115 191 191 191 F F F
124612024-04-20T01:13:42.650Z 18 360-379 115 115 115 191 191 191 F F F
124622024-04-20T01:13:42.650Z 19 380-399 111 111 111 186 186 186 F F F
124632024-04-20T01:13:42.650Z 20 400-419 114 114 114 190 190 190 F F F
124642024-04-20T01:13:42.650Z 21 420-439 115 115 115 191 191 191 F F F
124652024-04-20T01:13:42.650Z 22 440-459 112 112 112 187 187 187 F F F
124662024-04-20T01:13:42.650Z 23 460-479 115 115 115 191 191 191 F F F
124672024-04-20T01:13:42.650Z 24 480-499 112 112 112 187 187 187 F F F
124682024-04-20T01:13:42.651Z 25 500-519 107 107 107 181 181 181 F F F
124692024-04-20T01:13:42.651Z 26 520-539 103 103 103 176 176 176 F F F
124702024-04-20T01:13:42.651Z 27 540-559 115 115 115 191 191 191 F F F
124712024-04-20T01:13:42.651Z 28 560-579 115 115 115 191 191 191 F F F
124722024-04-20T01:13:42.651Z 29 580-599 115 115 115 191 191 191 F F F
124732024-04-20T01:13:42.651ZMax gen: 115, Max flush: 191
124742024-04-20T01:13:42.651Z
124752024-04-20T01:13:42.651ZBegin loop 58
124762024-04-20T01:13:42.651ZDownstairs to restart: 2
124772024-04-20T01:13:42.669Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171045884Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635}
124782024-04-20T01:13:42.677Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171296265Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635}
124792024-04-20T01:13:42.677Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171332615Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635}
124802024-04-20T01:13:42.677Z{"msg":"Crucible 29afe1cc-c2bd-4e5f-9cb6-16ce9073f5f2 has session id: bbddcc20-dc80-45ad-b3d3-3d38f829de66","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171355395Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124812024-04-20T01:13:42.677Z{Crucible runtime is spawned
124822024-04-20T01:13:42.678Z"msg":"Upstairs opts: Upstairs UUID: 29afe1cc-c2bd-4e5f-9cb6-16ce9073f5f2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171399915Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124832024-04-20T01:13:42.678Z{"msg":"Crucible stats registered with UUID: 29afe1cc-c2bd-4e5f-9cb6-16ce9073f5f2","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171437155Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124842024-04-20T01:13:42.678Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171461015Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635}
124852024-04-20T01:13:42.678Z{"msg":"29afe1cc-c2bd-4e5f-9cb6-16ce9073f5f2 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171547365Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124862024-04-20T01:13:42.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171596516Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"0","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124872024-04-20T01:13:42.678Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171637306Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"0","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124882024-04-20T01:13:42.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171666006Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"1","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124892024-04-20T01:13:42.678Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171710946Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"1","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124902024-04-20T01:13:42.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171747186Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"2","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124912024-04-20T01:13:42.678Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171782466Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"2","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124922024-04-20T01:13:42.678Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171810576Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124932024-04-20T01:13:42.678Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171847966Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124942024-04-20T01:13:42.678Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171883036Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124952024-04-20T01:13:42.678Z{"msg":"ds_connection connected from Ok(127.0.0.1:54748)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171907776Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124962024-04-20T01:13:42.678Z{"msg":"ds_connection connected from Ok(127.0.0.1:36216)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171943346Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124972024-04-20T01:13:42.678Z{"msg":"ds_connection connected from Ok(127.0.0.1:63245)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.171977086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124982024-04-20T01:13:42.678Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.172616168Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"0","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
124992024-04-20T01:13:42.678Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.172651608Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"0","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125002024-04-20T01:13:42.678Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.172670028Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"1","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125012024-04-20T01:13:42.678Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.172705428Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"1","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125022024-04-20T01:13:42.678Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.172747848Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"2","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125032024-04-20T01:13:42.679Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.172777908Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"2","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125042024-04-20T01:13:42.679Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.173192199Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"0","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125052024-04-20T01:13:42.679Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.173320569Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"2","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125062024-04-20T01:13:42.679Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.17335315Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"1","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125072024-04-20T01:13:42.679Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.173801791Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"0","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125082024-04-20T01:13:42.679Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.173867111Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"0","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125092024-04-20T01:13:42.679Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.173902101Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125102024-04-20T01:13:42.679Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.173915691Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"2","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125112024-04-20T01:13:42.679Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.173942991Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"2","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125122024-04-20T01:13:42.679Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.173957001Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125132024-04-20T01:13:42.679Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.173968911Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"1","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125142024-04-20T01:13:42.679Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.173988941Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"1","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125152024-04-20T01:13:42.679Z{"msg":"[0]R flush_numbers[0..12]: [191, 178, 190, 190, 188, 188, 185, 189, 191, 186, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174003681Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125162024-04-20T01:13:42.679Z{"msg":"[0]R generation[0..12]: [115, 105, 114, 114, 112, 112, 110, 113, 115, 111, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174017961Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125172024-04-20T01:13:42.679Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174039531Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125182024-04-20T01:13:42.679Z{"msg":"[1]R flush_numbers[0..12]: [191, 178, 190, 190, 188, 188, 185, 189, 191, 186, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174054831Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125192024-04-20T01:13:42.679Z{"msg":"[1]R generation[0..12]: [115, 105, 114, 114, 112, 112, 110, 113, 115, 111, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174075691Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125202024-04-20T01:13:42.679Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174090231Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125212024-04-20T01:13:42.679Z{"msg":"[2]R flush_numbers[0..12]: [191, 178, 190, 190, 188, 188, 185, 189, 191, 186, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174116901Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125222024-04-20T01:13:42.679Z{"Wait for a query_work_queue command to finish before sending IO
125232024-04-20T01:13:42.680Zmsg":"[2]R generation[0..12]: [115, 105, 114, 114, 112, 112, 110, 113, 115, 111, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174133261Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125242024-04-20T01:13:42.680Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174154411Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125252024-04-20T01:13:42.680Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174168042Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125262024-04-20T01:13:42.680Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174181531Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125272024-04-20T01:13:42.680Z{"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174201811Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125282024-04-20T01:13:42.680Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174215031Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"":"downstairs","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125292024-04-20T01:13:42.680Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174227871Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125302024-04-20T01:13:42.680Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174253512Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"0","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125312024-04-20T01:13:42.680Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174268072Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"1","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125322024-04-20T01:13:42.680Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174293872Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"client":"2","":"downstairs","session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125332024-04-20T01:13:42.680Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174319722Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125342024-04-20T01:13:42.680Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174335882Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125352024-04-20T01:13:42.680Z{"msg":"29afe1cc-c2bd-4e5f-9cb6-16ce9073f5f2 is now active with session: bbddcc20-dc80-45ad-b3d3-3d38f829de66","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174359482Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125362024-04-20T01:13:42.680Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174372002Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635,"session_id":"bbddcc20-dc80-45ad-b3d3-3d38f829de66"}
125372024-04-20T01:13:42.680Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:41.174384162Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635}
125382024-04-20T01:13:42.680ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
125392024-04-20T01:13:42.680ZLoading write count information from file "/tmp/test_repair_verify.data"
125402024-04-20T01:13:42.680ZRun Repair workload
125412024-04-20T01:13:42.684Z01/30 Write block 106 len 36864 data: 8 7 8 7 7 7 8 9 9
125422024-04-20T01:13:42.685Z02/30 Read block 141 len 28672
125432024-04-20T01:13:42.685Z03/30 Write block 296 len 20480 data: 7 7 5 6 6
125442024-04-20T01:13:42.685Z04/30 Read block 151 len 40960
125452024-04-20T01:13:42.685Z05/30 Write block 281 len 8192 data: 8 6
125462024-04-20T01:13:42.685Z06/30 Read block 153 len 20480
125472024-04-20T01:13:42.685Z07/30 Write block 484 len 20480 data: 9 10 10 8 7
125482024-04-20T01:13:42.685Z08/30 Read block 534 len 36864
125492024-04-20T01:13:42.685Z09/30 Read block 359 len 20480
125502024-04-20T01:13:42.685Z10/30 Read block 360 len 32768
125512024-04-20T01:13:42.685Z11/30 Read block 543 len 28672
125522024-04-20T01:13:42.685Z12/30 Read block 289 len 24576
125532024-04-20T01:13:42.689Z13/30 Write block 309 len 24576 data: 5 4 4 4 5 6
125542024-04-20T01:13:42.689Z14/30 Read block 129 len 40960
125552024-04-20T01:13:42.689Z15/30 Read block 284 len 40960
125562024-04-20T01:13:42.689Z16/30 Read block 245 len 12288
125572024-04-20T01:13:42.690Z17/30 Flush
125582024-04-20T01:13:42.693Z18/30 Write block 288 len 20480 data: 8 9 7 7 7
125592024-04-20T01:13:42.693Z19/30 Read block 506 len 4096
125602024-04-20T01:13:42.693Z20/30 Write block 194 len 12288 data: 8 8 7
125612024-04-20T01:13:42.693Z21/30 Read block 27 len 8192
125622024-04-20T01:13:42.697Z22/30 Read block 102 len 4096
125632024-04-20T01:13:42.697Z23/30 Write block 158 len 40960 data: 10 8 9 11 9 8 8 9 6 5
125642024-04-20T01:13:42.697Z24/30 Read block 470 len 28672
125652024-04-20T01:13:42.697Z25/30 Write block 565 len 8192 data: 9 11
125662024-04-20T01:13:42.697Z26/30 Read block 244 len 28672
125672024-04-20T01:13:42.701Z27/30 Read block 352 len 12288
125682024-04-20T01:13:42.701Z28/30 Read block 356 len 36864
125692024-04-20T01:13:42.701Z29/30 Read block 481 len 16384
125702024-04-20T01:13:42.701Z30/30 Write block 564 len 28672 data: 7 10 12 10 11 13 12
125712024-04-20T01:13:42.705Z----------------------------------------------------------------
125722024-04-20T01:13:42.705Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
125732024-04-20T01:13:42.705ZStates: Active Active Active
125742024-04-20T01:13:42.705ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
125752024-04-20T01:13:42.705Z 1 Acked 1000 Write 9 Done Done Done false
125762024-04-20T01:13:42.705Z 2 Acked 1001 Read 7 Done Done Done false
125772024-04-20T01:13:42.705Z 3 Acked 1002 Write 5 Done Done Done false
125782024-04-20T01:13:42.705Z 4 Acked 1003 Read 10 Done Done Done false
125792024-04-20T01:13:42.705Z 5 Acked 1004 Write 2 Done Done Done false
125802024-04-20T01:13:42.705Z 6 Acked 1005 Read 5 Done Done Done false
125812024-04-20T01:13:42.705Z 7 Acked 1006 Write 5 Done Done Sent false
125822024-04-20T01:13:42.705Z 8 Acked 1007 Read 9 Done Done Sent false
125832024-04-20T01:13:42.705Z 9 Acked 1008 Read 5 Done Done Sent false
125842024-04-20T01:13:42.706Z 10 Acked 1009 Read 8 Done Done Sent false
125852024-04-20T01:13:42.706Z 11 Acked 1010 Read 7 Done Done Sent false
125862024-04-20T01:13:42.706Z 12 Acked 1011 Read 6 Done Done Sent false
125872024-04-20T01:13:42.706Z 13 Acked 1012 Write 6 Done Done Sent false
125882024-04-20T01:13:42.706Z 14 Acked 1013 Read 10 Done Done Sent false
125892024-04-20T01:13:42.706Z 15 Acked 1014 Read 10 Done Done Sent false
125902024-04-20T01:13:42.706Z 16 Acked 1015 Read 3 Done Done Sent false
125912024-04-20T01:13:42.706Z 17 Acked 1016 Flush 0 Done Done Sent false
125922024-04-20T01:13:42.706Z 18 Acked 1017 Write 5 Done Done Sent false
125932024-04-20T01:13:42.706Z 19 Acked 1018 Read 1 Done Done Sent false
125942024-04-20T01:13:42.706Z 20 Acked 1019 Write 3 Done Done Sent false
125952024-04-20T01:13:42.706Z 21 Acked 1020 Read 2 Done Done Sent false
125962024-04-20T01:13:42.706Z 22 Acked 1021 Read 1 Done Done Sent false
125972024-04-20T01:13:42.706Z 23 Acked 1022 Write 10 Done Done Sent false
125982024-04-20T01:13:42.706Z 24 Acked 1023 Read 7 Done Done Sent false
125992024-04-20T01:13:42.706Z 25 Acked 1024 Write 2 Done Done Sent false
126002024-04-20T01:13:42.706Z 26 Acked 1025 Read 7 Done Done Sent false
126012024-04-20T01:13:42.706Z 27 Acked 1026 Read 3 Done Done Sent false
126022024-04-20T01:13:42.706Z 28 Acked 1027 Read 9 Done Done Sent false
126032024-04-20T01:13:42.706Z 29 Acked 1028 Read 4 Done Done Sent false
126042024-04-20T01:13:42.706Z 30 Acked 1029 Write 7 Sent Sent Sent false
126052024-04-20T01:13:42.706Z STATES DS:0 DS:1 DS:2 TOTAL
126062024-04-20T01:13:42.706Z New 0 0 0 0
126072024-04-20T01:13:42.707Z Sent 1 1 24 26
126082024-04-20T01:13:42.707Z Done 29 29 6 64
126092024-04-20T01:13:42.707Z Skipped 0 0 0 0
126102024-04-20T01:13:42.707Z Error 0 0 0 0
126112024-04-20T01:13:42.707ZLast Flush: 1016 1016 0
126122024-04-20T01:13:42.707ZDownstairs last five completed:
126132024-04-20T01:13:42.707ZUpstairs last five completed: 30 29 28 27 26
126142024-04-20T01:13:42.707Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:13:41.203280866Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1635}
126152024-04-20T01:13:42.707ZWrote out file "/tmp/test_repair_verify.data"
126162024-04-20T01:13:42.711Z
126172024-04-20T01:13:42.711ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
126182024-04-20T01:13:42.725ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126192024-04-20T01:13:42.725Z 0 000-019 115 115 115 191 191 191 F F F
126202024-04-20T01:13:42.725Z 1 020-039 105 105 105 178 178 178 F F F
126212024-04-20T01:13:42.725Z 2 040-059 114 114 114 190 190 190 F F F
126222024-04-20T01:13:42.725Z 3 060-079 114 114 114 190 190 190 F F F
126232024-04-20T01:13:42.725Z 4 080-099 112 112 112 188 188 188 F F F
126242024-04-20T01:13:42.725Z 5 100-119 116 116 112 192 192 188 F F T <---
126252024-04-20T01:13:42.725Z 6 120-139 110 110 110 185 185 185 F F F
126262024-04-20T01:13:42.725Z 7 140-159 113 113 113 189 189 189 T T F <---
126272024-04-20T01:13:42.725Z 8 160-179 115 115 115 191 191 191 T T F <---
126282024-04-20T01:13:42.729Z 9 180-199 111 111 111 186 186 186 T T F <---
126292024-04-20T01:13:42.729Z 10 200-219 115 115 115 191 191 191 F F F
126302024-04-20T01:13:42.729Z 11 220-239 115 115 115 191 191 191 F F F
126312024-04-20T01:13:42.729Z 12 240-259 108 108 108 182 182 182 F F F
126322024-04-20T01:13:42.729Z 13 260-279 115 115 115 191 191 191 F F F
126332024-04-20T01:13:42.729Z 14 280-299 116 116 114 192 192 190 T T T <---
126342024-04-20T01:13:42.729Z 15 300-319 116 116 114 192 192 190 F F T <---
126352024-04-20T01:13:42.729Z 16 320-339 114 114 114 190 190 190 F F F
126362024-04-20T01:13:42.729Z 17 340-359 115 115 115 191 191 191 F F F
126372024-04-20T01:13:42.729Z 18 360-379 115 115 115 191 191 191 F F F
126382024-04-20T01:13:42.729Z 19 380-399 111 111 111 186 186 186 F F F
126392024-04-20T01:13:42.729Z 20 400-419 114 114 114 190 190 190 F F F
126402024-04-20T01:13:42.729Z 21 420-439 115 115 115 191 191 191 F F F
126412024-04-20T01:13:42.729Z 22 440-459 112 112 112 187 187 187 F F F
126422024-04-20T01:13:42.729Z 23 460-479 115 115 115 191 191 191 F F F
126432024-04-20T01:13:42.729Z 24 480-499 116 116 112 192 192 187 F F F <---
126442024-04-20T01:13:42.729Z 25 500-519 107 107 107 181 181 181 F F F
126452024-04-20T01:13:42.730Z 26 520-539 103 103 103 176 176 176 F F F
126462024-04-20T01:13:42.730Z 27 540-559 115 115 115 191 191 191 F F F
126472024-04-20T01:13:42.730Z 28 560-579 115 115 115 191 191 191 T T F <---
126482024-04-20T01:13:42.730Z 29 580-599 115 115 115 191 191 191 F F F
126492024-04-20T01:13:42.730ZMax gen: 116, Max flush: 192
126502024-04-20T01:13:42.730ZError: Difference in extent metadata found!
126512024-04-20T01:13:42.730ZA Difference in extent metadata is expected here
126522024-04-20T01:13:42.730ZOn loop 58
126532024-04-20T01:13:42.730Z
126542024-04-20T01:13:42.737ZVerifying data now
126552024-04-20T01:13:57.134ZNo change after verify
126562024-04-20T01:13:57.134ZLoop: 58 Downstairs dump after verify (and repair):
126572024-04-20T01:13:57.147ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126582024-04-20T01:13:57.147Z 0 000-019 115 115 115 191 191 191 F F F
126592024-04-20T01:13:57.148Z 1 020-039 105 105 105 178 178 178 F F F
126602024-04-20T01:13:57.148Z 2 040-059 114 114 114 190 190 190 F F F
126612024-04-20T01:13:57.148Z 3 060-079 114 114 114 190 190 190 F F F
126622024-04-20T01:13:57.148Z 4 080-099 112 112 112 188 188 188 F F F
126632024-04-20T01:13:57.151Z 5 100-119 116 116 116 192 192 192 F F F
126642024-04-20T01:13:57.152Z 6 120-139 110 110 110 185 185 185 F F F
126652024-04-20T01:13:57.152Z 7 140-159 117 117 117 193 193 193 F F F
126662024-04-20T01:13:57.152Z 8 160-179 117 117 117 193 193 193 F F F
126672024-04-20T01:13:57.152Z 9 180-199 117 117 117 193 193 193 F F F
126682024-04-20T01:13:57.152Z 10 200-219 115 115 115 191 191 191 F F F
126692024-04-20T01:13:57.152Z 11 220-239 115 115 115 191 191 191 F F F
126702024-04-20T01:13:57.152Z 12 240-259 108 108 108 182 182 182 F F F
126712024-04-20T01:13:57.152Z 13 260-279 115 115 115 191 191 191 F F F
126722024-04-20T01:13:57.152Z 14 280-299 117 117 117 193 193 193 F F F
126732024-04-20T01:13:57.152Z 15 300-319 116 116 116 192 192 192 F F F
126742024-04-20T01:13:57.152Z 16 320-339 114 114 114 190 190 190 F F F
126752024-04-20T01:13:57.152Z 17 340-359 115 115 115 191 191 191 F F F
126762024-04-20T01:13:57.152Z 18 360-379 115 115 115 191 191 191 F F F
126772024-04-20T01:13:57.152Z 19 380-399 111 111 111 186 186 186 F F F
126782024-04-20T01:13:57.152Z 20 400-419 114 114 114 190 190 190 F F F
126792024-04-20T01:13:57.152Z 21 420-439 115 115 115 191 191 191 F F F
126802024-04-20T01:13:57.152Z 22 440-459 112 112 112 187 187 187 F F F
126812024-04-20T01:13:57.152Z 23 460-479 115 115 115 191 191 191 F F F
126822024-04-20T01:13:57.152Z 24 480-499 116 116 116 192 192 192 F F F
126832024-04-20T01:13:57.152Z 25 500-519 107 107 107 181 181 181 F F F
126842024-04-20T01:13:57.152Z 26 520-539 103 103 103 176 176 176 F F F
126852024-04-20T01:13:57.153Z 27 540-559 115 115 115 191 191 191 F F F
126862024-04-20T01:13:57.153Z 28 560-579 117 117 117 193 193 193 F F F
126872024-04-20T01:13:57.153Z 29 580-599 115 115 115 191 191 191 F F F
126882024-04-20T01:13:57.153ZMax gen: 117, Max flush: 193
126892024-04-20T01:13:57.153Z
126902024-04-20T01:13:57.153ZBegin loop 59
126912024-04-20T01:13:57.153ZDownstairs to restart: 2
126922024-04-20T01:13:57.169Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.670715281Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643}
126932024-04-20T01:13:57.177Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671033571Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643}
126942024-04-20T01:13:57.177Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671080172Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643}
126952024-04-20T01:13:57.177Z{"msg":"Crucible 7c2ae6ab-4596-408f-8bb0-d8f90fe25e3a has session id: 46306681-1758-4be3-8770-86a9abbf9ed6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671105652Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
126962024-04-20T01:13:57.177Z{"msg":"Upstairs opts: Upstairs UUID: 7c2ae6ab-4596-408f-8bb0-d8f90fe25e3a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
126972024-04-20T01:13:57.178Z,"time":"2024-04-20T01:13:55.671172502Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
126982024-04-20T01:13:57.178Z{"msg":"Crucible stats registered with UUID: 7c2ae6ab-4596-408f-8bb0-d8f90fe25e3a","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671210462Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
126992024-04-20T01:13:57.178Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671249972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643}
127002024-04-20T01:13:57.178Z{"msg":"7c2ae6ab-4596-408f-8bb0-d8f90fe25e3a active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671335462Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127012024-04-20T01:13:57.178Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671361242Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"0","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127022024-04-20T01:13:57.178Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671376062Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"0","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127032024-04-20T01:13:57.178Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671389692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"1","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127042024-04-20T01:13:57.178Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671419742Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"1","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127052024-04-20T01:13:57.178Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671443862Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"2","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127062024-04-20T01:13:57.178Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671479032Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"2","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127072024-04-20T01:13:57.178Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671542173Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127082024-04-20T01:13:57.178Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671580453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127092024-04-20T01:13:57.178Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671607843Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127102024-04-20T01:13:57.178Z{"msg":"ds_connection connected from Ok(127.0.0.1:44522)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671644983Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127112024-04-20T01:13:57.178Z{"msg":"ds_connection connected from Ok(127.0.0.1:61558)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671744203Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127122024-04-20T01:13:57.178Z{"msg":"ds_connection connected from Ok(127.0.0.1:57206)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.671772753Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127132024-04-20T01:13:57.178Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.672375664Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"0","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127142024-04-20T01:13:57.178Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.672413504Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"0","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127152024-04-20T01:13:57.178Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.672428995Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"1","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127162024-04-20T01:13:57.178Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.672443255Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"1","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127172024-04-20T01:13:57.178Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.672808295Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"2","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127182024-04-20T01:13:57.178Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.672854475Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"2","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127192024-04-20T01:13:57.178Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.672985396Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"1","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127202024-04-20T01:13:57.179Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673057766Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"0","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127212024-04-20T01:13:57.179Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673457987Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"2","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127222024-04-20T01:13:57.179Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673629167Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"0","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127232024-04-20T01:13:57.179Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673664797Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"0","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127242024-04-20T01:13:57.179Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673680257Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127252024-04-20T01:13:57.179Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673693347Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"1","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127262024-04-20T01:13:57.179Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673706537Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"1","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127272024-04-20T01:13:57.179Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673726937Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127282024-04-20T01:13:57.179Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673885208Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"2","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127292024-04-20T01:13:57.179Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673916158Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"2","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127302024-04-20T01:13:57.179Z{"msg":"[0]R flush_numbers[0..12]: [191, 178, 190, 190, 188, 192, 185, 193, 193, 193, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673932288Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127312024-04-20T01:13:57.179Z{"msg":"[0]R generation[0..12]: [115, 105, 114, 114, 112, 116, 110, 117, 117, 117, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673946408Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127322024-04-20T01:13:57.179Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673968268Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127332024-04-20T01:13:57.179Z{"msg":"[1]R flush_numbers[0..12]: [191, 178, 190, 190, 188, 192, 185, 193, 193, 193, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673982558Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127342024-04-20T01:13:57.179Z{"msg":"[1]R generation[0..12]: [115, 105, 114, 114, 112, 116, 110, 117, 117, 117, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.673996358Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127352024-04-20T01:13:57.179Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674018708Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127362024-04-20T01:13:57.179Z{"msg":"[2]R flush_numbers[0..12]: [191, 178, 190, 190, 188, 192, 185, 193, 193, 193, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674032748Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127372024-04-20T01:13:57.179Z{"msg":"[2]R generation[0..12]: [115, 105, 114, 114, 112, 116, 110, 117, 117, 117, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674046388Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127382024-04-20T01:13:57.179Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
127392024-04-20T01:13:57.179Z2024-04-20T01:13:55.674068158Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127402024-04-20T01:13:57.179Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674084628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127412024-04-20T01:13:57.179Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674097868Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127422024-04-20T01:13:57.179Z{"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674117848Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127432024-04-20T01:13:57.180Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674130978Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"":"downstairs","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127442024-04-20T01:13:57.180Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674143968Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127452024-04-20T01:13:57.180Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674163048Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"0","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127462024-04-20T01:13:57.180Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674176608Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"1","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127472024-04-20T01:13:57.180Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674189408Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"client":"2","":"downstairs","session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127482024-04-20T01:13:57.180Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674209388Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127492024-04-20T01:13:57.180Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674221888Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127502024-04-20T01:13:57.180Z{"msg":"7c2ae6ab-4596-408f-8bb0-d8f90fe25e3a is now active with session: 46306681-1758-4be3-8770-86a9abbf9ed6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674235468Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127512024-04-20T01:13:57.180Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674254539Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643,"session_id":"46306681-1758-4be3-8770-86a9abbf9ed6"}
127522024-04-20T01:13:57.180Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:13:55.674267059Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643}
127532024-04-20T01:13:57.180ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
127542024-04-20T01:13:57.184ZLoading write count information from file "/tmp/test_repair_verify.data"
127552024-04-20T01:13:57.184ZRun Repair workload
127562024-04-20T01:13:57.184Z01/30 Write block 171 len 32768 data: 8 9 12 11 10 11 11 13
127572024-04-20T01:13:57.184Z02/30 Write block 423 len 8192 data: 8 6
127582024-04-20T01:13:57.184Z03/30 Write block 470 len 12288 data: 13 11 11
127592024-04-20T01:13:57.184Z04/30 Read block 153 len 16384
127602024-04-20T01:13:57.184Z05/30 Write block 198 len 4096 data: 11
127612024-04-20T01:13:57.184Z06/30 Read block 11 len 32768
127622024-04-20T01:13:57.184Z07/30 Write block 414 len 28672 data: 13 13 14 10 11 11 10
127632024-04-20T01:13:57.184Z08/30 Read block 412 len 28672
127642024-04-20T01:13:57.184Z09/30 Flush
127652024-04-20T01:13:57.188Z10/30 Write block 209 len 40960 data: 9 10 10 10 11 11 9 10 14 13
127662024-04-20T01:13:57.188Z11/30 Read block 2 len 24576
127672024-04-20T01:13:57.188Z12/30 Read block 51 len 16384
127682024-04-20T01:13:57.188Z13/30 Read block 423 len 32768
127692024-04-20T01:13:57.188Z14/30 Write block 250 len 4096 data: 12
127702024-04-20T01:13:57.191Z15/30 Read block 226 len 16384
127712024-04-20T01:13:57.191Z16/30 Read block 582 len 4096
127722024-04-20T01:13:57.191Z17/30 Read block 178 len 32768
127732024-04-20T01:13:57.191Z18/30 Read block 120 len 40960
127742024-04-20T01:13:57.194Z19/30 Write block 376 len 40960 data: 10 12 13 12 13 10 9 8 6 3
127752024-04-20T01:13:57.195Z20/30 Flush
127762024-04-20T01:13:57.198Z21/30 Write block 438 len 36864 data: 9 10 10 11 12 9 10 9 9
127772024-04-20T01:13:57.198Z22/30 Write block 191 len 28672 data: 11 9 8 9 9 8 10
127782024-04-20T01:13:57.198Z23/30 Write block 74 len 4096 data: 10
127792024-04-20T01:13:57.201Z24/30 Flush
127802024-04-20T01:13:57.201Z25/30 Write block 408 len 8192 data: 10 11
127812024-04-20T01:13:57.204Z26/30 Read block 131 len 4096
127822024-04-20T01:13:57.204Z27/30 Write block 561 len 20480 data: 6 7 5 8 11
127832024-04-20T01:13:57.204Z28/30 Write block 129 len 20480 data: 11 13 12 13 11
127842024-04-20T01:13:57.208Z29/30 Write block 516 len 20480 data: 13 13 11 13 14
127852024-04-20T01:13:57.209Z30/30 Write block 276 len 40960 data: 5 7 7 7 9 9 7 5 6 6
127862024-04-20T01:13:57.209Z----------------------------------------------------------------
127872024-04-20T01:13:57.209Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30
127882024-04-20T01:13:57.209ZStates: Active Active Active
127892024-04-20T01:13:57.209ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127902024-04-20T01:13:57.209Z 1 Acked 1000 Write 8 Done Done Done false
127912024-04-20T01:13:57.209Z 2 Acked 1001 Write 2 Done Done Done false
127922024-04-20T01:13:57.209Z 3 Acked 1002 Write 3 Done Done Done false
127932024-04-20T01:13:57.209Z 4 Acked 1003 Read 4 Done Done Done false
127942024-04-20T01:13:57.209Z 5 Acked 1004 Write 1 Done Done Done false
127952024-04-20T01:13:57.209Z 6 Acked 1005 Read 8 Done Done Sent false
127962024-04-20T01:13:57.209Z 7 Acked 1006 Write 7 Done Done Sent false
127972024-04-20T01:13:57.209Z 8 Acked 1007 Read 7 Done Done Sent false
127982024-04-20T01:13:57.209Z 9 Acked 1008 Flush 0 Done Done Sent false
127992024-04-20T01:13:57.209Z 10 Acked 1009 Write 10 Done Done Sent false
128002024-04-20T01:13:57.209Z 11 Acked 1010 Read 6 Done Done Sent false
128012024-04-20T01:13:57.209Z 12 Acked 1011 Read 4 Done Done Sent false
128022024-04-20T01:13:57.209Z 13 Acked 1012 Read 8 Done Done Sent false
128032024-04-20T01:13:57.209Z 14 Acked 1013 Write 1 Done Done Sent false
128042024-04-20T01:13:57.209Z 15 Acked 1014 Read 4 Done Done Sent false
128052024-04-20T01:13:57.209Z 16 Acked 1015 Read 1 Done Done Sent false
128062024-04-20T01:13:57.209Z 17 Acked 1016 Read 8 Done Done Sent false
128072024-04-20T01:13:57.210Z 18 Acked 1017 Read 10 Done Done Sent false
128082024-04-20T01:13:57.210Z 19 Acked 1018 Write 10 Done Done Sent false
128092024-04-20T01:13:57.210Z 20 Acked 1019 Flush 0 Done Done Sent false
128102024-04-20T01:13:57.210Z 21 Acked 1020 Write 9 Done Done Sent false
128112024-04-20T01:13:57.210Z 22 Acked 1021 Write 7 Done Done Sent false
128122024-04-20T01:13:57.210Z 23 Acked 1022 Write 1 Done Done Sent false
128132024-04-20T01:13:57.210Z 24 Acked 1023 Flush 0 Done Done Sent false
128142024-04-20T01:13:57.210Z 25 Acked 1024 Write 2 Done Done Sent false
128152024-04-20T01:13:57.210Z 26 Acked 1025 Read 1 Done Done Sent false
128162024-04-20T01:13:57.210Z 27 Acked 1026 Write 5 Done Done Sent false
128172024-04-20T01:13:57.210Z 28 Acked 1027 Write 5 Done Done Sent false
128182024-04-20T01:13:57.210Z 29 Acked 1028 Write 5 Done Done Sent false
128192024-04-20T01:13:57.210Z 30 Acked 1029 Write 10 Sent Sent Sent false
128202024-04-20T01:13:57.210Z STATES DS:0 DS:1 DS:2 TOTAL
128212024-04-20T01:13:57.210Z New 0 0 0 0
128222024-04-20T01:13:57.210Z Sent 1 1 25 27
128232024-04-20T01:13:57.210Z Done 29 29 5 63
128242024-04-20T01:13:57.210Z Skipped 0 0 0 0
128252024-04-20T01:13:57.210Z Error 0 0 0 0
128262024-04-20T01:13:57.210ZLast Flush: 1023 1023 0
128272024-04-20T01:13:57.210ZDownstairs last five completed:
128282024-04-20T01:13:57.210ZUpstairs last five completed: 30 29 28 27 26
128292024-04-20T01:13:57.210Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:13:55.708789915Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1643}
128302024-04-20T01:13:57.210ZWrote out file "/tmp/test_repair_verify.data"
128312024-04-20T01:13:57.219Z
128322024-04-20T01:13:57.219ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
128332024-04-20T01:13:57.232ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128342024-04-20T01:13:57.232Z 0 000-019 115 115 115 191 191 191 F F F
128352024-04-20T01:13:57.232Z 1 020-039 105 105 105 178 178 178 F F F
128362024-04-20T01:13:57.232Z 2 040-059 114 114 114 190 190 190 F F F
128372024-04-20T01:13:57.232Z 3 060-079 118 118 114 196 196 190 F F F <---
128382024-04-20T01:13:57.232Z 4 080-099 112 112 112 188 188 188 F F F
128392024-04-20T01:13:57.232Z 5 100-119 116 116 116 192 192 192 F F F
128402024-04-20T01:13:57.232Z 6 120-139 110 110 110 185 185 185 T T F <---
128412024-04-20T01:13:57.232Z 7 140-159 117 117 117 193 193 193 F F F
128422024-04-20T01:13:57.232Z 8 160-179 118 118 117 194 194 193 F F T <---
128432024-04-20T01:13:57.292Z 9 180-199 118 118 117 196 196 193 F F T <---
128442024-04-20T01:13:57.292Z 10 200-219 118 118 115 195 195 191 F F F <---
128452024-04-20T01:13:57.292Z 11 220-239 115 115 115 191 191 191 F F F
128462024-04-20T01:13:57.292Z 12 240-259 118 118 108 195 195 182 F F F <---
128472024-04-20T01:13:57.292Z 13 260-279 115 115 115 191 191 191 T T F <---
128482024-04-20T01:13:57.292Z 14 280-299 117 117 117 193 193 193 T T F <---
128492024-04-20T01:13:57.292Z 15 300-319 116 116 116 192 192 192 F F F
128502024-04-20T01:13:57.292Z 16 320-339 114 114 114 190 190 190 F F F
128512024-04-20T01:13:57.292Z 17 340-359 115 115 115 191 191 191 F F F
128522024-04-20T01:13:57.292Z 18 360-379 118 118 115 195 195 191 F F F <---
128532024-04-20T01:13:57.292Z 19 380-399 118 118 111 195 195 186 F F F <---
128542024-04-20T01:13:57.292Z 20 400-419 118 118 114 194 194 190 T T F <---
128552024-04-20T01:13:57.292Z 21 420-439 118 118 115 196 196 191 F F T <---
128562024-04-20T01:13:57.293Z 22 440-459 118 118 112 196 196 187 F F F <---
128572024-04-20T01:13:57.293Z 23 460-479 118 118 115 194 194 191 F F T <---
128582024-04-20T01:13:57.293Z 24 480-499 116 116 116 192 192 192 F F F
128592024-04-20T01:13:57.293Z 25 500-519 107 107 107 181 181 181 T T F <---
128602024-04-20T01:13:57.293Z 26 520-539 103 103 103 176 176 176 T T F <---
128612024-04-20T01:13:57.293Z 27 540-559 115 115 115 191 191 191 F F F
128622024-04-20T01:13:57.293Z 28 560-579 117 117 117 193 193 193 T T F <---
128632024-04-20T01:13:57.293Z 29 580-599 115 115 115 191 191 191 F F F
128642024-04-20T01:13:57.293ZMax gen: 118, Max flush: 196
128652024-04-20T01:13:57.293ZError: Difference in extent metadata found!
128662024-04-20T01:13:57.293ZA Difference in extent metadata is expected here
128672024-04-20T01:13:57.293ZOn loop 59
128682024-04-20T01:13:57.293Z
128692024-04-20T01:13:57.297ZVerifying data now
128702024-04-20T01:14:12.009ZNo change after verify
128712024-04-20T01:14:12.009ZLoop: 59 Downstairs dump after verify (and repair):
128722024-04-20T01:14:12.020ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128732024-04-20T01:14:12.020Z 0 000-019 115 115 115 191 191 191 F F F
128742024-04-20T01:14:12.020Z 1 020-039 105 105 105 178 178 178 F F F
128752024-04-20T01:14:12.020Z 2 040-059 114 114 114 190 190 190 F F F
128762024-04-20T01:14:12.020Z 3 060-079 118 118 118 196 196 196 F F F
128772024-04-20T01:14:12.020Z 4 080-099 112 112 112 188 188 188 F F F
128782024-04-20T01:14:12.020Z 5 100-119 116 116 116 192 192 192 F F F
128792024-04-20T01:14:12.025Z 6 120-139 119 119 119 197 197 197 F F F
128802024-04-20T01:14:12.025Z 7 140-159 117 117 117 193 193 193 F F F
128812024-04-20T01:14:12.025Z 8 160-179 118 118 118 194 194 194 F F F
128822024-04-20T01:14:12.025Z 9 180-199 118 118 118 196 196 196 F F F
128832024-04-20T01:14:12.025Z 10 200-219 118 118 118 195 195 195 F F F
128842024-04-20T01:14:12.025Z 11 220-239 115 115 115 191 191 191 F F F
128852024-04-20T01:14:12.025Z 12 240-259 118 118 118 195 195 195 F F F
128862024-04-20T01:14:12.025Z 13 260-279 119 119 119 197 197 197 F F F
128872024-04-20T01:14:12.025Z 14 280-299 119 119 119 197 197 197 F F F
128882024-04-20T01:14:12.025Z 15 300-319 116 116 116 192 192 192 F F F
128892024-04-20T01:14:12.025Z 16 320-339 114 114 114 190 190 190 F F F
128902024-04-20T01:14:12.025Z 17 340-359 115 115 115 191 191 191 F F F
128912024-04-20T01:14:12.025Z 18 360-379 118 118 118 195 195 195 F F F
128922024-04-20T01:14:12.025Z 19 380-399 118 118 118 195 195 195 F F F
128932024-04-20T01:14:12.026Z 20 400-419 119 119 119 197 197 197 F F F
128942024-04-20T01:14:12.026Z 21 420-439 118 118 118 196 196 196 F F F
128952024-04-20T01:14:12.026Z 22 440-459 118 118 118 196 196 196 F F F
128962024-04-20T01:14:12.026Z 23 460-479 118 118 118 194 194 194 F F F
128972024-04-20T01:14:12.026Z 24 480-499 116 116 116 192 192 192 F F F
128982024-04-20T01:14:12.026Z 25 500-519 119 119 119 197 197 197 F F F
128992024-04-20T01:14:12.026Z 26 520-539 119 119 119 197 197 197 F F F
129002024-04-20T01:14:12.026Z 27 540-559 115 115 115 191 191 191 F F F
129012024-04-20T01:14:12.026Z 28 560-579 119 119 119 197 197 197 F F F
129022024-04-20T01:14:12.026Z 29 580-599 115 115 115 191 191 191 F F F
129032024-04-20T01:14:12.026ZMax gen: 119, Max flush: 197
129042024-04-20T01:14:12.026Z
129052024-04-20T01:14:12.026ZBegin loop 60
129062024-04-20T01:14:12.026ZDownstairs to restart: 1
129072024-04-20T01:14:12.042Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543072546Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651}
129082024-04-20T01:14:12.042Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543327007Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651}
129092024-04-20T01:14:12.048Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543362647Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651}
129102024-04-20T01:14:12.048Z{"msg":"Crucible 9568d3fd-f5a3-42e2-a606-184e2a61098c has session id: 5c73db7d-8a49-42aa-8683-22a5ea93d1aa","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543410907Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129112024-04-20T01:14:12.048Z{Crucible runtime is spawned
129122024-04-20T01:14:12.048Z"msg":"Upstairs opts: Upstairs UUID: 9568d3fd-f5a3-42e2-a606-184e2a61098c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543468197Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129132024-04-20T01:14:12.048Z{"msg":"Crucible stats registered with UUID: 9568d3fd-f5a3-42e2-a606-184e2a61098c","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543504807Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129142024-04-20T01:14:12.048Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543519558Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651}
129152024-04-20T01:14:12.048Z{"msg":"9568d3fd-f5a3-42e2-a606-184e2a61098c active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543619168Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129162024-04-20T01:14:12.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543657298Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"0","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129172024-04-20T01:14:12.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543684128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"0","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129182024-04-20T01:14:12.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543708958Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"1","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129192024-04-20T01:14:12.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543740138Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"1","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129202024-04-20T01:14:12.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543763788Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"2","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129212024-04-20T01:14:12.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543809418Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"2","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129222024-04-20T01:14:12.048Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543837138Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129232024-04-20T01:14:12.048Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543862058Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129242024-04-20T01:14:12.048Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.543889198Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129252024-04-20T01:14:12.048Z{"msg":"ds_connection connected from Ok(127.0.0.1:44684)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.544018039Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129262024-04-20T01:14:12.048Z{"msg":"ds_connection connected from Ok(127.0.0.1:55421)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.544064099Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129272024-04-20T01:14:12.049Z{"msg":"ds_connection connected from Ok(127.0.0.1:52007)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.544099869Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129282024-04-20T01:14:12.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.54482869Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"2","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129292024-04-20T01:14:12.049Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.54488139Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"2","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129302024-04-20T01:14:12.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.544911141Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"0","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129312024-04-20T01:14:12.049Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.544950961Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"0","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129322024-04-20T01:14:12.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.545264561Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"1","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129332024-04-20T01:14:12.049Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.545323102Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"1","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129342024-04-20T01:14:12.049Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.545358082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"2","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129352024-04-20T01:14:12.049Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.545484722Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"0","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129362024-04-20T01:14:12.049Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.545905363Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"1","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129372024-04-20T01:14:12.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.545942093Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"0","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129382024-04-20T01:14:12.049Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.545958563Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"0","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129392024-04-20T01:14:12.049Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.545985193Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129402024-04-20T01:14:12.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.545998553Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"2","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129412024-04-20T01:14:12.049Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546025583Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"2","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129422024-04-20T01:14:12.049Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546049853Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129432024-04-20T01:14:12.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546486504Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"1","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129442024-04-20T01:14:12.049Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546521164Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"1","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129452024-04-20T01:14:12.049Z{"msg":"[0]R flush_numbers[0..12]: [191, 178, 190, 196, 188, 192, 197, 193, 194, 196, 195, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546537514Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129462024-04-20T01:14:12.049Z{"msg":"[0]R generation[0..12]: [115, 105, 114, 118, 112, 116, 119, 117, 118, 118, 118, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546559444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129472024-04-20T01:14:12.049Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546574324Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129482024-04-20T01:14:12.049Z{"msg":"[1]R flush_numbers[0..12]: [191, 178, 190, 196, 188, 192, 197, 193, 194, 196, 195, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546595634Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129492024-04-20T01:14:12.049Z{"msg":"[1]R generation[0..12]: [115, 105, 114, 118, 112, 116, 119, 117, 118, 118, 118, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546610024Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129502024-04-20T01:14:12.050Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546636574Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129512024-04-20T01:14:12.050Z{"msg":"[2]R flush_numbers[0..12]: [191, 178, 190, 196, 188, 192, 197, 193, 194, 196, 195, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546650834Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129522024-04-20T01:14:12.050Z{"msg":"[2]R generation[0..12]: [115, 105, 114, 118, 112, 116, 119, 117, 118, 118, 118, 115]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
129532024-04-20T01:14:12.050Z,"time":"2024-04-20T01:14:10.546664305Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129542024-04-20T01:14:12.050Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546688234Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129552024-04-20T01:14:12.050Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546701274Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129562024-04-20T01:14:12.050Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546721705Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129572024-04-20T01:14:12.050Z{"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546739895Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129582024-04-20T01:14:12.050Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546759395Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"":"downstairs","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129592024-04-20T01:14:12.050Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546772925Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129602024-04-20T01:14:12.050Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546791645Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"0","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129612024-04-20T01:14:12.050Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546805045Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"1","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129622024-04-20T01:14:12.050Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546818075Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"client":"2","":"downstairs","session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129632024-04-20T01:14:12.050Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546837745Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129642024-04-20T01:14:12.050Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546850305Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129652024-04-20T01:14:12.050Z{"msg":"9568d3fd-f5a3-42e2-a606-184e2a61098c is now active with session: 5c73db7d-8a49-42aa-8683-22a5ea93d1aa","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546863015Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129662024-04-20T01:14:12.050Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546881565Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651,"session_id":"5c73db7d-8a49-42aa-8683-22a5ea93d1aa"}
129672024-04-20T01:14:12.050Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:10.546893975Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651}
129682024-04-20T01:14:12.050ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
129692024-04-20T01:14:12.055ZLoading write count information from file "/tmp/test_repair_verify.data"
129702024-04-20T01:14:12.055ZRun Repair workload
129712024-04-20T01:14:12.055Z01/30 Write block 592 len 24576 data: 8 9 11 10 10 7
129722024-04-20T01:14:12.055Z02/30 Read block 406 len 20480
129732024-04-20T01:14:12.055Z03/30 Read block 544 len 28672
129742024-04-20T01:14:12.056Z04/30 Write block 115 len 40960 data: 9 8 7 8 8 8 8 9 9 12
129752024-04-20T01:14:12.056Z05/30 Read block 118 len 40960
129762024-04-20T01:14:12.056Z06/30 Read block 405 len 16384
129772024-04-20T01:14:12.056Z07/30 Read block 64 len 32768
129782024-04-20T01:14:12.056Z08/30 Read block 505 len 40960
129792024-04-20T01:14:12.078Z09/30 Write block 255 len 12288 data: 11 11 8
129802024-04-20T01:14:12.079Z10/30 Read block 21 len 32768
129812024-04-20T01:14:12.079Z11/30 Read block 203 len 36864
129822024-04-20T01:14:12.079Z12/30 Read block 222 len 28672
129832024-04-20T01:14:12.079Z13/30 Flush
129842024-04-20T01:14:12.079Z14/30 Write block 529 len 28672 data: 8 8 10 10 9 11 9
129852024-04-20T01:14:12.079Z15/30 Flush
129862024-04-20T01:14:12.080Z16/30 Write block 135 len 4096 data: 10
129872024-04-20T01:14:12.080Z17/30 Flush
129882024-04-20T01:14:12.080Z18/30 Write block 534 len 28672 data: 12 10 8 7 7 6 7
129892024-04-20T01:14:12.080Z19/30 Write block 584 len 16384 data: 11 14 17 15
129902024-04-20T01:14:12.080Z20/30 Read block 367 len 16384
129912024-04-20T01:14:12.080Z21/30 Write block 14 len 36864 data: 6 9 10 8 8 8 8 7 7
129922024-04-20T01:14:12.080Z22/30 Read block 504 len 8192
129932024-04-20T01:14:12.081Z23/30 Read block 474 len 36864
129942024-04-20T01:14:12.081Z24/30 Read block 361 len 20480
129952024-04-20T01:14:12.081Z25/30 Read block 248 len 40960
129962024-04-20T01:14:12.081Z26/30 Read block 219 len 32768
129972024-04-20T01:14:12.081Z27/30 Write block 114 len 32768 data: 10 10 9 8 9 9 9 9
129982024-04-20T01:14:12.086Z28/30 Write block 182 len 12288 data: 12 8 6
129992024-04-20T01:14:12.086Z29/30 Read block 192 len 28672
130002024-04-20T01:14:12.086Z30/30 Write block 369 len 28672 data: 9 11 15 14 15 15 14
130012024-04-20T01:14:12.086Z----------------------------------------------------------------
130022024-04-20T01:14:12.086Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
130032024-04-20T01:14:12.086ZStates: Active Active Active
130042024-04-20T01:14:12.086ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
130052024-04-20T01:14:12.086Z 1 Acked 1000 Write 6 Done Done Done false
130062024-04-20T01:14:12.086Z 2 Acked 1001 Read 5 Done Done Done false
130072024-04-20T01:14:12.086Z 3 Acked 1002 Read 7 Done Done Done false
130082024-04-20T01:14:12.086Z 4 Acked 1003 Write 10 Done Done Done false
130092024-04-20T01:14:12.086Z 5 Acked 1004 Read 10 Done Done Done false
130102024-04-20T01:14:12.086Z 6 Acked 1005 Read 4 Done Sent Done false
130112024-04-20T01:14:12.086Z 7 Acked 1006 Read 8 Done Sent Done false
130122024-04-20T01:14:12.086Z 8 Acked 1007 Read 10 Done Sent Done false
130132024-04-20T01:14:12.086Z 9 Acked 1008 Write 3 Done Sent Done false
130142024-04-20T01:14:12.086Z 10 Acked 1009 Read 8 Done Sent Done false
130152024-04-20T01:14:12.086Z 11 Acked 1010 Read 9 Done Sent Done false
130162024-04-20T01:14:12.086Z 12 Acked 1011 Read 7 Done Sent Done false
130172024-04-20T01:14:12.086Z 13 Acked 1012 Flush 0 Done Sent Done false
130182024-04-20T01:14:12.086Z 14 Acked 1013 Write 7 Done Sent Done false
130192024-04-20T01:14:12.086Z 15 Acked 1014 Flush 0 Done Sent Done false
130202024-04-20T01:14:12.087Z 16 Acked 1015 Write 1 Done Sent Done false
130212024-04-20T01:14:12.087Z 17 Acked 1016 Flush 0 Done Sent Done false
130222024-04-20T01:14:12.087Z 18 Acked 1017 Write 7 Done Sent Done false
130232024-04-20T01:14:12.087Z 19 Acked 1018 Write 4 Done Sent Done false
130242024-04-20T01:14:12.087Z 20 Acked 1019 Read 4 Done Sent Done false
130252024-04-20T01:14:12.087Z 21 Acked 1020 Write 9 Done Sent Done false
130262024-04-20T01:14:12.087Z 22 Acked 1021 Read 2 Done Sent Done false
130272024-04-20T01:14:12.087Z 23 Acked 1022 Read 9 Done Sent Done false
130282024-04-20T01:14:12.087Z 24 Acked 1023 Read 5 Done Sent Done false
130292024-04-20T01:14:12.087Z 25 Acked 1024 Read 10 Done Sent Done false
130302024-04-20T01:14:12.087Z 26 Acked 1025 Read 8 Done Sent Done false
130312024-04-20T01:14:12.087Z 27 Acked 1026 Write 8 Done Sent Done false
130322024-04-20T01:14:12.087Z 28 Acked 1027 Write 3 Done Sent Done false
130332024-04-20T01:14:12.087Z 29 Acked 1028 Read 7 Done Sent Done false
130342024-04-20T01:14:12.087Z 30 Acked 1029 Write 7 Sent Sent Sent false
130352024-04-20T01:14:12.087Z STATES DS:0 DS:1 DS:2 TOTAL
130362024-04-20T01:14:12.087Z New 0 0 0 0
130372024-04-20T01:14:12.087Z Sent 1 25 1 27
130382024-04-20T01:14:12.087Z Done 29 5 29 63
130392024-04-20T01:14:12.087Z Skipped 0 0 0 0
130402024-04-20T01:14:12.087Z Error 0 0 0 0
130412024-04-20T01:14:12.087ZLast Flush: 1016 0 1016
130422024-04-20T01:14:12.087ZDownstairs last five completed:
130432024-04-20T01:14:12.087ZUpstairs last five completed: 30 29 28 27 26
130442024-04-20T01:14:12.088Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:14:10.580044139Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1651}
130452024-04-20T01:14:12.088ZWrote out file "/tmp/test_repair_verify.data"
130462024-04-20T01:14:12.088Z
130472024-04-20T01:14:12.091ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
130482024-04-20T01:14:12.104ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130492024-04-20T01:14:12.104Z 0 000-019 115 115 115 191 191 191 T F T <---
130502024-04-20T01:14:12.104Z 1 020-039 105 105 105 178 178 178 T F T <---
130512024-04-20T01:14:12.105Z 2 040-059 114 114 114 190 190 190 F F F
130522024-04-20T01:14:12.105Z 3 060-079 118 118 118 196 196 196 F F F
130532024-04-20T01:14:12.105Z 4 080-099 112 112 112 188 188 188 F F F
130542024-04-20T01:14:12.105Z 5 100-119 120 116 120 198 192 198 T T T <---
130552024-04-20T01:14:12.105Z 6 120-139 120 119 120 200 197 200 T T T <---
130562024-04-20T01:14:12.105Z 7 140-159 117 117 117 193 193 193 F F F
130572024-04-20T01:14:12.109Z 8 160-179 118 118 118 194 194 194 F F F
130582024-04-20T01:14:12.109Z 9 180-199 118 118 118 196 196 196 T F T <---
130592024-04-20T01:14:12.109Z 10 200-219 118 118 118 195 195 195 F F F
130602024-04-20T01:14:12.109Z 11 220-239 115 115 115 191 191 191 F F F
130612024-04-20T01:14:12.109Z 12 240-259 120 118 120 198 195 198 F F F <---
130622024-04-20T01:14:12.109Z 13 260-279 119 119 119 197 197 197 F F F
130632024-04-20T01:14:12.109Z 14 280-299 119 119 119 197 197 197 F F F
130642024-04-20T01:14:12.109Z 15 300-319 116 116 116 192 192 192 F F F
130652024-04-20T01:14:12.109Z 16 320-339 114 114 114 190 190 190 F F F
130662024-04-20T01:14:12.109Z 17 340-359 115 115 115 191 191 191 F F F
130672024-04-20T01:14:12.109Z 18 360-379 118 118 118 195 195 195 T F T <---
130682024-04-20T01:14:12.109Z 19 380-399 118 118 118 195 195 195 F F F
130692024-04-20T01:14:12.109Z 20 400-419 119 119 119 197 197 197 F F F
130702024-04-20T01:14:12.109Z 21 420-439 118 118 118 196 196 196 F F F
130712024-04-20T01:14:12.109Z 22 440-459 118 118 118 196 196 196 F F F
130722024-04-20T01:14:12.109Z 23 460-479 118 118 118 194 194 194 F F F
130732024-04-20T01:14:12.109Z 24 480-499 116 116 116 192 192 192 F F F
130742024-04-20T01:14:12.109Z 25 500-519 119 119 119 197 197 197 F F F
130752024-04-20T01:14:12.109Z 26 520-539 120 119 120 199 197 199 T F T <---
130762024-04-20T01:14:12.109Z 27 540-559 115 115 115 191 191 191 T F T <---
130772024-04-20T01:14:12.110Z 28 560-579 119 119 119 197 197 197 F F F
130782024-04-20T01:14:12.110Z 29 580-599 120 115 120 198 191 198 T T T <---
130792024-04-20T01:14:12.110ZMax gen: 120, Max flush: 200
130802024-04-20T01:14:12.110ZError: Difference in extent metadata found!
130812024-04-20T01:14:12.110ZA Difference in extent metadata is expected here
130822024-04-20T01:14:12.110ZOn loop 60
130832024-04-20T01:14:12.110Z
130842024-04-20T01:14:12.114ZVerifying data now
130852024-04-20T01:14:26.593ZNo change after verify
130862024-04-20T01:14:26.593ZLoop: 60 Downstairs dump after verify (and repair):
130872024-04-20T01:14:26.607ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130882024-04-20T01:14:26.607Z 0 000-019 121 121 121 201 201 201 F F F
130892024-04-20T01:14:26.607Z 1 020-039 121 121 121 201 201 201 F F F
130902024-04-20T01:14:26.607Z 2 040-059 114 114 114 190 190 190 F F F
130912024-04-20T01:14:26.607Z 3 060-079 118 118 118 196 196 196 F F F
130922024-04-20T01:14:26.607Z 4 080-099 112 112 112 188 188 188 F F F
130932024-04-20T01:14:26.607Z 5 100-119 121 121 121 201 201 201 F F F
130942024-04-20T01:14:26.607Z 6 120-139 121 121 121 201 201 201 F F F
130952024-04-20T01:14:26.607Z 7 140-159 117 117 117 193 193 193 F F F
130962024-04-20T01:14:26.607Z 8 160-179 118 118 118 194 194 194 F F F
130972024-04-20T01:14:26.607Z 9 180-199 121 121 121 201 201 201 F F F
130982024-04-20T01:14:26.607Z 10 200-219 118 118 118 195 195 195 F F F
130992024-04-20T01:14:26.607Z 11 220-239 115 115 115 191 191 191 F F F
131002024-04-20T01:14:26.607Z 12 240-259 120 120 120 198 198 198 F F F
131012024-04-20T01:14:26.611Z 13 260-279 119 119 119 197 197 197 F F F
131022024-04-20T01:14:26.611Z 14 280-299 119 119 119 197 197 197 F F F
131032024-04-20T01:14:26.611Z 15 300-319 116 116 116 192 192 192 F F F
131042024-04-20T01:14:26.611Z 16 320-339 114 114 114 190 190 190 F F F
131052024-04-20T01:14:26.611Z 17 340-359 115 115 115 191 191 191 F F F
131062024-04-20T01:14:26.611Z 18 360-379 121 121 121 201 201 201 F F F
131072024-04-20T01:14:26.611Z 19 380-399 118 118 118 195 195 195 F F F
131082024-04-20T01:14:26.611Z 20 400-419 119 119 119 197 197 197 F F F
131092024-04-20T01:14:26.611Z 21 420-439 118 118 118 196 196 196 F F F
131102024-04-20T01:14:26.611Z 22 440-459 118 118 118 196 196 196 F F F
131112024-04-20T01:14:26.611Z 23 460-479 118 118 118 194 194 194 F F F
131122024-04-20T01:14:26.611Z 24 480-499 116 116 116 192 192 192 F F F
131132024-04-20T01:14:26.611Z 25 500-519 119 119 119 197 197 197 F F F
131142024-04-20T01:14:26.612Z 26 520-539 121 121 121 201 201 201 F F F
131152024-04-20T01:14:26.612Z 27 540-559 121 121 121 201 201 201 F F F
131162024-04-20T01:14:26.612Z 28 560-579 119 119 119 197 197 197 F F F
131172024-04-20T01:14:26.612Z 29 580-599 121 121 121 201 201 201 F F F
131182024-04-20T01:14:26.612ZMax gen: 121, Max flush: 201
131192024-04-20T01:14:26.612Z
131202024-04-20T01:14:26.612ZBegin loop 61
131212024-04-20T01:14:26.612ZDownstairs to restart: 0
131222024-04-20T01:14:26.629Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.129507636Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659}
131232024-04-20T01:14:26.629Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.129733876Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659}
131242024-04-20T01:14:26.629Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.129768556Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659}
131252024-04-20T01:14:26.635Z{"msg":"Crucible 6b5b25d6-5e10-4e07-8eb9-c769108f4aa5 has session id: a1935177-52c7-49db-b68e-561cd7c00c76","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.129791887Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131262024-04-20T01:14:26.635Z{Crucible runtime is spawned
131272024-04-20T01:14:26.635Z"msg":"Upstairs opts: Upstairs UUID: 6b5b25d6-5e10-4e07-8eb9-c769108f4aa5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.129856267Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131282024-04-20T01:14:26.635Z{"msg":"Crucible stats registered with UUID: 6b5b25d6-5e10-4e07-8eb9-c769108f4aa5","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.129903937Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131292024-04-20T01:14:26.635Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.129927997Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659}
131302024-04-20T01:14:26.635Z{"msg":"6b5b25d6-5e10-4e07-8eb9-c769108f4aa5 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130065127Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131312024-04-20T01:14:26.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130091547Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"0","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131322024-04-20T01:14:26.635Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130106897Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"0","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131332024-04-20T01:14:26.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130120807Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"1","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131342024-04-20T01:14:26.635Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130150797Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"1","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131352024-04-20T01:14:26.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130174677Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"2","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131362024-04-20T01:14:26.635Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130267238Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"2","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131372024-04-20T01:14:26.635Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130346048Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131382024-04-20T01:14:26.635Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130412968Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131392024-04-20T01:14:26.635Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130462158Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131402024-04-20T01:14:26.636Z{"msg":"ds_connection connected from Ok(127.0.0.1:53813)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130494208Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131412024-04-20T01:14:26.636Z{"msg":"ds_connection connected from Ok(127.0.0.1:64956)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130510728Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131422024-04-20T01:14:26.636Z{"msg":"ds_connection connected from Ok(127.0.0.1:62463)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.130527058Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131432024-04-20T01:14:26.636Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.13115967Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"1","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131442024-04-20T01:14:26.636Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.1312005Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"1","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131452024-04-20T01:14:26.636Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.13122705Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"2","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131462024-04-20T01:14:26.636Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.13126335Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"2","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131472024-04-20T01:14:26.636Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.131739421Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"1","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131482024-04-20T01:14:26.636Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.131795921Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"0","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131492024-04-20T01:14:26.636Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.131829211Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"0","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131502024-04-20T01:14:26.636Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.131858861Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"2","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131512024-04-20T01:14:26.636Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132261712Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"1","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131522024-04-20T01:14:26.636Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132309962Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"1","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131532024-04-20T01:14:26.636Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132342482Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131542024-04-20T01:14:26.636Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132366702Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"2","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131552024-04-20T01:14:26.636Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132380922Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"2","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131562024-04-20T01:14:26.636Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132394722Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131572024-04-20T01:14:26.636Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132469082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"0","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131582024-04-20T01:14:26.636Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132836394Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"0","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131592024-04-20T01:14:26.636Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132873063Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"0","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131602024-04-20T01:14:26.636Z{"msg":"[0]R flush_numbers[0..12]: [201, 201, 190, 196, 188, 201, 201, 193, 194, 201, 195, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132889263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131612024-04-20T01:14:26.637Z{"msg":"[0]R generation[0..12]: [121, 121, 114, 118, 112, 121, 121, 117, 118, 121, 118, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132903594Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131622024-04-20T01:14:26.637Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132925204Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131632024-04-20T01:14:26.637Z{"msg":"[1]R flush_numbers[0..12]: [201, 201, 190, 196, 188, 201, 201, 193, 194, 201, 195, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132939394Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131642024-04-20T01:14:26.637Z{"msg":"[1]R generation[0..12]: [121, 121, 114, 118, 112, 121, 121, 117, 118, 121, 118, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132953184Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131652024-04-20T01:14:26.637Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132974534Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131662024-04-20T01:14:26.637Z{"msg":"[2]R flush_numbers[0..12]: [201, 201, 190, 196, 188, 201, 201, 193, 194, 201, 195, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.132989034Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131672024-04-20T01:14:26.637Z{"msg":"[2]R generation[0..12]: [121, 121, 114, 118, 112, 121, 121, 117, 118, 121, 118, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133002964Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131682024-04-20T01:14:26.637Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133018884Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs",Wait for a query_work_queue command to finish before sending IO
131692024-04-20T01:14:26.637Z"":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131702024-04-20T01:14:26.637Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133036264Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131712024-04-20T01:14:26.637Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133049674Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131722024-04-20T01:14:26.637Z{"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133069884Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131732024-04-20T01:14:26.637Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133082864Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"":"downstairs","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131742024-04-20T01:14:26.637Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133095824Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131752024-04-20T01:14:26.637Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133114904Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"0","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131762024-04-20T01:14:26.637Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133128564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"1","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131772024-04-20T01:14:26.637Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133154474Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"client":"2","":"downstairs","session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131782024-04-20T01:14:26.637Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133168604Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131792024-04-20T01:14:26.637Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133180994Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131802024-04-20T01:14:26.637Z{"msg":"6b5b25d6-5e10-4e07-8eb9-c769108f4aa5 is now active with session: a1935177-52c7-49db-b68e-561cd7c00c76","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133200634Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131812024-04-20T01:14:26.637Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133213144Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659,"session_id":"a1935177-52c7-49db-b68e-561cd7c00c76"}
131822024-04-20T01:14:26.637Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:25.133225254Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659}
131832024-04-20T01:14:26.638ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131842024-04-20T01:14:26.642ZLoading write count information from file "/tmp/test_repair_verify.data"
131852024-04-20T01:14:26.642ZRun Repair workload
131862024-04-20T01:14:26.642Z01/30 Write block 584 len 20480 data: 12 15 18 16 14
131872024-04-20T01:14:26.642Z02/30 Read block 206 len 24576
131882024-04-20T01:14:26.642Z03/30 Read block 406 len 36864
131892024-04-20T01:14:26.642Z04/30 Flush
131902024-04-20T01:14:26.642Z05/30 Flush
131912024-04-20T01:14:26.642Z06/30 Write block 159 len 40960 data: 9 10 12 10 9 9 10 7 6 7
131922024-04-20T01:14:26.642Z07/30 Read block 286 len 20480
131932024-04-20T01:14:26.642Z08/30 Read block 429 len 20480
131942024-04-20T01:14:26.646Z09/30 Flush
131952024-04-20T01:14:26.646Z10/30 Write block 109 len 4096 data: 8
131962024-04-20T01:14:26.649Z11/30 Write block 555 len 40960 data: 9 7 8 6 6 5 7 8 6 9
131972024-04-20T01:14:26.649Z12/30 Write block 336 len 4096 data: 9
131982024-04-20T01:14:26.649Z13/30 Write block 338 len 8192 data: 11 9
131992024-04-20T01:14:26.649Z14/30 Flush
132002024-04-20T01:14:26.653Z15/30 Write block 104 len 36864 data: 8 7 9 8 9 9 8 8 9
132012024-04-20T01:14:26.653Z16/30 Read block 79 len 12288
132022024-04-20T01:14:26.656Z17/30 Write block 96 len 8192 data: 4 5
132032024-04-20T01:14:26.656Z18/30 Write block 279 len 16384 data: 8 10 10 8
132042024-04-20T01:14:26.656Z19/30 Write block 273 len 36864 data: 12 9 8 6 8 8 9 11 11
132052024-04-20T01:14:26.659Z20/30 Read block 274 len 32768
132062024-04-20T01:14:26.660Z21/30 Read block 258 len 28672
132072024-04-20T01:14:26.660Z22/30 Write block 487 len 20480 data: 9 8 8 9 9
132082024-04-20T01:14:26.660Z23/30 Flush
132092024-04-20T01:14:26.663Z24/30 Write block 566 len 16384 data: 13 11 12 14
132102024-04-20T01:14:26.663Z25/30 Read block 284 len 16384
132112024-04-20T01:14:26.666Z26/30 Write block 391 len 24576 data: 5 5 6 6 6 6
132122024-04-20T01:14:26.666Z27/30 Write block 181 len 32768 data: 16 13 9 7 7 8 8 10
132132024-04-20T01:14:26.666Z28/30 Write block 575 len 28672 data: 5 6 8 8 11 11 11
132142024-04-20T01:14:26.670Z29/30 Read block 25 len 28672
132152024-04-20T01:14:26.670Z30/30 Write block 542 len 12288 data: 12 13 11
132162024-04-20T01:14:26.670Z----------------------------------------------------------------
132172024-04-20T01:14:26.670Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
132182024-04-20T01:14:26.670ZStates: Active Active Active
132192024-04-20T01:14:26.670ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
132202024-04-20T01:14:26.670Z 1 Acked 1000 Write 5 Sent Done Done false
132212024-04-20T01:14:26.670Z 2 Acked 1001 Read 6 Sent Done Done false
132222024-04-20T01:14:26.670Z 3 Acked 1002 Read 9 Sent Done Done false
132232024-04-20T01:14:26.670Z 4 Acked 1003 Flush 0 Sent Done Done false
132242024-04-20T01:14:26.670Z 5 Acked 1004 Flush 0 Sent Done Done false
132252024-04-20T01:14:26.671Z 6 Acked 1005 Write 10 Sent Done Done false
132262024-04-20T01:14:26.671Z 7 Acked 1006 Read 5 Sent Done Done false
132272024-04-20T01:14:26.671Z 8 Acked 1007 Read 5 Sent Done Done false
132282024-04-20T01:14:26.671Z 9 Acked 1008 Flush 0 Sent Done Done false
132292024-04-20T01:14:26.671Z 10 Acked 1009 Write 1 Sent Done Done false
132302024-04-20T01:14:26.671Z 11 Acked 1010 Write 10 Sent Done Done false
132312024-04-20T01:14:26.671Z 12 Acked 1011 Write 1 Sent Done Done false
132322024-04-20T01:14:26.671Z 13 Acked 1012 Write 2 Sent Done Done false
132332024-04-20T01:14:26.671Z 14 Acked 1013 Flush 0 Sent Done Done false
132342024-04-20T01:14:26.671Z 15 Acked 1014 Write 9 Sent Done Done false
132352024-04-20T01:14:26.671Z 16 Acked 1015 Read 3 Sent Done Done false
132362024-04-20T01:14:26.671Z 17 Acked 1016 Write 2 Sent Done Done false
132372024-04-20T01:14:26.671Z 18 Acked 1017 Write 4 Sent Done Done false
132382024-04-20T01:14:26.671Z 19 Acked 1018 Write 9 Sent Done Done false
132392024-04-20T01:14:26.671Z 20 Acked 1019 Read 8 Sent Done Done false
132402024-04-20T01:14:26.671Z 21 Acked 1020 Read 7 Sent Done Done false
132412024-04-20T01:14:26.671Z 22 Acked 1021 Write 5 Sent Done Done false
132422024-04-20T01:14:26.671Z 23 Acked 1022 Flush 0 Sent Done Done false
132432024-04-20T01:14:26.671Z 24 Acked 1023 Write 4 Sent Done Done false
132442024-04-20T01:14:26.671Z 25 Acked 1024 Read 4 Sent Done Done false
132452024-04-20T01:14:26.671Z 26 Acked 1025 Write 6 Sent Done Done false
132462024-04-20T01:14:26.671Z 27 Acked 1026 Write 8 Sent Done Done false
132472024-04-20T01:14:26.672Z 28 Acked 1027 Write 7 Sent Done Done false
132482024-04-20T01:14:26.672Z 29 Acked 1028 Read 7 Sent Done Done false
132492024-04-20T01:14:26.672Z 30 Acked 1029 Write 3 Sent Sent Sent false
132502024-04-20T01:14:26.672Z STATES DS:0 DS:1 DS:2 TOTAL
132512024-04-20T01:14:26.672Z New 0 0 0 0
132522024-04-20T01:14:26.672Z Sent 30 1 1 32
132532024-04-20T01:14:26.672Z Done 0 29 29 58
132542024-04-20T01:14:26.672Z Skipped 0 0 0 0
132552024-04-20T01:14:26.672Z Error 0 0 0 0
132562024-04-20T01:14:26.672ZLast Flush: 0 1022 1022
132572024-04-20T01:14:26.672ZDownstairs last five completed:
132582024-04-20T01:14:26.672ZUpstairs last five completed: 30 29 28 27 26
132592024-04-20T01:14:26.672Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:14:25.169432455Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1659}
132602024-04-20T01:14:26.672ZWrote out file "/tmp/test_repair_verify.data"
132612024-04-20T01:14:26.676Z
132622024-04-20T01:14:26.676ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
132632024-04-20T01:14:26.696ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132642024-04-20T01:14:26.696Z 0 000-019 121 121 121 201 201 201 F F F
132652024-04-20T01:14:26.696Z 1 020-039 121 121 121 201 201 201 F F F
132662024-04-20T01:14:26.696Z 2 040-059 114 114 114 190 190 190 F F F
132672024-04-20T01:14:26.696Z 3 060-079 118 118 118 196 196 196 F F F
132682024-04-20T01:14:26.696Z 4 080-099 112 122 122 188 206 206 F F F <---
132692024-04-20T01:14:26.696Z 5 100-119 121 122 122 201 206 206 F F F <---
132702024-04-20T01:14:26.700Z 6 120-139 121 121 121 201 201 201 F F F
132712024-04-20T01:14:26.700Z 7 140-159 117 122 122 193 204 204 F F F <---
132722024-04-20T01:14:26.700Z 8 160-179 118 122 122 194 204 204 F F F <---
132732024-04-20T01:14:26.700Z 9 180-199 121 121 121 201 201 201 F T T <---
132742024-04-20T01:14:26.700Z 10 200-219 118 118 118 195 195 195 F F F
132752024-04-20T01:14:26.700Z 11 220-239 115 115 115 191 191 191 F F F
132762024-04-20T01:14:26.700Z 12 240-259 120 120 120 198 198 198 F F F
132772024-04-20T01:14:26.701Z 13 260-279 119 122 122 197 206 206 F F F <---
132782024-04-20T01:14:26.701Z 14 280-299 119 122 122 197 206 206 F F F <---
132792024-04-20T01:14:26.701Z 15 300-319 116 116 116 192 192 192 F F F
132802024-04-20T01:14:26.701Z 16 320-339 114 122 122 190 205 205 F F F <---
132812024-04-20T01:14:26.701Z 17 340-359 115 115 115 191 191 191 F F F
132822024-04-20T01:14:26.701Z 18 360-379 121 121 121 201 201 201 F F F
132832024-04-20T01:14:26.701Z 19 380-399 118 118 118 195 195 195 F T T <---
132842024-04-20T01:14:26.701Z 20 400-419 119 119 119 197 197 197 F F F
132852024-04-20T01:14:26.701Z 21 420-439 118 118 118 196 196 196 F F F
132862024-04-20T01:14:26.701Z 22 440-459 118 118 118 196 196 196 F F F
132872024-04-20T01:14:26.701Z 23 460-479 118 118 118 194 194 194 F F F
132882024-04-20T01:14:26.701Z 24 480-499 116 122 122 192 206 206 F F F <---
132892024-04-20T01:14:26.701Z 25 500-519 119 119 119 197 197 197 F F F
132902024-04-20T01:14:26.701Z 26 520-539 121 121 121 201 201 201 F F F
132912024-04-20T01:14:26.701Z 27 540-559 121 122 122 201 205 205 F T T <---
132922024-04-20T01:14:26.701Z 28 560-579 119 122 122 197 205 205 F T T <---
132932024-04-20T01:14:26.701Z 29 580-599 121 122 122 201 202 202 F T T <---
132942024-04-20T01:14:26.701ZMax gen: 122, Max flush: 206
132952024-04-20T01:14:26.701ZError: Difference in extent metadata found!
132962024-04-20T01:14:26.702ZA Difference in extent metadata is expected here
132972024-04-20T01:14:26.702ZOn loop 61
132982024-04-20T01:14:26.702Z
132992024-04-20T01:14:26.702ZVerifying data now
133002024-04-20T01:14:31.291ZNo change after verify
133012024-04-20T01:14:31.291ZLoop: 61 Downstairs dump after verify (and repair):
133022024-04-20T01:14:31.305ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
133032024-04-20T01:14:31.305Z 0 000-019 121 121 121 201 201 201 F F F
133042024-04-20T01:14:31.305Z 1 020-039 121 121 121 201 201 201 F F F
133052024-04-20T01:14:31.305Z 2 040-059 114 114 114 190 190 190 F F F
133062024-04-20T01:14:31.305Z 3 060-079 118 118 118 196 196 196 F F F
133072024-04-20T01:14:31.305Z 4 080-099 122 122 122 206 206 206 F F F
133082024-04-20T01:14:31.305Z 5 100-119 122 122 122 206 206 206 F F F
133092024-04-20T01:14:31.305Z 6 120-139 121 121 121 201 201 201 F F F
133102024-04-20T01:14:31.305Z 7 140-159 122 122 122 204 204 204 F F F
133112024-04-20T01:14:31.305Z 8 160-179 122 122 122 204 204 204 F F F
133122024-04-20T01:14:31.309Z 9 180-199 123 123 123 207 207 207 F F F
133132024-04-20T01:14:31.309Z 10 200-219 118 118 118 195 195 195 F F F
133142024-04-20T01:14:31.309Z 11 220-239 115 115 115 191 191 191 F F F
133152024-04-20T01:14:31.309Z 12 240-259 120 120 120 198 198 198 F F F
133162024-04-20T01:14:31.310Z 13 260-279 122 122 122 206 206 206 F F F
133172024-04-20T01:14:31.310Z 14 280-299 122 122 122 206 206 206 F F F
133182024-04-20T01:14:31.310Z 15 300-319 116 116 116 192 192 192 F F F
133192024-04-20T01:14:31.310Z 16 320-339 122 122 122 205 205 205 F F F
133202024-04-20T01:14:31.310Z 17 340-359 115 115 115 191 191 191 F F F
133212024-04-20T01:14:31.310Z 18 360-379 121 121 121 201 201 201 F F F
133222024-04-20T01:14:31.310Z 19 380-399 123 123 123 207 207 207 F F F
133232024-04-20T01:14:31.310Z 20 400-419 119 119 119 197 197 197 F F F
133242024-04-20T01:14:31.310Z 21 420-439 118 118 118 196 196 196 F F F
133252024-04-20T01:14:31.310Z 22 440-459 118 118 118 196 196 196 F F F
133262024-04-20T01:14:31.310Z 23 460-479 118 118 118 194 194 194 F F F
133272024-04-20T01:14:31.310Z 24 480-499 122 122 122 206 206 206 F F F
133282024-04-20T01:14:31.310Z 25 500-519 119 119 119 197 197 197 F F F
133292024-04-20T01:14:31.310Z 26 520-539 121 121 121 201 201 201 F F F
133302024-04-20T01:14:31.310Z 27 540-559 123 123 123 207 207 207 F F F
133312024-04-20T01:14:31.310Z 28 560-579 123 123 123 207 207 207 F F F
133322024-04-20T01:14:31.310Z 29 580-599 123 123 123 207 207 207 F F F
133332024-04-20T01:14:31.310ZMax gen: 123, Max flush: 207
133342024-04-20T01:14:31.310Z
133352024-04-20T01:14:31.310ZBegin loop 62
133362024-04-20T01:14:31.310ZDownstairs to restart: 0
133372024-04-20T01:14:31.326Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827339343Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667}
133382024-04-20T01:14:31.326Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827592274Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667}
133392024-04-20T01:14:31.332Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827626264Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667}
133402024-04-20T01:14:31.333Z{"msg":"Crucible a4ca39a8-e6d4-406a-bb8d-c1ad57a51722 has session id: a676fa96-6274-4629-a311-7fb1b0f32562"Crucible runtime is spawned
133412024-04-20T01:14:31.333Z,"v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827660954Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133422024-04-20T01:14:31.333Z{"msg":"Upstairs opts: Upstairs UUID: a4ca39a8-e6d4-406a-bb8d-c1ad57a51722, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827719954Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133432024-04-20T01:14:31.333Z{"msg":"Crucible stats registered with UUID: a4ca39a8-e6d4-406a-bb8d-c1ad57a51722","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827757014Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133442024-04-20T01:14:31.333Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827771944Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667}
133452024-04-20T01:14:31.333Z{"msg":"a4ca39a8-e6d4-406a-bb8d-c1ad57a51722 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827788774Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133462024-04-20T01:14:31.333Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827821074Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"0","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133472024-04-20T01:14:31.333Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827836424Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"0","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133482024-04-20T01:14:31.333Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827850364Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"1","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133492024-04-20T01:14:31.333Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827872264Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"1","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133502024-04-20T01:14:31.333Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.827896414Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"2","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133512024-04-20T01:14:31.333Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.828044975Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"2","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133522024-04-20T01:14:31.333Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.828085055Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133532024-04-20T01:14:31.333Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.828131505Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133542024-04-20T01:14:31.333Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.828157685Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133552024-04-20T01:14:31.333Z{"msg":"ds_connection connected from Ok(127.0.0.1:41402)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.828173185Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133562024-04-20T01:14:31.333Z{"msg":"ds_connection connected from Ok(127.0.0.1:59438)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.828187325Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133572024-04-20T01:14:31.333Z{"msg":"ds_connection connected from Ok(127.0.0.1:62398)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.828202725Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133582024-04-20T01:14:31.333Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.828910257Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"1","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133592024-04-20T01:14:31.333Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.828967027Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"1","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133602024-04-20T01:14:31.333Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.829005487Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"2","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133612024-04-20T01:14:31.333Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.829023507Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"2","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133622024-04-20T01:14:31.334Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.829392828Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"2","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133632024-04-20T01:14:31.334Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.829525128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"1","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133642024-04-20T01:14:31.334Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.829560888Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"0","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133652024-04-20T01:14:31.334Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.829576478Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"0","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133662024-04-20T01:14:31.334Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.829814019Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"2","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133672024-04-20T01:14:31.334Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.829856679Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"2","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133682024-04-20T01:14:31.334Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.829884259Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133692024-04-20T01:14:31.334Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830066849Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"1","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133702024-04-20T01:14:31.334Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830098719Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"1","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133712024-04-20T01:14:31.334Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830113419Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133722024-04-20T01:14:31.334Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830126639Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"0","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133732024-04-20T01:14:31.334Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.83055188Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"0","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133742024-04-20T01:14:31.334Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.83057804Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"0","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133752024-04-20T01:14:31.334Z{"msg":"[0]R flush_numbers[0..12]: [201, 201, 190, 196, 206, 206, 201, 204, 204, 207, 195, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.83059291Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133762024-04-20T01:14:31.334Z{"msg":"[0]R generation[0..12]: [121, 121, 114, 118, 122, 122, 121, 122, 122, 123, 118, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.83061401Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133772024-04-20T01:14:31.334Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.83062844Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133782024-04-20T01:14:31.334Z{"msg":"[1]R flush_numbers[0..12]: [201, 201, 190, 196, 206, 206, 201, 204, 204, 207, 195, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830648951Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133792024-04-20T01:14:31.334Z{"msg":"[1]R generation[0..12]: [121, 121, 114, 118, 122, 122, 121, 122, 122, 123, 118, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.83066308Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133802024-04-20T01:14:31.334Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.8306839Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133812024-04-20T01:14:31.334Z{"msg":"[2]R flush_numbers[0..12]: [201, 201, 190, 196, 206, 206, 201, 204, 204, 207, 195, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830710021Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133822024-04-20T01:14:31.334Z{"msg":"[2]R generation[0..12]: [121, 121, 114, 118, 122, 122, 121, 122, 122, 123, 118, 115]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830724091Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133832024-04-20T01:14:31.334Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830738201Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133842024-04-20T01:14:31.334ZWait for a query_work_queue command to finish before sending IO
133852024-04-20T01:14:31.334Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830763151Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133862024-04-20T01:14:31.335Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830777041Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133872024-04-20T01:14:31.335Z{"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830790671Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133882024-04-20T01:14:31.335Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830803981Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"":"downstairs","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133892024-04-20T01:14:31.335Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830830801Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133902024-04-20T01:14:31.335Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830861271Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"0","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133912024-04-20T01:14:31.335Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830886001Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"1","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133922024-04-20T01:14:31.335Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830919281Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"client":"2","":"downstairs","session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133932024-04-20T01:14:31.335Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830944731Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133942024-04-20T01:14:31.335Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830974391Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133952024-04-20T01:14:31.335Z{"msg":"a4ca39a8-e6d4-406a-bb8d-c1ad57a51722 is now active with session: a676fa96-6274-4629-a311-7fb1b0f32562","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.830999201Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133962024-04-20T01:14:31.335Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.831023671Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667,"session_id":"a676fa96-6274-4629-a311-7fb1b0f32562"}
133972024-04-20T01:14:31.335Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:29.831040051Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667}
133982024-04-20T01:14:31.335ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133992024-04-20T01:14:31.339ZLoading write count information from file "/tmp/test_repair_verify.data"
134002024-04-20T01:14:31.339ZRun Repair workload
134012024-04-20T01:14:31.339Z01/30 Write block 226 len 4096 data: 16
134022024-04-20T01:14:31.339Z02/30 Write block 132 len 28672 data: 14 12 10 11 10 10 12
134032024-04-20T01:14:31.339Z03/30 Write block 321 len 32768 data: 13 13 13 11 10 11 9 10
134042024-04-20T01:14:31.339Z04/30 Read block 366 len 24576
134052024-04-20T01:14:31.339Z05/30 Flush
134062024-04-20T01:14:31.342Z06/30 Write block 270 len 40960 data: 8 9 10 13 10 9 7 9 9 10
134072024-04-20T01:14:31.342Z07/30 Write block 105 len 24576 data: 8 10 9 10 10 9
134082024-04-20T01:14:31.342Z08/30 Flush
134092024-04-20T01:14:31.346Z09/30 Write block 543 len 12288 data: 14 12 10
134102024-04-20T01:14:31.346Z10/30 Flush
134112024-04-20T01:14:31.346Z11/30 Flush
134122024-04-20T01:14:31.349Z12/30 Write block 381 len 20480 data: 11 10 9 7 4
134132024-04-20T01:14:31.349Z13/30 Write block 315 len 8192 data: 5 7
134142024-04-20T01:14:31.350Z14/30 Write block 88 len 16384 data: 10 11 12 10
134152024-04-20T01:14:31.350Z15/30 Read block 326 len 24576
134162024-04-20T01:14:31.350Z16/30 Read block 431 len 8192
134172024-04-20T01:14:31.353Z17/30 Write block 143 len 36864 data: 13 13 13 12 10 9 7 6 8
134182024-04-20T01:14:31.353Z18/30 Flush
134192024-04-20T01:14:31.356Z19/30 Write block 207 len 28672 data: 9 9 10 11 11 11 12
134202024-04-20T01:14:31.357Z20/30 Write block 425 len 8192 data: 9 8
134212024-04-20T01:14:31.357Z21/30 Read block 155 len 24576
134222024-04-20T01:14:31.360Z22/30 Read block 75 len 24576
134232024-04-20T01:14:31.360Z23/30 Flush
134242024-04-20T01:14:31.363Z24/30 Flush
134252024-04-20T01:14:31.363Z25/30 Write block 42 len 28672 data: 6 4 5 5 6 6 7
134262024-04-20T01:14:31.363Z26/30 Flush
134272024-04-20T01:14:31.367Z27/30 Write block 78 len 40960 data: 9 9 8 8 7 9 8 8 10 10
134282024-04-20T01:14:31.367Z28/30 Write block 362 len 36864 data: 10 10 8 8 10 13 10 10 12
134292024-04-20T01:14:31.371Z29/30 Read block 424 len 40960
134302024-04-20T01:14:31.371Z30/30 Read block 510 len 40960
134312024-04-20T01:14:31.371Z----------------------------------------------------------------
134322024-04-20T01:14:31.371Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
134332024-04-20T01:14:31.371ZStates: Active Active Active
134342024-04-20T01:14:31.371ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
134352024-04-20T01:14:31.371Z 1 Acked 1000 Write 1 Done Done Done false
134362024-04-20T01:14:31.371Z 2 Acked 1001 Write 7 Done Done Done false
134372024-04-20T01:14:31.371Z 3 Acked 1002 Write 8 Sent Done Done false
134382024-04-20T01:14:31.371Z 4 Acked 1003 Read 6 Sent Done Done false
134392024-04-20T01:14:31.371Z 5 Acked 1004 Flush 0 Sent Done Done false
134402024-04-20T01:14:31.371Z 6 Acked 1005 Write 10 Sent Done Done false
134412024-04-20T01:14:31.371Z 7 Acked 1006 Write 6 Sent Done Done false
134422024-04-20T01:14:31.371Z 8 Acked 1007 Flush 0 Sent Done Done false
134432024-04-20T01:14:31.371Z 9 Acked 1008 Write 3 Sent Done Done false
134442024-04-20T01:14:31.371Z 10 Acked 1009 Flush 0 Sent Done Done false
134452024-04-20T01:14:31.371Z 11 Acked 1010 Flush 0 Sent Done Done false
134462024-04-20T01:14:31.372Z 12 Acked 1011 Write 5 Sent Done Done false
134472024-04-20T01:14:31.372Z 13 Acked 1012 Write 2 Sent Done Done false
134482024-04-20T01:14:31.372Z 14 Acked 1013 Write 4 Sent Done Done false
134492024-04-20T01:14:31.372Z 15 Acked 1014 Read 6 Sent Done Done false
134502024-04-20T01:14:31.372Z 16 Acked 1015 Read 2 Sent Done Done false
134512024-04-20T01:14:31.372Z 17 Acked 1016 Write 9 Sent Done Done false
134522024-04-20T01:14:31.372Z 18 Acked 1017 Flush 0 Sent Done Done false
134532024-04-20T01:14:31.372Z 19 Acked 1018 Write 7 Sent Done Done false
134542024-04-20T01:14:31.372Z 20 Acked 1019 Write 2 Sent Done Done false
134552024-04-20T01:14:31.372Z 21 Acked 1020 Read 6 Sent Done Done false
134562024-04-20T01:14:31.372Z 22 Acked 1021 Read 6 Sent Done Done false
134572024-04-20T01:14:31.372Z 23 Acked 1022 Flush 0 Sent Done Done false
134582024-04-20T01:14:31.372Z 24 Acked 1023 Flush 0 Sent Done Done false
134592024-04-20T01:14:31.372Z 25 Acked 1024 Write 7 Sent Done Done false
134602024-04-20T01:14:31.372Z 26 Acked 1025 Flush 0 Sent Done Done false
134612024-04-20T01:14:31.372Z 27 Acked 1026 Write 10 Sent Done Done false
134622024-04-20T01:14:31.372Z 28 Acked 1027 Write 9 Sent Done Done false
134632024-04-20T01:14:31.372Z 29 Acked 1028 Read 10 Sent Done Done false
134642024-04-20T01:14:31.372Z 30 Acked 1029 Read 10 Sent Done Done false
134652024-04-20T01:14:31.372Z STATES DS:0 DS:1 DS:2 TOTAL
134662024-04-20T01:14:31.372Z New 0 0 0 0
134672024-04-20T01:14:31.372Z Sent 28 0 0 28
134682024-04-20T01:14:31.372Z Done 2 30 30 62
134692024-04-20T01:14:31.372Z Skipped 0 0 0 0
134702024-04-20T01:14:31.373Z Error 0 0 0 0
134712024-04-20T01:14:31.373ZLast Flush: 0 1025 1025
134722024-04-20T01:14:31.373ZDownstairs last five completed:
134732024-04-20T01:14:31.373ZUpstairs last five completed: 30 29 28 27 26
134742024-04-20T01:14:31.373Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:14:29.870325499Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1667}
134752024-04-20T01:14:31.373ZWrote out file "/tmp/test_repair_verify.data"
134762024-04-20T01:14:31.376Z
134772024-04-20T01:14:31.377ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134782024-04-20T01:14:31.392ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134792024-04-20T01:14:31.392Z 0 000-019 121 121 121 201 201 201 F F F
134802024-04-20T01:14:31.392Z 1 020-039 121 121 121 201 201 201 F F F
134812024-04-20T01:14:31.392Z 2 040-059 114 124 124 190 215 215 F F F <---
134822024-04-20T01:14:31.392Z 3 060-079 118 118 118 196 196 196 F T T <---
134832024-04-20T01:14:31.392Z 4 080-099 122 124 124 206 212 212 F T T <---
134842024-04-20T01:14:31.396Z 5 100-119 122 124 124 206 209 209 F F F <---
134852024-04-20T01:14:31.396Z 6 120-139 121 124 124 201 208 208 T F F <---
134862024-04-20T01:14:31.396Z 7 140-159 122 124 124 204 212 212 F F F <---
134872024-04-20T01:14:31.396Z 8 160-179 122 122 122 204 204 204 F F F
134882024-04-20T01:14:31.396Z 9 180-199 123 123 123 207 207 207 F F F
134892024-04-20T01:14:31.396Z 10 200-219 118 124 124 195 213 213 F F F <---
134902024-04-20T01:14:31.396Z 11 220-239 115 124 124 191 208 208 T F F <---
134912024-04-20T01:14:31.396Z 12 240-259 120 120 120 198 198 198 F F F
134922024-04-20T01:14:31.396Z 13 260-279 122 124 124 206 209 209 F F F <---
134932024-04-20T01:14:31.396Z 14 280-299 122 122 122 206 206 206 F F F
134942024-04-20T01:14:31.396Z 15 300-319 116 124 124 192 212 212 F F F <---
134952024-04-20T01:14:31.396Z 16 320-339 122 124 124 205 208 208 F F F <---
134962024-04-20T01:14:31.396Z 17 340-359 115 115 115 191 191 191 F F F
134972024-04-20T01:14:31.396Z 18 360-379 121 121 121 201 201 201 F T T <---
134982024-04-20T01:14:31.396Z 19 380-399 123 124 124 207 212 212 F F F <---
134992024-04-20T01:14:31.396Z 20 400-419 119 119 119 197 197 197 F F F
135002024-04-20T01:14:31.396Z 21 420-439 118 124 124 196 213 213 F F F <---
135012024-04-20T01:14:31.397Z 22 440-459 118 118 118 196 196 196 F F F
135022024-04-20T01:14:31.397Z 23 460-479 118 118 118 194 194 194 F F F
135032024-04-20T01:14:31.397Z 24 480-499 122 122 122 206 206 206 F F F
135042024-04-20T01:14:31.397Z 25 500-519 119 119 119 197 197 197 F F F
135052024-04-20T01:14:31.397Z 26 520-539 121 121 121 201 201 201 F F F
135062024-04-20T01:14:31.397Z 27 540-559 123 124 124 207 210 210 F F F <---
135072024-04-20T01:14:31.397Z 28 560-579 123 123 123 207 207 207 F F F
135082024-04-20T01:14:31.397Z 29 580-599 123 123 123 207 207 207 F F F
135092024-04-20T01:14:31.397ZMax gen: 124, Max flush: 215
135102024-04-20T01:14:31.397ZError: Difference in extent metadata found!
135112024-04-20T01:14:31.397ZA Difference in extent metadata is expected here
135122024-04-20T01:14:31.397ZOn loop 62
135132024-04-20T01:14:31.397Z
135142024-04-20T01:14:31.401ZVerifying data now
135152024-04-20T01:14:46.061ZNo change after verify
135162024-04-20T01:14:46.061ZLoop: 62 Downstairs dump after verify (and repair):
135172024-04-20T01:14:46.074ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135182024-04-20T01:14:46.074Z 0 000-019 121 121 121 201 201 201 F F F
135192024-04-20T01:14:46.074Z 1 020-039 121 121 121 201 201 201 F F F
135202024-04-20T01:14:46.074Z 2 040-059 124 124 124 215 215 215 F F F
135212024-04-20T01:14:46.074Z 3 060-079 125 125 125 216 216 216 F F F
135222024-04-20T01:14:46.074Z 4 080-099 125 125 125 216 216 216 F F F
135232024-04-20T01:14:46.074Z 5 100-119 124 124 124 209 209 209 F F F
135242024-04-20T01:14:46.074Z 6 120-139 124 124 124 208 208 208 F F F
135252024-04-20T01:14:46.074Z 7 140-159 124 124 124 212 212 212 F F F
135262024-04-20T01:14:46.074Z 8 160-179 122 122 122 204 204 204 F F F
135272024-04-20T01:14:46.078Z 9 180-199 123 123 123 207 207 207 F F F
135282024-04-20T01:14:46.078Z 10 200-219 124 124 124 213 213 213 F F F
135292024-04-20T01:14:46.078Z 11 220-239 124 124 124 208 208 208 F F F
135302024-04-20T01:14:46.078Z 12 240-259 120 120 120 198 198 198 F F F
135312024-04-20T01:14:46.078Z 13 260-279 124 124 124 209 209 209 F F F
135322024-04-20T01:14:46.078Z 14 280-299 122 122 122 206 206 206 F F F
135332024-04-20T01:14:46.078Z 15 300-319 124 124 124 212 212 212 F F F
135342024-04-20T01:14:46.078Z 16 320-339 124 124 124 208 208 208 F F F
135352024-04-20T01:14:46.078Z 17 340-359 115 115 115 191 191 191 F F F
135362024-04-20T01:14:46.078Z 18 360-379 125 125 125 216 216 216 F F F
135372024-04-20T01:14:46.078Z 19 380-399 124 124 124 212 212 212 F F F
135382024-04-20T01:14:46.078Z 20 400-419 119 119 119 197 197 197 F F F
135392024-04-20T01:14:46.078Z 21 420-439 124 124 124 213 213 213 F F F
135402024-04-20T01:14:46.079Z 22 440-459 118 118 118 196 196 196 F F F
135412024-04-20T01:14:46.079Z 23 460-479 118 118 118 194 194 194 F F F
135422024-04-20T01:14:46.079Z 24 480-499 122 122 122 206 206 206 F F F
135432024-04-20T01:14:46.079Z 25 500-519 119 119 119 197 197 197 F F F
135442024-04-20T01:14:46.079Z 26 520-539 121 121 121 201 201 201 F F F
135452024-04-20T01:14:46.079Z 27 540-559 124 124 124 210 210 210 F F F
135462024-04-20T01:14:46.079Z 28 560-579 123 123 123 207 207 207 F F F
135472024-04-20T01:14:46.079Z 29 580-599 123 123 123 207 207 207 F F F
135482024-04-20T01:14:46.079ZMax gen: 125, Max flush: 216
135492024-04-20T01:14:46.079Z
135502024-04-20T01:14:46.079ZBegin loop 63
135512024-04-20T01:14:46.079ZDownstairs to restart: 1
135522024-04-20T01:14:46.097Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.5969837Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675}
135532024-04-20T01:14:46.097Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597280021Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675}
135542024-04-20T01:14:46.102Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597317901Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675}
135552024-04-20T01:14:46.102Z{"msg":"Crucible d00b4272-a345-434d-bb51-ad66ff725bc4 has session id: 03ce29a0-682e-41a6-aab6-0c72e419fe24","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597341101Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135562024-04-20T01:14:46.102ZCrucible runtime is spawned
135572024-04-20T01:14:46.102Z{"msg":"Upstairs opts: Upstairs UUID: d00b4272-a345-434d-bb51-ad66ff725bc4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597396721Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135582024-04-20T01:14:46.102Z{"msg":"Crucible stats registered with UUID: d00b4272-a345-434d-bb51-ad66ff725bc4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597429671Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135592024-04-20T01:14:46.102Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597452921Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675}
135602024-04-20T01:14:46.102Z{"msg":"d00b4272-a345-434d-bb51-ad66ff725bc4 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597525551Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135612024-04-20T01:14:46.103Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597554782Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"0","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135622024-04-20T01:14:46.103Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597579972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"0","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135632024-04-20T01:14:46.103Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597600782Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"1","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135642024-04-20T01:14:46.103Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597625842Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"1","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135652024-04-20T01:14:46.103Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597656582Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"2","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135662024-04-20T01:14:46.103Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597681272Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"2","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135672024-04-20T01:14:46.103Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597704322Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135682024-04-20T01:14:46.103Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597727692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135692024-04-20T01:14:46.103Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597760332Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135702024-04-20T01:14:46.103Z{"msg":"ds_connection connected from Ok(127.0.0.1:40795)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597885032Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135712024-04-20T01:14:46.103Z{"msg":"ds_connection connected from Ok(127.0.0.1:40594)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597931602Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135722024-04-20T01:14:46.103Z{"msg":"ds_connection connected from Ok(127.0.0.1:56137)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.597948992Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135732024-04-20T01:14:46.103Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.598686984Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"0","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135742024-04-20T01:14:46.103Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.598750604Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"0","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135752024-04-20T01:14:46.103Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.598820254Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"2","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135762024-04-20T01:14:46.103Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.598848165Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"2","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135772024-04-20T01:14:46.103Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.598872205Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"1","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135782024-04-20T01:14:46.103Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.598896685Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"1","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135792024-04-20T01:14:46.103Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.599291855Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"0","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135802024-04-20T01:14:46.103Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.599347675Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"2","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135812024-04-20T01:14:46.103Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.599382046Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"1","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135822024-04-20T01:14:46.103Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.599834867Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"0","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135832024-04-20T01:14:46.103Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.599875917Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"0","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135842024-04-20T01:14:46.104Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.599901857Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135852024-04-20T01:14:46.104Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.599930847Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"2","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135862024-04-20T01:14:46.104Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.599955557Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"2","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135872024-04-20T01:14:46.104Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.599987297Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135882024-04-20T01:14:46.104Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600015887Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"1","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135892024-04-20T01:14:46.104Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
135902024-04-20T01:14:46.104Z,"time":"2024-04-20T01:14:44.600039047Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"1","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135912024-04-20T01:14:46.104Z{"msg":"[0]R flush_numbers[0..12]: [201, 201, 215, 216, 216, 209, 208, 212, 204, 207, 213, 208]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600070637Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135922024-04-20T01:14:46.104Z{"msg":"[0]R generation[0..12]: [121, 121, 124, 125, 125, 124, 124, 124, 122, 123, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600104767Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135932024-04-20T01:14:46.104Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600131667Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135942024-04-20T01:14:46.104Z{"msg":"[1]R flush_numbers[0..12]: [201, 201, 215, 216, 216, 209, 208, 212, 204, 207, 213, 208]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600163347Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135952024-04-20T01:14:46.104Z{"msg":"[1]R generation[0..12]: [121, 121, 124, 125, 125, 124, 124, 124, 122, 123, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600188388Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135962024-04-20T01:14:46.104Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600218468Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135972024-04-20T01:14:46.104Z{"msg":"[2]R flush_numbers[0..12]: [201, 201, 215, 216, 216, 209, 208, 212, 204, 207, 213, 208]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600244108Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135982024-04-20T01:14:46.104Z{"msg":"[2]R generation[0..12]: [121, 121, 124, 125, 125, 124, 124, 124, 122, 123, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600276428Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
135992024-04-20T01:14:46.104Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600310148Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136002024-04-20T01:14:46.104Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600340228Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136012024-04-20T01:14:46.104Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600363788Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136022024-04-20T01:14:46.104Z{"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600389348Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136032024-04-20T01:14:46.104Z{"msg":"All extents match","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
136042024-04-20T01:14:46.104Z,"time":"2024-04-20T01:14:44.600411428Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"":"downstairs","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136052024-04-20T01:14:46.104Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600442488Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136062024-04-20T01:14:46.104Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600472098Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"0","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136072024-04-20T01:14:46.105Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600496028Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"1","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136082024-04-20T01:14:46.105Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600523628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"client":"2","":"downstairs","session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136092024-04-20T01:14:46.105Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600547028Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136102024-04-20T01:14:46.105Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600568958Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136112024-04-20T01:14:46.105Z{"msg":"d00b4272-a345-434d-bb51-ad66ff725bc4 is now active with session: 03ce29a0-682e-41a6-aab6-0c72e419fe24","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600597518Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136122024-04-20T01:14:46.105Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600618878Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675,"session_id":"03ce29a0-682e-41a6-aab6-0c72e419fe24"}
136132024-04-20T01:14:46.105Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:44.600644949Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675}
136142024-04-20T01:14:46.109ZLoading write count information from file "/tmp/test_repair_verify.data"
136152024-04-20T01:14:46.109ZRun Repair workload
136162024-04-20T01:14:46.109Z01/30 Write block 45 len 8192 data: 6 7
136172024-04-20T01:14:46.109Z02/30 Read block 513 len 12288
136182024-04-20T01:14:46.109Z03/30 Write block 6 len 4096 data: 7
136192024-04-20T01:14:46.109Z04/30 Write block 338 len 36864 data: 12 10 9 9 8 9 8 11 8
136202024-04-20T01:14:46.109Z05/30 Read block 426 len 20480
136212024-04-20T01:14:46.109Z06/30 Read block 366 len 4096
136222024-04-20T01:14:46.109Z07/30 Read block 206 len 28672
136232024-04-20T01:14:46.109Z08/30 Read block 135 len 12288
136242024-04-20T01:14:46.109Z09/30 Read block 424 len 16384
136252024-04-20T01:14:46.109Z10/30 Write block 408 len 24576 data: 11 12 12 14 13 14
136262024-04-20T01:14:46.113Z11/30 Write block 521 len 8192 data: 14 13
136272024-04-20T01:14:46.113Z12/30 Read block 132 len 16384
136282024-04-20T01:14:46.113Z13/30 Write block 286 len 16384 data: 6 8 9 10
136292024-04-20T01:14:46.113Z14/30 Read block 391 len 32768
136302024-04-20T01:14:46.113Z15/30 Read block 427 len 36864
136312024-04-20T01:14:46.116Z16/30 Write block 238 len 16384 data: 12 12 9 8
136322024-04-20T01:14:46.116Z17/30 Write block 86 len 12288 data: 11 11 11
136332024-04-20T01:14:46.117Z18/30 Read block 515 len 24576
136342024-04-20T01:14:46.117Z19/30 Write block 462 len 24576 data: 13 14 11 12 15 15
136352024-04-20T01:14:46.120Z20/30 Write block 367 len 36864 data: 14 11 11 13 16 15 16 16 15
136362024-04-20T01:14:46.120Z21/30 Read block 79 len 20480
136372024-04-20T01:14:46.120Z22/30 Write block 143 len 12288 data: 14 14 14
136382024-04-20T01:14:46.120Z23/30 Write block 564 len 32768 data: 10 12 14 12 13 15 13 12
136392024-04-20T01:14:46.120Z24/30 Read block 58 len 40960
136402024-04-20T01:14:46.123Z25/30 Read block 59 len 12288
136412024-04-20T01:14:46.123Z26/30 Write block 281 len 40960 data: 12 9 6 7 7 7 9 10 11 8
136422024-04-20T01:14:46.123Z27/30 Read block 387 len 36864
136432024-04-20T01:14:46.127Z28/30 Write block 165 len 40960 data: 11 8 7 8 5 8 9 10 13 12
136442024-04-20T01:14:46.127Z29/30 Read block 411 len 8192
136452024-04-20T01:14:46.127Z30/30 Read block 270 len 28672
136462024-04-20T01:14:46.131Z----------------------------------------------------------------
136472024-04-20T01:14:46.131Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
136482024-04-20T01:14:46.131ZStates: Active Active Active
136492024-04-20T01:14:46.131ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
136502024-04-20T01:14:46.131Z 1 Acked 1000 Write 2 Done Done Done false
136512024-04-20T01:14:46.131Z 2 Acked 1001 Read 3 Done Done Done false
136522024-04-20T01:14:46.131Z 3 Acked 1002 Write 1 Done Done Done false
136532024-04-20T01:14:46.131Z 4 Acked 1003 Write 9 Done Done Done false
136542024-04-20T01:14:46.131Z 5 Acked 1004 Read 5 Done Sent Done false
136552024-04-20T01:14:46.131Z 6 Acked 1005 Read 1 Done Sent Done false
136562024-04-20T01:14:46.131Z 7 Acked 1006 Read 7 Done Sent Done false
136572024-04-20T01:14:46.131Z 8 Acked 1007 Read 3 Done Sent Done false
136582024-04-20T01:14:46.131Z 9 Acked 1008 Read 4 Done Sent Done false
136592024-04-20T01:14:46.131Z 10 Acked 1009 Write 6 Done Sent Done false
136602024-04-20T01:14:46.131Z 11 Acked 1010 Write 2 Done Sent Done false
136612024-04-20T01:14:46.131Z 12 Acked 1011 Read 4 Done Sent Done false
136622024-04-20T01:14:46.131Z 13 Acked 1012 Write 4 Done Sent Done false
136632024-04-20T01:14:46.132Z 14 Acked 1013 Read 8 Done Sent Done false
136642024-04-20T01:14:46.132Z 15 Acked 1014 Read 9 Done Sent Done false
136652024-04-20T01:14:46.132Z 16 Acked 1015 Write 4 Done Sent Done false
136662024-04-20T01:14:46.132Z 17 Acked 1016 Write 3 Done Sent Done false
136672024-04-20T01:14:46.132Z 18 Acked 1017 Read 6 Done Sent Done false
136682024-04-20T01:14:46.132Z 19 Acked 1018 Write 6 Done Sent Done false
136692024-04-20T01:14:46.132Z 20 Acked 1019 Write 9 Done Sent Done false
136702024-04-20T01:14:46.132Z 21 Acked 1020 Read 5 Done Sent Done false
136712024-04-20T01:14:46.132Z 22 Acked 1021 Write 3 Done Sent Done false
136722024-04-20T01:14:46.132Z 23 Acked 1022 Write 8 Done Sent Done false
136732024-04-20T01:14:46.132Z 24 Acked 1023 Read 10 Done Sent Done false
136742024-04-20T01:14:46.132Z 25 Acked 1024 Read 3 Done Sent Done false
136752024-04-20T01:14:46.132Z 26 Acked 1025 Write 10 Done Sent Done false
136762024-04-20T01:14:46.132Z 27 Acked 1026 Read 9 Done Sent Done false
136772024-04-20T01:14:46.132Z 28 Acked 1027 Write 10 Done Sent Done false
136782024-04-20T01:14:46.132Z 29 Acked 1028 Read 2 Done Sent Done false
136792024-04-20T01:14:46.132Z 30 Acked 1029 Read 7 Done Sent Done false
136802024-04-20T01:14:46.132Z STATES DS:0 DS:1 DS:2 TOTAL
136812024-04-20T01:14:46.132Z New 0 0 0 0
136822024-04-20T01:14:46.132Z Sent 0 26 0 26
136832024-04-20T01:14:46.132Z Done 30 4 30 64
136842024-04-20T01:14:46.132Z Skipped 0 0 0 0
136852024-04-20T01:14:46.132Z Error 0 0 0 0
136862024-04-20T01:14:46.132ZLast Flush: 0 0 0
136872024-04-20T01:14:46.133ZDownstairs last five completed:
136882024-04-20T01:14:46.133ZUpstairs last five completed: 30 29 28 27 26
136892024-04-20T01:14:46.133Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:14:44.627984819Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1675}
136902024-04-20T01:14:46.133ZWrote out file "/tmp/test_repair_verify.data"
136912024-04-20T01:14:46.136Z
136922024-04-20T01:14:46.136ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136932024-04-20T01:14:46.154ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136942024-04-20T01:14:46.154Z 0 000-019 121 121 121 201 201 201 T T T
136952024-04-20T01:14:46.154Z 1 020-039 121 121 121 201 201 201 F F F
136962024-04-20T01:14:46.154Z 2 040-059 124 124 124 215 215 215 T T T
136972024-04-20T01:14:46.154Z 3 060-079 125 125 125 216 216 216 F F F
136982024-04-20T01:14:46.154Z 4 080-099 125 125 125 216 216 216 T F T <---
136992024-04-20T01:14:46.155Z 5 100-119 124 124 124 209 209 209 F F F
137002024-04-20T01:14:46.155Z 6 120-139 124 124 124 208 208 208 F F F
137012024-04-20T01:14:46.155Z 7 140-159 124 124 124 212 212 212 T F T <---
137022024-04-20T01:14:46.158Z 8 160-179 122 122 122 204 204 204 T F T <---
137032024-04-20T01:14:46.158Z 9 180-199 123 123 123 207 207 207 F F F
137042024-04-20T01:14:46.158Z 10 200-219 124 124 124 213 213 213 F F F
137052024-04-20T01:14:46.159Z 11 220-239 124 124 124 208 208 208 T F T <---
137062024-04-20T01:14:46.159Z 12 240-259 120 120 120 198 198 198 T F T <---
137072024-04-20T01:14:46.159Z 13 260-279 124 124 124 209 209 209 F F F
137082024-04-20T01:14:46.159Z 14 280-299 122 122 122 206 206 206 T F T <---
137092024-04-20T01:14:46.159Z 15 300-319 124 124 124 212 212 212 F F F
137102024-04-20T01:14:46.159Z 16 320-339 124 124 124 208 208 208 T T T
137112024-04-20T01:14:46.159Z 17 340-359 115 115 115 191 191 191 T T T
137122024-04-20T01:14:46.159Z 18 360-379 125 125 125 216 216 216 T F T <---
137132024-04-20T01:14:46.159Z 19 380-399 124 124 124 212 212 212 F F F
137142024-04-20T01:14:46.159Z 20 400-419 119 119 119 197 197 197 T F T <---
137152024-04-20T01:14:46.159Z 21 420-439 124 124 124 213 213 213 F F F
137162024-04-20T01:14:46.159Z 22 440-459 118 118 118 196 196 196 F F F
137172024-04-20T01:14:46.159Z 23 460-479 118 118 118 194 194 194 T F T <---
137182024-04-20T01:14:46.159Z 24 480-499 122 122 122 206 206 206 F F F
137192024-04-20T01:14:46.159Z 25 500-519 119 119 119 197 197 197 F F F
137202024-04-20T01:14:46.159Z 26 520-539 121 121 121 201 201 201 T F T <---
137212024-04-20T01:14:46.159Z 27 540-559 124 124 124 210 210 210 F F F
137222024-04-20T01:14:46.159Z 28 560-579 123 123 123 207 207 207 T F T <---
137232024-04-20T01:14:46.159Z 29 580-599 123 123 123 207 207 207 F F F
137242024-04-20T01:14:46.159ZMax gen: 125, Max flush: 216
137252024-04-20T01:14:46.159ZError: Difference in extent metadata found!
137262024-04-20T01:14:46.159ZA Difference in extent metadata is expected here
137272024-04-20T01:14:46.159ZOn loop 63
137282024-04-20T01:14:46.160Z
137292024-04-20T01:14:46.163ZVerifying data now
137302024-04-20T01:15:00.862ZNo change after verify
137312024-04-20T01:15:00.862ZLoop: 63 Downstairs dump after verify (and repair):
137322024-04-20T01:15:00.876ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137332024-04-20T01:15:00.876Z 0 000-019 126 126 126 217 217 217 F F F
137342024-04-20T01:15:00.876Z 1 020-039 121 121 121 201 201 201 F F F
137352024-04-20T01:15:00.876Z 2 040-059 126 126 126 217 217 217 F F F
137362024-04-20T01:15:00.876Z 3 060-079 125 125 125 216 216 216 F F F
137372024-04-20T01:15:00.876Z 4 080-099 126 126 126 217 217 217 F F F
137382024-04-20T01:15:00.876Z 5 100-119 124 124 124 209 209 209 F F F
137392024-04-20T01:15:00.876Z 6 120-139 124 124 124 208 208 208 F F F
137402024-04-20T01:15:00.876Z 7 140-159 126 126 126 217 217 217 F F F
137412024-04-20T01:15:00.877Z 8 160-179 126 126 126 217 217 217 F F F
137422024-04-20T01:15:00.877Z 9 180-199 123 123 123 207 207 207 F F F
137432024-04-20T01:15:00.880Z 10 200-219 124 124 124 213 213 213 F F F
137442024-04-20T01:15:00.880Z 11 220-239 126 126 126 217 217 217 F F F
137452024-04-20T01:15:00.880Z 12 240-259 126 126 126 217 217 217 F F F
137462024-04-20T01:15:00.880Z 13 260-279 124 124 124 209 209 209 F F F
137472024-04-20T01:15:00.880Z 14 280-299 126 126 126 217 217 217 F F F
137482024-04-20T01:15:00.880Z 15 300-319 124 124 124 212 212 212 F F F
137492024-04-20T01:15:00.880Z 16 320-339 126 126 126 217 217 217 F F F
137502024-04-20T01:15:00.880Z 17 340-359 126 126 126 217 217 217 F F F
137512024-04-20T01:15:00.881Z 18 360-379 126 126 126 217 217 217 F F F
137522024-04-20T01:15:00.881Z 19 380-399 124 124 124 212 212 212 F F F
137532024-04-20T01:15:00.881Z 20 400-419 126 126 126 217 217 217 F F F
137542024-04-20T01:15:00.881Z 21 420-439 124 124 124 213 213 213 F F F
137552024-04-20T01:15:00.881Z 22 440-459 118 118 118 196 196 196 F F F
137562024-04-20T01:15:00.881Z 23 460-479 126 126 126 217 217 217 F F F
137572024-04-20T01:15:00.881Z 24 480-499 122 122 122 206 206 206 F F F
137582024-04-20T01:15:00.881Z 25 500-519 119 119 119 197 197 197 F F F
137592024-04-20T01:15:00.881Z 26 520-539 126 126 126 217 217 217 F F F
137602024-04-20T01:15:00.881Z 27 540-559 124 124 124 210 210 210 F F F
137612024-04-20T01:15:00.881Z 28 560-579 126 126 126 217 217 217 F F F
137622024-04-20T01:15:00.881Z 29 580-599 123 123 123 207 207 207 F F F
137632024-04-20T01:15:00.881ZMax gen: 126, Max flush: 217
137642024-04-20T01:15:00.881Z
137652024-04-20T01:15:00.881ZBegin loop 64
137662024-04-20T01:15:00.881ZDownstairs to restart: 1
137672024-04-20T01:15:00.897Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.397420306Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683}
137682024-04-20T01:15:00.898Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.397703477Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683}
137692024-04-20T01:15:00.903Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.397738127Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683}
137702024-04-20T01:15:00.903ZCrucible runtime is spawned
137712024-04-20T01:15:00.903Z{"msg":"Crucible c94eedf0-5935-467c-8e7a-99ea41c4b649 has session id: 352f1476-4fd3-41c5-a133-7b13a2621e73","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.397827907Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137722024-04-20T01:15:00.903Z{"msg":"Upstairs opts: Upstairs UUID: c94eedf0-5935-467c-8e7a-99ea41c4b649, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.397884257Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137732024-04-20T01:15:00.903Z{"msg":"Crucible stats registered with UUID: c94eedf0-5935-467c-8e7a-99ea41c4b649","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.397918387Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137742024-04-20T01:15:00.903Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.397941857Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683}
137752024-04-20T01:15:00.903Z{"msg":"c94eedf0-5935-467c-8e7a-99ea41c4b649 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.397968948Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137762024-04-20T01:15:00.903Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.397991548Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"0","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137772024-04-20T01:15:00.904Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.398017048Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"0","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137782024-04-20T01:15:00.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.398060638Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"1","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137792024-04-20T01:15:00.904Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.398085188Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"1","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137802024-04-20T01:15:00.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.398121398Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"2","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137812024-04-20T01:15:00.904Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.398165738Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"2","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137822024-04-20T01:15:00.904Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.398212128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137832024-04-20T01:15:00.904Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.398239858Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137842024-04-20T01:15:00.904Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.398260898Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137852024-04-20T01:15:00.904Z{"msg":"ds_connection connected from Ok(127.0.0.1:41150)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.398381158Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137862024-04-20T01:15:00.904Z{"msg":"ds_connection connected from Ok(127.0.0.1:34988)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.398418558Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137872024-04-20T01:15:00.904Z{"msg":"ds_connection connected from Ok(127.0.0.1:39253)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.398444049Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137882024-04-20T01:15:00.904Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.39922977Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"2","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137892024-04-20T01:15:00.904Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.39927929Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"2","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137902024-04-20T01:15:00.904Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.39931006Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"0","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137912024-04-20T01:15:00.904Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.399337731Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"0","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137922024-04-20T01:15:00.904Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.399657111Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"1","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137932024-04-20T01:15:00.904Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.399695351Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"1","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137942024-04-20T01:15:00.904Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.399723631Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"0","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137952024-04-20T01:15:00.904Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.399986222Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"2","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137962024-04-20T01:15:00.904Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400233373Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"0","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137972024-04-20T01:15:00.904Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400266793Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"0","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137982024-04-20T01:15:00.904Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400290603Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
137992024-04-20T01:15:00.905Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400320693Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"1","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138002024-04-20T01:15:00.905Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400407983Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"2","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138012024-04-20T01:15:00.905Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400462173Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"2","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138022024-04-20T01:15:00.905Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400498413Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138032024-04-20T01:15:00.905Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400729034Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"1","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138042024-04-20T01:15:00.905Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400781364Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"1","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138052024-04-20T01:15:00.905Z{"msg":"[0]R flush_numbers[0..12]: [217, 201, 217, 216, 217, 209, 208, 217, 217, 207, 213, 217]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400821984Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138062024-04-20T01:15:00.905Z{"msg":"[0]R generation[0..12]: [126, 121, 126, 125, 126, 124, 124, 126, 126, 123, 124, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400838744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138072024-04-20T01:15:00.905Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400862294Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138082024-04-20T01:15:00.905Z{"msg":"[1]R flush_numbers[0..12]: [217, 201, 217, 216, 217, 209, 208, 217, 217, 207, 213, 217]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400894004Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
138092024-04-20T01:15:00.905Z,"":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138102024-04-20T01:15:00.905Z{"msg":"[1]R generation[0..12]: [126, 121, 126, 125, 126, 124, 124, 126, 126, 123, 124, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400932014Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138112024-04-20T01:15:00.905Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400967294Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138122024-04-20T01:15:00.905Z{"msg":"[2]R flush_numbers[0..12]: [217, 201, 217, 216, 217, 209, 208, 217, 217, 207, 213, 217]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.400984304Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138132024-04-20T01:15:00.905Z{"msg":"[2]R generation[0..12]: [126, 121, 126, 125, 126, 124, 124, 126, 126, 123, 124, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401005614Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138142024-04-20T01:15:00.905Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401020014Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138152024-04-20T01:15:00.905Z{"msg":"Max found gen is 127","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401033714Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138162024-04-20T01:15:00.905Z{"msg":"Generation requested: 128 >= found:127","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401066824Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138172024-04-20T01:15:00.905Z{"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401102604Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138182024-04-20T01:15:00.905Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401126995Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"":"downstairs","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138192024-04-20T01:15:00.905Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401159184Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138202024-04-20T01:15:00.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401183625Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"0","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138212024-04-20T01:15:00.906Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401215445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"1","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138222024-04-20T01:15:00.906Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401239175Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"client":"2","":"downstairs","session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138232024-04-20T01:15:00.906Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401264845Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138242024-04-20T01:15:00.906Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401291065Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138252024-04-20T01:15:00.906Z{"msg":"c94eedf0-5935-467c-8e7a-99ea41c4b649 is now active with session: 352f1476-4fd3-41c5-a133-7b13a2621e73","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401314865Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"}
138262024-04-20T01:15:00.906Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401347085Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683,"session_id":"352f1476-4fd3-41c5-a133-7b13a2621e73"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
138272024-04-20T01:15:00.906Z}
138282024-04-20T01:15:00.906Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:14:59.401385245Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683}
138292024-04-20T01:15:00.910ZLoading write count information from file "/tmp/test_repair_verify.data"
138302024-04-20T01:15:00.910ZRun Repair workload
138312024-04-20T01:15:00.911Z01/30 Write block 457 len 32768 data: 15 13 10 12 15 14 15 12
138322024-04-20T01:15:00.911Z02/30 Read block 236 len 28672
138332024-04-20T01:15:00.911Z03/30 Write block 480 len 24576 data: 7 12 12 11 10 11
138342024-04-20T01:15:00.911Z04/30 Read block 166 len 8192
138352024-04-20T01:15:00.911Z05/30 Write block 445 len 20480 data: 10 10 11 11 12
138362024-04-20T01:15:00.911Z06/30 Write block 110 len 8192 data: 10 9
138372024-04-20T01:15:00.911Z07/30 Write block 570 len 20480 data: 14 13 9 7 6
138382024-04-20T01:15:00.911Z08/30 Read block 523 len 40960
138392024-04-20T01:15:00.915Z09/30 Write block 64 len 24576 data: 10 10 13 11 13 14
138402024-04-20T01:15:00.915Z10/30 Write block 172 len 20480 data: 11 14 13 11 12
138412024-04-20T01:15:00.915Z11/30 Flush
138422024-04-20T01:15:00.918Z12/30 Write block 191 len 4096 data: 12
138432024-04-20T01:15:00.918Z13/30 Read block 118 len 24576
138442024-04-20T01:15:00.918Z14/30 Read block 291 len 20480
138452024-04-20T01:15:00.918Z15/30 Read block 316 len 40960
138462024-04-20T01:15:00.918Z16/30 Read block 507 len 4096
138472024-04-20T01:15:00.922Z17/30 Write block 232 len 28672 data: 12 15 16 14 14 15 13
138482024-04-20T01:15:00.922Z18/30 Read block 538 len 32768
138492024-04-20T01:15:00.922Z19/30 Write block 518 len 32768 data: 12 14 15 15 14 14 12 11
138502024-04-20T01:15:00.922Z20/30 Write block 294 len 16384 data: 9 7 8 8
138512024-04-20T01:15:00.926Z21/30 Read block 68 len 36864
138522024-04-20T01:15:00.926Z22/30 Read block 228 len 24576
138532024-04-20T01:15:00.926Z23/30 Read block 48 len 12288
138542024-04-20T01:15:00.926Z24/30 Read block 108 len 12288
138552024-04-20T01:15:00.929Z25/30 Write block 176 len 20480 data: 13 12 14 15 14
138562024-04-20T01:15:00.930Z26/30 Read block 200 len 40960
138572024-04-20T01:15:00.930Z27/30 Write block 543 len 36864 data: 15 13 11 11 7 9 11 13 12
138582024-04-20T01:15:00.930Z28/30 Write block 53 len 32768 data: 17 16 16 15 13 13 12 13
138592024-04-20T01:15:00.934Z29/30 Read block 76 len 4096
138602024-04-20T01:15:00.934Z30/30 Read block 581 len 32768
138612024-04-20T01:15:00.934Z----------------------------------------------------------------
138622024-04-20T01:15:00.934Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
138632024-04-20T01:15:00.934ZStates: Active Active Active
138642024-04-20T01:15:00.934ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138652024-04-20T01:15:00.934Z 1 Acked 1000 Write 8 Done Sent Done false
138662024-04-20T01:15:00.934Z 2 Acked 1001 Read 7 Done Sent Done false
138672024-04-20T01:15:00.934Z 3 Acked 1002 Write 6 Done Sent Done false
138682024-04-20T01:15:00.934Z 4 Acked 1003 Read 2 Done Sent Done false
138692024-04-20T01:15:00.934Z 5 Acked 1004 Write 5 Done Sent Done false
138702024-04-20T01:15:00.934Z 6 Acked 1005 Write 2 Done Sent Done false
138712024-04-20T01:15:00.934Z 7 Acked 1006 Write 5 Done Sent Done false
138722024-04-20T01:15:00.934Z 8 Acked 1007 Read 10 Done Sent Done false
138732024-04-20T01:15:00.934Z 9 Acked 1008 Write 6 Done Sent Done false
138742024-04-20T01:15:00.935Z 10 Acked 1009 Write 5 Done Sent Done false
138752024-04-20T01:15:00.935Z 11 Acked 1010 Flush 0 Done Sent Done false
138762024-04-20T01:15:00.935Z 12 Acked 1011 Write 1 Done Sent Done false
138772024-04-20T01:15:00.935Z 13 Acked 1012 Read 6 Done Sent Done false
138782024-04-20T01:15:00.935Z 14 Acked 1013 Read 5 Done Sent Done false
138792024-04-20T01:15:00.935Z 15 Acked 1014 Read 10 Done Sent Done false
138802024-04-20T01:15:00.935Z 16 Acked 1015 Read 1 Done Sent Done false
138812024-04-20T01:15:00.935Z 17 Acked 1016 Write 7 Done Sent Done false
138822024-04-20T01:15:00.935Z 18 Acked 1017 Read 8 Done Sent Done false
138832024-04-20T01:15:00.935Z 19 Acked 1018 Write 8 Done Sent Done false
138842024-04-20T01:15:00.935Z 20 Acked 1019 Write 4 Done Sent Done false
138852024-04-20T01:15:00.935Z 21 Acked 1020 Read 9 Done Sent Done false
138862024-04-20T01:15:00.935Z 22 Acked 1021 Read 6 Done Sent Done false
138872024-04-20T01:15:00.935Z 23 Acked 1022 Read 3 Done Sent Done false
138882024-04-20T01:15:00.935Z 24 Acked 1023 Read 3 Done Sent Done false
138892024-04-20T01:15:00.935Z 25 Acked 1024 Write 5 Done Sent Done false
138902024-04-20T01:15:00.935Z 26 Acked 1025 Read 10 Done Sent Done false
138912024-04-20T01:15:00.935Z 27 Acked 1026 Write 9 Done Sent Done false
138922024-04-20T01:15:00.935Z 28 Acked 1027 Write 8 Done Sent Done false
138932024-04-20T01:15:00.935Z 29 Acked 1028 Read 1 Done Sent Done false
138942024-04-20T01:15:00.935Z 30 Acked 1029 Read 8 Sent Sent Done false
138952024-04-20T01:15:00.935Z STATES DS:0 DS:1 DS:2 TOTAL
138962024-04-20T01:15:00.935Z New 0 0 0 0
138972024-04-20T01:15:00.936Z Sent 1 30 0 31
138982024-04-20T01:15:00.936Z Done 29 0 30 59
138992024-04-20T01:15:00.936Z Skipped 0 0 0 0
139002024-04-20T01:15:00.936Z Error 0 0 0 0
139012024-04-20T01:15:00.936ZLast Flush: 1010 0 1010
139022024-04-20T01:15:00.936ZDownstairs last five completed:
139032024-04-20T01:15:00.936ZUpstairs last five completed: 30 29 28 27 26
139042024-04-20T01:15:00.936Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:14:59.431849803Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1683}
139052024-04-20T01:15:00.936ZWrote out file "/tmp/test_repair_verify.data"
139062024-04-20T01:15:00.939Z
139072024-04-20T01:15:00.940ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
139082024-04-20T01:15:00.957ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139092024-04-20T01:15:00.957Z 0 000-019 126 126 126 217 217 217 F F F
139102024-04-20T01:15:00.957Z 1 020-039 121 121 121 201 201 201 F F F
139112024-04-20T01:15:00.957Z 2 040-059 126 126 126 217 217 217 T F T <---
139122024-04-20T01:15:00.957Z 3 060-079 128 125 128 218 216 218 T F T <---
139132024-04-20T01:15:00.957Z 4 080-099 126 126 126 217 217 217 F F F
139142024-04-20T01:15:00.957Z 5 100-119 128 124 128 218 209 218 F F F <---
139152024-04-20T01:15:00.957Z 6 120-139 124 124 124 208 208 208 F F F
139162024-04-20T01:15:00.957Z 7 140-159 126 126 126 217 217 217 F F F
139172024-04-20T01:15:00.957Z 8 160-179 128 126 128 218 217 218 T F T <---
139182024-04-20T01:15:00.964Z 9 180-199 123 123 123 207 207 207 T F T <---
139192024-04-20T01:15:00.964Z 10 200-219 124 124 124 213 213 213 F F F
139202024-04-20T01:15:00.964Z 11 220-239 126 126 126 217 217 217 T F T <---
139212024-04-20T01:15:00.964Z 12 240-259 126 126 126 217 217 217 F F F
139222024-04-20T01:15:00.964Z 13 260-279 124 124 124 209 209 209 F F F
139232024-04-20T01:15:00.964Z 14 280-299 126 126 126 217 217 217 T F T <---
139242024-04-20T01:15:00.964Z 15 300-319 124 124 124 212 212 212 F F F
139252024-04-20T01:15:00.964Z 16 320-339 126 126 126 217 217 217 F F F
139262024-04-20T01:15:00.964Z 17 340-359 126 126 126 217 217 217 F F F
139272024-04-20T01:15:00.964Z 18 360-379 126 126 126 217 217 217 F F F
139282024-04-20T01:15:00.964Z 19 380-399 124 124 124 212 212 212 F F F
139292024-04-20T01:15:00.964Z 20 400-419 126 126 126 217 217 217 F F F
139302024-04-20T01:15:00.964Z 21 420-439 124 124 124 213 213 213 F F F
139312024-04-20T01:15:00.964Z 22 440-459 128 118 128 218 196 218 F F F <---
139322024-04-20T01:15:00.964Z 23 460-479 128 126 128 218 217 218 F F F <---
139332024-04-20T01:15:00.964Z 24 480-499 128 122 128 218 206 218 F F F <---
139342024-04-20T01:15:00.964Z 25 500-519 119 119 119 197 197 197 T F T <---
139352024-04-20T01:15:00.964Z 26 520-539 126 126 126 217 217 217 T F T <---
139362024-04-20T01:15:00.965Z 27 540-559 124 124 124 210 210 210 T F T <---
139372024-04-20T01:15:00.965Z 28 560-579 128 126 128 218 217 218 F F F <---
139382024-04-20T01:15:00.965Z 29 580-599 123 123 123 207 207 207 F F F
139392024-04-20T01:15:00.965ZMax gen: 128, Max flush: 218
139402024-04-20T01:15:00.965ZError: Difference in extent metadata found!
139412024-04-20T01:15:00.965ZA Difference in extent metadata is expected here
139422024-04-20T01:15:00.965ZOn loop 64
139432024-04-20T01:15:00.965Z
139442024-04-20T01:15:00.970ZVerifying data now
139452024-04-20T01:15:05.603ZNo change after verify
139462024-04-20T01:15:05.603ZLoop: 64 Downstairs dump after verify (and repair):
139472024-04-20T01:15:05.617ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139482024-04-20T01:15:05.617Z 0 000-019 126 126 126 217 217 217 F F F
139492024-04-20T01:15:05.617Z 1 020-039 121 121 121 201 201 201 F F F
139502024-04-20T01:15:05.617Z 2 040-059 129 129 129 219 219 219 F F F
139512024-04-20T01:15:05.617Z 3 060-079 129 129 129 219 219 219 F F F
139522024-04-20T01:15:05.617Z 4 080-099 126 126 126 217 217 217 F F F
139532024-04-20T01:15:05.617Z 5 100-119 128 128 128 218 218 218 F F F
139542024-04-20T01:15:05.621Z 6 120-139 124 124 124 208 208 208 F F F
139552024-04-20T01:15:05.621Z 7 140-159 126 126 126 217 217 217 F F F
139562024-04-20T01:15:05.621Z 8 160-179 129 129 129 219 219 219 F F F
139572024-04-20T01:15:05.621Z 9 180-199 129 129 129 219 219 219 F F F
139582024-04-20T01:15:05.621Z 10 200-219 124 124 124 213 213 213 F F F
139592024-04-20T01:15:05.621Z 11 220-239 129 129 129 219 219 219 F F F
139602024-04-20T01:15:05.621Z 12 240-259 126 126 126 217 217 217 F F F
139612024-04-20T01:15:05.621Z 13 260-279 124 124 124 209 209 209 F F F
139622024-04-20T01:15:05.622Z 14 280-299 129 129 129 219 219 219 F F F
139632024-04-20T01:15:05.622Z 15 300-319 124 124 124 212 212 212 F F F
139642024-04-20T01:15:05.622Z 16 320-339 126 126 126 217 217 217 F F F
139652024-04-20T01:15:05.622Z 17 340-359 126 126 126 217 217 217 F F F
139662024-04-20T01:15:05.622Z 18 360-379 126 126 126 217 217 217 F F F
139672024-04-20T01:15:05.622Z 19 380-399 124 124 124 212 212 212 F F F
139682024-04-20T01:15:05.622Z 20 400-419 126 126 126 217 217 217 F F F
139692024-04-20T01:15:05.622Z 21 420-439 124 124 124 213 213 213 F F F
139702024-04-20T01:15:05.622Z 22 440-459 128 128 128 218 218 218 F F F
139712024-04-20T01:15:05.622Z 23 460-479 128 128 128 218 218 218 F F F
139722024-04-20T01:15:05.622Z 24 480-499 128 128 128 218 218 218 F F F
139732024-04-20T01:15:05.622Z 25 500-519 129 129 129 219 219 219 F F F
139742024-04-20T01:15:05.622Z 26 520-539 129 129 129 219 219 219 F F F
139752024-04-20T01:15:05.622Z 27 540-559 129 129 129 219 219 219 F F F
139762024-04-20T01:15:05.622Z 28 560-579 128 128 128 218 218 218 F F F
139772024-04-20T01:15:05.622Z 29 580-599 123 123 123 207 207 207 F F F
139782024-04-20T01:15:05.622ZMax gen: 129, Max flush: 219
139792024-04-20T01:15:05.622Z
139802024-04-20T01:15:05.622ZBegin loop 65
139812024-04-20T01:15:05.622ZDownstairs to restart: 2
139822024-04-20T01:15:05.639Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.138892981Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691}
139832024-04-20T01:15:05.639Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139140462Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691}
139842024-04-20T01:15:05.646Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139158642Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691}
139852024-04-20T01:15:05.646Z{"msg":"Crucible b88a7c3e-7e4b-45f9-9ad2-3ebd604c4e94 has session id: dd3cda68-e225-4180-9636-45f3b6414ef8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139199902Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
139862024-04-20T01:15:05.646Z{"msg":"Upstairs opts: Upstairs UUID: b88a7c3e-7e4b-45f9-9ad2-3ebd604c4e94, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139249142Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"session_id":"Crucible runtime is spawned
139872024-04-20T01:15:05.646Zdd3cda68-e225-4180-9636-45f3b6414ef8"}
139882024-04-20T01:15:05.647Z{"msg":"Crucible stats registered with UUID: b88a7c3e-7e4b-45f9-9ad2-3ebd604c4e94","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139297322Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
139892024-04-20T01:15:05.647Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139352222Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691}
139902024-04-20T01:15:05.647Z{"msg":"b88a7c3e-7e4b-45f9-9ad2-3ebd604c4e94 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139414692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
139912024-04-20T01:15:05.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139460552Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"0","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
139922024-04-20T01:15:05.647Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139490472Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"0","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
139932024-04-20T01:15:05.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139516232Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"1","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
139942024-04-20T01:15:05.647Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139592123Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"1","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
139952024-04-20T01:15:05.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139631412Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"2","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
139962024-04-20T01:15:05.647Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139657483Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"2","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
139972024-04-20T01:15:05.647Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139692463Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
139982024-04-20T01:15:05.647Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139772673Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
139992024-04-20T01:15:05.647Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139814883Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140002024-04-20T01:15:05.647Z{"msg":"ds_connection connected from Ok(127.0.0.1:56728)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139841923Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140012024-04-20T01:15:05.647Z{"msg":"ds_connection connected from Ok(127.0.0.1:53451)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139876663Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140022024-04-20T01:15:05.647Z{"msg":"ds_connection connected from Ok(127.0.0.1:37012)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.139911813Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140032024-04-20T01:15:05.647Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.140429934Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"0","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140042024-04-20T01:15:05.647Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.140472514Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"0","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140052024-04-20T01:15:05.647Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.140513474Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"1","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140062024-04-20T01:15:05.647Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.140544834Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"1","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140072024-04-20T01:15:05.647Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.140721485Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"2","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140082024-04-20T01:15:05.647Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.140758675Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"2","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140092024-04-20T01:15:05.647Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.141181046Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"0","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140102024-04-20T01:15:05.648Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.141292116Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"1","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140112024-04-20T01:15:05.648Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.141333476Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"2","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140122024-04-20T01:15:05.648Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.141869168Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"0","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140132024-04-20T01:15:05.648Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.141909778Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"0","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140142024-04-20T01:15:05.648Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.141924868Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140152024-04-20T01:15:05.648Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.141940208Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"2","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140162024-04-20T01:15:05.648Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.141971588Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"2","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140172024-04-20T01:15:05.648Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.141987058Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140182024-04-20T01:15:05.648Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142007598Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"1","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140192024-04-20T01:15:05.648Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142037718Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"1","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140202024-04-20T01:15:05.648Z{"msg":"[0]R flush_numbers[0..12]: [217, 201, 219, 219, 217, 218, 208, 217, 219, 219, 213, 219]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142053498Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140212024-04-20T01:15:05.648Z{"msg":"[0]R generation[0..12]: [126, 121, 129, 129, 126, 128, 124, 126, 129, 129, 124, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142085548Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140222024-04-20T01:15:05.648Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142104168Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO
140232024-04-20T01:15:05.648Z","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140242024-04-20T01:15:05.648Z{"msg":"[1]R flush_numbers[0..12]: [217, 201, 219, 219, 217, 218, 208, 217, 219, 219, 213, 219]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142136528Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140252024-04-20T01:15:05.648Z{"msg":"[1]R generation[0..12]: [126, 121, 129, 129, 126, 128, 124, 126, 129, 129, 124, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142151688Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140262024-04-20T01:15:05.648Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142184498Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140272024-04-20T01:15:05.648Z{"msg":"[2]R flush_numbers[0..12]: [217, 201, 219, 219, 217, 218, 208, 217, 219, 219, 213, 219]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142201618Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140282024-04-20T01:15:05.648Z{"msg":"[2]R generation[0..12]: [126, 121, 129, 129, 126, 128, 124, 126, 129, 129, 124, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142232338Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140292024-04-20T01:15:05.648Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142247988Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140302024-04-20T01:15:05.648Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142278398Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140312024-04-20T01:15:05.648Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142294138Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140322024-04-20T01:15:05.648Z{"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142324109Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140332024-04-20T01:15:05.648Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142339909Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"":"downstairs","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
140342024-04-20T01:15:05.649Z}
140352024-04-20T01:15:05.649Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142372869Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140362024-04-20T01:15:05.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142386019Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"0","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140372024-04-20T01:15:05.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142399429Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"1","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140382024-04-20T01:15:05.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142420039Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"client":"2","":"downstairs","session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140392024-04-20T01:15:05.649Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142447589Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140402024-04-20T01:15:05.649Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142462729Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140412024-04-20T01:15:05.649Z{"msg":"b88a7c3e-7e4b-45f9-9ad2-3ebd604c4e94 is now active with session: dd3cda68-e225-4180-9636-45f3b6414ef8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142491389Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140422024-04-20T01:15:05.649Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142509889Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691,"session_id":"dd3cda68-e225-4180-9636-45f3b6414ef8"}
140432024-04-20T01:15:05.649Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:04.142544729Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691}
140442024-04-20T01:15:05.649ZLoading write count information from file "/tmp/test_repair_verify.data"
140452024-04-20T01:15:05.649ZRun Repair workload
140462024-04-20T01:15:05.654Z01/30 Write block 391 len 28672 data: 6 6 7 7 7 7 6
140472024-04-20T01:15:05.654Z02/30 Flush
140482024-04-20T01:15:05.654Z03/30 Write block 34 len 24576 data: 11 11 9 10 11 11
140492024-04-20T01:15:05.654Z04/30 Read block 120 len 24576
140502024-04-20T01:15:05.654Z05/30 Write block 50 len 12288 data: 12 14 15
140512024-04-20T01:15:05.654Z06/30 Read block 224 len 16384
140522024-04-20T01:15:05.654Z07/30 Write block 511 len 12288 data: 12 13 12
140532024-04-20T01:15:05.654Z08/30 Read block 199 len 4096
140542024-04-20T01:15:05.658Z09/30 Write block 326 len 16384 data: 12 10 11 9
140552024-04-20T01:15:05.658Z10/30 Read block 240 len 24576
140562024-04-20T01:15:05.658Z11/30 Flush
140572024-04-20T01:15:05.662Z12/30 Write block 144 len 24576 data: 15 15 13 11 10 8
140582024-04-20T01:15:05.662Z13/30 Write block 421 len 36864 data: 9 8 9 7 10 9 8 7 11
140592024-04-20T01:15:05.662Z14/30 Read block 476 len 8192
140602024-04-20T01:15:05.662Z15/30 Read block 365 len 12288
140612024-04-20T01:15:05.670Z16/30 Write block 511 len 28672 data: 13 14 13 11 10 14 14
140622024-04-20T01:15:05.670Z17/30 Read block 322 len 20480
140632024-04-20T01:15:05.670Z18/30 Write block 99 len 24576 data: 7 10 10 10 10 9
140642024-04-20T01:15:05.670Z19/30 Read block 470 len 16384
140652024-04-20T01:15:05.670Z20/30 Write block 579 len 8192 data: 12 12
140662024-04-20T01:15:05.670Z21/30 Write block 137 len 40960 data: 11 13 12 13 14 14 15 16 16 14
140672024-04-20T01:15:05.670Z22/30 Write block 366 len 24576 data: 11 15 12 12 14 17
140682024-04-20T01:15:05.671Z23/30 Read block 296 len 32768
140692024-04-20T01:15:05.675Z24/30 Write block 253 len 32768 data: 8 12 12 12 9 8 9 7
140702024-04-20T01:15:05.675Z25/30 Write block 223 len 4096 data: 12
140712024-04-20T01:15:05.675Z26/30 Write block 549 len 32768 data: 12 14 13 9 7 9 10 8
140722024-04-20T01:15:05.675Z27/30 Write block 500 len 16384 data: 9 9 9 12
140732024-04-20T01:15:05.675Z28/30 Write block 423 len 4096 data: 10
140742024-04-20T01:15:05.679Z29/30 Write block 569 len 40960 data: 16 15 14 10 8 7 6 7 9 9
140752024-04-20T01:15:05.679Z30/30 Read block 569 len 20480
140762024-04-20T01:15:05.679Z----------------------------------------------------------------
140772024-04-20T01:15:05.679Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:28
140782024-04-20T01:15:05.679ZStates: Active Active Active
140792024-04-20T01:15:05.679ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140802024-04-20T01:15:05.679Z 3 Acked 1002 Write 6 Done Done Done false
140812024-04-20T01:15:05.679Z 4 Acked 1003 Read 6 Done Done Done false
140822024-04-20T01:15:05.680Z 5 Acked 1004 Write 3 Done Done Done false
140832024-04-20T01:15:05.680Z 6 Acked 1005 Read 4 Done Done Sent false
140842024-04-20T01:15:05.680Z 7 Acked 1006 Write 3 Done Done Sent false
140852024-04-20T01:15:05.680Z 8 Acked 1007 Read 1 Done Done Sent false
140862024-04-20T01:15:05.680Z 9 Acked 1008 Write 4 Done Done Sent false
140872024-04-20T01:15:05.680Z 10 Acked 1009 Read 6 Done Done Sent false
140882024-04-20T01:15:05.680Z 11 Acked 1010 Flush 0 Done Done Sent false
140892024-04-20T01:15:05.680Z 12 Acked 1011 Write 6 Done Done Sent false
140902024-04-20T01:15:05.680Z 13 Acked 1012 Write 9 Done Done Sent false
140912024-04-20T01:15:05.680Z 14 Acked 1013 Read 2 Done Done Sent false
140922024-04-20T01:15:05.680Z 15 Acked 1014 Read 3 Done Done Sent false
140932024-04-20T01:15:05.680Z 16 Acked 1015 Write 7 Done Done Sent false
140942024-04-20T01:15:05.680Z 17 Acked 1016 Read 5 Done Done Sent false
140952024-04-20T01:15:05.680Z 18 Acked 1017 Write 6 Done Done Sent false
140962024-04-20T01:15:05.680Z 19 Acked 1018 Read 4 Done Done Sent false
140972024-04-20T01:15:05.680Z 20 Acked 1019 Write 2 Done Done Sent false
140982024-04-20T01:15:05.680Z 21 Acked 1020 Write 10 Done Done Sent false
140992024-04-20T01:15:05.680Z 22 Acked 1021 Write 6 Done Done Sent false
141002024-04-20T01:15:05.680Z 23 Acked 1022 Read 8 Done Done Sent false
141012024-04-20T01:15:05.680Z 24 Acked 1023 Write 8 Done Done Sent false
141022024-04-20T01:15:05.680Z 25 Acked 1024 Write 1 Done Done Sent false
141032024-04-20T01:15:05.680Z 26 Acked 1025 Write 8 Done Done Sent false
141042024-04-20T01:15:05.681Z 27 Acked 1026 Write 4 Done Done Sent false
141052024-04-20T01:15:05.681Z 28 Acked 1027 Write 1 Done Done Sent false
141062024-04-20T01:15:05.681Z 29 Acked 1028 Write 10 Done Done Sent false
141072024-04-20T01:15:05.681Z 30 Acked 1029 Read 5 Done Done Sent false
141082024-04-20T01:15:05.681Z STATES DS:0 DS:1 DS:2 TOTAL
141092024-04-20T01:15:05.681Z New 0 0 0 0
141102024-04-20T01:15:05.681Z Sent 0 0 25 25
141112024-04-20T01:15:05.681Z Done 28 28 3 59
141122024-04-20T01:15:05.681Z Skipped 0 0 0 0
141132024-04-20T01:15:05.681Z Error 0 0 0 0
141142024-04-20T01:15:05.681ZLast Flush: 1010 1010 1001
141152024-04-20T01:15:05.681ZDownstairs last five completed: 1001 1000
141162024-04-20T01:15:05.681ZUpstairs last five completed: 30 29 28 27 26
141172024-04-20T01:15:05.681Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:15:04.177034936Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1691}
141182024-04-20T01:15:05.681ZWrote out file "/tmp/test_repair_verify.data"
141192024-04-20T01:15:05.685Z
141202024-04-20T01:15:05.685ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
141212024-04-20T01:15:05.704ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141222024-04-20T01:15:05.704Z 0 000-019 126 126 126 217 217 217 F F F
141232024-04-20T01:15:05.704Z 1 020-039 130 130 121 221 221 201 F F T <---
141242024-04-20T01:15:05.705Z 2 040-059 130 130 129 221 221 219 F F T <---
141252024-04-20T01:15:05.705Z 3 060-079 129 129 129 219 219 219 F F F
141262024-04-20T01:15:05.705Z 4 080-099 126 126 126 217 217 217 T T F <---
141272024-04-20T01:15:05.705Z 5 100-119 128 128 128 218 218 218 T T F <---
141282024-04-20T01:15:05.705Z 6 120-139 124 124 124 208 208 208 T T F <---
141292024-04-20T01:15:05.705Z 7 140-159 126 126 126 217 217 217 T T F <---
141302024-04-20T01:15:05.708Z 8 160-179 129 129 129 219 219 219 F F F
141312024-04-20T01:15:05.708Z 9 180-199 129 129 129 219 219 219 F F F
141322024-04-20T01:15:05.709Z 10 200-219 124 124 124 213 213 213 F F F
141332024-04-20T01:15:05.709Z 11 220-239 129 129 129 219 219 219 T T F <---
141342024-04-20T01:15:05.709Z 12 240-259 126 126 126 217 217 217 T T F <---
141352024-04-20T01:15:05.709Z 13 260-279 124 124 124 209 209 209 T T F <---
141362024-04-20T01:15:05.709Z 14 280-299 129 129 129 219 219 219 F F F
141372024-04-20T01:15:05.709Z 15 300-319 124 124 124 212 212 212 F F F
141382024-04-20T01:15:05.709Z 16 320-339 130 130 126 221 221 217 F F F <---
141392024-04-20T01:15:05.709Z 17 340-359 126 126 126 217 217 217 F F F
141402024-04-20T01:15:05.709Z 18 360-379 126 126 126 217 217 217 T T F <---
141412024-04-20T01:15:05.709Z 19 380-399 130 130 130 220 220 220 F F F
141422024-04-20T01:15:05.709Z 20 400-419 126 126 126 217 217 217 F F F
141432024-04-20T01:15:05.709Z 21 420-439 124 124 124 213 213 213 T T F <---
141442024-04-20T01:15:05.709Z 22 440-459 128 128 128 218 218 218 F F F
141452024-04-20T01:15:05.709Z 23 460-479 128 128 128 218 218 218 F F F
141462024-04-20T01:15:05.709Z 24 480-499 128 128 128 218 218 218 F F F
141472024-04-20T01:15:05.709Z 25 500-519 130 130 129 221 221 219 T T F <---
141482024-04-20T01:15:05.709Z 26 520-539 129 129 129 219 219 219 F F F
141492024-04-20T01:15:05.709Z 27 540-559 129 129 129 219 219 219 T T F <---
141502024-04-20T01:15:05.709Z 28 560-579 128 128 128 218 218 218 T T F <---
141512024-04-20T01:15:05.709Z 29 580-599 123 123 123 207 207 207 T T F <---
141522024-04-20T01:15:05.709ZMax gen: 130, Max flush: 221
141532024-04-20T01:15:05.709ZError: Difference in extent metadata found!
141542024-04-20T01:15:05.709ZA Difference in extent metadata is expected here
141552024-04-20T01:15:05.710ZOn loop 65
141562024-04-20T01:15:05.710Z
141572024-04-20T01:15:05.715ZVerifying data now
141582024-04-20T01:15:20.442ZNo change after verify
141592024-04-20T01:15:20.442ZLoop: 65 Downstairs dump after verify (and repair):
141602024-04-20T01:15:20.456ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141612024-04-20T01:15:20.456Z 0 000-019 126 126 126 217 217 217 F F F
141622024-04-20T01:15:20.456Z 1 020-039 130 130 130 221 221 221 F F F
141632024-04-20T01:15:20.456Z 2 040-059 130 130 130 221 221 221 F F F
141642024-04-20T01:15:20.456Z 3 060-079 129 129 129 219 219 219 F F F
141652024-04-20T01:15:20.456Z 4 080-099 131 131 131 222 222 222 F F F
141662024-04-20T01:15:20.460Z 5 100-119 131 131 131 222 222 222 F F F
141672024-04-20T01:15:20.460Z 6 120-139 131 131 131 222 222 222 F F F
141682024-04-20T01:15:20.460Z 7 140-159 131 131 131 222 222 222 F F F
141692024-04-20T01:15:20.460Z 8 160-179 129 129 129 219 219 219 F F F
141702024-04-20T01:15:20.460Z 9 180-199 129 129 129 219 219 219 F F F
141712024-04-20T01:15:20.460Z 10 200-219 124 124 124 213 213 213 F F F
141722024-04-20T01:15:20.461Z 11 220-239 131 131 131 222 222 222 F F F
141732024-04-20T01:15:20.461Z 12 240-259 131 131 131 222 222 222 F F F
141742024-04-20T01:15:20.461Z 13 260-279 131 131 131 222 222 222 F F F
141752024-04-20T01:15:20.461Z 14 280-299 129 129 129 219 219 219 F F F
141762024-04-20T01:15:20.461Z 15 300-319 124 124 124 212 212 212 F F F
141772024-04-20T01:15:20.461Z 16 320-339 130 130 130 221 221 221 F F F
141782024-04-20T01:15:20.461Z 17 340-359 126 126 126 217 217 217 F F F
141792024-04-20T01:15:20.461Z 18 360-379 131 131 131 222 222 222 F F F
141802024-04-20T01:15:20.461Z 19 380-399 130 130 130 220 220 220 F F F
141812024-04-20T01:15:20.461Z 20 400-419 126 126 126 217 217 217 F F F
141822024-04-20T01:15:20.461Z 21 420-439 131 131 131 222 222 222 F F F
141832024-04-20T01:15:20.461Z 22 440-459 128 128 128 218 218 218 F F F
141842024-04-20T01:15:20.461Z 23 460-479 128 128 128 218 218 218 F F F
141852024-04-20T01:15:20.461Z 24 480-499 128 128 128 218 218 218 F F F
141862024-04-20T01:15:20.461Z 25 500-519 131 131 131 222 222 222 F F F
141872024-04-20T01:15:20.461Z 26 520-539 129 129 129 219 219 219 F F F
141882024-04-20T01:15:20.461Z 27 540-559 131 131 131 222 222 222 F F F
141892024-04-20T01:15:20.461Z 28 560-579 131 131 131 222 222 222 F F F
141902024-04-20T01:15:20.461Z 29 580-599 131 131 131 222 222 222 F F F
141912024-04-20T01:15:20.461ZMax gen: 131, Max flush: 222
141922024-04-20T01:15:20.461Z
141932024-04-20T01:15:20.461ZBegin loop 66
141942024-04-20T01:15:20.462ZDownstairs to restart: 2
141952024-04-20T01:15:20.480Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.977814392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699}
141962024-04-20T01:15:20.480Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978060313Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699}
141972024-04-20T01:15:20.487Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978094213Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699}
141982024-04-20T01:15:20.487Z{"msg":"Crucible 99be1583-c817-4ac6-96aa-40a2fc130257 has session id: 35e752d3-e796-4b32-b169-1dd0e1d18b34","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978135023Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
141992024-04-20T01:15:20.487Z{Crucible runtime is spawned
142002024-04-20T01:15:20.487Z"msg":"Upstairs opts: Upstairs UUID: 99be1583-c817-4ac6-96aa-40a2fc130257, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978195423Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142012024-04-20T01:15:20.487Z{"msg":"Crucible stats registered with UUID: 99be1583-c817-4ac6-96aa-40a2fc130257","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978255463Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142022024-04-20T01:15:20.487Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978272733Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699}
142032024-04-20T01:15:20.487Z{"msg":"99be1583-c817-4ac6-96aa-40a2fc130257 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978379263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142042024-04-20T01:15:20.487Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978427134Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"0","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142052024-04-20T01:15:20.487Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978449094Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"0","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142062024-04-20T01:15:20.487Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978474164Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"1","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142072024-04-20T01:15:20.487Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978496134Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"1","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142082024-04-20T01:15:20.487Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978523054Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"2","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142092024-04-20T01:15:20.487Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978576684Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"2","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142102024-04-20T01:15:20.487Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978621874Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142112024-04-20T01:15:20.487Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978650594Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142122024-04-20T01:15:20.487Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978675964Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142132024-04-20T01:15:20.487Z{"msg":"ds_connection connected from Ok(127.0.0.1:55373)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978702444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142142024-04-20T01:15:20.487Z{"msg":"ds_connection connected from Ok(127.0.0.1:42469)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978805864Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142152024-04-20T01:15:20.487Z{"msg":"ds_connection connected from Ok(127.0.0.1:61834)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.978876335Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142162024-04-20T01:15:20.487Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.979471506Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"0","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142172024-04-20T01:15:20.487Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.979521256Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"0","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142182024-04-20T01:15:20.488Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.979539666Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"1","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142192024-04-20T01:15:20.488Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.979559126Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"1","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142202024-04-20T01:15:20.488Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.979583346Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"2","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142212024-04-20T01:15:20.488Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.979607776Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"2","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142222024-04-20T01:15:20.488Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980081777Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"0","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142232024-04-20T01:15:20.488Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980134878Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"1","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142242024-04-20T01:15:20.488Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980241148Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"2","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142252024-04-20T01:15:20.488Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980539008Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"1","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142262024-04-20T01:15:20.488Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980579759Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"1","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142272024-04-20T01:15:20.488Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980626708Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142282024-04-20T01:15:20.488Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980659549Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"0","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142292024-04-20T01:15:20.488Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980684889Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"0","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142302024-04-20T01:15:20.488Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980707319Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142312024-04-20T01:15:20.488Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980729169Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"2","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142322024-04-20T01:15:20.488Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980760189Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"2","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142332024-04-20T01:15:20.488Z{"msg":"[0]R flush_numbers[0..12]: [217, 221, 221, 219, 222, 222, 222, 222, 219, 219, 213, 222]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980785149Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142342024-04-20T01:15:20.488Z{"msg":"[0]R generation[0..12]: [126, 130, 130, 129, 131, 131, 131, 131, 129, 129, 124, 131]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980813029Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142352024-04-20T01:15:20.488Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980838339Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142362024-04-20T01:15:20.488Z{"msg":"[1]R flush_numbers[0..12]: [217, 221, 221, 219, 222, 222, 222, 222, 219, 219, 213, 222]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980865859Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142372024-04-20T01:15:20.488Z{"msg":"[1]R generation[0..12]: [126, 130, 130, 129, 131, 131, 131, 131, 129, 129, 124, 131]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980905679Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142382024-04-20T01:15:20.488Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
142392024-04-20T01:15:20.488Z,"time":"2024-04-20T01:15:18.980930519Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142402024-04-20T01:15:20.488Z{"msg":"[2]R flush_numbers[0..12]: [217, 221, 221, 219, 222, 222, 222, 222, 219, 219, 213, 222]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980960509Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142412024-04-20T01:15:20.489Z{"msg":"[2]R generation[0..12]: [126, 130, 130, 129, 131, 131, 131, 131, 129, 129, 124, 131]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.980993629Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142422024-04-20T01:15:20.489Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.981020189Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142432024-04-20T01:15:20.489Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.981042389Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142442024-04-20T01:15:20.489Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.981083869Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142452024-04-20T01:15:20.489Z{"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.98111756Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142462024-04-20T01:15:20.489Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.98114099Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"":"downstairs","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142472024-04-20T01:15:20.489Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.98116851Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142482024-04-20T01:15:20.489Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.98118985Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"0","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142492024-04-20T01:15:20.489Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.98121761Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"1","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142502024-04-20T01:15:20.489Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.98124068Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"client":"2","":"downstairs","session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142512024-04-20T01:15:20.489Z{"msg":"All required reconciliation work is completed","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142522024-04-20T01:15:20.489Zcrucible","level":30,"time":"2024-04-20T01:15:18.98127386Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142532024-04-20T01:15:20.489Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.98129651Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142542024-04-20T01:15:20.489Z{"msg":"99be1583-c817-4ac6-96aa-40a2fc130257 is now active with session: 35e752d3-e796-4b32-b169-1dd0e1d18b34","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.98131925Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142552024-04-20T01:15:20.489Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.98133977Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699,"session_id":"35e752d3-e796-4b32-b169-1dd0e1d18b34"}
142562024-04-20T01:15:20.489Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:18.98136064Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699}
142572024-04-20T01:15:20.489ZLoading write count information from file "/tmp/test_repair_verify.data"
142582024-04-20T01:15:20.489ZRun Repair workload
142592024-04-20T01:15:20.489Z01/30 Write block 247 len 40960 data: 10 14 14 13 10 9 9 13 13 13
142602024-04-20T01:15:20.489Z02/30 Write block 199 len 4096 data: 13
142612024-04-20T01:15:20.494Z03/30 Write block 174 len 16384 data: 14 12 14 13
142622024-04-20T01:15:20.494Z04/30 Write block 416 len 20480 data: 15 11 12 12 11
142632024-04-20T01:15:20.494Z05/30 Read block 34 len 40960
142642024-04-20T01:15:20.494Z06/30 Write block 52 len 16384 data: 16 18 17 17
142652024-04-20T01:15:20.494Z07/30 Read block 442 len 40960
142662024-04-20T01:15:20.494Z08/30 Read block 173 len 32768
142672024-04-20T01:15:20.494Z09/30 Read block 377 len 20480
142682024-04-20T01:15:20.494Z10/30 Read block 580 len 28672
142692024-04-20T01:15:20.494Z11/30 Read block 494 len 32768
142702024-04-20T01:15:20.494Z12/30 Read block 82 len 8192
142712024-04-20T01:15:20.494Z13/30 Write block 42 len 12288 data: 7 5 6
142722024-04-20T01:15:20.498Z14/30 Write block 336 len 16384 data: 10 9 13 11
142732024-04-20T01:15:20.498Z15/30 Read block 244 len 40960
142742024-04-20T01:15:20.498Z16/30 Read block 266 len 12288
142752024-04-20T01:15:20.498Z17/30 Read block 94 len 28672
142762024-04-20T01:15:20.498Z18/30 Read block 62 len 8192
142772024-04-20T01:15:20.498Z19/30 Read block 14 len 16384
142782024-04-20T01:15:20.498Z20/30 Read block 248 len 32768
142792024-04-20T01:15:20.502Z21/30 Read block 477 len 8192
142802024-04-20T01:15:20.502Z22/30 Write block 355 len 36864 data: 9 11 10 10 11 9 9 11 11
142812024-04-20T01:15:20.502Z23/30 Flush
142822024-04-20T01:15:20.507Z24/30 Write block 463 len 40960 data: 16 13 13 16 16 13 15 14 12 12
142832024-04-20T01:15:20.512Z25/30 Read block 268 len 40960
142842024-04-20T01:15:20.512Z26/30 Read block 542 len 32768
142852024-04-20T01:15:20.513Z27/30 Write block 355 len 8192 data: 10 12
142862024-04-20T01:15:20.513Z28/30 Write block 412 len 24576 data: 14 15 14 14 16 12
142872024-04-20T01:15:20.513Z29/30 Read block 515 len 36864
142882024-04-20T01:15:20.518Z30/30 Write block 583 len 24576 data: 8 13 16 19 17 15
142892024-04-20T01:15:20.518Z----------------------------------------------------------------
142902024-04-20T01:15:20.518Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
142912024-04-20T01:15:20.518ZStates: Active Active Active
142922024-04-20T01:15:20.518ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
142932024-04-20T01:15:20.518Z 1 Acked 1000 Write 10 Done Done Done false
142942024-04-20T01:15:20.518Z 2 Acked 1001 Write 1 Done Done Done false
142952024-04-20T01:15:20.518Z 3 Acked 1002 Write 4 Done Done Done false
142962024-04-20T01:15:20.518Z 4 Acked 1003 Write 5 Done Done Sent false
142972024-04-20T01:15:20.518Z 5 Acked 1004 Read 10 Done Done Sent false
142982024-04-20T01:15:20.518Z 6 Acked 1005 Write 4 Done Done Sent false
142992024-04-20T01:15:20.518Z 7 Acked 1006 Read 10 Done Done Sent false
143002024-04-20T01:15:20.518Z 8 Acked 1007 Read 8 Done Done Sent false
143012024-04-20T01:15:20.518Z 9 Acked 1008 Read 5 Done Done Sent false
143022024-04-20T01:15:20.518Z 10 Acked 1009 Read 7 Done Done Sent false
143032024-04-20T01:15:20.518Z 11 Acked 1010 Read 8 Done Done Sent false
143042024-04-20T01:15:20.518Z 12 Acked 1011 Read 2 Done Done Sent false
143052024-04-20T01:15:20.518Z 13 Acked 1012 Write 3 Done Done Sent false
143062024-04-20T01:15:20.518Z 14 Acked 1013 Write 4 Done Done Sent false
143072024-04-20T01:15:20.519Z 15 Acked 1014 Read 10 Done Done Sent false
143082024-04-20T01:15:20.519Z 16 Acked 1015 Read 3 Done Done Sent false
143092024-04-20T01:15:20.519Z 17 Acked 1016 Read 7 Done Done Sent false
143102024-04-20T01:15:20.519Z 18 Acked 1017 Read 2 Done Done Sent false
143112024-04-20T01:15:20.519Z 19 Acked 1018 Read 4 Done Done Sent false
143122024-04-20T01:15:20.519Z 20 Acked 1019 Read 8 Done Done Sent false
143132024-04-20T01:15:20.519Z 21 Acked 1020 Read 2 Done Done Sent false
143142024-04-20T01:15:20.519Z 22 Acked 1021 Write 9 Done Done Sent false
143152024-04-20T01:15:20.519Z 23 Acked 1022 Flush 0 Done Done Sent false
143162024-04-20T01:15:20.519Z 24 Acked 1023 Write 10 Done Done Sent false
143172024-04-20T01:15:20.519Z 25 Acked 1024 Read 10 Done Done Sent false
143182024-04-20T01:15:20.519Z 26 Acked 1025 Read 8 Done Done Sent false
143192024-04-20T01:15:20.519Z 27 Acked 1026 Write 2 Done Done Sent false
143202024-04-20T01:15:20.519Z 28 Acked 1027 Write 6 Done Done Sent false
143212024-04-20T01:15:20.519Z 29 Acked 1028 Read 9 Done Done Sent false
143222024-04-20T01:15:20.519Z 30 Acked 1029 Write 6 Sent Sent Sent false
143232024-04-20T01:15:20.519Z STATES DS:0 DS:1 DS:2 TOTAL
143242024-04-20T01:15:20.519Z New 0 0 0 0
143252024-04-20T01:15:20.519Z Sent 1 1 27 29
143262024-04-20T01:15:20.519Z Done 29 29 3 61
143272024-04-20T01:15:20.519Z Skipped 0 0 0 0
143282024-04-20T01:15:20.519Z Error 0 0 0 0
143292024-04-20T01:15:20.520ZLast Flush: 1022 1022 0
143302024-04-20T01:15:20.520ZDownstairs last five completed:
143312024-04-20T01:15:20.520ZUpstairs last five completed: 30 29 28 27 26
143322024-04-20T01:15:20.520Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:15:19.01269558Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1699}
143332024-04-20T01:15:20.520ZWrote out file "/tmp/test_repair_verify.data"
143342024-04-20T01:15:20.523Z
143352024-04-20T01:15:20.523ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
143362024-04-20T01:15:20.539ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143372024-04-20T01:15:20.539Z 0 000-019 126 126 126 217 217 217 F F F
143382024-04-20T01:15:20.543Z 1 020-039 130 130 130 221 221 221 F F F
143392024-04-20T01:15:20.543Z 2 040-059 132 132 130 223 223 221 F F F <---
143402024-04-20T01:15:20.543Z 3 060-079 129 129 129 219 219 219 F F F
143412024-04-20T01:15:20.543Z 4 080-099 131 131 131 222 222 222 F F F
143422024-04-20T01:15:20.543Z 5 100-119 131 131 131 222 222 222 F F F
143432024-04-20T01:15:20.543Z 6 120-139 131 131 131 222 222 222 F F F
143442024-04-20T01:15:20.543Z 7 140-159 131 131 131 222 222 222 F F F
143452024-04-20T01:15:20.543Z 8 160-179 132 132 129 223 223 219 F F T <---
143462024-04-20T01:15:20.543Z 9 180-199 132 132 129 223 223 219 F F T <---
143472024-04-20T01:15:20.543Z 10 200-219 124 124 124 213 213 213 F F F
143482024-04-20T01:15:20.543Z 11 220-239 131 131 131 222 222 222 F F F
143492024-04-20T01:15:20.544Z 12 240-259 132 132 131 223 223 222 F F T <---
143502024-04-20T01:15:20.544Z 13 260-279 131 131 131 222 222 222 F F F
143512024-04-20T01:15:20.544Z 14 280-299 129 129 129 219 219 219 F F F
143522024-04-20T01:15:20.544Z 15 300-319 124 124 124 212 212 212 F F F
143532024-04-20T01:15:20.544Z 16 320-339 132 132 130 223 223 221 F F F <---
143542024-04-20T01:15:20.544Z 17 340-359 132 132 126 223 223 217 T T F <---
143552024-04-20T01:15:20.544Z 18 360-379 132 132 131 223 223 222 F F F <---
143562024-04-20T01:15:20.544Z 19 380-399 130 130 130 220 220 220 F F F
143572024-04-20T01:15:20.544Z 20 400-419 132 132 126 223 223 217 T T F <---
143582024-04-20T01:15:20.544Z 21 420-439 132 132 131 223 223 222 F F F <---
143592024-04-20T01:15:20.544Z 22 440-459 128 128 128 218 218 218 F F F
143602024-04-20T01:15:20.544Z 23 460-479 128 128 128 218 218 218 T T F <---
143612024-04-20T01:15:20.544Z 24 480-499 128 128 128 218 218 218 F F F
143622024-04-20T01:15:20.544Z 25 500-519 131 131 131 222 222 222 F F F
143632024-04-20T01:15:20.544Z 26 520-539 129 129 129 219 219 219 F F F
143642024-04-20T01:15:20.544Z 27 540-559 131 131 131 222 222 222 F F F
143652024-04-20T01:15:20.544Z 28 560-579 131 131 131 222 222 222 F F F
143662024-04-20T01:15:20.544Z 29 580-599 131 131 131 222 222 222 T T F <---
143672024-04-20T01:15:20.544ZMax gen: 132, Max flush: 223
143682024-04-20T01:15:20.544ZError: Difference in extent metadata found!
143692024-04-20T01:15:20.544ZA Difference in extent metadata is expected here
143702024-04-20T01:15:20.545ZOn loop 66
143712024-04-20T01:15:20.545Z
143722024-04-20T01:15:20.548ZVerifying data now
143732024-04-20T01:15:25.053ZNo change after verify
143742024-04-20T01:15:25.053ZLoop: 66 Downstairs dump after verify (and repair):
143752024-04-20T01:15:25.068ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143762024-04-20T01:15:25.068Z 0 000-019 126 126 126 217 217 217 F F F
143772024-04-20T01:15:25.068Z 1 020-039 130 130 130 221 221 221 F F F
143782024-04-20T01:15:25.068Z 2 040-059 132 132 132 223 223 223 F F F
143792024-04-20T01:15:25.068Z 3 060-079 129 129 129 219 219 219 F F F
143802024-04-20T01:15:25.068Z 4 080-099 131 131 131 222 222 222 F F F
143812024-04-20T01:15:25.068Z 5 100-119 131 131 131 222 222 222 F F F
143822024-04-20T01:15:25.068Z 6 120-139 131 131 131 222 222 222 F F F
143832024-04-20T01:15:25.068Z 7 140-159 131 131 131 222 222 222 F F F
143842024-04-20T01:15:25.068Z 8 160-179 132 132 132 223 223 223 F F F
143852024-04-20T01:15:25.068Z 9 180-199 132 132 132 223 223 223 F F F
143862024-04-20T01:15:25.075Z 10 200-219 124 124 124 213 213 213 F F F
143872024-04-20T01:15:25.075Z 11 220-239 131 131 131 222 222 222 F F F
143882024-04-20T01:15:25.075Z 12 240-259 132 132 132 223 223 223 F F F
143892024-04-20T01:15:25.075Z 13 260-279 131 131 131 222 222 222 F F F
143902024-04-20T01:15:25.075Z 14 280-299 129 129 129 219 219 219 F F F
143912024-04-20T01:15:25.075Z 15 300-319 124 124 124 212 212 212 F F F
143922024-04-20T01:15:25.075Z 16 320-339 132 132 132 223 223 223 F F F
143932024-04-20T01:15:25.075Z 17 340-359 133 133 133 224 224 224 F F F
143942024-04-20T01:15:25.075Z 18 360-379 132 132 132 223 223 223 F F F
143952024-04-20T01:15:25.075Z 19 380-399 130 130 130 220 220 220 F F F
143962024-04-20T01:15:25.075Z 20 400-419 133 133 133 224 224 224 F F F
143972024-04-20T01:15:25.075Z 21 420-439 132 132 132 223 223 223 F F F
143982024-04-20T01:15:25.076Z 22 440-459 128 128 128 218 218 218 F F F
143992024-04-20T01:15:25.076Z 23 460-479 133 133 133 224 224 224 F F F
144002024-04-20T01:15:25.076Z 24 480-499 128 128 128 218 218 218 F F F
144012024-04-20T01:15:25.076Z 25 500-519 131 131 131 222 222 222 F F F
144022024-04-20T01:15:25.076Z 26 520-539 129 129 129 219 219 219 F F F
144032024-04-20T01:15:25.076Z 27 540-559 131 131 131 222 222 222 F F F
144042024-04-20T01:15:25.076Z 28 560-579 131 131 131 222 222 222 F F F
144052024-04-20T01:15:25.076Z 29 580-599 133 133 133 224 224 224 F F F
144062024-04-20T01:15:25.076ZMax gen: 133, Max flush: 224
144072024-04-20T01:15:25.076Z
144082024-04-20T01:15:25.076ZBegin loop 67
144092024-04-20T01:15:25.076ZDownstairs to restart: 0
144102024-04-20T01:15:25.091Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589159467Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707}
144112024-04-20T01:15:25.091Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589386798Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707}
144122024-04-20T01:15:25.091Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589413428Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707}
144132024-04-20T01:15:25.091Z{"msg":"Crucible fce4d40a-b3b5-4ece-a871-52f453493265 has session id: 0ed72468-9e86-4dd6-8d8f-c9d68b5b7536","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589446858Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144142024-04-20T01:15:25.098Z{"msg":"Upstairs opts: Upstairs UUID: fce4d40a-b3b5-4ece-a871-52f453493265, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589489728Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144152024-04-20T01:15:25.098ZCrucible runtime is spawned
144162024-04-20T01:15:25.098Z{"msg":"Crucible stats registered with UUID: fce4d40a-b3b5-4ece-a871-52f453493265","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589538088Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144172024-04-20T01:15:25.098Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589608078Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707}
144182024-04-20T01:15:25.098Z{"msg":"fce4d40a-b3b5-4ece-a871-52f453493265 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589713948Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144192024-04-20T01:15:25.098Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589740428Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"0","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144202024-04-20T01:15:25.098Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589756088Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"0","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144212024-04-20T01:15:25.098Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589770359Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"1","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144222024-04-20T01:15:25.098Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589800679Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"1","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144232024-04-20T01:15:25.098Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589824588Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"2","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144242024-04-20T01:15:25.098Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589859059Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"2","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144252024-04-20T01:15:25.098Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589884209Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144262024-04-20T01:15:25.098Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589926039Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144272024-04-20T01:15:25.098Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.589952049Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144282024-04-20T01:15:25.098Z{"msg":"ds_connection connected from Ok(127.0.0.1:43849)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.590033179Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144292024-04-20T01:15:25.098Z{"msg":"ds_connection connected from Ok(127.0.0.1:57031)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.590090819Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144302024-04-20T01:15:25.098Z{"msg":"ds_connection connected from Ok(127.0.0.1:59319)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.590127429Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144312024-04-20T01:15:25.098Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.590805151Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"2","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144322024-04-20T01:15:25.098Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.590870491Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"2","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144332024-04-20T01:15:25.098Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.590981471Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"1","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144342024-04-20T01:15:25.099Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.591025961Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"1","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144352024-04-20T01:15:25.099Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.591050271Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"0","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144362024-04-20T01:15:25.099Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.591079391Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"0","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144372024-04-20T01:15:25.099Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.591289162Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"2","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144382024-04-20T01:15:25.099Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.591462322Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"1","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144392024-04-20T01:15:25.099Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.591705683Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"0","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144402024-04-20T01:15:25.099Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.591868693Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"2","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144412024-04-20T01:15:25.099Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.591904243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"2","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144422024-04-20T01:15:25.099Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.591921003Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144432024-04-20T01:15:25.099Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.591966243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"1","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144442024-04-20T01:15:25.099Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592023193Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"1","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144452024-04-20T01:15:25.099Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592061384Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144462024-04-20T01:15:25.099Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592163694Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"0","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144472024-04-20T01:15:25.099Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592192844Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"0","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144482024-04-20T01:15:25.099Z{"msg":"[0]R flush_numbers[0..12]: [217, 221, 223, 219, 222, 222, 222, 222, 223, 223, 213, 222]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592208544Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144492024-04-20T01:15:25.099Z{"msg":"[0]R generation[0..12]: [126, 130, 132, 129, 131, 131, 131, 131, 132, 132, 124, 131]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592284124Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144502024-04-20T01:15:25.099Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592302574Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144512024-04-20T01:15:25.099Z{"msg":"[1]R flush_numbers[0..12]: [217, 221, 223, 219, 222, 222, 222, 222, 223, 223, 213, 222]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592317274Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144522024-04-20T01:15:25.099Z{"msg":"[1]R generation[0..12]: [126, 130, 132, 129, 131, 131, 131, 131, 132, 132, 124, 131]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592344374Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144532024-04-20T01:15:25.099Z{"msg":"Wait for a query_work_queue command to finish before sending IO
144542024-04-20T01:15:25.099Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592361584Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144552024-04-20T01:15:25.099Z{"msg":"[2]R flush_numbers[0..12]: [217, 221, 223, 219, 222, 222, 222, 222, 223, 223, 213, 222]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592383224Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144562024-04-20T01:15:25.100Z{"msg":"[2]R generation[0..12]: [126, 130, 132, 129, 131, 131, 131, 131, 132, 132, 124, 131]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592397664Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144572024-04-20T01:15:25.100Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592419874Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144582024-04-20T01:15:25.100Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592433174Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144592024-04-20T01:15:25.100Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592453224Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144602024-04-20T01:15:25.100Z{"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592466754Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144612024-04-20T01:15:25.100Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592481394Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"":"downstairs","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144622024-04-20T01:15:25.100Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592494685Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144632024-04-20T01:15:25.100Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592513834Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"0","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144642024-04-20T01:15:25.100Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592527794Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"1","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144652024-04-20T01:15:25.100Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592541334Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"client":"2","":"downstairs","session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144662024-04-20T01:15:25.100Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592554465Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144672024-04-20T01:15:25.100Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592573245Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144682024-04-20T01:15:25.100Z{"msg":"fce4d40a-b3b5-4ece-a871-52f453493265 is now active with session: 0ed72468-9e86-4dd6-8d8f-c9d68b5b7536","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592587185Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144692024-04-20T01:15:25.100Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592600825Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707,"session_id":"0ed72468-9e86-4dd6-8d8f-c9d68b5b7536"}
144702024-04-20T01:15:25.100Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:23.592620185Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707}
144712024-04-20T01:15:25.100ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
144722024-04-20T01:15:25.100ZLoading write count information from file "/tmp/test_repair_verify.data"
144732024-04-20T01:15:25.100ZRun Repair workload
144742024-04-20T01:15:25.100Z01/30 Write block 159 len 28672 data: 10 11 13 11 10 10 12
144752024-04-20T01:15:25.104Z02/30 Read block 333 len 12288
144762024-04-20T01:15:25.104Z03/30 Write block 103 len 40960 data: 11 10 9 11 10 11 11 11 10 10
144772024-04-20T01:15:25.104Z04/30 Write block 281 len 24576 data: 13 10 7 8 8 8
144782024-04-20T01:15:25.104Z05/30 Read block 434 len 16384
144792024-04-20T01:15:25.104Z06/30 Write block 166 len 8192 data: 9 8
144802024-04-20T01:15:25.104Z07/30 Read block 451 len 32768
144812024-04-20T01:15:25.104Z08/30 Write block 480 len 20480 data: 8 13 13 12 11
144822024-04-20T01:15:25.104Z09/30 Read block 345 len 28672
144832024-04-20T01:15:25.104Z10/30 Read block 501 len 32768
144842024-04-20T01:15:25.109Z11/30 Write block 343 len 36864 data: 10 9 12 9 8 8 7 5 6
144852024-04-20T01:15:25.109Z12/30 Write block 83 len 36864 data: 10 9 9 12 12 12 12 13 11
144862024-04-20T01:15:25.109Z13/30 Write block 67 len 8192 data: 12 14
144872024-04-20T01:15:25.113Z14/30 Write block 470 len 32768 data: 15 13 13 12 12 14 11 12
144882024-04-20T01:15:25.113Z15/30 Write block 322 len 8192 data: 14 14
144892024-04-20T01:15:25.113Z16/30 Write block 316 len 40960 data: 8 8 14 14 14 14 15 15 12 11
144902024-04-20T01:15:25.113Z17/30 Read block 161 len 16384
144912024-04-20T01:15:25.113Z18/30 Write block 154 len 12288 data: 11 13 10
144922024-04-20T01:15:25.113Z19/30 Read block 563 len 16384
144932024-04-20T01:15:25.116Z20/30 Flush
144942024-04-20T01:15:25.120Z21/30 Write block 401 len 8192 data: 9 9
144952024-04-20T01:15:25.120Z22/30 Flush
144962024-04-20T01:15:25.123Z23/30 Write block 190 len 8192 data: 12 13
144972024-04-20T01:15:25.123Z24/30 Read block 559 len 32768
144982024-04-20T01:15:25.123Z25/30 Write block 379 len 36864 data: 13 14 12 11 10 8 5 6 4
144992024-04-20T01:15:25.126Z26/30 Write block 354 len 24576 data: 7 11 13 11 11 12
145002024-04-20T01:15:25.127Z27/30 Read block 519 len 12288
145012024-04-20T01:15:25.127Z28/30 Read block 53 len 12288
145022024-04-20T01:15:25.127Z29/30 Read block 588 len 20480
145032024-04-20T01:15:25.131Z30/30 Write block 160 len 36864 data: 12 14 12 11 11 13 10 9 9
145042024-04-20T01:15:25.131Z----------------------------------------------------------------
145052024-04-20T01:15:25.131Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
145062024-04-20T01:15:25.131ZStates: Active Active Active
145072024-04-20T01:15:25.131ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
145082024-04-20T01:15:25.132Z 1 Acked 1000 Write 7 Done Done Done false
145092024-04-20T01:15:25.132Z 2 Acked 1001 Read 3 Done Done Done false
145102024-04-20T01:15:25.132Z 3 Acked 1002 Write 10 Done Done Done false
145112024-04-20T01:15:25.132Z 4 Acked 1003 Write 6 Done Done Done false
145122024-04-20T01:15:25.132Z 5 Acked 1004 Read 4 Done Done Done false
145132024-04-20T01:15:25.132Z 6 Acked 1005 Write 2 Sent Done Done false
145142024-04-20T01:15:25.132Z 7 Acked 1006 Read 8 Sent Done Done false
145152024-04-20T01:15:25.132Z 8 Acked 1007 Write 5 Sent Done Done false
145162024-04-20T01:15:25.132Z 9 Acked 1008 Read 7 Sent Done Done false
145172024-04-20T01:15:25.132Z 10 Acked 1009 Read 8 Sent Done Done false
145182024-04-20T01:15:25.132Z 11 Acked 1010 Write 9 Sent Done Done false
145192024-04-20T01:15:25.132Z 12 Acked 1011 Write 9 Sent Done Done false
145202024-04-20T01:15:25.132Z 13 Acked 1012 Write 2 Sent Done Done false
145212024-04-20T01:15:25.132Z 14 Acked 1013 Write 8 Sent Done Done false
145222024-04-20T01:15:25.132Z 15 Acked 1014 Write 2 Sent Done Done false
145232024-04-20T01:15:25.132Z 16 Acked 1015 Write 10 Sent Done Done false
145242024-04-20T01:15:25.132Z 17 Acked 1016 Read 4 Sent Done Done false
145252024-04-20T01:15:25.132Z 18 Acked 1017 Write 3 Sent Done Done false
145262024-04-20T01:15:25.132Z 19 Acked 1018 Read 4 Sent Done Done false
145272024-04-20T01:15:25.132Z 20 Acked 1019 Flush 0 Sent Done Done false
145282024-04-20T01:15:25.132Z 21 Acked 1020 Write 2 Sent Done Done false
145292024-04-20T01:15:25.132Z 22 Acked 1021 Flush 0 Sent Done Done false
145302024-04-20T01:15:25.133Z 23 Acked 1022 Write 2 Sent Done Done false
145312024-04-20T01:15:25.133Z 24 Acked 1023 Read 8 Sent Done Done false
145322024-04-20T01:15:25.133Z 25 Acked 1024 Write 9 Sent Done Done false
145332024-04-20T01:15:25.133Z 26 Acked 1025 Write 6 Sent Done Done false
145342024-04-20T01:15:25.133Z 27 Acked 1026 Read 3 Sent Done Done false
145352024-04-20T01:15:25.133Z 28 Acked 1027 Read 3 Sent Done Done false
145362024-04-20T01:15:25.133Z 29 Acked 1028 Read 5 Sent Done Done false
145372024-04-20T01:15:25.133Z 30 Acked 1029 Write 9 Sent Sent Sent false
145382024-04-20T01:15:25.133Z STATES DS:0 DS:1 DS:2 TOTAL
145392024-04-20T01:15:25.133Z New 0 0 0 0
145402024-04-20T01:15:25.133Z Sent 25 1 1 27
145412024-04-20T01:15:25.133Z Done 5 29 29 63
145422024-04-20T01:15:25.133Z Skipped 0 0 0 0
145432024-04-20T01:15:25.133Z Error 0 0 0 0
145442024-04-20T01:15:25.133ZLast Flush: 0 1021 1021
145452024-04-20T01:15:25.133ZDownstairs last five completed:
145462024-04-20T01:15:25.133ZUpstairs last five completed: 30 29 28 27 26
145472024-04-20T01:15:25.133Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:15:23.628433994Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1707}
145482024-04-20T01:15:25.133ZWrote out file "/tmp/test_repair_verify.data"
145492024-04-20T01:15:25.137Z
145502024-04-20T01:15:25.137ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
145512024-04-20T01:15:25.152ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145522024-04-20T01:15:25.152Z 0 000-019 126 126 126 217 217 217 F F F
145532024-04-20T01:15:25.152Z 1 020-039 130 130 130 221 221 221 F F F
145542024-04-20T01:15:25.152Z 2 040-059 132 132 132 223 223 223 F F F
145552024-04-20T01:15:25.152Z 3 060-079 129 134 134 219 225 225 F F F <---
145562024-04-20T01:15:25.153Z 4 080-099 131 134 134 222 225 225 F F F <---
145572024-04-20T01:15:25.153Z 5 100-119 131 134 134 222 225 225 T F F <---
145582024-04-20T01:15:25.153Z 6 120-139 131 131 131 222 222 222 F F F
145592024-04-20T01:15:25.153Z 7 140-159 131 134 134 222 225 225 T F F <---
145602024-04-20T01:15:25.153Z 8 160-179 132 134 134 223 225 225 T T T <---
145612024-04-20T01:15:25.153Z 9 180-199 132 132 132 223 223 223 F T T <---
145622024-04-20T01:15:25.157Z 10 200-219 124 124 124 213 213 213 F F F
145632024-04-20T01:15:25.157Z 11 220-239 131 131 131 222 222 222 F F F
145642024-04-20T01:15:25.157Z 12 240-259 132 132 132 223 223 223 F F F
145652024-04-20T01:15:25.157Z 13 260-279 131 131 131 222 222 222 F F F
145662024-04-20T01:15:25.157Z 14 280-299 129 134 134 219 225 225 T F F <---
145672024-04-20T01:15:25.157Z 15 300-319 124 134 134 212 225 225 F F F <---
145682024-04-20T01:15:25.157Z 16 320-339 132 134 134 223 225 225 F F F <---
145692024-04-20T01:15:25.157Z 17 340-359 133 134 134 224 225 225 F T T <---
145702024-04-20T01:15:25.157Z 18 360-379 132 132 132 223 223 223 F T T <---
145712024-04-20T01:15:25.157Z 19 380-399 130 130 130 220 220 220 F T T <---
145722024-04-20T01:15:25.157Z 20 400-419 133 134 134 224 226 226 F F F <---
145732024-04-20T01:15:25.157Z 21 420-439 132 132 132 223 223 223 F F F
145742024-04-20T01:15:25.157Z 22 440-459 128 128 128 218 218 218 F F F
145752024-04-20T01:15:25.157Z 23 460-479 133 134 134 224 225 225 F F F <---
145762024-04-20T01:15:25.157Z 24 480-499 128 134 134 218 225 225 F F F <---
145772024-04-20T01:15:25.157Z 25 500-519 131 131 131 222 222 222 F F F
145782024-04-20T01:15:25.157Z 26 520-539 129 129 129 219 219 219 F F F
145792024-04-20T01:15:25.157Z 27 540-559 131 131 131 222 222 222 F F F
145802024-04-20T01:15:25.157Z 28 560-579 131 131 131 222 222 222 F F F
145812024-04-20T01:15:25.157Z 29 580-599 133 133 133 224 224 224 F F F
145822024-04-20T01:15:25.157ZMax gen: 134, Max flush: 226
145832024-04-20T01:15:25.157ZError: Difference in extent metadata found!
145842024-04-20T01:15:25.158ZA Difference in extent metadata is expected here
145852024-04-20T01:15:25.158ZOn loop 67
145862024-04-20T01:15:25.158Z
145872024-04-20T01:15:25.161ZVerifying data now
145882024-04-20T01:15:39.842ZNo change after verify
145892024-04-20T01:15:39.842ZLoop: 67 Downstairs dump after verify (and repair):
145902024-04-20T01:15:39.852ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145912024-04-20T01:15:39.852Z 0 000-019 126 126 126 217 217 217 F F F
145922024-04-20T01:15:39.852Z 1 020-039 130 130 130 221 221 221 F F F
145932024-04-20T01:15:39.852Z 2 040-059 132 132 132 223 223 223 F F F
145942024-04-20T01:15:39.852Z 3 060-079 134 134 134 225 225 225 F F F
145952024-04-20T01:15:39.852Z 4 080-099 134 134 134 225 225 225 F F F
145962024-04-20T01:15:39.852Z 5 100-119 134 134 134 225 225 225 F F F
145972024-04-20T01:15:39.852Z 6 120-139 131 131 131 222 222 222 F F F
145982024-04-20T01:15:39.852Z 7 140-159 134 134 134 225 225 225 F F F
145992024-04-20T01:15:39.852Z 8 160-179 135 135 135 227 227 227 F F F
146002024-04-20T01:15:39.852Z 9 180-199 135 135 135 227 227 227 F F F
146012024-04-20T01:15:39.852Z 10 200-219 124 124 124 213 213 213 F F F
146022024-04-20T01:15:39.856Z 11 220-239 131 131 131 222 222 222 F F F
146032024-04-20T01:15:39.856Z 12 240-259 132 132 132 223 223 223 F F F
146042024-04-20T01:15:39.856Z 13 260-279 131 131 131 222 222 222 F F F
146052024-04-20T01:15:39.856Z 14 280-299 134 134 134 225 225 225 F F F
146062024-04-20T01:15:39.856Z 15 300-319 134 134 134 225 225 225 F F F
146072024-04-20T01:15:39.856Z 16 320-339 134 134 134 225 225 225 F F F
146082024-04-20T01:15:39.856Z 17 340-359 135 135 135 227 227 227 F F F
146092024-04-20T01:15:39.856Z 18 360-379 135 135 135 227 227 227 F F F
146102024-04-20T01:15:39.856Z 19 380-399 135 135 135 227 227 227 F F F
146112024-04-20T01:15:39.856Z 20 400-419 134 134 134 226 226 226 F F F
146122024-04-20T01:15:39.856Z 21 420-439 132 132 132 223 223 223 F F F
146132024-04-20T01:15:39.856Z 22 440-459 128 128 128 218 218 218 F F F
146142024-04-20T01:15:39.856Z 23 460-479 134 134 134 225 225 225 F F F
146152024-04-20T01:15:39.856Z 24 480-499 134 134 134 225 225 225 F F F
146162024-04-20T01:15:39.856Z 25 500-519 131 131 131 222 222 222 F F F
146172024-04-20T01:15:39.857Z 26 520-539 129 129 129 219 219 219 F F F
146182024-04-20T01:15:39.857Z 27 540-559 131 131 131 222 222 222 F F F
146192024-04-20T01:15:39.857Z 28 560-579 131 131 131 222 222 222 F F F
146202024-04-20T01:15:39.857Z 29 580-599 133 133 133 224 224 224 F F F
146212024-04-20T01:15:39.857ZMax gen: 135, Max flush: 227
146222024-04-20T01:15:39.857Z
146232024-04-20T01:15:39.857ZBegin loop 68
146242024-04-20T01:15:39.857ZDownstairs to restart: 0
146252024-04-20T01:15:39.873Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372071584Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715}
146262024-04-20T01:15:39.873Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372345445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715}
146272024-04-20T01:15:39.881Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372384974Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715}
146282024-04-20T01:15:39.881Z{"Crucible runtime is spawned
146292024-04-20T01:15:39.881Zmsg":"Crucible 0f7c37b1-e060-4e3c-8bb1-12c322b252ec has session id: 4375f5df-907f-4b63-8ae6-ae68253f9383","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372489525Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146302024-04-20T01:15:39.881Z{"msg":"Upstairs opts: Upstairs UUID: 0f7c37b1-e060-4e3c-8bb1-12c322b252ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372550255Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146312024-04-20T01:15:39.882Z{"msg":"Crucible stats registered with UUID: 0f7c37b1-e060-4e3c-8bb1-12c322b252ec","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372586225Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146322024-04-20T01:15:39.882Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372607485Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715}
146332024-04-20T01:15:39.882Z{"msg":"0f7c37b1-e060-4e3c-8bb1-12c322b252ec active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372637165Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146342024-04-20T01:15:39.882Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372657515Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"0","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146352024-04-20T01:15:39.882Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372698355Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"0","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146362024-04-20T01:15:39.882Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372721665Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"1","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146372024-04-20T01:15:39.882Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372760975Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"1","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146382024-04-20T01:15:39.882Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372784085Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"2","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146392024-04-20T01:15:39.882Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372814986Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"2","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146402024-04-20T01:15:39.882Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372846046Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146412024-04-20T01:15:39.882Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372869135Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146422024-04-20T01:15:39.882Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372891406Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146432024-04-20T01:15:39.882Z{"msg":"ds_connection connected from Ok(127.0.0.1:42533)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.372986236Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146442024-04-20T01:15:39.882Z{"msg":"ds_connection connected from Ok(127.0.0.1:36340)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.373042546Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146452024-04-20T01:15:39.882Z{"msg":"ds_connection connected from Ok(127.0.0.1:57522)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.373088226Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146462024-04-20T01:15:39.882Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.373686387Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"1","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146472024-04-20T01:15:39.882Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.373736437Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"1","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146482024-04-20T01:15:39.882Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.373765488Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"2","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146492024-04-20T01:15:39.882Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.373799858Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"2","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146502024-04-20T01:15:39.882Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.374242979Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"1","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146512024-04-20T01:15:39.882Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.374357379Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"2","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146522024-04-20T01:15:39.883Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.374483049Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"0","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146532024-04-20T01:15:39.883Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.374548999Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"0","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146542024-04-20T01:15:39.883Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.37460739Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"1","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146552024-04-20T01:15:39.883Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.37464767Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"1","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146562024-04-20T01:15:39.883Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.37466319Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146572024-04-20T01:15:39.883Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.37478028Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"2","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146582024-04-20T01:15:39.883Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.37483149Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"2","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146592024-04-20T01:15:39.883Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.37486575Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146602024-04-20T01:15:39.883Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.37506772Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"0","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146612024-04-20T01:15:39.883Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375489612Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"0","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146622024-04-20T01:15:39.883Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375537492Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"0","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146632024-04-20T01:15:39.883Z{"msg":"[0]R flush_numbers[0..12]: [217, 221, 223, 225, 225, 225, 222, 225, 227, 227, 213, 222]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375565612Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146642024-04-20T01:15:39.883Z{"msg":"[0]R generation[0..12]: [126, 130, 132, 134, 134, 134, 131, 134, 135, 135, 124, 131]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375596552Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146652024-04-20T01:15:39.883Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375623202Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146662024-04-20T01:15:39.883Z{"msg":"[1]R flush_numbers[0..12]: [217, 221, 223, 225, 225, 225, 222, 225, 227, 227, 213, 222]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375652312Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146672024-04-20T01:15:39.883Z{"msg":"[1]R generation[0..12]: [126, 130, 132, 134, 134, 134, 131, 134, 135, 135, 124, 131]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
146682024-04-20T01:15:39.883Z,"time":"2024-04-20T01:15:38.375676912Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146692024-04-20T01:15:39.883Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375712142Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146702024-04-20T01:15:39.883Z{"msg":"[2]R flush_numbers[0..12]: [217, 221, 223, 225, 225, 225, 222, 225, 227, 227, 213, 222]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375745382Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146712024-04-20T01:15:39.883Z{"msg":"[2]R generation[0..12]: [126, 130, 132, 134, 134, 134, 131, 134, 135, 135, 124, 131]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375769082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146722024-04-20T01:15:39.883Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375795222Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146732024-04-20T01:15:39.883Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375823422Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146742024-04-20T01:15:39.883Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375847592Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146752024-04-20T01:15:39.884Z{"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375878632Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146762024-04-20T01:15:39.884Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375901682Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"":"downstairs","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146772024-04-20T01:15:39.884Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375928752Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146782024-04-20T01:15:39.884Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375949722Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"0","":"downstairsRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
146792024-04-20T01:15:39.884Z","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146802024-04-20T01:15:39.884Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.375978733Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"1","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146812024-04-20T01:15:39.884Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.376011433Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"client":"2","":"downstairs","session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146822024-04-20T01:15:39.884Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.376035853Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146832024-04-20T01:15:39.884Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.376062753Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146842024-04-20T01:15:39.884Z{"msg":"0f7c37b1-e060-4e3c-8bb1-12c322b252ec is now active with session: 4375f5df-907f-4b63-8ae6-ae68253f9383","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.376086533Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146852024-04-20T01:15:39.884Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.376112103Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715,"session_id":"4375f5df-907f-4b63-8ae6-ae68253f9383"}
146862024-04-20T01:15:39.884Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:38.376133793Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715}
146872024-04-20T01:15:39.884ZLoading write count information from file "/tmp/test_repair_verify.data"
146882024-04-20T01:15:39.884ZRun Repair workload
146892024-04-20T01:15:39.888Z01/30 Write block 69 len 28672 data: 15 9 9 7 9 11 7
146902024-04-20T01:15:39.888Z02/30 Flush
146912024-04-20T01:15:39.888Z03/30 Write block 24 len 40960 data: 6 6 8 8 8 9 8 8 9 9
146922024-04-20T01:15:39.888Z04/30 Read block 55 len 20480
146932024-04-20T01:15:39.888Z05/30 Flush
146942024-04-20T01:15:39.889Z06/30 Write block 575 len 36864 data: 7 8 10 10 13 13 12 11 9
146952024-04-20T01:15:39.889Z07/30 Write block 188 len 36864 data: 11 10 13 14 10 9 10 10 9
146962024-04-20T01:15:39.892Z08/30 Write block 20 len 8192 data: 9 8
146972024-04-20T01:15:39.892Z09/30 Write block 345 len 36864 data: 13 10 9 9 8 6 7 7 7
146982024-04-20T01:15:39.892Z10/30 Read block 398 len 16384
146992024-04-20T01:15:39.896Z11/30 Write block 197 len 32768 data: 11 12 14 9 10 12 9 8
147002024-04-20T01:15:39.896Z12/30 Read block 77 len 40960
147012024-04-20T01:15:39.896Z13/30 Read block 47 len 4096
147022024-04-20T01:15:39.896Z14/30 Read block 533 len 36864
147032024-04-20T01:15:39.896Z15/30 Read block 326 len 20480
147042024-04-20T01:15:39.899Z16/30 Write block 216 len 24576 data: 11 15 14 10 12 10
147052024-04-20T01:15:39.899Z17/30 Read block 410 len 32768
147062024-04-20T01:15:39.899Z18/30 Read block 486 len 36864
147072024-04-20T01:15:39.899Z19/30 Flush
147082024-04-20T01:15:39.904Z20/30 Write block 140 len 4096 data: 14
147092024-04-20T01:15:39.905Z21/30 Write block 23 len 4096 data: 5
147102024-04-20T01:15:39.905Z22/30 Read block 188 len 40960
147112024-04-20T01:15:39.908Z23/30 Write block 559 len 40960 data: 7 6 8 9 7 11 13 15 13 14
147122024-04-20T01:15:39.908Z24/30 Read block 413 len 4096
147132024-04-20T01:15:39.908Z25/30 Write block 191 len 16384 data: 15 11 10 11
147142024-04-20T01:15:39.908Z26/30 Read block 152 len 24576
147152024-04-20T01:15:39.908Z27/30 Write block 233 len 8192 data: 16 17
147162024-04-20T01:15:39.913Z28/30 Write block 385 len 40960 data: 6 7 5 5 6 6 7 7 8 8
147172024-04-20T01:15:39.913Z29/30 Read block 329 len 12288
147182024-04-20T01:15:39.913Z30/30 Read block 504 len 28672
147192024-04-20T01:15:39.913Z----------------------------------------------------------------
147202024-04-20T01:15:39.913Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:28
147212024-04-20T01:15:39.913ZStates: Active Active Active
147222024-04-20T01:15:39.913ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
147232024-04-20T01:15:39.913Z 3 Acked 1002 Write 10 Sent Done Done false
147242024-04-20T01:15:39.913Z 4 Acked 1003 Read 5 Sent Done Done false
147252024-04-20T01:15:39.913Z 5 Acked 1004 Flush 0 Sent Done Done false
147262024-04-20T01:15:39.913Z 6 Acked 1005 Write 9 Sent Done Done false
147272024-04-20T01:15:39.913Z 7 Acked 1006 Write 9 Sent Done Done false
147282024-04-20T01:15:39.913Z 8 Acked 1007 Write 2 Sent Done Done false
147292024-04-20T01:15:39.913Z 9 Acked 1008 Write 9 Sent Done Done false
147302024-04-20T01:15:39.913Z 10 Acked 1009 Read 4 Sent Done Done false
147312024-04-20T01:15:39.913Z 11 Acked 1010 Write 8 Sent Done Done false
147322024-04-20T01:15:39.913Z 12 Acked 1011 Read 10 Sent Done Done false
147332024-04-20T01:15:39.913Z 13 Acked 1012 Read 1 Sent Done Done false
147342024-04-20T01:15:39.914Z 14 Acked 1013 Read 9 Sent Done Done false
147352024-04-20T01:15:39.914Z 15 Acked 1014 Read 5 Sent Done Done false
147362024-04-20T01:15:39.914Z 16 Acked 1015 Write 6 Sent Done Done false
147372024-04-20T01:15:39.914Z 17 Acked 1016 Read 8 Sent Done Done false
147382024-04-20T01:15:39.914Z 18 Acked 1017 Read 9 Sent Done Done false
147392024-04-20T01:15:39.914Z 19 Acked 1018 Flush 0 Sent Done Done false
147402024-04-20T01:15:39.914Z 20 Acked 1019 Write 1 Sent Done Done false
147412024-04-20T01:15:39.914Z 21 Acked 1020 Write 1 Sent Done Done false
147422024-04-20T01:15:39.914Z 22 Acked 1021 Read 10 Sent Done Done false
147432024-04-20T01:15:39.914Z 23 Acked 1022 Write 10 Sent Done Done false
147442024-04-20T01:15:39.914Z 24 Acked 1023 Read 1 Sent Done Done false
147452024-04-20T01:15:39.914Z 25 Acked 1024 Write 4 Sent Done Done false
147462024-04-20T01:15:39.914Z 26 Acked 1025 Read 6 Sent Done Done false
147472024-04-20T01:15:39.914Z 27 Acked 1026 Write 2 Sent Done Done false
147482024-04-20T01:15:39.914Z 28 Acked 1027 Write 10 Sent Done Done false
147492024-04-20T01:15:39.914Z 29 Acked 1028 Read 3 Sent Done Done false
147502024-04-20T01:15:39.914Z 30 Acked 1029 Read 7 Sent Sent Done false
147512024-04-20T01:15:39.914Z STATES DS:0 DS:1 DS:2 TOTAL
147522024-04-20T01:15:39.914Z New 0 0 0 0
147532024-04-20T01:15:39.914Z Sent 28 1 0 29
147542024-04-20T01:15:39.914Z Done 0 27 28 55
147552024-04-20T01:15:39.914Z Skipped 0 0 0 0
147562024-04-20T01:15:39.914Z Error 0 0 0 0
147572024-04-20T01:15:39.915ZLast Flush: 1001 1018 1018
147582024-04-20T01:15:39.915ZDownstairs last five completed: 1001 1000
147592024-04-20T01:15:39.915ZUpstairs last five completed: 30 29 28 27 26
147602024-04-20T01:15:39.915Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:15:38.41069059Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1715}
147612024-04-20T01:15:39.915ZWrote out file "/tmp/test_repair_verify.data"
147622024-04-20T01:15:39.919Z
147632024-04-20T01:15:39.919ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
147642024-04-20T01:15:39.935ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147652024-04-20T01:15:39.935Z 0 000-019 126 126 126 217 217 217 F F F
147662024-04-20T01:15:39.935Z 1 020-039 130 136 136 221 230 230 F T T <---
147672024-04-20T01:15:39.935Z 2 040-059 132 132 132 223 223 223 F F F
147682024-04-20T01:15:39.935Z 3 060-079 136 136 136 228 228 228 F F F
147692024-04-20T01:15:39.935Z 4 080-099 134 134 134 225 225 225 F F F
147702024-04-20T01:15:39.935Z 5 100-119 134 134 134 225 225 225 F F F
147712024-04-20T01:15:39.935Z 6 120-139 131 131 131 222 222 222 F F F
147722024-04-20T01:15:39.935Z 7 140-159 134 134 134 225 225 225 F T T <---
147732024-04-20T01:15:39.939Z 8 160-179 135 135 135 227 227 227 F F F
147742024-04-20T01:15:39.939Z 9 180-199 135 136 136 227 230 230 F T T <---
147752024-04-20T01:15:39.939Z 10 200-219 124 136 136 213 230 230 F F F <---
147762024-04-20T01:15:39.939Z 11 220-239 131 136 136 222 230 230 F T T <---
147772024-04-20T01:15:39.939Z 12 240-259 132 132 132 223 223 223 F F F
147782024-04-20T01:15:39.939Z 13 260-279 131 131 131 222 222 222 F F F
147792024-04-20T01:15:39.939Z 14 280-299 134 134 134 225 225 225 F F F
147802024-04-20T01:15:39.939Z 15 300-319 134 134 134 225 225 225 F F F
147812024-04-20T01:15:39.939Z 16 320-339 134 134 134 225 225 225 F F F
147822024-04-20T01:15:39.939Z 17 340-359 135 136 136 227 230 230 F F F <---
147832024-04-20T01:15:39.939Z 18 360-379 135 135 135 227 227 227 F F F
147842024-04-20T01:15:39.939Z 19 380-399 135 135 135 227 227 227 F T T <---
147852024-04-20T01:15:39.939Z 20 400-419 134 134 134 226 226 226 F F F
147862024-04-20T01:15:39.939Z 21 420-439 132 132 132 223 223 223 F F F
147872024-04-20T01:15:39.939Z 22 440-459 128 128 128 218 218 218 F F F
147882024-04-20T01:15:39.940Z 23 460-479 134 134 134 225 225 225 F F F
147892024-04-20T01:15:39.940Z 24 480-499 134 134 134 225 225 225 F F F
147902024-04-20T01:15:39.940Z 25 500-519 131 131 131 222 222 222 F F F
147912024-04-20T01:15:39.940Z 26 520-539 129 129 129 219 219 219 F F F
147922024-04-20T01:15:39.940Z 27 540-559 131 131 131 222 222 222 F T T <---
147932024-04-20T01:15:39.940Z 28 560-579 131 136 136 222 230 230 F T T <---
147942024-04-20T01:15:39.940Z 29 580-599 133 136 136 224 230 230 F F F <---
147952024-04-20T01:15:39.940ZMax gen: 136, Max flush: 230
147962024-04-20T01:15:39.940ZError: Difference in extent metadata found!
147972024-04-20T01:15:39.940ZA Difference in extent metadata is expected here
147982024-04-20T01:15:39.940ZOn loop 68
147992024-04-20T01:15:39.940Z
148002024-04-20T01:15:39.944ZVerifying data now
148012024-04-20T01:15:44.420ZNo change after verify
148022024-04-20T01:15:44.421ZLoop: 68 Downstairs dump after verify (and repair):
148032024-04-20T01:15:44.435ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
148042024-04-20T01:15:44.435Z 0 000-019 126 126 126 217 217 217 F F F
148052024-04-20T01:15:44.435Z 1 020-039 137 137 137 231 231 231 F F F
148062024-04-20T01:15:44.435Z 2 040-059 132 132 132 223 223 223 F F F
148072024-04-20T01:15:44.435Z 3 060-079 136 136 136 228 228 228 F F F
148082024-04-20T01:15:44.435Z 4 080-099 134 134 134 225 225 225 F F F
148092024-04-20T01:15:44.435Z 5 100-119 134 134 134 225 225 225 F F F
148102024-04-20T01:15:44.435Z 6 120-139 131 131 131 222 222 222 F F F
148112024-04-20T01:15:44.435Z 7 140-159 137 137 137 231 231 231 F F F
148122024-04-20T01:15:44.439Z 8 160-179 135 135 135 227 227 227 F F F
148132024-04-20T01:15:44.439Z 9 180-199 137 137 137 231 231 231 F F F
148142024-04-20T01:15:44.439Z 10 200-219 136 136 136 230 230 230 F F F
148152024-04-20T01:15:44.439Z 11 220-239 137 137 137 231 231 231 F F F
148162024-04-20T01:15:44.439Z 12 240-259 132 132 132 223 223 223 F F F
148172024-04-20T01:15:44.439Z 13 260-279 131 131 131 222 222 222 F F F
148182024-04-20T01:15:44.439Z 14 280-299 134 134 134 225 225 225 F F F
148192024-04-20T01:15:44.439Z 15 300-319 134 134 134 225 225 225 F F F
148202024-04-20T01:15:44.440Z 16 320-339 134 134 134 225 225 225 F F F
148212024-04-20T01:15:44.440Z 17 340-359 136 136 136 230 230 230 F F F
148222024-04-20T01:15:44.440Z 18 360-379 135 135 135 227 227 227 F F F
148232024-04-20T01:15:44.440Z 19 380-399 137 137 137 231 231 231 F F F
148242024-04-20T01:15:44.440Z 20 400-419 134 134 134 226 226 226 F F F
148252024-04-20T01:15:44.440Z 21 420-439 132 132 132 223 223 223 F F F
148262024-04-20T01:15:44.440Z 22 440-459 128 128 128 218 218 218 F F F
148272024-04-20T01:15:44.440Z 23 460-479 134 134 134 225 225 225 F F F
148282024-04-20T01:15:44.440Z 24 480-499 134 134 134 225 225 225 F F F
148292024-04-20T01:15:44.440Z 25 500-519 131 131 131 222 222 222 F F F
148302024-04-20T01:15:44.440Z 26 520-539 129 129 129 219 219 219 F F F
148312024-04-20T01:15:44.440Z 27 540-559 137 137 137 231 231 231 F F F
148322024-04-20T01:15:44.440Z 28 560-579 137 137 137 231 231 231 F F F
148332024-04-20T01:15:44.440Z 29 580-599 136 136 136 230 230 230 F F F
148342024-04-20T01:15:44.440ZMax gen: 137, Max flush: 231
148352024-04-20T01:15:44.440Z
148362024-04-20T01:15:44.440ZBegin loop 69
148372024-04-20T01:15:44.440ZDownstairs to restart: 2
148382024-04-20T01:15:44.460Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.957730721Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723}
148392024-04-20T01:15:44.460Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958023552Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723}
148402024-04-20T01:15:44.460Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958052742Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723}
148412024-04-20T01:15:44.466Z{"msg":"Crucible 2c405c9f-b9c9-4973-80bb-7a19f8018beb has session id: 296e2c96-6c0d-4778-bb72-a968322c0fa6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958070272Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148422024-04-20T01:15:44.467Z{"msg":"Upstairs opts: Upstairs UUID: 2c405c9f-b9c9-4973-80bb-7a19f8018beb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":Crucible runtime is spawned
148432024-04-20T01:15:44.467Z30,"time":"2024-04-20T01:15:42.958182192Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148442024-04-20T01:15:44.467Z{"msg":"Crucible stats registered with UUID: 2c405c9f-b9c9-4973-80bb-7a19f8018beb","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958246253Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148452024-04-20T01:15:44.467Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958284513Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723}
148462024-04-20T01:15:44.467Z{"msg":"2c405c9f-b9c9-4973-80bb-7a19f8018beb active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958309033Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148472024-04-20T01:15:44.467Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958346503Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"0","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148482024-04-20T01:15:44.467Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958389773Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"0","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148492024-04-20T01:15:44.467Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958432703Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"1","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148502024-04-20T01:15:44.467Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958463443Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"1","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148512024-04-20T01:15:44.467Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958510323Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"2","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148522024-04-20T01:15:44.467Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958555743Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"2","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148532024-04-20T01:15:44.467Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958590243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148542024-04-20T01:15:44.467Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958624953Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148552024-04-20T01:15:44.467Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958669274Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148562024-04-20T01:15:44.467Z{"msg":"ds_connection connected from Ok(127.0.0.1:58964)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958693394Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148572024-04-20T01:15:44.467Z{"msg":"ds_connection connected from Ok(127.0.0.1:42298)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958726834Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148582024-04-20T01:15:44.467Z{"msg":"ds_connection connected from Ok(127.0.0.1:65387)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.958765604Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148592024-04-20T01:15:44.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.959327055Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"1","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148602024-04-20T01:15:44.467Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.959374735Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"1","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148612024-04-20T01:15:44.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.959433645Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"0","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148622024-04-20T01:15:44.467Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.959461835Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"0","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148632024-04-20T01:15:44.468Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.959830446Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"0","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148642024-04-20T01:15:44.468Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.959971136Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"2","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148652024-04-20T01:15:44.468Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.960006286Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"2","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148662024-04-20T01:15:44.468Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.960032086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"1","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148672024-04-20T01:15:44.468Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.960377267Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"0","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148682024-04-20T01:15:44.468Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.960435027Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"0","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148692024-04-20T01:15:44.468Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.960480307Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148702024-04-20T01:15:44.468Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.960543888Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"1","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148712024-04-20T01:15:44.468Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.960563548Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"1","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148722024-04-20T01:15:44.468Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.960577198Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148732024-04-20T01:15:44.468Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.960659658Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"2","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148742024-04-20T01:15:44.468Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961033679Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"2","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148752024-04-20T01:15:44.468Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961067269Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"2","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148762024-04-20T01:15:44.468Z{"msg":"[0]R flush_numbers[0..12]: [217, 231, 223, 228, 225, 225, 222, 231, 227, 231, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961082229Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148772024-04-20T01:15:44.468Z{"msg":"[0]R generation[0..12]: [126, 137, 132, 136, 134, 134, 131, 137, 135, 137, 136, 137]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961104129Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148782024-04-20T01:15:44.468Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961118869Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148792024-04-20T01:15:44.468Z{"msg":"[1]R flush_numbers[0..12]: [217, 231, 223, 228, 225, 225, 222, 231, 227, 231, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961139199Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148802024-04-20T01:15:44.468Z{"msg":"[1]R generation[0..12]: [126, 137, 132, 136, 134, 134, 131, 137, 135, 137, 136, 137]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961153289Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148812024-04-20T01:15:44.468Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961174759Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148822024-04-20T01:15:44.468Z{"msg":"[2]R flush_numbers[0..12]: [217, 231, 223, 228, 225, 225, 222, 231, 227, 231, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961189169Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148832024-04-20T01:15:44.468Z{"msg":"[2]R generation[0..12]: [126, 137, 132, 136, 134, 134, 131, 137, 135, 137, 136, 137]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
148842024-04-20T01:15:44.468Zlevel":30,"time":"2024-04-20T01:15:42.961208569Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148852024-04-20T01:15:44.468Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961223389Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148862024-04-20T01:15:44.468Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961236779Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148872024-04-20T01:15:44.469Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961249859Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148882024-04-20T01:15:44.469Z{"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961269929Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148892024-04-20T01:15:44.469Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961283239Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"":"downstairs","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148902024-04-20T01:15:44.469Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961296859Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148912024-04-20T01:15:44.469Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961315639Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"0","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148922024-04-20T01:15:44.469Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961329019Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"1","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148932024-04-20T01:15:44.469Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961341989Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"client":"2","":"downstairs","session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148942024-04-20T01:15:44.469Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.96136214Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148952024-04-20T01:15:44.469Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961374469Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148962024-04-20T01:15:44.469Z{"msg":"2c405c9f-b9c9-4973-80bb-7a19f8018beb is now active with session: 296e2c96-6c0d-4778-bb72-a968322c0fa6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.96138722Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148972024-04-20T01:15:44.469Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.961406709Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723,"session_id":"296e2c96-6c0d-4778-bb72-a968322c0fa6"}
148982024-04-20T01:15:44.469Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:42.9614195Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723}
148992024-04-20T01:15:44.469ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
149002024-04-20T01:15:44.469ZLoading write count information from file "/tmp/test_repair_verify.data"
149012024-04-20T01:15:44.469ZRun Repair workload
149022024-04-20T01:15:44.469Z01/30 Write block 459 len 24576 data: 11 13 16 15 17 14
149032024-04-20T01:15:44.473Z02/30 Write block 465 len 32768 data: 14 17 17 14 16 16 14 14
149042024-04-20T01:15:44.473Z03/30 Write block 519 len 32768 data: 15 16 16 15 15 13 12 11
149052024-04-20T01:15:44.473Z04/30 Write block 14 len 28672 data: 7 10 11 9 9 9 10
149062024-04-20T01:15:44.473Z05/30 Write block 412 len 24576 data: 15 16 15 15 17 13
149072024-04-20T01:15:44.473Z06/30 Read block 265 len 20480
149082024-04-20T01:15:44.473Z07/30 Read block 405 len 40960
149092024-04-20T01:15:44.473Z08/30 Read block 311 len 4096
149102024-04-20T01:15:44.473Z09/30 Read block 115 len 4096
149112024-04-20T01:15:44.473Z10/30 Read block 374 len 16384
149122024-04-20T01:15:44.474Z11/30 Read block 593 len 8192
149132024-04-20T01:15:44.477Z12/30 Write block 479 len 32768 data: 6 9 14 14 13 12 12 11
149142024-04-20T01:15:44.477Z13/30 Flush
149152024-04-20T01:15:44.481Z14/30 Write block 366 len 20480 data: 12 16 13 13 15
149162024-04-20T01:15:44.481Z15/30 Read block 302 len 28672
149172024-04-20T01:15:44.481Z16/30 Read block 37 len 12288
149182024-04-20T01:15:44.481Z17/30 Read block 521 len 36864
149192024-04-20T01:15:44.484Z18/30 Read block 233 len 12288
149202024-04-20T01:15:44.484Z19/30 Write block 373 len 4096 data: 17
149212024-04-20T01:15:44.484Z20/30 Read block 278 len 12288
149222024-04-20T01:15:44.485Z21/30 Write block 241 len 24576 data: 9 6 5 10 12 11
149232024-04-20T01:15:44.488Z22/30 Write block 160 len 20480 data: 13 15 13 12 12
149242024-04-20T01:15:44.488Z23/30 Read block 542 len 20480
149252024-04-20T01:15:44.488Z24/30 Write block 501 len 40960 data: 10 10 13 12 10 11 8 8 12 12
149262024-04-20T01:15:44.488Z25/30 Read block 361 len 40960
149272024-04-20T01:15:44.492Z26/30 Read block 566 len 12288
149282024-04-20T01:15:44.492Z27/30 Read block 434 len 8192
149292024-04-20T01:15:44.492Z28/30 Read block 566 len 28672
149302024-04-20T01:15:44.492Z29/30 Read block 182 len 12288
149312024-04-20T01:15:44.493Z30/30 Read block 258 len 28672
149322024-04-20T01:15:44.493Z----------------------------------------------------------------
149332024-04-20T01:15:44.493Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
149342024-04-20T01:15:44.493ZStates: Active Active Active
149352024-04-20T01:15:44.493ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
149362024-04-20T01:15:44.493Z 1 Acked 1000 Write 6 Done Done Done false
149372024-04-20T01:15:44.493Z 2 Acked 1001 Write 8 Done Done Done false
149382024-04-20T01:15:44.493Z 3 Acked 1002 Write 8 Done Done Done false
149392024-04-20T01:15:44.493Z 4 Acked 1003 Write 7 Done Done Done false
149402024-04-20T01:15:44.493Z 5 Acked 1004 Write 6 Done Done Sent false
149412024-04-20T01:15:44.493Z 6 Acked 1005 Read 5 Done Done Sent false
149422024-04-20T01:15:44.493Z 7 Acked 1006 Read 10 Done Done Sent false
149432024-04-20T01:15:44.493Z 8 Acked 1007 Read 1 Done Done Sent false
149442024-04-20T01:15:44.493Z 9 Acked 1008 Read 1 Done Done Sent false
149452024-04-20T01:15:44.493Z 10 Acked 1009 Read 4 Done Done Sent false
149462024-04-20T01:15:44.493Z 11 Acked 1010 Read 2 Done Done Sent false
149472024-04-20T01:15:44.493Z 12 Acked 1011 Write 8 Done Done Sent false
149482024-04-20T01:15:44.493Z 13 Acked 1012 Flush 0 Done Done Sent false
149492024-04-20T01:15:44.493Z 14 Acked 1013 Write 5 Done Done Sent false
149502024-04-20T01:15:44.493Z 15 Acked 1014 Read 7 Done Done Sent false
149512024-04-20T01:15:44.493Z 16 Acked 1015 Read 3 Done Done Sent false
149522024-04-20T01:15:44.493Z 17 Acked 1016 Read 9 Done Done Sent false
149532024-04-20T01:15:44.493Z 18 Acked 1017 Read 3 Done Done Sent false
149542024-04-20T01:15:44.493Z 19 Acked 1018 Write 1 Done Done Sent false
149552024-04-20T01:15:44.494Z 20 Acked 1019 Read 3 Done Done Sent false
149562024-04-20T01:15:44.494Z 21 Acked 1020 Write 6 Done Done Sent false
149572024-04-20T01:15:44.494Z 22 Acked 1021 Write 5 Done Done Sent false
149582024-04-20T01:15:44.494Z 23 Acked 1022 Read 5 Done Done Sent false
149592024-04-20T01:15:44.494Z 24 Acked 1023 Write 10 Done Done Sent false
149602024-04-20T01:15:44.494Z 25 Acked 1024 Read 10 Done Done Sent false
149612024-04-20T01:15:44.494Z 26 Acked 1025 Read 3 Done Done Sent false
149622024-04-20T01:15:44.494Z 27 Acked 1026 Read 2 Done Done Sent false
149632024-04-20T01:15:44.494Z 28 Acked 1027 Read 7 Done Done Sent false
149642024-04-20T01:15:44.494Z 29 Acked 1028 Read 3 Done Done Sent false
149652024-04-20T01:15:44.494Z 30 Acked 1029 Read 7 Done Sent Sent false
149662024-04-20T01:15:44.494Z STATES DS:0 DS:1 DS:2 TOTAL
149672024-04-20T01:15:44.494Z New 0 0 0 0
149682024-04-20T01:15:44.494Z Sent 0 1 26 27
149692024-04-20T01:15:44.494Z Done 30 29 4 63
149702024-04-20T01:15:44.494Z Skipped 0 0 0 0
149712024-04-20T01:15:44.494Z Error 0 0 0 0
149722024-04-20T01:15:44.494ZLast Flush: 1012 1012 0
149732024-04-20T01:15:44.494ZDownstairs last five completed:
149742024-04-20T01:15:44.494ZUpstairs last five completed: 30 29 28 27 26
149752024-04-20T01:15:44.494Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:15:42.991468527Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1723}
149762024-04-20T01:15:44.498ZWrote out file "/tmp/test_repair_verify.data"
149772024-04-20T01:15:44.498Z
149782024-04-20T01:15:44.501ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
149792024-04-20T01:15:44.517ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149802024-04-20T01:15:44.517Z 0 000-019 138 138 126 232 232 217 F F T <---
149812024-04-20T01:15:44.517Z 1 020-039 138 138 137 232 232 231 F F T <---
149822024-04-20T01:15:44.517Z 2 040-059 132 132 132 223 223 223 F F F
149832024-04-20T01:15:44.517Z 3 060-079 136 136 136 228 228 228 F F F
149842024-04-20T01:15:44.517Z 4 080-099 134 134 134 225 225 225 F F F
149852024-04-20T01:15:44.517Z 5 100-119 134 134 134 225 225 225 F F F
149862024-04-20T01:15:44.517Z 6 120-139 131 131 131 222 222 222 F F F
149872024-04-20T01:15:44.517Z 7 140-159 137 137 137 231 231 231 F F F
149882024-04-20T01:15:44.517Z 8 160-179 135 135 135 227 227 227 T T F <---
149892024-04-20T01:15:44.517Z 9 180-199 137 137 137 231 231 231 F F F
149902024-04-20T01:15:44.521Z 10 200-219 136 136 136 230 230 230 F F F
149912024-04-20T01:15:44.521Z 11 220-239 137 137 137 231 231 231 F F F
149922024-04-20T01:15:44.521Z 12 240-259 132 132 132 223 223 223 T T F <---
149932024-04-20T01:15:44.521Z 13 260-279 131 131 131 222 222 222 F F F
149942024-04-20T01:15:44.521Z 14 280-299 134 134 134 225 225 225 F F F
149952024-04-20T01:15:44.521Z 15 300-319 134 134 134 225 225 225 F F F
149962024-04-20T01:15:44.521Z 16 320-339 134 134 134 225 225 225 F F F
149972024-04-20T01:15:44.521Z 17 340-359 136 136 136 230 230 230 F F F
149982024-04-20T01:15:44.521Z 18 360-379 135 135 135 227 227 227 T T F <---
149992024-04-20T01:15:44.522Z 19 380-399 137 137 137 231 231 231 F F F
150002024-04-20T01:15:44.522Z 20 400-419 138 138 134 232 232 226 F F F <---
150012024-04-20T01:15:44.522Z 21 420-439 132 132 132 223 223 223 F F F
150022024-04-20T01:15:44.522Z 22 440-459 138 138 128 232 232 218 F F T <---
150032024-04-20T01:15:44.522Z 23 460-479 138 138 134 232 232 225 F F T <---
150042024-04-20T01:15:44.522Z 24 480-499 138 138 134 232 232 225 F F F <---
150052024-04-20T01:15:44.522Z 25 500-519 138 138 131 232 232 222 T T T <---
150062024-04-20T01:15:44.522Z 26 520-539 138 138 129 232 232 219 F F T <---
150072024-04-20T01:15:44.522Z 27 540-559 137 137 137 231 231 231 F F F
150082024-04-20T01:15:44.522Z 28 560-579 137 137 137 231 231 231 F F F
150092024-04-20T01:15:44.522Z 29 580-599 136 136 136 230 230 230 F F F
150102024-04-20T01:15:44.522ZMax gen: 138, Max flush: 232
150112024-04-20T01:15:44.522ZError: Difference in extent metadata found!
150122024-04-20T01:15:44.522ZA Difference in extent metadata is expected here
150132024-04-20T01:15:44.522ZOn loop 69
150142024-04-20T01:15:44.522Z
150152024-04-20T01:15:44.526ZVerifying data now
150162024-04-20T01:15:59.039ZNo change after verify
150172024-04-20T01:15:59.039ZLoop: 69 Downstairs dump after verify (and repair):
150182024-04-20T01:15:59.053ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150192024-04-20T01:15:59.053Z 0 000-019 138 138 138 232 232 232 F F F
150202024-04-20T01:15:59.053Z 1 020-039 138 138 138 232 232 232 F F F
150212024-04-20T01:15:59.053Z 2 040-059 132 132 132 223 223 223 F F F
150222024-04-20T01:15:59.053Z 3 060-079 136 136 136 228 228 228 F F F
150232024-04-20T01:15:59.053Z 4 080-099 134 134 134 225 225 225 F F F
150242024-04-20T01:15:59.053Z 5 100-119 134 134 134 225 225 225 F F F
150252024-04-20T01:15:59.057Z 6 120-139 131 131 131 222 222 222 F F F
150262024-04-20T01:15:59.057Z 7 140-159 137 137 137 231 231 231 F F F
150272024-04-20T01:15:59.057Z 8 160-179 139 139 139 233 233 233 F F F
150282024-04-20T01:15:59.057Z 9 180-199 137 137 137 231 231 231 F F F
150292024-04-20T01:15:59.057Z 10 200-219 136 136 136 230 230 230 F F F
150302024-04-20T01:15:59.057Z 11 220-239 137 137 137 231 231 231 F F F
150312024-04-20T01:15:59.057Z 12 240-259 139 139 139 233 233 233 F F F
150322024-04-20T01:15:59.057Z 13 260-279 131 131 131 222 222 222 F F F
150332024-04-20T01:15:59.057Z 14 280-299 134 134 134 225 225 225 F F F
150342024-04-20T01:15:59.057Z 15 300-319 134 134 134 225 225 225 F F F
150352024-04-20T01:15:59.057Z 16 320-339 134 134 134 225 225 225 F F F
150362024-04-20T01:15:59.058Z 17 340-359 136 136 136 230 230 230 F F F
150372024-04-20T01:15:59.058Z 18 360-379 139 139 139 233 233 233 F F F
150382024-04-20T01:15:59.058Z 19 380-399 137 137 137 231 231 231 F F F
150392024-04-20T01:15:59.058Z 20 400-419 138 138 138 232 232 232 F F F
150402024-04-20T01:15:59.058Z 21 420-439 132 132 132 223 223 223 F F F
150412024-04-20T01:15:59.058Z 22 440-459 138 138 138 232 232 232 F F F
150422024-04-20T01:15:59.058Z 23 460-479 138 138 138 232 232 232 F F F
150432024-04-20T01:15:59.058Z 24 480-499 138 138 138 232 232 232 F F F
150442024-04-20T01:15:59.058Z 25 500-519 139 139 139 233 233 233 F F F
150452024-04-20T01:15:59.058Z 26 520-539 138 138 138 232 232 232 F F F
150462024-04-20T01:15:59.058Z 27 540-559 137 137 137 231 231 231 F F F
150472024-04-20T01:15:59.058Z 28 560-579 137 137 137 231 231 231 F F F
150482024-04-20T01:15:59.058Z 29 580-599 136 136 136 230 230 230 F F F
150492024-04-20T01:15:59.058ZMax gen: 139, Max flush: 233
150502024-04-20T01:15:59.058Z
150512024-04-20T01:15:59.058ZBegin loop 70
150522024-04-20T01:15:59.058ZDownstairs to restart: 0
150532024-04-20T01:15:59.076Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.574870729Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731}
150542024-04-20T01:15:59.076Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.5751326Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731}
150552024-04-20T01:15:59.082Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.57516973Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731}
150562024-04-20T01:15:59.082Z{"msg":"Crucible 7a573175-190e-4ca6-8e20-50f6557aaa40 has session id: 05914d15-f205-442d-a814-103eb110c733","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.57520458Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"session_id":"05914d15-f205-442d-a814-103eb110c733"}
150572024-04-20T01:15:59.082Z{"msg":"Upstairs opts: Upstairs UUID: 7a573175-190e-4ca6-8e20-50f6557aaa40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.57524928Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"session_id":"05914d15-f205-442d-a814-103eb110c733"}
150582024-04-20T01:15:59.082Z{"msg":"Crucible stats registered with UUID: 7a573175-190e-4ca6-8e20-50f6557aaa40","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.57528548Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"session_id":"05914d15-f205-442d-a814-103eb110c733"Crucible runtime is spawned
150592024-04-20T01:15:59.082Z}
150602024-04-20T01:15:59.082Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.575416Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731}
150612024-04-20T01:15:59.082Z{"msg":"7a573175-190e-4ca6-8e20-50f6557aaa40 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.57545725Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"session_id":"05914d15-f205-442d-a814-103eb110c733"}
150622024-04-20T01:15:59.082Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.575485271Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"0","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150632024-04-20T01:15:59.082Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.575505931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"0","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150642024-04-20T01:15:59.082Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.57554104Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"1","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150652024-04-20T01:15:59.082Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.575566821Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"1","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150662024-04-20T01:15:59.082Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.575581571Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"2","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150672024-04-20T01:15:59.082Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.575602491Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"2","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150682024-04-20T01:15:59.082Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.575616021Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150692024-04-20T01:15:59.082Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.575632021Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150702024-04-20T01:15:59.082Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.575652341Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150712024-04-20T01:15:59.082Z{"msg":"ds_connection connected from Ok(127.0.0.1:51634)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.575942472Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150722024-04-20T01:15:59.083Z{"msg":"ds_connection connected from Ok(127.0.0.1:35046)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.576067152Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150732024-04-20T01:15:59.083Z{"msg":"ds_connection connected from Ok(127.0.0.1:41873)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.576122972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150742024-04-20T01:15:59.083Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.576554583Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"1","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150752024-04-20T01:15:59.083Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.576594893Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"1","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150762024-04-20T01:15:59.083Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.576610183Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"2","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150772024-04-20T01:15:59.083Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.576645473Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"2","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150782024-04-20T01:15:59.083Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.576721273Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"0","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150792024-04-20T01:15:59.083Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.576775503Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"0","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150802024-04-20T01:15:59.083Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.577192944Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"1","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150812024-04-20T01:15:59.083Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.577221104Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"2","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150822024-04-20T01:15:59.083Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.577237384Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"0","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150832024-04-20T01:15:59.083Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.577736776Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"1","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150842024-04-20T01:15:59.083Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.577773736Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"1","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150852024-04-20T01:15:59.083Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.577791196Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"session_id":"05914d15-f205-442d-a814-103eb110c733"}
150862024-04-20T01:15:59.083Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.577823026Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"2","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150872024-04-20T01:15:59.083Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.577855646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"2","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150882024-04-20T01:15:59.083Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.577888196Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"session_id":"05914d15-f205-442d-a814-103eb110c733"}
150892024-04-20T01:15:59.083Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.577917786Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"0","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150902024-04-20T01:15:59.083Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.577950856Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"0","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150912024-04-20T01:15:59.083Z{"msg":"[0]R flush_numbers[0..12]: [232, 232, 223, 228, 225, 225, 222, 231, 233, 231, 230, 231]","Wait for a query_work_queue command to finish before sending IO
150922024-04-20T01:15:59.083Zv":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.577985316Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150932024-04-20T01:15:59.083Z{"msg":"[0]R generation[0..12]: [138, 138, 132, 136, 134, 134, 131, 137, 139, 137, 136, 137]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578010776Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150942024-04-20T01:15:59.084Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578034736Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150952024-04-20T01:15:59.084Z{"msg":"[1]R flush_numbers[0..12]: [232, 232, 223, 228, 225, 225, 222, 231, 233, 231, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578062016Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150962024-04-20T01:15:59.084Z{"msg":"[1]R generation[0..12]: [138, 138, 132, 136, 134, 134, 131, 137, 139, 137, 136, 137]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578077566Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150972024-04-20T01:15:59.084Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578092166Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150982024-04-20T01:15:59.084Z{"msg":"[2]R flush_numbers[0..12]: [232, 232, 223, 228, 225, 225, 222, 231, 233, 231, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578119656Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
150992024-04-20T01:15:59.084Z{"msg":"[2]R generation[0..12]: [138, 138, 132, 136, 134, 134, 131, 137, 139, 137, 136, 137]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578139486Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
151002024-04-20T01:15:59.084Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578162566Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
151012024-04-20T01:15:59.084Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578176766Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
151022024-04-20T01:15:59.084Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578191277Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
151032024-04-20T01:15:59.084Z{"msg":"Next flush: 234","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578205017Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
151042024-04-20T01:15:59.084Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578218997Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"":"downstairs","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
151052024-04-20T01:15:59.084Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578232917Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"session_id":"05914d15-f205-442d-a814-103eb110c733"}
151062024-04-20T01:15:59.084Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578258137Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"0","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
151072024-04-20T01:15:59.084Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578273367Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"1","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
151082024-04-20T01:15:59.084Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578295047Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"client":"2","":"downstairs","session_id":"05914d15-f205-442d-a814-103eb110c733"}
151092024-04-20T01:15:59.084ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
151102024-04-20T01:15:59.084Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578315647Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"session_id":"05914d15-f205-442d-a814-103eb110c733"}
151112024-04-20T01:15:59.084Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578336207Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"session_id":"05914d15-f205-442d-a814-103eb110c733"}
151122024-04-20T01:15:59.084Z{"msg":"7a573175-190e-4ca6-8e20-50f6557aaa40 is now active with session: 05914d15-f205-442d-a814-103eb110c733","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578349837Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"session_id":"05914d15-f205-442d-a814-103eb110c733"}
151132024-04-20T01:15:59.084Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578362337Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731,"session_id":"05914d15-f205-442d-a814-103eb110c733"}
151142024-04-20T01:15:59.084Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:15:57.578382687Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731}
151152024-04-20T01:15:59.084ZLoading write count information from file "/tmp/test_repair_verify.data"
151162024-04-20T01:15:59.085ZRun Repair workload
151172024-04-20T01:15:59.089Z01/30 Write block 414 len 16384 data: 16 16 18 14
151182024-04-20T01:15:59.090Z02/30 Write block 84 len 40960 data: 10 10 13 13 13 13 14 12 9 6
151192024-04-20T01:15:59.090Z03/30 Read block 214 len 20480
151202024-04-20T01:15:59.090Z04/30 Read block 375 len 32768
151212024-04-20T01:15:59.090Z05/30 Write block 97 len 8192 data: 6 7
151222024-04-20T01:15:59.090Z06/30 Read block 556 len 32768
151232024-04-20T01:15:59.090Z07/30 Read block 114 len 20480
151242024-04-20T01:15:59.090Z08/30 Read block 510 len 12288
151252024-04-20T01:15:59.090Z09/30 Read block 148 len 12288
151262024-04-20T01:15:59.090Z10/30 Read block 295 len 16384
151272024-04-20T01:15:59.090Z11/30 Write block 360 len 8192 data: 10 10
151282024-04-20T01:15:59.094Z12/30 Write block 87 len 40960 data: 14 14 14 15 13 10 7 7 5 5
151292024-04-20T01:15:59.094Z13/30 Read block 358 len 36864
151302024-04-20T01:15:59.094Z14/30 Read block 333 len 32768
151312024-04-20T01:15:59.094Z15/30 Read block 12 len 40960
151322024-04-20T01:15:59.094Z16/30 Read block 523 len 40960
151332024-04-20T01:15:59.097Z17/30 Write block 77 len 40960 data: 9 10 10 9 9 8 11 11 11 14
151342024-04-20T01:15:59.097Z18/30 Write block 372 len 36864 data: 16 18 17 16 11 13 14 14 15
151352024-04-20T01:15:59.097Z19/30 Read block 434 len 12288
151362024-04-20T01:15:59.100Z20/30 Read block 212 len 20480
151372024-04-20T01:15:59.100Z21/30 Read block 204 len 4096
151382024-04-20T01:15:59.100Z22/30 Read block 64 len 12288
151392024-04-20T01:15:59.101Z23/30 Read block 23 len 20480
151402024-04-20T01:15:59.101Z24/30 Read block 74 len 16384
151412024-04-20T01:15:59.104Z25/30 Read block 312 len 40960
151422024-04-20T01:15:59.104Z26/30 Read block 408 len 32768
151432024-04-20T01:15:59.104Z27/30 Write block 67 len 36864 data: 13 15 16 10 10 8 10 12 8
151442024-04-20T01:15:59.108Z28/30 Write block 546 len 12288 data: 12 8 10
151452024-04-20T01:15:59.108Z29/30 Read block 521 len 40960
151462024-04-20T01:15:59.108Z30/30 Read block 430 len 40960
151472024-04-20T01:15:59.108Z----------------------------------------------------------------
151482024-04-20T01:15:59.108Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
151492024-04-20T01:15:59.108ZStates: Active Active Active
151502024-04-20T01:15:59.108ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151512024-04-20T01:15:59.108Z 1 Acked 1000 Write 4 Done Done Done false
151522024-04-20T01:15:59.108Z 2 Acked 1001 Write 10 Sent Done Done false
151532024-04-20T01:15:59.108Z 3 Acked 1002 Read 5 Sent Done Done false
151542024-04-20T01:15:59.108Z 4 Acked 1003 Read 8 Sent Done Done false
151552024-04-20T01:15:59.108Z 5 Acked 1004 Write 2 Sent Done Done false
151562024-04-20T01:15:59.108Z 6 Acked 1005 Read 8 Sent Done Done false
151572024-04-20T01:15:59.108Z 7 Acked 1006 Read 5 Sent Done Done false
151582024-04-20T01:15:59.108Z 8 Acked 1007 Read 3 Sent Done Done false
151592024-04-20T01:15:59.108Z 9 Acked 1008 Read 3 Sent Done Done false
151602024-04-20T01:15:59.108Z 10 Acked 1009 Read 4 Sent Done Done false
151612024-04-20T01:15:59.108Z 11 Acked 1010 Write 2 Sent Done Done false
151622024-04-20T01:15:59.108Z 12 Acked 1011 Write 10 Sent Done Done false
151632024-04-20T01:15:59.109Z 13 Acked 1012 Read 9 Sent Done Done false
151642024-04-20T01:15:59.109Z 14 Acked 1013 Read 8 Sent Done Done false
151652024-04-20T01:15:59.109Z 15 Acked 1014 Read 10 Sent Done Done false
151662024-04-20T01:15:59.109Z 16 Acked 1015 Read 10 Sent Done Done false
151672024-04-20T01:15:59.109Z 17 Acked 1016 Write 10 Sent Done Done false
151682024-04-20T01:15:59.109Z 18 Acked 1017 Write 9 Sent Done Done false
151692024-04-20T01:15:59.109Z 19 Acked 1018 Read 3 Sent Done Done false
151702024-04-20T01:15:59.109Z 20 Acked 1019 Read 5 Sent Done Done false
151712024-04-20T01:15:59.109Z 21 Acked 1020 Read 1 Sent Done Done false
151722024-04-20T01:15:59.109Z 22 Acked 1021 Read 3 Sent Done Done false
151732024-04-20T01:15:59.109Z 23 Acked 1022 Read 5 Sent Done Done false
151742024-04-20T01:15:59.109Z 24 Acked 1023 Read 4 Sent Done Done false
151752024-04-20T01:15:59.109Z 25 Acked 1024 Read 10 Sent Done Done false
151762024-04-20T01:15:59.109Z 26 Acked 1025 Read 8 Sent Done Done false
151772024-04-20T01:15:59.109Z 27 Acked 1026 Write 9 Sent Done Done false
151782024-04-20T01:15:59.109Z 28 Acked 1027 Write 3 Sent Done Done false
151792024-04-20T01:15:59.109Z 29 Acked 1028 Read 10 Sent Done Done false
151802024-04-20T01:15:59.109Z 30 Acked 1029 Read 10 Sent Done Sent false
151812024-04-20T01:15:59.109Z STATES DS:0 DS:1 DS:2 TOTAL
151822024-04-20T01:15:59.109Z New 0 0 0 0
151832024-04-20T01:15:59.109Z Sent 29 0 1 30
151842024-04-20T01:15:59.109Z Done 1 30 29 60
151852024-04-20T01:15:59.110Z Skipped 0 0 0 0
151862024-04-20T01:15:59.110Z Error 0 0 0 0
151872024-04-20T01:15:59.110ZLast Flush: 0 0 0
151882024-04-20T01:15:59.110ZDownstairs last five completed:
151892024-04-20T01:15:59.110ZUpstairs last five completed: 30 29 28 27 26
151902024-04-20T01:15:59.110Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:15:57.605428907Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1731}
151912024-04-20T01:15:59.110ZWrote out file "/tmp/test_repair_verify.data"
151922024-04-20T01:15:59.113Z
151932024-04-20T01:15:59.113ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151942024-04-20T01:15:59.130ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151952024-04-20T01:15:59.130Z 0 000-019 138 138 138 232 232 232 F F F
151962024-04-20T01:15:59.130Z 1 020-039 138 138 138 232 232 232 F F F
151972024-04-20T01:15:59.130Z 2 040-059 132 132 132 223 223 223 F F F
151982024-04-20T01:15:59.130Z 3 060-079 136 136 136 228 228 228 F T T <---
151992024-04-20T01:15:59.130Z 4 080-099 134 134 134 225 225 225 F T T <---
152002024-04-20T01:15:59.134Z 5 100-119 134 134 134 225 225 225 F F F
152012024-04-20T01:15:59.134Z 6 120-139 131 131 131 222 222 222 F F F
152022024-04-20T01:15:59.134Z 7 140-159 137 137 137 231 231 231 F F F
152032024-04-20T01:15:59.134Z 8 160-179 139 139 139 233 233 233 F F F
152042024-04-20T01:15:59.134Z 9 180-199 137 137 137 231 231 231 F F F
152052024-04-20T01:15:59.134Z 10 200-219 136 136 136 230 230 230 F F F
152062024-04-20T01:15:59.134Z 11 220-239 137 137 137 231 231 231 F F F
152072024-04-20T01:15:59.134Z 12 240-259 139 139 139 233 233 233 F F F
152082024-04-20T01:15:59.134Z 13 260-279 131 131 131 222 222 222 F F F
152092024-04-20T01:15:59.134Z 14 280-299 134 134 134 225 225 225 F F F
152102024-04-20T01:15:59.134Z 15 300-319 134 134 134 225 225 225 F F F
152112024-04-20T01:15:59.134Z 16 320-339 134 134 134 225 225 225 F F F
152122024-04-20T01:15:59.134Z 17 340-359 136 136 136 230 230 230 F F F
152132024-04-20T01:15:59.134Z 18 360-379 139 139 139 233 233 233 F T T <---
152142024-04-20T01:15:59.134Z 19 380-399 137 137 137 231 231 231 F T T <---
152152024-04-20T01:15:59.135Z 20 400-419 138 138 138 232 232 232 T T T
152162024-04-20T01:15:59.135Z 21 420-439 132 132 132 223 223 223 F F F
152172024-04-20T01:15:59.135Z 22 440-459 138 138 138 232 232 232 F F F
152182024-04-20T01:15:59.135Z 23 460-479 138 138 138 232 232 232 F F F
152192024-04-20T01:15:59.135Z 24 480-499 138 138 138 232 232 232 F F F
152202024-04-20T01:15:59.135Z 25 500-519 139 139 139 233 233 233 F F F
152212024-04-20T01:15:59.135Z 26 520-539 138 138 138 232 232 232 F F F
152222024-04-20T01:15:59.135Z 27 540-559 137 137 137 231 231 231 F T T <---
152232024-04-20T01:15:59.135Z 28 560-579 137 137 137 231 231 231 F F F
152242024-04-20T01:15:59.135Z 29 580-599 136 136 136 230 230 230 F F F
152252024-04-20T01:15:59.135ZMax gen: 139, Max flush: 233
152262024-04-20T01:15:59.135ZError: Difference in extent metadata found!
152272024-04-20T01:15:59.135ZA Difference in extent metadata is expected here
152282024-04-20T01:15:59.135ZOn loop 70
152292024-04-20T01:15:59.135Z
152302024-04-20T01:15:59.139ZVerifying data now
152312024-04-20T01:16:03.455ZNo change after verify
152322024-04-20T01:16:03.455ZLoop: 70 Downstairs dump after verify (and repair):
152332024-04-20T01:16:03.468ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152342024-04-20T01:16:03.468Z 0 000-019 138 138 138 232 232 232 F F F
152352024-04-20T01:16:03.472Z 1 020-039 138 138 138 232 232 232 F F F
152362024-04-20T01:16:03.472Z 2 040-059 132 132 132 223 223 223 F F F
152372024-04-20T01:16:03.472Z 3 060-079 140 140 140 234 234 234 F F F
152382024-04-20T01:16:03.472Z 4 080-099 140 140 140 234 234 234 F F F
152392024-04-20T01:16:03.472Z 5 100-119 134 134 134 225 225 225 F F F
152402024-04-20T01:16:03.472Z 6 120-139 131 131 131 222 222 222 F F F
152412024-04-20T01:16:03.472Z 7 140-159 137 137 137 231 231 231 F F F
152422024-04-20T01:16:03.472Z 8 160-179 139 139 139 233 233 233 F F F
152432024-04-20T01:16:03.473Z 9 180-199 137 137 137 231 231 231 F F F
152442024-04-20T01:16:03.473Z 10 200-219 136 136 136 230 230 230 F F F
152452024-04-20T01:16:03.473Z 11 220-239 137 137 137 231 231 231 F F F
152462024-04-20T01:16:03.473Z 12 240-259 139 139 139 233 233 233 F F F
152472024-04-20T01:16:03.473Z 13 260-279 131 131 131 222 222 222 F F F
152482024-04-20T01:16:03.473Z 14 280-299 134 134 134 225 225 225 F F F
152492024-04-20T01:16:03.473Z 15 300-319 134 134 134 225 225 225 F F F
152502024-04-20T01:16:03.473Z 16 320-339 134 134 134 225 225 225 F F F
152512024-04-20T01:16:03.473Z 17 340-359 136 136 136 230 230 230 F F F
152522024-04-20T01:16:03.473Z 18 360-379 140 140 140 234 234 234 F F F
152532024-04-20T01:16:03.473Z 19 380-399 140 140 140 234 234 234 F F F
152542024-04-20T01:16:03.473Z 20 400-419 140 140 140 234 234 234 F F F
152552024-04-20T01:16:03.473Z 21 420-439 132 132 132 223 223 223 F F F
152562024-04-20T01:16:03.473Z 22 440-459 138 138 138 232 232 232 F F F
152572024-04-20T01:16:03.473Z 23 460-479 138 138 138 232 232 232 F F F
152582024-04-20T01:16:03.473Z 24 480-499 138 138 138 232 232 232 F F F
152592024-04-20T01:16:03.473Z 25 500-519 139 139 139 233 233 233 F F F
152602024-04-20T01:16:03.473Z 26 520-539 138 138 138 232 232 232 F F F
152612024-04-20T01:16:03.473Z 27 540-559 140 140 140 234 234 234 F F F
152622024-04-20T01:16:03.473Z 28 560-579 137 137 137 231 231 231 F F F
152632024-04-20T01:16:03.473Z 29 580-599 136 136 136 230 230 230 F F F
152642024-04-20T01:16:03.473ZMax gen: 140, Max flush: 234
152652024-04-20T01:16:03.473Z
152662024-04-20T01:16:03.474ZBegin loop 71
152672024-04-20T01:16:03.474ZDownstairs to restart: 0
152682024-04-20T01:16:03.491Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989236246Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739}
152692024-04-20T01:16:03.497Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989564606Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739}
152702024-04-20T01:16:03.497ZCrucible runtime is spawned
152712024-04-20T01:16:03.497Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989624537Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739}
152722024-04-20T01:16:03.497Z{"msg":"Crucible f871314f-c3a0-4078-ad3e-be16ec51ff21 has session id: ee23ba5d-b946-44c7-a5c3-63c7c75702b0","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989667067Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152732024-04-20T01:16:03.497Z{"msg":"Upstairs opts: Upstairs UUID: f871314f-c3a0-4078-ad3e-be16ec51ff21, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989744667Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152742024-04-20T01:16:03.497Z{"msg":"Crucible stats registered with UUID: f871314f-c3a0-4078-ad3e-be16ec51ff21","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989791477Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152752024-04-20T01:16:03.497Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989809497Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739}
152762024-04-20T01:16:03.497Z{"msg":"f871314f-c3a0-4078-ad3e-be16ec51ff21 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989824517Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152772024-04-20T01:16:03.497Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989839767Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"0","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152782024-04-20T01:16:03.497Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989888227Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"0","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152792024-04-20T01:16:03.497Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989919987Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"1","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152802024-04-20T01:16:03.497Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989955217Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"1","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152812024-04-20T01:16:03.497Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.989991977Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"2","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152822024-04-20T01:16:03.497Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.990032447Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"2","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152832024-04-20T01:16:03.497Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.990069887Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152842024-04-20T01:16:03.497Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.990165588Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152852024-04-20T01:16:03.497Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.990204728Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152862024-04-20T01:16:03.497Z{"msg":"ds_connection connected from Ok(127.0.0.1:53633)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.990223238Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152872024-04-20T01:16:03.497Z{"msg":"ds_connection connected from Ok(127.0.0.1:61116)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.990276228Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152882024-04-20T01:16:03.497Z{"msg":"ds_connection connected from Ok(127.0.0.1:56779)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.990338918Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152892024-04-20T01:16:03.498Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.990915749Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"1","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152902024-04-20T01:16:03.498Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.9909575Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"1","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152912024-04-20T01:16:03.498Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.99103599Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"2","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152922024-04-20T01:16:03.498Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.99107485Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"2","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152932024-04-20T01:16:03.498Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.991501571Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"1","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152942024-04-20T01:16:03.498Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.991543931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"2","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152952024-04-20T01:16:03.498Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.991569411Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"0","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152962024-04-20T01:16:03.498Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.991605161Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"0","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152972024-04-20T01:16:03.498Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992112862Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"1","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152982024-04-20T01:16:03.498Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992155622Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"1","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
152992024-04-20T01:16:03.498Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992171482Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153002024-04-20T01:16:03.498Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992183942Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"2","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153012024-04-20T01:16:03.498Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992205472Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"2","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153022024-04-20T01:16:03.498Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992219792Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153032024-04-20T01:16:03.498Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992232592Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"0","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153042024-04-20T01:16:03.498Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992659253Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"0","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153052024-04-20T01:16:03.498Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992700364Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"0","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153062024-04-20T01:16:03.498Z{"msg":"[0]R flush_numbers[0..12]: [232, 232, 223, 234, 234, 225, 222, 231, 233, 231, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992717244Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153072024-04-20T01:16:03.498Z{"msg":"[0]R generation[0..12]: [138, 138, 132, 140, 140, 134, 131, 137, 139, 137, 136, 137]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992732653Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153082024-04-20T01:16:03.498Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992754144Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153092024-04-20T01:16:03.498Z{"msg":"[1]R flush_numbers[0..12]: [232, 232, 223, 234, 234, 225, 222, 231, 233, 231, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992768613Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153102024-04-20T01:16:03.498Z{"msg":"[1]R generation[0..12]: [138, 138, 132, 140, 140, 134, 131, 137, 139, 137, 136, 137]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992789264Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153112024-04-20T01:16:03.498Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992803854Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153122024-04-20T01:16:03.499Z{"msg":"[2]R flush_numbers[0..12]: [232, 232, 223, 234, 234, 225, 222, 231, 233, 231, 230, 231]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992828954Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153132024-04-20T01:16:03.499Z{"msg":"[2]R generation[0..12]: [138, 138, 132, 140, 140, 134, 131, 137, 139, 137, 136, 137]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
153142024-04-20T01:16:03.499Z,"time":"2024-04-20T01:16:01.992846484Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153152024-04-20T01:16:03.499Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992869524Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153162024-04-20T01:16:03.499Z{"msg":"Max found gen is 141","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992882894Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153172024-04-20T01:16:03.499Z{"msg":"Generation requested: 142 >= found:141","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992895984Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153182024-04-20T01:16:03.499Z{"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992916054Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153192024-04-20T01:16:03.499Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992929474Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"":"downstairs","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153202024-04-20T01:16:03.499Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992947444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153212024-04-20T01:16:03.499Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.992979604Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"0","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153222024-04-20T01:16:03.499Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.993004554Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"1","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153232024-04-20T01:16:03.499Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.993039154Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"client":"2","":"downstairs","session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153242024-04-20T01:16:03.499Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.993066764Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153252024-04-20T01:16:03.499Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.993100184Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
153262024-04-20T01:16:03.499Zw-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153272024-04-20T01:16:03.499Z{"msg":"f871314f-c3a0-4078-ad3e-be16ec51ff21 is now active with session: ee23ba5d-b946-44c7-a5c3-63c7c75702b0","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.993138785Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153282024-04-20T01:16:03.499Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.993165674Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739,"session_id":"ee23ba5d-b946-44c7-a5c3-63c7c75702b0"}
153292024-04-20T01:16:03.503Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:01.993178654Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739}
153302024-04-20T01:16:03.503ZLoading write count information from file "/tmp/test_repair_verify.data"
153312024-04-20T01:16:03.503ZRun Repair workload
153322024-04-20T01:16:03.503Z01/30 Write block 425 len 24576 data: 11 10 9 8 12 12
153332024-04-20T01:16:03.503Z02/30 Write block 430 len 8192 data: 13 10
153342024-04-20T01:16:03.503Z03/30 Write block 115 len 36864 data: 11 10 9 10 10 10 10 10 10
153352024-04-20T01:16:03.504Z04/30 Write block 588 len 4096 data: 16
153362024-04-20T01:16:03.504Z05/30 Write block 83 len 12288 data: 12 12 12
153372024-04-20T01:16:03.504Z06/30 Write block 185 len 40960 data: 8 9 9 12 11 14 16 12 11 12
153382024-04-20T01:16:03.504Z07/30 Read block 36 len 32768
153392024-04-20T01:16:03.507Z08/30 Write block 392 len 36864 data: 8 9 9 8 8 7 8 10 9
153402024-04-20T01:16:03.508Z09/30 Write block 265 len 20480 data: 4 6 5 4 7
153412024-04-20T01:16:03.508Z10/30 Read block 35 len 28672
153422024-04-20T01:16:03.508Z11/30 Read block 349 len 28672
153432024-04-20T01:16:03.508Z12/30 Read block 50 len 8192
153442024-04-20T01:16:03.511Z13/30 Write block 563 len 12288 data: 8 12 14
153452024-04-20T01:16:03.512Z14/30 Write block 535 len 12288 data: 11 9 8
153462024-04-20T01:16:03.512Z15/30 Flush
153472024-04-20T01:16:03.515Z16/30 Write block 193 len 40960 data: 12 13 11 10 12 13 15 10 11 13
153482024-04-20T01:16:03.515Z17/30 Read block 423 len 40960
153492024-04-20T01:16:03.518Z18/30 Read block 205 len 16384
153502024-04-20T01:16:03.518Z19/30 Read block 397 len 40960
153512024-04-20T01:16:03.518Z20/30 Read block 403 len 8192
153522024-04-20T01:16:03.518Z21/30 Flush
153532024-04-20T01:16:03.525Z22/30 Write block 104 len 16384 data: 11 10 12 11
153542024-04-20T01:16:03.525Z23/30 Write block 369 len 24576 data: 14 16 18 17 19 18
153552024-04-20T01:16:03.525Z24/30 Read block 261 len 12288
153562024-04-20T01:16:03.529Z25/30 Write block 510 len 40960 data: 13 14 15 14 12 11 15 15 13 16
153572024-04-20T01:16:03.529Z26/30 Read block 35 len 40960
153582024-04-20T01:16:03.529Z27/30 Write block 325 len 20480 data: 12 13 11 12 10
153592024-04-20T01:16:03.529Z28/30 Read block 316 len 28672
153602024-04-20T01:16:03.529Z29/30 Read block 323 len 16384
153612024-04-20T01:16:03.529Z30/30 Write block 219 len 28672 data: 11 13 11 12 13 13 12
153622024-04-20T01:16:03.529Z----------------------------------------------------------------
153632024-04-20T01:16:03.529Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
153642024-04-20T01:16:03.529ZStates: Active Active Active
153652024-04-20T01:16:03.529ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153662024-04-20T01:16:03.530Z 1 Acked 1000 Write 6 Done Done Done false
153672024-04-20T01:16:03.530Z 2 Acked 1001 Write 2 Done Done Done false
153682024-04-20T01:16:03.530Z 3 Acked 1002 Write 9 Done Done Done false
153692024-04-20T01:16:03.530Z 4 Acked 1003 Write 1 Done Done Done false
153702024-04-20T01:16:03.530Z 5 Acked 1004 Write 3 Done Done Done false
153712024-04-20T01:16:03.530Z 6 Acked 1005 Write 10 Done Done Done false
153722024-04-20T01:16:03.530Z 7 Acked 1006 Read 8 Done Done Done false
153732024-04-20T01:16:03.530Z 8 Acked 1007 Write 9 Sent Done Done false
153742024-04-20T01:16:03.530Z 9 Acked 1008 Write 5 Sent Done Done false
153752024-04-20T01:16:03.530Z 10 Acked 1009 Read 7 Sent Done Done false
153762024-04-20T01:16:03.530Z 11 Acked 1010 Read 7 Sent Done Done false
153772024-04-20T01:16:03.530Z 12 Acked 1011 Read 2 Sent Done Done false
153782024-04-20T01:16:03.530Z 13 Acked 1012 Write 3 Sent Done Done false
153792024-04-20T01:16:03.530Z 14 Acked 1013 Write 3 Sent Done Done false
153802024-04-20T01:16:03.530Z 15 Acked 1014 Flush 0 Sent Done Done false
153812024-04-20T01:16:03.530Z 16 Acked 1015 Write 10 Sent Done Done false
153822024-04-20T01:16:03.530Z 17 Acked 1016 Read 10 Sent Done Done false
153832024-04-20T01:16:03.530Z 18 Acked 1017 Read 4 Sent Done Done false
153842024-04-20T01:16:03.530Z 19 Acked 1018 Read 10 Sent Done Done false
153852024-04-20T01:16:03.530Z 20 Acked 1019 Read 2 Sent Done Done false
153862024-04-20T01:16:03.530Z 21 Acked 1020 Flush 0 Sent Done Done false
153872024-04-20T01:16:03.530Z 22 Acked 1021 Write 4 Sent Done Done false
153882024-04-20T01:16:03.531Z 23 Acked 1022 Write 6 Sent Done Done false
153892024-04-20T01:16:03.531Z 24 Acked 1023 Read 3 Sent Done Done false
153902024-04-20T01:16:03.535Z 25 Acked 1024 Write 10 Sent Done Done false
153912024-04-20T01:16:03.535Z 26 Acked 1025 Read 10 Sent Done Done false
153922024-04-20T01:16:03.535Z 27 Acked 1026 Write 5 Sent Done Done false
153932024-04-20T01:16:03.535Z 28 Acked 1027 Read 7 Sent Done Done false
153942024-04-20T01:16:03.535Z 29 Acked 1028 Read 4 Sent Done Done false
153952024-04-20T01:16:03.535Z 30 Acked 1029 Write 7 Sent Sent Sent false
153962024-04-20T01:16:03.535Z STATES DS:0 DS:1 DS:2 TOTAL
153972024-04-20T01:16:03.535Z New 0 0 0 0
153982024-04-20T01:16:03.535Z Sent 23 1 1 25
153992024-04-20T01:16:03.535Z Done 7 29 29 65
154002024-04-20T01:16:03.535Z Skipped 0 0 0 0
154012024-04-20T01:16:03.535Z Error 0 0 0 0
154022024-04-20T01:16:03.535ZLast Flush: 0 1020 1020
154032024-04-20T01:16:03.535ZDownstairs last five completed:
154042024-04-20T01:16:03.535ZUpstairs last five completed: 30 29 28 27 26
154052024-04-20T01:16:03.535Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:16:02.027873882Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1739}
154062024-04-20T01:16:03.535ZWrote out file "/tmp/test_repair_verify.data"
154072024-04-20T01:16:03.535Z
154082024-04-20T01:16:03.539ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
154092024-04-20T01:16:03.554ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154102024-04-20T01:16:03.555Z 0 000-019 138 138 138 232 232 232 F F F
154112024-04-20T01:16:03.555Z 1 020-039 138 138 138 232 232 232 F F F
154122024-04-20T01:16:03.555Z 2 040-059 132 132 132 223 223 223 F F F
154132024-04-20T01:16:03.555Z 3 060-079 140 140 140 234 234 234 F F F
154142024-04-20T01:16:03.555Z 4 080-099 140 142 142 234 235 235 T F F <---
154152024-04-20T01:16:03.559Z 5 100-119 134 142 142 225 235 235 T T T <---
154162024-04-20T01:16:03.559Z 6 120-139 131 142 142 222 235 235 T F F <---
154172024-04-20T01:16:03.559Z 7 140-159 137 137 137 231 231 231 F F F
154182024-04-20T01:16:03.559Z 8 160-179 139 139 139 233 233 233 F F F
154192024-04-20T01:16:03.559Z 9 180-199 137 142 142 231 236 236 T F F <---
154202024-04-20T01:16:03.559Z 10 200-219 136 142 142 230 236 236 F T T <---
154212024-04-20T01:16:03.559Z 11 220-239 137 137 137 231 231 231 F T T <---
154222024-04-20T01:16:03.559Z 12 240-259 139 139 139 233 233 233 F F F
154232024-04-20T01:16:03.559Z 13 260-279 131 142 142 222 235 235 F F F <---
154242024-04-20T01:16:03.559Z 14 280-299 134 134 134 225 225 225 F F F
154252024-04-20T01:16:03.559Z 15 300-319 134 134 134 225 225 225 F F F
154262024-04-20T01:16:03.559Z 16 320-339 134 134 134 225 225 225 F T T <---
154272024-04-20T01:16:03.559Z 17 340-359 136 136 136 230 230 230 F F F
154282024-04-20T01:16:03.559Z 18 360-379 140 140 140 234 234 234 F T T <---
154292024-04-20T01:16:03.559Z 19 380-399 140 142 142 234 235 235 F F F <---
154302024-04-20T01:16:03.559Z 20 400-419 140 142 142 234 235 235 F F F <---
154312024-04-20T01:16:03.559Z 21 420-439 132 142 142 223 235 235 T F F <---
154322024-04-20T01:16:03.559Z 22 440-459 138 138 138 232 232 232 F F F
154332024-04-20T01:16:03.559Z 23 460-479 138 138 138 232 232 232 F F F
154342024-04-20T01:16:03.559Z 24 480-499 138 138 138 232 232 232 F F F
154352024-04-20T01:16:03.560Z 25 500-519 139 139 139 233 233 233 F T T <---
154362024-04-20T01:16:03.560Z 26 520-539 138 142 142 232 235 235 F F F <---
154372024-04-20T01:16:03.560Z 27 540-559 140 140 140 234 234 234 F F F
154382024-04-20T01:16:03.560Z 28 560-579 137 142 142 231 235 235 F F F <---
154392024-04-20T01:16:03.560Z 29 580-599 136 142 142 230 235 235 T F F <---
154402024-04-20T01:16:03.560ZMax gen: 142, Max flush: 236
154412024-04-20T01:16:03.560ZError: Difference in extent metadata found!
154422024-04-20T01:16:03.560ZA Difference in extent metadata is expected here
154432024-04-20T01:16:03.560ZOn loop 71
154442024-04-20T01:16:03.560Z
154452024-04-20T01:16:03.564ZVerifying data now
154462024-04-20T01:16:22.791ZNo change after verify
154472024-04-20T01:16:22.792ZLoop: 71 Downstairs dump after verify (and repair):
154482024-04-20T01:16:22.792ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154492024-04-20T01:16:22.792Z 0 000-019 138 138 138 232 232 232 F F F
154502024-04-20T01:16:22.792Z 1 020-039 138 138 138 232 232 232 F F F
154512024-04-20T01:16:22.792Z 2 040-059 132 132 132 223 223 223 F F F
154522024-04-20T01:16:22.792Z 3 060-079 140 140 140 234 234 234 F F F
154532024-04-20T01:16:22.793Z 4 080-099 142 142 142 235 235 235 F F F
154542024-04-20T01:16:22.793Z 5 100-119 143 143 143 237 237 237 F F F
154552024-04-20T01:16:22.793Z 6 120-139 142 142 142 235 235 235 F F F
154562024-04-20T01:16:22.793Z 7 140-159 137 137 137 231 231 231 F F F
154572024-04-20T01:16:22.793Z 8 160-179 139 139 139 233 233 233 F F F
154582024-04-20T01:16:22.793Z 9 180-199 142 142 142 236 236 236 F F F
154592024-04-20T01:16:22.793Z 10 200-219 143 143 143 237 237 237 F F F
154602024-04-20T01:16:22.793Z 11 220-239 143 143 143 237 237 237 F F F
154612024-04-20T01:16:22.793Z 12 240-259 139 139 139 233 233 233 F F F
154622024-04-20T01:16:22.793Z 13 260-279 142 142 142 235 235 235 F F F
154632024-04-20T01:16:22.793Z 14 280-299 134 134 134 225 225 225 F F F
154642024-04-20T01:16:22.793Z 15 300-319 134 134 134 225 225 225 F F F
154652024-04-20T01:16:22.793Z 16 320-339 143 143 143 237 237 237 F F F
154662024-04-20T01:16:22.793Z 17 340-359 136 136 136 230 230 230 F F F
154672024-04-20T01:16:22.793Z 18 360-379 143 143 143 237 237 237 F F F
154682024-04-20T01:16:22.793Z 19 380-399 142 142 142 235 235 235 F F F
154692024-04-20T01:16:22.793Z 20 400-419 142 142 142 235 235 235 F F F
154702024-04-20T01:16:22.805Z 21 420-439 142 142 142 235 235 235 F F F
154712024-04-20T01:16:22.805Z 22 440-459 138 138 138 232 232 232 F F F
154722024-04-20T01:16:22.805Z 23 460-479 138 138 138 232 232 232 F F F
154732024-04-20T01:16:22.805Z 24 480-499 138 138 138 232 232 232 F F F
154742024-04-20T01:16:22.805Z 25 500-519 143 143 143 237 237 237 F F F
154752024-04-20T01:16:22.805Z 26 520-539 142 142 142 235 235 235 F F F
154762024-04-20T01:16:22.805Z 27 540-559 140 140 140 234 234 234 F F F
154772024-04-20T01:16:22.805Z 28 560-579 142 142 142 235 235 235 F F F
154782024-04-20T01:16:22.805Z 29 580-599 142 142 142 235 235 235 F F F
154792024-04-20T01:16:22.805ZMax gen: 143, Max flush: 237
154802024-04-20T01:16:22.806Z
154812024-04-20T01:16:22.806ZBegin loop 72
154822024-04-20T01:16:22.806ZDownstairs to restart: 2
154832024-04-20T01:16:22.806Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.807669471Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747}
154842024-04-20T01:16:22.806Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.807931012Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747}
154852024-04-20T01:16:22.806Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.807955752Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747}
154862024-04-20T01:16:22.806Z{"msg":"Crucible 34f96159-fe39-48a7-8063-fb6751d743c0 has session id: 6b7e5663-59ac-40da-8c17-f2865d343b51","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.807969902Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
154872024-04-20T01:16:22.806Z{"msg":"Upstairs opts: Upstairs UUID: 34f96159-fe39-48a7-8063-fb6751d743c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808004912Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
154882024-04-20T01:16:22.806Z{"msg":"Crucible stats registered with UUID: 34f96159-fe39-48a7-8063-fb6751d743c0","v":0,"name":"crucible","level":30Crucible runtime is spawned
154892024-04-20T01:16:22.806Z,"time":"2024-04-20T01:16:16.808027602Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
154902024-04-20T01:16:22.806Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808112392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747}
154912024-04-20T01:16:22.806Z{"msg":"34f96159-fe39-48a7-8063-fb6751d743c0 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808209423Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
154922024-04-20T01:16:22.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808252443Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"0","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
154932024-04-20T01:16:22.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808267683Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"0","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
154942024-04-20T01:16:22.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808288643Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"1","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
154952024-04-20T01:16:22.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808303353Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"1","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
154962024-04-20T01:16:22.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808337413Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"2","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
154972024-04-20T01:16:22.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808358153Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"2","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
154982024-04-20T01:16:22.806Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808420263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
154992024-04-20T01:16:22.807Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808483233Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155002024-04-20T01:16:22.807Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808530113Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155012024-04-20T01:16:22.807Z{"msg":"ds_connection connected from Ok(127.0.0.1:57346)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808574453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155022024-04-20T01:16:22.807Z{"msg":"ds_connection connected from Ok(127.0.0.1:35745)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808621264Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155032024-04-20T01:16:22.807Z{"msg":"ds_connection connected from Ok(127.0.0.1:44861)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.808649053Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155042024-04-20T01:16:22.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.809224505Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"0","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155052024-04-20T01:16:22.807Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.809271895Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"0","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155062024-04-20T01:16:22.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.809301465Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"1","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155072024-04-20T01:16:22.807Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.809328095Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"1","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155082024-04-20T01:16:22.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.809723766Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"2","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155092024-04-20T01:16:22.810Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.809775836Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"2","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155102024-04-20T01:16:22.810Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.809808866Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"1","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155112024-04-20T01:16:22.810Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.810013796Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"0","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155122024-04-20T01:16:22.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.810294457Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"1","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155132024-04-20T01:16:22.811Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.810338347Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"1","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155142024-04-20T01:16:22.811Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.810373597Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155152024-04-20T01:16:22.811Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.810407677Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"2","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155162024-04-20T01:16:22.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.810496598Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"0","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155172024-04-20T01:16:22.811Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.810532848Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"0","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155182024-04-20T01:16:22.811Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.810566788Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155192024-04-20T01:16:22.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.810901098Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"2","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155202024-04-20T01:16:22.811Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.810945669Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"2","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155212024-04-20T01:16:22.811Z{"msg":"[0]R flush_numbers[0..12]: [232, 232, 223, 234, 235, 237, 235, 231, 233, 236, 237, 237]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.810983529Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155222024-04-20T01:16:22.811Z{"msg":"[0]R generation[0..12]: [138, 138, 132, 140, 142, 143, 142, 137, 139, 142, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.811037819Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155232024-04-20T01:16:22.812Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.811076009Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155242024-04-20T01:16:22.812Z{"msg":"Wait for a query_work_queue command to finish before sending IO
155252024-04-20T01:16:22.812Z[1]R flush_numbers[0..12]: [232, 232, 223, 234, 235, 237, 235, 231, 233, 236, 237, 237]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.811103829Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155262024-04-20T01:16:22.812Z{"msg":"[1]R generation[0..12]: [138, 138, 132, 140, 142, 143, 142, 137, 139, 142, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.811140159Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155272024-04-20T01:16:22.812Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.811179779Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155282024-04-20T01:16:22.812Z{"msg":"[2]R flush_numbers[0..12]: [232, 232, 223, 234, 235, 237, 235, 231, 233, 236, 237, 237]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.811219709Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155292024-04-20T01:16:22.812Z{"msg":"[2]R generation[0..12]: [138, 138, 132, 140, 142, 143, 142, 137, 139, 142, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.811258799Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155302024-04-20T01:16:22.812Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.811297959Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155312024-04-20T01:16:22.812Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.811336559Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155322024-04-20T01:16:22.813Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.81137289Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155332024-04-20T01:16:22.813Z{"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.81140818Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155342024-04-20T01:16:22.813ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
155352024-04-20T01:16:22.813Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.81144756Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"":"downstairs","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155362024-04-20T01:16:22.813Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.81148036Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155372024-04-20T01:16:22.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.81150047Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"0","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155382024-04-20T01:16:22.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.81152075Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"1","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155392024-04-20T01:16:22.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.81154052Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"client":"2","":"downstairs","session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155402024-04-20T01:16:22.813Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.81156058Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155412024-04-20T01:16:22.813Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.81157965Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155422024-04-20T01:16:22.813Z{"msg":"34f96159-fe39-48a7-8063-fb6751d743c0 is now active with session: 6b7e5663-59ac-40da-8c17-f2865d343b51","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.81159932Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155432024-04-20T01:16:22.814Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.81161814Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747,"session_id":"6b7e5663-59ac-40da-8c17-f2865d343b51"}
155442024-04-20T01:16:22.814Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:16.81163692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747}
155452024-04-20T01:16:22.814ZLoading write count information from file "/tmp/test_repair_verify.data"
155462024-04-20T01:16:22.814ZError: Difference in extent metadata found!
155472024-04-20T01:16:22.814ZRun Repair workload
155482024-04-20T01:16:22.814Z01/30 Write block 39 len 16384 data: 12 10 9 8
155492024-04-20T01:16:22.814Z02/30 Write block 215 len 40960 data: 10 12 16 15 12 14 12 13 14 14
155502024-04-20T01:16:22.814Z03/30 Read block 568 len 24576
155512024-04-20T01:16:22.814Z04/30 Write block 460 len 4096 data: 14
155522024-04-20T01:16:22.815Z05/30 Read block 467 len 28672
155532024-04-20T01:16:22.815Z06/30 Read block 567 len 4096
155542024-04-20T01:16:22.815Z07/30 Read block 86 len 32768
155552024-04-20T01:16:22.815Z08/30 Write block 520 len 32768 data: 17 17 16 16 14 13 12 10
155562024-04-20T01:16:22.815Z09/30 Write block 237 len 12288 data: 16 14 13
155572024-04-20T01:16:22.815Z10/30 Read block 103 len 24576
155582024-04-20T01:16:22.815Z11/30 Write block 4 len 8192 data: 8 7
155592024-04-20T01:16:22.815Z12/30 Read block 492 len 20480
155602024-04-20T01:16:22.815Z13/30 Write block 579 len 16384 data: 14 14 13 12
155612024-04-20T01:16:22.815Z14/30 Write block 279 len 32768 data: 11 12 14 11 8 9 9 9
155622024-04-20T01:16:22.816Z15/30 Write block 456 len 4096 data: 13
155632024-04-20T01:16:22.816Z16/30 Write block 549 len 28672 data: 13 15 14 10 8 10 11
155642024-04-20T01:16:22.816Z17/30 Flush
155652024-04-20T01:16:22.816Z18/30 Write block 516 len 12288 data: 16 16 14
155662024-04-20T01:16:22.816Z19/30 Write block 301 len 24576 data: 8 9 9 9 8 7
155672024-04-20T01:16:22.816Z20/30 Write block 502 len 4096 data: 11
155682024-04-20T01:16:22.816Z21/30 Read block 498 len 32768
155692024-04-20T01:16:22.816Z22/30 Read block 249 len 8192
155702024-04-20T01:16:22.828Z23/30 Write block 200 len 40960 data: 11 12 14 10 9 9 10 10 10 11
155712024-04-20T01:16:22.829Z24/30 Read block 556 len 24576
155722024-04-20T01:16:22.829Z25/30 Read block 423 len 28672
155732024-04-20T01:16:22.829Z26/30 Flush
155742024-04-20T01:16:22.829Z27/30 Write block 262 len 20480 data: 6 4 4 5 7
155752024-04-20T01:16:22.829Z28/30 Write block 178 len 12288 data: 15 16 15
155762024-04-20T01:16:22.829Z29/30 Read block 92 len 16384
155772024-04-20T01:16:22.829Z30/30 Read block 308 len 32768
155782024-04-20T01:16:22.829Z----------------------------------------------------------------
155792024-04-20T01:16:22.829Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
155802024-04-20T01:16:22.829ZStates: Active Active Active
155812024-04-20T01:16:22.829ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
155822024-04-20T01:16:22.830Z 1 Acked 1000 Write 4 Done Done Done false
155832024-04-20T01:16:22.830Z 2 Acked 1001 Write 10 Done Done Done false
155842024-04-20T01:16:22.830Z 3 Acked 1002 Read 6 Done Done Done false
155852024-04-20T01:16:22.830Z 4 Acked 1003 Write 1 Done Done Done false
155862024-04-20T01:16:22.830Z 5 Acked 1004 Read 7 Done Done Done false
155872024-04-20T01:16:22.830Z 6 Acked 1005 Read 1 Done Done Done false
155882024-04-20T01:16:22.830Z 7 Acked 1006 Read 8 Done Done Done false
155892024-04-20T01:16:22.830Z 8 Acked 1007 Write 8 Done Done Done false
155902024-04-20T01:16:22.830Z 9 Acked 1008 Write 3 Done Done Done false
155912024-04-20T01:16:22.830Z 10 Acked 1009 Read 6 Done Done Done false
155922024-04-20T01:16:22.831Z 11 Acked 1010 Write 2 Done Done Sent false
155932024-04-20T01:16:22.831Z 12 Acked 1011 Read 5 Done Done Sent false
155942024-04-20T01:16:22.831Z 13 Acked 1012 Write 4 Done Done Sent false
155952024-04-20T01:16:22.831Z 14 Acked 1013 Write 8 Done Done Sent false
155962024-04-20T01:16:22.831Z 15 Acked 1014 Write 1 Done Done Sent false
155972024-04-20T01:16:22.831Z 16 Acked 1015 Write 7 Done Done Sent false
155982024-04-20T01:16:22.831Z 17 Acked 1016 Flush 0 Done Done Sent false
155992024-04-20T01:16:22.831Z 18 Acked 1017 Write 3 Done Done Sent false
156002024-04-20T01:16:22.831Z 19 Acked 1018 Write 6 Done Done Sent false
156012024-04-20T01:16:22.831Z 20 Acked 1019 Write 1 Done Done Sent false
156022024-04-20T01:16:22.831Z 21 Acked 1020 Read 8 Done Done Sent false
156032024-04-20T01:16:22.831Z 22 Acked 1021 Read 2 Done Done Sent false
156042024-04-20T01:16:22.831Z 23 Acked 1022 Write 10 Done Done Sent false
156052024-04-20T01:16:22.832Z 24 Acked 1023 Read 6 Done Done Sent false
156062024-04-20T01:16:22.832Z 25 Acked 1024 Read 7 Done Done Sent false
156072024-04-20T01:16:22.832Z 26 Acked 1025 Flush 0 Done Done Sent false
156082024-04-20T01:16:22.832Z 27 Acked 1026 Write 5 Done Done Sent false
156092024-04-20T01:16:22.832Z 28 Acked 1027 Write 3 Done Done Sent false
156102024-04-20T01:16:22.832Z 29 Acked 1028 Read 4 Done Done Sent false
156112024-04-20T01:16:22.832Z 30 Acked 1029 Read 8 Done Sent Sent false
156122024-04-20T01:16:22.832Z STATES DS:0 DS:1 DS:2 TOTAL
156132024-04-20T01:16:22.832Z New 0 0 0 0
156142024-04-20T01:16:22.832Z Sent 0 1 20 21
156152024-04-20T01:16:22.833Z Done 30 29 10 69
156162024-04-20T01:16:22.833Z Skipped 0 0 0 0
156172024-04-20T01:16:22.833Z Error 0 0 0 0
156182024-04-20T01:16:22.833ZLast Flush: 1025 1025 0
156192024-04-20T01:16:22.833ZDownstairs last five completed:
156202024-04-20T01:16:22.833ZUpstairs last five completed: 30 29 28 27 26
156212024-04-20T01:16:22.833Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:16:16.843274041Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1747}
156222024-04-20T01:16:22.833ZWrote out file "/tmp/test_repair_verify.data"
156232024-04-20T01:16:22.833Z
156242024-04-20T01:16:22.833ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
156252024-04-20T01:16:22.833ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156262024-04-20T01:16:22.833Z 0 000-019 144 144 138 238 238 232 F F F <---
156272024-04-20T01:16:22.833Z 1 020-039 144 144 138 238 238 232 F F T <---
156282024-04-20T01:16:22.834Z 2 040-059 144 144 132 238 238 223 F F T <---
156292024-04-20T01:16:22.834Z 3 060-079 140 140 140 234 234 234 F F F
156302024-04-20T01:16:22.834Z 4 080-099 142 142 142 235 235 235 F F F
156312024-04-20T01:16:22.834Z 5 100-119 143 143 143 237 237 237 F F F
156322024-04-20T01:16:22.834Z 6 120-139 142 142 142 235 235 235 F F F
156332024-04-20T01:16:22.834Z 7 140-159 137 137 137 231 231 231 F F F
156342024-04-20T01:16:22.834Z 8 160-179 139 139 139 233 233 233 T T F <---
156352024-04-20T01:16:22.834Z 9 180-199 142 142 142 236 236 236 T T F <---
156362024-04-20T01:16:22.834Z 10 200-219 144 144 143 239 239 237 F F T <---
156372024-04-20T01:16:22.834Z 11 220-239 144 144 143 238 238 237 F F T <---
156382024-04-20T01:16:22.834Z 12 240-259 139 139 139 233 233 233 F F F
156392024-04-20T01:16:22.835Z 13 260-279 144 144 142 238 238 235 T T F <---
156402024-04-20T01:16:22.835Z 14 280-299 144 144 134 238 238 225 F F F <---
156412024-04-20T01:16:22.835Z 15 300-319 144 144 134 239 239 225 F F F <---
156422024-04-20T01:16:22.835Z 16 320-339 143 143 143 237 237 237 F F F
156432024-04-20T01:16:22.835Z 17 340-359 136 136 136 230 230 230 F F F
156442024-04-20T01:16:22.835Z 18 360-379 143 143 143 237 237 237 F F F
156452024-04-20T01:16:22.835Z 19 380-399 142 142 142 235 235 235 F F F
156462024-04-20T01:16:22.835Z 20 400-419 142 142 142 235 235 235 F F F
156472024-04-20T01:16:22.835Z 21 420-439 142 142 142 235 235 235 F F F
156482024-04-20T01:16:22.835Z 22 440-459 144 144 138 238 238 232 F F F <---
156492024-04-20T01:16:22.835Z 23 460-479 144 144 138 238 238 232 F F T <---
156502024-04-20T01:16:22.835Z 24 480-499 138 138 138 232 232 232 F F F
156512024-04-20T01:16:22.836Z 25 500-519 144 144 143 239 239 237 F F F <---
156522024-04-20T01:16:22.836Z 26 520-539 144 144 142 238 238 235 F F T <---
156532024-04-20T01:16:22.836Z 27 540-559 144 144 140 238 238 234 F F F <---
156542024-04-20T01:16:22.836Z 28 560-579 144 144 142 238 238 235 F F F <---
156552024-04-20T01:16:22.836Z 29 580-599 144 144 142 238 238 235 F F F <---
156562024-04-20T01:16:22.836ZMax gen: 144, Max flush: 239
156572024-04-20T01:16:22.836ZA Difference in extent metadata is expected here
156582024-04-20T01:16:22.836ZOn loop 72
156592024-04-20T01:16:22.836Z
156602024-04-20T01:16:22.836ZVerifying data now
156612024-04-20T01:16:33.123ZNo change after verify
156622024-04-20T01:16:33.123ZLoop: 72 Downstairs dump after verify (and repair):
156632024-04-20T01:16:33.138ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156642024-04-20T01:16:33.138Z 0 000-019 144 144 144 238 238 238 F F F
156652024-04-20T01:16:33.138Z 1 020-039 144 144 144 238 238 238 F F F
156662024-04-20T01:16:33.138Z 2 040-059 144 144 144 238 238 238 F F F
156672024-04-20T01:16:33.138Z 3 060-079 140 140 140 234 234 234 F F F
156682024-04-20T01:16:33.138Z 4 080-099 142 142 142 235 235 235 F F F
156692024-04-20T01:16:33.138Z 5 100-119 143 143 143 237 237 237 F F F
156702024-04-20T01:16:33.138Z 6 120-139 142 142 142 235 235 235 F F F
156712024-04-20T01:16:33.138Z 7 140-159 137 137 137 231 231 231 F F F
156722024-04-20T01:16:33.138Z 8 160-179 145 145 145 240 240 240 F F F
156732024-04-20T01:16:33.138Z 9 180-199 145 145 145 240 240 240 F F F
156742024-04-20T01:16:33.138Z 10 200-219 144 144 144 239 239 239 F F F
156752024-04-20T01:16:33.139Z 11 220-239 144 144 144 238 238 238 F F F
156762024-04-20T01:16:33.143Z 12 240-259 139 139 139 233 233 233 F F F
156772024-04-20T01:16:33.143Z 13 260-279 145 145 145 240 240 240 F F F
156782024-04-20T01:16:33.143Z 14 280-299 144 144 144 238 238 238 F F F
156792024-04-20T01:16:33.143Z 15 300-319 144 144 144 239 239 239 F F F
156802024-04-20T01:16:33.143Z 16 320-339 143 143 143 237 237 237 F F F
156812024-04-20T01:16:33.143Z 17 340-359 136 136 136 230 230 230 F F F
156822024-04-20T01:16:33.143Z 18 360-379 143 143 143 237 237 237 F F F
156832024-04-20T01:16:33.143Z 19 380-399 142 142 142 235 235 235 F F F
156842024-04-20T01:16:33.143Z 20 400-419 142 142 142 235 235 235 F F F
156852024-04-20T01:16:33.143Z 21 420-439 142 142 142 235 235 235 F F F
156862024-04-20T01:16:33.143Z 22 440-459 144 144 144 238 238 238 F F F
156872024-04-20T01:16:33.143Z 23 460-479 144 144 144 238 238 238 F F F
156882024-04-20T01:16:33.143Z 24 480-499 138 138 138 232 232 232 F F F
156892024-04-20T01:16:33.143Z 25 500-519 144 144 144 239 239 239 F F F
156902024-04-20T01:16:33.143Z 26 520-539 144 144 144 238 238 238 F F F
156912024-04-20T01:16:33.143Z 27 540-559 144 144 144 238 238 238 F F F
156922024-04-20T01:16:33.143Z 28 560-579 144 144 144 238 238 238 F F F
156932024-04-20T01:16:33.143Z 29 580-599 144 144 144 238 238 238 F F F
156942024-04-20T01:16:33.143ZMax gen: 145, Max flush: 240
156952024-04-20T01:16:33.143Z
156962024-04-20T01:16:33.143ZBegin loop 73
156972024-04-20T01:16:33.144ZDownstairs to restart: 0
156982024-04-20T01:16:33.161Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.659237151Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755}
156992024-04-20T01:16:33.161Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.659503442Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755}
157002024-04-20T01:16:33.161Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.659523252Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755}
157012024-04-20T01:16:33.161Z{"msg":"Crucible 11385f04-007d-48d1-8b68-dfe1f96e21ec has session id: 43a2ed87-fe1c-4c30-97e2-fe340cfa069c","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.659556622Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157022024-04-20T01:16:33.169Z{"msg":"Upstairs opts: Upstairs UUID: 11385f04-007d-48d1-8b68-dfe1f96e21ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.659605952Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157032024-04-20T01:16:33.169Z{"msg":"Crucible stats registered with UUID: 11385f04-007d-48d1-8b68-dfe1f96e21ec","v":0,"name":"crucible","level":30Crucible runtime is spawned
157042024-04-20T01:16:33.169Z,"time":"2024-04-20T01:16:31.659651442Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157052024-04-20T01:16:33.169Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.659742662Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755}
157062024-04-20T01:16:33.169Z{"msg":"11385f04-007d-48d1-8b68-dfe1f96e21ec active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.659798782Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157072024-04-20T01:16:33.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.659818362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"0","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157082024-04-20T01:16:33.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.659863012Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"0","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157092024-04-20T01:16:33.170Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.659889863Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"1","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157102024-04-20T01:16:33.170Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.659937033Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"1","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157112024-04-20T01:16:33.170Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.659967313Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"2","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157122024-04-20T01:16:33.170Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.660003143Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"2","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157132024-04-20T01:16:33.170Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.660031903Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157142024-04-20T01:16:33.170Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.660069433Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157152024-04-20T01:16:33.170Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.660109953Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157162024-04-20T01:16:33.170Z{"msg":"ds_connection connected from Ok(127.0.0.1:52105)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.660143263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157172024-04-20T01:16:33.170Z{"msg":"ds_connection connected from Ok(127.0.0.1:57877)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.660178433Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157182024-04-20T01:16:33.170Z{"msg":"ds_connection connected from Ok(127.0.0.1:46796)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.660368833Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157192024-04-20T01:16:33.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.661000245Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"1","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157202024-04-20T01:16:33.170Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.661043725Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"1","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157212024-04-20T01:16:33.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.661121595Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"2","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157222024-04-20T01:16:33.170Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.661165015Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"2","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157232024-04-20T01:16:33.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.661204826Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"0","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157242024-04-20T01:16:33.170Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.661243026Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"0","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157252024-04-20T01:16:33.170Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.661625516Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"2","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157262024-04-20T01:16:33.170Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.661669636Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"1","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157272024-04-20T01:16:33.170Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.661706316Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"0","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157282024-04-20T01:16:33.170Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662134508Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"2","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157292024-04-20T01:16:33.170Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662171518Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"2","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157302024-04-20T01:16:33.170Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662186898Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157312024-04-20T01:16:33.171Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662215608Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"1","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157322024-04-20T01:16:33.171Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662249268Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"1","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157332024-04-20T01:16:33.171Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662270808Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157342024-04-20T01:16:33.171Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662310408Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"0","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157352024-04-20T01:16:33.171Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662335058Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"0","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157362024-04-20T01:16:33.171Z{"msg":"[0]R flush_numbers[0..12]: [238, 238, 238, 234, 235, 237, 235, 231, 240, 240, 239, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662350898Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157372024-04-20T01:16:33.171Z{"msg":"[0]R generation[0..12]: [144, 144, 144, 140, 142, 143, 142, 137, 145, 145, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662365218Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157382024-04-20T01:16:33.171Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662387518Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157392024-04-20T01:16:33.171Z{"msg":"[1]R flush_numbers[0..12]: [238, 238, 238, 234, 235, 237, 235, 231, 240, 240, 239, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662401948Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157402024-04-20T01:16:33.171Z{"msg":"[1]R generation[0..12]: [144, 144, 144, 140, 142, 143, 142, 137, 145, 145, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662415818Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157412024-04-20T01:16:33.171Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662437128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157422024-04-20T01:16:33.171Z{"msg":"[2]R flush_numbers[0..12]: [238, 238, 238, 234, 235, 237, 235, 231, 240, 240, 239, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662451038Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157432024-04-20T01:16:33.171Z{"msg":"[2]R generation[0..12]: [144, 144, 144, 140, 142, 143, 142, 137, 145, 145, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662464698Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157442024-04-20T01:16:33.171Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662485208Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157452024-04-20T01:16:33.171Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662498438Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157462024-04-20T01:16:33.171Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662512388Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157472024-04-20T01:16:33.171Z{"msg":"Next flush: 241","v":0,"name"Wait for a query_work_queue command to finish before sending IO
157482024-04-20T01:16:33.171Z:"crucible","level":30,"time":"2024-04-20T01:16:31.662547678Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157492024-04-20T01:16:33.171Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662573549Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"":"downstairs","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157502024-04-20T01:16:33.171Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662609478Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157512024-04-20T01:16:33.171Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662632239Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"0","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157522024-04-20T01:16:33.171Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662666829Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"1","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157532024-04-20T01:16:33.171Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662689959Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"client":"2","":"downstairs","session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157542024-04-20T01:16:33.172Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662725029Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157552024-04-20T01:16:33.172Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662764469Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157562024-04-20T01:16:33.172Z{"msg":"11385f04-007d-48d1-8b68-dfe1f96e21ec is now active with session: 43a2ed87-fe1c-4c30-97e2-fe340cfa069c","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662789479Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157572024-04-20T01:16:33.172Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662818229Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755,"session_id":"43a2ed87-fe1c-4c30-97e2-fe340cfa069c"}
157582024-04-20T01:16:33.172Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:31.662852569Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755}
157592024-04-20T01:16:33.172ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
157602024-04-20T01:16:33.172ZLoading write count information from file "/tmp/test_repair_verify.data"
157612024-04-20T01:16:33.172ZRun Repair workload
157622024-04-20T01:16:33.177Z01/30 Write block 380 len 28672 data: 16 13 12 11 9 7 8
157632024-04-20T01:16:33.177Z02/30 Write block 410 len 16384 data: 13 15 16 17
157642024-04-20T01:16:33.177Z03/30 Read block 116 len 4096
157652024-04-20T01:16:33.177Z04/30 Read block 353 len 16384
157662024-04-20T01:16:33.177Z05/30 Write block 77 len 24576 data: 10 11 11 10 10 9
157672024-04-20T01:16:33.177Z06/30 Read block 62 len 8192
157682024-04-20T01:16:33.177Z07/30 Read block 349 len 24576
157692024-04-20T01:16:33.177Z08/30 Write block 406 len 4096 data: 11
157702024-04-20T01:16:33.177Z09/30 Write block 509 len 12288 data: 13 14 15
157712024-04-20T01:16:33.177Z10/30 Read block 133 len 28672
157722024-04-20T01:16:33.177Z11/30 Read block 494 len 12288
157732024-04-20T01:16:33.177Z12/30 Read block 550 len 16384
157742024-04-20T01:16:33.177Z13/30 Write block 532 len 32768 data: 11 10 13 12 10 9 8 7
157752024-04-20T01:16:33.181Z14/30 Read block 302 len 28672
157762024-04-20T01:16:33.181Z15/30 Write block 559 len 8192 data: 8 7
157772024-04-20T01:16:33.181Z16/30 Read block 371 len 40960
157782024-04-20T01:16:33.181Z17/30 Read block 140 len 40960
157792024-04-20T01:16:33.181Z18/30 Read block 127 len 16384
157802024-04-20T01:16:33.184Z19/30 Write block 344 len 8192 data: 10 14
157812024-04-20T01:16:33.184Z20/30 Read block 110 len 28672
157822024-04-20T01:16:33.184Z21/30 Read block 456 len 4096
157832024-04-20T01:16:33.184Z22/30 Read block 233 len 8192
157842024-04-20T01:16:33.185Z23/30 Read block 110 len 20480
157852024-04-20T01:16:33.185Z24/30 Read block 30 len 40960
157862024-04-20T01:16:33.189Z25/30 Read block 381 len 36864
157872024-04-20T01:16:33.189Z26/30 Read block 431 len 8192
157882024-04-20T01:16:33.189Z27/30 Write block 74 len 8192 data: 13 9
157892024-04-20T01:16:33.189Z28/30 Write block 204 len 24576 data: 10 10 11 11 11 12
157902024-04-20T01:16:33.194Z29/30 Write block 109 len 24576 data: 12 12 11 11 10 11
157912024-04-20T01:16:33.194Z30/30 Write block 295 len 12288 data: 8 9 9
157922024-04-20T01:16:33.194Z----------------------------------------------------------------
157932024-04-20T01:16:33.194Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
157942024-04-20T01:16:33.194ZStates: Active Active Active
157952024-04-20T01:16:33.194ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157962024-04-20T01:16:33.194Z 1 Acked 1000 Write 7 Done Done Done false
157972024-04-20T01:16:33.194Z 2 Acked 1001 Write 4 Done Done Done false
157982024-04-20T01:16:33.194Z 3 Acked 1002 Read 1 Done Done Done false
157992024-04-20T01:16:33.194Z 4 Acked 1003 Read 4 Done Done Done false
158002024-04-20T01:16:33.194Z 5 Acked 1004 Write 6 Done Done Done false
158012024-04-20T01:16:33.194Z 6 Acked 1005 Read 2 Done Done Done false
158022024-04-20T01:16:33.194Z 7 Acked 1006 Read 6 Done Done Done false
158032024-04-20T01:16:33.194Z 8 Acked 1007 Write 1 Done Done Done false
158042024-04-20T01:16:33.194Z 9 Acked 1008 Write 3 Done Done Done false
158052024-04-20T01:16:33.194Z 10 Acked 1009 Read 7 Done Done Done false
158062024-04-20T01:16:33.194Z 11 Acked 1010 Read 3 Done Done Done false
158072024-04-20T01:16:33.194Z 12 Acked 1011 Read 4 Done Done Done false
158082024-04-20T01:16:33.194Z 13 Acked 1012 Write 8 Done Done Done false
158092024-04-20T01:16:33.195Z 14 Acked 1013 Read 7 Done Done Done false
158102024-04-20T01:16:33.195Z 15 Acked 1014 Write 2 Done Done Done false
158112024-04-20T01:16:33.195Z 16 Acked 1015 Read 10 Done Done Done false
158122024-04-20T01:16:33.195Z 17 Acked 1016 Read 10 Done Done Done false
158132024-04-20T01:16:33.195Z 18 Acked 1017 Read 4 Done Done Done false
158142024-04-20T01:16:33.195Z 19 Acked 1018 Write 2 Done Done Done false
158152024-04-20T01:16:33.195Z 20 Acked 1019 Read 7 Done Done Done false
158162024-04-20T01:16:33.195Z 21 Acked 1020 Read 1 Done Done Done false
158172024-04-20T01:16:33.195Z 22 Acked 1021 Read 2 Done Done Done false
158182024-04-20T01:16:33.195Z 23 Acked 1022 Read 5 Sent Done Done false
158192024-04-20T01:16:33.195Z 24 Acked 1023 Read 10 Sent Done Done false
158202024-04-20T01:16:33.195Z 25 Acked 1024 Read 9 Sent Done Done false
158212024-04-20T01:16:33.195Z 26 Acked 1025 Read 2 Sent Done Done false
158222024-04-20T01:16:33.195Z 27 Acked 1026 Write 2 Sent Done Done false
158232024-04-20T01:16:33.195Z 28 Acked 1027 Write 6 Sent Done Done false
158242024-04-20T01:16:33.195Z 29 Acked 1028 Write 6 Sent Done Done false
158252024-04-20T01:16:33.195Z 30 Acked 1029 Write 3 Sent Sent Sent false
158262024-04-20T01:16:33.195Z STATES DS:0 DS:1 DS:2 TOTAL
158272024-04-20T01:16:33.195Z New 0 0 0 0
158282024-04-20T01:16:33.195Z Sent 8 1 1 10
158292024-04-20T01:16:33.195Z Done 22 29 29 80
158302024-04-20T01:16:33.195Z Skipped 0 0 0 0
158312024-04-20T01:16:33.195Z Error 0 0 0 0
158322024-04-20T01:16:33.195ZLast Flush: 0 0 0
158332024-04-20T01:16:33.196ZDownstairs last five completed:
158342024-04-20T01:16:33.196ZUpstairs last five completed: 30 29 28 27 26
158352024-04-20T01:16:33.196Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:16:31.688358336Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1755}
158362024-04-20T01:16:33.196ZWrote out file "/tmp/test_repair_verify.data"
158372024-04-20T01:16:33.200Z
158382024-04-20T01:16:33.200ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
158392024-04-20T01:16:33.217ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158402024-04-20T01:16:33.217Z 0 000-019 144 144 144 238 238 238 F F F
158412024-04-20T01:16:33.217Z 1 020-039 144 144 144 238 238 238 F F F
158422024-04-20T01:16:33.217Z 2 040-059 144 144 144 238 238 238 F F F
158432024-04-20T01:16:33.217Z 3 060-079 140 140 140 234 234 234 T T T
158442024-04-20T01:16:33.217Z 4 080-099 142 142 142 235 235 235 T T T
158452024-04-20T01:16:33.217Z 5 100-119 143 143 143 237 237 237 F T T <---
158462024-04-20T01:16:33.217Z 6 120-139 142 142 142 235 235 235 F F F
158472024-04-20T01:16:33.217Z 7 140-159 137 137 137 231 231 231 F F F
158482024-04-20T01:16:33.217Z 8 160-179 145 145 145 240 240 240 F F F
158492024-04-20T01:16:33.217Z 9 180-199 145 145 145 240 240 240 F F F
158502024-04-20T01:16:33.217Z 10 200-219 144 144 144 239 239 239 F T T <---
158512024-04-20T01:16:33.218Z 11 220-239 144 144 144 238 238 238 F F F
158522024-04-20T01:16:33.222Z 12 240-259 139 139 139 233 233 233 F F F
158532024-04-20T01:16:33.222Z 13 260-279 145 145 145 240 240 240 F F F
158542024-04-20T01:16:33.222Z 14 280-299 144 144 144 238 238 238 F T T <---
158552024-04-20T01:16:33.222Z 15 300-319 144 144 144 239 239 239 F F F
158562024-04-20T01:16:33.222Z 16 320-339 143 143 143 237 237 237 F F F
158572024-04-20T01:16:33.222Z 17 340-359 136 136 136 230 230 230 T T T
158582024-04-20T01:16:33.222Z 18 360-379 143 143 143 237 237 237 F F F
158592024-04-20T01:16:33.222Z 19 380-399 142 142 142 235 235 235 T T T
158602024-04-20T01:16:33.222Z 20 400-419 142 142 142 235 235 235 T T T
158612024-04-20T01:16:33.222Z 21 420-439 142 142 142 235 235 235 F F F
158622024-04-20T01:16:33.222Z 22 440-459 144 144 144 238 238 238 F F F
158632024-04-20T01:16:33.222Z 23 460-479 144 144 144 238 238 238 F F F
158642024-04-20T01:16:33.222Z 24 480-499 138 138 138 232 232 232 F F F
158652024-04-20T01:16:33.223Z 25 500-519 144 144 144 239 239 239 T T T
158662024-04-20T01:16:33.223Z 26 520-539 144 144 144 238 238 238 T T T
158672024-04-20T01:16:33.223Z 27 540-559 144 144 144 238 238 238 T T T
158682024-04-20T01:16:33.223Z 28 560-579 144 144 144 238 238 238 T T T
158692024-04-20T01:16:33.223Z 29 580-599 144 144 144 238 238 238 F F F
158702024-04-20T01:16:33.223ZMax gen: 145, Max flush: 240
158712024-04-20T01:16:33.223ZError: Difference in extent metadata found!
158722024-04-20T01:16:33.223ZA Difference in extent metadata is expected here
158732024-04-20T01:16:33.223ZOn loop 73
158742024-04-20T01:16:33.223Z
158752024-04-20T01:16:33.226ZVerifying data now
158762024-04-20T01:16:47.820ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
158772024-04-20T01:16:47.824Z77,78c77,78
158782024-04-20T01:16:47.824Z< 12,
158792024-04-20T01:16:47.824Z< 8,
158802024-04-20T01:16:47.824Z---
158812024-04-20T01:16:47.824Z> 13,
158822024-04-20T01:16:47.824Z> 9,
158832024-04-20T01:16:47.824Zdiff found after verify
158842024-04-20T01:16:47.828ZLoop: 73 Downstairs dump after verify (and repair):
158852024-04-20T01:16:47.842ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158862024-04-20T01:16:47.842Z 0 000-019 144 144 144 238 238 238 F F F
158872024-04-20T01:16:47.846Z 1 020-039 144 144 144 238 238 238 F F F
158882024-04-20T01:16:47.846Z 2 040-059 144 144 144 238 238 238 F F F
158892024-04-20T01:16:47.846Z 3 060-079 146 146 146 241 241 241 F F F
158902024-04-20T01:16:47.846Z 4 080-099 146 146 146 241 241 241 F F F
158912024-04-20T01:16:47.846Z 5 100-119 146 146 146 241 241 241 F F F
158922024-04-20T01:16:47.846Z 6 120-139 142 142 142 235 235 235 F F F
158932024-04-20T01:16:47.846Z 7 140-159 137 137 137 231 231 231 F F F
158942024-04-20T01:16:47.846Z 8 160-179 145 145 145 240 240 240 F F F
158952024-04-20T01:16:47.846Z 9 180-199 145 145 145 240 240 240 F F F
158962024-04-20T01:16:47.846Z 10 200-219 146 146 146 241 241 241 F F F
158972024-04-20T01:16:47.846Z 11 220-239 144 144 144 238 238 238 F F F
158982024-04-20T01:16:47.846Z 12 240-259 139 139 139 233 233 233 F F F
158992024-04-20T01:16:47.846Z 13 260-279 145 145 145 240 240 240 F F F
159002024-04-20T01:16:47.846Z 14 280-299 146 146 146 241 241 241 F F F
159012024-04-20T01:16:47.846Z 15 300-319 144 144 144 239 239 239 F F F
159022024-04-20T01:16:47.847Z 16 320-339 143 143 143 237 237 237 F F F
159032024-04-20T01:16:47.847Z 17 340-359 146 146 146 241 241 241 F F F
159042024-04-20T01:16:47.847Z 18 360-379 143 143 143 237 237 237 F F F
159052024-04-20T01:16:47.847Z 19 380-399 146 146 146 241 241 241 F F F
159062024-04-20T01:16:47.847Z 20 400-419 146 146 146 241 241 241 F F F
159072024-04-20T01:16:47.847Z 21 420-439 142 142 142 235 235 235 F F F
159082024-04-20T01:16:47.847Z 22 440-459 144 144 144 238 238 238 F F F
159092024-04-20T01:16:47.847Z 23 460-479 144 144 144 238 238 238 F F F
159102024-04-20T01:16:47.847Z 24 480-499 138 138 138 232 232 232 F F F
159112024-04-20T01:16:47.847Z 25 500-519 146 146 146 241 241 241 F F F
159122024-04-20T01:16:47.847Z 26 520-539 146 146 146 241 241 241 F F F
159132024-04-20T01:16:47.847Z 27 540-559 146 146 146 241 241 241 F F F
159142024-04-20T01:16:47.847Z 28 560-579 146 146 146 241 241 241 F F F
159152024-04-20T01:16:47.847Z 29 580-599 144 144 144 238 238 238 F F F
159162024-04-20T01:16:47.847ZMax gen: 146, Max flush: 241
159172024-04-20T01:16:47.847Z
159182024-04-20T01:16:47.847ZBegin loop 74
159192024-04-20T01:16:47.847ZDownstairs to restart: 1
159202024-04-20T01:16:47.863Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361014737Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766}
159212024-04-20T01:16:47.863Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361253348Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766}
159222024-04-20T01:16:47.869Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361312248Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766}
159232024-04-20T01:16:47.869Z{"msg":"Crucible a778a2d1-6f97-4e70-9389-c9bfd3e1988e has session id: 86cd48c3-4201-4c2e-9828-10bcf718fd29","v":0,"name":"crucible","level":30Crucible runtime is spawned
159242024-04-20T01:16:47.869Z,"time":"2024-04-20T01:16:46.361369608Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159252024-04-20T01:16:47.869Z{"msg":"Upstairs opts: Upstairs UUID: a778a2d1-6f97-4e70-9389-c9bfd3e1988e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361443568Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159262024-04-20T01:16:47.869Z{"msg":"Crucible stats registered with UUID: a778a2d1-6f97-4e70-9389-c9bfd3e1988e","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361477338Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159272024-04-20T01:16:47.869Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361500569Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766}
159282024-04-20T01:16:47.869Z{"msg":"a778a2d1-6f97-4e70-9389-c9bfd3e1988e active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361529538Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159292024-04-20T01:16:47.869Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361551478Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"0","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159302024-04-20T01:16:47.869Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361581149Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"0","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159312024-04-20T01:16:47.869Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361612959Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"1","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159322024-04-20T01:16:47.869Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361636139Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"1","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159332024-04-20T01:16:47.869Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361668819Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"2","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159342024-04-20T01:16:47.869Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361684879Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"2","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159352024-04-20T01:16:47.870Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361709469Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159362024-04-20T01:16:47.870Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361733409Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159372024-04-20T01:16:47.870Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361798589Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159382024-04-20T01:16:47.870Z{"msg":"ds_connection connected from Ok(127.0.0.1:44181)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.361842499Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159392024-04-20T01:16:47.870Z{"msg":"ds_connection connected from Ok(127.0.0.1:40032)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.36207349Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159402024-04-20T01:16:47.870Z{"msg":"ds_connection connected from Ok(127.0.0.1:53867)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.36214076Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159412024-04-20T01:16:47.870Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.362551161Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"0","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159422024-04-20T01:16:47.870Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.362596891Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"0","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159432024-04-20T01:16:47.870Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.362631161Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"2","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159442024-04-20T01:16:47.870Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.362671241Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"2","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159452024-04-20T01:16:47.870Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.363197602Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"2","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159462024-04-20T01:16:47.870Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.363249402Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"0","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159472024-04-20T01:16:47.870Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.363399053Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"1","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159482024-04-20T01:16:47.870Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.363438383Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"1","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159492024-04-20T01:16:47.870Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.363639373Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"0","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159502024-04-20T01:16:47.870Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.363672563Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"0","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159512024-04-20T01:16:47.870Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.363693483Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159522024-04-20T01:16:47.870Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.363719563Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"2","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159532024-04-20T01:16:47.870Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.363734534Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"2","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159542024-04-20T01:16:47.870Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.363754923Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159552024-04-20T01:16:47.870Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.363982844Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"1","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159562024-04-20T01:16:47.870Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364416445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"1","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159572024-04-20T01:16:47.870Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364442595Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"1","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159582024-04-20T01:16:47.871Z{"msg":"[0]R flush_numbers[0..12]: [238, 238, 238, 241, 241, 241, 235, 231, 240, 240, 241, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364457475Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159592024-04-20T01:16:47.871Z{"msg":"[0]R generation[0..12]: [144, 144, 144, 146, 146, 146, 142, 137, 145, 145, 146, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364471715Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159602024-04-20T01:16:47.871Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364492665Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159612024-04-20T01:16:47.871Z{"msg":"[1]R flush_numbers[0..12]: [238, 238, 238, 241, 241, 241, 235, 231, 240, 240, 241, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364519625Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159622024-04-20T01:16:47.871Z{"msg":"[1]R generation[0..12]: [144, 144, 144, 146, 146, 146, 142, 137, 145, 145, 146, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364541145Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159632024-04-20T01:16:47.871Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364555445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159642024-04-20T01:16:47.871Z{"msg":"[2]R flush_numbers[0..12]: [238, 238, 238, 241, 241, 241, 235, 231, 240, 240, 241, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364577215Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159652024-04-20T01:16:47.871Z{"msg":"[2]R generation[0..12]: [144, 144, 144, 146, 146, 146, 142, 137, 145, 145, 146, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364591115Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159662024-04-20T01:16:47.871Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
159672024-04-20T01:16:47.871Z2024-04-20T01:16:46.364605395Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159682024-04-20T01:16:47.871Z{"msg":"Max found gen is 147","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364621596Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159692024-04-20T01:16:47.871Z{"msg":"Generation requested: 148 >= found:147","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364642245Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159702024-04-20T01:16:47.871Z{"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364655445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159712024-04-20T01:16:47.871Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364668356Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"":"downstairs","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159722024-04-20T01:16:47.871Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364688696Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159732024-04-20T01:16:47.871Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364700756Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"0","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159742024-04-20T01:16:47.871Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364713936Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"1","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159752024-04-20T01:16:47.871Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364733566Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"client":"2","":"downstairs","session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159762024-04-20T01:16:47.871Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364747356Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159772024-04-20T01:16:47.871Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364759696Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159782024-04-20T01:16:47.871Z{"msg":"a778a2d1-6f97-4e70-9389-c9bfd3e1988e is now active with session: 86cd48c3-4201-4c2e-9828-10bcf718fd29","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364779546Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159792024-04-20T01:16:47.871Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:46.364791796Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766,"session_id":"86cd48c3-4201-4c2e-9828-10bcf718fd29"}
159802024-04-20T01:16:47.871Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
159812024-04-20T01:16:47.871Z2024-04-20T01:16:46.364803606Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766}
159822024-04-20T01:16:47.876ZLoading write count information from file "/tmp/test_repair_verify.data"
159832024-04-20T01:16:47.876ZRun Repair workload
159842024-04-20T01:16:47.876Z01/30 Write block 242 len 36864 data: 7 6 11 13 12 11 15 15 14
159852024-04-20T01:16:47.876Z02/30 Write block 476 len 32768 data: 12 13 10 7 10 15 15 14
159862024-04-20T01:16:47.876Z03/30 Read block 327 len 12288
159872024-04-20T01:16:47.876Z04/30 Write block 479 len 20480 data: 8 11 16 16 15
159882024-04-20T01:16:47.876Z05/30 Write block 0 len 36864 data: 3 4 6 7 9 8 8 6 6
159892024-04-20T01:16:47.876Z06/30 Read block 219 len 40960
159902024-04-20T01:16:47.876Z07/30 Read block 493 len 16384
159912024-04-20T01:16:47.876Z08/30 Read block 523 len 12288
159922024-04-20T01:16:47.880Z09/30 Write block 22 len 16384 data: 8 6 7 7
159932024-04-20T01:16:47.880Z10/30 Write block 555 len 4096 data: 12
159942024-04-20T01:16:47.880Z11/30 Read block 5 len 12288
159952024-04-20T01:16:47.880Z12/30 Read block 492 len 4096
159962024-04-20T01:16:47.880Z13/30 Flush
159972024-04-20T01:16:47.883Z14/30 Flush
159982024-04-20T01:16:47.883Z15/30 Write block 305 len 40960 data: 9 8 7 7 6 5 5 5 6 7
159992024-04-20T01:16:47.886Z16/30 Write block 460 len 32768 data: 15 17 16 18 15 15 18 18
160002024-04-20T01:16:47.886Z17/30 Read block 279 len 16384
160012024-04-20T01:16:47.886Z18/30 Write block 490 len 16384 data: 10 10 10 9
160022024-04-20T01:16:47.890Z19/30 Write block 332 len 20480 data: 14 12 12 10 11
160032024-04-20T01:16:47.890Z20/30 Read block 333 len 20480
160042024-04-20T01:16:47.890Z21/30 Write block 1 len 12288 data: 5 7 8
160052024-04-20T01:16:47.890Z22/30 Read block 442 len 16384
160062024-04-20T01:16:47.890Z23/30 Read block 105 len 36864
160072024-04-20T01:16:47.893Z24/30 Read block 384 len 40960
160082024-04-20T01:16:47.893Z25/30 Read block 318 len 8192
160092024-04-20T01:16:47.893Z26/30 Read block 315 len 8192
160102024-04-20T01:16:47.893Z27/30 Write block 373 len 8192 data: 20 19
160112024-04-20T01:16:47.893Z28/30 Read block 408 len 24576
160122024-04-20T01:16:47.897Z29/30 Write block 439 len 32768 data: 11 11 12 13 10 11 11 11
160132024-04-20T01:16:47.897Z30/30 Read block 500 len 24576
160142024-04-20T01:16:47.897Z----------------------------------------------------------------
160152024-04-20T01:16:47.897Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
160162024-04-20T01:16:47.897ZStates: Active Active Active
160172024-04-20T01:16:47.897ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
160182024-04-20T01:16:47.898Z 1 Acked 1000 Write 9 Done Sent Done false
160192024-04-20T01:16:47.898Z 2 Acked 1001 Write 8 Done Sent Done false
160202024-04-20T01:16:47.898Z 3 Acked 1002 Read 3 Done Sent Done false
160212024-04-20T01:16:47.898Z 4 Acked 1003 Write 5 Done Sent Done false
160222024-04-20T01:16:47.898Z 5 Acked 1004 Write 9 Done Sent Done false
160232024-04-20T01:16:47.898Z 6 Acked 1005 Read 10 Done Sent Done false
160242024-04-20T01:16:47.898Z 7 Acked 1006 Read 4 Done Sent Done false
160252024-04-20T01:16:47.898Z 8 Acked 1007 Read 3 Done Sent Done false
160262024-04-20T01:16:47.898Z 9 Acked 1008 Write 4 Done Sent Done false
160272024-04-20T01:16:47.898Z 10 Acked 1009 Write 1 Done Sent Done false
160282024-04-20T01:16:47.898Z 11 Acked 1010 Read 3 Done Sent Done false
160292024-04-20T01:16:47.898Z 12 Acked 1011 Read 1 Done Sent Done false
160302024-04-20T01:16:47.898Z 13 Acked 1012 Flush 0 Done Sent Done false
160312024-04-20T01:16:47.898Z 14 Acked 1013 Flush 0 Done Sent Done false
160322024-04-20T01:16:47.898Z 15 Acked 1014 Write 10 Done Sent Done false
160332024-04-20T01:16:47.898Z 16 Acked 1015 Write 8 Done Sent Done false
160342024-04-20T01:16:47.898Z 17 Acked 1016 Read 4 Done Sent Done false
160352024-04-20T01:16:47.898Z 18 Acked 1017 Write 4 Done Sent Done false
160362024-04-20T01:16:47.898Z 19 Acked 1018 Write 5 Done Sent Done false
160372024-04-20T01:16:47.898Z 20 Acked 1019 Read 5 Done Sent Done false
160382024-04-20T01:16:47.898Z 21 Acked 1020 Write 3 Done Sent Done false
160392024-04-20T01:16:47.898Z 22 Acked 1021 Read 4 Done Sent Done false
160402024-04-20T01:16:47.898Z 23 Acked 1022 Read 9 Done Sent Done false
160412024-04-20T01:16:47.899Z 24 Acked 1023 Read 10 Done Sent Done false
160422024-04-20T01:16:47.899Z 25 Acked 1024 Read 2 Done Sent Done false
160432024-04-20T01:16:47.899Z 26 Acked 1025 Read 2 Done Sent Done false
160442024-04-20T01:16:47.899Z 27 Acked 1026 Write 2 Done Sent Done false
160452024-04-20T01:16:47.899Z 28 Acked 1027 Read 6 Done Sent Done false
160462024-04-20T01:16:47.899Z 29 Acked 1028 Write 8 Done Sent Done false
160472024-04-20T01:16:47.899Z 30 Acked 1029 Read 6 Done Sent Sent false
160482024-04-20T01:16:47.899Z STATES DS:0 DS:1 DS:2 TOTAL
160492024-04-20T01:16:47.899Z New 0 0 0 0
160502024-04-20T01:16:47.899Z Sent 0 30 1 31
160512024-04-20T01:16:47.899Z Done 30 0 29 59
160522024-04-20T01:16:47.899Z Skipped 0 0 0 0
160532024-04-20T01:16:47.899Z Error 0 0 0 0
160542024-04-20T01:16:47.899ZLast Flush: 1013 0 1013
160552024-04-20T01:16:47.899ZDownstairs last five completed:
160562024-04-20T01:16:47.899ZUpstairs last five completed: 30 29 28 27 26
160572024-04-20T01:16:47.899Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:16:46.39383333Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1766}
160582024-04-20T01:16:47.899ZWrote out file "/tmp/test_repair_verify.data"
160592024-04-20T01:16:47.903Z
160602024-04-20T01:16:47.903ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
160612024-04-20T01:16:47.920ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160622024-04-20T01:16:47.920Z 0 000-019 148 144 148 242 238 242 T F T <---
160632024-04-20T01:16:47.920Z 1 020-039 148 144 148 242 238 242 F F F <---
160642024-04-20T01:16:47.920Z 2 040-059 144 144 144 238 238 238 F F F
160652024-04-20T01:16:47.920Z 3 060-079 146 146 146 241 241 241 F F F
160662024-04-20T01:16:47.920Z 4 080-099 146 146 146 241 241 241 F F F
160672024-04-20T01:16:47.924Z 5 100-119 146 146 146 241 241 241 F F F
160682024-04-20T01:16:47.924Z 6 120-139 142 142 142 235 235 235 F F F
160692024-04-20T01:16:47.924Z 7 140-159 137 137 137 231 231 231 F F F
160702024-04-20T01:16:47.924Z 8 160-179 145 145 145 240 240 240 F F F
160712024-04-20T01:16:47.924Z 9 180-199 145 145 145 240 240 240 F F F
160722024-04-20T01:16:47.924Z 10 200-219 146 146 146 241 241 241 F F F
160732024-04-20T01:16:47.924Z 11 220-239 144 144 144 238 238 238 F F F
160742024-04-20T01:16:47.924Z 12 240-259 148 139 148 242 233 242 F F F <---
160752024-04-20T01:16:47.925Z 13 260-279 145 145 145 240 240 240 F F F
160762024-04-20T01:16:47.925Z 14 280-299 146 146 146 241 241 241 F F F
160772024-04-20T01:16:47.925Z 15 300-319 144 144 144 239 239 239 T F T <---
160782024-04-20T01:16:47.925Z 16 320-339 143 143 143 237 237 237 T F T <---
160792024-04-20T01:16:47.925Z 17 340-359 146 146 146 241 241 241 F F F
160802024-04-20T01:16:47.925Z 18 360-379 143 143 143 237 237 237 T F T <---
160812024-04-20T01:16:47.925Z 19 380-399 146 146 146 241 241 241 F F F
160822024-04-20T01:16:47.925Z 20 400-419 146 146 146 241 241 241 F F F
160832024-04-20T01:16:47.925Z 21 420-439 142 142 142 235 235 235 T F T <---
160842024-04-20T01:16:47.925Z 22 440-459 144 144 144 238 238 238 T F T <---
160852024-04-20T01:16:47.925Z 23 460-479 148 144 148 242 238 242 T F T <---
160862024-04-20T01:16:47.925Z 24 480-499 148 138 148 242 232 242 T F T <---
160872024-04-20T01:16:47.925Z 25 500-519 146 146 146 241 241 241 F F F
160882024-04-20T01:16:47.925Z 26 520-539 146 146 146 241 241 241 F F F
160892024-04-20T01:16:47.925Z 27 540-559 148 146 148 242 241 242 F F F <---
160902024-04-20T01:16:47.925Z 28 560-579 146 146 146 241 241 241 F F F
160912024-04-20T01:16:47.925Z 29 580-599 144 144 144 238 238 238 F F F
160922024-04-20T01:16:47.925ZMax gen: 148, Max flush: 242
160932024-04-20T01:16:47.925ZError: Difference in extent metadata found!
160942024-04-20T01:16:47.925ZA Difference in extent metadata is expected here
160952024-04-20T01:16:47.925ZOn loop 74
160962024-04-20T01:16:47.925Z
160972024-04-20T01:16:47.929ZVerifying data now
160982024-04-20T01:16:52.443ZNo change after verify
160992024-04-20T01:16:52.443ZLoop: 74 Downstairs dump after verify (and repair):
161002024-04-20T01:16:52.457ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
161012024-04-20T01:16:52.457Z 0 000-019 149 149 149 243 243 243 F F F
161022024-04-20T01:16:52.457Z 1 020-039 148 148 148 242 242 242 F F F
161032024-04-20T01:16:52.457Z 2 040-059 144 144 144 238 238 238 F F F
161042024-04-20T01:16:52.457Z 3 060-079 146 146 146 241 241 241 F F F
161052024-04-20T01:16:52.457Z 4 080-099 146 146 146 241 241 241 F F F
161062024-04-20T01:16:52.458Z 5 100-119 146 146 146 241 241 241 F F F
161072024-04-20T01:16:52.458Z 6 120-139 142 142 142 235 235 235 F F F
161082024-04-20T01:16:52.458Z 7 140-159 137 137 137 231 231 231 F F F
161092024-04-20T01:16:52.458Z 8 160-179 145 145 145 240 240 240 F F F
161102024-04-20T01:16:52.461Z 9 180-199 145 145 145 240 240 240 F F F
161112024-04-20T01:16:52.461Z 10 200-219 146 146 146 241 241 241 F F F
161122024-04-20T01:16:52.461Z 11 220-239 144 144 144 238 238 238 F F F
161132024-04-20T01:16:52.462Z 12 240-259 148 148 148 242 242 242 F F F
161142024-04-20T01:16:52.462Z 13 260-279 145 145 145 240 240 240 F F F
161152024-04-20T01:16:52.462Z 14 280-299 146 146 146 241 241 241 F F F
161162024-04-20T01:16:52.462Z 15 300-319 149 149 149 243 243 243 F F F
161172024-04-20T01:16:52.462Z 16 320-339 149 149 149 243 243 243 F F F
161182024-04-20T01:16:52.462Z 17 340-359 146 146 146 241 241 241 F F F
161192024-04-20T01:16:52.462Z 18 360-379 149 149 149 243 243 243 F F F
161202024-04-20T01:16:52.462Z 19 380-399 146 146 146 241 241 241 F F F
161212024-04-20T01:16:52.462Z 20 400-419 146 146 146 241 241 241 F F F
161222024-04-20T01:16:52.462Z 21 420-439 149 149 149 243 243 243 F F F
161232024-04-20T01:16:52.462Z 22 440-459 149 149 149 243 243 243 F F F
161242024-04-20T01:16:52.462Z 23 460-479 149 149 149 243 243 243 F F F
161252024-04-20T01:16:52.462Z 24 480-499 149 149 149 243 243 243 F F F
161262024-04-20T01:16:52.462Z 25 500-519 146 146 146 241 241 241 F F F
161272024-04-20T01:16:52.462Z 26 520-539 146 146 146 241 241 241 F F F
161282024-04-20T01:16:52.462Z 27 540-559 148 148 148 242 242 242 F F F
161292024-04-20T01:16:52.462Z 28 560-579 146 146 146 241 241 241 F F F
161302024-04-20T01:16:52.462Z 29 580-599 144 144 144 238 238 238 F F F
161312024-04-20T01:16:52.462ZMax gen: 149, Max flush: 243
161322024-04-20T01:16:52.462Z
161332024-04-20T01:16:52.462ZBegin loop 75
161342024-04-20T01:16:52.463ZDownstairs to restart: 0
161352024-04-20T01:16:52.479Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977169923Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774}
161362024-04-20T01:16:52.479Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977389353Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774}
161372024-04-20T01:16:52.479Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977416143Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774}
161382024-04-20T01:16:52.486Z{"msg":"Crucible 530a5c9f-1fce-489b-8832-b02f401c9652 has session id: eb8dd6cb-132b-4512-b3fd-608bf5adf310","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977449843Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161392024-04-20T01:16:52.486Z{"msg":"Crucible runtime is spawned
161402024-04-20T01:16:52.486ZUpstairs opts: Upstairs UUID: 530a5c9f-1fce-489b-8832-b02f401c9652, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977515093Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161412024-04-20T01:16:52.486Z{"msg":"Crucible stats registered with UUID: 530a5c9f-1fce-489b-8832-b02f401c9652","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977553464Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161422024-04-20T01:16:52.486Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977570813Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774}
161432024-04-20T01:16:52.486Z{"msg":"530a5c9f-1fce-489b-8832-b02f401c9652 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977656564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161442024-04-20T01:16:52.486Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977694344Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"0","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161452024-04-20T01:16:52.486Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977721534Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"0","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161462024-04-20T01:16:52.486Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977745204Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"1","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161472024-04-20T01:16:52.486Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977780354Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"1","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161482024-04-20T01:16:52.486Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977800034Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"2","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161492024-04-20T01:16:52.486Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977833504Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"2","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161502024-04-20T01:16:52.486Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977858904Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161512024-04-20T01:16:52.486Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977884034Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161522024-04-20T01:16:52.486Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977908924Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161532024-04-20T01:16:52.486Z{"msg":"ds_connection connected from Ok(127.0.0.1:33593)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977942724Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161542024-04-20T01:16:52.486Z{"msg":"ds_connection connected from Ok(127.0.0.1:45309)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977968225Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161552024-04-20T01:16:52.487Z{"msg":"ds_connection connected from Ok(127.0.0.1:56886)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.977996954Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161562024-04-20T01:16:52.487Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.978690576Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"2","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161572024-04-20T01:16:52.487Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.978731906Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"2","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161582024-04-20T01:16:52.487Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.978747956Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"1","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161592024-04-20T01:16:52.487Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.978766896Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"1","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161602024-04-20T01:16:52.487Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.979223077Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"1","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161612024-04-20T01:16:52.487Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.979259227Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"2","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161622024-04-20T01:16:52.487Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.979682248Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"0","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161632024-04-20T01:16:52.487Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.979714948Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"0","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161642024-04-20T01:16:52.487Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.979861589Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"1","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161652024-04-20T01:16:52.487Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.979892659Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"1","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161662024-04-20T01:16:52.487Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.979931619Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161672024-04-20T01:16:52.487Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.979951609Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"2","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161682024-04-20T01:16:52.487Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.979965409Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"2","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161692024-04-20T01:16:52.487Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.979979399Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161702024-04-20T01:16:52.487Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.98020237Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"0","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161712024-04-20T01:16:52.487Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.98061296Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"0","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161722024-04-20T01:16:52.487Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980652651Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"0","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161732024-04-20T01:16:52.487Z{"msg":"[0]R flush_numbers[0..12]: [243, 242, 238, 241, 241, 241, 235, 231, 240, 240, 241, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980669191Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161742024-04-20T01:16:52.487Z{"msg":"[0]R generation[0..12]: [149, 148, 144, 146, 146, 146, 142, 137, 145, 145, 146, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.98069064Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161752024-04-20T01:16:52.487Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.98070602Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161762024-04-20T01:16:52.487Z{"msg":"[1]R flush_numbers[0..12]: [243, 242, 238, 241, 241, 241, 235, 231, 240, 240, 241, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.98072722Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161772024-04-20T01:16:52.488Z{"msg":"[1]R generation[0..12]: [149, 148, 144, 146, 146, 146, 142, 137, 145, 145, 146, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980741171Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161782024-04-20T01:16:52.488Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980756551Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161792024-04-20T01:16:52.488Z{"msg":"[2]R flush_numbers[0..12]: [243, 242, 238, 241, 241, 241, 235, 231, 240, 240, 241, 238]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
161802024-04-20T01:16:52.488Z,"time":"2024-04-20T01:16:50.980777691Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161812024-04-20T01:16:52.488Z{"msg":"[2]R generation[0..12]: [149, 148, 144, 146, 146, 146, 142, 137, 145, 145, 146, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980791871Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161822024-04-20T01:16:52.488Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980817991Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161832024-04-20T01:16:52.488Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980831041Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161842024-04-20T01:16:52.488Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980851401Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161852024-04-20T01:16:52.488Z{"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980864621Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161862024-04-20T01:16:52.488Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980878611Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"":"downstairs","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161872024-04-20T01:16:52.488Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980892091Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161882024-04-20T01:16:52.488Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980904361Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"0","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161892024-04-20T01:16:52.488Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980925621Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"1","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161902024-04-20T01:16:52.488Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980947501Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"client":"2","":"downstairs","session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161912024-04-20T01:16:52.488Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980961151Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161922024-04-20T01:16:52.488Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980973181Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161932024-04-20T01:16:52.488Z{"msg":"530a5c9f-1fce-489b-8832-b02f401c9652 is now active with session: eb8dd6cb-132b-4512-b3fd-608bf5adf310","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.980986701Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161942024-04-20T01:16:52.488Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.981026881Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774,"session_id":"eb8dd6cb-132b-4512-b3fd-608bf5adf310"}
161952024-04-20T01:16:52.488Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:16:50.981042081Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774}
161962024-04-20T01:16:52.488ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
161972024-04-20T01:16:52.493ZLoading write count information from file "/tmp/test_repair_verify.data"
161982024-04-20T01:16:52.493ZRun Repair workload
161992024-04-20T01:16:52.493Z01/30 Write block 521 len 8192 data: 18 17
162002024-04-20T01:16:52.493Z02/30 Read block 404 len 28672
162012024-04-20T01:16:52.493Z03/30 Write block 13 len 20480 data: 6 8 11 12 10
162022024-04-20T01:16:52.493Z04/30 Write block 568 len 40960 data: 15 17 16 15 11 9 8 8 9 11
162032024-04-20T01:16:52.493Z05/30 Read block 377 len 36864
162042024-04-20T01:16:52.493Z06/30 Read block 391 len 28672
162052024-04-20T01:16:52.493Z07/30 Read block 562 len 24576
162062024-04-20T01:16:52.493Z08/30 Read block 360 len 12288
162072024-04-20T01:16:52.493Z09/30 Read block 191 len 12288
162082024-04-20T01:16:52.496Z10/30 Write block 300 len 32768 data: 7 9 10 10 10 10 9 8
162092024-04-20T01:16:52.497Z11/30 Write block 166 len 4096 data: 11
162102024-04-20T01:16:52.497Z12/30 Read block 411 len 16384
162112024-04-20T01:16:52.497Z13/30 Read block 436 len 16384
162122024-04-20T01:16:52.497Z14/30 Read block 146 len 16384
162132024-04-20T01:16:52.497Z15/30 Read block 313 len 8192
162142024-04-20T01:16:52.500Z16/30 Write block 42 len 32768 data: 9 6 7 7 8 7 8 10
162152024-04-20T01:16:52.500Z17/30 Read block 517 len 40960
162162024-04-20T01:16:52.500Z18/30 Write block 363 len 20480 data: 12 9 9 13 17
162172024-04-20T01:16:52.500Z19/30 Flush
162182024-04-20T01:16:52.504Z20/30 Write block 577 len 36864 data: 12 11 15 15 14 13 10 14 17
162192024-04-20T01:16:52.507Z21/30 Read block 425 len 36864
162202024-04-20T01:16:52.507Z22/30 Write block 273 len 40960 data: 14 11 10 8 10 10 12 13 15 12
162212024-04-20T01:16:52.507Z23/30 Flush
162222024-04-20T01:16:52.510Z24/30 Write block 542 len 8192 data: 13 16
162232024-04-20T01:16:52.511Z25/30 Flush
162242024-04-20T01:16:52.514Z26/30 Write block 94 len 40960 data: 8 6 6 7 8 8 11 11 11 12
162252024-04-20T01:16:52.514Z27/30 Read block 202 len 28672
162262024-04-20T01:16:52.514Z28/30 Read block 53 len 20480
162272024-04-20T01:16:52.521Z29/30 Read block 397 len 40960
162282024-04-20T01:16:52.521Z30/30 Write block 542 len 36864 data: 14 17 14 12 13 9 11 14 16
162292024-04-20T01:16:52.521Z----------------------------------------------------------------
162302024-04-20T01:16:52.521Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
162312024-04-20T01:16:52.521ZStates: Active Active Active
162322024-04-20T01:16:52.521ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
162332024-04-20T01:16:52.521Z 1 Acked 1000 Write 2 Done Done Done false
162342024-04-20T01:16:52.522Z 2 Acked 1001 Read 7 Done Done Done false
162352024-04-20T01:16:52.522Z 3 Acked 1002 Write 5 Done Done Done false
162362024-04-20T01:16:52.522Z 4 Acked 1003 Write 10 Done Done Done false
162372024-04-20T01:16:52.522Z 5 Acked 1004 Read 9 Done Done Done false
162382024-04-20T01:16:52.522Z 6 Acked 1005 Read 7 Done Done Done false
162392024-04-20T01:16:52.522Z 7 Acked 1006 Read 6 Sent Done Done false
162402024-04-20T01:16:52.522Z 8 Acked 1007 Read 3 Sent Done Done false
162412024-04-20T01:16:52.522Z 9 Acked 1008 Read 3 Sent Done Done false
162422024-04-20T01:16:52.522Z 10 Acked 1009 Write 8 Sent Done Done false
162432024-04-20T01:16:52.522Z 11 Acked 1010 Write 1 Sent Done Done false
162442024-04-20T01:16:52.522Z 12 Acked 1011 Read 4 Sent Done Done false
162452024-04-20T01:16:52.522Z 13 Acked 1012 Read 4 Sent Done Done false
162462024-04-20T01:16:52.522Z 14 Acked 1013 Read 4 Sent Done Done false
162472024-04-20T01:16:52.522Z 15 Acked 1014 Read 2 Sent Done Done false
162482024-04-20T01:16:52.522Z 16 Acked 1015 Write 8 Sent Done Done false
162492024-04-20T01:16:52.522Z 17 Acked 1016 Read 10 Sent Done Done false
162502024-04-20T01:16:52.522Z 18 Acked 1017 Write 5 Sent Done Done false
162512024-04-20T01:16:52.522Z 19 Acked 1018 Flush 0 Sent Done Done false
162522024-04-20T01:16:52.522Z 20 Acked 1019 Write 9 Sent Done Done false
162532024-04-20T01:16:52.522Z 21 Acked 1020 Read 9 Sent Done Done false
162542024-04-20T01:16:52.522Z 22 Acked 1021 Write 10 Sent Done Done false
162552024-04-20T01:16:52.522Z 23 Acked 1022 Flush 0 Sent Done Done false
162562024-04-20T01:16:52.522Z 24 Acked 1023 Write 2 Sent Done Done false
162572024-04-20T01:16:52.522Z 25 Acked 1024 Flush 0 Sent Done Done false
162582024-04-20T01:16:52.523Z 26 Acked 1025 Write 10 Sent Done Done false
162592024-04-20T01:16:52.523Z 27 Acked 1026 Read 7 Sent Done Done false
162602024-04-20T01:16:52.523Z 28 Acked 1027 Read 5 Sent Done Done false
162612024-04-20T01:16:52.523Z 29 Acked 1028 Read 10 Sent Done Done false
162622024-04-20T01:16:52.523Z 30 Acked 1029 Write 9 Sent Sent Sent false
162632024-04-20T01:16:52.523Z STATES DS:0 DS:1 DS:2 TOTAL
162642024-04-20T01:16:52.523Z New 0 0 0 0
162652024-04-20T01:16:52.523Z Sent 24 1 1 26
162662024-04-20T01:16:52.523Z Done 6 29 29 64
162672024-04-20T01:16:52.523Z Skipped 0 0 0 0
162682024-04-20T01:16:52.523Z Error 0 0 0 0
162692024-04-20T01:16:52.523ZLast Flush: 0 1024 1024
162702024-04-20T01:16:52.523ZDownstairs last five completed:
162712024-04-20T01:16:52.523ZUpstairs last five completed: 30 29 28 27 26
162722024-04-20T01:16:52.523Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:16:51.015844039Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1774}
162732024-04-20T01:16:52.523ZWrote out file "/tmp/test_repair_verify.data"
162742024-04-20T01:16:52.527Z
162752024-04-20T01:16:52.527ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
162762024-04-20T01:16:52.541ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162772024-04-20T01:16:52.541Z 0 000-019 149 150 150 243 244 244 T F F <---
162782024-04-20T01:16:52.541Z 1 020-039 148 148 148 242 242 242 F F F
162792024-04-20T01:16:52.541Z 2 040-059 144 150 150 238 244 244 F F F <---
162802024-04-20T01:16:52.541Z 3 060-079 146 146 146 241 241 241 F F F
162812024-04-20T01:16:52.541Z 4 080-099 146 146 146 241 241 241 F T T <---
162822024-04-20T01:16:52.541Z 5 100-119 146 146 146 241 241 241 F T T <---
162832024-04-20T01:16:52.541Z 6 120-139 142 142 142 235 235 235 F F F
162842024-04-20T01:16:52.541Z 7 140-159 137 137 137 231 231 231 F F F
162852024-04-20T01:16:52.541Z 8 160-179 145 150 150 240 244 244 F F F <---
162862024-04-20T01:16:52.545Z 9 180-199 145 145 145 240 240 240 F F F
162872024-04-20T01:16:52.545Z 10 200-219 146 146 146 241 241 241 F F F
162882024-04-20T01:16:52.545Z 11 220-239 144 144 144 238 238 238 F F F
162892024-04-20T01:16:52.545Z 12 240-259 148 148 148 242 242 242 F F F
162902024-04-20T01:16:52.545Z 13 260-279 145 150 150 240 245 245 F F F <---
162912024-04-20T01:16:52.545Z 14 280-299 146 150 150 241 245 245 F F F <---
162922024-04-20T01:16:52.545Z 15 300-319 149 150 150 243 244 244 F F F <---
162932024-04-20T01:16:52.545Z 16 320-339 149 149 149 243 243 243 F F F
162942024-04-20T01:16:52.545Z 17 340-359 146 146 146 241 241 241 F F F
162952024-04-20T01:16:52.546Z 18 360-379 149 150 150 243 244 244 F F F <---
162962024-04-20T01:16:52.546Z 19 380-399 146 146 146 241 241 241 F F F
162972024-04-20T01:16:52.546Z 20 400-419 146 146 146 241 241 241 F F F
162982024-04-20T01:16:52.546Z 21 420-439 149 149 149 243 243 243 F F F
162992024-04-20T01:16:52.546Z 22 440-459 149 149 149 243 243 243 F F F
163002024-04-20T01:16:52.546Z 23 460-479 149 149 149 243 243 243 F F F
163012024-04-20T01:16:52.546Z 24 480-499 149 149 149 243 243 243 F F F
163022024-04-20T01:16:52.546Z 25 500-519 146 146 146 241 241 241 F F F
163032024-04-20T01:16:52.546Z 26 520-539 146 150 150 241 244 244 T F F <---
163042024-04-20T01:16:52.546Z 27 540-559 148 150 150 242 246 246 F T T <---
163052024-04-20T01:16:52.546Z 28 560-579 146 150 150 241 245 245 T F F <---
163062024-04-20T01:16:52.546Z 29 580-599 144 150 150 238 245 245 F F F <---
163072024-04-20T01:16:52.546ZMax gen: 150, Max flush: 246
163082024-04-20T01:16:52.546ZError: Difference in extent metadata found!
163092024-04-20T01:16:52.546ZA Difference in extent metadata is expected here
163102024-04-20T01:16:52.546ZOn loop 75
163112024-04-20T01:16:52.546Z
163122024-04-20T01:16:52.550ZVerifying data now
163132024-04-20T01:17:07.141ZNo change after verify
163142024-04-20T01:17:07.141ZLoop: 75 Downstairs dump after verify (and repair):
163152024-04-20T01:17:07.156ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163162024-04-20T01:17:07.156Z 0 000-019 150 150 150 244 244 244 F F F
163172024-04-20T01:17:07.162Z 1 020-039 148 148 148 242 242 242 F F F
163182024-04-20T01:17:07.163Z 2 040-059 150 150 150 244 244 244 F F F
163192024-04-20T01:17:07.163Z 3 060-079 146 146 146 241 241 241 F F F
163202024-04-20T01:17:07.163Z 4 080-099 151 151 151 247 247 247 F F F
163212024-04-20T01:17:07.163Z 5 100-119 151 151 151 247 247 247 F F F
163222024-04-20T01:17:07.163Z 6 120-139 142 142 142 235 235 235 F F F
163232024-04-20T01:17:07.163Z 7 140-159 137 137 137 231 231 231 F F F
163242024-04-20T01:17:07.163Z 8 160-179 150 150 150 244 244 244 F F F
163252024-04-20T01:17:07.163Z 9 180-199 145 145 145 240 240 240 F F F
163262024-04-20T01:17:07.163Z 10 200-219 146 146 146 241 241 241 F F F
163272024-04-20T01:17:07.163Z 11 220-239 144 144 144 238 238 238 F F F
163282024-04-20T01:17:07.163Z 12 240-259 148 148 148 242 242 242 F F F
163292024-04-20T01:17:07.163Z 13 260-279 150 150 150 245 245 245 F F F
163302024-04-20T01:17:07.163Z 14 280-299 150 150 150 245 245 245 F F F
163312024-04-20T01:17:07.163Z 15 300-319 150 150 150 244 244 244 F F F
163322024-04-20T01:17:07.164Z 16 320-339 149 149 149 243 243 243 F F F
163332024-04-20T01:17:07.164Z 17 340-359 146 146 146 241 241 241 F F F
163342024-04-20T01:17:07.164Z 18 360-379 150 150 150 244 244 244 F F F
163352024-04-20T01:17:07.164Z 19 380-399 146 146 146 241 241 241 F F F
163362024-04-20T01:17:07.164Z 20 400-419 146 146 146 241 241 241 F F F
163372024-04-20T01:17:07.164Z 21 420-439 149 149 149 243 243 243 F F F
163382024-04-20T01:17:07.164Z 22 440-459 149 149 149 243 243 243 F F F
163392024-04-20T01:17:07.164Z 23 460-479 149 149 149 243 243 243 F F F
163402024-04-20T01:17:07.164Z 24 480-499 149 149 149 243 243 243 F F F
163412024-04-20T01:17:07.164Z 25 500-519 146 146 146 241 241 241 F F F
163422024-04-20T01:17:07.164Z 26 520-539 150 150 150 244 244 244 F F F
163432024-04-20T01:17:07.164Z 27 540-559 151 151 151 247 247 247 F F F
163442024-04-20T01:17:07.164Z 28 560-579 150 150 150 245 245 245 F F F
163452024-04-20T01:17:07.164Z 29 580-599 150 150 150 245 245 245 F F F
163462024-04-20T01:17:07.165ZMax gen: 151, Max flush: 247
163472024-04-20T01:17:07.165Z
163482024-04-20T01:17:07.165ZBegin loop 76
163492024-04-20T01:17:07.165ZDownstairs to restart: 0
163502024-04-20T01:17:07.179Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676035412Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782}
163512024-04-20T01:17:07.179Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676284623Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782}
163522024-04-20T01:17:07.179Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676302183Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782}
163532024-04-20T01:17:07.185Z{"msg":"Crucible db8e43c9-758b-4517-990e-1d00f7e02294 has session id: 800c8181-a69b-4cb7-ba29-b3834171c345","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676336173Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163542024-04-20T01:17:07.185Z{"msg":"Upstairs opts: Upstairs UUID: db8e43c9-758b-4517-990e-1d00f7e02294, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"Crucible runtime is spawned
163552024-04-20T01:17:07.185Zname":"crucible","level":30,"time":"2024-04-20T01:17:05.676414513Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163562024-04-20T01:17:07.185Z{"msg":"Crucible stats registered with UUID: db8e43c9-758b-4517-990e-1d00f7e02294","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676442213Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163572024-04-20T01:17:07.185Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676462763Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782}
163582024-04-20T01:17:07.185Z{"msg":"db8e43c9-758b-4517-990e-1d00f7e02294 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676610733Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163592024-04-20T01:17:07.185Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676644994Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"0","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163602024-04-20T01:17:07.185Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676670054Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"0","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163612024-04-20T01:17:07.185Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676693674Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"1","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163622024-04-20T01:17:07.185Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676715964Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"1","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163632024-04-20T01:17:07.185Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676740034Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"2","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163642024-04-20T01:17:07.185Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676765494Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"2","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163652024-04-20T01:17:07.185Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676792004Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163662024-04-20T01:17:07.185Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.676994505Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163672024-04-20T01:17:07.185Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.677032434Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163682024-04-20T01:17:07.185Z{"msg":"ds_connection connected from Ok(127.0.0.1:35240)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.677078175Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163692024-04-20T01:17:07.185Z{"msg":"ds_connection connected from Ok(127.0.0.1:55661)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.677107955Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163702024-04-20T01:17:07.185Z{"msg":"ds_connection connected from Ok(127.0.0.1:35740)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.677144975Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163712024-04-20T01:17:07.186Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.677676746Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"1","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163722024-04-20T01:17:07.186Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.677719106Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"1","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163732024-04-20T01:17:07.186Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.677795506Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"2","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163742024-04-20T01:17:07.186Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.677847806Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"2","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163752024-04-20T01:17:07.186Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.678227927Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"0","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163762024-04-20T01:17:07.186Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.678288217Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"0","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163772024-04-20T01:17:07.186Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.678333907Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"1","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163782024-04-20T01:17:07.186Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.678359788Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"2","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163792024-04-20T01:17:07.186Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.678689238Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"1","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163802024-04-20T01:17:07.186Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.678742628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"1","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163812024-04-20T01:17:07.186Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.678791569Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163822024-04-20T01:17:07.186Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.678840349Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"0","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163832024-04-20T01:17:07.186Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.678887159Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"2","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163842024-04-20T01:17:07.186Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.678927589Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"2","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163852024-04-20T01:17:07.186Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.678952879Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163862024-04-20T01:17:07.186Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.679217579Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"0","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163872024-04-20T01:17:07.186Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67926317Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"0","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163882024-04-20T01:17:07.186Z{"msg":"[0]R flush_numbers[0..12]: [244, 242, 244, 241, 247, 247, 235, 231, 244, 240, 241, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67930001Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163892024-04-20T01:17:07.186Z{"msg":"[0]R generation[0..12]: [150, 148, 150, 146, 151, 151, 142, 137, 150, 145, 146, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67933675Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163902024-04-20T01:17:07.186Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67937306Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163912024-04-20T01:17:07.186Z{"msg":"[1]R flush_numbers[0..12]: [244, 242, 244, 241, 247, 247, 235, 231, 244, 240, 241, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67941754Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163922024-04-20T01:17:07.187ZWait for a query_work_queue command to finish before sending IO
163932024-04-20T01:17:07.187Z{"msg":"[1]R generation[0..12]: [150, 148, 150, 146, 151, 151, 142, 137, 150, 145, 146, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67945953Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163942024-04-20T01:17:07.187Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67950077Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163952024-04-20T01:17:07.187Z{"msg":"[2]R flush_numbers[0..12]: [244, 242, 244, 241, 247, 247, 235, 231, 244, 240, 241, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67952965Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163962024-04-20T01:17:07.187Z{"msg":"[2]R generation[0..12]: [150, 148, 150, 146, 151, 151, 142, 137, 150, 145, 146, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67955732Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163972024-04-20T01:17:07.187Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67959879Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163982024-04-20T01:17:07.187Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67963434Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
163992024-04-20T01:17:07.187Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67967093Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
164002024-04-20T01:17:07.187Z{"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.67970716Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
164012024-04-20T01:17:07.187Z"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
164022024-04-20T01:17:07.187Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.679748691Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"":"downstairs","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
164032024-04-20T01:17:07.187Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.679775811Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
164042024-04-20T01:17:07.187Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.679789511Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"0","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
164052024-04-20T01:17:07.187Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.679810361Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"1","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
164062024-04-20T01:17:07.187Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.679824491Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"client":"2","":"downstairs","session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
164072024-04-20T01:17:07.187Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.679845281Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
164082024-04-20T01:17:07.187Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.679858191Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
164092024-04-20T01:17:07.187Z{"msg":"db8e43c9-758b-4517-990e-1d00f7e02294 is now active with session: 800c8181-a69b-4cb7-ba29-b3834171c345","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.679879311Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
164102024-04-20T01:17:07.187Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.679891821Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782,"session_id":"800c8181-a69b-4cb7-ba29-b3834171c345"}
164112024-04-20T01:17:07.187Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:05.679911681Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782}
164122024-04-20T01:17:07.187ZLoading write count information from file "/tmp/test_repair_verify.data"
164132024-04-20T01:17:07.188ZRun Repair workload
164142024-04-20T01:17:07.300Z01/30 Write block 463 len 20480 data: 19 16 16 19 19
164152024-04-20T01:17:07.300Z02/30 Write block 431 len 20480 data: 11 9 9 11 12
164162024-04-20T01:17:07.300Z03/30 Read block 190 len 4096
164172024-04-20T01:17:07.300Z04/30 Read block 301 len 4096
164182024-04-20T01:17:07.300Z05/30 Read block 70 len 32768
164192024-04-20T01:17:07.300Z06/30 Write block 512 len 24576 data: 16 15 13 12 17 17
164202024-04-20T01:17:07.300Z07/30 Read block 477 len 32768
164212024-04-20T01:17:07.300Z08/30 Write block 93 len 24576 data: 8 9 7 7 8 9
164222024-04-20T01:17:07.300Z09/30 Read block 125 len 24576
164232024-04-20T01:17:07.301Z10/30 Write block 540 len 24576 data: 8 9 15 18 15 13
164242024-04-20T01:17:07.301Z11/30 Flush
164252024-04-20T01:17:07.301Z12/30 Write block 215 len 16384 data: 11 13 17 16
164262024-04-20T01:17:07.301Z13/30 Write block 293 len 12288 data: 7 10 9
164272024-04-20T01:17:07.301Z14/30 Write block 376 len 4096 data: 12
164282024-04-20T01:17:07.301Z15/30 Flush
164292024-04-20T01:17:07.301Z16/30 Write block 332 len 4096 data: 15
164302024-04-20T01:17:07.301Z17/30 Read block 411 len 12288
164312024-04-20T01:17:07.301Z18/30 Write block 37 len 12288 data: 11 12 13
164322024-04-20T01:17:07.301Z19/30 Write block 187 len 36864 data: 10 13 12 15 17 13 13 14 12
164332024-04-20T01:17:07.301Z20/30 Write block 279 len 32768 data: 13 14 16 13 9 10 10 10
164342024-04-20T01:17:07.301Z21/30 Flush
164352024-04-20T01:17:07.301Z22/30 Write block 477 len 16384 data: 14 11 9 12
164362024-04-20T01:17:07.301Z23/30 Flush
164372024-04-20T01:17:07.301Z24/30 Flush
164382024-04-20T01:17:07.301Z25/30 Write block 153 len 4096 data: 9
164392024-04-20T01:17:07.301Z26/30 Write block 144 len 36864 data: 17 17 15 12 11 9 7 9 8
164402024-04-20T01:17:07.310Z27/30 Write block 342 len 40960 data: 9 11 11 15 11 10 10 9 7 8
164412024-04-20T01:17:07.310Z28/30 Write block 325 len 28672 data: 13 14 12 13 11 14 12
164422024-04-20T01:17:07.310Z29/30 Read block 49 len 36864
164432024-04-20T01:17:07.310Z30/30 Read block 201 len 4096
164442024-04-20T01:17:07.310Z----------------------------------------------------------------
164452024-04-20T01:17:07.311Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
164462024-04-20T01:17:07.311ZStates: Active Active Active
164472024-04-20T01:17:07.311ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
164482024-04-20T01:17:07.311Z 1 Acked 1000 Write 5 Done Done Done false
164492024-04-20T01:17:07.311Z 2 Acked 1001 Write 5 Done Done Done false
164502024-04-20T01:17:07.311Z 3 Acked 1002 Read 1 Sent Done Done false
164512024-04-20T01:17:07.311Z 4 Acked 1003 Read 1 Sent Done Done false
164522024-04-20T01:17:07.311Z 5 Acked 1004 Read 8 Sent Done Done false
164532024-04-20T01:17:07.311Z 6 Acked 1005 Write 6 Sent Done Done false
164542024-04-20T01:17:07.311Z 7 Acked 1006 Read 8 Sent Done Done false
164552024-04-20T01:17:07.311Z 8 Acked 1007 Write 6 Sent Done Done false
164562024-04-20T01:17:07.311Z 9 Acked 1008 Read 6 Sent Done Done false
164572024-04-20T01:17:07.311Z 10 Acked 1009 Write 6 Sent Done Done false
164582024-04-20T01:17:07.311Z 11 Acked 1010 Flush 0 Sent Done Done false
164592024-04-20T01:17:07.311Z 12 Acked 1011 Write 4 Sent Done Done false
164602024-04-20T01:17:07.311Z 13 Acked 1012 Write 3 Sent Done Done false
164612024-04-20T01:17:07.311Z 14 Acked 1013 Write 1 Sent Done Done false
164622024-04-20T01:17:07.311Z 15 Acked 1014 Flush 0 Sent Done Done false
164632024-04-20T01:17:07.311Z 16 Acked 1015 Write 1 Sent Done Done false
164642024-04-20T01:17:07.312Z 17 Acked 1016 Read 3 Sent Done Done false
164652024-04-20T01:17:07.312Z 18 Acked 1017 Write 3 Sent Done Done false
164662024-04-20T01:17:07.312Z 19 Acked 1018 Write 9 Sent Done Done false
164672024-04-20T01:17:07.312Z 20 Acked 1019 Write 8 Sent Done Done false
164682024-04-20T01:17:07.312Z 21 Acked 1020 Flush 0 Sent Done Done false
164692024-04-20T01:17:07.312Z 22 Acked 1021 Write 4 Sent Done Done false
164702024-04-20T01:17:07.312Z 23 Acked 1022 Flush 0 Sent Done Done false
164712024-04-20T01:17:07.312Z 24 Acked 1023 Flush 0 Sent Done Done false
164722024-04-20T01:17:07.312Z 25 Acked 1024 Write 1 Sent Done Done false
164732024-04-20T01:17:07.312Z 26 Acked 1025 Write 9 Sent Done Done false
164742024-04-20T01:17:07.312Z 27 Acked 1026 Write 10 Sent Done Done false
164752024-04-20T01:17:07.312Z 28 Acked 1027 Write 7 Sent Done Done false
164762024-04-20T01:17:07.312Z 29 Acked 1028 Read 9 Sent Done Done false
164772024-04-20T01:17:07.312Z 30 Acked 1029 Read 1 Sent Sent Done false
164782024-04-20T01:17:07.312Z STATES DS:0 DS:1 DS:2 TOTAL
164792024-04-20T01:17:07.312Z New 0 0 0 0
164802024-04-20T01:17:07.312Z Sent 28 1 0 29
164812024-04-20T01:17:07.312Z Done 2 29 30 61
164822024-04-20T01:17:07.312Z Skipped 0 0 0 0
164832024-04-20T01:17:07.312Z Error 0 0 0 0
164842024-04-20T01:17:07.312ZLast Flush: 0 1023 1023
164852024-04-20T01:17:07.312ZDownstairs last five completed:
164862024-04-20T01:17:07.312ZUpstairs last five completed: 30 29 28 27 26
164872024-04-20T01:17:07.313Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:17:05.714184137Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1782}
164882024-04-20T01:17:07.313ZWrote out file "/tmp/test_repair_verify.data"
164892024-04-20T01:17:07.313Z
164902024-04-20T01:17:07.313ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
164912024-04-20T01:17:07.313ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164922024-04-20T01:17:07.313Z 0 000-019 150 150 150 244 244 244 F F F
164932024-04-20T01:17:07.313Z 1 020-039 148 152 152 242 250 250 F F F <---
164942024-04-20T01:17:07.313Z 2 040-059 150 150 150 244 244 244 F F F
164952024-04-20T01:17:07.313Z 3 060-079 146 146 146 241 241 241 F F F
164962024-04-20T01:17:07.313Z 4 080-099 151 152 152 247 248 248 F F F <---
164972024-04-20T01:17:07.313Z 5 100-119 151 151 151 247 247 247 F F F
164982024-04-20T01:17:07.313Z 6 120-139 142 142 142 235 235 235 F F F
164992024-04-20T01:17:07.313Z 7 140-159 137 137 137 231 231 231 F T T <---
165002024-04-20T01:17:07.313Z 8 160-179 150 150 150 244 244 244 F F F
165012024-04-20T01:17:07.313Z 9 180-199 145 152 152 240 250 250 F F F <---
165022024-04-20T01:17:07.313Z 10 200-219 146 152 152 241 249 249 F F F <---
165032024-04-20T01:17:07.313Z 11 220-239 144 144 144 238 238 238 F F F
165042024-04-20T01:17:07.313Z 12 240-259 148 148 148 242 242 242 F F F
165052024-04-20T01:17:07.313Z 13 260-279 150 152 152 245 250 250 F F F <---
165062024-04-20T01:17:07.314Z 14 280-299 150 152 152 245 250 250 F F F <---
165072024-04-20T01:17:07.314Z 15 300-319 150 150 150 244 244 244 F F F
165082024-04-20T01:17:07.314Z 16 320-339 149 152 152 243 250 250 F T T <---
165092024-04-20T01:17:07.314Z 17 340-359 146 146 146 241 241 241 F T T <---
165102024-04-20T01:17:07.314Z 18 360-379 150 152 152 244 249 249 F F F <---
165112024-04-20T01:17:07.314Z 19 380-399 146 146 146 241 241 241 F F F
165122024-04-20T01:17:07.314Z 20 400-419 146 146 146 241 241 241 F F F
165132024-04-20T01:17:07.314Z 21 420-439 149 152 152 243 248 248 T F F <---
165142024-04-20T01:17:07.314Z 22 440-459 149 149 149 243 243 243 F F F
165152024-04-20T01:17:07.314Z 23 460-479 149 152 152 243 251 251 T F F <---
165162024-04-20T01:17:07.314Z 24 480-499 149 152 152 243 251 251 F F F <---
165172024-04-20T01:17:07.314Z 25 500-519 146 152 152 241 248 248 F F F <---
165182024-04-20T01:17:07.314Z 26 520-539 150 150 150 244 244 244 F F F
165192024-04-20T01:17:07.314Z 27 540-559 151 152 152 247 248 248 F F F <---
165202024-04-20T01:17:07.314Z 28 560-579 150 150 150 245 245 245 F F F
165212024-04-20T01:17:07.314Z 29 580-599 150 150 150 245 245 245 F F F
165222024-04-20T01:17:07.314ZMax gen: 152, Max flush: 251
165232024-04-20T01:17:07.314ZError: Difference in extent metadata found!
165242024-04-20T01:17:07.314ZA Difference in extent metadata is expected here
165252024-04-20T01:17:07.314ZOn loop 76
165262024-04-20T01:17:07.314Z
165272024-04-20T01:17:07.314ZVerifying data now
165282024-04-20T01:17:21.913ZNo change after verify
165292024-04-20T01:17:21.913ZLoop: 76 Downstairs dump after verify (and repair):
165302024-04-20T01:17:21.928ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
165312024-04-20T01:17:21.928Z 0 000-019 150 150 150 244 244 244 F F F
165322024-04-20T01:17:21.928Z 1 020-039 152 152 152 250 250 250 F F F
165332024-04-20T01:17:21.928Z 2 040-059 150 150 150 244 244 244 F F F
165342024-04-20T01:17:21.928Z 3 060-079 146 146 146 241 241 241 F F F
165352024-04-20T01:17:21.928Z 4 080-099 152 152 152 248 248 248 F F F
165362024-04-20T01:17:21.928Z 5 100-119 151 151 151 247 247 247 F F F
165372024-04-20T01:17:21.929Z 6 120-139 142 142 142 235 235 235 F F F
165382024-04-20T01:17:21.929Z 7 140-159 153 153 153 252 252 252 F F F
165392024-04-20T01:17:21.929Z 8 160-179 150 150 150 244 244 244 F F F
165402024-04-20T01:17:21.934Z 9 180-199 152 152 152 250 250 250 F F F
165412024-04-20T01:17:21.934Z 10 200-219 152 152 152 249 249 249 F F F
165422024-04-20T01:17:21.934Z 11 220-239 144 144 144 238 238 238 F F F
165432024-04-20T01:17:21.935Z 12 240-259 148 148 148 242 242 242 F F F
165442024-04-20T01:17:21.935Z 13 260-279 152 152 152 250 250 250 F F F
165452024-04-20T01:17:21.935Z 14 280-299 152 152 152 250 250 250 F F F
165462024-04-20T01:17:21.935Z 15 300-319 150 150 150 244 244 244 F F F
165472024-04-20T01:17:21.935Z 16 320-339 153 153 153 252 252 252 F F F
165482024-04-20T01:17:21.935Z 17 340-359 153 153 153 252 252 252 F F F
165492024-04-20T01:17:21.935Z 18 360-379 152 152 152 249 249 249 F F F
165502024-04-20T01:17:21.936Z 19 380-399 146 146 146 241 241 241 F F F
165512024-04-20T01:17:21.936Z 20 400-419 146 146 146 241 241 241 F F F
165522024-04-20T01:17:21.936Z 21 420-439 152 152 152 248 248 248 F F F
165532024-04-20T01:17:21.936Z 22 440-459 149 149 149 243 243 243 F F F
165542024-04-20T01:17:21.936Z 23 460-479 152 152 152 251 251 251 F F F
165552024-04-20T01:17:21.936Z 24 480-499 152 152 152 251 251 251 F F F
165562024-04-20T01:17:21.936Z 25 500-519 152 152 152 248 248 248 F F F
165572024-04-20T01:17:21.936Z 26 520-539 150 150 150 244 244 244 F F F
165582024-04-20T01:17:21.936Z 27 540-559 152 152 152 248 248 248 F F F
165592024-04-20T01:17:21.936Z 28 560-579 150 150 150 245 245 245 F F F
165602024-04-20T01:17:21.937Z 29 580-599 150 150 150 245 245 245 F F F
165612024-04-20T01:17:21.937ZMax gen: 153, Max flush: 252
165622024-04-20T01:17:21.937Z
165632024-04-20T01:17:21.937ZBegin loop 77
165642024-04-20T01:17:21.937ZDownstairs to restart: 2
165652024-04-20T01:17:21.950Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.446587002Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790}
165662024-04-20T01:17:21.950Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.446825942Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790}
165672024-04-20T01:17:21.950Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.446849102Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790}
165682024-04-20T01:17:21.959Z{"msg":"Crucible 983147b8-9813-4792-a523-d3c971f6bc74 has session id: 077f5675-47de-484a-978a-18ac5fe3fe58","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.446900102Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165692024-04-20T01:17:21.960Z{"msg":"Upstairs opts: Upstairs UUID: 983147b8-9813-4792-a523-d3c971f6bc74, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
165702024-04-20T01:17:21.960Z,"time":"2024-04-20T01:17:20.446947883Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165712024-04-20T01:17:21.960Z{"msg":"Crucible stats registered with UUID: 983147b8-9813-4792-a523-d3c971f6bc74","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447003403Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165722024-04-20T01:17:21.960Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447039453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790}
165732024-04-20T01:17:21.960Z{"msg":"983147b8-9813-4792-a523-d3c971f6bc74 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447136593Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165742024-04-20T01:17:21.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447163753Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"0","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165752024-04-20T01:17:21.961Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447179493Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"0","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165762024-04-20T01:17:21.961Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447218733Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"1","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165772024-04-20T01:17:21.961Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447242903Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"1","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165782024-04-20T01:17:21.961Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447277293Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"2","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165792024-04-20T01:17:21.961Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447317043Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"2","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165802024-04-20T01:17:21.961Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447362904Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165812024-04-20T01:17:21.961Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447403973Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165822024-04-20T01:17:21.962Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447512784Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165832024-04-20T01:17:21.962Z{"msg":"ds_connection connected from Ok(127.0.0.1:48219)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447533934Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165842024-04-20T01:17:21.962Z{"msg":"ds_connection connected from Ok(127.0.0.1:65488)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447578394Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165852024-04-20T01:17:21.962Z{"msg":"ds_connection connected from Ok(127.0.0.1:36387)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.447629414Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165862024-04-20T01:17:21.962Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.448147895Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"0","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165872024-04-20T01:17:21.962Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.448174725Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"0","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165882024-04-20T01:17:21.962Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.448189765Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"1","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165892024-04-20T01:17:21.963Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.448218475Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"1","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165902024-04-20T01:17:21.963Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.448751476Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"0","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165912024-04-20T01:17:21.963Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.448809997Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"1","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165922024-04-20T01:17:21.963Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.449167317Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"2","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165932024-04-20T01:17:21.963Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.449201248Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"2","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165942024-04-20T01:17:21.963Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.449389418Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"1","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165952024-04-20T01:17:21.963Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.449425868Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"1","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165962024-04-20T01:17:21.964Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.449477848Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165972024-04-20T01:17:21.964Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.449531848Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"0","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165982024-04-20T01:17:21.964Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.449588398Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"0","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
165992024-04-20T01:17:21.964Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.449610869Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166002024-04-20T01:17:21.964Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.449718719Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"2","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166012024-04-20T01:17:21.964Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45010989Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"2","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166022024-04-20T01:17:21.965Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45013975Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"2","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166032024-04-20T01:17:21.965Z{"msg":"[0]R flush_numbers[0..12]: [244, 250, 244, 241, 248, 247, 235, 252, 244, 250, 249, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45015584Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166042024-04-20T01:17:21.965Z{"msg":"[0]R generation[0..12]: [150, 152, 150, 146, 152, 151, 142, 153, 150, 152, 152, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45017099Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166052024-04-20T01:17:21.965Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45019376Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166062024-04-20T01:17:21.965Z{"msg":"[1]R flush_numbers[0..12]: [244, 250, 244, 241, 248, 247, 235, 252, 244, 250, 249, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45020824Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166072024-04-20T01:17:21.965Z{"msg":"[1]R generation[0..12]: [150, 152, 150, 146, 152, 151, 142, 153, 150, 152, 152, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45023011Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166082024-04-20T01:17:21.966Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.4502562Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
166092024-04-20T01:17:21.966Zsession_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166102024-04-20T01:17:21.966Z{"msg":"[2]R flush_numbers[0..12]: [244, 250, 244, 241, 248, 247, 235, 252, 244, 250, 249, 238]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45029508Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166112024-04-20T01:17:21.966Z{"msg":"[2]R generation[0..12]: [150, 152, 150, 146, 152, 151, 142, 153, 150, 152, 152, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45032742Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166122024-04-20T01:17:21.966Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45035005Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166132024-04-20T01:17:21.966Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45038462Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166142024-04-20T01:17:21.966Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45041261Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166152024-04-20T01:17:21.966Z{"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45042659Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166162024-04-20T01:17:21.966Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45045653Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"":"downstairs","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166172024-04-20T01:17:21.967Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45049091Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166182024-04-20T01:17:21.967Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.450517971Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"0","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166192024-04-20T01:17:21.967Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.450532411Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"1","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166202024-04-20T01:17:21.967Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.45056376Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"client":"2","":"downstairs","session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166212024-04-20T01:17:21.967Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.450598041Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166222024-04-20T01:17:21.967Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.450612941Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166232024-04-20T01:17:21.967Z{"msg":"983147b8-9813-4792-a523-d3c971f6bc74 is now active with session: 077f5675-47de-484a-978a-18ac5fe3fe58","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.450645281Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166242024-04-20T01:17:21.968Z{"msg":"Set Active after no reconciliation","v":0,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
166252024-04-20T01:17:21.968Zname":"crucible","level":30,"time":"2024-04-20T01:17:20.450683561Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790,"session_id":"077f5675-47de-484a-978a-18ac5fe3fe58"}
166262024-04-20T01:17:21.968Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:20.450715551Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790}
166272024-04-20T01:17:21.968ZLoading write count information from file "/tmp/test_repair_verify.data"
166282024-04-20T01:17:21.968ZRun Repair workload
166292024-04-20T01:17:21.968Z01/30 Write block 336 len 4096 data: 12
166302024-04-20T01:17:21.968Z02/30 Write block 265 len 16384 data: 6 8 6 5
166312024-04-20T01:17:21.969Z03/30 Read block 189 len 28672
166322024-04-20T01:17:21.974Z04/30 Read block 320 len 36864
166332024-04-20T01:17:21.974Z05/30 Write block 227 len 16384 data: 10 12 9 10
166342024-04-20T01:17:21.974Z06/30 Read block 222 len 12288
166352024-04-20T01:17:21.974Z07/30 Read block 364 len 20480
166362024-04-20T01:17:21.974Z08/30 Read block 165 len 8192
166372024-04-20T01:17:21.974Z09/30 Read block 535 len 12288
166382024-04-20T01:17:21.975Z10/30 Read block 299 len 20480
166392024-04-20T01:17:21.975Z11/30 Write block 409 len 36864 data: 13 14 16 17 18 17 17 19 15
166402024-04-20T01:17:21.975Z12/30 Read block 239 len 8192
166412024-04-20T01:17:21.975Z13/30 Flush
166422024-04-20T01:17:21.975Z14/30 Flush
166432024-04-20T01:17:21.975Z15/30 Write block 39 len 16384 data: 14 11 10 10
166442024-04-20T01:17:21.975Z16/30 Write block 354 len 12288 data: 8 12 14
166452024-04-20T01:17:21.975Z17/30 Write block 530 len 20480 data: 9 11 12 11 14
166462024-04-20T01:17:21.975Z18/30 Read block 71 len 8192
166472024-04-20T01:17:21.975Z19/30 Write block 90 len 28672 data: 16 14 11 9 10 8 8
166482024-04-20T01:17:21.976Z20/30 Read block 75 len 36864
166492024-04-20T01:17:21.976Z21/30 Read block 242 len 36864
166502024-04-20T01:17:21.976Z22/30 Write block 93 len 28672 data: 10 11 9 9 9 10 9
166512024-04-20T01:17:21.980Z23/30 Write block 286 len 36864 data: 11 10 11 12 9 8 8 8 11
166522024-04-20T01:17:21.980Z24/30 Write block 583 len 12288 data: 11 15 18
166532024-04-20T01:17:21.980Z25/30 Write block 315 len 16384 data: 6 9 9 15
166542024-04-20T01:17:21.980Z26/30 Write block 434 len 32768 data: 12 13 14 13 10 12 12 13
166552024-04-20T01:17:21.980Z27/30 Write block 376 len 8192 data: 13 14
166562024-04-20T01:17:21.980Z28/30 Write block 53 len 4096 data: 19
166572024-04-20T01:17:21.980Z29/30 Read block 8 len 32768
166582024-04-20T01:17:21.990Z30/30 Read block 372 len 28672
166592024-04-20T01:17:21.990Z----------------------------------------------------------------
166602024-04-20T01:17:21.991Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
166612024-04-20T01:17:21.991ZStates: Active Active Active
166622024-04-20T01:17:21.991ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
166632024-04-20T01:17:21.991Z 1 Acked 1000 Write 1 Done Done Done false
166642024-04-20T01:17:21.991Z 2 Acked 1001 Write 4 Done Done Done false
166652024-04-20T01:17:21.991Z 3 Acked 1002 Read 7 Done Done Done false
166662024-04-20T01:17:21.991Z 4 Acked 1003 Read 9 Done Done Done false
166672024-04-20T01:17:21.991Z 5 Acked 1004 Write 4 Done Done Done false
166682024-04-20T01:17:21.991Z 6 Acked 1005 Read 3 Done Done Done false
166692024-04-20T01:17:21.991Z 7 Acked 1006 Read 5 Done Done Sent false
166702024-04-20T01:17:21.991Z 8 Acked 1007 Read 2 Done Done Sent false
166712024-04-20T01:17:21.992Z 9 Acked 1008 Read 3 Done Done Sent false
166722024-04-20T01:17:21.992Z 10 Acked 1009 Read 5 Done Done Sent false
166732024-04-20T01:17:21.992Z 11 Acked 1010 Write 9 Done Done Sent false
166742024-04-20T01:17:21.992Z 12 Acked 1011 Read 2 Done Done Sent false
166752024-04-20T01:17:21.992Z 13 Acked 1012 Flush 0 Done Done Sent false
166762024-04-20T01:17:21.992Z 14 Acked 1013 Flush 0 Done Done Sent false
166772024-04-20T01:17:21.992Z 15 Acked 1014 Write 4 Done Done Sent false
166782024-04-20T01:17:21.992Z 16 Acked 1015 Write 3 Done Done Sent false
166792024-04-20T01:17:21.992Z 17 Acked 1016 Write 5 Done Done Sent false
166802024-04-20T01:17:21.992Z 18 Acked 1017 Read 2 Done Done Sent false
166812024-04-20T01:17:21.992Z 19 Acked 1018 Write 7 Done Done Sent false
166822024-04-20T01:17:21.993Z 20 Acked 1019 Read 9 Done Done Sent false
166832024-04-20T01:17:21.993Z 21 Acked 1020 Read 9 Done Done Sent false
166842024-04-20T01:17:21.993Z 22 Acked 1021 Write 7 Done Done Sent false
166852024-04-20T01:17:21.993Z 23 Acked 1022 Write 9 Done Done Sent false
166862024-04-20T01:17:21.993Z 24 Acked 1023 Write 3 Done Done Sent false
166872024-04-20T01:17:21.993Z 25 Acked 1024 Write 4 Done Done Sent false
166882024-04-20T01:17:21.993Z 26 Acked 1025 Write 8 Done Done Sent false
166892024-04-20T01:17:21.993Z 27 Acked 1026 Write 2 Done Done Sent false
166902024-04-20T01:17:21.993Z 28 Acked 1027 Write 1 Done Done Sent false
166912024-04-20T01:17:21.993Z 29 Acked 1028 Read 8 Done Done Sent false
166922024-04-20T01:17:21.993Z 30 Acked 1029 Read 7 Done Done Sent false
166932024-04-20T01:17:21.994Z STATES DS:0 DS:1 DS:2 TOTAL
166942024-04-20T01:17:21.994Z New 0 0 0 0
166952024-04-20T01:17:21.994Z Sent 0 0 24 24
166962024-04-20T01:17:21.994Z Done 30 30 6 66
166972024-04-20T01:17:21.994Z Skipped 0 0 0 0
166982024-04-20T01:17:21.994Z Error 0 0 0 0
166992024-04-20T01:17:21.994ZLast Flush: 1013 1013 0
167002024-04-20T01:17:21.994ZDownstairs last five completed:
167012024-04-20T01:17:21.994ZUpstairs last five completed: 30 29 28 27 26
167022024-04-20T01:17:21.994Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:17:20.478658873Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1790}
167032024-04-20T01:17:21.994ZWrote out file "/tmp/test_repair_verify.data"
167042024-04-20T01:17:22.000Z
167052024-04-20T01:17:22.000ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
167062024-04-20T01:17:22.007ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167072024-04-20T01:17:22.007Z 0 000-019 150 150 150 244 244 244 F F F
167082024-04-20T01:17:22.007Z 1 020-039 152 152 152 250 250 250 T T F <---
167092024-04-20T01:17:22.008Z 2 040-059 150 150 150 244 244 244 T T F <---
167102024-04-20T01:17:22.008Z 3 060-079 146 146 146 241 241 241 F F F
167112024-04-20T01:17:22.008Z 4 080-099 152 152 152 248 248 248 T T F <---
167122024-04-20T01:17:22.008Z 5 100-119 151 151 151 247 247 247 F F F
167132024-04-20T01:17:22.008Z 6 120-139 142 142 142 235 235 235 F F F
167142024-04-20T01:17:22.008Z 7 140-159 153 153 153 252 252 252 F F F
167152024-04-20T01:17:22.013Z 8 160-179 150 150 150 244 244 244 F F F
167162024-04-20T01:17:22.013Z 9 180-199 152 152 152 250 250 250 F F F
167172024-04-20T01:17:22.013Z 10 200-219 152 152 152 249 249 249 F F F
167182024-04-20T01:17:22.013Z 11 220-239 154 154 144 253 253 238 F F T <---
167192024-04-20T01:17:22.013Z 12 240-259 148 148 148 242 242 242 F F F
167202024-04-20T01:17:22.013Z 13 260-279 154 154 152 253 253 250 F F T <---
167212024-04-20T01:17:22.014Z 14 280-299 152 152 152 250 250 250 T T F <---
167222024-04-20T01:17:22.014Z 15 300-319 150 150 150 244 244 244 T T F <---
167232024-04-20T01:17:22.014Z 16 320-339 154 154 153 253 253 252 F F T <---
167242024-04-20T01:17:22.014Z 17 340-359 153 153 153 252 252 252 T T F <---
167252024-04-20T01:17:22.014Z 18 360-379 152 152 152 249 249 249 T T F <---
167262024-04-20T01:17:22.014Z 19 380-399 146 146 146 241 241 241 F F F
167272024-04-20T01:17:22.014Z 20 400-419 154 154 146 253 253 241 F F F <---
167282024-04-20T01:17:22.015Z 21 420-439 152 152 152 248 248 248 T T F <---
167292024-04-20T01:17:22.015Z 22 440-459 149 149 149 243 243 243 T T F <---
167302024-04-20T01:17:22.015Z 23 460-479 152 152 152 251 251 251 F F F
167312024-04-20T01:17:22.015Z 24 480-499 152 152 152 251 251 251 F F F
167322024-04-20T01:17:22.015Z 25 500-519 152 152 152 248 248 248 F F F
167332024-04-20T01:17:22.016Z 26 520-539 150 150 150 244 244 244 T T F <---
167342024-04-20T01:17:22.016Z 27 540-559 152 152 152 248 248 248 F F F
167352024-04-20T01:17:22.016Z 28 560-579 150 150 150 245 245 245 F F F
167362024-04-20T01:17:22.016Z 29 580-599 150 150 150 245 245 245 T T F <---
167372024-04-20T01:17:22.016ZMax gen: 154, Max flush: 253
167382024-04-20T01:17:22.016ZError: Difference in extent metadata found!
167392024-04-20T01:17:22.016ZA Difference in extent metadata is expected here
167402024-04-20T01:17:22.016ZOn loop 77
167412024-04-20T01:17:22.016Z
167422024-04-20T01:17:22.021ZVerifying data now
167432024-04-20T01:17:36.709ZNo change after verify
167442024-04-20T01:17:36.709ZLoop: 77 Downstairs dump after verify (and repair):
167452024-04-20T01:17:36.722ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167462024-04-20T01:17:36.723Z 0 000-019 150 150 150 244 244 244 F F F
167472024-04-20T01:17:36.723Z 1 020-039 155 155 155 254 254 254 F F F
167482024-04-20T01:17:36.723Z 2 040-059 155 155 155 254 254 254 F F F
167492024-04-20T01:17:36.723Z 3 060-079 146 146 146 241 241 241 F F F
167502024-04-20T01:17:36.723Z 4 080-099 155 155 155 254 254 254 F F F
167512024-04-20T01:17:36.723Z 5 100-119 151 151 151 247 247 247 F F F
167522024-04-20T01:17:36.723Z 6 120-139 142 142 142 235 235 235 F F F
167532024-04-20T01:17:36.723Z 7 140-159 153 153 153 252 252 252 F F F
167542024-04-20T01:17:36.723Z 8 160-179 150 150 150 244 244 244 F F F
167552024-04-20T01:17:36.728Z 9 180-199 152 152 152 250 250 250 F F F
167562024-04-20T01:17:36.728Z 10 200-219 152 152 152 249 249 249 F F F
167572024-04-20T01:17:36.728Z 11 220-239 154 154 154 253 253 253 F F F
167582024-04-20T01:17:36.728Z 12 240-259 148 148 148 242 242 242 F F F
167592024-04-20T01:17:36.728Z 13 260-279 154 154 154 253 253 253 F F F
167602024-04-20T01:17:36.728Z 14 280-299 155 155 155 254 254 254 F F F
167612024-04-20T01:17:36.728Z 15 300-319 155 155 155 254 254 254 F F F
167622024-04-20T01:17:36.728Z 16 320-339 154 154 154 253 253 253 F F F
167632024-04-20T01:17:36.728Z 17 340-359 155 155 155 254 254 254 F F F
167642024-04-20T01:17:36.728Z 18 360-379 155 155 155 254 254 254 F F F
167652024-04-20T01:17:36.728Z 19 380-399 146 146 146 241 241 241 F F F
167662024-04-20T01:17:36.728Z 20 400-419 154 154 154 253 253 253 F F F
167672024-04-20T01:17:36.728Z 21 420-439 155 155 155 254 254 254 F F F
167682024-04-20T01:17:36.728Z 22 440-459 155 155 155 254 254 254 F F F
167692024-04-20T01:17:36.729Z 23 460-479 152 152 152 251 251 251 F F F
167702024-04-20T01:17:36.729Z 24 480-499 152 152 152 251 251 251 F F F
167712024-04-20T01:17:36.729Z 25 500-519 152 152 152 248 248 248 F F F
167722024-04-20T01:17:36.729Z 26 520-539 155 155 155 254 254 254 F F F
167732024-04-20T01:17:36.729Z 27 540-559 152 152 152 248 248 248 F F F
167742024-04-20T01:17:36.729Z 28 560-579 150 150 150 245 245 245 F F F
167752024-04-20T01:17:36.729Z 29 580-599 155 155 155 254 254 254 F F F
167762024-04-20T01:17:36.729ZMax gen: 155, Max flush: 254
167772024-04-20T01:17:36.729Z
167782024-04-20T01:17:36.729ZBegin loop 78
167792024-04-20T01:17:36.729ZDownstairs to restart: 1
167802024-04-20T01:17:36.744Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241074704Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798}
167812024-04-20T01:17:36.750Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241311115Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798}
167822024-04-20T01:17:36.750Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241348475Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798}
167832024-04-20T01:17:36.750Z{"msg":"Crucible 48f8b1d7-1497-41ca-b4d0-b6f2e0096711 has session id: 79e3eba8-c3df-46ad-b2e9-154ded6ac7bd","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241382195Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167842024-04-20T01:17:36.751ZCrucible runtime is spawned
167852024-04-20T01:17:36.751Z{"msg":"Upstairs opts: Upstairs UUID: 48f8b1d7-1497-41ca-b4d0-b6f2e0096711, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241440985Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167862024-04-20T01:17:36.751Z{"msg":"Crucible stats registered with UUID: 48f8b1d7-1497-41ca-b4d0-b6f2e0096711","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241488935Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167872024-04-20T01:17:36.751Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241507625Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798}
167882024-04-20T01:17:36.751Z{"msg":"48f8b1d7-1497-41ca-b4d0-b6f2e0096711 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241548245Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167892024-04-20T01:17:36.751Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241575225Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"0","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167902024-04-20T01:17:36.751Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241590525Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"0","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167912024-04-20T01:17:36.751Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241618205Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"1","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167922024-04-20T01:17:36.751Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241632945Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"1","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167932024-04-20T01:17:36.751Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241661905Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"2","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167942024-04-20T01:17:36.751Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241688345Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"2","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167952024-04-20T01:17:36.751Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241711266Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167962024-04-20T01:17:36.751Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241734246Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167972024-04-20T01:17:36.751Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241757416Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167982024-04-20T01:17:36.751Z{"msg":"ds_connection connected from Ok(127.0.0.1:36533)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.241960946Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
167992024-04-20T01:17:36.751Z{"msg":"ds_connection connected from Ok(127.0.0.1:36922)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.242007646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168002024-04-20T01:17:36.751Z{"msg":"ds_connection connected from Ok(127.0.0.1:48616)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.242035406Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168012024-04-20T01:17:36.751Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.242735828Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"0","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168022024-04-20T01:17:36.751Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.242776528Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"0","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168032024-04-20T01:17:36.751Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.242806188Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"2","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168042024-04-20T01:17:36.751Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.242846438Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"2","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168052024-04-20T01:17:36.751Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.243346169Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"0","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168062024-04-20T01:17:36.751Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.24347656Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"2","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168072024-04-20T01:17:36.752Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.24375591Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"1","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168082024-04-20T01:17:36.752Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.24381032Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"1","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168092024-04-20T01:17:36.752Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.24386378Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"0","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168102024-04-20T01:17:36.752Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.24389073Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"0","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168112024-04-20T01:17:36.752Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.24395305Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168122024-04-20T01:17:36.752Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.243969481Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"2","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168132024-04-20T01:17:36.752Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244001601Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"2","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168142024-04-20T01:17:36.752Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244019271Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168152024-04-20T01:17:36.752Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244292491Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"1","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168162024-04-20T01:17:36.752Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244713272Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"1","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168172024-04-20T01:17:36.752Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244764112Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"1","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168182024-04-20T01:17:36.752Z{"msg":"[0]R flush_numbers[0..12]: [244, 254, 254, 241, 254, 247, 235, 252, 244, 250, 249, 253]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244791512Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168192024-04-20T01:17:36.752Z{"msg":"[0]R generation[0..12]: [150, 155, 155, 146, 155, 151, 142, 153, 150, 152, 152, 154]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244816552Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168202024-04-20T01:17:36.752Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244840343Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168212024-04-20T01:17:36.752Z{"msg":"[1]R flush_numbers[0..12]: [244, 254, 254, 241, 254, 247, 235, 252, 244, 250, 249, 253]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244865063Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168222024-04-20T01:17:36.752Z{"msg":"[1]R generation[0..12]: [150, 155, 155, 146, 155, 151, 142, 153, 150, 152, 152, 154]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
168232024-04-20T01:17:36.752Z,"time":"2024-04-20T01:17:35.244879893Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168242024-04-20T01:17:36.752Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244894463Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168252024-04-20T01:17:36.752Z{"msg":"[2]R flush_numbers[0..12]: [244, 254, 254, 241, 254, 247, 235, 252, 244, 250, 249, 253]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244916253Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168262024-04-20T01:17:36.752Z{"msg":"[2]R generation[0..12]: [150, 155, 155, 146, 155, 151, 142, 153, 150, 152, 152, 154]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244930893Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168272024-04-20T01:17:36.752Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244945823Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168282024-04-20T01:17:36.752Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244958743Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168292024-04-20T01:17:36.752Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244972173Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168302024-04-20T01:17:36.753Z{"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244985133Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168312024-04-20T01:17:36.753Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.244998333Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"":"downstairs","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168322024-04-20T01:17:36.753Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.245020263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168332024-04-20T01:17:36.753Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.245033243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"0","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168342024-04-20T01:17:36.753Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.245046373Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"1","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168352024-04-20T01:17:36.753Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.245067283Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"client":"2","":"downstairs","session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168362024-04-20T01:17:36.753Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.245081573Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168372024-04-20T01:17:36.753Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.245094253Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168382024-04-20T01:17:36.753Z{"msg":"48f8b1d7-1497-41ca-b4d0-b6f2e0096711 is now active with session: 79e3eba8-c3df-46ad-b2e9-154ded6ac7bd","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.245119913Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168392024-04-20T01:17:36.753Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.245142003Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798,"session_id":"79e3eba8-c3df-46ad-b2e9-154ded6ac7bd"}
168402024-04-20T01:17:36.757Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:35.245156403Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798}
168412024-04-20T01:17:36.757ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
168422024-04-20T01:17:36.757ZLoading write count information from file "/tmp/test_repair_verify.data"
168432024-04-20T01:17:36.757ZRun Repair workload
168442024-04-20T01:17:36.757Z01/30 Write block 434 len 12288 data: 13 14 15
168452024-04-20T01:17:36.757Z02/30 Flush
168462024-04-20T01:17:36.757Z03/30 Write block 491 len 8192 data: 11 11
168472024-04-20T01:17:36.757Z04/30 Read block 249 len 8192
168482024-04-20T01:17:36.758Z05/30 Write block 258 len 20480 data: 9 10 8 6 7
168492024-04-20T01:17:36.758Z06/30 Write block 412 len 32768 data: 18 19 18 18 20 16 13 13
168502024-04-20T01:17:36.758Z07/30 Read block 423 len 40960
168512024-04-20T01:17:36.758Z08/30 Flush
168522024-04-20T01:17:36.761Z09/30 Write block 475 len 16384 data: 15 13 15 12
168532024-04-20T01:17:36.761Z10/30 Read block 527 len 28672
168542024-04-20T01:17:36.761Z11/30 Write block 421 len 12288 data: 10 9 11
168552024-04-20T01:17:36.765Z12/30 Read block 309 len 36864
168562024-04-20T01:17:36.765Z13/30 Write block 43 len 36864 data: 7 8 8 9 8 9 11 13 15
168572024-04-20T01:17:36.765Z14/30 Write block 75 len 20480 data: 9 6 11 12 12
168582024-04-20T01:17:36.768Z15/30 Write block 42 len 24576 data: 11 8 9 9 10 9
168592024-04-20T01:17:36.768Z16/30 Read block 156 len 32768
168602024-04-20T01:17:36.769Z17/30 Read block 72 len 24576
168612024-04-20T01:17:36.769Z18/30 Read block 31 len 24576
168622024-04-20T01:17:36.769Z19/30 Read block 270 len 12288
168632024-04-20T01:17:36.772Z20/30 Write block 486 len 40960 data: 12 10 9 9 11 12 12 10 7 8
168642024-04-20T01:17:36.772Z21/30 Read block 478 len 40960
168652024-04-20T01:17:36.772Z22/30 Read block 291 len 40960
168662024-04-20T01:17:36.772Z23/30 Flush
168672024-04-20T01:17:36.776Z24/30 Write block 321 len 32768 data: 15 16 16 13 14 15 13 14
168682024-04-20T01:17:36.783Z25/30 Write block 391 len 4096 data: 8
168692024-04-20T01:17:36.783Z26/30 Write block 548 len 32768 data: 12 15 17 15 11 9 11 13
168702024-04-20T01:17:36.783Z27/30 Write block 491 len 36864 data: 13 13 11 8 9 8 7 7 8
168712024-04-20T01:17:36.783Z28/30 Write block 30 len 12288 data: 9 9 10
168722024-04-20T01:17:36.783Z29/30 Write block 304 len 36864 data: 11 11 10 9 8 7 6 6 6
168732024-04-20T01:17:36.783Z30/30 Read block 73 len 4096
168742024-04-20T01:17:36.784Z----------------------------------------------------------------
168752024-04-20T01:17:36.784Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:28
168762024-04-20T01:17:36.784ZStates: Active Active Active
168772024-04-20T01:17:36.784ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
168782024-04-20T01:17:36.784Z 3 Acked 1002 Write 2 Done Sent Done false
168792024-04-20T01:17:36.784Z 4 Acked 1003 Read 2 Done Sent Done false
168802024-04-20T01:17:36.784Z 5 Acked 1004 Write 5 Done Sent Done false
168812024-04-20T01:17:36.784Z 6 Acked 1005 Write 8 Done Sent Done false
168822024-04-20T01:17:36.784Z 7 Acked 1006 Read 10 Done Sent Done false
168832024-04-20T01:17:36.784Z 8 Acked 1007 Flush 0 Done Sent Done false
168842024-04-20T01:17:36.784Z 9 Acked 1008 Write 4 Done Sent Done false
168852024-04-20T01:17:36.784Z 10 Acked 1009 Read 7 Done Sent Done false
168862024-04-20T01:17:36.784Z 11 Acked 1010 Write 3 Done Sent Done false
168872024-04-20T01:17:36.784Z 12 Acked 1011 Read 9 Done Sent Done false
168882024-04-20T01:17:36.784Z 13 Acked 1012 Write 9 Done Sent Done false
168892024-04-20T01:17:36.784Z 14 Acked 1013 Write 5 Done Sent Done false
168902024-04-20T01:17:36.784Z 15 Acked 1014 Write 6 Done Sent Done false
168912024-04-20T01:17:36.784Z 16 Acked 1015 Read 8 Done Sent Done false
168922024-04-20T01:17:36.784Z 17 Acked 1016 Read 6 Done Sent Done false
168932024-04-20T01:17:36.784Z 18 Acked 1017 Read 6 Done Sent Done false
168942024-04-20T01:17:36.784Z 19 Acked 1018 Read 3 Done Sent Done false
168952024-04-20T01:17:36.784Z 20 Acked 1019 Write 10 Done Sent Done false
168962024-04-20T01:17:36.784Z 21 Acked 1020 Read 10 Done Sent Done false
168972024-04-20T01:17:36.785Z 22 Acked 1021 Read 10 Done Sent Done false
168982024-04-20T01:17:36.785Z 23 Acked 1022 Flush 0 Done Sent Done false
168992024-04-20T01:17:36.785Z 24 Acked 1023 Write 8 Done Sent Done false
169002024-04-20T01:17:36.785Z 25 Acked 1024 Write 1 Done Sent Done false
169012024-04-20T01:17:36.785Z 26 Acked 1025 Write 8 Done Sent Done false
169022024-04-20T01:17:36.785Z 27 Acked 1026 Write 9 Done Sent Done false
169032024-04-20T01:17:36.785Z 28 Acked 1027 Write 3 Done Sent Done false
169042024-04-20T01:17:36.785Z 29 Acked 1028 Write 9 Done Sent Sent false
169052024-04-20T01:17:36.785Z 30 Acked 1029 Read 1 Done Sent Sent false
169062024-04-20T01:17:36.785Z STATES DS:0 DS:1 DS:2 TOTAL
169072024-04-20T01:17:36.785Z New 0 0 0 0
169082024-04-20T01:17:36.785Z Sent 0 28 2 30
169092024-04-20T01:17:36.785Z Done 28 0 26 54
169102024-04-20T01:17:36.785Z Skipped 0 0 0 0
169112024-04-20T01:17:36.785Z Error 0 0 0 0
169122024-04-20T01:17:36.785ZLast Flush: 1022 1001 1022
169132024-04-20T01:17:36.785ZDownstairs last five completed: 1001 1000
169142024-04-20T01:17:36.785ZUpstairs last five completed: 30 29 28 27 26
169152024-04-20T01:17:36.789Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:17:35.280178921Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1798}
169162024-04-20T01:17:36.789ZWrote out file "/tmp/test_repair_verify.data"
169172024-04-20T01:17:36.789Z
169182024-04-20T01:17:36.792ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
169192024-04-20T01:17:36.807ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169202024-04-20T01:17:36.807Z 0 000-019 150 150 150 244 244 244 F F F
169212024-04-20T01:17:36.807Z 1 020-039 155 155 155 254 254 254 T F T <---
169222024-04-20T01:17:36.807Z 2 040-059 156 155 156 257 254 257 F F F <---
169232024-04-20T01:17:36.807Z 3 060-079 156 146 156 257 241 257 F F F <---
169242024-04-20T01:17:36.807Z 4 080-099 155 155 155 254 254 254 F F F
169252024-04-20T01:17:36.807Z 5 100-119 151 151 151 247 247 247 F F F
169262024-04-20T01:17:36.807Z 6 120-139 142 142 142 235 235 235 F F F
169272024-04-20T01:17:36.807Z 7 140-159 153 153 153 252 252 252 F F F
169282024-04-20T01:17:36.807Z 8 160-179 150 150 150 244 244 244 F F F
169292024-04-20T01:17:36.807Z 9 180-199 152 152 152 250 250 250 F F F
169302024-04-20T01:17:36.807Z 10 200-219 152 152 152 249 249 249 F F F
169312024-04-20T01:17:36.807Z 11 220-239 154 154 154 253 253 253 F F F
169322024-04-20T01:17:36.807Z 12 240-259 156 148 156 256 242 256 F F F <---
169332024-04-20T01:17:36.807Z 13 260-279 156 154 156 256 253 256 F F F <---
169342024-04-20T01:17:36.807Z 14 280-299 155 155 155 254 254 254 F F F
169352024-04-20T01:17:36.807Z 15 300-319 155 155 155 254 254 254 T F T <---
169362024-04-20T01:17:36.807Z 16 320-339 154 154 154 253 253 253 T F T <---
169372024-04-20T01:17:36.808Z 17 340-359 155 155 155 254 254 254 F F F
169382024-04-20T01:17:36.811Z 18 360-379 155 155 155 254 254 254 F F F
169392024-04-20T01:17:36.811Z 19 380-399 146 146 146 241 241 241 T F T <---
169402024-04-20T01:17:36.811Z 20 400-419 156 154 156 256 253 256 F F F <---
169412024-04-20T01:17:36.811Z 21 420-439 156 156 156 257 255 257 F F F <---
169422024-04-20T01:17:36.811Z 22 440-459 155 155 155 254 254 254 F F F
169432024-04-20T01:17:36.811Z 23 460-479 156 152 156 257 251 257 F F F <---
169442024-04-20T01:17:36.811Z 24 480-499 156 152 156 257 251 257 T F T <---
169452024-04-20T01:17:36.811Z 25 500-519 152 152 152 248 248 248 F F F
169462024-04-20T01:17:36.812Z 26 520-539 155 155 155 254 254 254 F F F
169472024-04-20T01:17:36.812Z 27 540-559 152 152 152 248 248 248 T F T <---
169482024-04-20T01:17:36.812Z 28 560-579 150 150 150 245 245 245 F F F
169492024-04-20T01:17:36.812Z 29 580-599 155 155 155 254 254 254 F F F
169502024-04-20T01:17:36.812ZMax gen: 156, Max flush: 257
169512024-04-20T01:17:36.812ZError: Difference in extent metadata found!
169522024-04-20T01:17:36.812ZA Difference in extent metadata is expected here
169532024-04-20T01:17:36.812ZOn loop 78
169542024-04-20T01:17:36.812Z
169552024-04-20T01:17:36.815ZVerifying data now
169562024-04-20T01:17:41.406ZNo change after verify
169572024-04-20T01:17:41.406ZLoop: 78 Downstairs dump after verify (and repair):
169582024-04-20T01:17:41.420ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169592024-04-20T01:17:41.420Z 0 000-019 150 150 150 244 244 244 F F F
169602024-04-20T01:17:41.420Z 1 020-039 157 157 157 258 258 258 F F F
169612024-04-20T01:17:41.420Z 2 040-059 156 156 156 257 257 257 F F F
169622024-04-20T01:17:41.420Z 3 060-079 156 156 156 257 257 257 F F F
169632024-04-20T01:17:41.420Z 4 080-099 155 155 155 254 254 254 F F F
169642024-04-20T01:17:41.425Z 5 100-119 151 151 151 247 247 247 F F F
169652024-04-20T01:17:41.425Z 6 120-139 142 142 142 235 235 235 F F F
169662024-04-20T01:17:41.425Z 7 140-159 153 153 153 252 252 252 F F F
169672024-04-20T01:17:41.425Z 8 160-179 150 150 150 244 244 244 F F F
169682024-04-20T01:17:41.425Z 9 180-199 152 152 152 250 250 250 F F F
169692024-04-20T01:17:41.425Z 10 200-219 152 152 152 249 249 249 F F F
169702024-04-20T01:17:41.425Z 11 220-239 154 154 154 253 253 253 F F F
169712024-04-20T01:17:41.425Z 12 240-259 156 156 156 256 256 256 F F F
169722024-04-20T01:17:41.425Z 13 260-279 156 156 156 256 256 256 F F F
169732024-04-20T01:17:41.425Z 14 280-299 155 155 155 254 254 254 F F F
169742024-04-20T01:17:41.425Z 15 300-319 157 157 157 258 258 258 F F F
169752024-04-20T01:17:41.425Z 16 320-339 157 157 157 258 258 258 F F F
169762024-04-20T01:17:41.426Z 17 340-359 155 155 155 254 254 254 F F F
169772024-04-20T01:17:41.426Z 18 360-379 155 155 155 254 254 254 F F F
169782024-04-20T01:17:41.426Z 19 380-399 157 157 157 258 258 258 F F F
169792024-04-20T01:17:41.426Z 20 400-419 156 156 156 256 256 256 F F F
169802024-04-20T01:17:41.426Z 21 420-439 156 156 156 257 257 257 F F F
169812024-04-20T01:17:41.426Z 22 440-459 155 155 155 254 254 254 F F F
169822024-04-20T01:17:41.426Z 23 460-479 156 156 156 257 257 257 F F F
169832024-04-20T01:17:41.426Z 24 480-499 157 157 157 258 258 258 F F F
169842024-04-20T01:17:41.426Z 25 500-519 152 152 152 248 248 248 F F F
169852024-04-20T01:17:41.426Z 26 520-539 155 155 155 254 254 254 F F F
169862024-04-20T01:17:41.426Z 27 540-559 157 157 157 258 258 258 F F F
169872024-04-20T01:17:41.426Z 28 560-579 150 150 150 245 245 245 F F F
169882024-04-20T01:17:41.426Z 29 580-599 155 155 155 254 254 254 F F F
169892024-04-20T01:17:41.426ZMax gen: 157, Max flush: 258
169902024-04-20T01:17:41.426Z
169912024-04-20T01:17:41.426ZBegin loop 79
169922024-04-20T01:17:41.426ZDownstairs to restart: 1
169932024-04-20T01:17:41.442Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.93829542Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806}
169942024-04-20T01:17:41.442Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.938552261Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806}
169952024-04-20T01:17:41.442Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.938569531Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806}
169962024-04-20T01:17:41.449Z{"msg":"Crucible 094a8041-7cf8-45d8-ae5e-914d9daeb53e has session id: f64e1010-977c-49e8-a1a5-6d0342ad1b46","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.938602991Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
169972024-04-20T01:17:41.449Z{Crucible runtime is spawned
169982024-04-20T01:17:41.449Z"msg":"Upstairs opts: Upstairs UUID: 094a8041-7cf8-45d8-ae5e-914d9daeb53e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.938658191Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
169992024-04-20T01:17:41.449Z{"msg":"Crucible stats registered with UUID: 094a8041-7cf8-45d8-ae5e-914d9daeb53e","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.938693451Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170002024-04-20T01:17:41.449Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.938708291Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806}
170012024-04-20T01:17:41.449Z{"msg":"094a8041-7cf8-45d8-ae5e-914d9daeb53e active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.938803152Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170022024-04-20T01:17:41.449Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.938836861Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"0","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170032024-04-20T01:17:41.449Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.938880992Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"0","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170042024-04-20T01:17:41.449Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.938905152Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"1","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170052024-04-20T01:17:41.449Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.938952912Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"1","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170062024-04-20T01:17:41.449Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.938989362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"2","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170072024-04-20T01:17:41.449Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.939009972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"2","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170082024-04-20T01:17:41.449Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.939033012Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170092024-04-20T01:17:41.449Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.939065852Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170102024-04-20T01:17:41.449Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.939090822Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170112024-04-20T01:17:41.449Z{"msg":"ds_connection connected from Ok(127.0.0.1:49851)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.939242973Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170122024-04-20T01:17:41.449Z{"msg":"ds_connection connected from Ok(127.0.0.1:39561)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.939298433Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170132024-04-20T01:17:41.449Z{"msg":"ds_connection connected from Ok(127.0.0.1:48598)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.939384183Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170142024-04-20T01:17:41.450Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.939940464Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"0","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170152024-04-20T01:17:41.450Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.939985514Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"0","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170162024-04-20T01:17:41.450Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.940073734Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"2","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170172024-04-20T01:17:41.450Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.940105614Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"2","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170182024-04-20T01:17:41.450Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.940364635Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"1","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170192024-04-20T01:17:41.450Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.940455465Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"1","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170202024-04-20T01:17:41.450Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.940488505Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"0","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170212024-04-20T01:17:41.450Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.940522145Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"2","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170222024-04-20T01:17:41.450Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.940974716Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"0","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170232024-04-20T01:17:41.450Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941013496Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"0","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170242024-04-20T01:17:41.450Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941029786Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170252024-04-20T01:17:41.450Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941061377Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"1","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170262024-04-20T01:17:41.450Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941095557Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"2","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170272024-04-20T01:17:41.450Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941127147Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"2","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170282024-04-20T01:17:41.450Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941152157Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170292024-04-20T01:17:41.450Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941531807Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"1","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170302024-04-20T01:17:41.450Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941593498Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"1","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170312024-04-20T01:17:41.450Z{"msg":"[0]R flush_numbers[0..12]: [244, 258, 257, 257, 254, 247, 235, 252, 244, 250, 249, 253]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941628898Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170322024-04-20T01:17:41.450Z{"msg":"[0]R generation[0..12]: [150, 157, 156, 156, 155, 151, 142, 153, 150, 152, 152, 154]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941644428Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170332024-04-20T01:17:41.450Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941666268Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170342024-04-20T01:17:41.450Z{"msg":"[1]R flush_numbers[0..12]: [244, 258, 257, 257, 254, 247, 235, 252, 244, 250, 249, 253]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941680418Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170352024-04-20T01:17:41.450Z{"msg":"[1]R generation[0..12]: [150, 157, 156, 156, 155, 151, 142, 153, 150, 152, 152, 154]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
170362024-04-20T01:17:41.450Z,"time":"2024-04-20T01:17:39.941701568Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170372024-04-20T01:17:41.451Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941725858Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170382024-04-20T01:17:41.451Z{"msg":"[2]R flush_numbers[0..12]: [244, 258, 257, 257, 254, 247, 235, 252, 244, 250, 249, 253]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941749438Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170392024-04-20T01:17:41.451Z{"msg":"[2]R generation[0..12]: [150, 157, 156, 156, 155, 151, 142, 153, 150, 152, 152, 154]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941778248Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170402024-04-20T01:17:41.451Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941793568Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170412024-04-20T01:17:41.451Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941813628Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170422024-04-20T01:17:41.451Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941827428Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170432024-04-20T01:17:41.451Z{"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941847258Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170442024-04-20T01:17:41.451Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941860418Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"":"downstairs","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170452024-04-20T01:17:41.451Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941880998Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170462024-04-20T01:17:41.451Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941893268Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"0","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170472024-04-20T01:17:41.451Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941912868Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"1","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170482024-04-20T01:17:41.451Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941926068Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"client":"2","":"downstairs","session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170492024-04-20T01:17:41.451Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941946448Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170502024-04-20T01:17:41.451Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
170512024-04-20T01:17:41.451Z"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941966908Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170522024-04-20T01:17:41.451Z{"msg":"094a8041-7cf8-45d8-ae5e-914d9daeb53e is now active with session: f64e1010-977c-49e8-a1a5-6d0342ad1b46","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.941989908Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170532024-04-20T01:17:41.451Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.942008239Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806,"session_id":"f64e1010-977c-49e8-a1a5-6d0342ad1b46"}
170542024-04-20T01:17:41.451Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:39.942027649Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806}
170552024-04-20T01:17:41.451ZLoading write count information from file "/tmp/test_repair_verify.data"
170562024-04-20T01:17:41.451ZRun Repair workload
170572024-04-20T01:17:41.451Z01/30 Write block 367 len 4096 data: 18
170582024-04-20T01:17:41.456Z02/30 Write block 591 len 24576 data: 10 9 10 12 11 11
170592024-04-20T01:17:41.456Z03/30 Flush
170602024-04-20T01:17:41.456Z04/30 Write block 447 len 32768 data: 12 12 13 12 12 11 9 10
170612024-04-20T01:17:41.456Z05/30 Read block 73 len 28672
170622024-04-20T01:17:41.456Z06/30 Read block 582 len 8192
170632024-04-20T01:17:41.456Z07/30 Read block 238 len 36864
170642024-04-20T01:17:41.456Z08/30 Read block 315 len 12288
170652024-04-20T01:17:41.456Z09/30 Read block 217 len 12288
170662024-04-20T01:17:41.456Z10/30 Write block 73 len 16384 data: 11 13 10 7
170672024-04-20T01:17:41.460Z11/30 Write block 68 len 4096 data: 16
170682024-04-20T01:17:41.460Z12/30 Read block 412 len 40960
170692024-04-20T01:17:41.460Z13/30 Read block 133 len 24576
170702024-04-20T01:17:41.460Z14/30 Flush
170712024-04-20T01:17:41.463Z15/30 Write block 293 len 40960 data: 9 12 10 10 10 6 7 8 10 11
170722024-04-20T01:17:41.463Z16/30 Write block 315 len 28672 data: 7 10 10 16 15 15 16
170732024-04-20T01:17:41.463Z17/30 Read block 136 len 12288
170742024-04-20T01:17:41.467Z18/30 Write block 92 len 24576 data: 12 11 12 10 10 10
170752024-04-20T01:17:41.467Z19/30 Write block 500 len 40960 data: 10 11 12 14 13 11 12 9 9 14
170762024-04-20T01:17:41.467Z20/30 Read block 343 len 40960
170772024-04-20T01:17:41.470Z21/30 Flush
170782024-04-20T01:17:41.474Z22/30 Write block 217 len 32768 data: 18 17 13 15 13 14 15 15
170792024-04-20T01:17:41.474Z23/30 Read block 251 len 36864
170802024-04-20T01:17:41.474Z24/30 Write block 459 len 28672 data: 12 16 18 17 20 17 17
170812024-04-20T01:17:41.474Z25/30 Read block 332 len 8192
170822024-04-20T01:17:41.480Z26/30 Write block 347 len 4096 data: 11
170832024-04-20T01:17:41.480Z27/30 Read block 348 len 24576
170842024-04-20T01:17:41.480Z28/30 Read block 176 len 24576
170852024-04-20T01:17:41.480Z29/30 Read block 365 len 32768
170862024-04-20T01:17:41.480Z30/30 Read block 138 len 28672
170872024-04-20T01:17:41.484Z----------------------------------------------------------------
170882024-04-20T01:17:41.484Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
170892024-04-20T01:17:41.484ZStates: Active Active Active
170902024-04-20T01:17:41.484ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
170912024-04-20T01:17:41.484Z 1 Acked 1000 Write 1 Done Done Done false
170922024-04-20T01:17:41.484Z 2 Acked 1001 Write 6 Done Sent Done false
170932024-04-20T01:17:41.484Z 3 Acked 1002 Flush 0 Done Sent Done false
170942024-04-20T01:17:41.484Z 4 Acked 1003 Write 8 Done Sent Done false
170952024-04-20T01:17:41.484Z 5 Acked 1004 Read 7 Done Sent Done false
170962024-04-20T01:17:41.484Z 6 Acked 1005 Read 2 Done Sent Done false
170972024-04-20T01:17:41.485Z 7 Acked 1006 Read 9 Done Sent Done false
170982024-04-20T01:17:41.485Z 8 Acked 1007 Read 3 Done Sent Done false
170992024-04-20T01:17:41.485Z 9 Acked 1008 Read 3 Done Sent Done false
171002024-04-20T01:17:41.485Z 10 Acked 1009 Write 4 Done Sent Done false
171012024-04-20T01:17:41.485Z 11 Acked 1010 Write 1 Done Sent Done false
171022024-04-20T01:17:41.485Z 12 Acked 1011 Read 10 Done Sent Done false
171032024-04-20T01:17:41.485Z 13 Acked 1012 Read 6 Done Sent Done false
171042024-04-20T01:17:41.485Z 14 Acked 1013 Flush 0 Done Sent Done false
171052024-04-20T01:17:41.485Z 15 Acked 1014 Write 10 Done Sent Done false
171062024-04-20T01:17:41.485Z 16 Acked 1015 Write 7 Done Sent Done false
171072024-04-20T01:17:41.485Z 17 Acked 1016 Read 3 Done Sent Done false
171082024-04-20T01:17:41.485Z 18 Acked 1017 Write 6 Done Sent Done false
171092024-04-20T01:17:41.485Z 19 Acked 1018 Write 10 Done Sent Done false
171102024-04-20T01:17:41.485Z 20 Acked 1019 Read 10 Done Sent Done false
171112024-04-20T01:17:41.485Z 21 Acked 1020 Flush 0 Done Sent Done false
171122024-04-20T01:17:41.485Z 22 Acked 1021 Write 8 Done Sent Done false
171132024-04-20T01:17:41.485Z 23 Acked 1022 Read 9 Done Sent Done false
171142024-04-20T01:17:41.485Z 24 Acked 1023 Write 7 Done Sent Done false
171152024-04-20T01:17:41.485Z 25 Acked 1024 Read 2 Done Sent Done false
171162024-04-20T01:17:41.485Z 26 Acked 1025 Write 1 Done Sent Done false
171172024-04-20T01:17:41.485Z 27 Acked 1026 Read 6 Done Sent Done false
171182024-04-20T01:17:41.485Z 28 Acked 1027 Read 6 Done Sent Done false
171192024-04-20T01:17:41.485Z 29 Acked 1028 Read 8 Done Sent Done false
171202024-04-20T01:17:41.486Z 30 Acked 1029 Read 7 Done Sent Done false
171212024-04-20T01:17:41.486Z STATES DS:0 DS:1 DS:2 TOTAL
171222024-04-20T01:17:41.486Z New 0 0 0 0
171232024-04-20T01:17:41.486Z Sent 0 29 0 29
171242024-04-20T01:17:41.486Z Done 30 1 30 61
171252024-04-20T01:17:41.486Z Skipped 0 0 0 0
171262024-04-20T01:17:41.486Z Error 0 0 0 0
171272024-04-20T01:17:41.486ZLast Flush: 1020 0 1020
171282024-04-20T01:17:41.486ZDownstairs last five completed:
171292024-04-20T01:17:41.486ZUpstairs last five completed: 30 29 28 27 26
171302024-04-20T01:17:41.486Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:17:39.975105362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1806}
171312024-04-20T01:17:41.486ZWrote out file "/tmp/test_repair_verify.data"
171322024-04-20T01:17:41.486Z
171332024-04-20T01:17:41.490ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
171342024-04-20T01:17:41.501ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171352024-04-20T01:17:41.501Z 0 000-019 150 150 150 244 244 244 F F F
171362024-04-20T01:17:41.501Z 1 020-039 157 157 157 258 258 258 F F F
171372024-04-20T01:17:41.501Z 2 040-059 156 156 156 257 257 257 F F F
171382024-04-20T01:17:41.502Z 3 060-079 158 156 158 260 257 260 F F F <---
171392024-04-20T01:17:41.506Z 4 080-099 158 155 158 261 254 261 F F F <---
171402024-04-20T01:17:41.506Z 5 100-119 151 151 151 247 247 247 F F F
171412024-04-20T01:17:41.506Z 6 120-139 142 142 142 235 235 235 F F F
171422024-04-20T01:17:41.506Z 7 140-159 153 153 153 252 252 252 F F F
171432024-04-20T01:17:41.506Z 8 160-179 150 150 150 244 244 244 F F F
171442024-04-20T01:17:41.506Z 9 180-199 152 152 152 250 250 250 F F F
171452024-04-20T01:17:41.506Z 10 200-219 152 152 152 249 249 249 T F T <---
171462024-04-20T01:17:41.506Z 11 220-239 154 154 154 253 253 253 T F T <---
171472024-04-20T01:17:41.506Z 12 240-259 156 156 156 256 256 256 F F F
171482024-04-20T01:17:41.506Z 13 260-279 156 156 156 256 256 256 F F F
171492024-04-20T01:17:41.506Z 14 280-299 158 155 158 261 254 261 F F F <---
171502024-04-20T01:17:41.506Z 15 300-319 158 157 158 261 258 261 F F F <---
171512024-04-20T01:17:41.506Z 16 320-339 158 157 158 261 258 261 F F F <---
171522024-04-20T01:17:41.506Z 17 340-359 155 155 155 254 254 254 T F T <---
171532024-04-20T01:17:41.507Z 18 360-379 158 155 158 259 254 259 F T F <---
171542024-04-20T01:17:41.507Z 19 380-399 157 157 157 258 258 258 F F F
171552024-04-20T01:17:41.507Z 20 400-419 156 156 156 256 256 256 F F F
171562024-04-20T01:17:41.507Z 21 420-439 156 156 156 257 257 257 F F F
171572024-04-20T01:17:41.507Z 22 440-459 158 155 158 260 254 260 T F T <---
171582024-04-20T01:17:41.507Z 23 460-479 156 156 156 257 257 257 T F T <---
171592024-04-20T01:17:41.507Z 24 480-499 157 157 157 258 258 258 F F F
171602024-04-20T01:17:41.507Z 25 500-519 158 152 158 261 248 261 F F F <---
171612024-04-20T01:17:41.507Z 26 520-539 155 155 155 254 254 254 F F F
171622024-04-20T01:17:41.507Z 27 540-559 157 157 157 258 258 258 F F F
171632024-04-20T01:17:41.507Z 28 560-579 150 150 150 245 245 245 F F F
171642024-04-20T01:17:41.507Z 29 580-599 158 155 158 259 254 259 F F F <---
171652024-04-20T01:17:41.507ZMax gen: 158, Max flush: 261
171662024-04-20T01:17:41.507ZError: Difference in extent metadata found!
171672024-04-20T01:17:41.507ZA Difference in extent metadata is expected here
171682024-04-20T01:17:41.507ZOn loop 79
171692024-04-20T01:17:41.507Z
171702024-04-20T01:17:41.511ZVerifying data now
171712024-04-20T01:17:56.109ZNo change after verify
171722024-04-20T01:17:56.110ZLoop: 79 Downstairs dump after verify (and repair):
171732024-04-20T01:17:56.128ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171742024-04-20T01:17:56.128Z 0 000-019 150 150 150 244 244 244 F F F
171752024-04-20T01:17:56.128Z 1 020-039 157 157 157 258 258 258 F F F
171762024-04-20T01:17:56.128Z 2 040-059 156 156 156 257 257 257 F F F
171772024-04-20T01:17:56.129Z 3 060-079 158 158 158 260 260 260 F F F
171782024-04-20T01:17:56.129Z 4 080-099 158 158 158 261 261 261 F F F
171792024-04-20T01:17:56.129Z 5 100-119 151 151 151 247 247 247 F F F
171802024-04-20T01:17:56.138Z 6 120-139 142 142 142 235 235 235 F F F
171812024-04-20T01:17:56.139Z 7 140-159 153 153 153 252 252 252 F F F
171822024-04-20T01:17:56.139Z 8 160-179 150 150 150 244 244 244 F F F
171832024-04-20T01:17:56.139Z 9 180-199 152 152 152 250 250 250 F F F
171842024-04-20T01:17:56.139Z 10 200-219 159 159 159 262 262 262 F F F
171852024-04-20T01:17:56.139Z 11 220-239 159 159 159 262 262 262 F F F
171862024-04-20T01:17:56.140Z 12 240-259 156 156 156 256 256 256 F F F
171872024-04-20T01:17:56.140Z 13 260-279 156 156 156 256 256 256 F F F
171882024-04-20T01:17:56.140Z 14 280-299 158 158 158 261 261 261 F F F
171892024-04-20T01:17:56.140Z 15 300-319 158 158 158 261 261 261 F F F
171902024-04-20T01:17:56.140Z 16 320-339 158 158 158 261 261 261 F F F
171912024-04-20T01:17:56.140Z 17 340-359 159 159 159 262 262 262 F F F
171922024-04-20T01:17:56.140Z 18 360-379 158 158 158 259 259 259 F F F
171932024-04-20T01:17:56.140Z 19 380-399 157 157 157 258 258 258 F F F
171942024-04-20T01:17:56.140Z 20 400-419 156 156 156 256 256 256 F F F
171952024-04-20T01:17:56.141Z 21 420-439 156 156 156 257 257 257 F F F
171962024-04-20T01:17:56.141Z 22 440-459 159 159 159 262 262 262 F F F
171972024-04-20T01:17:56.141Z 23 460-479 159 159 159 262 262 262 F F F
171982024-04-20T01:17:56.141Z 24 480-499 157 157 157 258 258 258 F F F
171992024-04-20T01:17:56.141Z 25 500-519 158 158 158 261 261 261 F F F
172002024-04-20T01:17:56.141Z 26 520-539 155 155 155 254 254 254 F F F
172012024-04-20T01:17:56.141Z 27 540-559 157 157 157 258 258 258 F F F
172022024-04-20T01:17:56.141Z 28 560-579 150 150 150 245 245 245 F F F
172032024-04-20T01:17:56.141Z 29 580-599 158 158 158 259 259 259 F F F
172042024-04-20T01:17:56.141ZMax gen: 159, Max flush: 262
172052024-04-20T01:17:56.142Z
172062024-04-20T01:17:56.142ZBegin loop 80
172072024-04-20T01:17:56.142ZDownstairs to restart: 2
172082024-04-20T01:17:56.160Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.641372899Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814}
172092024-04-20T01:17:56.161Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.6416052Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814}
172102024-04-20T01:17:56.161Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.64164157Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814}
172112024-04-20T01:17:56.161Z{"msg":"Crucible 5f1f304d-200c-444f-8da2-63ecfcf8aa12 has session id: b2d31b9f-901f-4832-9d29-f8c0376f1bff","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.64166638Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172122024-04-20T01:17:56.161Z{"msg":"Upstairs opts: Upstairs UUID: 5f1f304d-200c-444f-8da2-63ecfcf8aa12, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.64171065Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"Crucible runtime is spawned
172132024-04-20T01:17:56.162Z}
172142024-04-20T01:17:56.162Z{"msg":"Crucible stats registered with UUID: 5f1f304d-200c-444f-8da2-63ecfcf8aa12","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.6417477Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172152024-04-20T01:17:56.162Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.64178232Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814}
172162024-04-20T01:17:56.163Z{"msg":"5f1f304d-200c-444f-8da2-63ecfcf8aa12 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.641876021Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172172024-04-20T01:17:56.163Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.641932841Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"0","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172182024-04-20T01:17:56.163Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.641948781Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"0","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172192024-04-20T01:17:56.163Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.641966211Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"1","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172202024-04-20T01:17:56.164Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.641995061Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"1","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172212024-04-20T01:17:56.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.642010581Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"2","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172222024-04-20T01:17:56.164Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.642024451Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"2","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172232024-04-20T01:17:56.164Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.642038371Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172242024-04-20T01:17:56.165Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.642052861Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172252024-04-20T01:17:56.165Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.642066761Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172262024-04-20T01:17:56.165Z{"msg":"ds_connection connected from Ok(127.0.0.1:50992)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.642268771Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172272024-04-20T01:17:56.165Z{"msg":"ds_connection connected from Ok(127.0.0.1:40577)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.642327762Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172282024-04-20T01:17:56.165Z{"msg":"ds_connection connected from Ok(127.0.0.1:36690)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.642353342Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172292024-04-20T01:17:56.165Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.643088603Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"0","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172302024-04-20T01:17:56.165Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.643134913Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"0","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172312024-04-20T01:17:56.165Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.643162963Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"1","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172322024-04-20T01:17:56.165Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.643191753Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"1","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172332024-04-20T01:17:56.165Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.643223713Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"2","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172342024-04-20T01:17:56.166Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.643252313Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"2","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172352024-04-20T01:17:56.166Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.643659104Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"0","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172362024-04-20T01:17:56.166Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.643693474Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"2","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172372024-04-20T01:17:56.166Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.643759495Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"1","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172382024-04-20T01:17:56.166Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644301076Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"2","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172392024-04-20T01:17:56.166Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644338176Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"2","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172402024-04-20T01:17:56.166Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644354936Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172412024-04-20T01:17:56.166Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644367886Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"0","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172422024-04-20T01:17:56.166Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644389806Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"0","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172432024-04-20T01:17:56.166Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644404066Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172442024-04-20T01:17:56.166Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644423616Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"1","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172452024-04-20T01:17:56.166Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644438046Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"1","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172462024-04-20T01:17:56.166Z{"msg":"[0]R flush_numbers[0..12]: [244, 258, 257, 260, 261, 247, 235, 252, 244, 250, 262, 262]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644465296Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172472024-04-20T01:17:56.166Z{"msg":"[0]R generation[0..12]: [150, 157, 156, 158, 158, 151, 142, 153, 150, 152, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644486076Z","hostnameWait for a query_work_queue command to finish before sending IO
172482024-04-20T01:17:56.166Z":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172492024-04-20T01:17:56.167Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644505736Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172502024-04-20T01:17:56.167Z{"msg":"[1]R flush_numbers[0..12]: [244, 258, 257, 260, 261, 247, 235, 252, 244, 250, 262, 262]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644528316Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172512024-04-20T01:17:56.167Z{"msg":"[1]R generation[0..12]: [150, 157, 156, 158, 158, 151, 142, 153, 150, 152, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644542446Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172522024-04-20T01:17:56.167Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644569027Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172532024-04-20T01:17:56.167Z{"msg":"[2]R flush_numbers[0..12]: [244, 258, 257, 260, 261, 247, 235, 252, 244, 250, 262, 262]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644588947Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172542024-04-20T01:17:56.167Z{"msg":"[2]R generation[0..12]: [150, 157, 156, 158, 158, 151, 142, 153, 150, 152, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644615576Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172552024-04-20T01:17:56.167Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644631047Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172562024-04-20T01:17:56.167Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644659627Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172572024-04-20T01:17:56.168Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644675707Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172582024-04-20T01:17:56.168Z{"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644696617Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172592024-04-20T01:17:56.168Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644710467Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"":"downstairs","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172602024-04-20T01:17:56.168Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644739487Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172612024-04-20T01:17:56.169Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644762137Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"0","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172622024-04-20T01:17:56.169Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644775797Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"1","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172632024-04-20T01:17:56.169Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644811437Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"client":"2","":"downstairs","session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172642024-04-20T01:17:56.169Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644835197Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172652024-04-20T01:17:56.169Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644868077Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172662024-04-20T01:17:56.170Z{"msg":"5f1f304d-200c-444f-8da2-63ecfcf8aa12 is now active with session: b2d31b9f-901f-4832-9d29-f8c0376f1bff","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
172672024-04-20T01:17:56.170Z,"time":"2024-04-20T01:17:54.644903557Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172682024-04-20T01:17:56.170Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644937877Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814,"session_id":"b2d31b9f-901f-4832-9d29-f8c0376f1bff"}
172692024-04-20T01:17:56.170Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:54.644953277Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814}
172702024-04-20T01:17:56.171ZLoading write count information from file "/tmp/test_repair_verify.data"
172712024-04-20T01:17:56.171ZRun Repair workload
172722024-04-20T01:17:56.171Z01/30 Flush
172732024-04-20T01:17:56.179Z02/30 Write block 139 len 28672 data: 13 15 15 15 16 18 18
172742024-04-20T01:17:56.179Z03/30 Flush
172752024-04-20T01:17:56.179Z04/30 Write block 141 len 12288 data: 16 16 17
172762024-04-20T01:17:56.179Z05/30 Read block 181 len 24576
172772024-04-20T01:17:56.180Z06/30 Read block 300 len 4096
172782024-04-20T01:17:56.180Z07/30 Write block 150 len 4096 data: 8
172792024-04-20T01:17:56.180Z08/30 Read block 447 len 32768
172802024-04-20T01:17:56.180Z09/30 Flush
172812024-04-20T01:17:56.180Z10/30 Write block 550 len 4096 data: 18
172822024-04-20T01:17:56.180Z11/30 Read block 192 len 12288
172832024-04-20T01:17:56.180Z12/30 Read block 321 len 36864
172842024-04-20T01:17:56.181Z13/30 Write block 351 len 32768 data: 9 8 8 9 13 15 12 12
172852024-04-20T01:17:56.181Z14/30 Read block 106 len 8192
172862024-04-20T01:17:56.181Z15/30 Read block 45 len 20480
172872024-04-20T01:17:56.181Z16/30 Read block 416 len 28672
172882024-04-20T01:17:56.181Z17/30 Read block 412 len 32768
172892024-04-20T01:17:56.181Z18/30 Write block 26 len 24576 data: 9 9 9 10 10 10
172902024-04-20T01:17:56.182Z19/30 Read block 455 len 36864
172912024-04-20T01:17:56.182Z20/30 Flush
172922024-04-20T01:17:56.182Z21/30 Write block 275 len 16384 data: 11 9 11 11
172932024-04-20T01:17:56.182Z22/30 Read block 303 len 16384
172942024-04-20T01:17:56.182Z23/30 Read block 4 len 12288
172952024-04-20T01:17:56.182Z24/30 Read block 317 len 36864
172962024-04-20T01:17:56.182Z25/30 Write block 281 len 4096 data: 17
172972024-04-20T01:17:56.183Z26/30 Write block 255 len 24576 data: 14 14 10 10 11 9
172982024-04-20T01:17:56.183Z27/30 Read block 56 len 40960
172992024-04-20T01:17:56.183Z28/30 Write block 594 len 8192 data: 13 12
173002024-04-20T01:17:56.183Z29/30 Write block 136 len 40960 data: 11 12 14 14 16 17 17 18 19 19
173012024-04-20T01:17:56.189Z30/30 Write block 213 len 24576 data: 13 12 12 14 19 18
173022024-04-20T01:17:56.189Z----------------------------------------------------------------
173032024-04-20T01:17:56.189Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:27
173042024-04-20T01:17:56.190ZStates: Active Active Active
173052024-04-20T01:17:56.190ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
173062024-04-20T01:17:56.190Z 4 Acked 1003 Write 3 Done Done Done false
173072024-04-20T01:17:56.190Z 5 Acked 1004 Read 6 Done Done Done false
173082024-04-20T01:17:56.190Z 6 Acked 1005 Read 1 Done Done Done false
173092024-04-20T01:17:56.190Z 7 Acked 1006 Write 1 Done Done Done false
173102024-04-20T01:17:56.190Z 8 Acked 1007 Read 8 Done Done Done false
173112024-04-20T01:17:56.191Z 9 Acked 1008 Flush 0 Done Done Sent false
173122024-04-20T01:17:56.191Z 10 Acked 1009 Write 1 Done Done Sent false
173132024-04-20T01:17:56.191Z 11 Acked 1010 Read 3 Done Done Sent false
173142024-04-20T01:17:56.191Z 12 Acked 1011 Read 9 Done Done Sent false
173152024-04-20T01:17:56.191Z 13 Acked 1012 Write 8 Done Done Sent false
173162024-04-20T01:17:56.191Z 14 Acked 1013 Read 2 Done Done Sent false
173172024-04-20T01:17:56.192Z 15 Acked 1014 Read 5 Done Done Sent false
173182024-04-20T01:17:56.192Z 16 Acked 1015 Read 7 Done Done Sent false
173192024-04-20T01:17:56.192Z 17 Acked 1016 Read 8 Done Done Sent false
173202024-04-20T01:17:56.192Z 18 Acked 1017 Write 6 Done Done Sent false
173212024-04-20T01:17:56.192Z 19 Acked 1018 Read 9 Done Done Sent false
173222024-04-20T01:17:56.192Z 20 Acked 1019 Flush 0 Done Done Sent false
173232024-04-20T01:17:56.192Z 21 Acked 1020 Write 4 Done Done Sent false
173242024-04-20T01:17:56.193Z 22 Acked 1021 Read 4 Done Done Sent false
173252024-04-20T01:17:56.193Z 23 Acked 1022 Read 3 Done Done Sent false
173262024-04-20T01:17:56.193Z 24 Acked 1023 Read 9 Done Done Sent false
173272024-04-20T01:17:56.193Z 25 Acked 1024 Write 1 Done Done Sent false
173282024-04-20T01:17:56.193Z 26 Acked 1025 Write 6 Done Done Sent false
173292024-04-20T01:17:56.193Z 27 Acked 1026 Read 10 Done Done Sent false
173302024-04-20T01:17:56.193Z 28 Acked 1027 Write 2 Done Done Sent false
173312024-04-20T01:17:56.193Z 29 Acked 1028 Write 10 Done Done Sent false
173322024-04-20T01:17:56.193Z 30 Acked 1029 Write 6 Sent Sent Sent false
173332024-04-20T01:17:56.193Z STATES DS:0 DS:1 DS:2 TOTAL
173342024-04-20T01:17:56.193Z New 0 0 0 0
173352024-04-20T01:17:56.194Z Sent 1 1 22 24
173362024-04-20T01:17:56.194Z Done 26 26 5 57
173372024-04-20T01:17:56.194Z Skipped 0 0 0 0
173382024-04-20T01:17:56.194Z Error 0 0 0 0
173392024-04-20T01:17:56.194ZLast Flush: 1019 1019 1002
173402024-04-20T01:17:56.194ZDownstairs last five completed: 1002 1001 1000
173412024-04-20T01:17:56.194ZUpstairs last five completed: 30 29 28 27 26
173422024-04-20T01:17:56.195Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:17:54.675236225Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1814}
173432024-04-20T01:17:56.195ZWrote out file "/tmp/test_repair_verify.data"
173442024-04-20T01:17:56.195Z
173452024-04-20T01:17:56.195ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
173462024-04-20T01:17:56.212ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173472024-04-20T01:17:56.212Z 0 000-019 150 150 150 244 244 244 F F F
173482024-04-20T01:17:56.213Z 1 020-039 160 160 157 266 266 258 F F F <---
173492024-04-20T01:17:56.213Z 2 040-059 156 156 156 257 257 257 F F F
173502024-04-20T01:17:56.213Z 3 060-079 158 158 158 260 260 260 F F F
173512024-04-20T01:17:56.213Z 4 080-099 158 158 158 261 261 261 F F F
173522024-04-20T01:17:56.222Z 5 100-119 151 151 151 247 247 247 F F F
173532024-04-20T01:17:56.222Z 6 120-139 160 160 160 264 264 264 T T F <---
173542024-04-20T01:17:56.223Z 7 140-159 160 160 160 265 265 264 T T T <---
173552024-04-20T01:17:56.223Z 8 160-179 150 150 150 244 244 244 F F F
173562024-04-20T01:17:56.223Z 9 180-199 152 152 152 250 250 250 F F F
173572024-04-20T01:17:56.223Z 10 200-219 159 159 159 262 262 262 T T F <---
173582024-04-20T01:17:56.224Z 11 220-239 159 159 159 262 262 262 F F F
173592024-04-20T01:17:56.224Z 12 240-259 156 156 156 256 256 256 T T F <---
173602024-04-20T01:17:56.224Z 13 260-279 156 156 156 256 256 256 T T F <---
173612024-04-20T01:17:56.224Z 14 280-299 158 158 158 261 261 261 T T F <---
173622024-04-20T01:17:56.225Z 15 300-319 158 158 158 261 261 261 F F F
173632024-04-20T01:17:56.225Z 16 320-339 158 158 158 261 261 261 F F F
173642024-04-20T01:17:56.225Z 17 340-359 160 160 159 266 266 262 F F F <---
173652024-04-20T01:17:56.225Z 18 360-379 158 158 158 259 259 259 F F F
173662024-04-20T01:17:56.225Z 19 380-399 157 157 157 258 258 258 F F F
173672024-04-20T01:17:56.226Z 20 400-419 156 156 156 256 256 256 F F F
173682024-04-20T01:17:56.226Z 21 420-439 156 156 156 257 257 257 F F F
173692024-04-20T01:17:56.226Z 22 440-459 159 159 159 262 262 262 F F F
173702024-04-20T01:17:56.226Z 23 460-479 159 159 159 262 262 262 F F F
173712024-04-20T01:17:56.227Z 24 480-499 157 157 157 258 258 258 F F F
173722024-04-20T01:17:56.227Z 25 500-519 158 158 158 261 261 261 F F F
173732024-04-20T01:17:56.227Z 26 520-539 155 155 155 254 254 254 F F F
173742024-04-20T01:17:56.227Z 27 540-559 160 160 157 266 266 258 F F F <---
173752024-04-20T01:17:56.227Z 28 560-579 150 150 150 245 245 245 F F F
173762024-04-20T01:17:56.227Z 29 580-599 158 158 158 259 259 259 T T F <---
173772024-04-20T01:17:56.228ZMax gen: 160, Max flush: 266
173782024-04-20T01:17:56.228ZError: Difference in extent metadata found!
173792024-04-20T01:17:56.228ZA Difference in extent metadata is expected here
173802024-04-20T01:17:56.228ZOn loop 80
173812024-04-20T01:17:56.228Z
173822024-04-20T01:17:56.228ZVerifying data now
173832024-04-20T01:18:00.718ZNo change after verify
173842024-04-20T01:18:00.718ZLoop: 80 Downstairs dump after verify (and repair):
173852024-04-20T01:18:00.730ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173862024-04-20T01:18:00.730Z 0 000-019 150 150 150 244 244 244 F F F
173872024-04-20T01:18:00.731Z 1 020-039 160 160 160 266 266 266 F F F
173882024-04-20T01:18:00.731Z 2 040-059 156 156 156 257 257 257 F F F
173892024-04-20T01:18:00.731Z 3 060-079 158 158 158 260 260 260 F F F
173902024-04-20T01:18:00.731Z 4 080-099 158 158 158 261 261 261 F F F
173912024-04-20T01:18:00.732Z 5 100-119 151 151 151 247 247 247 F F F
173922024-04-20T01:18:00.732Z 6 120-139 161 161 161 267 267 267 F F F
173932024-04-20T01:18:00.732Z 7 140-159 161 161 161 267 267 267 F F F
173942024-04-20T01:18:00.732Z 8 160-179 150 150 150 244 244 244 F F F
173952024-04-20T01:18:00.732Z 9 180-199 152 152 152 250 250 250 F F F
173962024-04-20T01:18:00.733Z 10 200-219 161 161 161 267 267 267 F F F
173972024-04-20T01:18:00.733Z 11 220-239 159 159 159 262 262 262 F F F
173982024-04-20T01:18:00.733Z 12 240-259 161 161 161 267 267 267 F F F
173992024-04-20T01:18:00.733Z 13 260-279 161 161 161 267 267 267 F F F
174002024-04-20T01:18:00.733Z 14 280-299 161 161 161 267 267 267 F F F
174012024-04-20T01:18:00.734Z 15 300-319 158 158 158 261 261 261 F F F
174022024-04-20T01:18:00.734Z 16 320-339 158 158 158 261 261 261 F F F
174032024-04-20T01:18:00.734Z 17 340-359 160 160 160 266 266 266 F F F
174042024-04-20T01:18:00.734Z 18 360-379 158 158 158 259 259 259 F F F
174052024-04-20T01:18:00.734Z 19 380-399 157 157 157 258 258 258 F F F
174062024-04-20T01:18:00.734Z 20 400-419 156 156 156 256 256 256 F F F
174072024-04-20T01:18:00.735Z 21 420-439 156 156 156 257 257 257 F F F
174082024-04-20T01:18:00.735Z 22 440-459 159 159 159 262 262 262 F F F
174092024-04-20T01:18:00.735Z 23 460-479 159 159 159 262 262 262 F F F
174102024-04-20T01:18:00.735Z 24 480-499 157 157 157 258 258 258 F F F
174112024-04-20T01:18:00.735Z 25 500-519 158 158 158 261 261 261 F F F
174122024-04-20T01:18:00.735Z 26 520-539 155 155 155 254 254 254 F F F
174132024-04-20T01:18:00.735Z 27 540-559 160 160 160 266 266 266 F F F
174142024-04-20T01:18:00.735Z 28 560-579 150 150 150 245 245 245 F F F
174152024-04-20T01:18:00.736Z 29 580-599 161 161 161 267 267 267 F F F
174162024-04-20T01:18:00.736ZMax gen: 161, Max flush: 267
174172024-04-20T01:18:00.736Z
174182024-04-20T01:18:00.736ZBegin loop 81
174192024-04-20T01:18:00.736ZDownstairs to restart: 0
174202024-04-20T01:18:00.753Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242226171Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822}
174212024-04-20T01:18:00.754Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242498431Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822}
174222024-04-20T01:18:00.774Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242534592Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822}
174232024-04-20T01:18:00.774Z{"msg":"Crucible b1dbe72a-dac1-4081-a6dc-03e455f8e91b has session id: cf0d9e2b-111b-4831-b9a3-b94790043ef9","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242576432Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174242024-04-20T01:18:00.774Z{"msg":"Upstairs opts: Upstairs UUID: b1dbe72a-dac1-4081-a6dc-03e455f8e91b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242619112Z","hostnameCrucible runtime is spawned
174252024-04-20T01:18:00.775Z":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174262024-04-20T01:18:00.775Z{"msg":"Crucible stats registered with UUID: b1dbe72a-dac1-4081-a6dc-03e455f8e91b","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242724592Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174272024-04-20T01:18:00.775Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242757652Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822}
174282024-04-20T01:18:00.775Z{"msg":"b1dbe72a-dac1-4081-a6dc-03e455f8e91b active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242772792Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174292024-04-20T01:18:00.775Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242793272Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"0","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174302024-04-20T01:18:00.775Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242807952Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"0","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174312024-04-20T01:18:00.775Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242828692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"1","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174322024-04-20T01:18:00.775Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242864792Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"1","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174332024-04-20T01:18:00.775Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242888883Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"2","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174342024-04-20T01:18:00.775Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242909752Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"2","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174352024-04-20T01:18:00.775Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242947792Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174362024-04-20T01:18:00.775Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.242975043Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174372024-04-20T01:18:00.775Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.243018983Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174382024-04-20T01:18:00.776Z{"msg":"ds_connection connected from Ok(127.0.0.1:62178)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.243141843Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174392024-04-20T01:18:00.776Z{"msg":"ds_connection connected from Ok(127.0.0.1:60141)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.243193073Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174402024-04-20T01:18:00.776Z{"msg":"ds_connection connected from Ok(127.0.0.1:47304)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.243209783Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174412024-04-20T01:18:00.776Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.243926485Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"1","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174422024-04-20T01:18:00.776Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.243973065Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"1","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174432024-04-20T01:18:00.776Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.244028995Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"2","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174442024-04-20T01:18:00.776Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.244073885Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"2","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174452024-04-20T01:18:00.776Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.244463156Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"2","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174462024-04-20T01:18:00.776Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.244664316Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"1","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174472024-04-20T01:18:00.776Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.244711027Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"0","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174482024-04-20T01:18:00.776Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.244777477Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"0","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174492024-04-20T01:18:00.776Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.244883467Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"2","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174502024-04-20T01:18:00.776Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.244925417Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"2","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174512024-04-20T01:18:00.776Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.244951597Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174522024-04-20T01:18:00.776Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.245250598Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"1","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174532024-04-20T01:18:00.776Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.245288528Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"1","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174542024-04-20T01:18:00.776Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.245304678Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174552024-04-20T01:18:00.776Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.245318168Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"0","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174562024-04-20T01:18:00.776Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.245738529Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"0","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174572024-04-20T01:18:00.776Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.245785269Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"0","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174582024-04-20T01:18:00.776Z{"msg":"[0]R flush_numbers[0..12]: [244, 266, 257, 260, 261, 247, 267, 267, 244, 250, 267, 262]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.245823489Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"downstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174592024-04-20T01:18:00.776Z{"msg":"[0]R generation[0..12]: [150, 160, 156, 158, 158, 151, 161, 161, 150, 152, 161, 159]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.245860429Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"downstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174602024-04-20T01:18:00.776Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.245896349Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"downstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174612024-04-20T01:18:00.776Z{"msg":"[1]R flush_numbers[0..12]: [244, 266, 257, 260, 261, 247, 267, 267, 244, 250, 267, 262]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.245932509Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"Wait for a query_work_queue command to finish before sending IO
174622024-04-20T01:18:00.777Zdownstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174632024-04-20T01:18:00.777Z{"msg":"[1]R generation[0..12]: [150, 160, 156, 158, 158, 151, 161, 161, 150, 152, 161, 159]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.246018559Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"downstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174642024-04-20T01:18:00.777Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.246050639Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"downstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174652024-04-20T01:18:00.777Z{"msg":"[2]R flush_numbers[0..12]: [244, 266, 257, 260, 261, 247, 267, 267, 244, 250, 267, 262]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.246066359Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"downstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174662024-04-20T01:18:00.777Z{"msg":"[2]R generation[0..12]: [150, 160, 156, 158, 158, 151, 161, 161, 150, 152, 161, 159]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24608765Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"downstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174672024-04-20T01:18:00.777Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.246102339Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"downstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174682024-04-20T01:18:00.777Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24612292Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"downstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174692024-04-20T01:18:00.777Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24613644Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"downstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174702024-04-20T01:18:00.777Z{"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24615667Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"downstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174712024-04-20T01:18:00.777Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24616995Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"":"downstairs","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174722024-04-20T01:18:00.777Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24619014Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174732024-04-20T01:18:00.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24620223Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"0","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174742024-04-20T01:18:00.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24622253Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"1","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174752024-04-20T01:18:00.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24623576Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"client":"2","":"downstairs","session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174762024-04-20T01:18:00.777Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24625524Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174772024-04-20T01:18:00.777Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24626775Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174782024-04-20T01:18:00.777Z{"msg":"b1dbe72a-dac1-4081-a6dc-03e455f8e91b is now active with session: cf0d9e2b-111b-4831-b9a3-b94790043ef9","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24628705Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174792024-04-20T01:18:00.777Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24629902Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822,"session_id":"cf0d9e2b-111b-4831-b9a3-b94790043ef9"}
174802024-04-20T01:18:00.777Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:17:59.24631103Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822}
174812024-04-20T01:18:00.777ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
174822024-04-20T01:18:00.777ZLoading write count information from file "/tmp/test_repair_verify.data"
174832024-04-20T01:18:00.777ZRun Repair workload
174842024-04-20T01:18:00.778Z01/30 Write block 489 len 20480 data: 10 12 14 14 12
174852024-04-20T01:18:00.778Z02/30 Write block 47 len 36864 data: 10 10 12 14 16 17 20 18 18
174862024-04-20T01:18:00.778Z03/30 Read block 417 len 36864
174872024-04-20T01:18:00.778Z04/30 Write block 218 len 16384 data: 19 14 16 14
174882024-04-20T01:18:00.778Z05/30 Read block 472 len 8192
174892024-04-20T01:18:00.778Z06/30 Write block 389 len 4096 data: 7
174902024-04-20T01:18:00.778Z07/30 Read block 178 len 28672
174912024-04-20T01:18:00.778Z08/30 Read block 78 len 40960
174922024-04-20T01:18:00.778Z09/30 Flush
174932024-04-20T01:18:00.784Z10/30 Write block 274 len 28672 data: 12 12 10 12 12 14 15
174942024-04-20T01:18:00.785Z11/30 Read block 231 len 32768
174952024-04-20T01:18:00.785Z12/30 Write block 12 len 8192 data: 6 7
174962024-04-20T01:18:00.785Z13/30 Read block 239 len 8192
174972024-04-20T01:18:00.785Z14/30 Read block 136 len 12288
174982024-04-20T01:18:00.785Z15/30 Write block 62 len 36864 data: 10 8 11 11 14 14 17 17 11
174992024-04-20T01:18:00.786Z16/30 Read block 287 len 40960
175002024-04-20T01:18:00.786Z17/30 Write block 380 len 36864 data: 17 14 13 12 10 8 9 6 6
175012024-04-20T01:18:00.786Z18/30 Write block 508 len 20480 data: 10 15 15 16 17
175022024-04-20T01:18:00.786Z19/30 Read block 104 len 20480
175032024-04-20T01:18:00.786Z20/30 Write block 493 len 20480 data: 13 9 10 9 8
175042024-04-20T01:18:00.786Z21/30 Read block 257 len 8192
175052024-04-20T01:18:00.787Z22/30 Write block 431 len 12288 data: 12 10 10
175062024-04-20T01:18:00.787Z23/30 Read block 339 len 40960
175072024-04-20T01:18:00.787Z24/30 Read block 47 len 28672
175082024-04-20T01:18:00.787Z25/30 Read block 253 len 40960
175092024-04-20T01:18:00.787Z26/30 Write block 159 len 32768 data: 11 14 16 14 13 13 14 12
175102024-04-20T01:18:00.787Z27/30 Read block 507 len 16384
175112024-04-20T01:18:00.788Z28/30 Read block 406 len 28672
175122024-04-20T01:18:00.788Z29/30 Write block 586 len 32768 data: 20 18 17 12 9 11 10 11
175132024-04-20T01:18:00.788Z30/30 Read block 406 len 4096
175142024-04-20T01:18:00.788Z----------------------------------------------------------------
175152024-04-20T01:18:00.788Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
175162024-04-20T01:18:00.789ZStates: Active Active Active
175172024-04-20T01:18:00.789ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
175182024-04-20T01:18:00.789Z 1 Acked 1000 Write 5 Done Done Done false
175192024-04-20T01:18:00.789Z 2 Acked 1001 Write 9 Sent Done Done false
175202024-04-20T01:18:00.789Z 3 Acked 1002 Read 9 Sent Done Done false
175212024-04-20T01:18:00.790Z 4 Acked 1003 Write 4 Sent Done Done false
175222024-04-20T01:18:00.790Z 5 Acked 1004 Read 2 Sent Done Done false
175232024-04-20T01:18:00.790Z 6 Acked 1005 Write 1 Sent Done Done false
175242024-04-20T01:18:00.790Z 7 Acked 1006 Read 7 Sent Done Done false
175252024-04-20T01:18:00.790Z 8 Acked 1007 Read 10 Sent Done Done false
175262024-04-20T01:18:00.790Z 9 Acked 1008 Flush 0 Sent Done Done false
175272024-04-20T01:18:00.790Z 10 Acked 1009 Write 7 Sent Done Done false
175282024-04-20T01:18:00.791Z 11 Acked 1010 Read 8 Sent Done Done false
175292024-04-20T01:18:00.791Z 12 Acked 1011 Write 2 Sent Done Done false
175302024-04-20T01:18:00.791Z 13 Acked 1012 Read 2 Sent Done Done false
175312024-04-20T01:18:00.791Z 14 Acked 1013 Read 3 Sent Done Done false
175322024-04-20T01:18:00.791Z 15 Acked 1014 Write 9 Sent Done Done false
175332024-04-20T01:18:00.791Z 16 Acked 1015 Read 10 Sent Done Done false
175342024-04-20T01:18:00.792Z 17 Acked 1016 Write 9 Sent Done Done false
175352024-04-20T01:18:00.792Z 18 Acked 1017 Write 5 Sent Done Done false
175362024-04-20T01:18:00.792Z 19 Acked 1018 Read 5 Sent Done Done false
175372024-04-20T01:18:00.792Z 20 Acked 1019 Write 5 Sent Done Done false
175382024-04-20T01:18:00.793Z 21 Acked 1020 Read 2 Sent Done Done false
175392024-04-20T01:18:00.793Z 22 Acked 1021 Write 3 Sent Done Done false
175402024-04-20T01:18:00.793Z 23 Acked 1022 Read 10 Sent Done Done false
175412024-04-20T01:18:00.793Z 24 Acked 1023 Read 7 Sent Done Done false
175422024-04-20T01:18:00.794Z 25 Acked 1024 Read 10 Sent Done Done false
175432024-04-20T01:18:00.794Z 26 Acked 1025 Write 8 Sent Done Done false
175442024-04-20T01:18:00.794Z 27 Acked 1026 Read 4 Sent Done Done false
175452024-04-20T01:18:00.794Z 28 Acked 1027 Read 7 Sent Done Done false
175462024-04-20T01:18:00.795Z 29 Acked 1028 Write 8 Sent Done Done false
175472024-04-20T01:18:00.795Z 30 Acked 1029 Read 1 Sent Done Done false
175482024-04-20T01:18:00.795Z STATES DS:0 DS:1 DS:2 TOTAL
175492024-04-20T01:18:00.795Z New 0 0 0 0
175502024-04-20T01:18:00.795Z Sent 29 0 0 29
175512024-04-20T01:18:00.796Z Done 1 30 30 61
175522024-04-20T01:18:00.796Z Skipped 0 0 0 0
175532024-04-20T01:18:00.796Z Error 0 0 0 0
175542024-04-20T01:18:00.796ZLast Flush: 0 1008 1008
175552024-04-20T01:18:00.796ZDownstairs last five completed:
175562024-04-20T01:18:00.797ZUpstairs last five completed: 30 29 28 27 26
175572024-04-20T01:18:00.797Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:17:59.276880038Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1822}
175582024-04-20T01:18:00.797ZWrote out file "/tmp/test_repair_verify.data"
175592024-04-20T01:18:00.805Z
175602024-04-20T01:18:00.805ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
175612024-04-20T01:18:00.817ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175622024-04-20T01:18:00.817Z 0 000-019 150 150 150 244 244 244 F T T <---
175632024-04-20T01:18:00.817Z 1 020-039 160 160 160 266 266 266 F F F
175642024-04-20T01:18:00.817Z 2 040-059 156 162 162 257 268 268 F F F <---
175652024-04-20T01:18:00.818Z 3 060-079 158 158 158 260 260 260 F T T <---
175662024-04-20T01:18:00.818Z 4 080-099 158 158 158 261 261 261 F F F
175672024-04-20T01:18:00.818Z 5 100-119 151 151 151 247 247 247 F F F
175682024-04-20T01:18:00.818Z 6 120-139 161 161 161 267 267 267 F F F
175692024-04-20T01:18:00.818Z 7 140-159 161 161 161 267 267 267 F T T <---
175702024-04-20T01:18:00.818Z 8 160-179 150 150 150 244 244 244 F T T <---
175712024-04-20T01:18:00.818Z 9 180-199 152 152 152 250 250 250 F F F
175722024-04-20T01:18:00.818Z 10 200-219 161 162 162 267 268 268 F F F <---
175732024-04-20T01:18:00.818Z 11 220-239 159 162 162 262 268 268 F F F <---
175742024-04-20T01:18:00.818Z 12 240-259 161 161 161 267 267 267 F F F
175752024-04-20T01:18:00.819Z 13 260-279 161 161 161 267 267 267 F T T <---
175762024-04-20T01:18:00.819Z 14 280-299 161 161 161 267 267 267 F T T <---
175772024-04-20T01:18:00.819Z 15 300-319 158 158 158 261 261 261 F F F
175782024-04-20T01:18:00.819Z 16 320-339 158 158 158 261 261 261 F F F
175792024-04-20T01:18:00.819Z 17 340-359 160 160 160 266 266 266 F F F
175802024-04-20T01:18:00.819Z 18 360-379 158 158 158 259 259 259 F F F
175812024-04-20T01:18:00.819Z 19 380-399 157 162 162 258 268 268 F T T <---
175822024-04-20T01:18:00.819Z 20 400-419 156 156 156 256 256 256 F F F
175832024-04-20T01:18:00.819Z 21 420-439 156 156 156 257 257 257 F T T <---
175842024-04-20T01:18:00.819Z 22 440-459 159 159 159 262 262 262 F F F
175852024-04-20T01:18:00.819Z 23 460-479 159 159 159 262 262 262 F F F
175862024-04-20T01:18:00.820Z 24 480-499 157 162 162 258 268 268 T T T <---
175872024-04-20T01:18:00.820Z 25 500-519 158 158 158 261 261 261 F T T <---
175882024-04-20T01:18:00.820Z 26 520-539 155 155 155 254 254 254 F F F
175892024-04-20T01:18:00.820Z 27 540-559 160 160 160 266 266 266 F F F
175902024-04-20T01:18:00.820Z 28 560-579 150 150 150 245 245 245 F F F
175912024-04-20T01:18:00.820Z 29 580-599 161 161 161 267 267 267 F T T <---
175922024-04-20T01:18:00.820ZMax gen: 162, Max flush: 268
175932024-04-20T01:18:00.820ZError: Difference in extent metadata found!
175942024-04-20T01:18:00.820ZA Difference in extent metadata is expected here
175952024-04-20T01:18:00.821ZOn loop 81
175962024-04-20T01:18:00.821Z
175972024-04-20T01:18:00.828ZVerifying data now
175982024-04-20T01:18:05.469ZNo change after verify
175992024-04-20T01:18:05.470ZLoop: 81 Downstairs dump after verify (and repair):
176002024-04-20T01:18:05.487ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176012024-04-20T01:18:05.487Z 0 000-019 163 163 163 269 269 269 F F F
176022024-04-20T01:18:05.487Z 1 020-039 160 160 160 266 266 266 F F F
176032024-04-20T01:18:05.487Z 2 040-059 162 162 162 268 268 268 F F F
176042024-04-20T01:18:05.488Z 3 060-079 163 163 163 269 269 269 F F F
176052024-04-20T01:18:05.488Z 4 080-099 158 158 158 261 261 261 F F F
176062024-04-20T01:18:05.488Z 5 100-119 151 151 151 247 247 247 F F F
176072024-04-20T01:18:05.910Z 6 120-139 161 161 161 267 267 267 F F F
176082024-04-20T01:18:05.910Z 7 140-159 163 163 163 269 269 269 F F F
176092024-04-20T01:18:05.910Z 8 160-179 163 163 163 269 269 269 F F F
176102024-04-20T01:18:05.910Z 9 180-199 152 152 152 250 250 250 F F F
176112024-04-20T01:18:05.911Z 10 200-219 162 162 162 268 268 268 F F F
176122024-04-20T01:18:05.911Z 11 220-239 162 162 162 268 268 268 F F F
176132024-04-20T01:18:05.911Z 12 240-259 161 161 161 267 267 267 F F F
176142024-04-20T01:18:05.911Z 13 260-279 163 163 163 269 269 269 F F F
176152024-04-20T01:18:05.911Z 14 280-299 163 163 163 269 269 269 F F F
176162024-04-20T01:18:05.911Z 15 300-319 158 158 158 261 261 261 F F F
176172024-04-20T01:18:05.912Z 16 320-339 158 158 158 261 261 261 F F F
176182024-04-20T01:18:05.912Z 17 340-359 160 160 160 266 266 266 F F F
176192024-04-20T01:18:05.912Z 18 360-379 158 158 158 259 259 259 F F F
176202024-04-20T01:18:05.912Z 19 380-399 163 163 163 269 269 269 F F F
176212024-04-20T01:18:05.912Z 20 400-419 156 156 156 256 256 256 F F F
176222024-04-20T01:18:05.912Z 21 420-439 163 163 163 269 269 269 F F F
176232024-04-20T01:18:05.913Z 22 440-459 159 159 159 262 262 262 F F F
176242024-04-20T01:18:05.913Z 23 460-479 159 159 159 262 262 262 F F F
176252024-04-20T01:18:05.913Z 24 480-499 163 163 163 269 269 269 F F F
176262024-04-20T01:18:05.913Z 25 500-519 163 163 163 269 269 269 F F F
176272024-04-20T01:18:05.913Z 26 520-539 155 155 155 254 254 254 F F F
176282024-04-20T01:18:05.913Z 27 540-559 160 160 160 266 266 266 F F F
176292024-04-20T01:18:05.914Z 28 560-579 150 150 150 245 245 245 F F F
176302024-04-20T01:18:05.914Z 29 580-599 163 163 163 269 269 269 F F F
176312024-04-20T01:18:05.914ZMax gen: 163, Max flush: 269
176322024-04-20T01:18:05.914Z
176332024-04-20T01:18:05.914ZBegin loop 82
176342024-04-20T01:18:05.914ZDownstairs to restart: 0
176352024-04-20T01:18:05.937Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.998764189Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830}
176362024-04-20T01:18:05.937Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.999081179Z","hostname":"Crucible runtime is spawned
176372024-04-20T01:18:05.937Zw-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830}
176382024-04-20T01:18:05.937Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.999137239Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830}
176392024-04-20T01:18:05.938Z{"msg":"Crucible 126064d0-24e1-458b-b7fa-f5590f0e7482 has session id: 6a9d68ac-d795-49f1-af4b-19fb5aae28a7","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.9991818Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176402024-04-20T01:18:05.938Z{"msg":"Upstairs opts: Upstairs UUID: 126064d0-24e1-458b-b7fa-f5590f0e7482, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.99924287Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176412024-04-20T01:18:05.938Z{"msg":"Crucible stats registered with UUID: 126064d0-24e1-458b-b7fa-f5590f0e7482","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.99928974Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176422024-04-20T01:18:05.938Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.99933362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830}
176432024-04-20T01:18:05.938Z{"msg":"126064d0-24e1-458b-b7fa-f5590f0e7482 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.99935854Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176442024-04-20T01:18:05.938Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.99938699Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"0","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176452024-04-20T01:18:05.939Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.9994452Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"0","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176462024-04-20T01:18:05.939Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.99948698Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"1","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176472024-04-20T01:18:05.939Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.999530951Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"1","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176482024-04-20T01:18:05.939Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.99956464Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"2","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176492024-04-20T01:18:05.940Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.999610241Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"2","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176502024-04-20T01:18:05.940Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.999652151Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176512024-04-20T01:18:05.940Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.999710201Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176522024-04-20T01:18:05.940Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.999742241Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176532024-04-20T01:18:05.940Z{"msg":"ds_connection connected from Ok(127.0.0.1:62411)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.999767241Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176542024-04-20T01:18:05.940Z{"msg":"ds_connection connected from Ok(127.0.0.1:51228)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.999796081Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176552024-04-20T01:18:05.940Z{"msg":"ds_connection connected from Ok(127.0.0.1:63632)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:03.999820751Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176562024-04-20T01:18:05.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.000430353Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"2","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176572024-04-20T01:18:05.941Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.000477942Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"2","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176582024-04-20T01:18:05.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.000498283Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"1","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176592024-04-20T01:18:05.941Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.000531063Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"1","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176602024-04-20T01:18:05.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.000867194Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"0","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176612024-04-20T01:18:05.941Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.000916383Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"0","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176622024-04-20T01:18:05.941Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.001060074Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"2","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176632024-04-20T01:18:05.941Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.001194104Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"1","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176642024-04-20T01:18:05.941Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.001403215Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"0","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176652024-04-20T01:18:05.941Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.001552825Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"2","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176662024-04-20T01:18:05.941Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.001603645Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"2","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176672024-04-20T01:18:05.942Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.001639735Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176682024-04-20T01:18:05.942Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.001672425Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"1","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176692024-04-20T01:18:05.942Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.001706145Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"1","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176702024-04-20T01:18:05.942Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.001739265Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176712024-04-20T01:18:05.942Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.001908116Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"0","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176722024-04-20T01:18:05.943Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.001968946Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"0","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176732024-04-20T01:18:05.943Z{"msg":"[0]R flush_numbers[0..12]: [269, 266, 268, 269, 261, 247, 267, 269, 269, 250, 268, 268]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002003326Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176742024-04-20T01:18:05.943Z{"msg":"[0]R generation[0..12]: [163, 160, 162, 163, 158, 151, 161, 163, 163, 152, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002018976Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176752024-04-20T01:18:05.943Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002034186Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176762024-04-20T01:18:05.943Z{"msg":"[1]R flush_numbers[0..12]: [269, 266, 268, 269, 261, 247, 267, 269, 269, 250, 268, 268]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002048606Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176772024-04-20T01:18:05.944Z{"Wait for a query_work_queue command to finish before sending IO
176782024-04-20T01:18:05.944Zmsg":"[1]R generation[0..12]: [163, 160, 162, 163, 158, 151, 161, 163, 163, 152, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002072686Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176792024-04-20T01:18:05.944Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002087486Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176802024-04-20T01:18:05.944Z{"msg":"[2]R flush_numbers[0..12]: [269, 266, 268, 269, 261, 247, 267, 269, 269, 250, 268, 268]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002101976Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176812024-04-20T01:18:05.944Z{"msg":"[2]R generation[0..12]: [163, 160, 162, 163, 158, 151, 161, 163, 163, 152, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002117976Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176822024-04-20T01:18:05.945Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002132446Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176832024-04-20T01:18:05.945Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002145496Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176842024-04-20T01:18:05.945Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002166346Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176852024-04-20T01:18:05.945Z{"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002179696Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176862024-04-20T01:18:05.946Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002192586Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"":"downstairs","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176872024-04-20T01:18:05.946Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002212886Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176882024-04-20T01:18:05.946Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002225326Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"0","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176892024-04-20T01:18:05.946Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002245606Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"1","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176902024-04-20T01:18:05.946Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002259106Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"client":"2","":"downstairs","session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176912024-04-20T01:18:05.946Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002272897Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176922024-04-20T01:18:05.947Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002285077Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176932024-04-20T01:18:05.947Z{"msg":"126064d0-24e1-458b-b7fa-f5590f0e7482 is now active with session: 6a9d68ac-d795-49f1-af4b-19fb5aae28a7","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002304637Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176942024-04-20T01:18:05.947Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002317507Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830,"session_id":"6a9d68ac-d795-49f1-af4b-19fb5aae28a7"}
176952024-04-20T01:18:05.947Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:04.002336927Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830}
176962024-04-20T01:18:05.947ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176972024-04-20T01:18:05.947ZLoading write count information from file "/tmp/test_repair_verify.data"
176982024-04-20T01:18:05.947ZRun Repair workload
176992024-04-20T01:18:05.947Z01/30 Write block 356 len 24576 data: 16 13 13 13 11 11
177002024-04-20T01:18:05.947Z02/30 Read block 91 len 40960
177012024-04-20T01:18:05.947Z03/30 Read block 286 len 40960
177022024-04-20T01:18:05.947Z04/30 Read block 589 len 24576
177032024-04-20T01:18:05.947Z05/30 Write block 127 len 4096 data: 11
177042024-04-20T01:18:05.947Z06/30 Write block 82 len 12288 data: 10 13 13
177052024-04-20T01:18:05.947Z07/30 Write block 327 len 4096 data: 14
177062024-04-20T01:18:05.947Z08/30 Write block 5 len 16384 data: 9 9 7 7
177072024-04-20T01:18:05.947Z09/30 Write block 321 len 20480 data: 17 17 17 14 15
177082024-04-20T01:18:05.947Z10/30 Read block 260 len 36864
177092024-04-20T01:18:05.948Z11/30 Write block 216 len 36864 data: 15 20 20 15 17 15 15 16 16
177102024-04-20T01:18:05.948Z12/30 Read block 339 len 12288
177112024-04-20T01:18:05.948Z13/30 Write block 539 len 4096 data: 8
177122024-04-20T01:18:05.948Z14/30 Read block 123 len 32768
177132024-04-20T01:18:05.948Z15/30 Read block 207 len 12288
177142024-04-20T01:18:05.949Z16/30 Write block 544 len 32768 data: 16 14 14 10 13 16 19 16
177152024-04-20T01:18:05.949Z17/30 Read block 173 len 4096
177162024-04-20T01:18:05.949Z18/30 Read block 517 len 20480
177172024-04-20T01:18:05.949Z19/30 Write block 354 len 28672 data: 10 14 17 14 14 14 12
177182024-04-20T01:18:05.949Z20/30 Write block 580 len 40960 data: 16 15 14 12 16 19 21 19 18 13
177192024-04-20T01:18:05.949Z21/30 Write block 191 len 36864 data: 18 14 14 15 13 11 13 14 16
177202024-04-20T01:18:05.950Z22/30 Read block 128 len 40960
177212024-04-20T01:18:05.950Z23/30 Write block 475 len 12288 data: 16 14 16
177222024-04-20T01:18:05.950Z24/30 Read block 443 len 20480
177232024-04-20T01:18:05.950Z25/30 Write block 7 len 4096 data: 8
177242024-04-20T01:18:05.950Z26/30 Flush
177252024-04-20T01:18:05.950Z27/30 Write block 251 len 40960 data: 11 10 10 14 15 15 11 11 12 10
177262024-04-20T01:18:05.951Z28/30 Read block 473 len 24576
177272024-04-20T01:18:05.951Z29/30 Write block 562 len 28672 data: 10 9 13 15 16 14 16
177282024-04-20T01:18:05.951Z30/30 Write block 553 len 8192 data: 10 12
177292024-04-20T01:18:05.951Z----------------------------------------------------------------
177302024-04-20T01:18:05.951Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
177312024-04-20T01:18:05.951ZStates: Active Active Active
177322024-04-20T01:18:05.951ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
177332024-04-20T01:18:05.952Z 1 Acked 1000 Write 6 Done Done Done false
177342024-04-20T01:18:05.952Z 2 Acked 1001 Read 10 Done Done Done false
177352024-04-20T01:18:05.972Z 3 Acked 1002 Read 10 Done Done Done false
177362024-04-20T01:18:05.972Z 4 Acked 1003 Read 6 Done Done Done false
177372024-04-20T01:18:05.973Z 5 Acked 1004 Write 1 Done Done Done false
177382024-04-20T01:18:05.973Z 6 Acked 1005 Write 3 Done Done Done false
177392024-04-20T01:18:05.973Z 7 Acked 1006 Write 1 Sent Done Done false
177402024-04-20T01:18:05.973Z 8 Acked 1007 Write 4 Sent Done Done false
177412024-04-20T01:18:05.974Z 9 Acked 1008 Write 5 Sent Done Done false
177422024-04-20T01:18:05.974Z 10 Acked 1009 Read 9 Sent Done Done false
177432024-04-20T01:18:05.974Z 11 Acked 1010 Write 9 Sent Done Done false
177442024-04-20T01:18:05.974Z 12 Acked 1011 Read 3 Sent Done Done false
177452024-04-20T01:18:05.974Z 13 Acked 1012 Write 1 Sent Done Done false
177462024-04-20T01:18:05.974Z 14 Acked 1013 Read 8 Sent Done Done false
177472024-04-20T01:18:05.974Z 15 Acked 1014 Read 3 Sent Done Done false
177482024-04-20T01:18:05.974Z 16 Acked 1015 Write 8 Sent Done Done false
177492024-04-20T01:18:05.974Z 17 Acked 1016 Read 1 Sent Done Done false
177502024-04-20T01:18:05.974Z 18 Acked 1017 Read 5 Sent Done Done false
177512024-04-20T01:18:05.974Z 19 Acked 1018 Write 7 Sent Done Done false
177522024-04-20T01:18:05.974Z 20 Acked 1019 Write 10 Sent Done Done false
177532024-04-20T01:18:05.974Z 21 Acked 1020 Write 9 Sent Done Done false
177542024-04-20T01:18:05.974Z 22 Acked 1021 Read 10 Sent Done Done false
177552024-04-20T01:18:05.975Z 23 Acked 1022 Write 3 Sent Done Done false
177562024-04-20T01:18:05.975Z 24 Acked 1023 Read 5 Sent Done Done false
177572024-04-20T01:18:05.975Z 25 Acked 1024 Write 1 Sent Done Done false
177582024-04-20T01:18:05.975Z 26 Acked 1025 Flush 0 Sent Done Done false
177592024-04-20T01:18:05.975Z 27 Acked 1026 Write 10 Sent Done Done false
177602024-04-20T01:18:05.975Z 28 Acked 1027 Read 6 Sent Done Done false
177612024-04-20T01:18:05.975Z 29 Acked 1028 Write 7 Sent Done Done false
177622024-04-20T01:18:05.976Z 30 Acked 1029 Write 2 Sent Sent Sent false
177632024-04-20T01:18:05.976Z STATES DS:0 DS:1 DS:2 TOTAL
177642024-04-20T01:18:05.976Z New 0 0 0 0
177652024-04-20T01:18:05.976Z Sent 24 1 1 26
177662024-04-20T01:18:05.976Z Done 6 29 29 64
177672024-04-20T01:18:05.977Z Skipped 0 0 0 0
177682024-04-20T01:18:05.977Z Error 0 0 0 0
177692024-04-20T01:18:05.977ZLast Flush: 0 1025 1025
177702024-04-20T01:18:05.977ZDownstairs last five completed:
177712024-04-20T01:18:05.977ZUpstairs last five completed: 30 29 28 27 26
177722024-04-20T01:18:05.977Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:18:04.035829651Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1830}
177732024-04-20T01:18:05.977ZWrote out file "/tmp/test_repair_verify.data"
177742024-04-20T01:18:05.977Z
177752024-04-20T01:18:05.977ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
177762024-04-20T01:18:05.977ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177772024-04-20T01:18:05.978Z 0 000-019 163 164 164 269 270 270 F F F <---
177782024-04-20T01:18:05.978Z 1 020-039 160 160 160 266 266 266 F F F
177792024-04-20T01:18:05.978Z 2 040-059 162 162 162 268 268 268 F F F
177802024-04-20T01:18:05.978Z 3 060-079 163 163 163 269 269 269 F F F
177812024-04-20T01:18:05.978Z 4 080-099 158 164 164 261 270 270 T F F <---
177822024-04-20T01:18:05.978Z 5 100-119 151 151 151 247 247 247 F F F
177832024-04-20T01:18:05.978Z 6 120-139 161 164 164 267 270 270 T F F <---
177842024-04-20T01:18:05.978Z 7 140-159 163 163 163 269 269 269 F F F
177852024-04-20T01:18:05.979Z 8 160-179 163 163 163 269 269 269 F F F
177862024-04-20T01:18:05.979Z 9 180-199 152 164 164 250 270 270 F F F <---
177872024-04-20T01:18:05.979Z 10 200-219 162 164 164 268 270 270 F F F <---
177882024-04-20T01:18:05.979Z 11 220-239 162 164 164 268 270 270 F F F <---
177892024-04-20T01:18:05.979Z 12 240-259 161 161 161 267 267 267 F T T <---
177902024-04-20T01:18:05.979Z 13 260-279 163 163 163 269 269 269 F T T <---
177912024-04-20T01:18:05.979Z 14 280-299 163 163 163 269 269 269 F F F
177922024-04-20T01:18:05.979Z 15 300-319 158 158 158 261 261 261 F F F
177932024-04-20T01:18:05.979Z 16 320-339 158 164 164 261 270 270 F F F <---
177942024-04-20T01:18:05.979Z 17 340-359 160 164 164 266 270 270 T F F <---
177952024-04-20T01:18:05.979Z 18 360-379 158 164 164 259 270 270 T F F <---
177962024-04-20T01:18:05.979Z 19 380-399 163 163 163 269 269 269 F F F
177972024-04-20T01:18:05.979Z 20 400-419 156 156 156 256 256 256 F F F
177982024-04-20T01:18:05.979Z 21 420-439 163 163 163 269 269 269 F F F
177992024-04-20T01:18:05.979Z 22 440-459 159 159 159 262 262 262 F F F
178002024-04-20T01:18:05.979Z 23 460-479 159 164 164 262 270 270 F F F <---
178012024-04-20T01:18:05.979Z 24 480-499 163 163 163 269 269 269 F F F
178022024-04-20T01:18:05.979Z 25 500-519 163 163 163 269 269 269 F F F
178032024-04-20T01:18:05.979Z 26 520-539 155 164 164 254 270 270 F F F <---
178042024-04-20T01:18:05.979Z 27 540-559 160 164 164 266 270 270 F T T <---
178052024-04-20T01:18:05.980Z 28 560-579 150 150 150 245 245 245 F T T <---
178062024-04-20T01:18:05.980Z 29 580-599 163 164 164 269 270 270 F F F <---
178072024-04-20T01:18:05.980ZMax gen: 164, Max flush: 270
178082024-04-20T01:18:05.980ZError: Difference in extent metadata found!
178092024-04-20T01:18:05.980ZA Difference in extent metadata is expected here
178102024-04-20T01:18:05.980ZOn loop 82
178112024-04-20T01:18:05.980Z
178122024-04-20T01:18:05.980ZVerifying data now
178132024-04-20T01:18:20.280ZNo change after verify
178142024-04-20T01:18:20.280ZLoop: 82 Downstairs dump after verify (and repair):
178152024-04-20T01:18:20.295ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178162024-04-20T01:18:20.295Z 0 000-019 164 164 164 270 270 270 F F F
178172024-04-20T01:18:20.301Z 1 020-039 160 160 160 266 266 266 F F F
178182024-04-20T01:18:20.301Z 2 040-059 162 162 162 268 268 268 F F F
178192024-04-20T01:18:20.301Z 3 060-079 163 163 163 269 269 269 F F F
178202024-04-20T01:18:20.301Z 4 080-099 164 164 164 270 270 270 F F F
178212024-04-20T01:18:20.301Z 5 100-119 151 151 151 247 247 247 F F F
178222024-04-20T01:18:20.301Z 6 120-139 164 164 164 270 270 270 F F F
178232024-04-20T01:18:20.301Z 7 140-159 163 163 163 269 269 269 F F F
178242024-04-20T01:18:20.301Z 8 160-179 163 163 163 269 269 269 F F F
178252024-04-20T01:18:20.301Z 9 180-199 164 164 164 270 270 270 F F F
178262024-04-20T01:18:20.301Z 10 200-219 164 164 164 270 270 270 F F F
178272024-04-20T01:18:20.301Z 11 220-239 164 164 164 270 270 270 F F F
178282024-04-20T01:18:20.301Z 12 240-259 165 165 165 271 271 271 F F F
178292024-04-20T01:18:20.301Z 13 260-279 165 165 165 271 271 271 F F F
178302024-04-20T01:18:20.301Z 14 280-299 163 163 163 269 269 269 F F F
178312024-04-20T01:18:20.301Z 15 300-319 158 158 158 261 261 261 F F F
178322024-04-20T01:18:20.301Z 16 320-339 164 164 164 270 270 270 F F F
178332024-04-20T01:18:20.301Z 17 340-359 164 164 164 270 270 270 F F F
178342024-04-20T01:18:20.302Z 18 360-379 164 164 164 270 270 270 F F F
178352024-04-20T01:18:20.302Z 19 380-399 163 163 163 269 269 269 F F F
178362024-04-20T01:18:20.302Z 20 400-419 156 156 156 256 256 256 F F F
178372024-04-20T01:18:20.302Z 21 420-439 163 163 163 269 269 269 F F F
178382024-04-20T01:18:20.302Z 22 440-459 159 159 159 262 262 262 F F F
178392024-04-20T01:18:20.302Z 23 460-479 164 164 164 270 270 270 F F F
178402024-04-20T01:18:20.302Z 24 480-499 163 163 163 269 269 269 F F F
178412024-04-20T01:18:20.302Z 25 500-519 163 163 163 269 269 269 F F F
178422024-04-20T01:18:20.302Z 26 520-539 164 164 164 270 270 270 F F F
178432024-04-20T01:18:20.302Z 27 540-559 165 165 165 271 271 271 F F F
178442024-04-20T01:18:20.302Z 28 560-579 165 165 165 271 271 271 F F F
178452024-04-20T01:18:20.302Z 29 580-599 164 164 164 270 270 270 F F F
178462024-04-20T01:18:20.302ZMax gen: 165, Max flush: 271
178472024-04-20T01:18:20.302Z
178482024-04-20T01:18:20.302ZBegin loop 83
178492024-04-20T01:18:20.302ZDownstairs to restart: 0
178502024-04-20T01:18:20.315Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.811734392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838}
178512024-04-20T01:18:20.323Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812021553Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838}
178522024-04-20T01:18:20.323ZCrucible runtime is spawned
178532024-04-20T01:18:20.323Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812074443Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838}
178542024-04-20T01:18:20.324Z{"msg":"Crucible c31406c5-1bb4-407b-8770-e0b48a75ff01 has session id: 3afc6bb9-ef82-469e-a753-fcd4c4f7138d","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812139153Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178552024-04-20T01:18:20.324Z{"msg":"Upstairs opts: Upstairs UUID: c31406c5-1bb4-407b-8770-e0b48a75ff01, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812198324Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178562024-04-20T01:18:20.324Z{"msg":"Crucible stats registered with UUID: c31406c5-1bb4-407b-8770-e0b48a75ff01","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812225884Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178572024-04-20T01:18:20.324Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812257354Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838}
178582024-04-20T01:18:20.324Z{"msg":"c31406c5-1bb4-407b-8770-e0b48a75ff01 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812275814Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178592024-04-20T01:18:20.324Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812294414Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"0","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178602024-04-20T01:18:20.324Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812328714Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"0","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178612024-04-20T01:18:20.324Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812355424Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"1","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178622024-04-20T01:18:20.324Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812394504Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"1","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178632024-04-20T01:18:20.324Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812422864Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"2","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178642024-04-20T01:18:20.324Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812450844Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"2","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178652024-04-20T01:18:20.324Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812488464Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178662024-04-20T01:18:20.324Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812516564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178672024-04-20T01:18:20.325Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812541744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178682024-04-20T01:18:20.325Z{"msg":"ds_connection connected from Ok(127.0.0.1:49178)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812565765Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178692024-04-20T01:18:20.325Z{"msg":"ds_connection connected from Ok(127.0.0.1:47068)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812590934Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178702024-04-20T01:18:20.325Z{"msg":"ds_connection connected from Ok(127.0.0.1:35322)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.812668185Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178712024-04-20T01:18:20.325Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.813382906Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"1","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178722024-04-20T01:18:20.325Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.813416366Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"1","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178732024-04-20T01:18:20.325Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.813434386Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"2","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178742024-04-20T01:18:20.325Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.813503267Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"2","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178752024-04-20T01:18:20.325Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.813958377Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"1","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178762024-04-20T01:18:20.326Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.814004128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"0","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178772024-04-20T01:18:20.326Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.814030448Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"0","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178782024-04-20T01:18:20.326Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.814055188Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"2","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178792024-04-20T01:18:20.326Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.814536489Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"1","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178802024-04-20T01:18:20.326Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.814576819Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"1","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178812024-04-20T01:18:20.326Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.814593539Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178822024-04-20T01:18:20.326Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.814606689Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"2","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178832024-04-20T01:18:20.326Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.814628279Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"2","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178842024-04-20T01:18:20.326Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.814642469Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178852024-04-20T01:18:20.326Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.81486077Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"0","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178862024-04-20T01:18:20.326Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.81527082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"0","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178872024-04-20T01:18:20.326Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.81531478Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"0","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178882024-04-20T01:18:20.326Z{"msg":"[0]R flush_numbers[0..12]: [270, 266, 268, 269, 270, 247, 270, 269, 269, 270, 270, 270]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815332571Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178892024-04-20T01:18:20.326Z{"msg":"[0]R generation[0..12]: [164, 160, 162, 163, 164, 151, 164, 163, 163, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815355841Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178902024-04-20T01:18:20.326Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815370921Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178912024-04-20T01:18:20.326Z{"msg":"[1]R flush_numbers[0..12]: [270, 266, 268, 269, 270, 247, 270, 269, 269, 270, 270, 270]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815392241Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178922024-04-20T01:18:20.327Z{"msg":"[1]R generation[0..12]: [164, 160, 162, 163, 164, 151, 164, 163, 163, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815406331Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178932024-04-20T01:18:20.327ZWait for a query_work_queue command to finish before sending IO
178942024-04-20T01:18:20.327Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815429861Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178952024-04-20T01:18:20.327Z{"msg":"[2]R flush_numbers[0..12]: [270, 266, 268, 269, 270, 247, 270, 269, 269, 270, 270, 270]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815444391Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178962024-04-20T01:18:20.327Z{"msg":"[2]R generation[0..12]: [164, 160, 162, 163, 164, 151, 164, 163, 163, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815465191Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178972024-04-20T01:18:20.327Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815479201Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178982024-04-20T01:18:20.327Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815499121Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
178992024-04-20T01:18:20.327Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815512941Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
179002024-04-20T01:18:20.327Z{"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815526471Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
179012024-04-20T01:18:20.327Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815545901Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"":"downstairs","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
179022024-04-20T01:18:20.327Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815559051Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
179032024-04-20T01:18:20.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815570731Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"0","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
179042024-04-20T01:18:20.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815590681Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"1","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
179052024-04-20T01:18:20.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815603991Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"client":"2","":"downstairs","session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
179062024-04-20T01:18:20.327Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815624561Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
179072024-04-20T01:18:20.327Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815636931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
179082024-04-20T01:18:20.327Z}
179092024-04-20T01:18:20.327Z{"msg":"c31406c5-1bb4-407b-8770-e0b48a75ff01 is now active with session: 3afc6bb9-ef82-469e-a753-fcd4c4f7138d","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815666211Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
179102024-04-20T01:18:20.327Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815677961Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838,"session_id":"3afc6bb9-ef82-469e-a753-fcd4c4f7138d"}
179112024-04-20T01:18:20.328Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:18.815690841Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838}
179122024-04-20T01:18:20.332ZLoading write count information from file "/tmp/test_repair_verify.data"
179132024-04-20T01:18:20.332ZRun Repair workload
179142024-04-20T01:18:20.332Z01/30 Write block 343 len 8192 data: 12 12
179152024-04-20T01:18:20.332Z02/30 Write block 93 len 16384 data: 12 13 11 11
179162024-04-20T01:18:20.332Z03/30 Write block 382 len 4096 data: 14
179172024-04-20T01:18:20.332Z04/30 Write block 428 len 16384 data: 9 13 14 13
179182024-04-20T01:18:20.332Z05/30 Write block 261 len 32768 data: 7 8 5 5 7 9 7 6
179192024-04-20T01:18:20.332Z06/30 Flush
179202024-04-20T01:18:20.333Z07/30 Write block 104 len 8192 data: 12 11
179212024-04-20T01:18:20.333Z08/30 Write block 402 len 36864 data: 10 8 8 9 12 9 12 14 15
179222024-04-20T01:18:20.333Z09/30 Write block 10 len 20480 data: 6 6 7 8 9
179232024-04-20T01:18:20.337Z10/30 Write block 43 len 32768 data: 9 10 10 11 11 11 13 15
179242024-04-20T01:18:20.337Z11/30 Read block 188 len 24576
179252024-04-20T01:18:20.337Z12/30 Write block 453 len 12288 data: 10 11 13
179262024-04-20T01:18:20.337Z13/30 Write block 478 len 24576 data: 13 10 13 17 17 16
179272024-04-20T01:18:20.337Z14/30 Read block 281 len 8192
179282024-04-20T01:18:20.337Z15/30 Read block 526 len 20480
179292024-04-20T01:18:20.337Z16/30 Read block 251 len 8192
179302024-04-20T01:18:20.341Z17/30 Read block 401 len 8192
179312024-04-20T01:18:20.341Z18/30 Read block 447 len 4096
179322024-04-20T01:18:20.341Z19/30 Flush
179332024-04-20T01:18:20.345Z20/30 Write block 201 len 16384 data: 13 15 11 11
179342024-04-20T01:18:20.345Z21/30 Read block 393 len 20480
179352024-04-20T01:18:20.345Z22/30 Read block 283 len 40960
179362024-04-20T01:18:20.345Z23/30 Write block 535 len 32768 data: 13 11 10 9 9 9 10 16
179372024-04-20T01:18:20.345Z24/30 Flush
179382024-04-20T01:18:20.349Z25/30 Write block 591 len 12288 data: 12 11 12
179392024-04-20T01:18:20.349Z26/30 Write block 589 len 32768 data: 14 10 13 12 13 14 13 12
179402024-04-20T01:18:20.357Z27/30 Write block 111 len 36864 data: 12 12 11 12 12 11 10 11 11
179412024-04-20T01:18:20.357Z28/30 Read block 41 len 4096
179422024-04-20T01:18:20.357Z29/30 Read block 453 len 4096
179432024-04-20T01:18:20.357Z30/30 Write block 486 len 8192 data: 13 11
179442024-04-20T01:18:20.357Z----------------------------------------------------------------
179452024-04-20T01:18:20.357Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30
179462024-04-20T01:18:20.357ZStates: Active Active Active
179472024-04-20T01:18:20.357ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
179482024-04-20T01:18:20.357Z 1 Acked 1000 Write 2 Sent Done Done false
179492024-04-20T01:18:20.357Z 2 Acked 1001 Write 4 Sent Done Done false
179502024-04-20T01:18:20.357Z 3 Acked 1002 Write 1 Sent Done Done false
179512024-04-20T01:18:20.357Z 4 Acked 1003 Write 4 Sent Done Done false
179522024-04-20T01:18:20.357Z 5 Acked 1004 Write 8 Sent Done Done false
179532024-04-20T01:18:20.357Z 6 Acked 1005 Flush 0 Sent Done Done false
179542024-04-20T01:18:20.357Z 7 Acked 1006 Write 2 Sent Done Done false
179552024-04-20T01:18:20.357Z 8 Acked 1007 Write 9 Sent Done Done false
179562024-04-20T01:18:20.357Z 9 Acked 1008 Write 5 Sent Done Done false
179572024-04-20T01:18:20.358Z 10 Acked 1009 Write 8 Sent Done Done false
179582024-04-20T01:18:20.358Z 11 Acked 1010 Read 6 Sent Done Done false
179592024-04-20T01:18:20.358Z 12 Acked 1011 Write 3 Sent Done Done false
179602024-04-20T01:18:20.358Z 13 Acked 1012 Write 6 Sent Done Done false
179612024-04-20T01:18:20.358Z 14 Acked 1013 Read 2 Sent Done Done false
179622024-04-20T01:18:20.358Z 15 Acked 1014 Read 5 Sent Done Done false
179632024-04-20T01:18:20.358Z 16 Acked 1015 Read 2 Sent Done Done false
179642024-04-20T01:18:20.358Z 17 Acked 1016 Read 2 Sent Done Done false
179652024-04-20T01:18:20.358Z 18 Acked 1017 Read 1 Sent Done Done false
179662024-04-20T01:18:20.358Z 19 Acked 1018 Flush 0 Sent Done Done false
179672024-04-20T01:18:20.359Z 20 Acked 1019 Write 4 Sent Done Done false
179682024-04-20T01:18:20.359Z 21 Acked 1020 Read 5 Sent Done Done false
179692024-04-20T01:18:20.359Z 22 Acked 1021 Read 10 Sent Done Done false
179702024-04-20T01:18:20.359Z 23 Acked 1022 Write 8 Sent Done Done false
179712024-04-20T01:18:20.359Z 24 Acked 1023 Flush 0 Sent Done Done false
179722024-04-20T01:18:20.359Z 25 Acked 1024 Write 3 Sent Done Done false
179732024-04-20T01:18:20.359Z 26 Acked 1025 Write 8 Sent Done Done false
179742024-04-20T01:18:20.359Z 27 Acked 1026 Write 9 Sent Done Done false
179752024-04-20T01:18:20.359Z 28 Acked 1027 Read 1 Sent Done Done false
179762024-04-20T01:18:20.359Z 29 Acked 1028 Read 1 Sent Done Done false
179772024-04-20T01:18:20.359Z 30 Acked 1029 Write 2 Sent Sent Sent false
179782024-04-20T01:18:20.359Z STATES DS:0 DS:1 DS:2 TOTAL
179792024-04-20T01:18:20.359Z New 0 0 0 0
179802024-04-20T01:18:20.360Z Sent 30 1 1 32
179812024-04-20T01:18:20.360Z Done 0 29 29 58
179822024-04-20T01:18:20.360Z Skipped 0 0 0 0
179832024-04-20T01:18:20.360Z Error 0 0 0 0
179842024-04-20T01:18:20.360ZLast Flush: 0 1023 1023
179852024-04-20T01:18:20.360ZDownstairs last five completed:
179862024-04-20T01:18:20.360ZUpstairs last five completed: 30 29 28 27 26
179872024-04-20T01:18:20.360Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:18:18.848676185Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1838}
179882024-04-20T01:18:20.360ZWrote out file "/tmp/test_repair_verify.data"
179892024-04-20T01:18:20.365Z
179902024-04-20T01:18:20.366ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
179912024-04-20T01:18:20.374ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179922024-04-20T01:18:20.374Z 0 000-019 164 166 166 270 273 273 F F F <---
179932024-04-20T01:18:20.374Z 1 020-039 160 160 160 266 266 266 F F F
179942024-04-20T01:18:20.375Z 2 040-059 162 166 166 268 273 273 F F F <---
179952024-04-20T01:18:20.375Z 3 060-079 163 163 163 269 269 269 F F F
179962024-04-20T01:18:20.380Z 4 080-099 164 166 166 270 272 272 F F F <---
179972024-04-20T01:18:20.380Z 5 100-119 151 166 166 247 273 273 F T T <---
179982024-04-20T01:18:20.380Z 6 120-139 164 164 164 270 270 270 F F F
179992024-04-20T01:18:20.380Z 7 140-159 163 163 163 269 269 269 F F F
180002024-04-20T01:18:20.380Z 8 160-179 163 163 163 269 269 269 F F F
180012024-04-20T01:18:20.380Z 9 180-199 164 164 164 270 270 270 F F F
180022024-04-20T01:18:20.380Z 10 200-219 164 166 166 270 274 274 F F F <---
180032024-04-20T01:18:20.380Z 11 220-239 164 164 164 270 270 270 F F F
180042024-04-20T01:18:20.380Z 12 240-259 165 165 165 271 271 271 F F F
180052024-04-20T01:18:20.380Z 13 260-279 165 166 166 271 272 272 F F F <---
180062024-04-20T01:18:20.380Z 14 280-299 163 163 163 269 269 269 F F F
180072024-04-20T01:18:20.380Z 15 300-319 158 158 158 261 261 261 F F F
180082024-04-20T01:18:20.380Z 16 320-339 164 164 164 270 270 270 F F F
180092024-04-20T01:18:20.380Z 17 340-359 164 166 166 270 272 272 F F F <---
180102024-04-20T01:18:20.380Z 18 360-379 164 164 164 270 270 270 F F F
180112024-04-20T01:18:20.380Z 19 380-399 163 166 166 269 272 272 F F F <---
180122024-04-20T01:18:20.380Z 20 400-419 156 166 166 256 273 273 F F F <---
180132024-04-20T01:18:20.381Z 21 420-439 163 166 166 269 272 272 F F F <---
180142024-04-20T01:18:20.381Z 22 440-459 159 166 166 262 273 273 F F F <---
180152024-04-20T01:18:20.381Z 23 460-479 164 166 166 270 273 273 F F F <---
180162024-04-20T01:18:20.381Z 24 480-499 163 166 166 269 273 273 F T T <---
180172024-04-20T01:18:20.381Z 25 500-519 163 163 163 269 269 269 F F F
180182024-04-20T01:18:20.381Z 26 520-539 164 166 166 270 274 274 F F F <---
180192024-04-20T01:18:20.381Z 27 540-559 165 166 166 271 274 274 F F F <---
180202024-04-20T01:18:20.381Z 28 560-579 165 165 165 271 271 271 F F F
180212024-04-20T01:18:20.381Z 29 580-599 164 164 164 270 270 270 F T T <---
180222024-04-20T01:18:20.381ZMax gen: 166, Max flush: 274
180232024-04-20T01:18:20.381ZError: Difference in extent metadata found!
180242024-04-20T01:18:20.381ZA Difference in extent metadata is expected here
180252024-04-20T01:18:20.381ZOn loop 83
180262024-04-20T01:18:20.381Z
180272024-04-20T01:18:20.386ZVerifying data now
180282024-04-20T01:18:25.090ZNo change after verify
180292024-04-20T01:18:25.090ZLoop: 83 Downstairs dump after verify (and repair):
180302024-04-20T01:18:25.105ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180312024-04-20T01:18:25.106Z 0 000-019 166 166 166 273 273 273 F F F
180322024-04-20T01:18:25.106Z 1 020-039 160 160 160 266 266 266 F F F
180332024-04-20T01:18:25.106Z 2 040-059 166 166 166 273 273 273 F F F
180342024-04-20T01:18:25.106Z 3 060-079 163 163 163 269 269 269 F F F
180352024-04-20T01:18:25.106Z 4 080-099 166 166 166 272 272 272 F F F
180362024-04-20T01:18:25.106Z 5 100-119 167 167 167 275 275 275 F F F
180372024-04-20T01:18:25.109Z 6 120-139 164 164 164 270 270 270 F F F
180382024-04-20T01:18:25.109Z 7 140-159 163 163 163 269 269 269 F F F
180392024-04-20T01:18:25.110Z 8 160-179 163 163 163 269 269 269 F F F
180402024-04-20T01:18:25.110Z 9 180-199 164 164 164 270 270 270 F F F
180412024-04-20T01:18:25.110Z 10 200-219 166 166 166 274 274 274 F F F
180422024-04-20T01:18:25.110Z 11 220-239 164 164 164 270 270 270 F F F
180432024-04-20T01:18:25.110Z 12 240-259 165 165 165 271 271 271 F F F
180442024-04-20T01:18:25.110Z 13 260-279 166 166 166 272 272 272 F F F
180452024-04-20T01:18:25.110Z 14 280-299 163 163 163 269 269 269 F F F
180462024-04-20T01:18:25.110Z 15 300-319 158 158 158 261 261 261 F F F
180472024-04-20T01:18:25.110Z 16 320-339 164 164 164 270 270 270 F F F
180482024-04-20T01:18:25.110Z 17 340-359 166 166 166 272 272 272 F F F
180492024-04-20T01:18:25.110Z 18 360-379 164 164 164 270 270 270 F F F
180502024-04-20T01:18:25.110Z 19 380-399 166 166 166 272 272 272 F F F
180512024-04-20T01:18:25.110Z 20 400-419 166 166 166 273 273 273 F F F
180522024-04-20T01:18:25.110Z 21 420-439 166 166 166 272 272 272 F F F
180532024-04-20T01:18:25.110Z 22 440-459 166 166 166 273 273 273 F F F
180542024-04-20T01:18:25.110Z 23 460-479 166 166 166 273 273 273 F F F
180552024-04-20T01:18:25.110Z 24 480-499 167 167 167 275 275 275 F F F
180562024-04-20T01:18:25.110Z 25 500-519 163 163 163 269 269 269 F F F
180572024-04-20T01:18:25.110Z 26 520-539 166 166 166 274 274 274 F F F
180582024-04-20T01:18:25.110Z 27 540-559 166 166 166 274 274 274 F F F
180592024-04-20T01:18:25.110Z 28 560-579 165 165 165 271 271 271 F F F
180602024-04-20T01:18:25.110Z 29 580-599 167 167 167 275 275 275 F F F
180612024-04-20T01:18:25.110ZMax gen: 167, Max flush: 275
180622024-04-20T01:18:25.111Z
180632024-04-20T01:18:25.111ZBegin loop 84
180642024-04-20T01:18:25.111ZDownstairs to restart: 2
180652024-04-20T01:18:25.128Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.623924164Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846}
180662024-04-20T01:18:25.128Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624131945Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846}
180672024-04-20T01:18:25.128Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624166535Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846}
180682024-04-20T01:18:25.136Z{"msg":"Crucible c3073587-6a49-4589-812e-37fdb68c6201 has session id: 8f96dd6d-1e4e-4222-928c-2ff05fe01f86","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624191245Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180692024-04-20T01:18:25.136Z{"msg":"Upstairs opts: Upstairs UUID: c3073587-6a49-4589-812e-37fdb68c6201, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
180702024-04-20T01:18:25.136Z,"time":"2024-04-20T01:18:23.624236975Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180712024-04-20T01:18:25.136Z{"msg":"Crucible stats registered with UUID: c3073587-6a49-4589-812e-37fdb68c6201","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624281495Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180722024-04-20T01:18:25.137Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624310955Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846}
180732024-04-20T01:18:25.137Z{"msg":"c3073587-6a49-4589-812e-37fdb68c6201 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624389585Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180742024-04-20T01:18:25.137Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624414936Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"0","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180752024-04-20T01:18:25.137Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624430825Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"0","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180762024-04-20T01:18:25.137Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624452066Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"1","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180772024-04-20T01:18:25.137Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624466186Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"1","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180782024-04-20T01:18:25.137Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624502836Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"2","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180792024-04-20T01:18:25.137Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624523206Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"2","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180802024-04-20T01:18:25.137Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624546416Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180812024-04-20T01:18:25.137Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624569476Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180822024-04-20T01:18:25.137Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624594636Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180832024-04-20T01:18:25.137Z{"msg":"ds_connection connected from Ok(127.0.0.1:51777)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624805666Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180842024-04-20T01:18:25.137Z{"msg":"ds_connection connected from Ok(127.0.0.1:54360)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624861706Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180852024-04-20T01:18:25.137Z{"msg":"ds_connection connected from Ok(127.0.0.1:53942)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.624908247Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180862024-04-20T01:18:25.137Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.625643258Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"0","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180872024-04-20T01:18:25.137Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.625685708Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"0","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180882024-04-20T01:18:25.137Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.625713588Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"1","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180892024-04-20T01:18:25.137Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.625787759Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"1","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180902024-04-20T01:18:25.137Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.626099069Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"2","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180912024-04-20T01:18:25.137Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.626143269Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"2","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180922024-04-20T01:18:25.137Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.62629902Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"1","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180932024-04-20T01:18:25.137Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.62641242Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"0","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180942024-04-20T01:18:25.138Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.626677161Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"2","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180952024-04-20T01:18:25.138Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.626848431Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"0","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180962024-04-20T01:18:25.138Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.626887741Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"0","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180972024-04-20T01:18:25.138Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.626932281Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180982024-04-20T01:18:25.138Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.626966071Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"1","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
180992024-04-20T01:18:25.138Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.626982411Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"1","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181002024-04-20T01:18:25.138Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627003491Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181012024-04-20T01:18:25.138Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627172842Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"2","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181022024-04-20T01:18:25.138Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627212092Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"2","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181032024-04-20T01:18:25.138Z{"msg":"[0]R flush_numbers[0..12]: [273, 266, 273, 269, 272, 275, 270, 269, 269, 270, 274, 270]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627229022Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181042024-04-20T01:18:25.138Z{"msg":"[0]R generation[0..12]: [166, 160, 166, 163, 166, 167, 164, 163, 163, 164, 166, 164]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627243352Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181052024-04-20T01:18:25.138Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627257742Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181062024-04-20T01:18:25.138Z{"msg":"[1]R flush_numbers[0..12]: [273, 266, 273, 269, 272, 275, 270, 269, 269, 270, 274, 270]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627278722Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181072024-04-20T01:18:25.138Z{"msg":"[1]R generation[0..12]: [166, 160, 166, 163, 166, 167, 164, 163, 163, 164, 166, 164]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627292902Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181082024-04-20T01:18:25.138Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627307102Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181092024-04-20T01:18:25.138Z{"msg":"[2]R flush_numbers[0..12]: [273, 266, 273, 269, 272, 275, 270, 269, 269, 270, 274, 270]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627328282Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181102024-04-20T01:18:25.138Z{"msg":"[2]R generation[0..12]: [166, 160, 166, 163, 166, 167, 164, 163, 163, 164, 166, 164]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627342702Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181112024-04-20T01:18:25.138Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627356452Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181122024-04-20T01:18:25.138Z{"msg":"Wait for a query_work_queue command to finish before sending IO
181132024-04-20T01:18:25.138ZMax found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627378362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181142024-04-20T01:18:25.138Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627391992Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181152024-04-20T01:18:25.138Z{"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627405052Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181162024-04-20T01:18:25.138Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627424812Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"":"downstairs","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181172024-04-20T01:18:25.139Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627438192Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181182024-04-20T01:18:25.139Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627450052Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"0","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181192024-04-20T01:18:25.139Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627470002Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"1","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181202024-04-20T01:18:25.139Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627483242Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"client":"2","":"downstairs","session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181212024-04-20T01:18:25.139Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627496392Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181222024-04-20T01:18:25.139Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627515692Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181232024-04-20T01:18:25.139Z{"msg":"c3073587-6a49-4589-812e-37fdb68c6201 is now active with session: 8f96dd6d-1e4e-4222-928c-2ff05fe01f86","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627528612Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181242024-04-20T01:18:25.139Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627547403Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846,"session_id":"8f96dd6d-1e4e-4222-928c-2ff05fe01f86"}
181252024-04-20T01:18:25.139Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:23.627559792Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846}
181262024-04-20T01:18:25.139ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
181272024-04-20T01:18:25.139ZLoading write count information from file "/tmp/test_repair_verify.data"
181282024-04-20T01:18:25.139ZRun Repair workload
181292024-04-20T01:18:25.143Z01/30 Write block 323 len 32768 data: 18 15 16 16 15 15 12 15
181302024-04-20T01:18:25.143Z02/30 Flush
181312024-04-20T01:18:25.143Z03/30 Write block 90 len 20480 data: 17 15 13 13 14
181322024-04-20T01:18:25.143Z04/30 Read block 565 len 40960
181332024-04-20T01:18:25.143Z05/30 Read block 97 len 40960
181342024-04-20T01:18:25.143Z06/30 Read block 422 len 16384
181352024-04-20T01:18:25.143Z07/30 Read block 122 len 24576
181362024-04-20T01:18:25.143Z08/30 Read block 211 len 20480
181372024-04-20T01:18:25.143Z09/30 Read block 497 len 12288
181382024-04-20T01:18:25.147Z10/30 Write block 273 len 32768 data: 15 13 13 11 13 13 15 16
181392024-04-20T01:18:25.147Z11/30 Read block 426 len 28672
181402024-04-20T01:18:25.147Z12/30 Read block 572 len 36864
181412024-04-20T01:18:25.147Z13/30 Read block 534 len 8192
181422024-04-20T01:18:25.147Z14/30 Flush
181432024-04-20T01:18:25.150Z15/30 Write block 4 len 20480 data: 10 10 10 9 8
181442024-04-20T01:18:25.155Z16/30 Flush
181452024-04-20T01:18:25.155Z17/30 Write block 158 len 8192 data: 11 12
181462024-04-20T01:18:25.155Z18/30 Write block 247 len 12288 data: 12 16 16
181472024-04-20T01:18:25.159Z19/30 Write block 305 len 12288 data: 12 11 10
181482024-04-20T01:18:25.159Z20/30 Write block 578 len 40960 data: 12 16 17 16 15 13 17 20 22 20
181492024-04-20T01:18:25.159Z21/30 Write block 14 len 20480 data: 10 12 13 11 10
181502024-04-20T01:18:25.159Z22/30 Write block 229 len 36864 data: 10 11 13 13 17 18 15 15 17
181512024-04-20T01:18:25.162Z23/30 Read block 37 len 32768
181522024-04-20T01:18:25.163Z24/30 Read block 142 len 12288
181532024-04-20T01:18:25.163Z25/30 Read block 508 len 16384
181542024-04-20T01:18:25.163Z26/30 Read block 307 len 40960
181552024-04-20T01:18:25.166Z27/30 Write block 229 len 32768 data: 11 12 14 14 18 19 16 16
181562024-04-20T01:18:25.166Z28/30 Write block 231 len 24576 data: 15 15 19 20 17 17
181572024-04-20T01:18:25.166Z29/30 Read block 60 len 40960
181582024-04-20T01:18:25.171Z30/30 Write block 521 len 32768 data: 19 18 17 15 14 13 11 8
181592024-04-20T01:18:25.171Z----------------------------------------------------------------
181602024-04-20T01:18:25.171Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:28
181612024-04-20T01:18:25.171ZStates: Active Active Active
181622024-04-20T01:18:25.171ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
181632024-04-20T01:18:25.171Z 3 Acked 1002 Write 5 Done Done Done false
181642024-04-20T01:18:25.171Z 4 Acked 1003 Read 10 Done Done Done false
181652024-04-20T01:18:25.171Z 5 Acked 1004 Read 10 Done Done Done false
181662024-04-20T01:18:25.171Z 6 Acked 1005 Read 4 Done Done Done false
181672024-04-20T01:18:25.171Z 7 Acked 1006 Read 6 Done Done Done false
181682024-04-20T01:18:25.171Z 8 Acked 1007 Read 5 Done Done Done false
181692024-04-20T01:18:25.171Z 9 Acked 1008 Read 3 Done Done Done false
181702024-04-20T01:18:25.171Z 10 Acked 1009 Write 8 Done Done Done false
181712024-04-20T01:18:25.172Z 11 Acked 1010 Read 7 Done Done Done false
181722024-04-20T01:18:25.172Z 12 Acked 1011 Read 9 Done Done Sent false
181732024-04-20T01:18:25.172Z 13 Acked 1012 Read 2 Done Done Sent false
181742024-04-20T01:18:25.172Z 14 Acked 1013 Flush 0 Done Done Sent false
181752024-04-20T01:18:25.172Z 15 Acked 1014 Write 5 Done Done Sent false
181762024-04-20T01:18:25.172Z 16 Acked 1015 Flush 0 Done Done Sent false
181772024-04-20T01:18:25.172Z 17 Acked 1016 Write 2 Done Done Sent false
181782024-04-20T01:18:25.172Z 18 Acked 1017 Write 3 Done Done Sent false
181792024-04-20T01:18:25.172Z 19 Acked 1018 Write 3 Done Done Sent false
181802024-04-20T01:18:25.172Z 20 Acked 1019 Write 10 Done Done Sent false
181812024-04-20T01:18:25.172Z 21 Acked 1020 Write 5 Done Done Sent false
181822024-04-20T01:18:25.172Z 22 Acked 1021 Write 9 Done Done Sent false
181832024-04-20T01:18:25.172Z 23 Acked 1022 Read 8 Done Done Sent false
181842024-04-20T01:18:25.172Z 24 Acked 1023 Read 3 Done Done Sent false
181852024-04-20T01:18:25.172Z 25 Acked 1024 Read 4 Done Done Sent false
181862024-04-20T01:18:25.172Z 26 Acked 1025 Read 10 Done Done Sent false
181872024-04-20T01:18:25.172Z 27 Acked 1026 Write 8 Done Done Sent false
181882024-04-20T01:18:25.172Z 28 Acked 1027 Write 6 Done Done Sent false
181892024-04-20T01:18:25.172Z 29 Acked 1028 Read 10 Done Done Sent false
181902024-04-20T01:18:25.172Z 30 Acked 1029 Write 8 Sent Sent Sent false
181912024-04-20T01:18:25.172Z STATES DS:0 DS:1 DS:2 TOTAL
181922024-04-20T01:18:25.172Z New 0 0 0 0
181932024-04-20T01:18:25.173Z Sent 1 1 19 21
181942024-04-20T01:18:25.173Z Done 27 27 9 63
181952024-04-20T01:18:25.173Z Skipped 0 0 0 0
181962024-04-20T01:18:25.173Z Error 0 0 0 0
181972024-04-20T01:18:25.173ZLast Flush: 1015 1015 1001
181982024-04-20T01:18:25.173ZDownstairs last five completed: 1001 1000
181992024-04-20T01:18:25.173ZUpstairs last five completed: 30 29 28 27 26
182002024-04-20T01:18:25.173Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:18:23.664235914Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1846}
182012024-04-20T01:18:25.173ZWrote out file "/tmp/test_repair_verify.data"
182022024-04-20T01:18:25.177Z
182032024-04-20T01:18:25.177ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
182042024-04-20T01:18:25.194ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182052024-04-20T01:18:25.194Z 0 000-019 168 168 166 278 278 273 T T F <---
182062024-04-20T01:18:25.194Z 1 020-039 160 160 160 266 266 266 F F F
182072024-04-20T01:18:25.194Z 2 040-059 166 166 166 273 273 273 F F F
182082024-04-20T01:18:25.194Z 3 060-079 163 163 163 269 269 269 F F F
182092024-04-20T01:18:25.200Z 4 080-099 168 168 166 277 277 272 F F T <---
182102024-04-20T01:18:25.200Z 5 100-119 167 167 167 275 275 275 F F F
182112024-04-20T01:18:25.200Z 6 120-139 164 164 164 270 270 270 F F F
182122024-04-20T01:18:25.200Z 7 140-159 163 163 163 269 269 269 T T F <---
182132024-04-20T01:18:25.200Z 8 160-179 163 163 163 269 269 269 F F F
182142024-04-20T01:18:25.200Z 9 180-199 164 164 164 270 270 270 F F F
182152024-04-20T01:18:25.200Z 10 200-219 166 166 166 274 274 274 F F F
182162024-04-20T01:18:25.200Z 11 220-239 164 164 164 270 270 270 T T F <---
182172024-04-20T01:18:25.200Z 12 240-259 165 165 165 271 271 271 T T F <---
182182024-04-20T01:18:25.200Z 13 260-279 168 168 166 277 277 272 F F T <---
182192024-04-20T01:18:25.200Z 14 280-299 168 168 163 277 277 269 F F T <---
182202024-04-20T01:18:25.200Z 15 300-319 158 158 158 261 261 261 T T F <---
182212024-04-20T01:18:25.200Z 16 320-339 168 168 168 276 276 276 F F F
182222024-04-20T01:18:25.200Z 17 340-359 166 166 166 272 272 272 F F F
182232024-04-20T01:18:25.200Z 18 360-379 164 164 164 270 270 270 F F F
182242024-04-20T01:18:25.200Z 19 380-399 166 166 166 272 272 272 F F F
182252024-04-20T01:18:25.200Z 20 400-419 166 166 166 273 273 273 F F F
182262024-04-20T01:18:25.200Z 21 420-439 166 166 166 272 272 272 F F F
182272024-04-20T01:18:25.200Z 22 440-459 166 166 166 273 273 273 F F F
182282024-04-20T01:18:25.200Z 23 460-479 166 166 166 273 273 273 F F F
182292024-04-20T01:18:25.201Z 24 480-499 167 167 167 275 275 275 F F F
182302024-04-20T01:18:25.201Z 25 500-519 163 163 163 269 269 269 F F F
182312024-04-20T01:18:25.201Z 26 520-539 166 166 166 274 274 274 T T F <---
182322024-04-20T01:18:25.201Z 27 540-559 166 166 166 274 274 274 F F F
182332024-04-20T01:18:25.201Z 28 560-579 165 165 165 271 271 271 T T F <---
182342024-04-20T01:18:25.201Z 29 580-599 167 167 167 275 275 275 T T F <---
182352024-04-20T01:18:25.201ZMax gen: 168, Max flush: 278
182362024-04-20T01:18:25.201ZError: Difference in extent metadata found!
182372024-04-20T01:18:25.201ZA Difference in extent metadata is expected here
182382024-04-20T01:18:25.201ZOn loop 84
182392024-04-20T01:18:25.201Z
182402024-04-20T01:18:25.206ZVerifying data now
182412024-04-20T01:18:39.736ZNo change after verify
182422024-04-20T01:18:39.746ZLoop: 84 Downstairs dump after verify (and repair):
182432024-04-20T01:18:39.756ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182442024-04-20T01:18:39.756Z 0 000-019 169 169 169 279 279 279 F F F
182452024-04-20T01:18:39.756Z 1 020-039 160 160 160 266 266 266 F F F
182462024-04-20T01:18:39.756Z 2 040-059 166 166 166 273 273 273 F F F
182472024-04-20T01:18:39.756Z 3 060-079 163 163 163 269 269 269 F F F
182482024-04-20T01:18:39.756Z 4 080-099 168 168 168 277 277 277 F F F
182492024-04-20T01:18:39.757Z 5 100-119 167 167 167 275 275 275 F F F
182502024-04-20T01:18:39.757Z 6 120-139 164 164 164 270 270 270 F F F
182512024-04-20T01:18:39.757Z 7 140-159 169 169 169 279 279 279 F F F
182522024-04-20T01:18:39.757Z 8 160-179 163 163 163 269 269 269 F F F
182532024-04-20T01:18:39.757Z 9 180-199 164 164 164 270 270 270 F F F
182542024-04-20T01:18:39.757Z 10 200-219 166 166 166 274 274 274 F F F
182552024-04-20T01:18:39.757Z 11 220-239 169 169 169 279 279 279 F F F
182562024-04-20T01:18:39.758Z 12 240-259 169 169 169 279 279 279 F F F
182572024-04-20T01:18:39.758Z 13 260-279 168 168 168 277 277 277 F F F
182582024-04-20T01:18:39.758Z 14 280-299 168 168 168 277 277 277 F F F
182592024-04-20T01:18:39.758Z 15 300-319 169 169 169 279 279 279 F F F
182602024-04-20T01:18:39.758Z 16 320-339 168 168 168 276 276 276 F F F
182612024-04-20T01:18:39.758Z 17 340-359 166 166 166 272 272 272 F F F
182622024-04-20T01:18:39.758Z 18 360-379 164 164 164 270 270 270 F F F
182632024-04-20T01:18:39.759Z 19 380-399 166 166 166 272 272 272 F F F
182642024-04-20T01:18:39.759Z 20 400-419 166 166 166 273 273 273 F F F
182652024-04-20T01:18:39.759Z 21 420-439 166 166 166 272 272 272 F F F
182662024-04-20T01:18:39.759Z 22 440-459 166 166 166 273 273 273 F F F
182672024-04-20T01:18:39.759Z 23 460-479 166 166 166 273 273 273 F F F
182682024-04-20T01:18:39.760Z 24 480-499 167 167 167 275 275 275 F F F
182692024-04-20T01:18:39.760Z 25 500-519 163 163 163 269 269 269 F F F
182702024-04-20T01:18:39.760Z 26 520-539 169 169 169 279 279 279 F F F
182712024-04-20T01:18:39.760Z 27 540-559 166 166 166 274 274 274 F F F
182722024-04-20T01:18:39.760Z 28 560-579 169 169 169 279 279 279 F F F
182732024-04-20T01:18:39.761Z 29 580-599 169 169 169 279 279 279 F F F
182742024-04-20T01:18:39.761ZMax gen: 169, Max flush: 279
182752024-04-20T01:18:39.761Z
182762024-04-20T01:18:39.761ZBegin loop 85
182772024-04-20T01:18:39.761ZDownstairs to restart: 2
182782024-04-20T01:18:39.768Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.261685248Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854}
182792024-04-20T01:18:39.768Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.261988489Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854}
182802024-04-20T01:18:39.769Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.262029869Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854}
182812024-04-20T01:18:39.769ZCrucible runtime is spawned
182822024-04-20T01:18:39.769Z{"msg":"Crucible 1eddfa8a-9c77-46a0-9a58-5ae3576f0fc4 has session id: 487cc62c-7afe-44dd-8d68-0a88cfc371e7","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.262075489Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182832024-04-20T01:18:39.769Z{"msg":"Upstairs opts: Upstairs UUID: 1eddfa8a-9c77-46a0-9a58-5ae3576f0fc4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.262126899Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182842024-04-20T01:18:39.769Z{"msg":"Crucible stats registered with UUID: 1eddfa8a-9c77-46a0-9a58-5ae3576f0fc4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.262165209Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182852024-04-20T01:18:39.769Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.262201179Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854}
182862024-04-20T01:18:39.770Z{"msg":"1eddfa8a-9c77-46a0-9a58-5ae3576f0fc4 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.262232719Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182872024-04-20T01:18:39.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.262256179Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"0","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182882024-04-20T01:18:39.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.262305089Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"0","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182892024-04-20T01:18:39.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.26234806Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"1","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182902024-04-20T01:18:39.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.26237553Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"1","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182912024-04-20T01:18:39.771Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.2623983Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"2","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182922024-04-20T01:18:39.771Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.26243669Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"2","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182932024-04-20T01:18:39.771Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.26247804Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182942024-04-20T01:18:39.771Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.26251699Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182952024-04-20T01:18:39.771Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.26254509Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182962024-04-20T01:18:39.771Z{"msg":"ds_connection connected from Ok(127.0.0.1:52429)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.26257124Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182972024-04-20T01:18:39.771Z{"msg":"ds_connection connected from Ok(127.0.0.1:57617)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.26277521Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182982024-04-20T01:18:39.772Z{"msg":"ds_connection connected from Ok(127.0.0.1:37954)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.262824651Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
182992024-04-20T01:18:39.788Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.263216882Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"1","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183002024-04-20T01:18:39.788Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.263280552Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"1","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183012024-04-20T01:18:39.789Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.263331622Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"0","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183022024-04-20T01:18:39.789Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.263361572Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"0","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183032024-04-20T01:18:39.789Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.263778893Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"0","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183042024-04-20T01:18:39.789Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.263916223Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"1","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183052024-04-20T01:18:39.789Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.264027073Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"2","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183062024-04-20T01:18:39.789Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.264059613Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"2","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183072024-04-20T01:18:39.789Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.264255464Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"0","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183082024-04-20T01:18:39.790Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.264301834Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"0","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183092024-04-20T01:18:39.790Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.264324314Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183102024-04-20T01:18:39.790Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.264492404Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"1","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183112024-04-20T01:18:39.790Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.264528944Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"1","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183122024-04-20T01:18:39.791Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.264545594Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183132024-04-20T01:18:39.791Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.264728215Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"2","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183142024-04-20T01:18:39.791Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265120246Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"2","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183152024-04-20T01:18:39.791Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265151376Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"2","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183162024-04-20T01:18:39.791Z{"msg":"[0]R flush_numbers[0..12]: [279, 266, 273, 269, 277, 275, 270, 279, 269, 270, 274, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265177696Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183172024-04-20T01:18:39.791Z{"msg":"[0]R generation[0..12]: [169, 160, 166, 163, 168, 167, 164, 169, 163, 164, 166, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265207686Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183182024-04-20T01:18:39.791Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265234166Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183192024-04-20T01:18:39.791Z{"msg":"[1]R flush_numbers[0..12]: [279, 266, 273, 269, 277, 275, 270, 279, 269, 270, 274, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265262626Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22"Wait for a query_work_queue command to finish before sending IO
183202024-04-20T01:18:39.791Z,"pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183212024-04-20T01:18:39.791Z{"msg":"[1]R generation[0..12]: [169, 160, 166, 163, 168, 167, 164, 169, 163, 164, 166, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265290576Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183222024-04-20T01:18:39.792Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265321996Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183232024-04-20T01:18:39.792Z{"msg":"[2]R flush_numbers[0..12]: [279, 266, 273, 269, 277, 275, 270, 279, 269, 270, 274, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265347256Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183242024-04-20T01:18:39.792Z{"msg":"[2]R generation[0..12]: [169, 160, 166, 163, 168, 167, 164, 169, 163, 164, 166, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265377346Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183252024-04-20T01:18:39.792Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265402516Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183262024-04-20T01:18:39.792Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265425646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183272024-04-20T01:18:39.792Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265457316Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183282024-04-20T01:18:39.792Z{"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265481647Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183292024-04-20T01:18:39.792Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265503636Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"":"downstairs","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183302024-04-20T01:18:39.792Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265535217Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183312024-04-20T01:18:39.792Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265557097Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"0","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183322024-04-20T01:18:39.792Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265587657Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"1","":"downstairs"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
183332024-04-20T01:18:39.793Z,"session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183342024-04-20T01:18:39.793Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265627197Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"client":"2","":"downstairs","session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183352024-04-20T01:18:39.793Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265647577Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183362024-04-20T01:18:39.793Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265665657Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183372024-04-20T01:18:39.793Z{"msg":"1eddfa8a-9c77-46a0-9a58-5ae3576f0fc4 is now active with session: 487cc62c-7afe-44dd-8d68-0a88cfc371e7","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265703557Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183382024-04-20T01:18:39.793Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265724967Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854,"session_id":"487cc62c-7afe-44dd-8d68-0a88cfc371e7"}
183392024-04-20T01:18:39.793Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:38.265750657Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854}
183402024-04-20T01:18:39.793ZLoading write count information from file "/tmp/test_repair_verify.data"
183412024-04-20T01:18:39.793ZRun Repair workload
183422024-04-20T01:18:39.793Z01/30 Write block 17 len 12288 data: 12 11 10
183432024-04-20T01:18:39.793Z02/30 Read block 16 len 32768
183442024-04-20T01:18:39.793Z03/30 Read block 439 len 28672
183452024-04-20T01:18:39.793Z04/30 Flush
183462024-04-20T01:18:39.794Z05/30 Write block 517 len 28672 data: 18 15 17 18 20 19 18
183472024-04-20T01:18:39.794Z06/30 Read block 532 len 16384
183482024-04-20T01:18:39.794Z07/30 Write block 186 len 24576 data: 10 11 14 13 16 19
183492024-04-20T01:18:39.801Z08/30 Write block 271 len 40960 data: 10 11 16 14 14 12 14 14 16 17
183502024-04-20T01:18:39.801Z09/30 Read block 324 len 4096
183512024-04-20T01:18:39.801Z10/30 Write block 119 len 36864 data: 12 11 11 11 11 13 12 11 12
183522024-04-20T01:18:39.801Z11/30 Write block 259 len 20480 data: 13 11 8 9 6
183532024-04-20T01:18:39.802Z12/30 Read block 417 len 40960
183542024-04-20T01:18:39.802Z13/30 Read block 302 len 36864
183552024-04-20T01:18:39.802Z14/30 Write block 40 len 28672 data: 12 11 12 10 11 11 12
183562024-04-20T01:18:39.802Z15/30 Read block 55 len 12288
183572024-04-20T01:18:39.802Z16/30 Flush
183582024-04-20T01:18:39.802Z17/30 Write block 266 len 32768 data: 10 8 7 8 9 11 12 17
183592024-04-20T01:18:39.802Z18/30 Read block 49 len 24576
183602024-04-20T01:18:39.803Z19/30 Flush
183612024-04-20T01:18:39.803Z20/30 Write block 590 len 28672 data: 11 14 13 14 15 14 13
183622024-04-20T01:18:39.803Z21/30 Read block 374 len 24576
183632024-04-20T01:18:39.803Z22/30 Write block 11 len 12288 data: 7 8 9
183642024-04-20T01:18:39.804Z23/30 Read block 543 len 8192
183652024-04-20T01:18:39.804Z24/30 Write block 570 len 12288 data: 17 16 12
183662024-04-20T01:18:39.810Z25/30 Write block 169 len 40960 data: 6 9 10 12 15 15 13 15 14 16
183672024-04-20T01:18:39.810Z26/30 Read block 589 len 16384
183682024-04-20T01:18:39.811Z27/30 Write block 293 len 20480 data: 10 13 11 11 11
183692024-04-20T01:18:39.811Z28/30 Write block 197 len 24576 data: 14 15 17 12 14 16
183702024-04-20T01:18:39.811Z29/30 Read block 91 len 20480
183712024-04-20T01:18:39.811Z30/30 Write block 371 len 28672 data: 19 18 21 20 17 14 15
183722024-04-20T01:18:39.811Z----------------------------------------------------------------
183732024-04-20T01:18:39.811Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
183742024-04-20T01:18:39.811ZStates: Active Active Active
183752024-04-20T01:18:39.811ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
183762024-04-20T01:18:39.811Z 1 Acked 1000 Write 3 Done Done Sent false
183772024-04-20T01:18:39.811Z 2 Acked 1001 Read 8 Done Done Sent false
183782024-04-20T01:18:39.811Z 3 Acked 1002 Read 7 Done Done Sent false
183792024-04-20T01:18:39.811Z 4 Acked 1003 Flush 0 Done Done Sent false
183802024-04-20T01:18:39.811Z 5 Acked 1004 Write 7 Done Done Sent false
183812024-04-20T01:18:39.812Z 6 Acked 1005 Read 4 Done Done Sent false
183822024-04-20T01:18:39.812Z 7 Acked 1006 Write 6 Done Done Sent false
183832024-04-20T01:18:39.812Z 8 Acked 1007 Write 10 Done Done Sent false
183842024-04-20T01:18:39.812Z 9 Acked 1008 Read 1 Done Done Sent false
183852024-04-20T01:18:39.812Z 10 Acked 1009 Write 9 Done Done Sent false
183862024-04-20T01:18:39.812Z 11 Acked 1010 Write 5 Done Done Sent false
183872024-04-20T01:18:39.812Z 12 Acked 1011 Read 10 Done Done Sent false
183882024-04-20T01:18:39.812Z 13 Acked 1012 Read 9 Done Done Sent false
183892024-04-20T01:18:39.812Z 14 Acked 1013 Write 7 Done Done Sent false
183902024-04-20T01:18:39.812Z 15 Acked 1014 Read 3 Done Done Sent false
183912024-04-20T01:18:39.812Z 16 Acked 1015 Flush 0 Done Done Sent false
183922024-04-20T01:18:39.812Z 17 Acked 1016 Write 8 Done Done Sent false
183932024-04-20T01:18:39.813Z 18 Acked 1017 Read 6 Done Done Sent false
183942024-04-20T01:18:39.813Z 19 Acked 1018 Flush 0 Done Done Sent false
183952024-04-20T01:18:39.813Z 20 Acked 1019 Write 7 Done Done Sent false
183962024-04-20T01:18:39.813Z 21 Acked 1020 Read 6 Done Done Sent false
183972024-04-20T01:18:39.813Z 22 Acked 1021 Write 3 Done Done Sent false
183982024-04-20T01:18:39.813Z 23 Acked 1022 Read 2 Done Done Sent false
183992024-04-20T01:18:39.813Z 24 Acked 1023 Write 3 Done Done Sent false
184002024-04-20T01:18:39.813Z 25 Acked 1024 Write 10 Done Done Sent false
184012024-04-20T01:18:39.813Z 26 Acked 1025 Read 4 Done Done Sent false
184022024-04-20T01:18:39.813Z 27 Acked 1026 Write 5 Done Done Sent false
184032024-04-20T01:18:39.813Z 28 Acked 1027 Write 6 Done Done Sent false
184042024-04-20T01:18:39.813Z 29 Acked 1028 Read 5 Done Done Sent false
184052024-04-20T01:18:39.814Z 30 Acked 1029 Write 7 Sent Sent Sent false
184062024-04-20T01:18:39.814Z STATES DS:0 DS:1 DS:2 TOTAL
184072024-04-20T01:18:39.814Z New 0 0 0 0
184082024-04-20T01:18:39.814Z Sent 1 1 30 32
184092024-04-20T01:18:39.814Z Done 29 29 0 58
184102024-04-20T01:18:39.814Z Skipped 0 0 0 0
184112024-04-20T01:18:39.814Z Error 0 0 0 0
184122024-04-20T01:18:39.814ZLast Flush: 1018 1018 0
184132024-04-20T01:18:39.814ZDownstairs last five completed:
184142024-04-20T01:18:39.814ZUpstairs last five completed: 30 29 28 27 26
184152024-04-20T01:18:39.814Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:18:38.301344726Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1854}
184162024-04-20T01:18:39.814ZWrote out file "/tmp/test_repair_verify.data"
184172024-04-20T01:18:39.826Z
184182024-04-20T01:18:39.827ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
184192024-04-20T01:18:39.842ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184202024-04-20T01:18:39.842Z 0 000-019 170 170 169 280 280 279 T T F <---
184212024-04-20T01:18:39.842Z 1 020-039 160 160 160 266 266 266 F F F
184222024-04-20T01:18:39.842Z 2 040-059 170 170 166 281 281 273 F F F <---
184232024-04-20T01:18:39.842Z 3 060-079 163 163 163 269 269 269 F F F
184242024-04-20T01:18:39.842Z 4 080-099 168 168 168 277 277 277 F F F
184252024-04-20T01:18:39.843Z 5 100-119 170 170 167 281 281 275 F F F <---
184262024-04-20T01:18:39.843Z 6 120-139 170 170 164 281 281 270 F F F <---
184272024-04-20T01:18:39.843Z 7 140-159 169 169 169 279 279 279 F F F
184282024-04-20T01:18:39.843Z 8 160-179 163 163 163 269 269 269 T T F <---
184292024-04-20T01:18:39.843Z 9 180-199 170 170 164 281 281 270 T T F <---
184302024-04-20T01:18:39.843Z 10 200-219 166 166 166 274 274 274 T T F <---
184312024-04-20T01:18:39.843Z 11 220-239 169 169 169 279 279 279 F F F
184322024-04-20T01:18:39.843Z 12 240-259 170 170 169 281 281 279 F F F <---
184332024-04-20T01:18:39.843Z 13 260-279 170 170 168 282 282 277 F F F <---
184342024-04-20T01:18:39.844Z 14 280-299 170 170 168 281 281 277 T T F <---
184352024-04-20T01:18:39.844Z 15 300-319 169 169 169 279 279 279 F F F
184362024-04-20T01:18:39.844Z 16 320-339 168 168 168 276 276 276 F F F
184372024-04-20T01:18:39.844Z 17 340-359 166 166 166 272 272 272 F F F
184382024-04-20T01:18:39.844Z 18 360-379 164 164 164 270 270 270 T T F <---
184392024-04-20T01:18:39.845Z 19 380-399 166 166 166 272 272 272 F F F
184402024-04-20T01:18:39.845Z 20 400-419 166 166 166 273 273 273 F F F
184412024-04-20T01:18:39.845Z 21 420-439 166 166 166 272 272 272 F F F
184422024-04-20T01:18:39.845Z 22 440-459 166 166 166 273 273 273 F F F
184432024-04-20T01:18:39.845Z 23 460-479 166 166 166 273 273 273 F F F
184442024-04-20T01:18:39.846Z 24 480-499 167 167 167 275 275 275 F F F
184452024-04-20T01:18:39.846Z 25 500-519 170 170 163 281 281 269 F F F <---
184462024-04-20T01:18:39.846Z 26 520-539 170 170 169 281 281 279 F F F <---
184472024-04-20T01:18:39.846Z 27 540-559 166 166 166 274 274 274 F F F
184482024-04-20T01:18:39.846Z 28 560-579 169 169 169 279 279 279 T T F <---
184492024-04-20T01:18:39.847Z 29 580-599 169 169 169 279 279 279 T T F <---
184502024-04-20T01:18:39.847ZMax gen: 170, Max flush: 282
184512024-04-20T01:18:39.847ZError: Difference in extent metadata found!
184522024-04-20T01:18:39.847ZA Difference in extent metadata is expected here
184532024-04-20T01:18:39.847ZOn loop 85
184542024-04-20T01:18:39.847Z
184552024-04-20T01:18:39.865ZVerifying data now
184562024-04-20T01:18:44.519ZNo change after verify
184572024-04-20T01:18:44.519ZLoop: 85 Downstairs dump after verify (and repair):
184582024-04-20T01:18:44.533ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184592024-04-20T01:18:44.533Z 0 000-019 171 171 171 283 283 283 F F F
184602024-04-20T01:18:44.533Z 1 020-039 160 160 160 266 266 266 F F F
184612024-04-20T01:18:44.533Z 2 040-059 170 170 170 281 281 281 F F F
184622024-04-20T01:18:44.533Z 3 060-079 163 163 163 269 269 269 F F F
184632024-04-20T01:18:44.533Z 4 080-099 168 168 168 277 277 277 F F F
184642024-04-20T01:18:44.533Z 5 100-119 170 170 170 281 281 281 F F F
184652024-04-20T01:18:44.538Z 6 120-139 170 170 170 281 281 281 F F F
184662024-04-20T01:18:44.538Z 7 140-159 169 169 169 279 279 279 F F F
184672024-04-20T01:18:44.538Z 8 160-179 171 171 171 283 283 283 F F F
184682024-04-20T01:18:44.538Z 9 180-199 171 171 171 283 283 283 F F F
184692024-04-20T01:18:44.538Z 10 200-219 171 171 171 283 283 283 F F F
184702024-04-20T01:18:44.538Z 11 220-239 169 169 169 279 279 279 F F F
184712024-04-20T01:18:44.538Z 12 240-259 170 170 170 281 281 281 F F F
184722024-04-20T01:18:44.538Z 13 260-279 170 170 170 282 282 282 F F F
184732024-04-20T01:18:44.538Z 14 280-299 171 171 171 283 283 283 F F F
184742024-04-20T01:18:44.538Z 15 300-319 169 169 169 279 279 279 F F F
184752024-04-20T01:18:44.538Z 16 320-339 168 168 168 276 276 276 F F F
184762024-04-20T01:18:44.538Z 17 340-359 166 166 166 272 272 272 F F F
184772024-04-20T01:18:44.538Z 18 360-379 171 171 171 283 283 283 F F F
184782024-04-20T01:18:44.538Z 19 380-399 166 166 166 272 272 272 F F F
184792024-04-20T01:18:44.538Z 20 400-419 166 166 166 273 273 273 F F F
184802024-04-20T01:18:44.539Z 21 420-439 166 166 166 272 272 272 F F F
184812024-04-20T01:18:44.539Z 22 440-459 166 166 166 273 273 273 F F F
184822024-04-20T01:18:44.539Z 23 460-479 166 166 166 273 273 273 F F F
184832024-04-20T01:18:44.539Z 24 480-499 167 167 167 275 275 275 F F F
184842024-04-20T01:18:44.539Z 25 500-519 170 170 170 281 281 281 F F F
184852024-04-20T01:18:44.539Z 26 520-539 170 170 170 281 281 281 F F F
184862024-04-20T01:18:44.539Z 27 540-559 166 166 166 274 274 274 F F F
184872024-04-20T01:18:44.539Z 28 560-579 171 171 171 283 283 283 F F F
184882024-04-20T01:18:44.539Z 29 580-599 171 171 171 283 283 283 F F F
184892024-04-20T01:18:44.539ZMax gen: 171, Max flush: 283
184902024-04-20T01:18:44.539Z
184912024-04-20T01:18:44.539ZBegin loop 86
184922024-04-20T01:18:44.539ZDownstairs to restart: 2
184932024-04-20T01:18:44.554Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049268105Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862}
184942024-04-20T01:18:44.554Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049490646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862}
184952024-04-20T01:18:44.554Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049508376Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862}
184962024-04-20T01:18:44.566Z{"msg":"Crucible 4f8e8c33-8657-481a-b263-ab3107951dcc has session id: 7694fbd4-6e38-4be0-9132-7c1b5519428b","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049541086Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
184972024-04-20T01:18:44.566Z{"msg":"Upstairs opts: Upstairs UUID: 4f8e8c33-8657-481a-b263-ab3107951dcc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"Crucible runtime is spawned
184982024-04-20T01:18:44.566Zcrucible","level":30,"time":"2024-04-20T01:18:43.049642846Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
184992024-04-20T01:18:44.566Z{"msg":"Crucible stats registered with UUID: 4f8e8c33-8657-481a-b263-ab3107951dcc","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049684996Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185002024-04-20T01:18:44.566Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049701726Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862}
185012024-04-20T01:18:44.566Z{"msg":"4f8e8c33-8657-481a-b263-ab3107951dcc active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049798206Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185022024-04-20T01:18:44.566Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049817137Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"0","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185032024-04-20T01:18:44.566Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049835167Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"0","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185042024-04-20T01:18:44.566Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049877467Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"1","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185052024-04-20T01:18:44.566Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049906917Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"1","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185062024-04-20T01:18:44.566Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049947247Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"2","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185072024-04-20T01:18:44.566Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.049969167Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"2","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185082024-04-20T01:18:44.566Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.050004037Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185092024-04-20T01:18:44.566Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.050032897Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185102024-04-20T01:18:44.567Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.050080277Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185112024-04-20T01:18:44.567Z{"msg":"ds_connection connected from Ok(127.0.0.1:57808)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.050160607Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185122024-04-20T01:18:44.567Z{"msg":"ds_connection connected from Ok(127.0.0.1:37481)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.050228547Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185132024-04-20T01:18:44.567Z{"msg":"ds_connection connected from Ok(127.0.0.1:58469)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.050256218Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185142024-04-20T01:18:44.567Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.050948389Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"0","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185152024-04-20T01:18:44.567Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.050997329Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"0","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185162024-04-20T01:18:44.567Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.051031039Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"1","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185172024-04-20T01:18:44.567Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.051058689Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"1","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185182024-04-20T01:18:44.567Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.05130154Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"2","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185192024-04-20T01:18:44.567Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.05134504Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"2","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185202024-04-20T01:18:44.567Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.05160743Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"1","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185212024-04-20T01:18:44.567Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.051703001Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"0","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185222024-04-20T01:18:44.567Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052003102Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"2","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185232024-04-20T01:18:44.567Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052190292Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"1","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185242024-04-20T01:18:44.567Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052236662Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"1","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185252024-04-20T01:18:44.567Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052297182Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185262024-04-20T01:18:44.567Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052337712Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"0","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185272024-04-20T01:18:44.567Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052356432Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"0","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185282024-04-20T01:18:44.567Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052377612Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185292024-04-20T01:18:44.567Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052411622Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"2","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185302024-04-20T01:18:44.567Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052434703Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"2","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185312024-04-20T01:18:44.567Z{"msg":"[0]R flush_numbers[0..12]: [283, 266, 281, 269, 277, 281, 281, 279, 283, 283, 283, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052450323Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185322024-04-20T01:18:44.567Z{"msg":"[0]R generation[0..12]: [171, 160, 170, 163, 168, 170, 170, 169, 171, 171, 171, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052471632Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185332024-04-20T01:18:44.568Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052486583Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185342024-04-20T01:18:44.568Z{"msg":"[1]R flush_numbers[0..12]: [283, 266, 281, 269, 277, 281, 281, 279, 283, 283, 283, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052507123Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185352024-04-20T01:18:44.568Z{"msg":"[1]R generation[0..12]: [171, 160, 170, 163, 168, 170, 170, 169, 171, 171, 171, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052521073Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185362024-04-20T01:18:44.568Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052541803Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185372024-04-20T01:18:44.568Z{"msg":"[2]R flush_numbers[0..12]: [283, 266, 281, 269, 277, 281, 281, 279, 283, 283, 283, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052555863Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185382024-04-20T01:18:44.568Z{"msg":"[2]R generation[0..12]: [171, 160, 170, 163, 168, 170, 170, 169, 171, 171, 171, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052569593Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185392024-04-20T01:18:44.568Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
185402024-04-20T01:18:44.568Zcrucible","level":30,"time":"2024-04-20T01:18:43.052586633Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185412024-04-20T01:18:44.568Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052607903Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185422024-04-20T01:18:44.568Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052621493Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185432024-04-20T01:18:44.568Z{"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052641563Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185442024-04-20T01:18:44.568Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052655213Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"":"downstairs","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185452024-04-20T01:18:44.568Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052688033Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185462024-04-20T01:18:44.568Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052720843Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"0","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185472024-04-20T01:18:44.568Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052750303Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"1","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185482024-04-20T01:18:44.568Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052767193Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"client":"2","":"downstairs","session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185492024-04-20T01:18:44.568Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
185502024-04-20T01:18:44.568Z,"time":"2024-04-20T01:18:43.052791883Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185512024-04-20T01:18:44.568Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052810423Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185522024-04-20T01:18:44.568Z{"msg":"4f8e8c33-8657-481a-b263-ab3107951dcc is now active with session: 7694fbd4-6e38-4be0-9132-7c1b5519428b","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052835093Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185532024-04-20T01:18:44.568Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052849613Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862,"session_id":"7694fbd4-6e38-4be0-9132-7c1b5519428b"}
185542024-04-20T01:18:44.568Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:43.052869623Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862}
185552024-04-20T01:18:44.569ZLoading write count information from file "/tmp/test_repair_verify.data"
185562024-04-20T01:18:44.569ZRun Repair workload
185572024-04-20T01:18:44.575Z01/30 Write block 23 len 28672 data: 7 8 8 10 10 10 11
185582024-04-20T01:18:44.575Z02/30 Read block 470 len 40960
185592024-04-20T01:18:44.575Z03/30 Read block 267 len 20480
185602024-04-20T01:18:44.575Z04/30 Read block 204 len 20480
185612024-04-20T01:18:44.575Z05/30 Write block 178 len 36864 data: 17 17 16 17 14 10 8 9 11
185622024-04-20T01:18:44.575Z06/30 Write block 66 len 16384 data: 15 15 18 18
185632024-04-20T01:18:44.575Z07/30 Read block 60 len 16384
185642024-04-20T01:18:44.575Z08/30 Read block 185 len 36864
185652024-04-20T01:18:44.575Z09/30 Read block 169 len 40960
185662024-04-20T01:18:44.575Z10/30 Write block 173 len 24576 data: 16 16 14 16 15 18
185672024-04-20T01:18:44.575Z11/30 Read block 512 len 16384
185682024-04-20T01:18:44.575Z12/30 Write block 392 len 36864 data: 9 10 10 9 9 8 9 11 10
185692024-04-20T01:18:44.575Z13/30 Write block 110 len 8192 data: 13 13
185702024-04-20T01:18:44.575Z14/30 Write block 404 len 28672 data: 9 10 13 10 13 15 16
185712024-04-20T01:18:44.575Z15/30 Write block 542 len 36864 data: 17 19 17 15 15 11 14 17 20
185722024-04-20T01:18:44.579Z16/30 Read block 475 len 36864
185732024-04-20T01:18:44.579Z17/30 Flush
185742024-04-20T01:18:44.579Z18/30 Write block 185 len 24576 data: 10 12 12 15 14 17
185752024-04-20T01:18:44.579Z19/30 Read block 409 len 36864
185762024-04-20T01:18:44.584Z20/30 Write block 364 len 4096 data: 10
185772024-04-20T01:18:44.584Z21/30 Write block 591 len 28672 data: 15 14 15 16 15 14 8
185782024-04-20T01:18:44.584Z22/30 Write block 513 len 24576 data: 16 14 13 18 19 16
185792024-04-20T01:18:44.584Z23/30 Read block 209 len 8192
185802024-04-20T01:18:44.584Z24/30 Read block 38 len 28672
185812024-04-20T01:18:44.589Z25/30 Read block 212 len 40960
185822024-04-20T01:18:44.589Z26/30 Write block 434 len 32768 data: 14 15 16 14 11 13 13 14
185832024-04-20T01:18:44.589Z27/30 Write block 529 len 12288 data: 9 10 12
185842024-04-20T01:18:44.589Z28/30 Write block 565 len 8192 data: 16 17
185852024-04-20T01:18:44.589Z29/30 Read block 536 len 28672
185862024-04-20T01:18:44.589Z30/30 Read block 344 len 12288
185872024-04-20T01:18:44.595Z----------------------------------------------------------------
185882024-04-20T01:18:44.595Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
185892024-04-20T01:18:44.595ZStates: Active Active Active
185902024-04-20T01:18:44.595ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185912024-04-20T01:18:44.595Z 1 Acked 1000 Write 7 Done Done Done false
185922024-04-20T01:18:44.595Z 2 Acked 1001 Read 10 Done Done Done false
185932024-04-20T01:18:44.595Z 3 Acked 1002 Read 5 Done Done Done false
185942024-04-20T01:18:44.595Z 4 Acked 1003 Read 5 Done Done Done false
185952024-04-20T01:18:44.595Z 5 Acked 1004 Write 9 Done Done Done false
185962024-04-20T01:18:44.595Z 6 Acked 1005 Write 4 Done Done Done false
185972024-04-20T01:18:44.595Z 7 Acked 1006 Read 4 Done Done Done false
185982024-04-20T01:18:44.595Z 8 Acked 1007 Read 9 Done Done Done false
185992024-04-20T01:18:44.595Z 9 Acked 1008 Read 10 Done Done Sent false
186002024-04-20T01:18:44.595Z 10 Acked 1009 Write 6 Done Done Sent false
186012024-04-20T01:18:44.595Z 11 Acked 1010 Read 4 Done Done Sent false
186022024-04-20T01:18:44.595Z 12 Acked 1011 Write 9 Done Done Sent false
186032024-04-20T01:18:44.595Z 13 Acked 1012 Write 2 Done Done Sent false
186042024-04-20T01:18:44.595Z 14 Acked 1013 Write 7 Done Done Sent false
186052024-04-20T01:18:44.595Z 15 Acked 1014 Write 9 Done Done Sent false
186062024-04-20T01:18:44.596Z 16 Acked 1015 Read 9 Done Done Sent false
186072024-04-20T01:18:44.596Z 17 Acked 1016 Flush 0 Done Done Sent false
186082024-04-20T01:18:44.596Z 18 Acked 1017 Write 6 Done Done Sent false
186092024-04-20T01:18:44.596Z 19 Acked 1018 Read 9 Done Done Sent false
186102024-04-20T01:18:44.596Z 20 Acked 1019 Write 1 Done Done Sent false
186112024-04-20T01:18:44.596Z 21 Acked 1020 Write 7 Done Done Sent false
186122024-04-20T01:18:44.596Z 22 Acked 1021 Write 6 Done Done Sent false
186132024-04-20T01:18:44.596Z 23 Acked 1022 Read 2 Done Done Sent false
186142024-04-20T01:18:44.596Z 24 Acked 1023 Read 7 Done Done Sent false
186152024-04-20T01:18:44.596Z 25 Acked 1024 Read 10 Done Done Sent false
186162024-04-20T01:18:44.596Z 26 Acked 1025 Write 8 Done Done Sent false
186172024-04-20T01:18:44.596Z 27 Acked 1026 Write 3 Done Done Sent false
186182024-04-20T01:18:44.596Z 28 Acked 1027 Write 2 Done Done Sent false
186192024-04-20T01:18:44.596Z 29 Acked 1028 Read 7 Done Done Sent false
186202024-04-20T01:18:44.596Z 30 Acked 1029 Read 3 Done Done Sent false
186212024-04-20T01:18:44.596Z STATES DS:0 DS:1 DS:2 TOTAL
186222024-04-20T01:18:44.596Z New 0 0 0 0
186232024-04-20T01:18:44.596Z Sent 0 0 22 22
186242024-04-20T01:18:44.596Z Done 30 30 8 68
186252024-04-20T01:18:44.596Z Skipped 0 0 0 0
186262024-04-20T01:18:44.596Z Error 0 0 0 0
186272024-04-20T01:18:44.596ZLast Flush: 1016 1016 0
186282024-04-20T01:18:44.596ZDownstairs last five completed:
186292024-04-20T01:18:44.597ZUpstairs last five completed: 30 29 28 27 26
186302024-04-20T01:18:44.597Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:18:43.084444934Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1862}
186312024-04-20T01:18:44.597ZWrote out file "/tmp/test_repair_verify.data"
186322024-04-20T01:18:44.597Z
186332024-04-20T01:18:44.601ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
186342024-04-20T01:18:44.614ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186352024-04-20T01:18:44.614Z 0 000-019 171 171 171 283 283 283 F F F
186362024-04-20T01:18:44.614Z 1 020-039 172 172 160 284 284 266 F F T <---
186372024-04-20T01:18:44.614Z 2 040-059 170 170 170 281 281 281 F F F
186382024-04-20T01:18:44.614Z 3 060-079 172 172 163 284 284 269 F F T <---
186392024-04-20T01:18:44.614Z 4 080-099 168 168 168 277 277 277 F F F
186402024-04-20T01:18:44.614Z 5 100-119 172 172 170 284 284 281 F F F <---
186412024-04-20T01:18:44.614Z 6 120-139 170 170 170 281 281 281 F F F
186422024-04-20T01:18:44.614Z 7 140-159 169 169 169 279 279 279 F F F
186432024-04-20T01:18:44.614Z 8 160-179 172 172 171 284 284 283 F F T <---
186442024-04-20T01:18:44.618Z 9 180-199 172 172 171 284 284 283 T T T <---
186452024-04-20T01:18:44.618Z 10 200-219 171 171 171 283 283 283 F F F
186462024-04-20T01:18:44.618Z 11 220-239 169 169 169 279 279 279 F F F
186472024-04-20T01:18:44.618Z 12 240-259 170 170 170 281 281 281 F F F
186482024-04-20T01:18:44.618Z 13 260-279 170 170 170 282 282 282 F F F
186492024-04-20T01:18:44.618Z 14 280-299 171 171 171 283 283 283 F F F
186502024-04-20T01:18:44.618Z 15 300-319 169 169 169 279 279 279 F F F
186512024-04-20T01:18:44.618Z 16 320-339 168 168 168 276 276 276 F F F
186522024-04-20T01:18:44.618Z 17 340-359 166 166 166 272 272 272 F F F
186532024-04-20T01:18:44.618Z 18 360-379 171 171 171 283 283 283 T T F <---
186542024-04-20T01:18:44.618Z 19 380-399 172 172 166 284 284 272 F F F <---
186552024-04-20T01:18:44.618Z 20 400-419 172 172 166 284 284 273 F F F <---
186562024-04-20T01:18:44.618Z 21 420-439 166 166 166 272 272 272 T T F <---
186572024-04-20T01:18:44.618Z 22 440-459 166 166 166 273 273 273 T T F <---
186582024-04-20T01:18:44.619Z 23 460-479 166 166 166 273 273 273 F F F
186592024-04-20T01:18:44.619Z 24 480-499 167 167 167 275 275 275 F F F
186602024-04-20T01:18:44.619Z 25 500-519 170 170 170 281 281 281 T T F <---
186612024-04-20T01:18:44.619Z 26 520-539 170 170 170 281 281 281 T T F <---
186622024-04-20T01:18:44.619Z 27 540-559 172 172 166 284 284 274 F F F <---
186632024-04-20T01:18:44.619Z 28 560-579 171 171 171 283 283 283 T T F <---
186642024-04-20T01:18:44.619Z 29 580-599 171 171 171 283 283 283 T T F <---
186652024-04-20T01:18:44.619ZMax gen: 172, Max flush: 284
186662024-04-20T01:18:44.619ZError: Difference in extent metadata found!
186672024-04-20T01:18:44.619ZA Difference in extent metadata is expected here
186682024-04-20T01:18:44.619ZOn loop 86
186692024-04-20T01:18:44.619Z
186702024-04-20T01:18:44.623ZVerifying data now
186712024-04-20T01:18:59.314ZNo change after verify
186722024-04-20T01:18:59.314ZLoop: 86 Downstairs dump after verify (and repair):
186732024-04-20T01:18:59.329ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186742024-04-20T01:18:59.330Z 0 000-019 171 171 171 283 283 283 F F F
186752024-04-20T01:18:59.330Z 1 020-039 172 172 172 284 284 284 F F F
186762024-04-20T01:18:59.330Z 2 040-059 170 170 170 281 281 281 F F F
186772024-04-20T01:18:59.330Z 3 060-079 172 172 172 284 284 284 F F F
186782024-04-20T01:18:59.330Z 4 080-099 168 168 168 277 277 277 F F F
186792024-04-20T01:18:59.333Z 5 100-119 172 172 172 284 284 284 F F F
186802024-04-20T01:18:59.333Z 6 120-139 170 170 170 281 281 281 F F F
186812024-04-20T01:18:59.333Z 7 140-159 169 169 169 279 279 279 F F F
186822024-04-20T01:18:59.333Z 8 160-179 172 172 172 284 284 284 F F F
186832024-04-20T01:18:59.334Z 9 180-199 173 173 173 285 285 285 F F F
186842024-04-20T01:18:59.334Z 10 200-219 171 171 171 283 283 283 F F F
186852024-04-20T01:18:59.334Z 11 220-239 169 169 169 279 279 279 F F F
186862024-04-20T01:18:59.334Z 12 240-259 170 170 170 281 281 281 F F F
186872024-04-20T01:18:59.334Z 13 260-279 170 170 170 282 282 282 F F F
186882024-04-20T01:18:59.334Z 14 280-299 171 171 171 283 283 283 F F F
186892024-04-20T01:18:59.334Z 15 300-319 169 169 169 279 279 279 F F F
186902024-04-20T01:18:59.334Z 16 320-339 168 168 168 276 276 276 F F F
186912024-04-20T01:18:59.334Z 17 340-359 166 166 166 272 272 272 F F F
186922024-04-20T01:18:59.334Z 18 360-379 173 173 173 285 285 285 F F F
186932024-04-20T01:18:59.334Z 19 380-399 172 172 172 284 284 284 F F F
186942024-04-20T01:18:59.334Z 20 400-419 172 172 172 284 284 284 F F F
186952024-04-20T01:18:59.334Z 21 420-439 173 173 173 285 285 285 F F F
186962024-04-20T01:18:59.334Z 22 440-459 173 173 173 285 285 285 F F F
186972024-04-20T01:18:59.334Z 23 460-479 166 166 166 273 273 273 F F F
186982024-04-20T01:18:59.334Z 24 480-499 167 167 167 275 275 275 F F F
186992024-04-20T01:18:59.334Z 25 500-519 173 173 173 285 285 285 F F F
187002024-04-20T01:18:59.334Z 26 520-539 173 173 173 285 285 285 F F F
187012024-04-20T01:18:59.334Z 27 540-559 172 172 172 284 284 284 F F F
187022024-04-20T01:18:59.334Z 28 560-579 173 173 173 285 285 285 F F F
187032024-04-20T01:18:59.334Z 29 580-599 173 173 173 285 285 285 F F F
187042024-04-20T01:18:59.334ZMax gen: 173, Max flush: 285
187052024-04-20T01:18:59.334Z
187062024-04-20T01:18:59.335ZBegin loop 87
187072024-04-20T01:18:59.335ZDownstairs to restart: 2
187082024-04-20T01:18:59.351Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846127323Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870}
187092024-04-20T01:18:59.351Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846336594Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870}
187102024-04-20T01:18:59.351Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846353594Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870}
187112024-04-20T01:18:59.351Z{"msg":"Crucible 83c37bd2-a0f1-42ee-8ccb-ca2718a44d61 has session id: a5eab753-2947-499b-be0b-c00ec0a66041","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846375464Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187122024-04-20T01:18:59.357Z{"msg":"Upstairs opts: Upstairs UUID: 83c37bd2-a0f1-42ee-8ccb-ca2718a44d61, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
187132024-04-20T01:18:59.358Z,"time":"2024-04-20T01:18:57.846420334Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187142024-04-20T01:18:59.358Z{"msg":"Crucible stats registered with UUID: 83c37bd2-a0f1-42ee-8ccb-ca2718a44d61","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846462454Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187152024-04-20T01:18:59.358Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846500744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870}
187162024-04-20T01:18:59.358Z{"msg":"83c37bd2-a0f1-42ee-8ccb-ca2718a44d61 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846596014Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187172024-04-20T01:18:59.358Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846621874Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"0","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187182024-04-20T01:18:59.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846636724Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"0","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187192024-04-20T01:18:59.358Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846650934Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"1","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187202024-04-20T01:18:59.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846677564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"1","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187212024-04-20T01:18:59.358Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846691704Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"2","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187222024-04-20T01:18:59.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846709235Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"2","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187232024-04-20T01:18:59.358Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846773364Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187242024-04-20T01:18:59.358Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846815155Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187252024-04-20T01:18:59.358Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846843895Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187262024-04-20T01:18:59.358Z{"msg":"ds_connection connected from Ok(127.0.0.1:44633)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.846976435Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187272024-04-20T01:18:59.358Z{"msg":"ds_connection connected from Ok(127.0.0.1:44431)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.847052495Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187282024-04-20T01:18:59.358Z{"msg":"ds_connection connected from Ok(127.0.0.1:48082)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.847110065Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187292024-04-20T01:18:59.358Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.847689977Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"0","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187302024-04-20T01:18:59.358Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.847733947Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"0","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187312024-04-20T01:18:59.358Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.847798817Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"1","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187322024-04-20T01:18:59.358Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.847845327Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"1","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187332024-04-20T01:18:59.358Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.848066258Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"2","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187342024-04-20T01:18:59.359Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.848108698Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"2","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187352024-04-20T01:18:59.359Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.848189448Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"1","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187362024-04-20T01:18:59.359Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.848298998Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"0","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187372024-04-20T01:18:59.359Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.848600399Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"2","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187382024-04-20T01:18:59.359Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.848816109Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"1","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187392024-04-20T01:18:59.359Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.848852299Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"1","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187402024-04-20T01:18:59.359Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.848868839Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187412024-04-20T01:18:59.359Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.848888789Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"0","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187422024-04-20T01:18:59.359Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.848902509Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"0","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187432024-04-20T01:18:59.359Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.848915969Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187442024-04-20T01:18:59.359Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.8490871Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"2","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187452024-04-20T01:18:59.359Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.8491327Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"2","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187462024-04-20T01:18:59.359Z{"msg":"[0]R flush_numbers[0..12]: [283, 284, 281, 284, 277, 284, 281, 279, 284, 285, 283, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.84917063Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187472024-04-20T01:18:59.359Z{"msg":"[0]R generation[0..12]: [171, 172, 170, 172, 168, 172, 170, 169, 172, 173, 171, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.84920685Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187482024-04-20T01:18:59.359Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.84924205Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187492024-04-20T01:18:59.359Z{"msg":"[1]R flush_numbers[0..12]: [283, 284, 281, 284, 277, 284, 281, 279, 284, 285, 283, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.84927863Z",Wait for a query_work_queue command to finish before sending IO
187502024-04-20T01:18:59.359Z"hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187512024-04-20T01:18:59.359Z{"msg":"[1]R generation[0..12]: [171, 172, 170, 172, 168, 172, 170, 169, 172, 173, 171, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.84931714Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187522024-04-20T01:18:59.359Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.84935684Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187532024-04-20T01:18:59.359Z{"msg":"[2]R flush_numbers[0..12]: [283, 284, 281, 284, 277, 284, 281, 279, 284, 285, 283, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849398021Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187542024-04-20T01:18:59.359Z{"msg":"[2]R generation[0..12]: [171, 172, 170, 172, 168, 172, 170, 169, 172, 173, 171, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.84943765Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187552024-04-20T01:18:59.359Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.84947531Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187562024-04-20T01:18:59.359Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849495221Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187572024-04-20T01:18:59.360Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849508961Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187582024-04-20T01:18:59.360Z{"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849529401Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187592024-04-20T01:18:59.360Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849542571Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"":"downstairs","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187602024-04-20T01:18:59.360Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849556381Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187612024-04-20T01:18:59.360Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849574951Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"0","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187622024-04-20T01:18:59.360Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849588561Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"1","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187632024-04-20T01:18:59.360Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849601411Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"client":"2","":"downstairs","session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187642024-04-20T01:18:59.360Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
187652024-04-20T01:18:59.360Z2024-04-20T01:18:57.849621501Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187662024-04-20T01:18:59.360Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849639231Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187672024-04-20T01:18:59.360Z{"msg":"83c37bd2-a0f1-42ee-8ccb-ca2718a44d61 is now active with session: a5eab753-2947-499b-be0b-c00ec0a66041","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849651931Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187682024-04-20T01:18:59.360Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849670591Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870,"session_id":"a5eab753-2947-499b-be0b-c00ec0a66041"}
187692024-04-20T01:18:59.360Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:18:57.849682961Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870}
187702024-04-20T01:18:59.360ZLoading write count information from file "/tmp/test_repair_verify.data"
187712024-04-20T01:18:59.360ZRun Repair workload
187722024-04-20T01:18:59.364Z01/30 Write block 378 len 24576 data: 15 15 18 15 15 13
187732024-04-20T01:18:59.364Z02/30 Read block 255 len 24576
187742024-04-20T01:18:59.364Z03/30 Read block 374 len 8192
187752024-04-20T01:18:59.364Z04/30 Read block 380 len 40960
187762024-04-20T01:18:59.364Z05/30 Write block 396 len 36864 data: 10 9 10 12 11 10 11 9 10
187772024-04-20T01:18:59.364Z06/30 Read block 560 len 4096
187782024-04-20T01:18:59.364Z07/30 Write block 82 len 4096 data: 11
187792024-04-20T01:18:59.364Z08/30 Read block 106 len 32768
187802024-04-20T01:18:59.364Z09/30 Flush
187812024-04-20T01:18:59.368Z10/30 Write block 116 len 32768 data: 12 11 12 13 12 12 12 12
187822024-04-20T01:18:59.368Z11/30 Read block 381 len 28672
187832024-04-20T01:18:59.368Z12/30 Flush
187842024-04-20T01:18:59.372Z13/30 Write block 71 len 36864 data: 11 9 12 14 11 8 12 13 13
187852024-04-20T01:18:59.372Z14/30 Write block 80 len 12288 data: 11 11 12
187862024-04-20T01:18:59.372Z15/30 Write block 212 len 28672 data: 12 14 13 13 16 21 21
187872024-04-20T01:18:59.375Z16/30 Write block 68 len 32768 data: 19 19 12 12 10 13 15 12
187882024-04-20T01:18:59.375Z17/30 Read block 513 len 32768
187892024-04-20T01:18:59.375Z18/30 Read block 521 len 4096
187902024-04-20T01:18:59.375Z19/30 Read block 558 len 16384
187912024-04-20T01:18:59.379Z20/30 Write block 470 len 4096 data: 17
187922024-04-20T01:18:59.379Z21/30 Write block 520 len 40960 data: 19 21 20 19 16 15 14 12 9 10
187932024-04-20T01:18:59.379Z22/30 Write block 87 len 28672 data: 15 15 15 18 16 14 14
187942024-04-20T01:18:59.379Z23/30 Write block 315 len 8192 data: 8 11
187952024-04-20T01:18:59.383Z24/30 Write block 292 len 16384 data: 9 11 14 12
187962024-04-20T01:18:59.383Z25/30 Flush
187972024-04-20T01:18:59.386Z26/30 Write block 118 len 24576 data: 13 14 13 13 13 13
187982024-04-20T01:18:59.386Z27/30 Write block 96 len 16384 data: 12 11 11 10
187992024-04-20T01:18:59.386Z28/30 Read block 63 len 40960
188002024-04-20T01:18:59.386Z29/30 Write block 340 len 16384 data: 10 10 10 13
188012024-04-20T01:18:59.386Z30/30 Read block 112 len 16384
188022024-04-20T01:18:59.390Z----------------------------------------------------------------
188032024-04-20T01:18:59.391Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
188042024-04-20T01:18:59.391ZStates: Active Active Active
188052024-04-20T01:18:59.391ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
188062024-04-20T01:18:59.391Z 1 Acked 1000 Write 6 Done Done Done false
188072024-04-20T01:18:59.391Z 2 Acked 1001 Read 6 Done Done Done false
188082024-04-20T01:18:59.391Z 3 Acked 1002 Read 2 Done Done Sent false
188092024-04-20T01:18:59.391Z 4 Acked 1003 Read 10 Done Done Sent false
188102024-04-20T01:18:59.391Z 5 Acked 1004 Write 9 Done Done Sent false
188112024-04-20T01:18:59.391Z 6 Acked 1005 Read 1 Done Done Sent false
188122024-04-20T01:18:59.391Z 7 Acked 1006 Write 1 Done Done Sent false
188132024-04-20T01:18:59.391Z 8 Acked 1007 Read 8 Done Done Sent false
188142024-04-20T01:18:59.391Z 9 Acked 1008 Flush 0 Done Done Sent false
188152024-04-20T01:18:59.391Z 10 Acked 1009 Write 8 Done Done Sent false
188162024-04-20T01:18:59.391Z 11 Acked 1010 Read 7 Done Done Sent false
188172024-04-20T01:18:59.391Z 12 Acked 1011 Flush 0 Done Done Sent false
188182024-04-20T01:18:59.391Z 13 Acked 1012 Write 9 Done Done Sent false
188192024-04-20T01:18:59.391Z 14 Acked 1013 Write 3 Done Done Sent false
188202024-04-20T01:18:59.391Z 15 Acked 1014 Write 7 Done Done Sent false
188212024-04-20T01:18:59.391Z 16 Acked 1015 Write 8 Done Done Sent false
188222024-04-20T01:18:59.391Z 17 Acked 1016 Read 8 Done Done Sent false
188232024-04-20T01:18:59.391Z 18 Acked 1017 Read 1 Done Done Sent false
188242024-04-20T01:18:59.391Z 19 Acked 1018 Read 4 Done Done Sent false
188252024-04-20T01:18:59.392Z 20 Acked 1019 Write 1 Done Done Sent false
188262024-04-20T01:18:59.392Z 21 Acked 1020 Write 10 Done Done Sent false
188272024-04-20T01:18:59.392Z 22 Acked 1021 Write 7 Done Done Sent false
188282024-04-20T01:18:59.392Z 23 Acked 1022 Write 2 Done Done Sent false
188292024-04-20T01:18:59.392Z 24 Acked 1023 Write 4 Done Done Sent false
188302024-04-20T01:18:59.392Z 25 Acked 1024 Flush 0 Done Done Sent false
188312024-04-20T01:18:59.392Z 26 Acked 1025 Write 6 Done Done Sent false
188322024-04-20T01:18:59.392Z 27 Acked 1026 Write 4 Done Done Sent false
188332024-04-20T01:18:59.392Z 28 Acked 1027 Read 10 Done Done Sent false
188342024-04-20T01:18:59.392Z 29 Acked 1028 Write 4 Done Done Sent false
188352024-04-20T01:18:59.392Z 30 Acked 1029 Read 4 Done Done Sent false
188362024-04-20T01:18:59.392Z STATES DS:0 DS:1 DS:2 TOTAL
188372024-04-20T01:18:59.392Z New 0 0 0 0
188382024-04-20T01:18:59.392Z Sent 0 0 28 28
188392024-04-20T01:18:59.392Z Done 30 30 2 62
188402024-04-20T01:18:59.392Z Skipped 0 0 0 0
188412024-04-20T01:18:59.392Z Error 0 0 0 0
188422024-04-20T01:18:59.392ZLast Flush: 1024 1024 0
188432024-04-20T01:18:59.392ZDownstairs last five completed:
188442024-04-20T01:18:59.392ZUpstairs last five completed: 30 29 28 27 26
188452024-04-20T01:18:59.392Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:18:57.882979225Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1870}
188462024-04-20T01:18:59.392ZWrote out file "/tmp/test_repair_verify.data"
188472024-04-20T01:18:59.396Z
188482024-04-20T01:18:59.396ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
188492024-04-20T01:18:59.410ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188502024-04-20T01:18:59.410Z 0 000-019 171 171 171 283 283 283 F F F
188512024-04-20T01:18:59.410Z 1 020-039 172 172 172 284 284 284 F F F
188522024-04-20T01:18:59.410Z 2 040-059 170 170 170 281 281 281 F F F
188532024-04-20T01:18:59.410Z 3 060-079 174 174 172 288 288 284 F F F <---
188542024-04-20T01:18:59.410Z 4 080-099 174 174 168 288 288 277 T T F <---
188552024-04-20T01:18:59.410Z 5 100-119 174 174 172 287 287 284 T T F <---
188562024-04-20T01:18:59.414Z 6 120-139 174 174 170 287 287 281 T T F <---
188572024-04-20T01:18:59.414Z 7 140-159 169 169 169 279 279 279 F F F
188582024-04-20T01:18:59.414Z 8 160-179 172 172 172 284 284 284 F F F
188592024-04-20T01:18:59.414Z 9 180-199 173 173 173 285 285 285 F F F
188602024-04-20T01:18:59.414Z 10 200-219 174 174 171 288 288 283 F F F <---
188612024-04-20T01:18:59.414Z 11 220-239 169 169 169 279 279 279 F F F
188622024-04-20T01:18:59.414Z 12 240-259 170 170 170 281 281 281 F F F
188632024-04-20T01:18:59.414Z 13 260-279 170 170 170 282 282 282 F F F
188642024-04-20T01:18:59.414Z 14 280-299 174 174 171 288 288 283 F F F <---
188652024-04-20T01:18:59.414Z 15 300-319 174 174 169 288 288 279 F F F <---
188662024-04-20T01:18:59.415Z 16 320-339 168 168 168 276 276 276 F F F
188672024-04-20T01:18:59.415Z 17 340-359 166 166 166 272 272 272 T T F <---
188682024-04-20T01:18:59.415Z 18 360-379 174 174 173 286 286 285 F F T <---
188692024-04-20T01:18:59.415Z 19 380-399 174 174 172 286 286 284 F F T <---
188702024-04-20T01:18:59.415Z 20 400-419 174 174 172 286 286 284 F F F <---
188712024-04-20T01:18:59.415Z 21 420-439 173 173 173 285 285 285 F F F
188722024-04-20T01:18:59.415Z 22 440-459 173 173 173 285 285 285 F F F
188732024-04-20T01:18:59.415Z 23 460-479 174 174 166 288 288 273 F F F <---
188742024-04-20T01:18:59.415Z 24 480-499 167 167 167 275 275 275 F F F
188752024-04-20T01:18:59.415Z 25 500-519 173 173 173 285 285 285 F F F
188762024-04-20T01:18:59.415Z 26 520-539 174 174 173 288 288 285 F F F <---
188772024-04-20T01:18:59.415Z 27 540-559 172 172 172 284 284 284 F F F
188782024-04-20T01:18:59.415Z 28 560-579 173 173 173 285 285 285 F F F
188792024-04-20T01:18:59.415Z 29 580-599 173 173 173 285 285 285 F F F
188802024-04-20T01:18:59.415ZMax gen: 174, Max flush: 288
188812024-04-20T01:18:59.415ZError: Difference in extent metadata found!
188822024-04-20T01:18:59.415ZA Difference in extent metadata is expected here
188832024-04-20T01:18:59.415ZOn loop 87
188842024-04-20T01:18:59.415Z
188852024-04-20T01:18:59.419ZVerifying data now
188862024-04-20T01:19:14.014ZNo change after verify
188872024-04-20T01:19:14.015ZLoop: 87 Downstairs dump after verify (and repair):
188882024-04-20T01:19:14.029ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188892024-04-20T01:19:14.029Z 0 000-019 171 171 171 283 283 283 F F F
188902024-04-20T01:19:14.033Z 1 020-039 172 172 172 284 284 284 F F F
188912024-04-20T01:19:14.033Z 2 040-059 170 170 170 281 281 281 F F F
188922024-04-20T01:19:14.033Z 3 060-079 174 174 174 288 288 288 F F F
188932024-04-20T01:19:14.033Z 4 080-099 175 175 175 289 289 289 F F F
188942024-04-20T01:19:14.033Z 5 100-119 175 175 175 289 289 289 F F F
188952024-04-20T01:19:14.034Z 6 120-139 175 175 175 289 289 289 F F F
188962024-04-20T01:19:14.034Z 7 140-159 169 169 169 279 279 279 F F F
188972024-04-20T01:19:14.034Z 8 160-179 172 172 172 284 284 284 F F F
188982024-04-20T01:19:14.034Z 9 180-199 173 173 173 285 285 285 F F F
188992024-04-20T01:19:14.034Z 10 200-219 174 174 174 288 288 288 F F F
189002024-04-20T01:19:14.034Z 11 220-239 169 169 169 279 279 279 F F F
189012024-04-20T01:19:14.034Z 12 240-259 170 170 170 281 281 281 F F F
189022024-04-20T01:19:14.034Z 13 260-279 170 170 170 282 282 282 F F F
189032024-04-20T01:19:14.034Z 14 280-299 174 174 174 288 288 288 F F F
189042024-04-20T01:19:14.034Z 15 300-319 174 174 174 288 288 288 F F F
189052024-04-20T01:19:14.034Z 16 320-339 168 168 168 276 276 276 F F F
189062024-04-20T01:19:14.034Z 17 340-359 175 175 175 289 289 289 F F F
189072024-04-20T01:19:14.034Z 18 360-379 174 174 174 286 286 286 F F F
189082024-04-20T01:19:14.034Z 19 380-399 174 174 174 286 286 286 F F F
189092024-04-20T01:19:14.034Z 20 400-419 174 174 174 286 286 286 F F F
189102024-04-20T01:19:14.034Z 21 420-439 173 173 173 285 285 285 F F F
189112024-04-20T01:19:14.034Z 22 440-459 173 173 173 285 285 285 F F F
189122024-04-20T01:19:14.034Z 23 460-479 174 174 174 288 288 288 F F F
189132024-04-20T01:19:14.034Z 24 480-499 167 167 167 275 275 275 F F F
189142024-04-20T01:19:14.034Z 25 500-519 173 173 173 285 285 285 F F F
189152024-04-20T01:19:14.034Z 26 520-539 174 174 174 288 288 288 F F F
189162024-04-20T01:19:14.034Z 27 540-559 172 172 172 284 284 284 F F F
189172024-04-20T01:19:14.034Z 28 560-579 173 173 173 285 285 285 F F F
189182024-04-20T01:19:14.035Z 29 580-599 173 173 173 285 285 285 F F F
189192024-04-20T01:19:14.035ZMax gen: 175, Max flush: 289
189202024-04-20T01:19:14.035Z
189212024-04-20T01:19:14.035ZBegin loop 88
189222024-04-20T01:19:14.035ZDownstairs to restart: 2
189232024-04-20T01:19:14.051Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.545874085Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878}
189242024-04-20T01:19:14.051Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546100085Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878}
189252024-04-20T01:19:14.057Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546145925Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878}
189262024-04-20T01:19:14.057Z{"msg":"Crucible 39d8d347-051b-4f99-a142-5c737a5cd5be has session id: 718a22de-252c-4ee1-9579-3ea8685f3fb4","v":0,"name":"crucible","level":30Crucible runtime is spawned
189272024-04-20T01:19:14.057Z,"time":"2024-04-20T01:19:12.546231405Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189282024-04-20T01:19:14.057Z{"msg":"Upstairs opts: Upstairs UUID: 39d8d347-051b-4f99-a142-5c737a5cd5be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546404336Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189292024-04-20T01:19:14.057Z{"msg":"Crucible stats registered with UUID: 39d8d347-051b-4f99-a142-5c737a5cd5be","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546441746Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189302024-04-20T01:19:14.057Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546472836Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878}
189312024-04-20T01:19:14.057Z{"msg":"39d8d347-051b-4f99-a142-5c737a5cd5be active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546490966Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189322024-04-20T01:19:14.058Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546508116Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"0","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189332024-04-20T01:19:14.058Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546543646Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"0","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189342024-04-20T01:19:14.058Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546569726Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"1","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189352024-04-20T01:19:14.058Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546604846Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"1","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189362024-04-20T01:19:14.058Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546630737Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"2","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189372024-04-20T01:19:14.058Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546655626Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"2","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189382024-04-20T01:19:14.058Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546679657Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189392024-04-20T01:19:14.058Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546703766Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189402024-04-20T01:19:14.058Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546728347Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189412024-04-20T01:19:14.058Z{"msg":"ds_connection connected from Ok(127.0.0.1:47865)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546891147Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189422024-04-20T01:19:14.058Z{"msg":"ds_connection connected from Ok(127.0.0.1:48494)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546941677Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189432024-04-20T01:19:14.058Z{"msg":"ds_connection connected from Ok(127.0.0.1:65000)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.546961577Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189442024-04-20T01:19:14.058Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.547626409Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"0","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189452024-04-20T01:19:14.058Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.547667979Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"0","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189462024-04-20T01:19:14.058Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.547734619Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"1","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189472024-04-20T01:19:14.058Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.547769539Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"1","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189482024-04-20T01:19:14.058Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.54815665Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"2","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189492024-04-20T01:19:14.058Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.54820859Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"2","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189502024-04-20T01:19:14.058Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.54823914Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"0","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189512024-04-20T01:19:14.058Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.54833444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"1","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189522024-04-20T01:19:14.058Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.548742641Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"0","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189532024-04-20T01:19:14.058Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.548788311Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"0","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189542024-04-20T01:19:14.058Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.548804391Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189552024-04-20T01:19:14.058Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.548830491Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"1","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189562024-04-20T01:19:14.059Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.548852311Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"1","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189572024-04-20T01:19:14.059Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.548875302Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189582024-04-20T01:19:14.059Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.548889442Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"2","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189592024-04-20T01:19:14.059Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549289082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"2","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189602024-04-20T01:19:14.059Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549322372Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"2","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189612024-04-20T01:19:14.059Z{"msg":"[0]R flush_numbers[0..12]: [283, 284, 281, 288, 289, 289, 289, 279, 284, 285, 288, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549337532Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189622024-04-20T01:19:14.059Z{"msg":"[0]R generation[0..12]: [171, 172, 170, 174, 175, 175, 175, 169, 172, 173, 174, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549352932Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189632024-04-20T01:19:14.059Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549367452Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189642024-04-20T01:19:14.059Z{"msg":"[1]R flush_numbers[0..12]: [283, 284, 281, 288, 289, 289, 289, 279, 284, 285, 288, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549381832Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189652024-04-20T01:19:14.059Z{"msg":"[1]R generation[0..12]: [171, 172, 170, 174, 175, 175, 175, 169, 172, 173, 174, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549404663Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189662024-04-20T01:19:14.059Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549419223Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189672024-04-20T01:19:14.059Z{"msg":"[2]R flush_numbers[0..12]: [283, 284, 281, 288, 289, 289, 289, 279, 284, 285, 288, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549433733Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189682024-04-20T01:19:14.059Z{"msg":"[2]R generation[0..12]: [171, 172, 170, 174, 175, 175, 175, 169, 172, 173, 174, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549455453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189692024-04-20T01:19:14.059Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549469793Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189702024-04-20T01:19:14.059Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549483293Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189712024-04-20T01:19:14.059Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
189722024-04-20T01:19:14.059Z,"time":"2024-04-20T01:19:12.549507043Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189732024-04-20T01:19:14.059Z{"msg":"Next flush: 290","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549527403Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189742024-04-20T01:19:14.059Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549541113Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"":"downstairs","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189752024-04-20T01:19:14.059Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549554953Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189762024-04-20T01:19:14.059Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549566993Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"0","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189772024-04-20T01:19:14.059Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549588203Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"1","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189782024-04-20T01:19:14.059Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549601683Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"client":"2","":"downstairs","session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189792024-04-20T01:19:14.060Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549627363Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189802024-04-20T01:19:14.060Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549640293Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189812024-04-20T01:19:14.060Z{"msg":"39d8d347-051b-4f99-a142-5c737a5cd5be is now active with session: 718a22de-252c-4ee1-9579-3ea8685f3fb4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549654123Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189822024-04-20T01:19:14.060Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549665833Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878,"session_id":"718a22de-252c-4ee1-9579-3ea8685f3fb4"}
189832024-04-20T01:19:14.060Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:12.549678883Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878}
189842024-04-20T01:19:14.060ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189852024-04-20T01:19:14.064ZLoading write count information from file "/tmp/test_repair_verify.data"
189862024-04-20T01:19:14.064ZRun Repair workload
189872024-04-20T01:19:14.064Z01/30 Write block 113 len 28672 data: 12 13 13 13 12 14 15
189882024-04-20T01:19:14.064Z02/30 Read block 464 len 32768
189892024-04-20T01:19:14.064Z03/30 Read block 49 len 28672
189902024-04-20T01:19:14.064Z04/30 Flush
189912024-04-20T01:19:14.064Z05/30 Write block 141 len 28672 data: 18 18 19 20 20 16 13
189922024-04-20T01:19:14.064Z06/30 Read block 443 len 12288
189932024-04-20T01:19:14.064Z07/30 Read block 337 len 4096
189942024-04-20T01:19:14.064Z08/30 Flush
189952024-04-20T01:19:14.068Z09/30 Write block 327 len 12288 data: 16 16 13
189962024-04-20T01:19:14.068Z10/30 Read block 557 len 16384
189972024-04-20T01:19:14.076Z11/30 Write block 136 len 32768 data: 12 13 15 15 17 19 19 20
189982024-04-20T01:19:14.076Z12/30 Flush
189992024-04-20T01:19:14.076Z13/30 Write block 267 len 4096 data: 9
190002024-04-20T01:19:14.076Z14/30 Flush
190012024-04-20T01:19:14.080Z15/30 Write block 155 len 16384 data: 14 11 10 12
190022024-04-20T01:19:14.080Z16/30 Read block 155 len 36864
190032024-04-20T01:19:14.080Z17/30 Read block 304 len 4096
190042024-04-20T01:19:14.080Z18/30 Read block 510 len 32768
190052024-04-20T01:19:14.080Z19/30 Read block 559 len 40960
190062024-04-20T01:19:14.080Z20/30 Read block 487 len 4096
190072024-04-20T01:19:14.080Z21/30 Write block 556 len 32768 data: 9 9 7 9 8 9 11 10
190082024-04-20T01:19:14.080Z22/30 Read block 441 len 40960
190092024-04-20T01:19:14.080Z23/30 Flush
190102024-04-20T01:19:14.084Z24/30 Write block 521 len 32768 data: 22 21 20 17 16 15 13 10
190112024-04-20T01:19:14.084Z25/30 Read block 217 len 32768
190122024-04-20T01:19:14.084Z26/30 Flush
190132024-04-20T01:19:14.087Z27/30 Write block 390 len 20480 data: 7 9 10 11 11
190142024-04-20T01:19:14.087Z28/30 Read block 148 len 12288
190152024-04-20T01:19:14.092Z29/30 Write block 126 len 28672 data: 12 13 12 12 14 13 15
190162024-04-20T01:19:14.092Z30/30 Read block 526 len 24576
190172024-04-20T01:19:14.092Z----------------------------------------------------------------
190182024-04-20T01:19:14.092Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
190192024-04-20T01:19:14.092ZStates: Active Active Active
190202024-04-20T01:19:14.092ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
190212024-04-20T01:19:14.092Z 1 Acked 1000 Write 7 Done Done Done false
190222024-04-20T01:19:14.092Z 2 Acked 1001 Read 8 Done Done Done false
190232024-04-20T01:19:14.092Z 3 Acked 1002 Read 7 Done Done Sent false
190242024-04-20T01:19:14.092Z 4 Acked 1003 Flush 0 Done Done Sent false
190252024-04-20T01:19:14.093Z 5 Acked 1004 Write 7 Done Done Sent false
190262024-04-20T01:19:14.093Z 6 Acked 1005 Read 3 Done Done Sent false
190272024-04-20T01:19:14.093Z 7 Acked 1006 Read 1 Done Done Sent false
190282024-04-20T01:19:14.093Z 8 Acked 1007 Flush 0 Done Done Sent false
190292024-04-20T01:19:14.093Z 9 Acked 1008 Write 3 Done Done Sent false
190302024-04-20T01:19:14.093Z 10 Acked 1009 Read 4 Done Done Sent false
190312024-04-20T01:19:14.093Z 11 Acked 1010 Write 8 Done Done Sent false
190322024-04-20T01:19:14.093Z 12 Acked 1011 Flush 0 Done Done Sent false
190332024-04-20T01:19:14.093Z 13 Acked 1012 Write 1 Done Done Sent false
190342024-04-20T01:19:14.093Z 14 Acked 1013 Flush 0 Done Done Sent false
190352024-04-20T01:19:14.093Z 15 Acked 1014 Write 4 Done Done Sent false
190362024-04-20T01:19:14.093Z 16 Acked 1015 Read 9 Done Done Sent false
190372024-04-20T01:19:14.093Z 17 Acked 1016 Read 1 Done Done Sent false
190382024-04-20T01:19:14.093Z 18 Acked 1017 Read 8 Done Done Sent false
190392024-04-20T01:19:14.093Z 19 Acked 1018 Read 10 Done Done Sent false
190402024-04-20T01:19:14.093Z 20 Acked 1019 Read 1 Done Done Sent false
190412024-04-20T01:19:14.093Z 21 Acked 1020 Write 8 Done Done Sent false
190422024-04-20T01:19:14.093Z 22 Acked 1021 Read 10 Done Done Sent false
190432024-04-20T01:19:14.093Z 23 Acked 1022 Flush 0 Done Done Sent false
190442024-04-20T01:19:14.093Z 24 Acked 1023 Write 8 Done Done Sent false
190452024-04-20T01:19:14.093Z 25 Acked 1024 Read 8 Done Done Sent false
190462024-04-20T01:19:14.093Z 26 Acked 1025 Flush 0 Done Done Sent false
190472024-04-20T01:19:14.093Z 27 Acked 1026 Write 5 Done Done Sent false
190482024-04-20T01:19:14.093Z 28 Acked 1027 Read 3 Done Done Sent false
190492024-04-20T01:19:14.094Z 29 Acked 1028 Write 7 Done Done Sent false
190502024-04-20T01:19:14.094Z 30 Acked 1029 Read 6 Done Done Sent false
190512024-04-20T01:19:14.094Z STATES DS:0 DS:1 DS:2 TOTAL
190522024-04-20T01:19:14.094Z New 0 0 0 0
190532024-04-20T01:19:14.094Z Sent 0 0 28 28
190542024-04-20T01:19:14.094Z Done 30 30 2 62
190552024-04-20T01:19:14.094Z Skipped 0 0 0 0
190562024-04-20T01:19:14.094Z Error 0 0 0 0
190572024-04-20T01:19:14.094ZLast Flush: 1025 1025 0
190582024-04-20T01:19:14.094ZDownstairs last five completed:
190592024-04-20T01:19:14.094ZUpstairs last five completed: 30 29 28 27 26
190602024-04-20T01:19:14.094Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:19:12.584933951Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1878}
190612024-04-20T01:19:14.094ZWrote out file "/tmp/test_repair_verify.data"
190622024-04-20T01:19:14.098Z
190632024-04-20T01:19:14.098ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
190642024-04-20T01:19:14.112ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190652024-04-20T01:19:14.112Z 0 000-019 171 171 171 283 283 283 F F F
190662024-04-20T01:19:14.112Z 1 020-039 172 172 172 284 284 284 F F F
190672024-04-20T01:19:14.112Z 2 040-059 170 170 170 281 281 281 F F F
190682024-04-20T01:19:14.116Z 3 060-079 174 174 174 288 288 288 F F F
190692024-04-20T01:19:14.116Z 4 080-099 175 175 175 289 289 289 F F F
190702024-04-20T01:19:14.116Z 5 100-119 176 176 175 290 290 289 F F T <---
190712024-04-20T01:19:14.116Z 6 120-139 176 176 175 292 292 289 T T F <---
190722024-04-20T01:19:14.117Z 7 140-159 176 176 169 294 294 279 F F F <---
190732024-04-20T01:19:14.117Z 8 160-179 172 172 172 284 284 284 F F F
190742024-04-20T01:19:14.117Z 9 180-199 173 173 173 285 285 285 F F F
190752024-04-20T01:19:14.117Z 10 200-219 174 174 174 288 288 288 F F F
190762024-04-20T01:19:14.117Z 11 220-239 169 169 169 279 279 279 F F F
190772024-04-20T01:19:14.117Z 12 240-259 170 170 170 281 281 281 F F F
190782024-04-20T01:19:14.117Z 13 260-279 176 176 170 293 293 282 F F F <---
190792024-04-20T01:19:14.117Z 14 280-299 174 174 174 288 288 288 F F F
190802024-04-20T01:19:14.117Z 15 300-319 174 174 174 288 288 288 F F F
190812024-04-20T01:19:14.117Z 16 320-339 176 176 168 292 292 276 F F F <---
190822024-04-20T01:19:14.117Z 17 340-359 175 175 175 289 289 289 F F F
190832024-04-20T01:19:14.117Z 18 360-379 174 174 174 286 286 286 F F F
190842024-04-20T01:19:14.117Z 19 380-399 174 174 174 286 286 286 T T F <---
190852024-04-20T01:19:14.117Z 20 400-419 174 174 174 286 286 286 F F F
190862024-04-20T01:19:14.117Z 21 420-439 173 173 173 285 285 285 F F F
190872024-04-20T01:19:14.117Z 22 440-459 173 173 173 285 285 285 F F F
190882024-04-20T01:19:14.117Z 23 460-479 174 174 174 288 288 288 F F F
190892024-04-20T01:19:14.117Z 24 480-499 167 167 167 275 275 275 F F F
190902024-04-20T01:19:14.117Z 25 500-519 173 173 173 285 285 285 F F F
190912024-04-20T01:19:14.117Z 26 520-539 176 176 174 295 295 288 F F F <---
190922024-04-20T01:19:14.117Z 27 540-559 176 176 172 294 294 284 F F F <---
190932024-04-20T01:19:14.117Z 28 560-579 176 176 173 294 294 285 F F F <---
190942024-04-20T01:19:14.117Z 29 580-599 173 173 173 285 285 285 F F F
190952024-04-20T01:19:14.117ZMax gen: 176, Max flush: 295
190962024-04-20T01:19:14.118ZError: Difference in extent metadata found!
190972024-04-20T01:19:14.118ZA Difference in extent metadata is expected here
190982024-04-20T01:19:14.118ZOn loop 88
190992024-04-20T01:19:14.118Z
191002024-04-20T01:19:14.121ZVerifying data now
191012024-04-20T01:19:28.551ZNo change after verify
191022024-04-20T01:19:28.551ZLoop: 88 Downstairs dump after verify (and repair):
191032024-04-20T01:19:28.566ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191042024-04-20T01:19:28.566Z 0 000-019 171 171 171 283 283 283 F F F
191052024-04-20T01:19:28.566Z 1 020-039 172 172 172 284 284 284 F F F
191062024-04-20T01:19:28.570Z 2 040-059 170 170 170 281 281 281 F F F
191072024-04-20T01:19:28.570Z 3 060-079 174 174 174 288 288 288 F F F
191082024-04-20T01:19:28.570Z 4 080-099 175 175 175 289 289 289 F F F
191092024-04-20T01:19:28.570Z 5 100-119 176 176 176 290 290 290 F F F
191102024-04-20T01:19:28.570Z 6 120-139 177 177 177 296 296 296 F F F
191112024-04-20T01:19:28.570Z 7 140-159 176 176 176 294 294 294 F F F
191122024-04-20T01:19:28.570Z 8 160-179 172 172 172 284 284 284 F F F
191132024-04-20T01:19:28.570Z 9 180-199 173 173 173 285 285 285 F F F
191142024-04-20T01:19:28.570Z 10 200-219 174 174 174 288 288 288 F F F
191152024-04-20T01:19:28.571Z 11 220-239 169 169 169 279 279 279 F F F
191162024-04-20T01:19:28.571Z 12 240-259 170 170 170 281 281 281 F F F
191172024-04-20T01:19:28.571Z 13 260-279 176 176 176 293 293 293 F F F
191182024-04-20T01:19:28.571Z 14 280-299 174 174 174 288 288 288 F F F
191192024-04-20T01:19:28.571Z 15 300-319 174 174 174 288 288 288 F F F
191202024-04-20T01:19:28.571Z 16 320-339 176 176 176 292 292 292 F F F
191212024-04-20T01:19:28.571Z 17 340-359 175 175 175 289 289 289 F F F
191222024-04-20T01:19:28.571Z 18 360-379 174 174 174 286 286 286 F F F
191232024-04-20T01:19:28.571Z 19 380-399 177 177 177 296 296 296 F F F
191242024-04-20T01:19:28.571Z 20 400-419 174 174 174 286 286 286 F F F
191252024-04-20T01:19:28.571Z 21 420-439 173 173 173 285 285 285 F F F
191262024-04-20T01:19:28.571Z 22 440-459 173 173 173 285 285 285 F F F
191272024-04-20T01:19:28.571Z 23 460-479 174 174 174 288 288 288 F F F
191282024-04-20T01:19:28.571Z 24 480-499 167 167 167 275 275 275 F F F
191292024-04-20T01:19:28.571Z 25 500-519 173 173 173 285 285 285 F F F
191302024-04-20T01:19:28.571Z 26 520-539 176 176 176 295 295 295 F F F
191312024-04-20T01:19:28.571Z 27 540-559 176 176 176 294 294 294 F F F
191322024-04-20T01:19:28.571Z 28 560-579 176 176 176 294 294 294 F F F
191332024-04-20T01:19:28.571Z 29 580-599 173 173 173 285 285 285 F F F
191342024-04-20T01:19:28.571ZMax gen: 177, Max flush: 296
191352024-04-20T01:19:28.571Z
191362024-04-20T01:19:28.571ZBegin loop 89
191372024-04-20T01:19:28.571ZDownstairs to restart: 0
191382024-04-20T01:19:28.588Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.082462523Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886}
191392024-04-20T01:19:28.588Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.082696254Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886}
191402024-04-20T01:19:28.588Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.082713104Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886}
191412024-04-20T01:19:28.594Z{"msg":"Crucible d6efe3a6-44be-41e8-8c5d-8d6183abf846 has session id: d35b0bb7-96a9-409f-953e-9454b55b351f","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.082745904Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191422024-04-20T01:19:28.594Z{Crucible runtime is spawned
191432024-04-20T01:19:28.594Z"msg":"Upstairs opts: Upstairs UUID: d6efe3a6-44be-41e8-8c5d-8d6183abf846, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.082866824Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191442024-04-20T01:19:28.594Z{"msg":"Crucible stats registered with UUID: d6efe3a6-44be-41e8-8c5d-8d6183abf846","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.082919804Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191452024-04-20T01:19:28.594Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.082968944Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886}
191462024-04-20T01:19:28.594Z{"msg":"d6efe3a6-44be-41e8-8c5d-8d6183abf846 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.082990804Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191472024-04-20T01:19:28.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.083014355Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"0","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191482024-04-20T01:19:28.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.083064635Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"0","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191492024-04-20T01:19:28.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.083097885Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"1","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191502024-04-20T01:19:28.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.083137355Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"1","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191512024-04-20T01:19:28.595Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.083166515Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"2","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191522024-04-20T01:19:28.595Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.083218235Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"2","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191532024-04-20T01:19:28.595Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.083249045Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191542024-04-20T01:19:28.595Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.083279945Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191552024-04-20T01:19:28.595Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.083310085Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191562024-04-20T01:19:28.595Z{"msg":"ds_connection connected from Ok(127.0.0.1:63779)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.083340805Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191572024-04-20T01:19:28.595Z{"msg":"ds_connection connected from Ok(127.0.0.1:57064)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.083366605Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191582024-04-20T01:19:28.595Z{"msg":"ds_connection connected from Ok(127.0.0.1:37928)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.083403075Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191592024-04-20T01:19:28.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.084111337Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"1","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191602024-04-20T01:19:28.595Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.084154717Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"1","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191612024-04-20T01:19:28.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.084223077Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"2","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191622024-04-20T01:19:28.595Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.084262787Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"2","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191632024-04-20T01:19:28.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.084320817Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"0","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191642024-04-20T01:19:28.595Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.084389647Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"0","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191652024-04-20T01:19:28.595Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.084713728Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"1","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191662024-04-20T01:19:28.595Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.084763898Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"2","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191672024-04-20T01:19:28.595Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.084959529Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"0","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191682024-04-20T01:19:28.595Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085237829Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"1","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191692024-04-20T01:19:28.595Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085274609Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"1","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191702024-04-20T01:19:28.595Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.08529026Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191712024-04-20T01:19:28.595Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.08531511Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"2","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191722024-04-20T01:19:28.595Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.08539163Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"2","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191732024-04-20T01:19:28.595Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.08545073Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191742024-04-20T01:19:28.595Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.08548219Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"0","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191752024-04-20T01:19:28.596Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.08550742Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"0","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191762024-04-20T01:19:28.596Z{"msg":"[0]R flush_numbers[0..12]: [283, 284, 281, 288, 289, 290, 296, 294, 284, 285, 288, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.08554147Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191772024-04-20T01:19:28.596Z{"msg":"[0]R generation[0..12]: [171, 172, 170, 174, 175, 176, 177, 176, 172, 173, 174, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.08556741Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191782024-04-20T01:19:28.596Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085597Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191792024-04-20T01:19:28.596Z{"msg":"Wait for a query_work_queue command to finish before sending IO
191802024-04-20T01:19:28.596Z[1]R flush_numbers[0..12]: [283, 284, 281, 288, 289, 290, 296, 294, 284, 285, 288, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.08563929Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191812024-04-20T01:19:28.596Z{"msg":"[1]R generation[0..12]: [171, 172, 170, 174, 175, 176, 177, 176, 172, 173, 174, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.08566632Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191822024-04-20T01:19:28.596Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085695841Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191832024-04-20T01:19:28.596Z{"msg":"[2]R flush_numbers[0..12]: [283, 284, 281, 288, 289, 290, 296, 294, 284, 285, 288, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.08572111Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191842024-04-20T01:19:28.596Z{"msg":"[2]R generation[0..12]: [171, 172, 170, 174, 175, 176, 177, 176, 172, 173, 174, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.08574899Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191852024-04-20T01:19:28.596Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085774101Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191862024-04-20T01:19:28.596Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085806141Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191872024-04-20T01:19:28.596Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085830151Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191882024-04-20T01:19:28.596Z{"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085858311Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191892024-04-20T01:19:28.596Z{"msg":"All extents match","v"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191902024-04-20T01:19:28.596Z:0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085898191Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"":"downstairs","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191912024-04-20T01:19:28.596Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085922441Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191922024-04-20T01:19:28.596Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085948401Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"0","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191932024-04-20T01:19:28.596Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085971591Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"1","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191942024-04-20T01:19:28.596Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.085997571Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"client":"2","":"downstairs","session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191952024-04-20T01:19:28.596Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.086020611Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191962024-04-20T01:19:28.596Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.086046631Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191972024-04-20T01:19:28.597Z{"msg":"d6efe3a6-44be-41e8-8c5d-8d6183abf846 is now active with session: d35b0bb7-96a9-409f-953e-9454b55b351f","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.086069171Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191982024-04-20T01:19:28.597Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.086094791Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886,"session_id":"d35b0bb7-96a9-409f-953e-9454b55b351f"}
191992024-04-20T01:19:28.597Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:27.086116311Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886}
192002024-04-20T01:19:28.597ZLoading write count information from file "/tmp/test_repair_verify.data"
192012024-04-20T01:19:28.597ZRun Repair workload
192022024-04-20T01:19:28.597Z01/30 Write block 421 len 4096 data: 11
192032024-04-20T01:19:28.601Z02/30 Write block 406 len 16384 data: 14 11 14 16
192042024-04-20T01:19:28.601Z03/30 Read block 104 len 36864
192052024-04-20T01:19:28.601Z04/30 Write block 14 len 8192 data: 11 13
192062024-04-20T01:19:28.601Z05/30 Write block 326 len 36864 data: 17 17 17 14 16 13 16 13 13
192072024-04-20T01:19:28.601Z06/30 Read block 405 len 20480
192082024-04-20T01:19:28.601Z07/30 Read block 437 len 32768
192092024-04-20T01:19:28.601Z08/30 Flush
192102024-04-20T01:19:28.601Z09/30 Write block 44 len 24576 data: 12 12 13 12 12 14
192112024-04-20T01:19:28.604Z10/30 Write block 300 len 28672 data: 9 11 12 11 12 13 12
192122024-04-20T01:19:28.604Z11/30 Write block 91 len 32768 data: 17 15 15 15 12 13 12 12
192132024-04-20T01:19:28.608Z12/30 Write block 38 len 24576 data: 13 15 13 12 13 11
192142024-04-20T01:19:28.608Z13/30 Write block 257 len 40960 data: 12 12 14 12 9 10 7 6 8 11
192152024-04-20T01:19:28.608Z14/30 Write block 377 len 4096 data: 16
192162024-04-20T01:19:28.608Z15/30 Flush
192172024-04-20T01:19:28.612Z16/30 Write block 195 len 8192 data: 14 12
192182024-04-20T01:19:28.612Z17/30 Read block 444 len 4096
192192024-04-20T01:19:28.612Z18/30 Read block 490 len 32768
192202024-04-20T01:19:28.615Z19/30 Write block 475 len 36864 data: 17 15 17 14 11 14 18 18 17
192212024-04-20T01:19:28.615Z20/30 Read block 174 len 12288
192222024-04-20T01:19:28.615Z21/30 Write block 411 len 16384 data: 17 19 20 19
192232024-04-20T01:19:28.615Z22/30 Read block 364 len 28672
192242024-04-20T01:19:28.619Z23/30 Read block 172 len 8192
192252024-04-20T01:19:28.619Z24/30 Flush
192262024-04-20T01:19:28.622Z25/30 Write block 355 len 36864 data: 15 18 15 15 15 13 12 12 13
192272024-04-20T01:19:28.622Z26/30 Read block 450 len 40960
192282024-04-20T01:19:28.622Z27/30 Read block 365 len 4096
192292024-04-20T01:19:28.622Z28/30 Read block 150 len 12288
192302024-04-20T01:19:28.627Z29/30 Write block 500 len 12288 data: 11 12 13
192312024-04-20T01:19:28.627Z30/30 Write block 192 len 4096 data: 15
192322024-04-20T01:19:28.627Z----------------------------------------------------------------
192332024-04-20T01:19:28.627Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
192342024-04-20T01:19:28.627ZStates: Active Active Active
192352024-04-20T01:19:28.627ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
192362024-04-20T01:19:28.627Z 1 Acked 1000 Write 1 Sent Done Done false
192372024-04-20T01:19:28.627Z 2 Acked 1001 Write 4 Sent Done Done false
192382024-04-20T01:19:28.627Z 3 Acked 1002 Read 9 Sent Done Done false
192392024-04-20T01:19:28.627Z 4 Acked 1003 Write 2 Sent Done Done false
192402024-04-20T01:19:28.627Z 5 Acked 1004 Write 9 Sent Done Done false
192412024-04-20T01:19:28.627Z 6 Acked 1005 Read 5 Sent Done Done false
192422024-04-20T01:19:28.627Z 7 Acked 1006 Read 8 Sent Done Done false
192432024-04-20T01:19:28.627Z 8 Acked 1007 Flush 0 Sent Done Done false
192442024-04-20T01:19:28.627Z 9 Acked 1008 Write 6 Sent Done Done false
192452024-04-20T01:19:28.627Z 10 Acked 1009 Write 7 Sent Done Done false
192462024-04-20T01:19:28.627Z 11 Acked 1010 Write 8 Sent Done Done false
192472024-04-20T01:19:28.627Z 12 Acked 1011 Write 6 Sent Done Done false
192482024-04-20T01:19:28.627Z 13 Acked 1012 Write 10 Sent Done Done false
192492024-04-20T01:19:28.627Z 14 Acked 1013 Write 1 Sent Done Done false
192502024-04-20T01:19:28.628Z 15 Acked 1014 Flush 0 Sent Done Done false
192512024-04-20T01:19:28.628Z 16 Acked 1015 Write 2 Sent Done Done false
192522024-04-20T01:19:28.628Z 17 Acked 1016 Read 1 Sent Done Done false
192532024-04-20T01:19:28.628Z 18 Acked 1017 Read 8 Sent Done Done false
192542024-04-20T01:19:28.628Z 19 Acked 1018 Write 9 Sent Done Done false
192552024-04-20T01:19:28.628Z 20 Acked 1019 Read 3 Sent Done Done false
192562024-04-20T01:19:28.628Z 21 Acked 1020 Write 4 Sent Done Done false
192572024-04-20T01:19:28.628Z 22 Acked 1021 Read 7 Sent Done Done false
192582024-04-20T01:19:28.628Z 23 Acked 1022 Read 2 Sent Done Done false
192592024-04-20T01:19:28.628Z 24 Acked 1023 Flush 0 Sent Done Done false
192602024-04-20T01:19:28.628Z 25 Acked 1024 Write 9 Sent Done Done false
192612024-04-20T01:19:28.628Z 26 Acked 1025 Read 10 Sent Done Done false
192622024-04-20T01:19:28.628Z 27 Acked 1026 Read 1 Sent Done Done false
192632024-04-20T01:19:28.628Z 28 Acked 1027 Read 3 Sent Done Done false
192642024-04-20T01:19:28.628Z 29 Acked 1028 Write 3 Sent Done Done false
192652024-04-20T01:19:28.628Z 30 Acked 1029 Write 1 Sent Sent Sent false
192662024-04-20T01:19:28.628Z STATES DS:0 DS:1 DS:2 TOTAL
192672024-04-20T01:19:28.628Z New 0 0 0 0
192682024-04-20T01:19:28.628Z Sent 30 1 1 32
192692024-04-20T01:19:28.628Z Done 0 29 29 58
192702024-04-20T01:19:28.628Z Skipped 0 0 0 0
192712024-04-20T01:19:28.628Z Error 0 0 0 0
192722024-04-20T01:19:28.628ZLast Flush: 0 1023 1023
192732024-04-20T01:19:28.628ZDownstairs last five completed:
192742024-04-20T01:19:28.629ZUpstairs last five completed: 30 29 28 27 26
192752024-04-20T01:19:28.629Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:19:27.119237635Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1886}
192762024-04-20T01:19:28.629ZWrote out file "/tmp/test_repair_verify.data"
192772024-04-20T01:19:28.632Z
192782024-04-20T01:19:28.632ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192792024-04-20T01:19:28.646ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192802024-04-20T01:19:28.646Z 0 000-019 171 178 178 283 297 297 F F F <---
192812024-04-20T01:19:28.646Z 1 020-039 172 178 178 284 298 298 F F F <---
192822024-04-20T01:19:28.646Z 2 040-059 170 178 178 281 298 298 F F F <---
192832024-04-20T01:19:28.647Z 3 060-079 174 174 174 288 288 288 F F F
192842024-04-20T01:19:28.647Z 4 080-099 175 178 178 289 298 298 F F F <---
192852024-04-20T01:19:28.647Z 5 100-119 176 176 176 290 290 290 F F F
192862024-04-20T01:19:28.651Z 6 120-139 177 177 177 296 296 296 F F F
192872024-04-20T01:19:28.651Z 7 140-159 176 176 176 294 294 294 F F F
192882024-04-20T01:19:28.651Z 8 160-179 172 172 172 284 284 284 F F F
192892024-04-20T01:19:28.651Z 9 180-199 173 178 178 285 299 299 F T T <---
192902024-04-20T01:19:28.651Z 10 200-219 174 174 174 288 288 288 F F F
192912024-04-20T01:19:28.651Z 11 220-239 169 169 169 279 279 279 F F F
192922024-04-20T01:19:28.651Z 12 240-259 170 178 178 281 298 298 F F F <---
192932024-04-20T01:19:28.651Z 13 260-279 176 178 178 293 298 298 F F F <---
192942024-04-20T01:19:28.651Z 14 280-299 174 174 174 288 288 288 F F F
192952024-04-20T01:19:28.651Z 15 300-319 174 178 178 288 298 298 F F F <---
192962024-04-20T01:19:28.651Z 16 320-339 176 178 178 292 297 297 F F F <---
192972024-04-20T01:19:28.652Z 17 340-359 175 175 175 289 289 289 F T T <---
192982024-04-20T01:19:28.652Z 18 360-379 174 178 178 286 298 298 F T T <---
192992024-04-20T01:19:28.652Z 19 380-399 177 177 177 296 296 296 F F F
193002024-04-20T01:19:28.652Z 20 400-419 174 178 178 286 299 299 F F F <---
193012024-04-20T01:19:28.652Z 21 420-439 173 178 178 285 297 297 F F F <---
193022024-04-20T01:19:28.652Z 22 440-459 173 173 173 285 285 285 F F F
193032024-04-20T01:19:28.652Z 23 460-479 174 178 178 288 299 299 F F F <---
193042024-04-20T01:19:28.652Z 24 480-499 167 178 178 275 299 299 F F F <---
193052024-04-20T01:19:28.652Z 25 500-519 173 173 173 285 285 285 F T T <---
193062024-04-20T01:19:28.652Z 26 520-539 176 176 176 295 295 295 F F F
193072024-04-20T01:19:28.652Z 27 540-559 176 176 176 294 294 294 F F F
193082024-04-20T01:19:28.652Z 28 560-579 176 176 176 294 294 294 F F F
193092024-04-20T01:19:28.652Z 29 580-599 173 173 173 285 285 285 F F F
193102024-04-20T01:19:28.652ZMax gen: 178, Max flush: 299
193112024-04-20T01:19:28.652ZError: Difference in extent metadata found!
193122024-04-20T01:19:28.652ZA Difference in extent metadata is expected here
193132024-04-20T01:19:28.652ZOn loop 89
193142024-04-20T01:19:28.652Z
193152024-04-20T01:19:28.656ZVerifying data now
193162024-04-20T01:19:33.364ZNo change after verify
193172024-04-20T01:19:33.365ZLoop: 89 Downstairs dump after verify (and repair):
193182024-04-20T01:19:33.386ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193192024-04-20T01:19:33.386Z 0 000-019 178 178 178 297 297 297 F F F
193202024-04-20T01:19:33.386Z 1 020-039 178 178 178 298 298 298 F F F
193212024-04-20T01:19:33.386Z 2 040-059 178 178 178 298 298 298 F F F
193222024-04-20T01:19:33.386Z 3 060-079 174 174 174 288 288 288 F F F
193232024-04-20T01:19:33.387Z 4 080-099 178 178 178 298 298 298 F F F
193242024-04-20T01:19:33.387Z 5 100-119 176 176 176 290 290 290 F F F
193252024-04-20T01:19:33.387Z 6 120-139 177 177 177 296 296 296 F F F
193262024-04-20T01:19:33.387Z 7 140-159 176 176 176 294 294 294 F F F
193272024-04-20T01:19:33.387Z 8 160-179 172 172 172 284 284 284 F F F
193282024-04-20T01:19:33.387Z 9 180-199 179 179 179 300 300 300 F F F
193292024-04-20T01:19:33.387Z 10 200-219 174 174 174 288 288 288 F F F
193302024-04-20T01:19:33.387Z 11 220-239 169 169 169 279 279 279 F F F
193312024-04-20T01:19:33.387Z 12 240-259 178 178 178 298 298 298 F F F
193322024-04-20T01:19:33.387Z 13 260-279 178 178 178 298 298 298 F F F
193332024-04-20T01:19:33.393Z 14 280-299 174 174 174 288 288 288 F F F
193342024-04-20T01:19:33.393Z 15 300-319 178 178 178 298 298 298 F F F
193352024-04-20T01:19:33.394Z 16 320-339 178 178 178 297 297 297 F F F
193362024-04-20T01:19:33.394Z 17 340-359 179 179 179 300 300 300 F F F
193372024-04-20T01:19:33.394Z 18 360-379 179 179 179 300 300 300 F F F
193382024-04-20T01:19:33.394Z 19 380-399 177 177 177 296 296 296 F F F
193392024-04-20T01:19:33.394Z 20 400-419 178 178 178 299 299 299 F F F
193402024-04-20T01:19:33.394Z 21 420-439 178 178 178 297 297 297 F F F
193412024-04-20T01:19:33.394Z 22 440-459 173 173 173 285 285 285 F F F
193422024-04-20T01:19:33.394Z 23 460-479 178 178 178 299 299 299 F F F
193432024-04-20T01:19:33.394Z 24 480-499 178 178 178 299 299 299 F F F
193442024-04-20T01:19:33.394Z 25 500-519 179 179 179 300 300 300 F F F
193452024-04-20T01:19:33.395Z 26 520-539 176 176 176 295 295 295 F F F
193462024-04-20T01:19:33.395Z 27 540-559 176 176 176 294 294 294 F F F
193472024-04-20T01:19:33.395Z 28 560-579 176 176 176 294 294 294 F F F
193482024-04-20T01:19:33.395Z 29 580-599 173 173 173 285 285 285 F F F
193492024-04-20T01:19:33.395ZMax gen: 179, Max flush: 300
193502024-04-20T01:19:33.395Z
193512024-04-20T01:19:33.395ZBegin loop 90
193522024-04-20T01:19:33.395ZDownstairs to restart: 2
193532024-04-20T01:19:33.408Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.89244908Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894}
193542024-04-20T01:19:33.408Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.892695371Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894}
193552024-04-20T01:19:33.408Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.892728951Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894}
193562024-04-20T01:19:33.408Z{"msg":"Crucible 9558caf3-eabe-43f3-bfc9-58ea96ecf142 has session id: 73cf7fe5-ad10-4848-9ee0-11603119f1b8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.892751761Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193572024-04-20T01:19:33.408Z{Crucible runtime is spawned
193582024-04-20T01:19:33.409Z"msg":"Upstairs opts: Upstairs UUID: 9558caf3-eabe-43f3-bfc9-58ea96ecf142, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.892810761Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193592024-04-20T01:19:33.409Z{"msg":"Crucible stats registered with UUID: 9558caf3-eabe-43f3-bfc9-58ea96ecf142","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.892846051Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193602024-04-20T01:19:33.409Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.892869771Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894}
193612024-04-20T01:19:33.409Z{"msg":"9558caf3-eabe-43f3-bfc9-58ea96ecf142 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.892904221Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193622024-04-20T01:19:33.409Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.892935451Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"0","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193632024-04-20T01:19:33.409Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.892960681Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"0","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193642024-04-20T01:19:33.409Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.892992921Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"1","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193652024-04-20T01:19:33.409Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.893025951Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"1","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193662024-04-20T01:19:33.409Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.893050711Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"2","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193672024-04-20T01:19:33.409Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.893073162Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"2","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193682024-04-20T01:19:33.409Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.893096591Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193692024-04-20T01:19:33.410Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.893132422Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193702024-04-20T01:19:33.410Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.893158192Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193712024-04-20T01:19:33.410Z{"msg":"ds_connection connected from Ok(127.0.0.1:61571)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.893245852Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193722024-04-20T01:19:33.410Z{"msg":"ds_connection connected from Ok(127.0.0.1:53151)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.893287272Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193732024-04-20T01:19:33.410Z{"msg":"ds_connection connected from Ok(127.0.0.1:38731)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.893344132Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193742024-04-20T01:19:33.410Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.894102244Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"0","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193752024-04-20T01:19:33.410Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.894154584Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"0","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193762024-04-20T01:19:33.410Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.894184534Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"1","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193772024-04-20T01:19:33.418Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.894255494Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"1","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193782024-04-20T01:19:33.418Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.894644155Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"0","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193792024-04-20T01:19:33.418Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.894764285Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"2","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193802024-04-20T01:19:33.418Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.894803355Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"2","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193812024-04-20T01:19:33.419Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.894832405Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"1","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193822024-04-20T01:19:33.419Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895068486Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"0","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193832024-04-20T01:19:33.419Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895112816Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"0","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193842024-04-20T01:19:33.419Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895130556Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193852024-04-20T01:19:33.419Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895290866Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"1","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193862024-04-20T01:19:33.419Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895325967Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"1","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193872024-04-20T01:19:33.419Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895341236Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193882024-04-20T01:19:33.419Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895359327Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"2","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193892024-04-20T01:19:33.419Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895775128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"2","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193902024-04-20T01:19:33.419Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895821168Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"2","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193912024-04-20T01:19:33.420Z{"msg":"[0]R flush_numbers[0..12]: [297, 298, 298, 288, 298, 290, 296, 294, 284, 300, 288, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895849688Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193922024-04-20T01:19:33.420Z{"msg":"[0]R generation[0..12]: [178, 178, 178, 174, 178, 176, 177, 176, 172, 179, 174, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895880098Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193932024-04-20T01:19:33.420Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895906008Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193942024-04-20T01:19:33.420Z{"msg":"[1]R flush_numbers[0..12]: [297, 298, 298, 288, 298, 290, 296, 294, 284, 300, 288, 279]"Wait for a query_work_queue command to finish before sending IO
193952024-04-20T01:19:33.420Z,"v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895939438Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193962024-04-20T01:19:33.420Z{"msg":"[1]R generation[0..12]: [178, 178, 178, 174, 178, 176, 177, 176, 172, 179, 174, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895965128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193972024-04-20T01:19:33.420Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.895995118Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193982024-04-20T01:19:33.420Z{"msg":"[2]R flush_numbers[0..12]: [297, 298, 298, 288, 298, 290, 296, 294, 284, 300, 288, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896021448Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
193992024-04-20T01:19:33.420Z{"msg":"[2]R generation[0..12]: [178, 178, 178, 174, 178, 176, 177, 176, 172, 179, 174, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896051798Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194002024-04-20T01:19:33.420Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896112618Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194012024-04-20T01:19:33.420Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896153158Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194022024-04-20T01:19:33.420Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896174718Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194032024-04-20T01:19:33.421Z{"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896215029Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194042024-04-20T01:19:33.421Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
194052024-04-20T01:19:33.421Z"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896246989Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"":"downstairs","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194062024-04-20T01:19:33.421Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896279669Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194072024-04-20T01:19:33.421Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896294709Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"0","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194082024-04-20T01:19:33.421Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896309739Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"1","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194092024-04-20T01:19:33.421Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896323419Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"client":"2","":"downstairs","session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194102024-04-20T01:19:33.421Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896337289Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194112024-04-20T01:19:33.421Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896357689Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194122024-04-20T01:19:33.421Z{"msg":"9558caf3-eabe-43f3-bfc9-58ea96ecf142 is now active with session: 73cf7fe5-ad10-4848-9ee0-11603119f1b8","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896371299Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194132024-04-20T01:19:33.421Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896383189Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894,"session_id":"73cf7fe5-ad10-4848-9ee0-11603119f1b8"}
194142024-04-20T01:19:33.422Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:31.896402799Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894}
194152024-04-20T01:19:33.422ZLoading write count information from file "/tmp/test_repair_verify.data"
194162024-04-20T01:19:33.422ZRun Repair workload
194172024-04-20T01:19:33.422Z01/30 Write block 209 len 4096 data: 13
194182024-04-20T01:19:33.422Z02/30 Write block 584 len 4096 data: 18
194192024-04-20T01:19:33.422Z03/30 Write block 264 len 40960 data: 7 9 12 10 8 9 10 12 13 18
194202024-04-20T01:19:33.422Z04/30 Flush
194212024-04-20T01:19:33.422Z05/30 Write block 213 len 16384 data: 15 14 14 17
194222024-04-20T01:19:33.422Z06/30 Read block 344 len 24576
194232024-04-20T01:19:33.423Z07/30 Read block 255 len 32768
194242024-04-20T01:19:33.423Z08/30 Read block 11 len 20480
194252024-04-20T01:19:33.423Z09/30 Write block 397 len 40960 data: 10 11 13 12 11 12 10 11 11 15
194262024-04-20T01:19:33.423Z10/30 Read block 125 len 20480
194272024-04-20T01:19:33.423Z11/30 Write block 131 len 12288 data: 14 16 13
194282024-04-20T01:19:33.431Z12/30 Read block 245 len 40960
194292024-04-20T01:19:33.431Z13/30 Write block 582 len 4096 data: 16
194302024-04-20T01:19:33.431Z14/30 Write block 405 len 4096 data: 12
194312024-04-20T01:19:33.431Z15/30 Read block 188 len 36864
194322024-04-20T01:19:33.431Z16/30 Flush
194332024-04-20T01:19:33.431Z17/30 Write block 275 len 40960 data: 15 13 15 15 17 18 18 14 10 11
194342024-04-20T01:19:33.431Z18/30 Read block 237 len 4096
194352024-04-20T01:19:33.431Z19/30 Read block 120 len 40960
194362024-04-20T01:19:33.431Z20/30 Write block 119 len 32768 data: 16 14 14 14 14 14 13 13
194372024-04-20T01:19:33.431Z21/30 Write block 448 len 24576 data: 13 14 13 13 12 11
194382024-04-20T01:19:33.431Z22/30 Read block 421 len 12288
194392024-04-20T01:19:33.431Z23/30 Read block 2 len 8192
194402024-04-20T01:19:33.432Z24/30 Write block 427 len 20480 data: 10 10 14 15 14
194412024-04-20T01:19:33.432Z25/30 Write block 442 len 4096 data: 14
194422024-04-20T01:19:33.446Z26/30 Write block 187 len 4096 data: 13
194432024-04-20T01:19:33.446Z27/30 Read block 522 len 16384
194442024-04-20T01:19:33.446Z28/30 Read block 471 len 8192
194452024-04-20T01:19:33.446Z29/30 Write block 209 len 4096 data: 14
194462024-04-20T01:19:33.446Z30/30 Write block 436 len 28672 data: 17 15 12 14 14 15 15
194472024-04-20T01:19:33.446Z----------------------------------------------------------------
194482024-04-20T01:19:33.446Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
194492024-04-20T01:19:33.447ZStates: Active Active Active
194502024-04-20T01:19:33.447ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
194512024-04-20T01:19:33.447Z 1 Acked 1000 Write 1 Done Done Sent false
194522024-04-20T01:19:33.447Z 2 Acked 1001 Write 1 Done Done Sent false
194532024-04-20T01:19:33.447Z 3 Acked 1002 Write 10 Done Done Sent false
194542024-04-20T01:19:33.447Z 4 Acked 1003 Flush 0 Done Done Sent false
194552024-04-20T01:19:33.447Z 5 Acked 1004 Write 4 Done Done Sent false
194562024-04-20T01:19:33.447Z 6 Acked 1005 Read 6 Done Done Sent false
194572024-04-20T01:19:33.447Z 7 Acked 1006 Read 8 Done Done Sent false
194582024-04-20T01:19:33.447Z 8 Acked 1007 Read 5 Done Done Sent false
194592024-04-20T01:19:33.447Z 9 Acked 1008 Write 10 Done Done Sent false
194602024-04-20T01:19:33.447Z 10 Acked 1009 Read 5 Done Done Sent false
194612024-04-20T01:19:33.447Z 11 Acked 1010 Write 3 Done Done Sent false
194622024-04-20T01:19:33.447Z 12 Acked 1011 Read 10 Done Done Sent false
194632024-04-20T01:19:33.447Z 13 Acked 1012 Write 1 Done Done Sent false
194642024-04-20T01:19:33.447Z 14 Acked 1013 Write 1 Done Done Sent false
194652024-04-20T01:19:33.448Z 15 Acked 1014 Read 9 Done Done Sent false
194662024-04-20T01:19:33.448Z 16 Acked 1015 Flush 0 Done Done Sent false
194672024-04-20T01:19:33.448Z 17 Acked 1016 Write 10 Done Done Sent false
194682024-04-20T01:19:33.448Z 18 Acked 1017 Read 1 Done Done Sent false
194692024-04-20T01:19:33.448Z 19 Acked 1018 Read 10 Done Done Sent false
194702024-04-20T01:19:33.448Z 20 Acked 1019 Write 8 Done Done Sent false
194712024-04-20T01:19:33.448Z 21 Acked 1020 Write 6 Done Done Sent false
194722024-04-20T01:19:33.448Z 22 Acked 1021 Read 3 Done Done Sent false
194732024-04-20T01:19:33.448Z 23 Acked 1022 Read 2 Done Done Sent false
194742024-04-20T01:19:33.448Z 24 Acked 1023 Write 5 Done Done Sent false
194752024-04-20T01:19:33.448Z 25 Acked 1024 Write 1 Done Done Sent false
194762024-04-20T01:19:33.448Z 26 Acked 1025 Write 1 Done Done Sent false
194772024-04-20T01:19:33.448Z 27 Acked 1026 Read 4 Done Done Sent false
194782024-04-20T01:19:33.448Z 28 Acked 1027 Read 2 Done Done Sent false
194792024-04-20T01:19:33.449Z 29 Acked 1028 Write 1 Done Done Sent false
194802024-04-20T01:19:33.449Z 30 Acked 1029 Write 7 Sent Sent Sent false
194812024-04-20T01:19:33.449Z STATES DS:0 DS:1 DS:2 TOTAL
194822024-04-20T01:19:33.449Z New 0 0 0 0
194832024-04-20T01:19:33.449Z Sent 1 1 30 32
194842024-04-20T01:19:33.449Z Done 29 29 0 58
194852024-04-20T01:19:33.449Z Skipped 0 0 0 0
194862024-04-20T01:19:33.449Z Error 0 0 0 0
194872024-04-20T01:19:33.449ZLast Flush: 1015 1015 0
194882024-04-20T01:19:33.449ZDownstairs last five completed:
194892024-04-20T01:19:33.449ZUpstairs last five completed: 30 29 28 27 26
194902024-04-20T01:19:33.450Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:19:31.926726556Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1894}
194912024-04-20T01:19:33.450ZWrote out file "/tmp/test_repair_verify.data"
194922024-04-20T01:19:33.450Z
194932024-04-20T01:19:33.462ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194942024-04-20T01:19:33.462ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194952024-04-20T01:19:33.462Z 0 000-019 178 178 178 297 297 297 F F F
194962024-04-20T01:19:33.462Z 1 020-039 178 178 178 298 298 298 F F F
194972024-04-20T01:19:33.462Z 2 040-059 178 178 178 298 298 298 F F F
194982024-04-20T01:19:33.462Z 3 060-079 174 174 174 288 288 288 F F F
194992024-04-20T01:19:33.462Z 4 080-099 178 178 178 298 298 298 F F F
195002024-04-20T01:19:33.462Z 5 100-119 176 176 176 290 290 290 T T F <---
195012024-04-20T01:19:33.462Z 6 120-139 180 180 177 302 302 296 T T F <---
195022024-04-20T01:19:33.462Z 7 140-159 176 176 176 294 294 294 F F F
195032024-04-20T01:19:33.462Z 8 160-179 172 172 172 284 284 284 F F F
195042024-04-20T01:19:33.462Z 9 180-199 179 179 179 300 300 300 T T F <---
195052024-04-20T01:19:33.463Z 10 200-219 180 180 174 302 302 288 T T F <---
195062024-04-20T01:19:33.463Z 11 220-239 169 169 169 279 279 279 F F F
195072024-04-20T01:19:33.463Z 12 240-259 178 178 178 298 298 298 F F F
195082024-04-20T01:19:33.463Z 13 260-279 180 180 178 301 301 298 T T F <---
195092024-04-20T01:19:33.463Z 14 280-299 174 174 174 288 288 288 T T F <---
195102024-04-20T01:19:33.463Z 15 300-319 178 178 178 298 298 298 F F F
195112024-04-20T01:19:33.463Z 16 320-339 178 178 178 297 297 297 F F F
195122024-04-20T01:19:33.464Z 17 340-359 179 179 179 300 300 300 F F F
195132024-04-20T01:19:33.464Z 18 360-379 179 179 179 300 300 300 F F F
195142024-04-20T01:19:33.464Z 19 380-399 180 180 177 302 302 296 F F F <---
195152024-04-20T01:19:33.464Z 20 400-419 180 180 178 302 302 299 F F F <---
195162024-04-20T01:19:33.464Z 21 420-439 178 178 178 297 297 297 T T F <---
195172024-04-20T01:19:33.464Z 22 440-459 173 173 173 285 285 285 T T F <---
195182024-04-20T01:19:33.464Z 23 460-479 178 178 178 299 299 299 F F F
195192024-04-20T01:19:33.464Z 24 480-499 178 178 178 299 299 299 F F F
195202024-04-20T01:19:33.464Z 25 500-519 179 179 179 300 300 300 F F F
195212024-04-20T01:19:33.464Z 26 520-539 176 176 176 295 295 295 F F F
195222024-04-20T01:19:33.465Z 27 540-559 176 176 176 294 294 294 F F F
195232024-04-20T01:19:33.465Z 28 560-579 176 176 176 294 294 294 F F F
195242024-04-20T01:19:33.465Z 29 580-599 180 180 173 302 302 285 F F F <---
195252024-04-20T01:19:33.465ZMax gen: 180, Max flush: 302
195262024-04-20T01:19:33.465ZError: Difference in extent metadata found!
195272024-04-20T01:19:33.472ZA Difference in extent metadata is expected here
195282024-04-20T01:19:33.472ZOn loop 90
195292024-04-20T01:19:33.472Z
195302024-04-20T01:19:33.472ZVerifying data now
195312024-04-20T01:19:33.990ZNo change after verify
195322024-04-20T01:19:33.990ZLoop: 90 Downstairs dump after verify (and repair):
195332024-04-20T01:19:34.005ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
195342024-04-20T01:19:34.005Z 0 000-019 178 178 178 297 297 297 F F F
195352024-04-20T01:19:34.010Z 1 020-039 178 178 178 298 298 298 F F F
195362024-04-20T01:19:34.010Z 2 040-059 178 178 178 298 298 298 F F F
195372024-04-20T01:19:34.010Z 3 060-079 174 174 174 288 288 288 F F F
195382024-04-20T01:19:34.010Z 4 080-099 178 178 178 298 298 298 F F F
195392024-04-20T01:19:34.010Z 5 100-119 181 181 181 303 303 303 F F F
195402024-04-20T01:19:34.010Z 6 120-139 181 181 181 303 303 303 F F F
195412024-04-20T01:19:34.010Z 7 140-159 176 176 176 294 294 294 F F F
195422024-04-20T01:19:34.010Z 8 160-179 172 172 172 284 284 284 F F F
195432024-04-20T01:19:34.010Z 9 180-199 181 181 181 303 303 303 F F F
195442024-04-20T01:19:34.010Z 10 200-219 181 181 181 303 303 303 F F F
195452024-04-20T01:19:34.010Z 11 220-239 169 169 169 279 279 279 F F F
195462024-04-20T01:19:34.011Z 12 240-259 178 178 178 298 298 298 F F F
195472024-04-20T01:19:34.011Z 13 260-279 181 181 181 303 303 303 F F F
195482024-04-20T01:19:34.011Z 14 280-299 181 181 181 303 303 303 F F F
195492024-04-20T01:19:34.011Z 15 300-319 178 178 178 298 298 298 F F F
195502024-04-20T01:19:34.011Z 16 320-339 178 178 178 297 297 297 F F F
195512024-04-20T01:19:34.011Z 17 340-359 179 179 179 300 300 300 F F F
195522024-04-20T01:19:34.011Z 18 360-379 179 179 179 300 300 300 F F F
195532024-04-20T01:19:34.011Z 19 380-399 180 180 180 302 302 302 F F F
195542024-04-20T01:19:34.011Z 20 400-419 180 180 180 302 302 302 F F F
195552024-04-20T01:19:34.011Z 21 420-439 181 181 181 303 303 303 F F F
195562024-04-20T01:19:34.011Z 22 440-459 181 181 181 303 303 303 F F F
195572024-04-20T01:19:34.011Z 23 460-479 178 178 178 299 299 299 F F F
195582024-04-20T01:19:34.011Z 24 480-499 178 178 178 299 299 299 F F F
195592024-04-20T01:19:34.011Z 25 500-519 179 179 179 300 300 300 F F F
195602024-04-20T01:19:34.011Z 26 520-539 176 176 176 295 295 295 F F F
195612024-04-20T01:19:34.011Z 27 540-559 176 176 176 294 294 294 F F F
195622024-04-20T01:19:34.011Z 28 560-579 176 176 176 294 294 294 F F F
195632024-04-20T01:19:34.011Z 29 580-599 180 180 180 302 302 302 F F F
195642024-04-20T01:19:34.011ZMax gen: 181, Max flush: 303
195652024-04-20T01:19:34.011Z
195662024-04-20T01:19:34.011ZBegin loop 91
195672024-04-20T01:19:34.011ZDownstairs to restart: 2
195682024-04-20T01:19:34.028Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522004472Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902}
195692024-04-20T01:19:34.028Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522216092Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902}
195702024-04-20T01:19:34.028Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522242102Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902}
195712024-04-20T01:19:34.035Z{"msg":"Crucible 37914242-bb25-48c1-998a-8aab10e748fe has session id: 4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522276182Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195722024-04-20T01:19:34.035Z{"msg":"Upstairs opts: Upstairs UUID: 37914242-bb25-48c1-998a-8aab10e748fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
195732024-04-20T01:19:34.035Z,"time":"2024-04-20T01:19:32.522322082Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195742024-04-20T01:19:34.035Z{"msg":"Crucible stats registered with UUID: 37914242-bb25-48c1-998a-8aab10e748fe","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522357522Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195752024-04-20T01:19:34.035Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522426412Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902}
195762024-04-20T01:19:34.035Z{"msg":"37914242-bb25-48c1-998a-8aab10e748fe active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522456043Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195772024-04-20T01:19:34.035Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522474723Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"0","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195782024-04-20T01:19:34.035Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522501443Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"0","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195792024-04-20T01:19:34.035Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522545533Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"1","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195802024-04-20T01:19:34.035Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522571243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"1","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195812024-04-20T01:19:34.036Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522595923Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"2","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195822024-04-20T01:19:34.036Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522619963Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"2","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195832024-04-20T01:19:34.036Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522645783Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195842024-04-20T01:19:34.036Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522678253Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195852024-04-20T01:19:34.036Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522705753Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195862024-04-20T01:19:34.036Z{"msg":"ds_connection connected from Ok(127.0.0.1:33915)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522765783Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195872024-04-20T01:19:34.036Z{"msg":"ds_connection connected from Ok(127.0.0.1:45032)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522796653Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195882024-04-20T01:19:34.036Z{"msg":"ds_connection connected from Ok(127.0.0.1:63440)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.522997284Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195892024-04-20T01:19:34.036Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.523536075Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"0","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195902024-04-20T01:19:34.036Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.523572915Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"0","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195912024-04-20T01:19:34.036Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.523599295Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"1","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195922024-04-20T01:19:34.036Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.523643445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"1","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195932024-04-20T01:19:34.036Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.524092036Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"1","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195942024-04-20T01:19:34.036Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.524265957Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"0","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195952024-04-20T01:19:34.036Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.524387907Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"2","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195962024-04-20T01:19:34.036Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.524450317Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"2","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195972024-04-20T01:19:34.036Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.524558327Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"1","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195982024-04-20T01:19:34.036Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.524603637Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"1","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
195992024-04-20T01:19:34.036Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.524709358Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196002024-04-20T01:19:34.036Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.524733658Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"0","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196012024-04-20T01:19:34.036Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.524749368Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"0","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196022024-04-20T01:19:34.036Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.524763428Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196032024-04-20T01:19:34.036Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.524950668Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"2","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196042024-04-20T01:19:34.036Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.525373489Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"2","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196052024-04-20T01:19:34.037Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.525403919Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"2","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196062024-04-20T01:19:34.037Z{"msg":"[0]R flush_numbers[0..12]: [297, 298, 298, 288, 298, 303, 303, 294, 284, 303, 303, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.525419219Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196072024-04-20T01:19:34.037Z{"msg":"[0]R generation[0..12]: [178, 178, 178, 174, 178, 181, 181, 176, 172, 181, 181, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.525433689Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196082024-04-20T01:19:34.037Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.525447879Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196092024-04-20T01:19:34.037Z{"msg":"[1]R flush_numbers[0..12]: [297, 298, 298, 288, 298, 303, 303, 294, 284, 303, 303, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.525470099Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196102024-04-20T01:19:34.037Z{"msg":"[1]R generation[0..12]: [178, 178, 178, 174, 178, 181, 181, 176, 172, 181, 181, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.525484169Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196112024-04-20T01:19:34.037Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.525498109Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196122024-04-20T01:19:34.037Z{"msg":"[2]R flush_numbers[0..12]: [297, 298, 298, 288, 298, 303, 303, 294, 284, 303, 303, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.525518979Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196132024-04-20T01:19:34.037Z{"msg":"[2]R generation[0..12]: [178, 178, 178, 174, 178, 181, 181, 176, 172, 181, 181, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.525532919Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196142024-04-20T01:19:34.037Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0Wait for a query_work_queue command to finish before sending IO
196152024-04-20T01:19:34.037Z,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.525550179Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196162024-04-20T01:19:34.037Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.52556303Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196172024-04-20T01:19:34.037Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.52557675Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196182024-04-20T01:19:34.037Z{"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.52558949Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196192024-04-20T01:19:34.037Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.52560265Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"":"downstairs","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196202024-04-20T01:19:34.037Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.52561555Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196212024-04-20T01:19:34.037Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.52563488Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"0","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196222024-04-20T01:19:34.037Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.52564893Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"1","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196232024-04-20T01:19:34.037Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.52566188Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"client":"2","":"downstairs","session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196242024-04-20T01:19:34.037Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.52568193Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196252024-04-20T01:19:34.037Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.5256943Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196262024-04-20T01:19:34.037Z{"msg":"37914242-bb25-48c1-998a-8aab10e748fe is now active with session: 4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.52570714Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196272024-04-20T01:19:34.037Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.5257191Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902,"session_id":"4bf3e35b-78e0-48f1-aecf-3ad2dc92d3fa"}
196282024-04-20T01:19:34.038Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:32.52573571Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902}
196292024-04-20T01:19:34.038ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
196302024-04-20T01:19:34.038ZLoading write count information from file "/tmp/test_repair_verify.data"
196312024-04-20T01:19:34.038ZRun Repair workload
196322024-04-20T01:19:34.042Z01/30 Write block 101 len 40960 data: 12 12 13 13 12 13 12 12 13 14
196332024-04-20T01:19:34.042Z02/30 Flush
196342024-04-20T01:19:34.042Z03/30 Write block 70 len 32768 data: 13 13 11 14 16 13 9 13
196352024-04-20T01:19:34.042Z04/30 Read block 21 len 8192
196362024-04-20T01:19:34.042Z05/30 Write block 313 len 12288 data: 7 8 9
196372024-04-20T01:19:34.042Z06/30 Read block 105 len 28672
196382024-04-20T01:19:34.042Z07/30 Read block 283 len 24576
196392024-04-20T01:19:34.046Z08/30 Read block 248 len 36864
196402024-04-20T01:19:34.046Z09/30 Flush
196412024-04-20T01:19:34.051Z10/30 Write block 439 len 12288 data: 15 15 16
196422024-04-20T01:19:34.052Z11/30 Read block 154 len 32768
196432024-04-20T01:19:34.052Z12/30 Read block 430 len 20480
196442024-04-20T01:19:34.052Z13/30 Write block 303 len 28672 data: 12 13 14 13 11 9 8
196452024-04-20T01:19:34.055Z14/30 Write block 246 len 20480 data: 13 13 17 17 15
196462024-04-20T01:19:34.055Z15/30 Write block 153 len 4096 data: 10
196472024-04-20T01:19:34.055Z16/30 Read block 419 len 8192
196482024-04-20T01:19:34.055Z17/30 Read block 372 len 36864
196492024-04-20T01:19:34.055Z18/30 Write block 72 len 36864 data: 12 15 17 14 10 14 14 14 12
196502024-04-20T01:19:34.055Z19/30 Flush
196512024-04-20T01:19:34.059Z20/30 Write block 362 len 40960 data: 13 14 11 10 14 19 14 15 17 20
196522024-04-20T01:19:34.059Z21/30 Write block 131 len 40960 data: 15 17 14 11 12 13 14 16 16 18
196532024-04-20T01:19:34.063Z22/30 Read block 444 len 4096
196542024-04-20T01:19:34.063Z23/30 Read block 128 len 28672
196552024-04-20T01:19:34.063Z24/30 Flush
196562024-04-20T01:19:34.067Z25/30 Write block 382 len 36864 data: 16 14 11 9 10 7 7 8 8
196572024-04-20T01:19:34.067Z26/30 Write block 525 len 40960 data: 17 16 14 11 11 11 13 13 12 15
196582024-04-20T01:19:34.067Z27/30 Read block 142 len 36864
196592024-04-20T01:19:34.072Z28/30 Write block 342 len 36864 data: 11 14 13 16 12 12 11 10 8
196602024-04-20T01:19:34.072Z29/30 Write block 537 len 28672 data: 11 10 10 10 11 18 20
196612024-04-20T01:19:34.072Z30/30 Write block 310 len 8192 data: 7 7
196622024-04-20T01:19:34.072Z----------------------------------------------------------------
196632024-04-20T01:19:34.072Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
196642024-04-20T01:19:34.072ZStates: Active Active Active
196652024-04-20T01:19:34.072ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
196662024-04-20T01:19:34.072Z 1 Acked 1000 Write 10 Done Done Sent false
196672024-04-20T01:19:34.072Z 2 Acked 1001 Flush 0 Done Done Sent false
196682024-04-20T01:19:34.072Z 3 Acked 1002 Write 8 Done Done Sent false
196692024-04-20T01:19:34.072Z 4 Acked 1003 Read 2 Done Done Sent false
196702024-04-20T01:19:34.072Z 5 Acked 1004 Write 3 Done Done Sent false
196712024-04-20T01:19:34.072Z 6 Acked 1005 Read 7 Done Done Sent false
196722024-04-20T01:19:34.072Z 7 Acked 1006 Read 6 Done Done Sent false
196732024-04-20T01:19:34.073Z 8 Acked 1007 Read 9 Done Done Sent false
196742024-04-20T01:19:34.073Z 9 Acked 1008 Flush 0 Done Done Sent false
196752024-04-20T01:19:34.073Z 10 Acked 1009 Write 3 Done Done Sent false
196762024-04-20T01:19:34.073Z 11 Acked 1010 Read 8 Done Done Sent false
196772024-04-20T01:19:34.073Z 12 Acked 1011 Read 5 Done Done Sent false
196782024-04-20T01:19:34.073Z 13 Acked 1012 Write 7 Done Done Sent false
196792024-04-20T01:19:34.073Z 14 Acked 1013 Write 5 Done Done Sent false
196802024-04-20T01:19:34.073Z 15 Acked 1014 Write 1 Done Done Sent false
196812024-04-20T01:19:34.073Z 16 Acked 1015 Read 2 Done Done Sent false
196822024-04-20T01:19:34.073Z 17 Acked 1016 Read 9 Done Done Sent false
196832024-04-20T01:19:34.073Z 18 Acked 1017 Write 9 Done Done Sent false
196842024-04-20T01:19:34.073Z 19 Acked 1018 Flush 0 Done Done Sent false
196852024-04-20T01:19:34.073Z 20 Acked 1019 Write 10 Done Done Sent false
196862024-04-20T01:19:34.073Z 21 Acked 1020 Write 10 Done Done Sent false
196872024-04-20T01:19:34.073Z 22 Acked 1021 Read 1 Done Done Sent false
196882024-04-20T01:19:34.073Z 23 Acked 1022 Read 7 Done Done Sent false
196892024-04-20T01:19:34.073Z 24 Acked 1023 Flush 0 Done Done Sent false
196902024-04-20T01:19:34.073Z 25 Acked 1024 Write 9 Done Done Sent false
196912024-04-20T01:19:34.073Z 26 Acked 1025 Write 10 Done Done Sent false
196922024-04-20T01:19:34.073Z 27 Acked 1026 Read 9 Done Done Sent false
196932024-04-20T01:19:34.073Z 28 Acked 1027 Write 9 Done Done Sent false
196942024-04-20T01:19:34.073Z 29 Acked 1028 Write 7 Done Done Sent false
196952024-04-20T01:19:34.073Z 30 Acked 1029 Write 2 Sent Sent Sent false
196962024-04-20T01:19:34.074Z STATES DS:0 DS:1 DS:2 TOTAL
196972024-04-20T01:19:34.074Z New 0 0 0 0
196982024-04-20T01:19:34.074Z Sent 1 1 30 32
196992024-04-20T01:19:34.074Z Done 29 29 0 58
197002024-04-20T01:19:34.074Z Skipped 0 0 0 0
197012024-04-20T01:19:34.074Z Error 0 0 0 0
197022024-04-20T01:19:34.074ZLast Flush: 1023 1023 0
197032024-04-20T01:19:34.074ZDownstairs last five completed:
197042024-04-20T01:19:34.074ZUpstairs last five completed: 30 29 28 27 26
197052024-04-20T01:19:34.074Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:19:32.564796167Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1902}
197062024-04-20T01:19:34.074ZWrote out file "/tmp/test_repair_verify.data"
197072024-04-20T01:19:34.078Z
197082024-04-20T01:19:34.078ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
197092024-04-20T01:19:34.096ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197102024-04-20T01:19:34.096Z 0 000-019 178 178 178 297 297 297 F F F
197112024-04-20T01:19:34.100Z 1 020-039 178 178 178 298 298 298 F F F
197122024-04-20T01:19:34.101Z 2 040-059 178 178 178 298 298 298 F F F
197132024-04-20T01:19:34.101Z 3 060-079 182 182 174 306 306 288 F F F <---
197142024-04-20T01:19:34.101Z 4 080-099 182 182 178 306 306 298 F F F <---
197152024-04-20T01:19:34.101Z 5 100-119 182 182 181 304 304 303 F F F <---
197162024-04-20T01:19:34.101Z 6 120-139 182 182 181 307 307 303 F F F <---
197172024-04-20T01:19:34.101Z 7 140-159 182 182 176 307 307 294 F F F <---
197182024-04-20T01:19:34.101Z 8 160-179 172 172 172 284 284 284 F F F
197192024-04-20T01:19:34.101Z 9 180-199 181 181 181 303 303 303 F F F
197202024-04-20T01:19:34.101Z 10 200-219 181 181 181 303 303 303 F F F
197212024-04-20T01:19:34.101Z 11 220-239 169 169 169 279 279 279 F F F
197222024-04-20T01:19:34.101Z 12 240-259 182 182 178 306 306 298 F F F <---
197232024-04-20T01:19:34.101Z 13 260-279 181 181 181 303 303 303 F F F
197242024-04-20T01:19:34.101Z 14 280-299 181 181 181 303 303 303 F F F
197252024-04-20T01:19:34.101Z 15 300-319 182 182 178 306 306 298 T T F <---
197262024-04-20T01:19:34.101Z 16 320-339 178 178 178 297 297 297 F F F
197272024-04-20T01:19:34.101Z 17 340-359 179 179 179 300 300 300 T T F <---
197282024-04-20T01:19:34.101Z 18 360-379 182 182 179 307 307 300 F F F <---
197292024-04-20T01:19:34.101Z 19 380-399 180 180 180 302 302 302 T T F <---
197302024-04-20T01:19:34.101Z 20 400-419 180 180 180 302 302 302 F F F
197312024-04-20T01:19:34.101Z 21 420-439 182 182 181 306 306 303 F F F <---
197322024-04-20T01:19:34.101Z 22 440-459 182 182 181 306 306 303 F F F <---
197332024-04-20T01:19:34.101Z 23 460-479 178 178 178 299 299 299 F F F
197342024-04-20T01:19:34.102Z 24 480-499 178 178 178 299 299 299 F F F
197352024-04-20T01:19:34.102Z 25 500-519 179 179 179 300 300 300 F F F
197362024-04-20T01:19:34.102Z 26 520-539 176 176 176 295 295 295 T T F <---
197372024-04-20T01:19:34.102Z 27 540-559 176 176 176 294 294 294 T T F <---
197382024-04-20T01:19:34.102Z 28 560-579 176 176 176 294 294 294 F F F
197392024-04-20T01:19:34.102Z 29 580-599 180 180 180 302 302 302 F F F
197402024-04-20T01:19:34.102ZMax gen: 182, Max flush: 307
197412024-04-20T01:19:34.102ZError: Difference in extent metadata found!
197422024-04-20T01:19:34.102ZA Difference in extent metadata is expected here
197432024-04-20T01:19:34.102ZOn loop 91
197442024-04-20T01:19:34.102Z
197452024-04-20T01:19:34.102ZVerifying data now
197462024-04-20T01:19:38.748ZNo change after verify
197472024-04-20T01:19:38.748ZLoop: 91 Downstairs dump after verify (and repair):
197482024-04-20T01:19:38.762ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197492024-04-20T01:19:38.762Z 0 000-019 178 178 178 297 297 297 F F F
197502024-04-20T01:19:38.762Z 1 020-039 178 178 178 298 298 298 F F F
197512024-04-20T01:19:38.763Z 2 040-059 178 178 178 298 298 298 F F F
197522024-04-20T01:19:38.763Z 3 060-079 182 182 182 306 306 306 F F F
197532024-04-20T01:19:38.763Z 4 080-099 182 182 182 306 306 306 F F F
197542024-04-20T01:19:38.763Z 5 100-119 182 182 182 304 304 304 F F F
197552024-04-20T01:19:38.763Z 6 120-139 182 182 182 307 307 307 F F F
197562024-04-20T01:19:38.763Z 7 140-159 182 182 182 307 307 307 F F F
197572024-04-20T01:19:38.764Z 8 160-179 172 172 172 284 284 284 F F F
197582024-04-20T01:19:38.764Z 9 180-199 181 181 181 303 303 303 F F F
197592024-04-20T01:19:38.764Z 10 200-219 181 181 181 303 303 303 F F F
197602024-04-20T01:19:38.764Z 11 220-239 169 169 169 279 279 279 F F F
197612024-04-20T01:19:38.764Z 12 240-259 182 182 182 306 306 306 F F F
197622024-04-20T01:19:38.764Z 13 260-279 181 181 181 303 303 303 F F F
197632024-04-20T01:19:38.764Z 14 280-299 181 181 181 303 303 303 F F F
197642024-04-20T01:19:38.764Z 15 300-319 183 183 183 308 308 308 F F F
197652024-04-20T01:19:38.764Z 16 320-339 178 178 178 297 297 297 F F F
197662024-04-20T01:19:38.765Z 17 340-359 183 183 183 308 308 308 F F F
197672024-04-20T01:19:38.765Z 18 360-379 182 182 182 307 307 307 F F F
197682024-04-20T01:19:38.765Z 19 380-399 183 183 183 308 308 308 F F F
197692024-04-20T01:19:38.765Z 20 400-419 180 180 180 302 302 302 F F F
197702024-04-20T01:19:38.765Z 21 420-439 182 182 182 306 306 306 F F F
197712024-04-20T01:19:38.765Z 22 440-459 182 182 182 306 306 306 F F F
197722024-04-20T01:19:38.765Z 23 460-479 178 178 178 299 299 299 F F F
197732024-04-20T01:19:38.765Z 24 480-499 178 178 178 299 299 299 F F F
197742024-04-20T01:19:38.765Z 25 500-519 179 179 179 300 300 300 F F F
197752024-04-20T01:19:38.765Z 26 520-539 183 183 183 308 308 308 F F F
197762024-04-20T01:19:38.766Z 27 540-559 183 183 183 308 308 308 F F F
197772024-04-20T01:19:38.766Z 28 560-579 176 176 176 294 294 294 F F F
197782024-04-20T01:19:38.766Z 29 580-599 180 180 180 302 302 302 F F F
197792024-04-20T01:19:38.766ZMax gen: 183, Max flush: 308
197802024-04-20T01:19:38.766Z
197812024-04-20T01:19:38.767ZBegin loop 92
197822024-04-20T01:19:38.767ZDownstairs to restart: 2
197832024-04-20T01:19:38.780Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.268713918Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910}
197842024-04-20T01:19:38.781Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.268998378Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910}
197852024-04-20T01:19:38.796Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.269031618Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910}
197862024-04-20T01:19:38.796Z{"msg":"Crucible de88a9ef-e5f5-440b-883b-9648a838aa03 has session id: 77d604c6-9e1b-4411-8a13-bef358f66d22","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.269054369Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
197872024-04-20T01:19:38.796Z{"msg":"Upstairs opts: Upstairs UUID: de88a9ef-e5f5-440b-883b-9648a838aa03, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.269164429Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
197882024-04-20T01:19:38.796ZCrucible runtime is spawned
197892024-04-20T01:19:38.797Z{"msg":"Crucible stats registered with UUID: de88a9ef-e5f5-440b-883b-9648a838aa03","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.269204139Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
197902024-04-20T01:19:38.797Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.269251669Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910}
197912024-04-20T01:19:38.797Z{"msg":"de88a9ef-e5f5-440b-883b-9648a838aa03 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.269346849Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
197922024-04-20T01:19:38.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.269371629Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"0","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
197932024-04-20T01:19:38.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.269388529Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"0","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
197942024-04-20T01:19:38.798Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.269424699Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"1","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
197952024-04-20T01:19:38.798Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.269448839Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"1","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
197962024-04-20T01:19:38.798Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.269495759Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"2","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
197972024-04-20T01:19:38.798Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.269523179Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"2","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
197982024-04-20T01:19:38.798Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.26955559Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
197992024-04-20T01:19:38.798Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.26957938Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198002024-04-20T01:19:38.798Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.26961876Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198012024-04-20T01:19:38.799Z{"msg":"ds_connection connected from Ok(127.0.0.1:57546)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.26965615Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198022024-04-20T01:19:38.799Z{"msg":"ds_connection connected from Ok(127.0.0.1:45467)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.26969388Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198032024-04-20T01:19:38.799Z{"msg":"ds_connection connected from Ok(127.0.0.1:37905)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.26972142Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198042024-04-20T01:19:38.799Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.270366661Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"1","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198052024-04-20T01:19:38.799Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.270412612Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"1","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198062024-04-20T01:19:38.800Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.270436221Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"0","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198072024-04-20T01:19:38.800Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.270502832Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"0","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198082024-04-20T01:19:38.800Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.270998483Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"1","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198092024-04-20T01:19:38.800Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.271045363Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"0","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198102024-04-20T01:19:38.800Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.271063743Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"2","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198112024-04-20T01:19:38.800Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.271087663Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"2","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198122024-04-20T01:19:38.800Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.271562914Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"1","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198132024-04-20T01:19:38.800Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.271626834Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"1","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198142024-04-20T01:19:38.800Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.271667534Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198152024-04-20T01:19:38.801Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.271702364Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"2","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198162024-04-20T01:19:38.801Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.271728424Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"0","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198172024-04-20T01:19:38.801Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.271770364Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"0","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198182024-04-20T01:19:38.801Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.271796075Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198192024-04-20T01:19:38.801Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272126825Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"2","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198202024-04-20T01:19:38.801Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272168985Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"2","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198212024-04-20T01:19:38.801Z{"msg":"[0]R flush_numbers[0..12]: [297, 298, 298, 306, 306, 304, 307, 307, 284, 303, 303, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272185856Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198222024-04-20T01:19:38.801Z{"msg":"[0]R generation[0..12]: [178, 178, 178, 182, 182, 182, 182, 182, 172, 181, 181, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272208605Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198232024-04-20T01:19:38.802Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272223316Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198242024-04-20T01:19:38.802Z{"msg":"[1]R flush_numbers[0..12]: [297, 298, 298, 306, 306, 304, 307, 307, 284, 303, 303, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272248325Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198252024-04-20T01:19:38.802Z{"msg":"[1]R generation[0..12]: [178, 178, 178, 182, 182, 182, 182, 182, 172, 181, 181, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272262766Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198262024-04-20T01:19:38.802Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272289716Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198272024-04-20T01:19:38.802Z{"msg":"[2]R flush_numbers[0..12]: [297, 298, 298, 306, 306, 304, 307, 307, 284, 303, 303, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272303816Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198282024-04-20T01:19:38.802ZWait for a query_work_queue command to finish before sending IO
198292024-04-20T01:19:38.802Z{"msg":"[2]R generation[0..12]: [178, 178, 178, 182, 182, 182, 182, 182, 172, 181, 181, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272327856Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198302024-04-20T01:19:38.802Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272343006Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198312024-04-20T01:19:38.802Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272363516Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198322024-04-20T01:19:38.802Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272377176Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198332024-04-20T01:19:38.802Z{"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272390606Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198342024-04-20T01:19:38.802Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272403916Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"":"downstairs","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198352024-04-20T01:19:38.802Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272419556Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198362024-04-20T01:19:38.803Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272444396Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"0","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198372024-04-20T01:19:38.803Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272458976Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"1","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198382024-04-20T01:19:38.803Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272479666Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"client":"2","":"downstairs","session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198392024-04-20T01:19:38.803Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272493416Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198402024-04-20T01:19:38.803Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272513096Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198412024-04-20T01:19:38.803Z{"msg":"de88a9ef-e5f5-440b-883b-9648a838aa03 is now active with session: 77d604c6-9e1b-4411-8a13-bef358f66d22","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272526136Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198422024-04-20T01:19:38.803Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272544586Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910,"session_id":"77d604c6-9e1b-4411-8a13-bef358f66d22"}
198432024-04-20T01:19:38.803Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:37.272557126Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910}
198442024-04-20T01:19:38.803ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
198452024-04-20T01:19:38.804ZLoading write count information from file "/tmp/test_repair_verify.data"
198462024-04-20T01:19:38.804ZRun Repair workload
198472024-04-20T01:19:38.804Z01/30 Write block 180 len 8192 data: 17 18
198482024-04-20T01:19:38.804Z02/30 Write block 355 len 4096 data: 16
198492024-04-20T01:19:38.804Z03/30 Read block 202 len 24576
198502024-04-20T01:19:38.804Z04/30 Read block 36 len 40960
198512024-04-20T01:19:38.804Z05/30 Write block 574 len 12288 data: 9 9 10
198522024-04-20T01:19:38.804Z06/30 Read block 190 len 4096
198532024-04-20T01:19:38.804Z07/30 Flush
198542024-04-20T01:19:38.813Z08/30 Write block 156 len 16384 data: 12 11 13 13
198552024-04-20T01:19:38.813Z09/30 Read block 452 len 12288
198562024-04-20T01:19:38.813Z10/30 Write block 582 len 24576 data: 17 14 19 21 23 21
198572024-04-20T01:19:38.813Z11/30 Read block 347 len 32768
198582024-04-20T01:19:38.813Z12/30 Read block 203 len 20480
198592024-04-20T01:19:38.813Z13/30 Read block 206 len 20480
198602024-04-20T01:19:38.813Z14/30 Flush
198612024-04-20T01:19:38.814Z15/30 Write block 179 len 8192 data: 18 18
198622024-04-20T01:19:38.814Z16/30 Write block 36 len 28672 data: 10 12 14 16 14 13 14
198632024-04-20T01:19:38.814Z17/30 Write block 114 len 36864 data: 14 14 14 13 15 17 15 15 15
198642024-04-20T01:19:38.814Z18/30 Read block 320 len 40960
198652024-04-20T01:19:38.814Z19/30 Read block 299 len 8192
198662024-04-20T01:19:38.814Z20/30 Read block 541 len 8192
198672024-04-20T01:19:38.814Z21/30 Write block 445 len 16384 data: 12 12 13 14
198682024-04-20T01:19:38.814Z22/30 Flush
198692024-04-20T01:19:38.814Z23/30 Write block 187 len 12288 data: 14 16 15
198702024-04-20T01:19:38.814Z24/30 Read block 359 len 32768
198712024-04-20T01:19:38.814Z25/30 Read block 538 len 4096
198722024-04-20T01:19:38.814Z26/30 Write block 279 len 24576 data: 18 19 19 15 11 12
198732024-04-20T01:19:38.814Z27/30 Read block 185 len 28672
198742024-04-20T01:19:38.814Z28/30 Write block 115 len 8192 data: 15 15
198752024-04-20T01:19:38.814Z29/30 Read block 482 len 40960
198762024-04-20T01:19:38.815Z30/30 Read block 508 len 8192
198772024-04-20T01:19:38.815Z----------------------------------------------------------------
198782024-04-20T01:19:38.815Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
198792024-04-20T01:19:38.815ZStates: Active Active Active
198802024-04-20T01:19:38.815ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
198812024-04-20T01:19:38.815Z 1 Acked 1000 Write 2 Done Done Sent false
198822024-04-20T01:19:38.815Z 2 Acked 1001 Write 1 Done Done Sent false
198832024-04-20T01:19:38.815Z 3 Acked 1002 Read 6 Done Done Sent false
198842024-04-20T01:19:38.815Z 4 Acked 1003 Read 10 Done Done Sent false
198852024-04-20T01:19:38.815Z 5 Acked 1004 Write 3 Done Done Sent false
198862024-04-20T01:19:38.815Z 6 Acked 1005 Read 1 Done Done Sent false
198872024-04-20T01:19:38.815Z 7 Acked 1006 Flush 0 Done Done Sent false
198882024-04-20T01:19:38.815Z 8 Acked 1007 Write 4 Done Done Sent false
198892024-04-20T01:19:38.816Z 9 Acked 1008 Read 3 Done Done Sent false
198902024-04-20T01:19:38.816Z 10 Acked 1009 Write 6 Done Done Sent false
198912024-04-20T01:19:38.816Z 11 Acked 1010 Read 8 Done Done Sent false
198922024-04-20T01:19:38.816Z 12 Acked 1011 Read 5 Done Done Sent false
198932024-04-20T01:19:38.816Z 13 Acked 1012 Read 5 Done Done Sent false
198942024-04-20T01:19:38.816Z 14 Acked 1013 Flush 0 Done Done Sent false
198952024-04-20T01:19:38.816Z 15 Acked 1014 Write 2 Done Done Sent false
198962024-04-20T01:19:38.816Z 16 Acked 1015 Write 7 Done Done Sent false
198972024-04-20T01:19:38.816Z 17 Acked 1016 Write 9 Done Done Sent false
198982024-04-20T01:19:38.816Z 18 Acked 1017 Read 10 Done Done Sent false
198992024-04-20T01:19:38.816Z 19 Acked 1018 Read 2 Done Done Sent false
199002024-04-20T01:19:38.816Z 20 Acked 1019 Read 2 Done Done Sent false
199012024-04-20T01:19:38.816Z 21 Acked 1020 Write 4 Done Done Sent false
199022024-04-20T01:19:38.816Z 22 Acked 1021 Flush 0 Done Done Sent false
199032024-04-20T01:19:38.816Z 23 Acked 1022 Write 3 Done Done Sent false
199042024-04-20T01:19:38.816Z 24 Acked 1023 Read 8 Done Done Sent false
199052024-04-20T01:19:38.817Z 25 Acked 1024 Read 1 Done Done Sent false
199062024-04-20T01:19:38.817Z 26 Acked 1025 Write 6 Done Done Sent false
199072024-04-20T01:19:38.817Z 27 Acked 1026 Read 7 Done Done Sent false
199082024-04-20T01:19:38.817Z 28 Acked 1027 Write 2 Done Done Sent false
199092024-04-20T01:19:38.817Z 29 Acked 1028 Read 10 Done Done Sent false
199102024-04-20T01:19:38.817Z 30 Acked 1029 Read 2 Done Done Sent false
199112024-04-20T01:19:38.817Z STATES DS:0 DS:1 DS:2 TOTAL
199122024-04-20T01:19:38.817Z New 0 0 0 0
199132024-04-20T01:19:38.817Z Sent 0 0 30 30
199142024-04-20T01:19:38.817Z Done 30 30 0 60
199152024-04-20T01:19:38.817Z Skipped 0 0 0 0
199162024-04-20T01:19:38.818Z Error 0 0 0 0
199172024-04-20T01:19:38.818ZLast Flush: 1021 1021 0
199182024-04-20T01:19:38.818ZDownstairs last five completed:
199192024-04-20T01:19:38.818ZUpstairs last five completed: 30 29 28 27 26
199202024-04-20T01:19:38.818Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:19:37.302916564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1910}
199212024-04-20T01:19:38.818ZWrote out file "/tmp/test_repair_verify.data"
199222024-04-20T01:19:38.826Z
199232024-04-20T01:19:38.827ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
199242024-04-20T01:19:38.840ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199252024-04-20T01:19:38.841Z 0 000-019 178 178 178 297 297 297 F F F
199262024-04-20T01:19:38.841Z 1 020-039 184 184 178 311 311 298 F F F <---
199272024-04-20T01:19:38.841Z 2 040-059 184 184 178 311 311 298 F F F <---
199282024-04-20T01:19:38.841Z 3 060-079 182 182 182 306 306 306 F F F
199292024-04-20T01:19:38.841Z 4 080-099 182 182 182 306 306 306 F F F
199302024-04-20T01:19:38.841Z 5 100-119 184 184 182 311 311 304 T T F <---
199312024-04-20T01:19:38.841Z 6 120-139 184 184 182 311 311 307 F F F <---
199322024-04-20T01:19:38.841Z 7 140-159 184 184 182 310 310 307 F F F <---
199332024-04-20T01:19:38.841Z 8 160-179 184 184 172 311 311 284 F F F <---
199342024-04-20T01:19:38.841Z 9 180-199 184 184 181 311 311 303 T T F <---
199352024-04-20T01:19:38.841Z 10 200-219 181 181 181 303 303 303 F F F
199362024-04-20T01:19:38.841Z 11 220-239 169 169 169 279 279 279 F F F
199372024-04-20T01:19:38.841Z 12 240-259 182 182 182 306 306 306 F F F
199382024-04-20T01:19:38.841Z 13 260-279 181 181 181 303 303 303 T T F <---
199392024-04-20T01:19:38.842Z 14 280-299 181 181 181 303 303 303 T T F <---
199402024-04-20T01:19:38.842Z 15 300-319 183 183 183 308 308 308 F F F
199412024-04-20T01:19:38.842Z 16 320-339 178 178 178 297 297 297 F F F
199422024-04-20T01:19:38.842Z 17 340-359 184 184 183 309 309 308 F F F <---
199432024-04-20T01:19:38.842Z 18 360-379 182 182 182 307 307 307 F F F
199442024-04-20T01:19:38.842Z 19 380-399 183 183 183 308 308 308 F F F
199452024-04-20T01:19:38.842Z 20 400-419 180 180 180 302 302 302 F F F
199462024-04-20T01:19:38.842Z 21 420-439 182 182 182 306 306 306 F F F
199472024-04-20T01:19:38.842Z 22 440-459 184 184 182 311 311 306 F F F <---
199482024-04-20T01:19:38.842Z 23 460-479 178 178 178 299 299 299 F F F
199492024-04-20T01:19:38.842Z 24 480-499 178 178 178 299 299 299 F F F
199502024-04-20T01:19:38.842Z 25 500-519 179 179 179 300 300 300 F F F
199512024-04-20T01:19:38.842Z 26 520-539 183 183 183 308 308 308 F F F
199522024-04-20T01:19:38.843Z 27 540-559 183 183 183 308 308 308 F F F
199532024-04-20T01:19:38.843Z 28 560-579 184 184 176 309 309 294 F F F <---
199542024-04-20T01:19:38.843Z 29 580-599 184 184 180 310 310 302 F F F <---
199552024-04-20T01:19:38.843ZMax gen: 184, Max flush: 311
199562024-04-20T01:19:38.843ZError: Difference in extent metadata found!
199572024-04-20T01:19:38.851ZA Difference in extent metadata is expected here
199582024-04-20T01:19:38.851ZOn loop 92
199592024-04-20T01:19:38.851Z
199602024-04-20T01:19:38.852ZVerifying data now
199612024-04-20T01:19:43.423ZNo change after verify
199622024-04-20T01:19:43.423ZLoop: 92 Downstairs dump after verify (and repair):
199632024-04-20T01:19:43.437ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199642024-04-20T01:19:43.438Z 0 000-019 178 178 178 297 297 297 F F F
199652024-04-20T01:19:43.438Z 1 020-039 184 184 184 311 311 311 F F F
199662024-04-20T01:19:43.438Z 2 040-059 184 184 184 311 311 311 F F F
199672024-04-20T01:19:43.438Z 3 060-079 182 182 182 306 306 306 F F F
199682024-04-20T01:19:43.438Z 4 080-099 182 182 182 306 306 306 F F F
199692024-04-20T01:19:43.441Z 5 100-119 185 185 185 312 312 312 F F F
199702024-04-20T01:19:43.442Z 6 120-139 184 184 184 311 311 311 F F F
199712024-04-20T01:19:43.442Z 7 140-159 184 184 184 310 310 310 F F F
199722024-04-20T01:19:43.442Z 8 160-179 184 184 184 311 311 311 F F F
199732024-04-20T01:19:43.442Z 9 180-199 185 185 185 312 312 312 F F F
199742024-04-20T01:19:43.442Z 10 200-219 181 181 181 303 303 303 F F F
199752024-04-20T01:19:43.442Z 11 220-239 169 169 169 279 279 279 F F F
199762024-04-20T01:19:43.442Z 12 240-259 182 182 182 306 306 306 F F F
199772024-04-20T01:19:43.442Z 13 260-279 185 185 185 312 312 312 F F F
199782024-04-20T01:19:43.442Z 14 280-299 185 185 185 312 312 312 F F F
199792024-04-20T01:19:43.442Z 15 300-319 183 183 183 308 308 308 F F F
199802024-04-20T01:19:43.442Z 16 320-339 178 178 178 297 297 297 F F F
199812024-04-20T01:19:43.442Z 17 340-359 184 184 184 309 309 309 F F F
199822024-04-20T01:19:43.442Z 18 360-379 182 182 182 307 307 307 F F F
199832024-04-20T01:19:43.442Z 19 380-399 183 183 183 308 308 308 F F F
199842024-04-20T01:19:43.442Z 20 400-419 180 180 180 302 302 302 F F F
199852024-04-20T01:19:43.442Z 21 420-439 182 182 182 306 306 306 F F F
199862024-04-20T01:19:43.442Z 22 440-459 184 184 184 311 311 311 F F F
199872024-04-20T01:19:43.442Z 23 460-479 178 178 178 299 299 299 F F F
199882024-04-20T01:19:43.442Z 24 480-499 178 178 178 299 299 299 F F F
199892024-04-20T01:19:43.442Z 25 500-519 179 179 179 300 300 300 F F F
199902024-04-20T01:19:43.442Z 26 520-539 183 183 183 308 308 308 F F F
199912024-04-20T01:19:43.442Z 27 540-559 183 183 183 308 308 308 F F F
199922024-04-20T01:19:43.442Z 28 560-579 184 184 184 309 309 309 F F F
199932024-04-20T01:19:43.443Z 29 580-599 184 184 184 310 310 310 F F F
199942024-04-20T01:19:43.443ZMax gen: 185, Max flush: 312
199952024-04-20T01:19:43.443Z
199962024-04-20T01:19:43.443ZBegin loop 93
199972024-04-20T01:19:43.443ZDownstairs to restart: 1
199982024-04-20T01:19:43.461Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.955731791Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918}
199992024-04-20T01:19:43.461Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.955976542Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918}
200002024-04-20T01:19:43.467Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956010042Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918}
200012024-04-20T01:19:43.467Z{"msg":"Crucible 15a924ab-2281-409b-becc-82834c3f7750 has session id: 7d2cfcfc-c590-49b0-ad23-25a61ff88344","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956043122Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200022024-04-20T01:19:43.467Z{"Crucible runtime is spawned
200032024-04-20T01:19:43.467Zmsg":"Upstairs opts: Upstairs UUID: 15a924ab-2281-409b-becc-82834c3f7750, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956155352Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200042024-04-20T01:19:43.467Z{"msg":"Crucible stats registered with UUID: 15a924ab-2281-409b-becc-82834c3f7750","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956202982Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200052024-04-20T01:19:43.467Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956236172Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918}
200062024-04-20T01:19:43.467Z{"msg":"15a924ab-2281-409b-becc-82834c3f7750 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956256492Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200072024-04-20T01:19:43.467Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956286452Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"0","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200082024-04-20T01:19:43.467Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956310732Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"0","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200092024-04-20T01:19:43.467Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956333682Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"1","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200102024-04-20T01:19:43.467Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956356872Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"1","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200112024-04-20T01:19:43.467Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956397323Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"2","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200122024-04-20T01:19:43.467Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956492963Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"2","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200132024-04-20T01:19:43.468Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956548723Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200142024-04-20T01:19:43.468Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956579623Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200152024-04-20T01:19:43.468Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956615893Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200162024-04-20T01:19:43.468Z{"msg":"ds_connection connected from Ok(127.0.0.1:60095)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956644903Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200172024-04-20T01:19:43.468Z{"msg":"ds_connection connected from Ok(127.0.0.1:51449)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956685253Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200182024-04-20T01:19:43.468Z{"msg":"ds_connection connected from Ok(127.0.0.1:56516)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.956713293Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200192024-04-20T01:19:43.468Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.957398715Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"0","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200202024-04-20T01:19:43.468Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.957480945Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"0","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200212024-04-20T01:19:43.468Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.957520145Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"2","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200222024-04-20T01:19:43.468Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.957555665Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"2","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200232024-04-20T01:19:43.468Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.957920146Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"2","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200242024-04-20T01:19:43.468Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.958041306Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"0","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200252024-04-20T01:19:43.468Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.958163416Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"1","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200262024-04-20T01:19:43.468Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.958196207Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"1","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200272024-04-20T01:19:43.468Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.958384437Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"2","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200282024-04-20T01:19:43.468Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.958429907Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"2","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200292024-04-20T01:19:43.468Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.958461447Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200302024-04-20T01:19:43.468Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.958503187Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"0","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200312024-04-20T01:19:43.468Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.958548377Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"0","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200322024-04-20T01:19:43.468Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.958590917Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200332024-04-20T01:19:43.468Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.958750128Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"1","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200342024-04-20T01:19:43.468Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959196299Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"1","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200352024-04-20T01:19:43.469Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959230899Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"1","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200362024-04-20T01:19:43.469Z{"msg":"[0]R flush_numbers[0..12]: [297, 311, 311, 306, 306, 312, 311, 310, 311, 312, 303, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959247289Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200372024-04-20T01:19:43.469Z{"msg":"[0]R generation[0..12]: [178, 184, 184, 182, 182, 185, 184, 184, 184, 185, 181, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959269959Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200382024-04-20T01:19:43.469Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959285379Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200392024-04-20T01:19:43.469Z{"msg":"[1]R flush_numbers[0..12]: [297, 311, 311, 306, 306, 312, 311, 310, 311, 312, 303, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959306989Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200402024-04-20T01:19:43.469Z{"msg":"[1]R generation[0..12]: [178, 184, 184, 182, 182, 185, 184, 184, 184, 185, 181, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959321409Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200412024-04-20T01:19:43.474Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
200422024-04-20T01:19:43.475Z,"time":"2024-04-20T01:19:41.959381399Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200432024-04-20T01:19:43.475Z{"msg":"[2]R flush_numbers[0..12]: [297, 311, 311, 306, 306, 312, 311, 310, 311, 312, 303, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959415929Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200442024-04-20T01:19:43.475Z{"msg":"[2]R generation[0..12]: [178, 184, 184, 182, 182, 185, 184, 184, 184, 185, 181, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959452769Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200452024-04-20T01:19:43.475Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959484969Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200462024-04-20T01:19:43.475Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959506019Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200472024-04-20T01:19:43.475Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.95954035Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200482024-04-20T01:19:43.475Z{"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.95956647Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200492024-04-20T01:19:43.475Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959593159Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"":"downstairs","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200502024-04-20T01:19:43.475Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.959607199Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200512024-04-20T01:19:43.475Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.95961963Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"0","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200522024-04-20T01:19:43.475Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.95963334Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"1","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200532024-04-20T01:19:43.475Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.95964616Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"client":"2","":"downstairs","session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200542024-04-20T01:19:43.475ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
200552024-04-20T01:19:43.475Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.95968753Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200562024-04-20T01:19:43.475Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.95970064Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200572024-04-20T01:19:43.476Z{"msg":"15a924ab-2281-409b-becc-82834c3f7750 is now active with session: 7d2cfcfc-c590-49b0-ad23-25a61ff88344","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.95971421Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200582024-04-20T01:19:43.476Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.95973303Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918,"session_id":"7d2cfcfc-c590-49b0-ad23-25a61ff88344"}
200592024-04-20T01:19:43.476Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:41.95974555Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918}
200602024-04-20T01:19:43.476ZLoading write count information from file "/tmp/test_repair_verify.data"
200612024-04-20T01:19:43.476ZRun Repair workload
200622024-04-20T01:19:43.476Z01/30 Write block 45 len 40960 data: 13 14 13 13 15 16 17 18 21 19
200632024-04-20T01:19:43.476Z02/30 Write block 308 len 8192 data: 10 9
200642024-04-20T01:19:43.476Z03/30 Read block 234 len 36864
200652024-04-20T01:19:43.476Z04/30 Write block 96 len 40960 data: 14 13 13 11 12 13 13 14 14 13
200662024-04-20T01:19:43.476Z05/30 Read block 375 len 40960
200672024-04-20T01:19:43.476Z06/30 Read block 84 len 40960
200682024-04-20T01:19:43.480Z07/30 Write block 495 len 24576 data: 11 10 9 8 9 12
200692024-04-20T01:19:43.480Z08/30 Write block 288 len 28672 data: 12 13 10 9 10 12 15
200702024-04-20T01:19:43.480Z09/30 Read block 76 len 20480
200712024-04-20T01:19:43.480Z10/30 Flush
200722024-04-20T01:19:43.480Z11/30 Write block 324 len 12288 data: 16 17 18
200732024-04-20T01:19:43.483Z12/30 Read block 103 len 40960
200742024-04-20T01:19:43.483Z13/30 Read block 90 len 12288
200752024-04-20T01:19:43.483Z14/30 Read block 377 len 32768
200762024-04-20T01:19:43.483Z15/30 Read block 277 len 8192
200772024-04-20T01:19:43.484Z16/30 Read block 144 len 4096
200782024-04-20T01:19:43.487Z17/30 Read block 481 len 16384
200792024-04-20T01:19:43.487Z18/30 Write block 327 len 28672 data: 18 18 15 17 14 17 14
200802024-04-20T01:19:43.487Z19/30 Read block 175 len 8192
200812024-04-20T01:19:43.487Z20/30 Read block 479 len 32768
200822024-04-20T01:19:43.487Z21/30 Read block 362 len 4096
200832024-04-20T01:19:43.491Z22/30 Read block 499 len 32768
200842024-04-20T01:19:43.491Z23/30 Read block 495 len 32768
200852024-04-20T01:19:43.491Z24/30 Write block 399 len 16384 data: 14 13 12 13
200862024-04-20T01:19:43.491Z25/30 Write block 538 len 12288 data: 11 11 11
200872024-04-20T01:19:43.491Z26/30 Flush
200882024-04-20T01:19:43.512Z27/30 Write block 409 len 12288 data: 17 17 18
200892024-04-20T01:19:43.512Z28/30 Write block 412 len 12288 data: 20 21 20
200902024-04-20T01:19:43.512Z29/30 Read block 563 len 24576
200912024-04-20T01:19:43.512Z30/30 Write block 480 len 40960 data: 15 19 19 18 13 13 14 12 10 11
200922024-04-20T01:19:43.512Z----------------------------------------------------------------
200932024-04-20T01:19:43.513Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
200942024-04-20T01:19:43.513ZStates: Active Active Active
200952024-04-20T01:19:43.513ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200962024-04-20T01:19:43.513Z 1 Acked 1000 Write 10 Done Done Done false
200972024-04-20T01:19:43.513Z 2 Acked 1001 Write 2 Done Done Done false
200982024-04-20T01:19:43.513Z 3 Acked 1002 Read 9 Done Sent Done false
200992024-04-20T01:19:43.513Z 4 Acked 1003 Write 10 Done Sent Done false
201002024-04-20T01:19:43.513Z 5 Acked 1004 Read 10 Done Sent Done false
201012024-04-20T01:19:43.513Z 6 Acked 1005 Read 10 Done Sent Done false
201022024-04-20T01:19:43.513Z 7 Acked 1006 Write 6 Done Sent Done false
201032024-04-20T01:19:43.513Z 8 Acked 1007 Write 7 Done Sent Done false
201042024-04-20T01:19:43.513Z 9 Acked 1008 Read 5 Done Sent Done false
201052024-04-20T01:19:43.513Z 10 Acked 1009 Flush 0 Done Sent Done false
201062024-04-20T01:19:43.513Z 11 Acked 1010 Write 3 Done Sent Done false
201072024-04-20T01:19:43.513Z 12 Acked 1011 Read 10 Done Sent Done false
201082024-04-20T01:19:43.513Z 13 Acked 1012 Read 3 Done Sent Done false
201092024-04-20T01:19:43.513Z 14 Acked 1013 Read 8 Done Sent Done false
201102024-04-20T01:19:43.513Z 15 Acked 1014 Read 2 Done Sent Done false
201112024-04-20T01:19:43.514Z 16 Acked 1015 Read 1 Done Sent Done false
201122024-04-20T01:19:43.514Z 17 Acked 1016 Read 4 Done Sent Done false
201132024-04-20T01:19:43.514Z 18 Acked 1017 Write 7 Done Sent Done false
201142024-04-20T01:19:43.514Z 19 Acked 1018 Read 2 Done Sent Done false
201152024-04-20T01:19:43.514Z 20 Acked 1019 Read 8 Done Sent Done false
201162024-04-20T01:19:43.514Z 21 Acked 1020 Read 1 Done Sent Done false
201172024-04-20T01:19:43.514Z 22 Acked 1021 Read 8 Done Sent Done false
201182024-04-20T01:19:43.514Z 23 Acked 1022 Read 8 Done Sent Done false
201192024-04-20T01:19:43.514Z 24 Acked 1023 Write 4 Done Sent Done false
201202024-04-20T01:19:43.514Z 25 Acked 1024 Write 3 Done Sent Done false
201212024-04-20T01:19:43.514Z 26 Acked 1025 Flush 0 Done Sent Done false
201222024-04-20T01:19:43.514Z 27 Acked 1026 Write 3 Done Sent Done false
201232024-04-20T01:19:43.514Z 28 Acked 1027 Write 3 Done Sent Done false
201242024-04-20T01:19:43.514Z 29 Acked 1028 Read 6 Done Sent Done false
201252024-04-20T01:19:43.515Z 30 Acked 1029 Write 10 Sent Sent Sent false
201262024-04-20T01:19:43.515Z STATES DS:0 DS:1 DS:2 TOTAL
201272024-04-20T01:19:43.515Z New 0 0 0 0
201282024-04-20T01:19:43.515Z Sent 1 28 1 30
201292024-04-20T01:19:43.515Z Done 29 2 29 60
201302024-04-20T01:19:43.515Z Skipped 0 0 0 0
201312024-04-20T01:19:43.515Z Error 0 0 0 0
201322024-04-20T01:19:43.515ZLast Flush: 1025 0 1025
201332024-04-20T01:19:43.515ZDownstairs last five completed:
201342024-04-20T01:19:43.515ZUpstairs last five completed: 30 29 28 27 26
201352024-04-20T01:19:43.516Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:19:41.9913539Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1918}
201362024-04-20T01:19:43.516ZWrote out file "/tmp/test_repair_verify.data"
201372024-04-20T01:19:43.528Z
201382024-04-20T01:19:43.528ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
201392024-04-20T01:19:43.528ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201402024-04-20T01:19:43.528Z 0 000-019 178 178 178 297 297 297 F F F
201412024-04-20T01:19:43.528Z 1 020-039 184 184 184 311 311 311 F F F
201422024-04-20T01:19:43.528Z 2 040-059 186 184 186 313 311 313 F T F <---
201432024-04-20T01:19:43.528Z 3 060-079 182 182 182 306 306 306 F F F
201442024-04-20T01:19:43.529Z 4 080-099 186 182 186 313 306 313 F F F <---
201452024-04-20T01:19:43.529Z 5 100-119 186 185 186 313 312 313 F F F <---
201462024-04-20T01:19:43.529Z 6 120-139 184 184 184 311 311 311 F F F
201472024-04-20T01:19:43.529Z 7 140-159 184 184 184 310 310 310 F F F
201482024-04-20T01:19:43.529Z 8 160-179 184 184 184 311 311 311 F F F
201492024-04-20T01:19:43.529Z 9 180-199 185 185 185 312 312 312 F F F
201502024-04-20T01:19:43.529Z 10 200-219 181 181 181 303 303 303 F F F
201512024-04-20T01:19:43.529Z 11 220-239 169 169 169 279 279 279 F F F
201522024-04-20T01:19:43.529Z 12 240-259 182 182 182 306 306 306 F F F
201532024-04-20T01:19:43.529Z 13 260-279 185 185 185 312 312 312 F F F
201542024-04-20T01:19:43.529Z 14 280-299 186 185 186 313 312 313 F F F <---
201552024-04-20T01:19:43.529Z 15 300-319 186 183 186 313 308 313 F T F <---
201562024-04-20T01:19:43.529Z 16 320-339 186 178 186 314 297 314 F F F <---
201572024-04-20T01:19:43.530Z 17 340-359 184 184 184 309 309 309 F F F
201582024-04-20T01:19:43.530Z 18 360-379 182 182 182 307 307 307 F F F
201592024-04-20T01:19:43.530Z 19 380-399 186 183 186 314 308 314 F F F <---
201602024-04-20T01:19:43.530Z 20 400-419 186 180 186 314 302 314 T F T <---
201612024-04-20T01:19:43.530Z 21 420-439 182 182 182 306 306 306 F F F
201622024-04-20T01:19:43.530Z 22 440-459 184 184 184 311 311 311 F F F
201632024-04-20T01:19:43.530Z 23 460-479 178 178 178 299 299 299 F F F
201642024-04-20T01:19:43.530Z 24 480-499 186 178 186 313 299 313 T F T <---
201652024-04-20T01:19:43.530Z 25 500-519 186 179 186 313 300 313 F F F <---
201662024-04-20T01:19:43.530Z 26 520-539 186 183 186 314 308 314 F F F <---
201672024-04-20T01:19:43.530Z 27 540-559 186 183 186 314 308 314 F F F <---
201682024-04-20T01:19:43.530Z 28 560-579 184 184 184 309 309 309 F F F
201692024-04-20T01:19:43.530Z 29 580-599 184 184 184 310 310 310 F F F
201702024-04-20T01:19:43.530ZMax gen: 186, Max flush: 314
201712024-04-20T01:19:43.530ZError: Difference in extent metadata found!
201722024-04-20T01:19:43.540ZA Difference in extent metadata is expected here
201732024-04-20T01:19:43.540ZOn loop 93
201742024-04-20T01:19:43.540Z
201752024-04-20T01:19:43.540ZVerifying data now
201762024-04-20T01:19:54.084ZNo change after verify
201772024-04-20T01:19:54.084ZLoop: 93 Downstairs dump after verify (and repair):
201782024-04-20T01:19:54.098ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201792024-04-20T01:19:54.098Z 0 000-019 178 178 178 297 297 297 F F F
201802024-04-20T01:19:54.098Z 1 020-039 184 184 184 311 311 311 F F F
201812024-04-20T01:19:54.098Z 2 040-059 186 186 186 313 313 313 F F F
201822024-04-20T01:19:54.098Z 3 060-079 182 182 182 306 306 306 F F F
201832024-04-20T01:19:54.098Z 4 080-099 186 186 186 313 313 313 F F F
201842024-04-20T01:19:54.098Z 5 100-119 186 186 186 313 313 313 F F F
201852024-04-20T01:19:54.098Z 6 120-139 184 184 184 311 311 311 F F F
201862024-04-20T01:19:54.098Z 7 140-159 184 184 184 310 310 310 F F F
201872024-04-20T01:19:54.104Z 8 160-179 184 184 184 311 311 311 F F F
201882024-04-20T01:19:54.104Z 9 180-199 185 185 185 312 312 312 F F F
201892024-04-20T01:19:54.104Z 10 200-219 181 181 181 303 303 303 F F F
201902024-04-20T01:19:54.104Z 11 220-239 169 169 169 279 279 279 F F F
201912024-04-20T01:19:54.104Z 12 240-259 182 182 182 306 306 306 F F F
201922024-04-20T01:19:54.104Z 13 260-279 185 185 185 312 312 312 F F F
201932024-04-20T01:19:54.104Z 14 280-299 186 186 186 313 313 313 F F F
201942024-04-20T01:19:54.104Z 15 300-319 186 186 186 313 313 313 F F F
201952024-04-20T01:19:54.104Z 16 320-339 186 186 186 314 314 314 F F F
201962024-04-20T01:19:54.104Z 17 340-359 184 184 184 309 309 309 F F F
201972024-04-20T01:19:54.104Z 18 360-379 182 182 182 307 307 307 F F F
201982024-04-20T01:19:54.104Z 19 380-399 186 186 186 314 314 314 F F F
201992024-04-20T01:19:54.104Z 20 400-419 187 187 187 315 315 315 F F F
202002024-04-20T01:19:54.104Z 21 420-439 182 182 182 306 306 306 F F F
202012024-04-20T01:19:54.104Z 22 440-459 184 184 184 311 311 311 F F F
202022024-04-20T01:19:54.104Z 23 460-479 178 178 178 299 299 299 F F F
202032024-04-20T01:19:54.104Z 24 480-499 187 187 187 315 315 315 F F F
202042024-04-20T01:19:54.104Z 25 500-519 186 186 186 313 313 313 F F F
202052024-04-20T01:19:54.104Z 26 520-539 186 186 186 314 314 314 F F F
202062024-04-20T01:19:54.105Z 27 540-559 186 186 186 314 314 314 F F F
202072024-04-20T01:19:54.105Z 28 560-579 184 184 184 309 309 309 F F F
202082024-04-20T01:19:54.105Z 29 580-599 184 184 184 310 310 310 F F F
202092024-04-20T01:19:54.105ZMax gen: 187, Max flush: 315
202102024-04-20T01:19:54.105Z
202112024-04-20T01:19:54.105ZBegin loop 94
202122024-04-20T01:19:54.105ZDownstairs to restart: 2
202132024-04-20T01:19:54.120Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614119887Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926}
202142024-04-20T01:19:54.120Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614357367Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926}
202152024-04-20T01:19:54.120Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614374817Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926}
202162024-04-20T01:19:54.126Z{"msg":"Crucible 03ef7a52-e1df-48ce-89ce-e2ec4d4b33da has session id: 6d13b257-6eaa-4773-82ff-3bb834e0bcae","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614408777Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202172024-04-20T01:19:54.126Z{"msg":"Upstairs opts: Upstairs UUID: 03ef7a52-e1df-48ce-89ce-e2ec4d4b33da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614465807Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202182024-04-20T01:19:54.126Z{"msg":"Crucible stats registered with UUID: 03ef7a52-e1df-48ce-89ce-e2ec4d4b33da","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614492937Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202192024-04-20T01:19:54.126ZCrucible runtime is spawned
202202024-04-20T01:19:54.126Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614603598Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926}
202212024-04-20T01:19:54.127Z{"msg":"03ef7a52-e1df-48ce-89ce-e2ec4d4b33da active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614628808Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202222024-04-20T01:19:54.127Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614642458Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"0","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202232024-04-20T01:19:54.127Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614656678Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"0","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202242024-04-20T01:19:54.127Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614682648Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"1","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202252024-04-20T01:19:54.127Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614719888Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"1","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202262024-04-20T01:19:54.127Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614747028Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"2","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202272024-04-20T01:19:54.127Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614787408Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"2","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202282024-04-20T01:19:54.127Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614815298Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202292024-04-20T01:19:54.127Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614849088Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202302024-04-20T01:19:54.127Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614875768Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202312024-04-20T01:19:54.127Z{"msg":"ds_connection connected from Ok(127.0.0.1:64746)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.614984249Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202322024-04-20T01:19:54.127Z{"msg":"ds_connection connected from Ok(127.0.0.1:62921)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.615034079Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202332024-04-20T01:19:54.127Z{"msg":"ds_connection connected from Ok(127.0.0.1:36567)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.615075769Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202342024-04-20T01:19:54.127Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.61578052Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"1","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202352024-04-20T01:19:54.127Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.61580773Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"1","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202362024-04-20T01:19:54.127Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.6158231Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"0","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202372024-04-20T01:19:54.127Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.615849211Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"0","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202382024-04-20T01:19:54.127Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.616203991Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"2","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202392024-04-20T01:19:54.127Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.616257791Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"2","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202402024-04-20T01:19:54.127Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.616443912Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"1","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202412024-04-20T01:19:54.127Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.616472902Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"0","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202422024-04-20T01:19:54.127Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.616819133Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"2","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202432024-04-20T01:19:54.127Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617009593Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"0","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202442024-04-20T01:19:54.128Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617059023Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"0","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202452024-04-20T01:19:54.128Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617094993Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202462024-04-20T01:19:54.128Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617127453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"1","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202472024-04-20T01:19:54.128Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617160803Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"1","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202482024-04-20T01:19:54.128Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617196274Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202492024-04-20T01:19:54.128Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617335144Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"2","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202502024-04-20T01:19:54.128Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617391054Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"2","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202512024-04-20T01:19:54.128Z{"msg":"[0]R flush_numbers[0..12]: [297, 311, 313, 306, 313, 313, 311, 310, 311, 312, 303, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617429304Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202522024-04-20T01:19:54.128Z{"msg":"[0]R generation[0..12]: [178, 184, 186, 182, 186, 186, 184, 184, 184, 185, 181, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617463714Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202532024-04-20T01:19:54.128Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617486284Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202542024-04-20T01:19:54.128Z{"msg":"[1]R flush_numbers[0..12]: [297, 311, 313, 306, 313, 313, 311, 310, 311, 312, 303, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617519024Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
202552024-04-20T01:19:54.128Zdownstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202562024-04-20T01:19:54.128Z{"msg":"[1]R generation[0..12]: [178, 184, 186, 182, 186, 186, 184, 184, 184, 185, 181, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617548124Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202572024-04-20T01:19:54.128Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617572324Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202582024-04-20T01:19:54.128Z{"msg":"[2]R flush_numbers[0..12]: [297, 311, 313, 306, 313, 313, 311, 310, 311, 312, 303, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617587344Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202592024-04-20T01:19:54.128Z{"msg":"[2]R generation[0..12]: [178, 184, 186, 182, 186, 186, 184, 184, 184, 185, 181, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617612614Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202602024-04-20T01:19:54.128Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617644975Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202612024-04-20T01:19:54.128Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617669564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202622024-04-20T01:19:54.128Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617700625Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202632024-04-20T01:19:54.128Z{"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617722955Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202642024-04-20T01:19:54.128Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617752165Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"":"downstairs","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202652024-04-20T01:19:54.128Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
202662024-04-20T01:19:54.128Z,"time":"2024-04-20T01:19:52.617782895Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202672024-04-20T01:19:54.129Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617805325Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"0","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202682024-04-20T01:19:54.129Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617830015Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"1","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202692024-04-20T01:19:54.129Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617843785Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"client":"2","":"downstairs","session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202702024-04-20T01:19:54.129Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617864625Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202712024-04-20T01:19:54.129Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617877605Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202722024-04-20T01:19:54.129Z{"msg":"03ef7a52-e1df-48ce-89ce-e2ec4d4b33da is now active with session: 6d13b257-6eaa-4773-82ff-3bb834e0bcae","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617897565Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202732024-04-20T01:19:54.129Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617909955Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926,"session_id":"6d13b257-6eaa-4773-82ff-3bb834e0bcae"}
202742024-04-20T01:19:54.129Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:52.617928775Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926}
202752024-04-20T01:19:54.129ZLoading write count information from file "/tmp/test_repair_verify.data"
202762024-04-20T01:19:54.129ZRun Repair workload
202772024-04-20T01:19:54.133Z01/30 Write block 409 len 24576 data: 18 18 19 21 22 21
202782024-04-20T01:19:54.133Z02/30 Write block 573 len 20480 data: 10 10 10 11 13
202792024-04-20T01:19:54.133Z03/30 Read block 214 len 28672
202802024-04-20T01:19:54.133Z04/30 Read block 316 len 8192
202812024-04-20T01:19:54.133Z05/30 Write block 418 len 28672 data: 14 14 12 12 10 12 8
202822024-04-20T01:19:54.133Z06/30 Flush
202832024-04-20T01:19:54.134Z07/30 Write block 178 len 4096 data: 19
202842024-04-20T01:19:54.134Z08/30 Read block 496 len 20480
202852024-04-20T01:19:54.134Z09/30 Read block 94 len 8192
202862024-04-20T01:19:54.137Z10/30 Write block 430 len 36864 data: 16 15 11 11 15 16 18 16 13
202872024-04-20T01:19:54.137Z11/30 Read block 101 len 36864
202882024-04-20T01:19:54.137Z12/30 Read block 570 len 20480
202892024-04-20T01:19:54.137Z13/30 Write block 290 len 4096 data: 11
202902024-04-20T01:19:54.137Z14/30 Read block 582 len 28672
202912024-04-20T01:19:54.141Z15/30 Write block 115 len 40960 data: 16 16 14 16 18 16 16 16 15 15
202922024-04-20T01:19:54.141Z16/30 Write block 270 len 24576 data: 11 13 14 19 15 16
202932024-04-20T01:19:54.141Z17/30 Read block 6 len 28672
202942024-04-20T01:19:54.141Z18/30 Read block 386 len 28672
202952024-04-20T01:19:54.144Z19/30 Read block 154 len 32768
202962024-04-20T01:19:54.145Z20/30 Write block 375 len 36864 data: 18 15 17 16 16 19 16 17 15
202972024-04-20T01:19:54.145Z21/30 Write block 561 len 20480 data: 10 12 11 14 17
202982024-04-20T01:19:54.148Z22/30 Write block 207 len 12288 data: 12 12 15
202992024-04-20T01:19:54.148Z23/30 Flush
203002024-04-20T01:19:54.152Z24/30 Write block 411 len 40960 data: 20 22 23 22 19 21 17 15 15 13
203012024-04-20T01:19:54.152Z25/30 Read block 127 len 36864
203022024-04-20T01:19:54.152Z26/30 Flush
203032024-04-20T01:19:54.155Z27/30 Write block 399 len 20480 data: 15 14 13 14 11
203042024-04-20T01:19:54.155Z28/30 Read block 470 len 12288
203052024-04-20T01:19:54.160Z29/30 Write block 593 len 16384 data: 16 17 16 15
203062024-04-20T01:19:54.160Z30/30 Write block 444 len 4096 data: 12
203072024-04-20T01:19:54.160Z----------------------------------------------------------------
203082024-04-20T01:19:54.160Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:24
203092024-04-20T01:19:54.160ZStates: Active Active Active
203102024-04-20T01:19:54.160ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
203112024-04-20T01:19:54.160Z 7 Acked 1006 Write 1 Done Done Done false
203122024-04-20T01:19:54.160Z 8 Acked 1007 Read 5 Done Done Done false
203132024-04-20T01:19:54.160Z 9 Acked 1008 Read 2 Done Done Done false
203142024-04-20T01:19:54.160Z 10 Acked 1009 Write 9 Done Done Sent false
203152024-04-20T01:19:54.160Z 11 Acked 1010 Read 9 Done Done Sent false
203162024-04-20T01:19:54.160Z 12 Acked 1011 Read 5 Done Done Sent false
203172024-04-20T01:19:54.160Z 13 Acked 1012 Write 1 Done Done Sent false
203182024-04-20T01:19:54.160Z 14 Acked 1013 Read 7 Done Done Sent false
203192024-04-20T01:19:54.160Z 15 Acked 1014 Write 10 Done Done Sent false
203202024-04-20T01:19:54.161Z 16 Acked 1015 Write 6 Done Done Sent false
203212024-04-20T01:19:54.161Z 17 Acked 1016 Read 7 Done Done Sent false
203222024-04-20T01:19:54.161Z 18 Acked 1017 Read 7 Done Done Sent false
203232024-04-20T01:19:54.161Z 19 Acked 1018 Read 8 Done Done Sent false
203242024-04-20T01:19:54.161Z 20 Acked 1019 Write 9 Done Done Sent false
203252024-04-20T01:19:54.161Z 21 Acked 1020 Write 5 Done Done Sent false
203262024-04-20T01:19:54.161Z 22 Acked 1021 Write 3 Done Done Sent false
203272024-04-20T01:19:54.161Z 23 Acked 1022 Flush 0 Done Done Sent false
203282024-04-20T01:19:54.161Z 24 Acked 1023 Write 10 Done Done Sent false
203292024-04-20T01:19:54.161Z 25 Acked 1024 Read 9 Done Done Sent false
203302024-04-20T01:19:54.161Z 26 Acked 1025 Flush 0 Done Done Sent false
203312024-04-20T01:19:54.161Z 27 Acked 1026 Write 5 Done Done Sent false
203322024-04-20T01:19:54.161Z 28 Acked 1027 Read 3 Done Done Sent false
203332024-04-20T01:19:54.161Z 29 Acked 1028 Write 4 Done Done Sent false
203342024-04-20T01:19:54.161Z 30 Acked 1029 Write 1 Sent Sent Sent false
203352024-04-20T01:19:54.161Z STATES DS:0 DS:1 DS:2 TOTAL
203362024-04-20T01:19:54.161Z New 0 0 0 0
203372024-04-20T01:19:54.161Z Sent 1 1 21 23
203382024-04-20T01:19:54.161Z Done 23 23 3 49
203392024-04-20T01:19:54.161Z Skipped 0 0 0 0
203402024-04-20T01:19:54.161Z Error 0 0 0 0
203412024-04-20T01:19:54.161ZLast Flush: 1025 1025 1005
203422024-04-20T01:19:54.161ZDownstairs last five completed: 1005 1004 1003 1002 1001
203432024-04-20T01:19:54.162ZUpstairs last five completed: 30 29 28 27 26
203442024-04-20T01:19:54.162Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:19:52.65139329Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1926}
203452024-04-20T01:19:54.162ZWrote out file "/tmp/test_repair_verify.data"
203462024-04-20T01:19:54.166Z
203472024-04-20T01:19:54.166ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
203482024-04-20T01:19:54.180ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203492024-04-20T01:19:54.180Z 0 000-019 178 178 178 297 297 297 F F F
203502024-04-20T01:19:54.180Z 1 020-039 184 184 184 311 311 311 F F F
203512024-04-20T01:19:54.180Z 2 040-059 186 186 186 313 313 313 F F F
203522024-04-20T01:19:54.180Z 3 060-079 182 182 182 306 306 306 F F F
203532024-04-20T01:19:54.180Z 4 080-099 186 186 186 313 313 313 F F F
203542024-04-20T01:19:54.183Z 5 100-119 188 188 186 317 317 313 F F F <---
203552024-04-20T01:19:54.183Z 6 120-139 188 188 184 317 317 311 F F F <---
203562024-04-20T01:19:54.183Z 7 140-159 184 184 184 310 310 310 F F F
203572024-04-20T01:19:54.184Z 8 160-179 188 188 184 317 317 311 F F T <---
203582024-04-20T01:19:54.184Z 9 180-199 185 185 185 312 312 312 F F F
203592024-04-20T01:19:54.184Z 10 200-219 188 188 181 317 317 303 F F F <---
203602024-04-20T01:19:54.184Z 11 220-239 169 169 169 279 279 279 F F F
203612024-04-20T01:19:54.184Z 12 240-259 182 182 182 306 306 306 F F F
203622024-04-20T01:19:54.184Z 13 260-279 188 188 185 317 317 312 F F F <---
203632024-04-20T01:19:54.184Z 14 280-299 188 188 186 317 317 313 F F F <---
203642024-04-20T01:19:54.184Z 15 300-319 186 186 186 313 313 313 F F F
203652024-04-20T01:19:54.184Z 16 320-339 186 186 186 314 314 314 F F F
203662024-04-20T01:19:54.184Z 17 340-359 184 184 184 309 309 309 F F F
203672024-04-20T01:19:54.184Z 18 360-379 188 188 182 317 317 307 F F F <---
203682024-04-20T01:19:54.184Z 19 380-399 188 188 186 317 317 314 T T F <---
203692024-04-20T01:19:54.184Z 20 400-419 188 188 188 318 318 316 T T F <---
203702024-04-20T01:19:54.184Z 21 420-439 188 188 188 318 318 316 F F F <---
203712024-04-20T01:19:54.184Z 22 440-459 184 184 184 311 311 311 T T F <---
203722024-04-20T01:19:54.184Z 23 460-479 178 178 178 299 299 299 F F F
203732024-04-20T01:19:54.184Z 24 480-499 187 187 187 315 315 315 F F F
203742024-04-20T01:19:54.184Z 25 500-519 186 186 186 313 313 313 F F F
203752024-04-20T01:19:54.184Z 26 520-539 186 186 186 314 314 314 F F F
203762024-04-20T01:19:54.184Z 27 540-559 186 186 186 314 314 314 F F F
203772024-04-20T01:19:54.184Z 28 560-579 188 188 188 317 317 316 F F F <---
203782024-04-20T01:19:54.184Z 29 580-599 184 184 184 310 310 310 T T F <---
203792024-04-20T01:19:54.184ZMax gen: 188, Max flush: 318
203802024-04-20T01:19:54.185ZError: Difference in extent metadata found!
203812024-04-20T01:19:54.185ZA Difference in extent metadata is expected here
203822024-04-20T01:19:54.185ZOn loop 94
203832024-04-20T01:19:54.185Z
203842024-04-20T01:19:54.188ZVerifying data now
203852024-04-20T01:19:58.794ZNo change after verify
203862024-04-20T01:19:58.794ZLoop: 94 Downstairs dump after verify (and repair):
203872024-04-20T01:19:58.804ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203882024-04-20T01:19:58.804Z 0 000-019 178 178 178 297 297 297 F F F
203892024-04-20T01:19:58.804Z 1 020-039 184 184 184 311 311 311 F F F
203902024-04-20T01:19:58.805Z 2 040-059 186 186 186 313 313 313 F F F
203912024-04-20T01:19:58.805Z 3 060-079 182 182 182 306 306 306 F F F
203922024-04-20T01:19:58.805Z 4 080-099 186 186 186 313 313 313 F F F
203932024-04-20T01:19:58.805Z 5 100-119 188 188 188 317 317 317 F F F
203942024-04-20T01:19:58.806Z 6 120-139 188 188 188 317 317 317 F F F
203952024-04-20T01:19:58.806Z 7 140-159 184 184 184 310 310 310 F F F
203962024-04-20T01:19:58.806Z 8 160-179 188 188 188 317 317 317 F F F
203972024-04-20T01:19:58.806Z 9 180-199 185 185 185 312 312 312 F F F
203982024-04-20T01:19:58.806Z 10 200-219 188 188 188 317 317 317 F F F
203992024-04-20T01:19:58.812Z 11 220-239 169 169 169 279 279 279 F F F
204002024-04-20T01:19:58.812Z 12 240-259 182 182 182 306 306 306 F F F
204012024-04-20T01:19:58.812Z 13 260-279 188 188 188 317 317 317 F F F
204022024-04-20T01:19:58.812Z 14 280-299 188 188 188 317 317 317 F F F
204032024-04-20T01:19:58.813Z 15 300-319 186 186 186 313 313 313 F F F
204042024-04-20T01:19:58.813Z 16 320-339 186 186 186 314 314 314 F F F
204052024-04-20T01:19:58.813Z 17 340-359 184 184 184 309 309 309 F F F
204062024-04-20T01:19:58.813Z 18 360-379 188 188 188 317 317 317 F F F
204072024-04-20T01:19:58.813Z 19 380-399 189 189 189 319 319 319 F F F
204082024-04-20T01:19:58.813Z 20 400-419 189 189 189 319 319 319 F F F
204092024-04-20T01:19:58.813Z 21 420-439 188 188 188 318 318 318 F F F
204102024-04-20T01:19:58.813Z 22 440-459 189 189 189 319 319 319 F F F
204112024-04-20T01:19:58.813Z 23 460-479 178 178 178 299 299 299 F F F
204122024-04-20T01:19:58.813Z 24 480-499 187 187 187 315 315 315 F F F
204132024-04-20T01:19:58.813Z 25 500-519 186 186 186 313 313 313 F F F
204142024-04-20T01:19:58.813Z 26 520-539 186 186 186 314 314 314 F F F
204152024-04-20T01:19:58.813Z 27 540-559 186 186 186 314 314 314 F F F
204162024-04-20T01:19:58.813Z 28 560-579 188 188 188 317 317 317 F F F
204172024-04-20T01:19:58.813Z 29 580-599 189 189 189 319 319 319 F F F
204182024-04-20T01:19:58.813ZMax gen: 189, Max flush: 319
204192024-04-20T01:19:58.813Z
204202024-04-20T01:19:58.814ZBegin loop 95
204212024-04-20T01:19:58.814ZDownstairs to restart: 1
204222024-04-20T01:19:58.823Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317090686Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934}
204232024-04-20T01:19:58.823Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317305846Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934}
204242024-04-20T01:19:58.823Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317331856Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934}
204252024-04-20T01:19:58.832Z{"msg":"Crucible bbaffc8b-5e81-44cf-abfe-e1c8f9aa6965 has session id: c81375c9-3b12-4844-9595-c7815a4a08db","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317376116Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204262024-04-20T01:19:58.832ZCrucible runtime is spawned
204272024-04-20T01:19:58.833Z{"msg":"Upstairs opts: Upstairs UUID: bbaffc8b-5e81-44cf-abfe-e1c8f9aa6965, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317433956Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204282024-04-20T01:19:58.833Z{"msg":"Crucible stats registered with UUID: bbaffc8b-5e81-44cf-abfe-e1c8f9aa6965","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317472297Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204292024-04-20T01:19:58.833Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317495677Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934}
204302024-04-20T01:19:58.833Z{"msg":"bbaffc8b-5e81-44cf-abfe-e1c8f9aa6965 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317515806Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204312024-04-20T01:19:58.833Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317547667Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"0","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204322024-04-20T01:19:58.833Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317583727Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"0","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204332024-04-20T01:19:58.833Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317607877Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"1","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204342024-04-20T01:19:58.833Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317640777Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"1","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204352024-04-20T01:19:58.833Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317688477Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"2","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204362024-04-20T01:19:58.833Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317714547Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"2","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204372024-04-20T01:19:58.833Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317796177Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204382024-04-20T01:19:58.833Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317846287Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204392024-04-20T01:19:58.834Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317883217Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204402024-04-20T01:19:58.834Z{"msg":"ds_connection connected from Ok(127.0.0.1:59059)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.317909927Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204412024-04-20T01:19:58.834Z{"msg":"ds_connection connected from Ok(127.0.0.1:65204)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.318005298Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204422024-04-20T01:19:58.834Z{"msg":"ds_connection connected from Ok(127.0.0.1:54877)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.318055178Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204432024-04-20T01:19:58.834Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.318563429Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"2","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204442024-04-20T01:19:58.834Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.318615819Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"2","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204452024-04-20T01:19:58.834Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.318643869Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"0","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204462024-04-20T01:19:58.834Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.318682989Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"0","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204472024-04-20T01:19:58.834Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.318799789Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"1","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204482024-04-20T01:19:58.834Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.318831889Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"1","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204492024-04-20T01:19:58.834Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.3192172Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"2","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204502024-04-20T01:19:58.834Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.319314921Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"0","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204512024-04-20T01:19:58.835Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.319506761Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"1","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204522024-04-20T01:19:58.835Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.319678241Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"0","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204532024-04-20T01:19:58.835Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.319724311Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"0","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204542024-04-20T01:19:58.835Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.319751622Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204552024-04-20T01:19:58.835Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.319783852Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"2","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204562024-04-20T01:19:58.835Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.319808292Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"2","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204572024-04-20T01:19:58.835Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.319835292Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204582024-04-20T01:19:58.835Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.319957212Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"1","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204592024-04-20T01:19:58.835Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320013632Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"1","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204602024-04-20T01:19:58.835Z{"msg":"[0]R flush_numbers[0..12]: [297, 311, 313, 306, 313, 317, 317, 310, 317, 312, 317, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320048522Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204612024-04-20T01:19:58.835Z{"msg":"[0]R generation[0..12]: [178, 184, 186, 182, 186, 188, 188, 184, 188, 185, 188, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320064572Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204622024-04-20T01:19:58.835Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320086362Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204632024-04-20T01:19:58.836Z{"msg":"[1]R flush_numbers[0..12]: [297, 311, 313, 306, 313, 317, 317, 310, 317, 312, 317, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320101622Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204642024-04-20T01:19:58.836ZWait for a query_work_queue command to finish before sending IO
204652024-04-20T01:19:58.836Z{"msg":"[1]R generation[0..12]: [178, 184, 186, 182, 186, 188, 188, 184, 188, 185, 188, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320124882Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204662024-04-20T01:19:58.836Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320140292Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204672024-04-20T01:19:58.836Z{"msg":"[2]R flush_numbers[0..12]: [297, 311, 313, 306, 313, 317, 317, 310, 317, 312, 317, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320160772Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204682024-04-20T01:19:58.836Z{"msg":"[2]R generation[0..12]: [178, 184, 186, 182, 186, 188, 188, 184, 188, 185, 188, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320174923Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204692024-04-20T01:19:58.836Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320188723Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204702024-04-20T01:19:58.836Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320214072Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204712024-04-20T01:19:58.836Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320227792Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204722024-04-20T01:19:58.836Z{"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320240763Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204732024-04-20T01:19:58.836Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320254323Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"":"downstairs","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204742024-04-20T01:19:58.836Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320280893Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204752024-04-20T01:19:58.836Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320294233Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"0","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204762024-04-20T01:19:58.837Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320307663Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"1","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204772024-04-20T01:19:58.837Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320331373Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"client":"2","":"downstairs","session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204782024-04-20T01:19:58.837Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320347643Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204792024-04-20T01:19:58.837Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
204802024-04-20T01:19:58.837Z,"time":"2024-04-20T01:19:57.320371453Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204812024-04-20T01:19:58.837Z{"msg":"bbaffc8b-5e81-44cf-abfe-e1c8f9aa6965 is now active with session: c81375c9-3b12-4844-9595-c7815a4a08db","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320389693Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204822024-04-20T01:19:58.837Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320402293Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934,"session_id":"c81375c9-3b12-4844-9595-c7815a4a08db"}
204832024-04-20T01:19:58.837Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:19:57.320423763Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934}
204842024-04-20T01:19:58.837ZLoading write count information from file "/tmp/test_repair_verify.data"
204852024-04-20T01:19:58.837ZRun Repair workload
204862024-04-20T01:19:58.842Z01/30 Write block 43 len 12288 data: 12 13 14
204872024-04-20T01:19:58.842Z02/30 Write block 88 len 32768 data: 16 16 19 18 16 16 16 13
204882024-04-20T01:19:58.842Z03/30 Read block 536 len 32768
204892024-04-20T01:19:58.842Z04/30 Read block 419 len 4096
204902024-04-20T01:19:58.842Z05/30 Write block 139 len 32768 data: 17 19 20 20 21 21 21 17
204912024-04-20T01:19:58.842Z06/30 Write block 135 len 12288 data: 13 14 15
204922024-04-20T01:19:58.842Z07/30 Write block 421 len 4096 data: 13
204932024-04-20T01:19:58.842Z08/30 Read block 516 len 24576
204942024-04-20T01:19:58.843Z09/30 Read block 353 len 24576
204952024-04-20T01:19:58.843Z10/30 Read block 588 len 16384
204962024-04-20T01:19:58.843Z11/30 Flush
204972024-04-20T01:19:58.843Z12/30 Flush
204982024-04-20T01:19:58.843Z13/30 Write block 169 len 8192 data: 7 10
204992024-04-20T01:19:58.843Z14/30 Flush
205002024-04-20T01:19:58.843Z15/30 Write block 490 len 4096 data: 13
205012024-04-20T01:19:58.848Z16/30 Read block 376 len 8192
205022024-04-20T01:19:58.848Z17/30 Read block 557 len 20480
205032024-04-20T01:19:58.848Z18/30 Write block 428 len 24576 data: 11 15 17 16 12 12
205042024-04-20T01:19:58.848Z19/30 Read block 23 len 40960
205052024-04-20T01:19:58.848Z20/30 Read block 384 len 36864
205062024-04-20T01:19:58.848Z21/30 Read block 69 len 20480
205072024-04-20T01:19:58.848Z22/30 Flush
205082024-04-20T01:19:58.857Z23/30 Write block 543 len 4096 data: 21
205092024-04-20T01:19:58.858Z24/30 Write block 529 len 32768 data: 12 12 14 14 13 16 14 12
205102024-04-20T01:19:58.858Z25/30 Read block 270 len 24576
205112024-04-20T01:19:58.858Z26/30 Read block 206 len 8192
205122024-04-20T01:19:58.858Z27/30 Read block 462 len 40960
205132024-04-20T01:19:58.858Z28/30 Write block 242 len 36864 data: 8 7 12 14 14 14 18 18 16
205142024-04-20T01:19:58.858Z29/30 Read block 162 len 28672
205152024-04-20T01:19:58.858Z30/30 Read block 484 len 12288
205162024-04-20T01:19:58.858Z----------------------------------------------------------------
205172024-04-20T01:19:58.858Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
205182024-04-20T01:19:58.858ZStates: Active Active Active
205192024-04-20T01:19:58.858ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
205202024-04-20T01:19:58.858Z 1 Acked 1000 Write 3 Done Sent Done false
205212024-04-20T01:19:58.858Z 2 Acked 1001 Write 8 Done Sent Done false
205222024-04-20T01:19:58.858Z 3 Acked 1002 Read 8 Done Sent Done false
205232024-04-20T01:19:58.858Z 4 Acked 1003 Read 1 Done Sent Done false
205242024-04-20T01:19:58.858Z 5 Acked 1004 Write 8 Done Sent Done false
205252024-04-20T01:19:58.858Z 6 Acked 1005 Write 3 Done Sent Done false
205262024-04-20T01:19:58.858Z 7 Acked 1006 Write 1 Done Sent Done false
205272024-04-20T01:19:58.858Z 8 Acked 1007 Read 6 Done Sent Done false
205282024-04-20T01:19:58.858Z 9 Acked 1008 Read 6 Done Sent Done false
205292024-04-20T01:19:58.858Z 10 Acked 1009 Read 4 Done Sent Done false
205302024-04-20T01:19:58.858Z 11 Acked 1010 Flush 0 Done Sent Done false
205312024-04-20T01:19:58.858Z 12 Acked 1011 Flush 0 Done Sent Done false
205322024-04-20T01:19:58.859Z 13 Acked 1012 Write 2 Done Sent Done false
205332024-04-20T01:19:58.859Z 14 Acked 1013 Flush 0 Done Sent Done false
205342024-04-20T01:19:58.859Z 15 Acked 1014 Write 1 Done Sent Done false
205352024-04-20T01:19:58.859Z 16 Acked 1015 Read 2 Done Sent Done false
205362024-04-20T01:19:58.859Z 17 Acked 1016 Read 5 Done Sent Done false
205372024-04-20T01:19:58.859Z 18 Acked 1017 Write 6 Done Sent Done false
205382024-04-20T01:19:58.859Z 19 Acked 1018 Read 10 Done Sent Done false
205392024-04-20T01:19:58.859Z 20 Acked 1019 Read 9 Done Sent Done false
205402024-04-20T01:19:58.859Z 21 Acked 1020 Read 5 Done Sent Done false
205412024-04-20T01:19:58.859Z 22 Acked 1021 Flush 0 Done Sent Done false
205422024-04-20T01:19:58.859Z 23 Acked 1022 Write 1 Done Sent Done false
205432024-04-20T01:19:58.859Z 24 Acked 1023 Write 8 Done Sent Done false
205442024-04-20T01:19:58.859Z 25 Acked 1024 Read 6 Done Sent Done false
205452024-04-20T01:19:58.859Z 26 Acked 1025 Read 2 Done Sent Done false
205462024-04-20T01:19:58.859Z 27 Acked 1026 Read 10 Done Sent Done false
205472024-04-20T01:19:58.859Z 28 Acked 1027 Write 9 Done Sent Done false
205482024-04-20T01:19:58.859Z 29 Acked 1028 Read 7 Done Sent Done false
205492024-04-20T01:19:58.859Z 30 Acked 1029 Read 3 Done Sent Done false
205502024-04-20T01:19:58.859Z STATES DS:0 DS:1 DS:2 TOTAL
205512024-04-20T01:19:58.859Z New 0 0 0 0
205522024-04-20T01:19:58.859Z Sent 0 30 0 30
205532024-04-20T01:19:58.859Z Done 30 0 30 60
205542024-04-20T01:19:58.859Z Skipped 0 0 0 0
205552024-04-20T01:19:58.860Z Error 0 0 0 0
205562024-04-20T01:19:58.860ZLast Flush: 1021 0 1021
205572024-04-20T01:19:58.860ZDownstairs last five completed:
205582024-04-20T01:19:58.860ZUpstairs last five completed: 30 29 28 27 26
205592024-04-20T01:19:58.860Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:19:57.350802961Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1934}
205602024-04-20T01:19:58.860ZWrote out file "/tmp/test_repair_verify.data"
205612024-04-20T01:19:58.863Z
205622024-04-20T01:19:58.863ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
205632024-04-20T01:19:58.879ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205642024-04-20T01:19:58.879Z 0 000-019 178 178 178 297 297 297 F F F
205652024-04-20T01:19:58.879Z 1 020-039 184 184 184 311 311 311 F F F
205662024-04-20T01:19:58.879Z 2 040-059 190 186 190 320 313 320 F F F <---
205672024-04-20T01:19:58.879Z 3 060-079 182 182 182 306 306 306 F F F
205682024-04-20T01:19:58.879Z 4 080-099 190 186 190 320 313 320 F F F <---
205692024-04-20T01:19:58.884Z 5 100-119 188 188 188 317 317 317 F F F
205702024-04-20T01:19:58.884Z 6 120-139 190 188 190 320 317 320 F F F <---
205712024-04-20T01:19:58.884Z 7 140-159 190 184 190 320 310 320 F F F <---
205722024-04-20T01:19:58.884Z 8 160-179 190 188 190 322 317 322 F F F <---
205732024-04-20T01:19:58.884Z 9 180-199 185 185 185 312 312 312 F F F
205742024-04-20T01:19:58.884Z 10 200-219 188 188 188 317 317 317 F F F
205752024-04-20T01:19:58.884Z 11 220-239 169 169 169 279 279 279 F F F
205762024-04-20T01:19:58.884Z 12 240-259 182 182 182 306 306 306 T F T <---
205772024-04-20T01:19:58.884Z 13 260-279 188 188 188 317 317 317 F F F
205782024-04-20T01:19:58.884Z 14 280-299 188 188 188 317 317 317 F F F
205792024-04-20T01:19:58.884Z 15 300-319 186 186 186 313 313 313 F F F
205802024-04-20T01:19:58.884Z 16 320-339 186 186 186 314 314 314 F F F
205812024-04-20T01:19:58.884Z 17 340-359 184 184 184 309 309 309 F F F
205822024-04-20T01:19:58.884Z 18 360-379 188 188 188 317 317 317 F F F
205832024-04-20T01:19:58.884Z 19 380-399 189 189 189 319 319 319 F F F
205842024-04-20T01:19:58.884Z 20 400-419 189 189 189 319 319 319 F F F
205852024-04-20T01:19:58.884Z 21 420-439 190 188 190 323 318 323 F F F <---
205862024-04-20T01:19:58.884Z 22 440-459 189 189 189 319 319 319 F F F
205872024-04-20T01:19:58.884Z 23 460-479 178 178 178 299 299 299 F F F
205882024-04-20T01:19:58.884Z 24 480-499 190 187 190 323 315 323 F F F <---
205892024-04-20T01:19:58.884Z 25 500-519 186 186 186 313 313 313 F F F
205902024-04-20T01:19:58.885Z 26 520-539 186 186 186 314 314 314 T F T <---
205912024-04-20T01:19:58.885Z 27 540-559 186 186 186 314 314 314 T F T <---
205922024-04-20T01:19:58.885Z 28 560-579 188 188 188 317 317 317 F F F
205932024-04-20T01:19:58.885Z 29 580-599 189 189 189 319 319 319 F F F
205942024-04-20T01:19:58.885ZMax gen: 190, Max flush: 323
205952024-04-20T01:19:58.885ZError: Difference in extent metadata found!
205962024-04-20T01:19:58.885ZA Difference in extent metadata is expected here
205972024-04-20T01:19:58.885ZOn loop 95
205982024-04-20T01:19:58.885Z
205992024-04-20T01:19:58.889ZVerifying data now
206002024-04-20T01:20:03.357ZNo change after verify
206012024-04-20T01:20:03.357ZLoop: 95 Downstairs dump after verify (and repair):
206022024-04-20T01:20:03.369ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206032024-04-20T01:20:03.369Z 0 000-019 178 178 178 297 297 297 F F F
206042024-04-20T01:20:03.373Z 1 020-039 184 184 184 311 311 311 F F F
206052024-04-20T01:20:03.373Z 2 040-059 190 190 190 320 320 320 F F F
206062024-04-20T01:20:03.373Z 3 060-079 182 182 182 306 306 306 F F F
206072024-04-20T01:20:03.373Z 4 080-099 190 190 190 320 320 320 F F F
206082024-04-20T01:20:03.374Z 5 100-119 188 188 188 317 317 317 F F F
206092024-04-20T01:20:03.374Z 6 120-139 190 190 190 320 320 320 F F F
206102024-04-20T01:20:03.374Z 7 140-159 190 190 190 320 320 320 F F F
206112024-04-20T01:20:03.374Z 8 160-179 190 190 190 322 322 322 F F F
206122024-04-20T01:20:03.374Z 9 180-199 185 185 185 312 312 312 F F F
206132024-04-20T01:20:03.374Z 10 200-219 188 188 188 317 317 317 F F F
206142024-04-20T01:20:03.374Z 11 220-239 169 169 169 279 279 279 F F F
206152024-04-20T01:20:03.374Z 12 240-259 191 191 191 324 324 324 F F F
206162024-04-20T01:20:03.374Z 13 260-279 188 188 188 317 317 317 F F F
206172024-04-20T01:20:03.374Z 14 280-299 188 188 188 317 317 317 F F F
206182024-04-20T01:20:03.374Z 15 300-319 186 186 186 313 313 313 F F F
206192024-04-20T01:20:03.374Z 16 320-339 186 186 186 314 314 314 F F F
206202024-04-20T01:20:03.374Z 17 340-359 184 184 184 309 309 309 F F F
206212024-04-20T01:20:03.374Z 18 360-379 188 188 188 317 317 317 F F F
206222024-04-20T01:20:03.374Z 19 380-399 189 189 189 319 319 319 F F F
206232024-04-20T01:20:03.374Z 20 400-419 189 189 189 319 319 319 F F F
206242024-04-20T01:20:03.374Z 21 420-439 190 190 190 323 323 323 F F F
206252024-04-20T01:20:03.374Z 22 440-459 189 189 189 319 319 319 F F F
206262024-04-20T01:20:03.374Z 23 460-479 178 178 178 299 299 299 F F F
206272024-04-20T01:20:03.374Z 24 480-499 190 190 190 323 323 323 F F F
206282024-04-20T01:20:03.374Z 25 500-519 186 186 186 313 313 313 F F F
206292024-04-20T01:20:03.374Z 26 520-539 191 191 191 324 324 324 F F F
206302024-04-20T01:20:03.374Z 27 540-559 191 191 191 324 324 324 F F F
206312024-04-20T01:20:03.375Z 28 560-579 188 188 188 317 317 317 F F F
206322024-04-20T01:20:03.375Z 29 580-599 189 189 189 319 319 319 F F F
206332024-04-20T01:20:03.375ZMax gen: 191, Max flush: 324
206342024-04-20T01:20:03.375Z
206352024-04-20T01:20:03.375ZBegin loop 96
206362024-04-20T01:20:03.375ZDownstairs to restart: 2
206372024-04-20T01:20:03.390Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.883724681Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942}
206382024-04-20T01:20:03.390Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.883932411Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942}
206392024-04-20T01:20:03.397Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.883951981Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942}
206402024-04-20T01:20:03.397Z{"msg":"Crucible 8202af4a-5428-42d3-808b-a30d6b43ea24 has session id: 6b58e170-8217-4000-811a-b84bc2cf8fbe","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.883984691Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206412024-04-20T01:20:03.397Z{"msg":"Upstairs opts: Upstairs UUID: 8202af4a-5428-42d3-808b-a30d6b43ea24, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884029591Z","hostname":"Crucible runtime is spawned
206422024-04-20T01:20:03.397Zw-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206432024-04-20T01:20:03.397Z{"msg":"Crucible stats registered with UUID: 8202af4a-5428-42d3-808b-a30d6b43ea24","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884075691Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206442024-04-20T01:20:03.397Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884112942Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942}
206452024-04-20T01:20:03.397Z{"msg":"8202af4a-5428-42d3-808b-a30d6b43ea24 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884215842Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206462024-04-20T01:20:03.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884245882Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"0","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206472024-04-20T01:20:03.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884271032Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"0","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206482024-04-20T01:20:03.398Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884293212Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"1","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206492024-04-20T01:20:03.398Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884314942Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"1","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206502024-04-20T01:20:03.398Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884338172Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"2","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206512024-04-20T01:20:03.398Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884437722Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"2","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206522024-04-20T01:20:03.398Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884494722Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206532024-04-20T01:20:03.398Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884524903Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206542024-04-20T01:20:03.398Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884561303Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206552024-04-20T01:20:03.398Z{"msg":"ds_connection connected from Ok(127.0.0.1:53922)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884590243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206562024-04-20T01:20:03.398Z{"msg":"ds_connection connected from Ok(127.0.0.1:55807)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884644663Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206572024-04-20T01:20:03.398Z{"msg":"ds_connection connected from Ok(127.0.0.1:40273)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.884674623Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206582024-04-20T01:20:03.398Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.885343744Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"1","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206592024-04-20T01:20:03.398Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.885422145Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"1","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206602024-04-20T01:20:03.398Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.885471585Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"0","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206612024-04-20T01:20:03.398Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.885499645Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"0","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206622024-04-20T01:20:03.398Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.885945655Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"1","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206632024-04-20T01:20:03.398Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.886122486Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"0","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206642024-04-20T01:20:03.398Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.886429647Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"1","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206652024-04-20T01:20:03.398Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.886508777Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"1","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206662024-04-20T01:20:03.398Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.886554907Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206672024-04-20T01:20:03.398Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.886575647Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"0","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206682024-04-20T01:20:03.398Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.886610457Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"0","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206692024-04-20T01:20:03.398Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.886646227Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206702024-04-20T01:20:03.398Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.886660577Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"2","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206712024-04-20T01:20:03.399Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.886689957Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"2","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206722024-04-20T01:20:03.399Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.887157788Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"2","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206732024-04-20T01:20:03.399Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.887540909Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"2","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206742024-04-20T01:20:03.399Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.887585279Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"2","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206752024-04-20T01:20:03.399Z{"msg":"[0]R flush_numbers[0..12]: [297, 311, 320, 306, 320, 317, 320, 320, 322, 312, 317, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.887603689Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206762024-04-20T01:20:03.399Z{"msg":"[0]R generation[0..12]: [178, 184, 190, 182, 190, 188, 190, 190, 190, 185, 188, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.887618609Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206772024-04-20T01:20:03.399Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.887641459Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206782024-04-20T01:20:03.399Z{"msg":"[1]R flush_numbers[0..12]: [297, 311, 320, 306, 320, 317, 320, 320, 322, 312, 317, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.887656319Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206792024-04-20T01:20:03.399Z{"msg":"[1]R generation[0..12]: [178, 184, 190, 182, 190, 188, 190, 190, 190, 185, 188, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88767757Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206802024-04-20T01:20:03.399Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88769279Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206812024-04-20T01:20:03.399Z{"msg":"[2]R flush_numbers[0..12]: [297, 311, 320, 306, 320, 317, 320, 320, 322, 312, 317, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88770953Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"Wait for a query_work_queue command to finish before sending IO
206822024-04-20T01:20:03.399Z}
206832024-04-20T01:20:03.399Z{"msg":"[2]R generation[0..12]: [178, 184, 190, 182, 190, 188, 190, 190, 190, 185, 188, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.887735619Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206842024-04-20T01:20:03.399Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88775126Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206852024-04-20T01:20:03.399Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88776514Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206862024-04-20T01:20:03.399Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88777847Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206872024-04-20T01:20:03.399Z{"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88779891Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206882024-04-20T01:20:03.399Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88781232Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"":"downstairs","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206892024-04-20T01:20:03.399Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88782616Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206902024-04-20T01:20:03.399Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88784513Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"0","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206912024-04-20T01:20:03.399Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88785873Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"1","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206922024-04-20T01:20:03.399Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88787257Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"client":"2","":"downstairs","session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206932024-04-20T01:20:03.399Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88790095Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206942024-04-20T01:20:03.400Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88792281Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206952024-04-20T01:20:03.400Z{"msg":"8202af4a-5428-42d3-808b-a30d6b43ea24 is now active with session: 6b58e170-8217-4000-811a-b84bc2cf8fbe","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88794484Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206962024-04-20T01:20:03.400Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.88795718Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942,"session_id":"6b58e170-8217-4000-811a-b84bc2cf8fbe"}
206972024-04-20T01:20:03.400Z{"msg":"The guest has finished waiting for activation with:192",Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206982024-04-20T01:20:03.400Z"v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:01.8879757Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942}
206992024-04-20T01:20:03.407ZLoading write count information from file "/tmp/test_repair_verify.data"
207002024-04-20T01:20:03.407ZRun Repair workload
207012024-04-20T01:20:03.407Z01/30 Write block 507 len 36864 data: 10 11 16 16 17 18 17 15 14
207022024-04-20T01:20:03.407Z02/30 Read block 426 len 16384
207032024-04-20T01:20:03.407Z03/30 Write block 342 len 40960 data: 12 15 14 17 13 13 12 11 9 10
207042024-04-20T01:20:03.407Z04/30 Read block 316 len 32768
207052024-04-20T01:20:03.407Z05/30 Read block 452 len 28672
207062024-04-20T01:20:03.407Z06/30 Write block 341 len 4096 data: 11
207072024-04-20T01:20:03.407Z07/30 Write block 263 len 8192 data: 8 8
207082024-04-20T01:20:03.407Z08/30 Read block 69 len 32768
207092024-04-20T01:20:03.407Z09/30 Flush
207102024-04-20T01:20:03.411Z10/30 Write block 502 len 32768 data: 14 15 14 12 13 11 12 17
207112024-04-20T01:20:03.411Z11/30 Write block 197 len 32768 data: 15 16 18 13 15 17 12 12
207122024-04-20T01:20:03.411Z12/30 Read block 272 len 36864
207132024-04-20T01:20:03.411Z13/30 Read block 138 len 4096
207142024-04-20T01:20:03.411Z14/30 Write block 367 len 8192 data: 20 15
207152024-04-20T01:20:03.411Z15/30 Write block 61 len 8192 data: 10 11
207162024-04-20T01:20:03.411Z16/30 Read block 192 len 4096
207172024-04-20T01:20:03.415Z17/30 Write block 336 len 40960 data: 13 10 14 12 11 12 13 16 15 18
207182024-04-20T01:20:03.415Z18/30 Read block 84 len 24576
207192024-04-20T01:20:03.415Z19/30 Write block 584 len 4096 data: 20
207202024-04-20T01:20:03.415Z20/30 Write block 79 len 40960 data: 15 13 12 13 14 14 13 15 16 17
207212024-04-20T01:20:03.415Z21/30 Read block 509 len 28672
207222024-04-20T01:20:03.419Z22/30 Write block 265 len 12288 data: 10 13 11
207232024-04-20T01:20:03.420Z23/30 Write block 399 len 28672 data: 16 15 14 15 12 12 13
207242024-04-20T01:20:03.420Z24/30 Read block 92 len 32768
207252024-04-20T01:20:03.420Z25/30 Read block 75 len 4096
207262024-04-20T01:20:03.420Z26/30 Write block 483 len 4096 data: 19
207272024-04-20T01:20:03.420Z27/30 Write block 17 len 4096 data: 13
207282024-04-20T01:20:03.431Z28/30 Write block 114 len 16384 data: 15 17 17 15
207292024-04-20T01:20:03.431Z29/30 Read block 410 len 16384
207302024-04-20T01:20:03.431Z30/30 Write block 99 len 12288 data: 12 13 14
207312024-04-20T01:20:03.431Z----------------------------------------------------------------
207322024-04-20T01:20:03.431Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
207332024-04-20T01:20:03.431ZStates: Active Active Active
207342024-04-20T01:20:03.431ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
207352024-04-20T01:20:03.431Z 1 Acked 1000 Write 9 Done Done Done false
207362024-04-20T01:20:03.431Z 2 Acked 1001 Read 4 Done Done Done false
207372024-04-20T01:20:03.431Z 3 Acked 1002 Write 10 Done Done Done false
207382024-04-20T01:20:03.431Z 4 Acked 1003 Read 8 Done Done Done false
207392024-04-20T01:20:03.432Z 5 Acked 1004 Read 7 Done Done Sent false
207402024-04-20T01:20:03.432Z 6 Acked 1005 Write 1 Done Done Sent false
207412024-04-20T01:20:03.432Z 7 Acked 1006 Write 2 Done Done Sent false
207422024-04-20T01:20:03.432Z 8 Acked 1007 Read 8 Done Done Sent false
207432024-04-20T01:20:03.432Z 9 Acked 1008 Flush 0 Done Done Sent false
207442024-04-20T01:20:03.432Z 10 Acked 1009 Write 8 Done Done Sent false
207452024-04-20T01:20:03.432Z 11 Acked 1010 Write 8 Done Done Sent false
207462024-04-20T01:20:03.432Z 12 Acked 1011 Read 9 Done Done Sent false
207472024-04-20T01:20:03.432Z 13 Acked 1012 Read 1 Done Done Sent false
207482024-04-20T01:20:03.432Z 14 Acked 1013 Write 2 Done Done Sent false
207492024-04-20T01:20:03.432Z 15 Acked 1014 Write 2 Done Done Sent false
207502024-04-20T01:20:03.432Z 16 Acked 1015 Read 1 Done Done Sent false
207512024-04-20T01:20:03.432Z 17 Acked 1016 Write 10 Done Done Sent false
207522024-04-20T01:20:03.432Z 18 Acked 1017 Read 6 Done Done Sent false
207532024-04-20T01:20:03.432Z 19 Acked 1018 Write 1 Done Done Sent false
207542024-04-20T01:20:03.432Z 20 Acked 1019 Write 10 Done Done Sent false
207552024-04-20T01:20:03.432Z 21 Acked 1020 Read 7 Done Done Sent false
207562024-04-20T01:20:03.432Z 22 Acked 1021 Write 3 Done Done Sent false
207572024-04-20T01:20:03.432Z 23 Acked 1022 Write 7 Done Done Sent false
207582024-04-20T01:20:03.432Z 24 Acked 1023 Read 8 Done Done Sent false
207592024-04-20T01:20:03.432Z 25 Acked 1024 Read 1 Done Done Sent false
207602024-04-20T01:20:03.432Z 26 Acked 1025 Write 1 Done Done Sent false
207612024-04-20T01:20:03.432Z 27 Acked 1026 Write 1 Done Done Sent false
207622024-04-20T01:20:03.432Z 28 Acked 1027 Write 4 Done Done Sent false
207632024-04-20T01:20:03.433Z 29 Acked 1028 Read 4 Done Done Sent false
207642024-04-20T01:20:03.433Z 30 Acked 1029 Write 3 Sent Sent Sent false
207652024-04-20T01:20:03.433Z STATES DS:0 DS:1 DS:2 TOTAL
207662024-04-20T01:20:03.433Z New 0 0 0 0
207672024-04-20T01:20:03.433Z Sent 1 1 26 28
207682024-04-20T01:20:03.433Z Done 29 29 4 62
207692024-04-20T01:20:03.433Z Skipped 0 0 0 0
207702024-04-20T01:20:03.433Z Error 0 0 0 0
207712024-04-20T01:20:03.433ZLast Flush: 1008 1008 0
207722024-04-20T01:20:03.433ZDownstairs last five completed:
207732024-04-20T01:20:03.433ZUpstairs last five completed: 30 29 28 27 26
207742024-04-20T01:20:03.433Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:20:01.916198403Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1942}
207752024-04-20T01:20:03.433ZWrote out file "/tmp/test_repair_verify.data"
207762024-04-20T01:20:03.437Z
207772024-04-20T01:20:03.437ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
207782024-04-20T01:20:03.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207792024-04-20T01:20:03.454Z 0 000-019 178 178 178 297 297 297 T T F <---
207802024-04-20T01:20:03.454Z 1 020-039 184 184 184 311 311 311 F F F
207812024-04-20T01:20:03.454Z 2 040-059 190 190 190 320 320 320 F F F
207822024-04-20T01:20:03.454Z 3 060-079 182 182 182 306 306 306 T T F <---
207832024-04-20T01:20:03.454Z 4 080-099 190 190 190 320 320 320 T T F <---
207842024-04-20T01:20:03.454Z 5 100-119 188 188 188 317 317 317 T T F <---
207852024-04-20T01:20:03.458Z 6 120-139 190 190 190 320 320 320 F F F
207862024-04-20T01:20:03.458Z 7 140-159 190 190 190 320 320 320 F F F
207872024-04-20T01:20:03.458Z 8 160-179 190 190 190 322 322 322 F F F
207882024-04-20T01:20:03.458Z 9 180-199 185 185 185 312 312 312 T T F <---
207892024-04-20T01:20:03.458Z 10 200-219 188 188 188 317 317 317 T T F <---
207902024-04-20T01:20:03.458Z 11 220-239 169 169 169 279 279 279 F F F
207912024-04-20T01:20:03.458Z 12 240-259 191 191 191 324 324 324 F F F
207922024-04-20T01:20:03.458Z 13 260-279 192 192 188 325 325 317 T T F <---
207932024-04-20T01:20:03.458Z 14 280-299 188 188 188 317 317 317 F F F
207942024-04-20T01:20:03.458Z 15 300-319 186 186 186 313 313 313 F F F
207952024-04-20T01:20:03.458Z 16 320-339 186 186 186 314 314 314 T T F <---
207962024-04-20T01:20:03.458Z 17 340-359 192 192 184 325 325 309 T T T <---
207972024-04-20T01:20:03.459Z 18 360-379 188 188 188 317 317 317 T T F <---
207982024-04-20T01:20:03.459Z 19 380-399 189 189 189 319 319 319 T T F <---
207992024-04-20T01:20:03.459Z 20 400-419 189 189 189 319 319 319 T T F <---
208002024-04-20T01:20:03.459Z 21 420-439 190 190 190 323 323 323 F F F
208012024-04-20T01:20:03.459Z 22 440-459 189 189 189 319 319 319 F F F
208022024-04-20T01:20:03.459Z 23 460-479 178 178 178 299 299 299 F F F
208032024-04-20T01:20:03.459Z 24 480-499 190 190 190 323 323 323 T T F <---
208042024-04-20T01:20:03.459Z 25 500-519 192 192 186 325 325 313 T T T <---
208052024-04-20T01:20:03.459Z 26 520-539 191 191 191 324 324 324 F F F
208062024-04-20T01:20:03.459Z 27 540-559 191 191 191 324 324 324 F F F
208072024-04-20T01:20:03.459Z 28 560-579 188 188 188 317 317 317 F F F
208082024-04-20T01:20:03.459Z 29 580-599 189 189 189 319 319 319 T T F <---
208092024-04-20T01:20:03.459ZMax gen: 192, Max flush: 325
208102024-04-20T01:20:03.459ZError: Difference in extent metadata found!
208112024-04-20T01:20:03.459ZA Difference in extent metadata is expected here
208122024-04-20T01:20:03.459ZOn loop 96
208132024-04-20T01:20:03.459Z
208142024-04-20T01:20:03.459ZVerifying data now
208152024-04-20T01:20:18.155ZNo change after verify
208162024-04-20T01:20:18.155ZLoop: 96 Downstairs dump after verify (and repair):
208172024-04-20T01:20:18.170ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
208182024-04-20T01:20:18.170Z 0 000-019 193 193 193 326 326 326 F F F
208192024-04-20T01:20:18.170Z 1 020-039 184 184 184 311 311 311 F F F
208202024-04-20T01:20:18.170Z 2 040-059 190 190 190 320 320 320 F F F
208212024-04-20T01:20:18.170Z 3 060-079 193 193 193 326 326 326 F F F
208222024-04-20T01:20:18.180Z 4 080-099 193 193 193 326 326 326 F F F
208232024-04-20T01:20:18.180Z 5 100-119 193 193 193 326 326 326 F F F
208242024-04-20T01:20:18.180Z 6 120-139 190 190 190 320 320 320 F F F
208252024-04-20T01:20:18.180Z 7 140-159 190 190 190 320 320 320 F F F
208262024-04-20T01:20:18.180Z 8 160-179 190 190 190 322 322 322 F F F
208272024-04-20T01:20:18.181Z 9 180-199 193 193 193 326 326 326 F F F
208282024-04-20T01:20:18.181Z 10 200-219 193 193 193 326 326 326 F F F
208292024-04-20T01:20:18.181Z 11 220-239 169 169 169 279 279 279 F F F
208302024-04-20T01:20:18.181Z 12 240-259 191 191 191 324 324 324 F F F
208312024-04-20T01:20:18.181Z 13 260-279 193 193 193 326 326 326 F F F
208322024-04-20T01:20:18.181Z 14 280-299 188 188 188 317 317 317 F F F
208332024-04-20T01:20:18.181Z 15 300-319 186 186 186 313 313 313 F F F
208342024-04-20T01:20:18.181Z 16 320-339 193 193 193 326 326 326 F F F
208352024-04-20T01:20:18.181Z 17 340-359 193 193 193 326 326 326 F F F
208362024-04-20T01:20:18.181Z 18 360-379 193 193 193 326 326 326 F F F
208372024-04-20T01:20:18.181Z 19 380-399 193 193 193 326 326 326 F F F
208382024-04-20T01:20:18.181Z 20 400-419 193 193 193 326 326 326 F F F
208392024-04-20T01:20:18.181Z 21 420-439 190 190 190 323 323 323 F F F
208402024-04-20T01:20:18.181Z 22 440-459 189 189 189 319 319 319 F F F
208412024-04-20T01:20:18.181Z 23 460-479 178 178 178 299 299 299 F F F
208422024-04-20T01:20:18.181Z 24 480-499 193 193 193 326 326 326 F F F
208432024-04-20T01:20:18.181Z 25 500-519 193 193 193 326 326 326 F F F
208442024-04-20T01:20:18.181Z 26 520-539 191 191 191 324 324 324 F F F
208452024-04-20T01:20:18.182Z 27 540-559 191 191 191 324 324 324 F F F
208462024-04-20T01:20:18.182Z 28 560-579 188 188 188 317 317 317 F F F
208472024-04-20T01:20:18.182Z 29 580-599 193 193 193 326 326 326 F F F
208482024-04-20T01:20:18.182ZMax gen: 193, Max flush: 326
208492024-04-20T01:20:18.182Z
208502024-04-20T01:20:18.182ZBegin loop 97
208512024-04-20T01:20:18.182ZDownstairs to restart: 0
208522024-04-20T01:20:18.191Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684135267Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950}
208532024-04-20T01:20:18.191Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684387597Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950}
208542024-04-20T01:20:18.198Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684412017Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950}
208552024-04-20T01:20:18.199Z{"msg":"Crucible 586c3f4d-c9b7-41f8-a57b-9aca175b0f0e has session id: 0135fe84-e03e-40da-84e0-8c7435300225","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684455177Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208562024-04-20T01:20:18.199Z{"msg":"Crucible runtime is spawned
208572024-04-20T01:20:18.199ZUpstairs opts: Upstairs UUID: 586c3f4d-c9b7-41f8-a57b-9aca175b0f0e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684564797Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208582024-04-20T01:20:18.199Z{"msg":"Crucible stats registered with UUID: 586c3f4d-c9b7-41f8-a57b-9aca175b0f0e","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684605528Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208592024-04-20T01:20:18.199Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684621348Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950}
208602024-04-20T01:20:18.199Z{"msg":"586c3f4d-c9b7-41f8-a57b-9aca175b0f0e active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684697998Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208612024-04-20T01:20:18.199Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684724518Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"0","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208622024-04-20T01:20:18.199Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684739918Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"0","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208632024-04-20T01:20:18.199Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684753898Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"1","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208642024-04-20T01:20:18.199Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684767588Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"1","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208652024-04-20T01:20:18.199Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684790118Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"2","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208662024-04-20T01:20:18.200Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684820108Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"2","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208672024-04-20T01:20:18.200Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684852588Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208682024-04-20T01:20:18.200Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684888688Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208692024-04-20T01:20:18.200Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.684909068Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208702024-04-20T01:20:18.200Z{"msg":"ds_connection connected from Ok(127.0.0.1:52338)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.685125029Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208712024-04-20T01:20:18.200Z{"msg":"ds_connection connected from Ok(127.0.0.1:42662)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.685172259Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208722024-04-20T01:20:18.200Z{"msg":"ds_connection connected from Ok(127.0.0.1:55770)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.685202639Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208732024-04-20T01:20:18.200Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.685910671Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"2","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208742024-04-20T01:20:18.200Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.68596026Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"2","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208752024-04-20T01:20:18.200Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.685985711Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"1","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208762024-04-20T01:20:18.200Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.686043701Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"1","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208772024-04-20T01:20:18.200Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.686160921Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"0","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208782024-04-20T01:20:18.201Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.686203371Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"0","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208792024-04-20T01:20:18.201Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.686516732Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"2","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208802024-04-20T01:20:18.201Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.686629972Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"1","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208812024-04-20T01:20:18.201Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.686730752Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"0","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208822024-04-20T01:20:18.201Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687099993Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"1","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208832024-04-20T01:20:18.201Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687141263Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"1","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208842024-04-20T01:20:18.201Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687167053Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208852024-04-20T01:20:18.201Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687205923Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"2","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208862024-04-20T01:20:18.201Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687229243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"2","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208872024-04-20T01:20:18.201Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687250794Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208882024-04-20T01:20:18.201Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687263374Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"0","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208892024-04-20T01:20:18.201Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687277274Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"0","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208902024-04-20T01:20:18.202Z{"msg":"[0]R flush_numbers[0..12]: [326, 311, 320, 326, 326, 326, 320, 320, 322, 326, 326, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687291343Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208912024-04-20T01:20:18.202Z{"msg":"[0]R generation[0..12]: [193, 184, 190, 193, 193, 193, 190, 190, 190, 193, 193, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687307594Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208922024-04-20T01:20:18.202Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
208932024-04-20T01:20:18.202Z,"time":"2024-04-20T01:20:16.687332534Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208942024-04-20T01:20:18.202Z{"msg":"[1]R flush_numbers[0..12]: [326, 311, 320, 326, 326, 326, 320, 320, 322, 326, 326, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687358094Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208952024-04-20T01:20:18.202Z{"msg":"[1]R generation[0..12]: [193, 184, 190, 193, 193, 193, 190, 190, 190, 193, 193, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687390104Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208962024-04-20T01:20:18.202Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687416734Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208972024-04-20T01:20:18.202Z{"msg":"[2]R flush_numbers[0..12]: [326, 311, 320, 326, 326, 326, 320, 320, 322, 326, 326, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687442824Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208982024-04-20T01:20:18.202Z{"msg":"[2]R generation[0..12]: [193, 184, 190, 193, 193, 193, 190, 190, 190, 193, 193, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687475914Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
208992024-04-20T01:20:18.202Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687503514Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209002024-04-20T01:20:18.202Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687539304Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209012024-04-20T01:20:18.202ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
209022024-04-20T01:20:18.202Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687578444Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209032024-04-20T01:20:18.203Z{"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687602104Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209042024-04-20T01:20:18.203Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687635044Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"":"downstairs","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209052024-04-20T01:20:18.203Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687667514Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209062024-04-20T01:20:18.203Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687698124Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"0","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209072024-04-20T01:20:18.203Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687729204Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"1","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209082024-04-20T01:20:18.203Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687759615Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"client":"2","":"downstairs","session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209092024-04-20T01:20:18.203Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687790615Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209102024-04-20T01:20:18.203Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687819735Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209112024-04-20T01:20:18.203Z{"msg":"586c3f4d-c9b7-41f8-a57b-9aca175b0f0e is now active with session: 0135fe84-e03e-40da-84e0-8c7435300225","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687849175Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209122024-04-20T01:20:18.203Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687876515Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950,"session_id":"0135fe84-e03e-40da-84e0-8c7435300225"}
209132024-04-20T01:20:18.203Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:16.687906435Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950}
209142024-04-20T01:20:18.203ZLoading write count information from file "/tmp/test_repair_verify.data"
209152024-04-20T01:20:18.204ZRun Repair workload
209162024-04-20T01:20:18.204Z01/30 Write block 327 len 8192 data: 19 19
209172024-04-20T01:20:18.208Z02/30 Read block 359 len 20480
209182024-04-20T01:20:18.208Z03/30 Read block 77 len 20480
209192024-04-20T01:20:18.208Z04/30 Read block 66 len 8192
209202024-04-20T01:20:18.208Z05/30 Write block 529 len 4096 data: 13
209212024-04-20T01:20:18.209Z06/30 Read block 76 len 16384
209222024-04-20T01:20:18.209Z07/30 Read block 481 len 32768
209232024-04-20T01:20:18.209Z08/30 Read block 420 len 12288
209242024-04-20T01:20:18.209Z09/30 Write block 415 len 20480 data: 20 22 18 16 16
209252024-04-20T01:20:18.209Z10/30 Write block 46 len 24576 data: 15 14 14 16 17 18
209262024-04-20T01:20:18.209Z11/30 Read block 132 len 32768
209272024-04-20T01:20:18.209Z12/30 Read block 471 len 32768
209282024-04-20T01:20:18.209Z13/30 Write block 434 len 16384 data: 16 17 19 17
209292024-04-20T01:20:18.209Z14/30 Write block 61 len 4096 data: 11
209302024-04-20T01:20:18.209Z15/30 Write block 259 len 24576 data: 15 13 10 11 9 9
209312024-04-20T01:20:18.209Z16/30 Write block 82 len 40960 data: 14 15 15 14 16 17 18 17 20 19
209322024-04-20T01:20:18.210Z17/30 Read block 65 len 4096
209332024-04-20T01:20:18.214Z18/30 Write block 254 len 28672 data: 15 16 16 13 13 16 14
209342024-04-20T01:20:18.215Z19/30 Write block 274 len 16384 data: 16 17 14 16
209352024-04-20T01:20:18.215Z20/30 Read block 352 len 24576
209362024-04-20T01:20:18.215Z21/30 Read block 423 len 28672
209372024-04-20T01:20:18.215Z22/30 Write block 405 len 4096 data: 14
209382024-04-20T01:20:18.215Z23/30 Flush
209392024-04-20T01:20:18.220Z24/30 Flush
209402024-04-20T01:20:18.220Z25/30 Write block 21 len 16384 data: 9 9 8 9
209412024-04-20T01:20:18.220Z26/30 Write block 323 len 20480 data: 19 17 18 19 20
209422024-04-20T01:20:18.220Z27/30 Read block 347 len 16384
209432024-04-20T01:20:18.225Z28/30 Write block 414 len 24576 data: 23 21 23 19 17 17
209442024-04-20T01:20:18.225Z29/30 Read block 78 len 36864
209452024-04-20T01:20:18.225Z30/30 Write block 332 len 20480 data: 18 15 14 11 14
209462024-04-20T01:20:18.225Z----------------------------------------------------------------
209472024-04-20T01:20:18.225Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
209482024-04-20T01:20:18.225ZStates: Active Active Active
209492024-04-20T01:20:18.225ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
209502024-04-20T01:20:18.226Z 1 Acked 1000 Write 2 Done Done Done false
209512024-04-20T01:20:18.226Z 2 Acked 1001 Read 5 Done Done Done false
209522024-04-20T01:20:18.226Z 3 Acked 1002 Read 5 Done Done Done false
209532024-04-20T01:20:18.226Z 4 Acked 1003 Read 2 Done Done Done false
209542024-04-20T01:20:18.226Z 5 Acked 1004 Write 1 Sent Done Done false
209552024-04-20T01:20:18.226Z 6 Acked 1005 Read 4 Sent Done Done false
209562024-04-20T01:20:18.226Z 7 Acked 1006 Read 8 Sent Done Done false
209572024-04-20T01:20:18.226Z 8 Acked 1007 Read 3 Sent Done Done false
209582024-04-20T01:20:18.226Z 9 Acked 1008 Write 5 Sent Done Done false
209592024-04-20T01:20:18.226Z 10 Acked 1009 Write 6 Sent Done Done false
209602024-04-20T01:20:18.226Z 11 Acked 1010 Read 8 Sent Done Done false
209612024-04-20T01:20:18.226Z 12 Acked 1011 Read 8 Sent Done Done false
209622024-04-20T01:20:18.227Z 13 Acked 1012 Write 4 Sent Done Done false
209632024-04-20T01:20:18.227Z 14 Acked 1013 Write 1 Sent Done Done false
209642024-04-20T01:20:18.227Z 15 Acked 1014 Write 6 Sent Done Done false
209652024-04-20T01:20:18.227Z 16 Acked 1015 Write 10 Sent Done Done false
209662024-04-20T01:20:18.227Z 17 Acked 1016 Read 1 Sent Done Done false
209672024-04-20T01:20:18.227Z 18 Acked 1017 Write 7 Sent Done Done false
209682024-04-20T01:20:18.227Z 19 Acked 1018 Write 4 Sent Done Done false
209692024-04-20T01:20:18.227Z 20 Acked 1019 Read 6 Sent Done Done false
209702024-04-20T01:20:18.227Z 21 Acked 1020 Read 7 Sent Done Done false
209712024-04-20T01:20:18.227Z 22 Acked 1021 Write 1 Sent Done Done false
209722024-04-20T01:20:18.227Z 23 Acked 1022 Flush 0 Sent Done Done false
209732024-04-20T01:20:18.228Z 24 Acked 1023 Flush 0 Sent Done Done false
209742024-04-20T01:20:18.228Z 25 Acked 1024 Write 4 Sent Done Done false
209752024-04-20T01:20:18.228Z 26 Acked 1025 Write 5 Sent Done Done false
209762024-04-20T01:20:18.228Z 27 Acked 1026 Read 4 Sent Done Done false
209772024-04-20T01:20:18.228Z 28 Acked 1027 Write 6 Sent Done Done false
209782024-04-20T01:20:18.228Z 29 Acked 1028 Read 9 Sent Done Done false
209792024-04-20T01:20:18.228Z 30 Acked 1029 Write 5 Sent Sent Sent false
209802024-04-20T01:20:18.228Z STATES DS:0 DS:1 DS:2 TOTAL
209812024-04-20T01:20:18.228Z New 0 0 0 0
209822024-04-20T01:20:18.228Z Sent 26 1 1 28
209832024-04-20T01:20:18.228Z Done 4 29 29 62
209842024-04-20T01:20:18.228Z Skipped 0 0 0 0
209852024-04-20T01:20:18.229Z Error 0 0 0 0
209862024-04-20T01:20:18.229ZLast Flush: 0 1023 1023
209872024-04-20T01:20:18.229ZDownstairs last five completed:
209882024-04-20T01:20:18.229ZUpstairs last five completed: 30 29 28 27 26
209892024-04-20T01:20:18.229Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:20:16.716325488Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1950}
209902024-04-20T01:20:18.229ZWrote out file "/tmp/test_repair_verify.data"
209912024-04-20T01:20:18.233Z
209922024-04-20T01:20:18.234ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209932024-04-20T01:20:18.246ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209942024-04-20T01:20:18.246Z 0 000-019 193 193 193 326 326 326 F F F
209952024-04-20T01:20:18.246Z 1 020-039 184 184 184 311 311 311 F T T <---
209962024-04-20T01:20:18.246Z 2 040-059 190 194 194 320 327 327 F F F <---
209972024-04-20T01:20:18.246Z 3 060-079 193 194 194 326 327 327 F F F <---
209982024-04-20T01:20:18.251Z 4 080-099 193 194 194 326 327 327 F F F <---
209992024-04-20T01:20:18.251Z 5 100-119 193 193 193 326 326 326 F F F
210002024-04-20T01:20:18.251Z 6 120-139 190 190 190 320 320 320 F F F
210012024-04-20T01:20:18.251Z 7 140-159 190 190 190 320 320 320 F F F
210022024-04-20T01:20:18.251Z 8 160-179 190 190 190 322 322 322 F F F
210032024-04-20T01:20:18.251Z 9 180-199 193 193 193 326 326 326 F F F
210042024-04-20T01:20:18.251Z 10 200-219 193 193 193 326 326 326 F F F
210052024-04-20T01:20:18.251Z 11 220-239 169 169 169 279 279 279 F F F
210062024-04-20T01:20:18.251Z 12 240-259 191 194 194 324 327 327 F F F <---
210072024-04-20T01:20:18.251Z 13 260-279 193 194 194 326 327 327 F F F <---
210082024-04-20T01:20:18.251Z 14 280-299 188 188 188 317 317 317 F F F
210092024-04-20T01:20:18.251Z 15 300-319 186 186 186 313 313 313 F F F
210102024-04-20T01:20:18.251Z 16 320-339 193 194 194 326 327 327 T T T <---
210112024-04-20T01:20:18.252Z 17 340-359 193 193 193 326 326 326 F F F
210122024-04-20T01:20:18.252Z 18 360-379 193 193 193 326 326 326 F F F
210132024-04-20T01:20:18.252Z 19 380-399 193 193 193 326 326 326 F F F
210142024-04-20T01:20:18.252Z 20 400-419 193 194 194 326 327 327 F T T <---
210152024-04-20T01:20:18.252Z 21 420-439 190 194 194 323 327 327 F F F <---
210162024-04-20T01:20:18.252Z 22 440-459 189 189 189 319 319 319 F F F
210172024-04-20T01:20:18.252Z 23 460-479 178 178 178 299 299 299 F F F
210182024-04-20T01:20:18.252Z 24 480-499 193 193 193 326 326 326 F F F
210192024-04-20T01:20:18.252Z 25 500-519 193 193 193 326 326 326 F F F
210202024-04-20T01:20:18.252Z 26 520-539 191 194 194 324 327 327 F F F <---
210212024-04-20T01:20:18.252Z 27 540-559 191 191 191 324 324 324 F F F
210222024-04-20T01:20:18.252Z 28 560-579 188 188 188 317 317 317 F F F
210232024-04-20T01:20:18.252Z 29 580-599 193 193 193 326 326 326 F F F
210242024-04-20T01:20:18.252ZMax gen: 194, Max flush: 327
210252024-04-20T01:20:18.252ZError: Difference in extent metadata found!
210262024-04-20T01:20:18.252ZA Difference in extent metadata is expected here
210272024-04-20T01:20:18.252ZOn loop 97
210282024-04-20T01:20:18.252Z
210292024-04-20T01:20:18.252ZVerifying data now
210302024-04-20T01:20:32.724ZNo change after verify
210312024-04-20T01:20:32.741ZLoop: 97 Downstairs dump after verify (and repair):
210322024-04-20T01:20:32.742ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210332024-04-20T01:20:32.742Z 0 000-019 193 193 193 326 326 326 F F F
210342024-04-20T01:20:32.742Z 1 020-039 195 195 195 328 328 328 F F F
210352024-04-20T01:20:32.742Z 2 040-059 194 194 194 327 327 327 F F F
210362024-04-20T01:20:32.743Z 3 060-079 194 194 194 327 327 327 F F F
210372024-04-20T01:20:32.743Z 4 080-099 194 194 194 327 327 327 F F F
210382024-04-20T01:20:32.743Z 5 100-119 193 193 193 326 326 326 F F F
210392024-04-20T01:20:32.743Z 6 120-139 190 190 190 320 320 320 F F F
210402024-04-20T01:20:32.743Z 7 140-159 190 190 190 320 320 320 F F F
210412024-04-20T01:20:32.744Z 8 160-179 190 190 190 322 322 322 F F F
210422024-04-20T01:20:32.744Z 9 180-199 193 193 193 326 326 326 F F F
210432024-04-20T01:20:32.744Z 10 200-219 193 193 193 326 326 326 F F F
210442024-04-20T01:20:32.745Z 11 220-239 169 169 169 279 279 279 F F F
210452024-04-20T01:20:32.745Z 12 240-259 194 194 194 327 327 327 F F F
210462024-04-20T01:20:32.745Z 13 260-279 194 194 194 327 327 327 F F F
210472024-04-20T01:20:32.745Z 14 280-299 188 188 188 317 317 317 F F F
210482024-04-20T01:20:32.745Z 15 300-319 186 186 186 313 313 313 F F F
210492024-04-20T01:20:32.746Z 16 320-339 195 195 195 328 328 328 F F F
210502024-04-20T01:20:32.746Z 17 340-359 193 193 193 326 326 326 F F F
210512024-04-20T01:20:32.746Z 18 360-379 193 193 193 326 326 326 F F F
210522024-04-20T01:20:32.746Z 19 380-399 193 193 193 326 326 326 F F F
210532024-04-20T01:20:32.747Z 20 400-419 195 195 195 328 328 328 F F F
210542024-04-20T01:20:32.747Z 21 420-439 194 194 194 327 327 327 F F F
210552024-04-20T01:20:32.747Z 22 440-459 189 189 189 319 319 319 F F F
210562024-04-20T01:20:32.747Z 23 460-479 178 178 178 299 299 299 F F F
210572024-04-20T01:20:32.747Z 24 480-499 193 193 193 326 326 326 F F F
210582024-04-20T01:20:32.747Z 25 500-519 193 193 193 326 326 326 F F F
210592024-04-20T01:20:32.747Z 26 520-539 194 194 194 327 327 327 F F F
210602024-04-20T01:20:32.747Z 27 540-559 191 191 191 324 324 324 F F F
210612024-04-20T01:20:32.747Z 28 560-579 188 188 188 317 317 317 F F F
210622024-04-20T01:20:32.747Z 29 580-599 193 193 193 326 326 326 F F F
210632024-04-20T01:20:32.747ZMax gen: 195, Max flush: 328
210642024-04-20T01:20:32.756Z
210652024-04-20T01:20:32.756ZBegin loop 98
210662024-04-20T01:20:32.757ZDownstairs to restart: 0
210672024-04-20T01:20:32.778Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.249178698Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958}
210682024-04-20T01:20:32.778Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.249413009Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958}
210692024-04-20T01:20:32.778Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.249436259Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958}
210702024-04-20T01:20:32.778Z{"msg":"Crucible 01bc36ff-4958-49a6-baf1-3b8cfe5dcfb4 has session id: 81e7cad2-f169-4dbd-acd7-9450dcb34b77","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.249470529Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210712024-04-20T01:20:32.779Z{"msg":"Upstairs opts: Upstairs UUID: 01bc36ff-4958-49a6-baf1-3b8cfe5dcfb4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.249514459Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"Crucible runtime is spawned
210722024-04-20T01:20:32.779Zsession_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210732024-04-20T01:20:32.779Z{"msg":"Crucible stats registered with UUID: 01bc36ff-4958-49a6-baf1-3b8cfe5dcfb4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.249556959Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210742024-04-20T01:20:32.779Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.24962997Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958}
210752024-04-20T01:20:32.779Z{"msg":"01bc36ff-4958-49a6-baf1-3b8cfe5dcfb4 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.24972405Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210762024-04-20T01:20:32.779Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.24974941Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"0","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210772024-04-20T01:20:32.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.24979321Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"0","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210782024-04-20T01:20:32.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.24981919Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"1","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210792024-04-20T01:20:32.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.24985663Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"1","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210802024-04-20T01:20:32.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.24988103Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"2","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210812024-04-20T01:20:32.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.24993851Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"2","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210822024-04-20T01:20:32.780Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.24999831Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210832024-04-20T01:20:32.781Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.25005142Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210842024-04-20T01:20:32.781Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.25010228Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210852024-04-20T01:20:32.781Z{"msg":"ds_connection connected from Ok(127.0.0.1:62806)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.250154891Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210862024-04-20T01:20:32.781Z{"msg":"ds_connection connected from Ok(127.0.0.1:49596)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.250210811Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210872024-04-20T01:20:32.781Z{"msg":"ds_connection connected from Ok(127.0.0.1:50082)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.250251641Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210882024-04-20T01:20:32.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.250712982Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"1","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210892024-04-20T01:20:32.782Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.250758072Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"1","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210902024-04-20T01:20:32.782Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.250783162Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"2","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210912024-04-20T01:20:32.782Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.250806602Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"2","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210922024-04-20T01:20:32.782Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.251111123Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"0","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210932024-04-20T01:20:32.782Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.251179243Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"0","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210942024-04-20T01:20:32.782Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.251253623Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"1","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210952024-04-20T01:20:32.783Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.251291903Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"2","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210962024-04-20T01:20:32.783Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.251717514Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"0","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210972024-04-20T01:20:32.783Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.251921324Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"1","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210982024-04-20T01:20:32.783Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.251968835Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"1","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
210992024-04-20T01:20:32.783Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252004035Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211002024-04-20T01:20:32.783Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252034165Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"2","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211012024-04-20T01:20:32.784Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252068235Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"2","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211022024-04-20T01:20:32.784Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252100565Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211032024-04-20T01:20:32.784Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252258575Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"0","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211042024-04-20T01:20:32.784Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252316965Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"0","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211052024-04-20T01:20:32.784Z{"msg":"[0]R flush_numbers[0..12]: [326, 328, 327, 327, 327, 326, 320, 320, 322, 326, 326, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252357445Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211062024-04-20T01:20:32.784Z{"msg":"[0]R generation[0..12]: [193, 195, 194, 194, 194, 193, 190, 190, 190, 193, 193, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252380876Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211072024-04-20T01:20:32.785Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
211082024-04-20T01:20:32.785Z30,"time":"2024-04-20T01:20:31.252413826Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211092024-04-20T01:20:32.785Z{"msg":"[1]R flush_numbers[0..12]: [326, 328, 327, 327, 327, 326, 320, 320, 322, 326, 326, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252431606Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211102024-04-20T01:20:32.785Z{"msg":"[1]R generation[0..12]: [193, 195, 194, 194, 194, 193, 190, 190, 190, 193, 193, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252462626Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211112024-04-20T01:20:32.785Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252483656Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211122024-04-20T01:20:32.785Z{"msg":"[2]R flush_numbers[0..12]: [326, 328, 327, 327, 327, 326, 320, 320, 322, 326, 326, 279]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252519216Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211132024-04-20T01:20:32.785Z{"msg":"[2]R generation[0..12]: [193, 195, 194, 194, 194, 193, 190, 190, 190, 193, 193, 169]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252554176Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211142024-04-20T01:20:32.786Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252587046Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211152024-04-20T01:20:32.786Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252618076Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211162024-04-20T01:20:32.786Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
211172024-04-20T01:20:32.786ZGeneration requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252652606Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211182024-04-20T01:20:32.786Z{"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252683666Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211192024-04-20T01:20:32.786Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252703456Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"":"downstairs","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211202024-04-20T01:20:32.786Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252731516Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211212024-04-20T01:20:32.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252748256Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"0","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211222024-04-20T01:20:32.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252776846Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"1","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211232024-04-20T01:20:32.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252791777Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"client":"2","":"downstairs","session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211242024-04-20T01:20:32.787Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252821966Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211252024-04-20T01:20:32.787Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252839306Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211262024-04-20T01:20:32.787Z{"msg":"01bc36ff-4958-49a6-baf1-3b8cfe5dcfb4 is now active with session: 81e7cad2-f169-4dbd-acd7-9450dcb34b77","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252867697Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211272024-04-20T01:20:32.788Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252894987Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958,"session_id":"81e7cad2-f169-4dbd-acd7-9450dcb34b77"}
211282024-04-20T01:20:32.788Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:31.252926487Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958}
211292024-04-20T01:20:32.788ZLoading write count information from file "/tmp/test_repair_verify.data"
211302024-04-20T01:20:32.788ZRun Repair workload
211312024-04-20T01:20:32.788Z01/30 Write block 141 len 32768 data: 21 21 22 22 22 18 14 12
211322024-04-20T01:20:33.200Z02/30 Write block 237 len 32768 data: 18 15 14 10 10 9 8 13
211332024-04-20T01:20:33.200Z03/30 Flush
211342024-04-20T01:20:33.200Z04/30 Write block 229 len 8192 data: 12 13
211352024-04-20T01:20:33.200Z05/30 Write block 332 len 12288 data: 19 16 15
211362024-04-20T01:20:33.200Z06/30 Flush
211372024-04-20T01:20:33.200Z07/30 Write block 130 len 28672 data: 15 16 18 15 12 14 15
211382024-04-20T01:20:33.200Z08/30 Read block 104 len 36864
211392024-04-20T01:20:33.200Z09/30 Write block 133 len 40960 data: 16 13 15 16 16 17 18 20 22 22
211402024-04-20T01:20:33.200Z10/30 Write block 31 len 40960 data: 11 11 10 12 12 11 13 15 17 15
211412024-04-20T01:20:33.200Z11/30 Read block 457 len 4096
211422024-04-20T01:20:33.200Z12/30 Flush
211432024-04-20T01:20:33.200Z13/30 Write block 442 len 32768 data: 16 11 13 13 13 14 15 15
211442024-04-20T01:20:33.200Z14/30 Write block 131 len 40960 data: 17 19 17 14 16 17 17 18 19 21
211452024-04-20T01:20:33.201Z15/30 Read block 278 len 36864
211462024-04-20T01:20:33.201Z16/30 Read block 368 len 4096
211472024-04-20T01:20:33.201Z17/30 Write block 453 len 28672 data: 12 12 14 14 16 14 13
211482024-04-20T01:20:33.201Z18/30 Read block 55 len 36864
211492024-04-20T01:20:33.201Z19/30 Write block 248 len 8192 data: 19 19
211502024-04-20T01:20:33.201Z20/30 Read block 520 len 36864
211512024-04-20T01:20:33.201Z21/30 Write block 483 len 20480 data: 20 14 14 15 13
211522024-04-20T01:20:33.201Z22/30 Write block 19 len 4096 data: 11
211532024-04-20T01:20:33.201Z23/30 Write block 555 len 36864 data: 14 10 10 8 10 9 11 13 12
211542024-04-20T01:20:33.201Z24/30 Read block 536 len 20480
211552024-04-20T01:20:33.201Z25/30 Flush
211562024-04-20T01:20:33.207Z26/30 Write block 304 len 12288 data: 14 15 14
211572024-04-20T01:20:33.207Z27/30 Write block 505 len 4096 data: 13
211582024-04-20T01:20:33.207Z28/30 Write block 95 len 4096 data: 14
211592024-04-20T01:20:33.207Z29/30 Read block 382 len 12288
211602024-04-20T01:20:33.207Z30/30 Read block 340 len 8192
211612024-04-20T01:20:33.207Z----------------------------------------------------------------
211622024-04-20T01:20:33.207Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:27
211632024-04-20T01:20:33.207ZStates: Active Active Active
211642024-04-20T01:20:33.207ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211652024-04-20T01:20:33.207Z 4 Acked 1003 Write 2 Sent Done Done false
211662024-04-20T01:20:33.207Z 5 Acked 1004 Write 3 Sent Done Done false
211672024-04-20T01:20:33.207Z 6 Acked 1005 Flush 0 Sent Done Done false
211682024-04-20T01:20:33.208Z 7 Acked 1006 Write 7 Sent Done Done false
211692024-04-20T01:20:33.208Z 8 Acked 1007 Read 9 Sent Done Done false
211702024-04-20T01:20:33.208Z 9 Acked 1008 Write 10 Sent Done Done false
211712024-04-20T01:20:33.208Z 10 Acked 1009 Write 10 Sent Done Done false
211722024-04-20T01:20:33.208Z 11 Acked 1010 Read 1 Sent Done Done false
211732024-04-20T01:20:33.208Z 12 Acked 1011 Flush 0 Sent Done Done false
211742024-04-20T01:20:33.208Z 13 Acked 1012 Write 8 Sent Done Done false
211752024-04-20T01:20:33.208Z 14 Acked 1013 Write 10 Sent Done Done false
211762024-04-20T01:20:33.208Z 15 Acked 1014 Read 9 Sent Done Done false
211772024-04-20T01:20:33.208Z 16 Acked 1015 Read 1 Sent Done Done false
211782024-04-20T01:20:33.208Z 17 Acked 1016 Write 7 Sent Done Done false
211792024-04-20T01:20:33.208Z 18 Acked 1017 Read 9 Sent Done Done false
211802024-04-20T01:20:33.208Z 19 Acked 1018 Write 2 Sent Done Done false
211812024-04-20T01:20:33.208Z 20 Acked 1019 Read 9 Sent Done Done false
211822024-04-20T01:20:33.208Z 21 Acked 1020 Write 5 Sent Done Done false
211832024-04-20T01:20:33.208Z 22 Acked 1021 Write 1 Sent Done Done false
211842024-04-20T01:20:33.208Z 23 Acked 1022 Write 9 Sent Done Done false
211852024-04-20T01:20:33.208Z 24 Acked 1023 Read 5 Sent Done Done false
211862024-04-20T01:20:33.208Z 25 Acked 1024 Flush 0 Sent Done Done false
211872024-04-20T01:20:33.208Z 26 Acked 1025 Write 3 Sent Done Done false
211882024-04-20T01:20:33.208Z 27 Acked 1026 Write 1 Sent Done Done false
211892024-04-20T01:20:33.208Z 28 Acked 1027 Write 1 Sent Done Done false
211902024-04-20T01:20:33.208Z 29 Acked 1028 Read 3 Sent Done Done false
211912024-04-20T01:20:33.209Z 30 Acked 1029 Read 2 Sent Done Done false
211922024-04-20T01:20:33.209Z STATES DS:0 DS:1 DS:2 TOTAL
211932024-04-20T01:20:33.209Z New 0 0 0 0
211942024-04-20T01:20:33.209Z Sent 27 0 0 27
211952024-04-20T01:20:33.209Z Done 0 27 27 54
211962024-04-20T01:20:33.209Z Skipped 0 0 0 0
211972024-04-20T01:20:33.209Z Error 0 0 0 0
211982024-04-20T01:20:33.209ZLast Flush: 1002 1024 1024
211992024-04-20T01:20:33.209ZDownstairs last five completed: 1002 1001 1000
212002024-04-20T01:20:33.209ZUpstairs last five completed: 30 29 28 27 26
212012024-04-20T01:20:33.209Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:20:31.291648203Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1958}
212022024-04-20T01:20:33.209ZWrote out file "/tmp/test_repair_verify.data"
212032024-04-20T01:20:33.209Z
212042024-04-20T01:20:33.209ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
212052024-04-20T01:20:33.209ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212062024-04-20T01:20:33.209Z 0 000-019 193 196 196 326 332 332 F F F <---
212072024-04-20T01:20:33.209Z 1 020-039 195 196 196 328 331 331 F F F <---
212082024-04-20T01:20:33.209Z 2 040-059 194 196 196 327 331 331 F F F <---
212092024-04-20T01:20:33.209Z 3 060-079 194 194 194 327 327 327 F F F
212102024-04-20T01:20:33.209Z 4 080-099 194 194 194 327 327 327 F T T <---
212112024-04-20T01:20:33.209Z 5 100-119 193 193 193 326 326 326 F F F
212122024-04-20T01:20:33.209Z 6 120-139 190 196 196 320 332 332 F F F <---
212132024-04-20T01:20:33.209Z 7 140-159 196 196 196 329 332 332 F F F <---
212142024-04-20T01:20:33.210Z 8 160-179 190 190 190 322 322 322 F F F
212152024-04-20T01:20:33.210Z 9 180-199 193 193 193 326 326 326 F F F
212162024-04-20T01:20:33.210Z 10 200-219 193 193 193 326 326 326 F F F
212172024-04-20T01:20:33.210Z 11 220-239 196 196 196 329 330 330 F F F <---
212182024-04-20T01:20:33.210Z 12 240-259 196 196 196 329 332 332 F F F <---
212192024-04-20T01:20:33.210Z 13 260-279 194 194 194 327 327 327 F F F
212202024-04-20T01:20:33.210Z 14 280-299 188 188 188 317 317 317 F F F
212212024-04-20T01:20:33.210Z 15 300-319 186 186 186 313 313 313 F T T <---
212222024-04-20T01:20:33.210Z 16 320-339 195 196 196 328 330 330 F F F <---
212232024-04-20T01:20:33.210Z 17 340-359 193 193 193 326 326 326 F F F
212242024-04-20T01:20:33.210Z 18 360-379 193 193 193 326 326 326 F F F
212252024-04-20T01:20:33.210Z 19 380-399 193 193 193 326 326 326 F F F
212262024-04-20T01:20:33.210Z 20 400-419 195 195 195 328 328 328 F F F
212272024-04-20T01:20:33.210Z 21 420-439 194 194 194 327 327 327 F F F
212282024-04-20T01:20:33.210Z 22 440-459 189 196 196 319 332 332 F F F <---
212292024-04-20T01:20:33.210Z 23 460-479 178 178 178 299 299 299 F F F
212302024-04-20T01:20:33.210Z 24 480-499 193 196 196 326 332 332 F F F <---
212312024-04-20T01:20:33.210Z 25 500-519 193 193 193 326 326 326 F T T <---
212322024-04-20T01:20:33.210Z 26 520-539 194 194 194 327 327 327 F F F
212332024-04-20T01:20:33.210Z 27 540-559 191 196 196 324 332 332 F F F <---
212342024-04-20T01:20:33.210Z 28 560-579 188 196 196 317 332 332 F F F <---
212352024-04-20T01:20:33.210Z 29 580-599 193 193 193 326 326 326 F F F
212362024-04-20T01:20:33.210ZMax gen: 196, Max flush: 332
212372024-04-20T01:20:33.210ZError: Difference in extent metadata found!
212382024-04-20T01:20:33.211ZA Difference in extent metadata is expected here
212392024-04-20T01:20:33.211ZOn loop 98
212402024-04-20T01:20:33.211Z
212412024-04-20T01:20:33.211ZVerifying data now
212422024-04-20T01:20:37.496ZNo change after verify
212432024-04-20T01:20:37.496ZLoop: 98 Downstairs dump after verify (and repair):
212442024-04-20T01:20:37.509ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212452024-04-20T01:20:37.509Z 0 000-019 196 196 196 332 332 332 F F F
212462024-04-20T01:20:37.509Z 1 020-039 196 196 196 331 331 331 F F F
212472024-04-20T01:20:37.509Z 2 040-059 196 196 196 331 331 331 F F F
212482024-04-20T01:20:37.509Z 3 060-079 194 194 194 327 327 327 F F F
212492024-04-20T01:20:37.509Z 4 080-099 197 197 197 333 333 333 F F F
212502024-04-20T01:20:37.515Z 5 100-119 193 193 193 326 326 326 F F F
212512024-04-20T01:20:37.515Z 6 120-139 196 196 196 332 332 332 F F F
212522024-04-20T01:20:37.515Z 7 140-159 196 196 196 332 332 332 F F F
212532024-04-20T01:20:37.515Z 8 160-179 190 190 190 322 322 322 F F F
212542024-04-20T01:20:37.515Z 9 180-199 193 193 193 326 326 326 F F F
212552024-04-20T01:20:37.515Z 10 200-219 193 193 193 326 326 326 F F F
212562024-04-20T01:20:37.515Z 11 220-239 196 196 196 330 330 330 F F F
212572024-04-20T01:20:37.515Z 12 240-259 196 196 196 332 332 332 F F F
212582024-04-20T01:20:37.515Z 13 260-279 194 194 194 327 327 327 F F F
212592024-04-20T01:20:37.515Z 14 280-299 188 188 188 317 317 317 F F F
212602024-04-20T01:20:37.515Z 15 300-319 197 197 197 333 333 333 F F F
212612024-04-20T01:20:37.515Z 16 320-339 196 196 196 330 330 330 F F F
212622024-04-20T01:20:37.515Z 17 340-359 193 193 193 326 326 326 F F F
212632024-04-20T01:20:37.515Z 18 360-379 193 193 193 326 326 326 F F F
212642024-04-20T01:20:37.515Z 19 380-399 193 193 193 326 326 326 F F F
212652024-04-20T01:20:37.515Z 20 400-419 195 195 195 328 328 328 F F F
212662024-04-20T01:20:37.515Z 21 420-439 194 194 194 327 327 327 F F F
212672024-04-20T01:20:37.515Z 22 440-459 196 196 196 332 332 332 F F F
212682024-04-20T01:20:37.515Z 23 460-479 178 178 178 299 299 299 F F F
212692024-04-20T01:20:37.515Z 24 480-499 196 196 196 332 332 332 F F F
212702024-04-20T01:20:37.515Z 25 500-519 197 197 197 333 333 333 F F F
212712024-04-20T01:20:37.515Z 26 520-539 194 194 194 327 327 327 F F F
212722024-04-20T01:20:37.516Z 27 540-559 196 196 196 332 332 332 F F F
212732024-04-20T01:20:37.516Z 28 560-579 196 196 196 332 332 332 F F F
212742024-04-20T01:20:37.516Z 29 580-599 193 193 193 326 326 326 F F F
212752024-04-20T01:20:37.516ZMax gen: 197, Max flush: 333
212762024-04-20T01:20:37.516Z
212772024-04-20T01:20:37.516ZBegin loop 99
212782024-04-20T01:20:37.516ZDownstairs to restart: 2
212792024-04-20T01:20:37.531Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.023912017Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966}
212802024-04-20T01:20:37.531Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024213858Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966}
212812024-04-20T01:20:37.541Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024256758Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966}
212822024-04-20T01:20:37.541Z{"msg":"Crucible e48bc3f1-3f82-4c64-b806-b9955e7ef6ca has session id: eb3e0d56-a941-45ff-bf05-f719d30456db"Crucible runtime is spawned
212832024-04-20T01:20:37.541Z,"v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024297758Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212842024-04-20T01:20:37.541Z{"msg":"Upstairs opts: Upstairs UUID: e48bc3f1-3f82-4c64-b806-b9955e7ef6ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024361338Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212852024-04-20T01:20:37.541Z{"msg":"Crucible stats registered with UUID: e48bc3f1-3f82-4c64-b806-b9955e7ef6ca","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024405278Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212862024-04-20T01:20:37.541Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024425298Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966}
212872024-04-20T01:20:37.541Z{"msg":"e48bc3f1-3f82-4c64-b806-b9955e7ef6ca active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024467958Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212882024-04-20T01:20:37.541Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024522228Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"0","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212892024-04-20T01:20:37.541Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024556968Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"0","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212902024-04-20T01:20:37.541Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024598068Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"1","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212912024-04-20T01:20:37.541Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024639908Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"1","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212922024-04-20T01:20:37.541Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024688599Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"2","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212932024-04-20T01:20:37.541Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024775639Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"2","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212942024-04-20T01:20:37.541Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024865859Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212952024-04-20T01:20:37.541Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024912609Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212962024-04-20T01:20:37.541Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024946399Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212972024-04-20T01:20:37.541Z{"msg":"ds_connection connected from Ok(127.0.0.1:62088)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024972859Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212982024-04-20T01:20:37.542Z{"msg":"ds_connection connected from Ok(127.0.0.1:38841)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.024996639Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
212992024-04-20T01:20:37.542Z{"msg":"ds_connection connected from Ok(127.0.0.1:55226)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.025021559Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213002024-04-20T01:20:37.542Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.025580941Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"0","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213012024-04-20T01:20:37.542Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.025627781Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"0","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213022024-04-20T01:20:37.542Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.025654821Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"1","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213032024-04-20T01:20:37.542Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.025699771Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"1","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213042024-04-20T01:20:37.542Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.026164272Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"2","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213052024-04-20T01:20:37.542Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.026215122Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"2","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213062024-04-20T01:20:37.542Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.026256422Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"0","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213072024-04-20T01:20:37.542Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.026290182Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"1","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213082024-04-20T01:20:37.542Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.026709123Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"1","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213092024-04-20T01:20:37.542Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.026796113Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"1","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213102024-04-20T01:20:37.542Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.026851473Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213112024-04-20T01:20:37.542Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.026885564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"2","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213122024-04-20T01:20:37.542Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.026901404Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"0","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213132024-04-20T01:20:37.542Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.026922934Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"0","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213142024-04-20T01:20:37.542Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.026936954Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213152024-04-20T01:20:37.542Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027190084Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"2","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213162024-04-20T01:20:37.542Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027227564Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"2","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213172024-04-20T01:20:37.542Z{"msg":"[0]R flush_numbers[0..12]: [332, 331, 331, 327, 333, 326, 332, 332, 322, 326, 326, 330]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027245574Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213182024-04-20T01:20:37.542Z{"msg":"[0]R generation[0..12]: [196, 196, 196, 194, 197, 193, 196, 196, 190, 193, 193, 196]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027268494Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213192024-04-20T01:20:37.542Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027283465Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213202024-04-20T01:20:37.542Z{"msg":"[1]R flush_numbers[0..12]: [332, 331, 331, 327, 333, 326, 332, 332, 322, 326, 326, 330]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027305165Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213212024-04-20T01:20:37.543Z{"msg":"[1]R generation[0..12]: [196, 196, 196, 194, 197, 193, 196, 196, 190, 193, 193, 196]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027319534Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213222024-04-20T01:20:37.543Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
213232024-04-20T01:20:37.543Z,"time":"2024-04-20T01:20:36.027341874Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213242024-04-20T01:20:37.543Z{"msg":"[2]R flush_numbers[0..12]: [332, 331, 331, 327, 333, 326, 332, 332, 322, 326, 326, 330]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027357785Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213252024-04-20T01:20:37.543Z{"msg":"[2]R generation[0..12]: [196, 196, 196, 194, 197, 193, 196, 196, 190, 193, 193, 196]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027371745Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213262024-04-20T01:20:37.543Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027393415Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213272024-04-20T01:20:37.543Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027406735Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213282024-04-20T01:20:37.543Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027422225Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213292024-04-20T01:20:37.543Z{"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027435595Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213302024-04-20T01:20:37.543Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027455635Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"":"downstairs","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213312024-04-20T01:20:37.543Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027469305Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213322024-04-20T01:20:37.543Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027488595Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"0","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213332024-04-20T01:20:37.543Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027502385Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"1","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213342024-04-20T01:20:37.543Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027522915Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"client":"2","":"downstairs","session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213352024-04-20T01:20:37.543Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027536525Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213362024-04-20T01:20:37.543Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027549185Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213372024-04-20T01:20:37.543Z{"msg":"e48bc3f1-3f82-4c64-b806-b9955e7ef6ca is now active with session: eb3e0d56-a941-45ff-bf05-f719d30456db","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027569645Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213382024-04-20T01:20:37.543Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027583125Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966,"session_id":"eb3e0d56-a941-45ff-bf05-f719d30456db"}
213392024-04-20T01:20:37.543Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:20:36.027596545Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966}
213402024-04-20T01:20:37.543ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
213412024-04-20T01:20:37.543ZLoading write count information from file "/tmp/test_repair_verify.data"
213422024-04-20T01:20:37.543ZRun Repair workload
213432024-04-20T01:20:37.547Z01/30 Write block 270 len 20480 data: 12 14 15 20 17
213442024-04-20T01:20:37.547Z02/30 Flush
213452024-04-20T01:20:37.548Z03/30 Write block 232 len 24576 data: 16 20 21 18 18 19
213462024-04-20T01:20:37.548Z04/30 Write block 49 len 16384 data: 17 18 19 19
213472024-04-20T01:20:37.548Z05/30 Read block 427 len 8192
213482024-04-20T01:20:37.548Z06/30 Read block 472 len 40960
213492024-04-20T01:20:37.548Z07/30 Read block 253 len 20480
213502024-04-20T01:20:37.548Z08/30 Write block 136 len 16384 data: 18 18 19 20
213512024-04-20T01:20:37.548Z09/30 Write block 273 len 8192 data: 21 18
213522024-04-20T01:20:37.548Z10/30 Read block 559 len 28672
213532024-04-20T01:20:37.548Z11/30 Read block 509 len 16384
213542024-04-20T01:20:37.548Z12/30 Read block 432 len 12288
213552024-04-20T01:20:37.548Z13/30 Read block 500 len 20480
213562024-04-20T01:20:37.548Z14/30 Flush
213572024-04-20T01:20:37.552Z15/30 Write block 14 len 8192 data: 12 14
213582024-04-20T01:20:37.552Z16/30 Read block 74 len 20480
213592024-04-20T01:20:37.556Z17/30 Write block 19 len 40960 data: 12 11 10 10 9 10 9 11 11 11
213602024-04-20T01:20:37.556Z18/30 Write block 211 len 36864 data: 12 13 16 15 15 18 22 22 16
213612024-04-20T01:20:37.556Z19/30 Read block 420 len 40960
213622024-04-20T01:20:37.560Z20/30 Write block 139 len 36864 data: 21 22 23 23 23 23 23 19 15
213632024-04-20T01:20:37.560Z21/30 Read block 537 len 16384
213642024-04-20T01:20:37.560Z22/30 Write block 320 len 16384 data: 16 18 18 20
213652024-04-20T01:20:37.560Z23/30 Read block 390 len 16384
213662024-04-20T01:20:37.561Z24/30 Read block 232 len 28672
213672024-04-20T01:20:37.564Z25/30 Write block 535 len 20480 data: 15 13 12 12 12
213682024-04-20T01:20:37.564Z26/30 Read block 583 len 36864
213692024-04-20T01:20:37.564Z27/30 Read block 458 len 8192
213702024-04-20T01:20:37.564Z28/30 Read block 87 len 8192
213712024-04-20T01:20:37.564Z29/30 Write block 531 len 12288 data: 15 15 14
213722024-04-20T01:20:37.568Z30/30 Write block 530 len 40960 data: 13 16 16 15 17 16 14 13 13 13
213732024-04-20T01:20:37.568Z----------------------------------------------------------------
213742024-04-20T01:20:37.568Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:28
213752024-04-20T01:20:37.568ZStates: Active Active Active
213762024-04-20T01:20:37.568ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213772024-04-20T01:20:37.568Z 3 Acked 1002 Write 6 Done Done Sent false
213782024-04-20T01:20:37.569Z 4 Acked 1003 Write 4 Done Done Sent false
213792024-04-20T01:20:37.569Z 5 Acked 1004 Read 2 Done Done Sent false
213802024-04-20T01:20:37.569Z 6 Acked 1005 Read 10 Done Done Sent false
213812024-04-20T01:20:37.569Z 7 Acked 1006 Read 5 Done Done Sent false
213822024-04-20T01:20:37.569Z 8 Acked 1007 Write 4 Done Done Sent false
213832024-04-20T01:20:37.569Z 9 Acked 1008 Write 2 Done Done Sent false
213842024-04-20T01:20:37.569Z 10 Acked 1009 Read 7 Done Done Sent false
213852024-04-20T01:20:37.569Z 11 Acked 1010 Read 4 Done Done Sent false
213862024-04-20T01:20:37.569Z 12 Acked 1011 Read 3 Done Done Sent false
213872024-04-20T01:20:37.569Z 13 Acked 1012 Read 5 Done Done Sent false
213882024-04-20T01:20:37.569Z 14 Acked 1013 Flush 0 Done Done Sent false
213892024-04-20T01:20:37.569Z 15 Acked 1014 Write 2 Done Done Sent false
213902024-04-20T01:20:37.569Z 16 Acked 1015 Read 5 Done Done Sent false
213912024-04-20T01:20:37.569Z 17 Acked 1016 Write 10 Done Done Sent false
213922024-04-20T01:20:37.569Z 18 Acked 1017 Write 9 Done Done Sent false
213932024-04-20T01:20:37.569Z 19 Acked 1018 Read 10 Done Done Sent false
213942024-04-20T01:20:37.569Z 20 Acked 1019 Write 9 Done Done Sent false
213952024-04-20T01:20:37.569Z 21 Acked 1020 Read 4 Done Done Sent false
213962024-04-20T01:20:37.569Z 22 Acked 1021 Write 4 Done Done Sent false
213972024-04-20T01:20:37.569Z 23 Acked 1022 Read 4 Done Done Sent false
213982024-04-20T01:20:37.569Z 24 Acked 1023 Read 7 Done Done Sent false
213992024-04-20T01:20:37.569Z 25 Acked 1024 Write 5 Done Done Sent false
214002024-04-20T01:20:37.569Z 26 Acked 1025 Read 9 Done Done Sent false
214012024-04-20T01:20:37.570Z 27 Acked 1026 Read 2 Done Done Sent false
214022024-04-20T01:20:37.570Z 28 Acked 1027 Read 2 Done Done Sent false
214032024-04-20T01:20:37.570Z 29 Acked 1028 Write 3 Done Done Sent false
214042024-04-20T01:20:37.570Z 30 Acked 1029 Write 10 Sent Sent Sent false
214052024-04-20T01:20:37.570Z STATES DS:0 DS:1 DS:2 TOTAL
214062024-04-20T01:20:37.570Z New 0 0 0 0
214072024-04-20T01:20:37.570Z Sent 1 1 28 30
214082024-04-20T01:20:37.570Z Done 27 27 0 54
214092024-04-20T01:20:37.570Z Skipped 0 0 0 0
214102024-04-20T01:20:37.570Z Error 0 0 0 0
214112024-04-20T01:20:37.570ZLast Flush: 1013 1013 1001
214122024-04-20T01:20:37.570ZDownstairs last five completed: 1001 1000
214132024-04-20T01:20:37.570ZUpstairs last five completed: 30 29 28 27 26
214142024-04-20T01:20:37.570Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:20:36.058452674Z","hostname":"w-01HVWGBKT3EF1RFQT1PYDDZP22","pid":1966}
214152024-04-20T01:20:37.570ZWrote out file "/tmp/test_repair_verify.data"
214162024-04-20T01:20:37.574Z
214172024-04-20T01:20:37.574ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
214182024-04-20T01:20:37.589ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214192024-04-20T01:20:37.589Z 0 000-019 196 196 196 332 332 332 T T F <---
214202024-04-20T01:20:37.589Z 1 020-039 196 196 196 331 331 331 T T F <---
214212024-04-20T01:20:37.589Z 2 040-059 198 198 196 335 335 331 F F F <---
214222024-04-20T01:20:37.589Z 3 060-079 194 194 194 327 327 327 F F F
214232024-04-20T01:20:37.589Z 4 080-099 197 197 197 333 333 333 F F F
214242024-04-20T01:20:37.593Z 5 100-119 193 193 193 326 326 326 F F F
214252024-04-20T01:20:37.593Z 6 120-139 198 198 196 335 335 332 T T F <---
214262024-04-20T01:20:37.593Z 7 140-159 196 196 196 332 332 332 T T F <---
214272024-04-20T01:20:37.593Z 8 160-179 190 190 190 322 322 322 F F F
214282024-04-20T01:20:37.593Z 9 180-199 193 193 193 326 326 326 F F F
214292024-04-20T01:20:37.593Z 10 200-219 193 193 193 326 326 326 T T F <---
214302024-04-20T01:20:37.594Z 11 220-239 198 198 196 335 335 330 F F F <---
214312024-04-20T01:20:37.594Z 12 240-259 196 196 196 332 332 332 F F F
214322024-04-20T01:20:37.594Z 13 260-279 198 198 198 335 335 334 F F F <---
214332024-04-20T01:20:37.594Z 14 280-299 188 188 188 317 317 317 F F F
214342024-04-20T01:20:37.594Z 15 300-319 197 197 197 333 333 333 F F F
214352024-04-20T01:20:37.594Z 16 320-339 196 196 196 330 330 330 T T F <---
214362024-04-20T01:20:37.594Z 17 340-359 193 193 193 326 326 326 F F F
214372024-04-20T01:20:37.594Z 18 360-379 193 193 193 326 326 326 F F F
214382024-04-20T01:20:37.594Z 19 380-399 193 193 193 326 326 326 F F F
214392024-04-20T01:20:37.594Z 20 400-419 195 195 195 328 328 328 F F F
214402024-04-20T01:20:37.594Z 21 420-439 194 194 194 327 327 327 F F F
214412024-04-20T01:20:37.594Z 22 440-459 196 196 196 332 332 332 F F F
214422024-04-20T01:20:37.594Z 23 460-479 178 178 178 299 299 299 F F F
214432024-04-20T01:20:37.594Z 24 480-499 196 196 196 332 332 332 F F F
214442024-04-20T01:20:37.594Z 25 500-519 197 197 197 333 333 333 F F F
214452024-04-20T01:20:37.594Z 26 520-539 194 194 194 327 327 327 T T F <---
214462024-04-20T01:20:37.594Z 27 540-559 196 196 196 332 332 332 F F F
214472024-04-20T01:20:37.594Z 28 560-579 196 196 196 332 332 332 F F F
214482024-04-20T01:20:37.594Z 29 580-599 193 193 193 326 326 326 F F F
214492024-04-20T01:20:37.594ZMax gen: 198, Max flush: 335
214502024-04-20T01:20:37.594ZError: Difference in extent metadata found!
214512024-04-20T01:20:37.594ZA Difference in extent metadata is expected here
214522024-04-20T01:20:37.594ZOn loop 99
214532024-04-20T01:20:37.595Z
214542024-04-20T01:20:37.598ZVerifying data now
214552024-04-20T01:20:42.079ZNo change after verify
214562024-04-20T01:20:42.079ZLoop: 99 Downstairs dump after verify (and repair):
214572024-04-20T01:20:42.094ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214582024-04-20T01:20:42.094Z 0 000-019 199 199 199 336 336 336 F F F
214592024-04-20T01:20:42.094Z 1 020-039 199 199 199 336 336 336 F F F
214602024-04-20T01:20:42.094Z 2 040-059 198 198 198 335 335 335 F F F
214612024-04-20T01:20:42.094Z 3 060-079 194 194 194 327 327 327 F F F
214622024-04-20T01:20:42.094Z 4 080-099 197 197 197 333 333 333 F F F
214632024-04-20T01:20:42.094Z 5 100-119 193 193 193 326 326 326 F F F
214642024-04-20T01:20:42.098Z 6 120-139 199 199 199 336 336 336 F F F
214652024-04-20T01:20:42.098Z 7 140-159 199 199 199 336 336 336 F F F
214662024-04-20T01:20:42.099Z 8 160-179 190 190 190 322 322 322 F F F
214672024-04-20T01:20:42.099Z 9 180-199 193 193 193 326 326 326 F F F
214682024-04-20T01:20:42.099Z 10 200-219 199 199 199 336 336 336 F F F
214692024-04-20T01:20:42.099Z 11 220-239 198 198 198 335 335 335 F F F
214702024-04-20T01:20:42.099Z 12 240-259 196 196 196 332 332 332 F F F
214712024-04-20T01:20:42.099Z 13 260-279 198 198 198 335 335 335 F F F
214722024-04-20T01:20:42.099Z 14 280-299 188 188 188 317 317 317 F F F
214732024-04-20T01:20:42.099Z 15 300-319 197 197 197 333 333 333 F F F
214742024-04-20T01:20:42.099Z 16 320-339 199 199 199 336 336 336 F F F
214752024-04-20T01:20:42.099Z 17 340-359 193 193 193 326 326 326 F F F
214762024-04-20T01:20:42.099Z 18 360-379 193 193 193 326 326 326 F F F
214772024-04-20T01:20:42.099Z 19 380-399 193 193 193 326 326 326 F F F
214782024-04-20T01:20:42.099Z 20 400-419 195 195 195 328 328 328 F F F
214792024-04-20T01:20:42.099Z 21 420-439 194 194 194 327 327 327 F F F
214802024-04-20T01:20:42.099Z 22 440-459 196 196 196 332 332 332 F F F
214812024-04-20T01:20:42.099Z 23 460-479 178 178 178 299 299 299 F F F
214822024-04-20T01:20:42.099Z 24 480-499 196 196 196 332 332 332 F F F
214832024-04-20T01:20:42.099Z 25 500-519 197 197 197 333 333 333 F F F
214842024-04-20T01:20:42.099Z 26 520-539 199 199 199 336 336 336 F F F
214852024-04-20T01:20:42.099Z 27 540-559 196 196 196 332 332 332 F F F
214862024-04-20T01:20:42.099Z 28 560-579 196 196 196 332 332 332 F F F
214872024-04-20T01:20:42.099Z 29 580-599 193 193 193 326 326 326 F F F
214882024-04-20T01:20:42.099ZMax gen: 199, Max flush: 336
214892024-04-20T01:20:42.099Z15:46 Test duration
214902024-04-20T01:20:42.100ZTest completed
214912024-04-20T01:20:42.100Z
214922024-04-20T01:20:42.103Zreal 15:46.111160733
214932024-04-20T01:20:42.104Zuser 1:26.698995600
214942024-04-20T01:20:42.104Zsys 17.679648842
214952024-04-20T01:20:42.104Ztrap 0.078692935
214962024-04-20T01:20:42.104Ztflt 0.045237952
214972024-04-20T01:20:42.104Zdflt 0.048655102
214982024-04-20T01:20:42.104Zkflt 0.003114341
214992024-04-20T01:20:42.104Zlock 22:46:35.262802688
215002024-04-20T01:20:42.104Zslp 1:16:42.658471909
215012024-04-20T01:20:42.104Zlat 6.878584595
215022024-04-20T01:20:42.104Zstop 0.050125088
215032024-04-20T01:20:42.104Zprocess exited: duration 953313 ms, exit code 0
 
215042024-04-20T01:20:42.118Zfound 4 output files
215052024-04-20T01:20:42.118Zuploading: /tmp/test_repair_ds8810.txt (51743 bytes)
215062024-04-20T01:20:43.164Zuploaded: /tmp/test_repair_ds8810.txt
215072024-04-20T01:20:43.164Zuploading: /tmp/test_repair_ds8820.txt (106563 bytes)
215082024-04-20T01:20:44.218Zuploaded: /tmp/test_repair_ds8820.txt
215092024-04-20T01:20:44.218Zuploading: /tmp/test_repair_ds8830.txt (18062 bytes)
215102024-04-20T01:20:45.243Zuploaded: /tmp/test_repair_ds8830.txt
215112024-04-20T01:20:45.243Zuploading: /tmp/test_repair_out.txt (57520 bytes)
215122024-04-20T01:20:46.681Zuploaded: /tmp/test_repair_out.txt