01J2CPRGJZV0M0HNNR9ZYADGB4: test-up-encrypted

BasicConfig {
    output_rules: [
        "/tmp/test_up/*.txt",
        "/tmp/test_up/dsc/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01J2CPRYMC6RW00GC43YVVZ48Y

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-07-09T21:48:35.048Zjob dependencies complete; ready to run (waiting for 12 m 53 s)
22024-07-09T21:49:47.547Zjob assigned to worker 01J2CQGX6HBYPH0PACK3598HBZ (queued for 1 m 12 s)
32024-07-09T21:49:55.474Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-07-09T21:49:56.519Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-07-09T21:49:56.519Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-07-09T21:49:57.413Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-07-09T21:49:57.413Zdownloading input: /input/build/work/bins/crutest.gz
82024-07-09T21:49:58.428Zdownloaded input: /input/build/work/bins/crutest.gz
92024-07-09T21:49:58.428Zdownloading input: /input/build/work/bins/dsc.gz
102024-07-09T21:49:59.117Zdownloaded input: /input/build/work/bins/dsc.gz
112024-07-09T21:49:59.120Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-07-09T21:49:59.146Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-07-09T21:49:59.146Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-07-09T21:49:59.161Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-07-09T21:49:59.161Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-07-09T21:49:59.182Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-07-09T21:49:59.182Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-07-09T21:49:59.199Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-07-09T21:49:59.199Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-07-09T21:49:59.221Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-07-09T21:49:59.221Zdownloading input: /input/build/work/scripts/test_up.sh
222024-07-09T21:49:59.250Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-07-09T21:49:59.250Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-07-09T21:49:59.279Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-07-09T21:49:59.283Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-07-09T21:50:00.017Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-07-09T21:50:00.017Zstarting task 0: "setup"
282024-07-09T21:50:00.068Z++ uname -s
292024-07-09T21:50:00.072Z+ kern=SunOS
302024-07-09T21:50:00.072Z+ case "$kern" in
312024-07-09T21:50:00.072Z+ groupadd -g 12345 build
322024-07-09T21:50:00.074Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-07-09T21:50:02.084Z+ zfs create -o mountpoint=/work rpool/work
342024-07-09T21:50:02.912Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-07-09T21:50:02.915Z+ home_fs=zfs
362024-07-09T21:50:02.915Z+ [[ zfs == autofs ]]
372024-07-09T21:50:02.915Z+ mkdir -p /home/build
382024-07-09T21:50:02.922Z+ chown build:build /home/build /work
392024-07-09T21:50:03.920Z+ chmod 0700 /home/build /work
402024-07-09T21:50:03.923Zprocess exited: duration 3904 ms, exit code 0
 
412024-07-09T21:50:03.929Zstarting task 1: "authentication"
422024-07-09T21:50:03.947Zprocess exited: duration 17 ms, exit code 0
 
432024-07-09T21:50:03.954Zstarting task 2: "build"
442024-07-09T21:50:03.957Z+ banner cores
452024-07-09T21:50:03.979Z
462024-07-09T21:50:03.979Z #### #### ##### ###### ####
472024-07-09T21:50:03.979Z # # # # # # # #
482024-07-09T21:50:03.979Z # # # # # ##### ####
492024-07-09T21:50:03.979Z # # # ##### # #
502024-07-09T21:50:03.979Z # # # # # # # # #
512024-07-09T21:50:03.979Z #### #### # # ###### ####
522024-07-09T21:50:03.979Z
532024-07-09T21:50:03.982Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-07-09T21:50:03.989Z+ echo 'input bins dir contains:'
552024-07-09T21:50:03.989Zinput bins dir contains:
562024-07-09T21:50:03.992Z+ ls -ltr /input/build/work/bins
572024-07-09T21:50:03.992Ztotal 683763
582024-07-09T21:50:03.992Z-rw-r--r-- 1 root root 103149412 Jul 9 21:49 crucible-downstairs.gz
592024-07-09T21:50:03.992Z-rw-r--r-- 1 root root 81519276 Jul 9 21:49 crucible-hammer.gz
602024-07-09T21:50:03.992Z-rw-r--r-- 1 root root 100894766 Jul 9 21:49 crutest.gz
612024-07-09T21:50:03.992Z-rw-r--r-- 1 root root 64288036 Jul 9 21:49 dsc.gz
622024-07-09T21:50:03.992Z+ banner unpack
632024-07-09T21:50:03.995Z
642024-07-09T21:50:03.995Z # # # # ##### ## #### # #
652024-07-09T21:50:03.995Z # # ## # # # # # # # # #
662024-07-09T21:50:03.995Z # # # # # # # # # # ####
672024-07-09T21:50:03.995Z # # # # # ##### ###### # # #
682024-07-09T21:50:03.995Z # # # ## # # # # # # #
692024-07-09T21:50:03.995Z #### # # # # # #### # #
702024-07-09T21:50:03.995Z
712024-07-09T21:50:03.995Z+ mkdir -p /var/tmp/bins
722024-07-09T21:50:03.995Z+ for t in "$input/bins/"*.gz
732024-07-09T21:50:03.998Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-07-09T21:50:03.998Z+ b=crucible-downstairs.gz
752024-07-09T21:50:03.998Z+ b=crucible-downstairs
762024-07-09T21:50:03.998Z+ gunzip
772024-07-09T21:50:06.173Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-07-09T21:50:06.177Z+ for t in "$input/bins/"*.gz
792024-07-09T21:50:06.180Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-07-09T21:50:06.180Z+ b=crucible-hammer.gz
812024-07-09T21:50:06.180Z+ b=crucible-hammer
822024-07-09T21:50:06.180Z+ gunzip
832024-07-09T21:50:07.947Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-07-09T21:50:07.949Z+ for t in "$input/bins/"*.gz
852024-07-09T21:50:07.952Z++ basename /input/build/work/bins/crutest.gz
862024-07-09T21:50:07.952Z+ b=crutest.gz
872024-07-09T21:50:07.952Z+ b=crutest
882024-07-09T21:50:07.952Z+ gunzip
892024-07-09T21:50:10.022Z+ chmod +x /var/tmp/bins/crutest
902024-07-09T21:50:10.025Z+ for t in "$input/bins/"*.gz
912024-07-09T21:50:10.028Z++ basename /input/build/work/bins/dsc.gz
922024-07-09T21:50:10.028Z+ b=dsc.gz
932024-07-09T21:50:10.028Z+ b=dsc
942024-07-09T21:50:10.028Z+ gunzip
952024-07-09T21:50:11.390Z+ chmod +x /var/tmp/bins/dsc
962024-07-09T21:50:11.393Z+ export BINDIR=/var/tmp/bins
972024-07-09T21:50:11.393Z+ BINDIR=/var/tmp/bins
982024-07-09T21:50:11.393Z+ banner test_up_encrypted
992024-07-09T21:50:11.395Z
1002024-07-09T21:50:11.395Z ##### ###### #### ##### # # ##### ###### # #
1012024-07-09T21:50:11.396Z # # # # # # # # # ## #
1022024-07-09T21:50:11.396Z # ##### #### # # # # # ##### # # #
1032024-07-09T21:50:11.396Z # # # # # # ##### # # # #
1042024-07-09T21:50:11.396Z # # # # # # # # # # ##
1052024-07-09T21:50:11.396Z # ###### #### # ####### #### # ####### ###### # #
1062024-07-09T21:50:11.396Z
1072024-07-09T21:50:11.396Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
1082024-07-09T21:50:11.422Z/input/build/work
1092024-07-09T21:50:11.429ZTurn off color for downstairs dump
1102024-07-09T21:50:11.548ZUpstairs using key: JNbE7w/ts23of1tnsp1kJwjAXmLd5A0zZM8sppXQfVI=
1112024-07-09T21:50:11.551Zdsc output goes to /tmp/test_up/dsc-out.txt
1122024-07-09T21:50:11.551ZCreating three downstairs regions
1132024-07-09T21:50:11.642ZStarting three downstairs
1142024-07-09T21:50:16.650ZDisable automatic restart on all downstairs
1152024-07-09T21:50:16.710Z
1162024-07-09T21:50:16.710ZBegin tests, output goes to /tmp/test_up/test_up_out.txt
1172024-07-09T21:50:16.715ZRunning test: span
1182024-07-09T21:50:20.763ZCompleted test: span
1192024-07-09T21:50:25.770ZRunning test: big
1202024-07-09T21:50:30.062ZCompleted test: big
1212024-07-09T21:50:35.069ZRunning test: dep
1222024-07-09T21:51:22.115ZCompleted test: dep
1232024-07-09T21:51:27.121ZRunning test: balloon
1242024-07-09T21:52:05.854ZCompleted test: balloon
1252024-07-09T21:52:10.860ZRunning test: deactivate
1262024-07-09T21:53:05.463ZCompleted test: deactivate
1272024-07-09T21:53:10.470ZRunning hammer
1282024-07-09T21:53:21.701ZRun repair tests
1292024-07-09T21:53:21.706Z/var/tmp/bins/crutest fill -g 31 -q --key JNbE7w/ts23of1tnsp1kJwjAXmLd5A0zZM8sppXQfVI= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file
1302024-07-09T21:53:21.714Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.395967519Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210}
1312024-07-09T21:53:21.714Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"23d2929dfa82e5493027add06127e56457f6c86e\",\n git_commit_timestamp: \"2024-07-09T21:34:58.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.396220887Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210}
1322024-07-09T21:53:21.714Z{"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.396252127Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210}
1332024-07-09T21:53:21.720Z{"msg":"Crucible 2e5d6575-e7d4-4992-9604-e01e25e02e35 has session id: b6b70b06-261a-404d-9e0d-57b6c55f1595","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.396736792Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1342024-07-09T21:53:21.720Z{"msg":"Upstairs opts: Upstairs UUID: 2e5d6575-e7d4-4992-9604-e01e25e02e35, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.396813411Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1352024-07-09T21:53:21.720Z{"msg":"Crucible stats registered with UUID: 2e5d6575-e7d4-4992-9604-e01e25e02e35","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.396843561Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1362024-07-09T21:53:21.721ZCrucible runtime is spawned
1372024-07-09T21:53:21.721Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.397178118Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210}
1382024-07-09T21:53:21.721Z{"msg":"2e5d6575-e7d4-4992-9604-e01e25e02e35 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.397264447Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1392024-07-09T21:53:21.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.397298917Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"0","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1402024-07-09T21:53:21.721Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.397316336Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"0","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1412024-07-09T21:53:21.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.397353756Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"1","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1422024-07-09T21:53:21.721Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.397377366Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"1","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1432024-07-09T21:53:21.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.397410285Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"2","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1442024-07-09T21:53:21.721Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.397434885Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"2","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1452024-07-09T21:53:21.721Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.397462395Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"io task","client":"2","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1462024-07-09T21:53:21.721Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.397489395Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"io task","client":"0","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1472024-07-09T21:53:21.721Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.397526954Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"io task","client":"1","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1482024-07-09T21:53:21.721Z{"msg":"ds_connection connected from Ok(127.0.0.1:55925)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.397723872Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"io task","client":"0","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1492024-07-09T21:53:21.722Z{"msg":"ds_connection connected from Ok(127.0.0.1:59205)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.39801151Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"io task","client":"2","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1502024-07-09T21:53:21.722Z{"msg":"ds_connection connected from Ok(127.0.0.1:64868)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.398061019Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"io task","client":"1","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1512024-07-09T21:53:21.722Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.398493035Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"2","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1522024-07-09T21:53:21.722Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.398534045Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"2","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1532024-07-09T21:53:21.722Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.398551574Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"0","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1542024-07-09T21:53:21.722Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.398566104Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"0","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1552024-07-09T21:53:21.722Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.398591974Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"1","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1562024-07-09T21:53:21.722Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.398621054Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"1","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1572024-07-09T21:53:21.722Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399164778Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"2","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1582024-07-09T21:53:21.722Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399242958Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"1","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1592024-07-09T21:53:21.722Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399281967Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"0","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1602024-07-09T21:53:21.722Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399551444Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"2","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1612024-07-09T21:53:21.722Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399598694Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"2","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1622024-07-09T21:53:21.722Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399620014Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1632024-07-09T21:53:21.722Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399673413Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"1","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1642024-07-09T21:53:21.722Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399712033Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"1","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1652024-07-09T21:53:21.722Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399745793Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1662024-07-09T21:53:21.722Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399760282Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"0","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1672024-07-09T21:53:21.723Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399780992Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"0","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1682024-07-09T21:53:21.723Z{"msg":"[0]R flush_numbers: [1669, 1669, 1669, 1669, 1669]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399794782Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1692024-07-09T21:53:21.723Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399815272Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1702024-07-09T21:53:21.723Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399829232Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1712024-07-09T21:53:21.723Z{"msg":"[1]R flush_numbers: [1669, 1669, 1669, 1669, 1669]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399849412Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1722024-07-09T21:53:21.723Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399862932Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1732024-07-09T21:53:21.723Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399884121Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1742024-07-09T21:53:21.723Z{"msg":"[2]R flush_numbers: [1669, 1669, 1669, 1669, 1669]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399897911Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1752024-07-09T21:53:21.723ZWait for a query_work_queue command to finish before sending IO
1762024-07-09T21:53:21.723Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399914411Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1772024-07-09T21:53:21.723Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399928571Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1782024-07-09T21:53:21.723Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399948421Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1792024-07-09T21:53:21.723Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.399968371Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1802024-07-09T21:53:21.723Z{"msg":"Next flush: 1670","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.40001697Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1812024-07-09T21:53:21.723Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.40005265Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"":"downstairs","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1822024-07-09T21:53:21.723Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.400077939Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1832024-07-09T21:53:21.724Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.400118889Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"0","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1842024-07-09T21:53:21.724Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.400146279Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"1","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1852024-07-09T21:53:21.724Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.400170129Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"client":"2","":"downstairs","session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1862024-07-09T21:53:21.724Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.400190958Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1872024-07-09T21:53:21.724Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.400225758Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1882024-07-09T21:53:21.724Z{"msg":"2e5d6575-e7d4-4992-9604-e01e25e02e35 is now active with session: b6b70b06-261a-404d-9e0d-57b6c55f1595","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.400254808Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1892024-07-09T21:53:21.724Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:20.400267758Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210,"session_id":"b6b70b06-261a-404d-9e0d-57b6c55f1595"}
1902024-07-09T21:53:21.724Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30Region: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
1912024-07-09T21:53:21.724Z,"time":"2024-07-09T21:53:20.400295237Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210}
1922024-07-09T21:53:21.724ZFill test
1932024-07-09T21:53:21.754ZRead and Verify all blocks (0..50 range:false)
1942024-07-09T21:53:21.795ZWrote out file "/var/tmp/test_up/verify_file"
1952024-07-09T21:53:21.795ZCLIENT: Tests done. All submitted work has been ACK'd
1962024-07-09T21:53:21.798Z----------------------------------------------------------------
1972024-07-09T21:53:21.798Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
1982024-07-09T21:53:21.798ZStates: Active Active Active
1992024-07-09T21:53:21.798ZGW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2002024-07-09T21:53:21.799Z 3 Acked 1002 Read 50 Done Done Done false
2012024-07-09T21:53:21.799Z STATES DS:0 DS:1 DS:2 TOTAL
2022024-07-09T21:53:21.799Z New 0 0 0 0
2032024-07-09T21:53:21.799Z Sent 0 0 0 0
2042024-07-09T21:53:21.799Z Done 1 1 1 3
2052024-07-09T21:53:21.799Z Skipped 0 0 0 0
2062024-07-09T21:53:21.799Z Error 0 0 0 0
2072024-07-09T21:53:21.799ZLast Flush: 1001 1001 1001
2082024-07-09T21:53:21.799ZDownstairs last five completed: 1001 1000
2092024-07-09T21:53:21.799ZUpstairs last five completed: 3 2 1
2102024-07-09T21:53:21.799ZCLIENT: Up:0 ds:1 act:3
2112024-07-09T21:53:25.796Z----------------------------------------------------------------
2122024-07-09T21:53:25.796Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2132024-07-09T21:53:25.797ZDownstairs last five completed: 1003 1002 1001 1000
2142024-07-09T21:53:25.797ZUpstairs last five completed: 4 3 2 1
2152024-07-09T21:53:25.797ZCLIENT: Up:0 ds:0 act:3
2162024-07-09T21:53:25.797ZCLIENT: All crucible jobs finished, exiting program
2172024-07-09T21:53:25.799Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-09T21:53:24.480801227Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1210}
2182024-07-09T21:53:25.804ZRepair setup passed
2192024-07-09T21:53:25.808ZCopy the 8830 file
2202024-07-09T21:53:25.811Zcp -r /var/tmp/test_up/8830 /var/tmp/test_up/previous
2212024-07-09T21:53:25.813Z/var/tmp/bins/crutest repair -g 32 -q --key JNbE7w/ts23of1tnsp1kJwjAXmLd5A0zZM8sppXQfVI= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file --verify-in /var/tmp/test_up/verify_file
2222024-07-09T21:53:25.824Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.507964453Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216}
2232024-07-09T21:53:25.824Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"23d2929dfa82e5493027add06127e56457f6c86e\",\n git_commit_timestamp: \"2024-07-09T21:34:58.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.508181651Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216}
2242024-07-09T21:53:25.827Z{"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.508204581Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216}
2252024-07-09T21:53:25.827Z{"msg":"Crucible 0964ad53-8d7a-490a-ae30-8192a4201a39 has session id: c8a99fa7-f0fa-4a54-9aeb-33759254391f","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.508804485Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2262024-07-09T21:53:25.827Z{"msg":"Upstairs opts: Upstairs UUID: 0964ad53-8d7a-490a-ae30-8192a4201a39, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.508874354Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2272024-07-09T21:53:25.827Z{"msg":"Crucible stats registered with UUID: 0964ad53-8d7a-490a-ae30-8192a4201a39","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.508907574Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2282024-07-09T21:53:25.827ZCrucible runtime is spawned
2292024-07-09T21:53:25.827Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.50929433Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216}
2302024-07-09T21:53:25.827Z{"msg":"0964ad53-8d7a-490a-ae30-8192a4201a39 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.50935403Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2312024-07-09T21:53:25.827Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.509388949Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"0","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2322024-07-09T21:53:25.827Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.509418199Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"0","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2332024-07-09T21:53:25.827Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.509463509Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"1","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2342024-07-09T21:53:25.827Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.509494838Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"1","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2352024-07-09T21:53:25.827Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.509536918Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"2","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2362024-07-09T21:53:25.827Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.509557988Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"2","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2372024-07-09T21:53:25.827Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.509588927Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2382024-07-09T21:53:25.827Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.509609047Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2392024-07-09T21:53:25.828Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.509634937Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2402024-07-09T21:53:25.828Z{"msg":"ds_connection connected from Ok(127.0.0.1:58269)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.510083952Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2412024-07-09T21:53:25.828Z{"msg":"ds_connection connected from Ok(127.0.0.1:49666)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.510146692Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2422024-07-09T21:53:25.828Z{"msg":"ds_connection connected from Ok(127.0.0.1:50245)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.510166982Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2432024-07-09T21:53:25.828Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.510904064Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"0","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2442024-07-09T21:53:25.828Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.510941804Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"0","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2452024-07-09T21:53:25.828Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.510958454Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"1","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2462024-07-09T21:53:25.828Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.510993064Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"1","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2472024-07-09T21:53:25.828Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.511016854Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"2","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2482024-07-09T21:53:25.828Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.511039123Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"2","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2492024-07-09T21:53:25.831Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.511630748Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"2","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2502024-07-09T21:53:25.831Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.511794626Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"1","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2512024-07-09T21:53:25.831Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.511831906Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"0","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2522024-07-09T21:53:25.831Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512015484Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"2","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2532024-07-09T21:53:25.831Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512078673Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"2","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2542024-07-09T21:53:25.831Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512112663Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2552024-07-09T21:53:25.831Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512242031Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"1","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2562024-07-09T21:53:25.831Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512270181Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"1","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2572024-07-09T21:53:25.831Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512285441Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2582024-07-09T21:53:25.831Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512297951Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"0","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2592024-07-09T21:53:25.831Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512312001Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"0","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2602024-07-09T21:53:25.831Z{"msg":"[0]R flush_numbers: [1670, 1670, 1670, 1670, 1670]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512326171Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2612024-07-09T21:53:25.831Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512339401Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2622024-07-09T21:53:25.831Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.51236087Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2632024-07-09T21:53:25.831Z{"msg":"[1]R flush_numbers: [1670, 1670, 1670, 1670, 1670]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.51237527Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2642024-07-09T21:53:25.831Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.51238938Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2652024-07-09T21:53:25.831Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.51241619Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2662024-07-09T21:53:25.832Z{"msg"Wait for a query_work_queue command to finish before sending IO
2672024-07-09T21:53:25.832Z:"[2]R flush_numbers: [1670, 1670, 1670, 1670, 1670]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.51244276Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2682024-07-09T21:53:25.832Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512473819Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2692024-07-09T21:53:25.832Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512488909Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2702024-07-09T21:53:25.832Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512501809Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2712024-07-09T21:53:25.832Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512529069Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2722024-07-09T21:53:25.832Z{"msg":"Next flush: 1671","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512554488Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2732024-07-09T21:53:25.832Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512610758Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"":"downstairs","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2742024-07-09T21:53:25.832Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512638048Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2752024-07-09T21:53:25.832Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512672157Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"0","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2762024-07-09T21:53:25.832Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512696987Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"1","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2772024-07-09T21:53:25.832Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512730627Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"client":"2","":"downstairs","session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2782024-07-09T21:53:25.832Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512754946Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2792024-07-09T21:53:25.832Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512786146Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2802024-07-09T21:53:25.832Z{"msg":"0964ad53-8d7a-490a-ae30-8192a4201a39 is now active with session: c8a99fa7-f0fa-4a54-9aeb-33759254391f","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512810916Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2812024-07-09T21:53:25.832Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512840316Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216,"session_id":"c8a99fa7-f0fa-4a54-9aeb-33759254391f"}
2822024-07-09T21:53:25.832Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.512854386Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216}
2832024-07-09T21:53:25.832ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2842024-07-09T21:53:25.832ZLoading write count information from file "/var/tmp/test_up/verify_file"
2852024-07-09T21:53:25.832ZRun Repair workload
2862024-07-09T21:53:25.832Z01/10 Write block 39 len 8192 data: 2 2
2872024-07-09T21:53:25.832Z02/10 Read block 27 len 24576
2882024-07-09T21:53:25.836Z03/10 Read block 5 len 32768
2892024-07-09T21:53:25.841Z04/10 Write block 14 len 32768 data: 2 2 2 2 2 2 2 2
2902024-07-09T21:53:25.846Z05/10 Read block 14 len 24576
2912024-07-09T21:53:25.853Z06/10 Write block 40 len 40960 data: 3 2 2 2 2 2 2 2 2 2
2922024-07-09T21:53:25.857Z07/10 Read block 33 len 8192
2932024-07-09T21:53:25.859Z08/10 Read block 41 len 36864
2942024-07-09T21:53:25.867Z09/10 Read block 10 len 4096
2952024-07-09T21:53:25.869Z10/10 Write block 42 len 8192 data: 3 3
2962024-07-09T21:53:25.872Z----------------------------------------------------------------
2972024-07-09T21:53:25.872Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:10
2982024-07-09T21:53:25.872ZStates: Active Active Active
2992024-07-09T21:53:25.872ZGW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3002024-07-09T21:53:25.872Z 1 Acked 1000 Write 2 Done Done Done false
3012024-07-09T21:53:25.873Z 2 Acked 1001 Read 6 Done Done Done false
3022024-07-09T21:53:25.873Z 3 Acked 1002 Read 8 Done Done Done false
3032024-07-09T21:53:25.873Z 4 Acked 1003 Write 8 Done Done Done false
3042024-07-09T21:53:25.873Z 5 Acked 1004 Read 6 Done Done Done false
3052024-07-09T21:53:25.873Z 6 Acked 1005 Write 10 Done Done Done false
3062024-07-09T21:53:25.873Z 7 Acked 1006 Read 2 Done Done Done false
3072024-07-09T21:53:25.873Z 8 Acked 1007 Read 9 Done Done Done false
3082024-07-09T21:53:25.873Z 9 Acked 1008 Read 1 Done Done Done false
3092024-07-09T21:53:25.873Z 10 Acked 1009 Write 2 Sent Sent Sent false
3102024-07-09T21:53:25.873Z STATES DS:0 DS:1 DS:2 TOTAL
3112024-07-09T21:53:25.873Z New 0 0 0 0
3122024-07-09T21:53:25.873Z Sent 1 1 1 3
3132024-07-09T21:53:25.873Z Done 9 9 9 27
3142024-07-09T21:53:25.873Z Skipped 0 0 0 0
3152024-07-09T21:53:25.873Z Error 0 0 0 0
3162024-07-09T21:53:25.873ZLast Flush: 0 0 0
3172024-07-09T21:53:25.873ZDownstairs last five completed:
3182024-07-09T21:53:25.873ZUpstairs last five completed: 10 9 8 7 6
3192024-07-09T21:53:25.873Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-09T21:53:24.555280654Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1216}
3202024-07-09T21:53:25.873ZWrote out file "/var/tmp/test_up/verify_file"
3212024-07-09T21:53:25.876ZRepair part 1 passed
3222024-07-09T21:53:25.876Z
3232024-07-09T21:53:25.876ZKill the current downstairs
3242024-07-09T21:53:25.988Zrm -rf /var/tmp/test_up/8830
3252024-07-09T21:53:25.988ZNow put back the original so we have a mismatch
3262024-07-09T21:53:25.988Zmv /var/tmp/test_up/previous /var/tmp/test_up/8830
3272024-07-09T21:53:25.992ZRestart downstairs with old directory
3282024-07-09T21:53:26.049Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830
3292024-07-09T21:53:26.064ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3302024-07-09T21:53:26.067Z 0 000-009 31 31 31 1670 1670 1670 F F F
3312024-07-09T21:53:26.067Z 1 010-019 31 31 31 1670 1670 1670 T T F <---
3322024-07-09T21:53:26.067Z 2 020-029 31 31 31 1670 1670 1670 T T F <---
3332024-07-09T21:53:26.067Z 3 030-039 31 31 31 1670 1670 1670 T T F <---
3342024-07-09T21:53:26.067Z 4 040-049 31 31 31 1670 1670 1670 T T F <---
3352024-07-09T21:53:26.067ZMax gen: 31, Max flush: 1670
3362024-07-09T21:53:26.067ZError: Difference in extent metadata found!
3372024-07-09T21:53:26.070Zdump test found error as expected
3382024-07-09T21:53:26.070Z
3392024-07-09T21:53:26.070Z
3402024-07-09T21:53:26.070Z/var/tmp/bins/crutest deactivate --range -g 33 -q --key JNbE7w/ts23of1tnsp1kJwjAXmLd5A0zZM8sppXQfVI= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file --verify-in /var/tmp/test_up/verify_file
3412024-07-09T21:53:26.082Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.765873251Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223}
3422024-07-09T21:53:26.082Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"23d2929dfa82e5493027add06127e56457f6c86e\",\n git_commit_timestamp: \"2024-07-09T21:34:58.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.766126599Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223}
3432024-07-09T21:53:26.085Z{"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.766160949Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223}
3442024-07-09T21:53:26.085Z{"msg":"Crucible e4ddfc0b-9897-4e24-85f2-4cd96d379acc has session id: 79c40952-8540-417e-bcb9-85e281c86399","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.766690624Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3452024-07-09T21:53:26.086Z{"msg":"Upstairs opts: Upstairs UUID: e4ddfc0b-9897-4e24-85f2-4cd96d379acc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.766745713Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3462024-07-09T21:53:26.086Z{"msg":"Crucible stats registered with UUID: e4ddfc0b-9897-4e24-85f2-4cd96d379acc","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.766764223Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3472024-07-09T21:53:26.086ZCrucible runtime is spawned
3482024-07-09T21:53:26.086Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767106239Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223}
3492024-07-09T21:53:26.086Z{"msg":"e4ddfc0b-9897-4e24-85f2-4cd96d379acc active request set","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767152439Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3502024-07-09T21:53:26.086Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767182699Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"0","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3512024-07-09T21:53:26.086Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767217809Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"0","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3522024-07-09T21:53:26.086Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767259918Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3532024-07-09T21:53:26.086Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767280978Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3542024-07-09T21:53:26.086Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767304638Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3552024-07-09T21:53:26.086Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767346127Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3562024-07-09T21:53:26.086Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767382547Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3572024-07-09T21:53:26.086Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767408097Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3582024-07-09T21:53:26.086Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767444076Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3592024-07-09T21:53:26.086Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-09T21:53:24.767476916Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3602024-07-09T21:53:26.086Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-09T21:53:24.767516526Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3612024-07-09T21:53:26.086Z{"msg":"ds_connection connected from Ok(127.0.0.1:59827)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767542775Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3622024-07-09T21:53:26.086Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-09T21:53:24.767571555Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3632024-07-09T21:53:26.086Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767605825Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3642024-07-09T21:53:26.086Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767639734Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3652024-07-09T21:53:26.086Z{"msg":"ds_connection connected from Ok(127.0.0.1:45642)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767675514Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3662024-07-09T21:53:26.086Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.767755523Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3672024-07-09T21:53:26.086Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.768218809Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"0","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3682024-07-09T21:53:26.086Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.768255999Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"0","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3692024-07-09T21:53:26.086Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.768271518Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3702024-07-09T21:53:26.086Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.768285738Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3712024-07-09T21:53:26.086Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.768792453Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3722024-07-09T21:53:26.086Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.768897842Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"0","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3732024-07-09T21:53:26.086Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.76913014Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3742024-07-09T21:53:26.086Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.76916479Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3752024-07-09T21:53:26.086Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.769179579Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3762024-07-09T21:53:26.086Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.769192709Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"0","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3772024-07-09T21:53:26.086Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.769213789Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"0","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3782024-07-09T21:53:26.086Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:24.769227299Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3792024-07-09T21:53:36.086Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.769181639Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3802024-07-09T21:53:36.086Z{"msg":"ds_connection connected from Ok(127.0.0.1:34286)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.769258479Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3812024-07-09T21:53:36.091Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.770416367Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3822024-07-09T21:53:36.091Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.770454507Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3832024-07-09T21:53:36.091Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.770991982Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3842024-07-09T21:53:36.091Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771367308Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3852024-07-09T21:53:36.091Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771411308Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3862024-07-09T21:53:36.091Z{"msg":"[0]R flush_numbers: [1670, 1670, 1670, 1670, 1670]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771428118Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3872024-07-09T21:53:36.091Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771442808Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3882024-07-09T21:53:36.091Z{"msg":"[0]R dirty: [false, true, true, true, true]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771456437Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3892024-07-09T21:53:36.091Z{"msg":"[1]R flush_numbers: [1670, 1670, 1670, 1670, 1670]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771470647Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3902024-07-09T21:53:36.091Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771483727Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3912024-07-09T21:53:36.091Z{"msg":"[1]R dirty: [false, true, true, true, true]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771497547Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3922024-07-09T21:53:36.091Z{"msg":"[2]R flush_numbers: [1670, 1670, 1670, 1670, 1670]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771511147Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3932024-07-09T21:53:36.091Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771532627Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3942024-07-09T21:53:36.091Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771546447Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3952024-07-09T21:53:36.091Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771559766Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3962024-07-09T21:53:36.091Z{"msg":"Generation requested: 33 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771580576Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3972024-07-09T21:53:36.091Z{"msg":"Next flush: 1671","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771594256Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3982024-07-09T21:53:36.091Z{"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771606916Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
3992024-07-09T21:53:36.091Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771627596Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4002024-07-09T21:53:36.091Z{"msg":"extent:1 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771641995Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4012024-07-09T21:53:36.091Z{"msg":"extent:1 flush: 1670 1670 1670 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771656955Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4022024-07-09T21:53:36.091Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771679255Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4032024-07-09T21:53:36.091Z{"msg":"extent:1 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771694045Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4042024-07-09T21:53:36.091Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771709455Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4052024-07-09T21:53:36.091Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771723475Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4062024-07-09T21:53:36.092Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771744725Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4072024-07-09T21:53:36.092Z{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771759645Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4082024-07-09T21:53:36.092Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771773664Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4092024-07-09T21:53:36.092Z{"msg":"extent:2 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771794594Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4102024-07-09T21:53:36.092Z{"msg":"extent:2 flush: 1670 1670 1670 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771809574Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4112024-07-09T21:53:36.092Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771836014Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4122024-07-09T21:53:36.092Z{"msg":"extent:2 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771864704Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4132024-07-09T21:53:36.092Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771880303Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4142024-07-09T21:53:36.092Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771894373Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4152024-07-09T21:53:36.092Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771908233Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4162024-07-09T21:53:36.092Z{"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771929283Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4172024-07-09T21:53:36.092Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771943673Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4182024-07-09T21:53:36.092Z{"msg":"extent:3 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771957603Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4192024-07-09T21:53:36.092Z{"msg":"extent:3 flush: 1670 1670 1670 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771978962Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4202024-07-09T21:53:36.092Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.771993572Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4212024-07-09T21:53:36.092Z{"msg":"extent:3 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772014212Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4222024-07-09T21:53:36.092Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772028642Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4232024-07-09T21:53:36.092Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772049552Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4242024-07-09T21:53:36.092Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772063882Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4252024-07-09T21:53:36.092Z{"msg":"Extents 4 dirty","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772084631Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4262024-07-09T21:53:36.092Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772098771Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4272024-07-09T21:53:36.092Z{"msg":"extent:4 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772112581Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4282024-07-09T21:53:36.092Z{"msg":"extent:4 flush: 1670 1670 1670 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772133851Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4292024-07-09T21:53:36.092Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772148621Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4302024-07-09T21:53:36.092Z{"msg":"extent:4 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.77216923Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4312024-07-09T21:53:36.092Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.77218382Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4322024-07-09T21:53:36.092Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.77220499Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4332024-07-09T21:53:36.092Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.77221942Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4342024-07-09T21:53:36.092Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.77224055Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"0","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4352024-07-09T21:53:36.092Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.77225406Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4362024-07-09T21:53:36.092Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772291449Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4372024-07-09T21:53:36.092Z{"msg":"starting reconciliation 8e8b2cf1-8a56-4579-a706-9386a952cde4: found 4 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772306269Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4382024-07-09T21:53:36.092Z{"msg":"Full repair list: {ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772325129Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4392024-07-09T21:53:36.092Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1671, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1671, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1671, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1671, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.772404398Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4402024-07-09T21:53:36.096Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.773444058Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4412024-07-09T21:53:36.096Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.773480488Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4422024-07-09T21:53:36.105Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.788530681Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4432024-07-09T21:53:36.105Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.78858247Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4442024-07-09T21:53:36.115Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.799299307Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4452024-07-09T21:53:36.115Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.799348136Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4462024-07-09T21:53:36.126Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.810100652Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4472024-07-09T21:53:36.126Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.810160721Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4482024-07-09T21:53:36.136Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.819705259Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"":"downstairs","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4492024-07-09T21:53:36.136ZWait for a query_work_queue command to finish before sending IO
4502024-07-09T21:53:36.136Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.819754298Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"0","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4512024-07-09T21:53:36.136Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.819770698Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"1","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4522024-07-09T21:53:36.136Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.819788958Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"client":"2","":"downstairs","session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4532024-07-09T21:53:36.139Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.819823018Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4542024-07-09T21:53:36.139Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.819853797Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4552024-07-09T21:53:36.139Z{"msg":"e4ddfc0b-9897-4e24-85f2-4cd96d379acc is now active with session: 79c40952-8540-417e-bcb9-85e281c86399","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.819870187Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223,"session_id":"79c40952-8540-417e-bcb9-85e281c86399"}
4562024-07-09T21:53:36.139Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-07-09T21:53:34.819890777Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223}
4572024-07-09T21:53:36.139ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4582024-07-09T21:53:36.139ZLoading write count information from file "/var/tmp/test_up/verify_file"
4592024-07-09T21:53:36.139ZRead and Verify all blocks (0..50 range:true)
4602024-07-09T21:53:36.171ZWrote out file "/var/tmp/test_up/verify_file"
4612024-07-09T21:53:36.171ZVerify test completed
4622024-07-09T21:53:36.174ZWrote out file "/var/tmp/test_up/verify_file"
4632024-07-09T21:53:36.174ZCLIENT: Tests done. All submitted work has been ACK'd
4642024-07-09T21:53:36.174Z----------------------------------------------------------------
4652024-07-09T21:53:36.174Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
4662024-07-09T21:53:36.174ZStates: Active Active Active
4672024-07-09T21:53:36.174ZGW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4682024-07-09T21:53:36.174Z 1 Acked 1000 Read 50 Done Done Done false
4692024-07-09T21:53:36.174Z STATES DS:0 DS:1 DS:2 TOTAL
4702024-07-09T21:53:36.174Z New 0 0 0 0
4712024-07-09T21:53:36.174Z Sent 0 0 0 0
4722024-07-09T21:53:36.174Z Done 1 1 1 3
4732024-07-09T21:53:36.174Z Skipped 0 0 0 0
4742024-07-09T21:53:36.174Z Error 0 0 0 0
4752024-07-09T21:53:36.174ZLast Flush: 0 0 0
4762024-07-09T21:53:36.174ZDownstairs last five completed:
4772024-07-09T21:53:36.174ZUpstairs last five completed: 1
4782024-07-09T21:53:36.174ZCLIENT: Up:0 ds:1 act:3
4792024-07-09T21:53:40.172Z----------------------------------------------------------------
4802024-07-09T21:53:40.172Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
4812024-07-09T21:53:40.172ZDownstairs last five completed: 1001 1000
4822024-07-09T21:53:40.172ZUpstairs last five completed: 2 1
4832024-07-09T21:53:40.172ZCLIENT: Up:0 ds:0 act:3
4842024-07-09T21:53:40.172ZCLIENT: All crucible jobs finished, exiting program
4852024-07-09T21:53:40.181Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-09T21:53:38.855877108Z","hostname":"w-01J2CQGX6HBYPH0PACK3598HBZ","pid":1223}
4862024-07-09T21:53:40.184ZRepair part 2 passed
4872024-07-09T21:53:40.184Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830
4882024-07-09T21:53:40.191ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4892024-07-09T21:53:40.191Z 0 000-009 31 31 31 1670 1670 1670 F F F
4902024-07-09T21:53:40.191Z 1 010-019 32 32 32 1671 1671 1671 F F F
4912024-07-09T21:53:40.191Z 2 020-029 32 32 32 1671 1671 1671 F F F
4922024-07-09T21:53:40.191Z 3 030-039 32 32 32 1671 1671 1671 F F F
4932024-07-09T21:53:40.191Z 4 040-049 32 32 32 1671 1671 1671 F F F
4942024-07-09T21:53:40.191ZMax gen: 32, Max flush: 1671
4952024-07-09T21:53:40.194Zdump test passed
4962024-07-09T21:53:40.194Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 -e 1
4972024-07-09T21:53:40.206Z Extent 1
4982024-07-09T21:53:40.209ZGEN 32 32 32
4992024-07-09T21:53:40.209ZFLUSH_ID 1671 1671 1671
5002024-07-09T21:53:40.209ZDIRTY
5012024-07-09T21:53:40.209Z
5022024-07-09T21:53:40.209ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5032024-07-09T21:53:40.209Z010 A A A A A A
5042024-07-09T21:53:40.212Z011 A A A A A A
5052024-07-09T21:53:40.212Z012 A A A A A A
5062024-07-09T21:53:40.215Z013 A A A A A A
5072024-07-09T21:53:40.215Z014 A A A A A A
5082024-07-09T21:53:40.218Z015 A A A A A A
5092024-07-09T21:53:40.218Z016 A A A A A A
5102024-07-09T21:53:40.218Z017 A A A A A A
5112024-07-09T21:53:40.221Z018 A A A A A A
5122024-07-09T21:53:40.221Z019 A A A A A A
5132024-07-09T21:53:40.224Zdump extent test passed
5142024-07-09T21:53:40.224Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 -b 20
5152024-07-09T21:53:40.237ZExtent 2 Block in extent 0 Actual block 20
5162024-07-09T21:53:40.240Z
5172024-07-09T21:53:40.240Z DATA SHA256 VER
5182024-07-09T21:53:40.240Z------ ---------------------------------------------------------------- ---
5192024-07-09T21:53:40.240Z 0 ad03125e0494fc3cb51daf1686c675812f9921e43f62eff6654317353f08d8e7 A
5202024-07-09T21:53:40.240Z 1 ad03125e0494fc3cb51daf1686c675812f9921e43f62eff6654317353f08d8e7 A
5212024-07-09T21:53:40.240Z 2 ad03125e0494fc3cb51daf1686c675812f9921e43f62eff6654317353f08d8e7 A
5222024-07-09T21:53:40.240Z
5232024-07-09T21:53:40.240ZNONCES 0 1 2 DIFF
5242024-07-09T21:53:40.241Z------ ------------------------ ------------------------ ------------------------ -----
5252024-07-09T21:53:40.241Z 0 d3d7a3892e62ea1a24bb28fc d3d7a3892e62ea1a24bb28fc d3d7a3892e62ea1a24bb28fc
5262024-07-09T21:53:40.241Z
5272024-07-09T21:53:40.241Z TAGS 0 1 2 DIFF
5282024-07-09T21:53:40.241Z------ -------------------------------- -------------------------------- -------------------------------- -----
5292024-07-09T21:53:40.241Z 0 421b5796638bf3b8901e2f7ee3bf843e 421b5796638bf3b8901e2f7ee3bf843e 421b5796638bf3b8901e2f7ee3bf843e
5302024-07-09T21:53:40.241Z
5312024-07-09T21:53:40.241ZHASHES 0 1 2 DIFF
5322024-07-09T21:53:40.241Z------ ---------------- ---------------- ---------------- -----
5332024-07-09T21:53:40.241Z 0 f8645ee9adcb747c f8645ee9adcb747c f8645ee9adcb747c
5342024-07-09T21:53:40.241Z
5352024-07-09T21:53:40.244Zdump block test passed
5362024-07-09T21:53:40.244ZUpstairs tests have completed, stopping all downstairs
5372024-07-09T21:53:40.300Z
5382024-07-09T21:53:40.300ZAll Tests have passed
5392024-07-09T21:53:40.303Z3:28 Test duration
5402024-07-09T21:53:40.304Z
5412024-07-09T21:53:40.304Zreal 3:28.882196683
5422024-07-09T21:53:40.304Zuser 2:08.704151474
5432024-07-09T21:53:40.304Zsys 7.262489829
5442024-07-09T21:53:40.304Ztrap 0.102061552
5452024-07-09T21:53:40.304Ztflt 0.025776473
5462024-07-09T21:53:40.304Zdflt 0.024344108
5472024-07-09T21:53:40.304Zkflt 0.000136189
5482024-07-09T21:53:40.304Zlock 57:26.770861069
5492024-07-09T21:53:40.304Zslp 6:46.234531657
5502024-07-09T21:53:40.304Zlat 6.341185752
5512024-07-09T21:53:40.304Zstop 0.031807061
5522024-07-09T21:53:40.304Zprocess exited: duration 216343 ms, exit code 0
 
5532024-07-09T21:53:40.310Zfound 5 output files
5542024-07-09T21:53:40.310Zuploading: /tmp/test_up/dsc-out.txt (4484 bytes)
5552024-07-09T21:53:41.321Zuploaded: /tmp/test_up/dsc-out.txt
5562024-07-09T21:53:41.321Zuploading: /tmp/test_up/test_up_out.txt (3986263 bytes)
5572024-07-09T21:53:42.365Zuploaded: /tmp/test_up/test_up_out.txt
5582024-07-09T21:53:42.367Zuploading: /tmp/test_up/dsc/downstairs-8810.txt (59724 bytes)
5592024-07-09T21:53:43.377Zuploaded: /tmp/test_up/dsc/downstairs-8810.txt
5602024-07-09T21:53:43.377Zuploading: /tmp/test_up/dsc/downstairs-8820.txt (55760 bytes)
5612024-07-09T21:53:44.388Zuploaded: /tmp/test_up/dsc/downstairs-8820.txt
5622024-07-09T21:53:44.391Zuploading: /tmp/test_up/dsc/downstairs-8830.txt (10249 bytes)
5632024-07-09T21:53:45.398Zuploaded: /tmp/test_up/dsc/downstairs-8830.txt