01HVWFBBD71NX6QE2ZER9JZE9J: test-up-unencrypted

BasicConfig {
    output_rules: [
        "/tmp/test_up/*.txt",
        "/tmp/test_up/dsc/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HVWFBQ38YMN3TN96SX22Q0QR

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-04-20T01:01:23.199Zjob dependencies complete; ready to run (waiting for 17 m 17 s)
22024-04-20T01:05:04.373Zjob assigned to worker 01HVWGH0W6F783SHC3TFR2QHB8 (queued for 3 m 41 s)
32024-04-20T01:05:04.660Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-04-20T01:05:13.380Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-04-20T01:05:13.380Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-04-20T01:05:18.670Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-04-20T01:05:18.670Zdownloading input: /input/build/work/bins/crutest.gz
82024-04-20T01:05:29.185Zdownloaded input: /input/build/work/bins/crutest.gz
92024-04-20T01:05:29.185Zdownloading input: /input/build/work/bins/dsc.gz
102024-04-20T01:05:39.079Zdownloaded input: /input/build/work/bins/dsc.gz
112024-04-20T01:05:39.105Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-04-20T01:05:39.131Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-04-20T01:05:39.131Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-04-20T01:05:39.156Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-04-20T01:05:39.156Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-04-20T01:05:39.248Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-04-20T01:05:39.248Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-04-20T01:05:39.287Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-04-20T01:05:39.287Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-04-20T01:05:39.342Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-04-20T01:05:39.367Zdownloading input: /input/build/work/scripts/test_up.sh
222024-04-20T01:05:39.393Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-04-20T01:05:39.393Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-04-20T01:05:39.423Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-04-20T01:05:39.447Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-04-20T01:05:48.672Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-04-20T01:05:48.672Zstarting task 0: "setup"
282024-04-20T01:05:48.699Z++ uname -s
292024-04-20T01:05:48.699Z+ kern=SunOS
302024-04-20T01:05:48.699Z+ case "$kern" in
312024-04-20T01:05:48.699Z+ groupadd -g 12345 build
322024-04-20T01:05:48.699Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-04-20T01:05:50.691Z+ zfs create -o mountpoint=/work rpool/work
342024-04-20T01:05:50.727Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-04-20T01:05:50.760Z+ home_fs=zfs
362024-04-20T01:05:50.760Z+ [[ zfs == autofs ]]
372024-04-20T01:05:50.760Z+ mkdir -p /home/build
382024-04-20T01:05:50.761Z+ chown build:build /home/build /work
392024-04-20T01:05:52.738Z+ chmod 0700 /home/build /work
402024-04-20T01:05:52.765Zprocess exited: duration 4079 ms, exit code 0
 
412024-04-20T01:05:52.823Zstarting task 1: "authentication"
422024-04-20T01:05:52.867Zprocess exited: duration 48 ms, exit code 0
 
432024-04-20T01:05:52.916Zstarting task 2: "build"
442024-04-20T01:05:52.940Z+ banner cores
452024-04-20T01:05:52.940Z
462024-04-20T01:05:52.940Z #### #### ##### ###### ####
472024-04-20T01:05:52.940Z # # # # # # # #
482024-04-20T01:05:52.940Z # # # # # ##### ####
492024-04-20T01:05:52.940Z # # # ##### # #
502024-04-20T01:05:52.940Z # # # # # # # # #
512024-04-20T01:05:52.940Z #### #### # # ###### ####
522024-04-20T01:05:52.941Z
532024-04-20T01:05:52.941Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-04-20T01:05:52.941Z+ echo 'input bins dir contains:'
552024-04-20T01:05:52.941Zinput bins dir contains:
562024-04-20T01:05:52.941Z+ ls -ltr /input/build/work/bins
572024-04-20T01:05:52.965Ztotal 658516
582024-04-20T01:05:52.965Z-rw-r--r-- 1 root root 98276289 Apr 20 01:05 crucible-downstairs.gz
592024-04-20T01:05:52.965Z-rw-r--r-- 1 root root 79513064 Apr 20 01:05 crucible-hammer.gz
602024-04-20T01:05:52.965Z-rw-r--r-- 1 root root 95398132 Apr 20 01:05 crutest.gz
612024-04-20T01:05:52.965Z-rw-r--r-- 1 root root 63730486 Apr 20 01:05 dsc.gz
622024-04-20T01:05:52.965Z+ banner unpack
632024-04-20T01:05:52.965Z
642024-04-20T01:05:52.965Z # # # # ##### ## #### # #
652024-04-20T01:05:52.965Z # # ## # # # # # # # # #
662024-04-20T01:05:52.965Z # # # # # # # # # # ####
672024-04-20T01:05:52.965Z # # # # # ##### ###### # # #
682024-04-20T01:05:52.965Z # # # ## # # # # # # #
692024-04-20T01:05:52.966Z #### # # # # # #### # #
702024-04-20T01:05:52.966Z
712024-04-20T01:05:52.966Z+ mkdir -p /var/tmp/bins
722024-04-20T01:05:52.966Z+ for t in "$input/bins/"*.gz
732024-04-20T01:05:52.966Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-04-20T01:05:52.966Z+ b=crucible-downstairs.gz
752024-04-20T01:05:52.966Z+ b=crucible-downstairs
762024-04-20T01:05:52.966Z+ gunzip
772024-04-20T01:05:55.679Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-04-20T01:05:55.704Z+ for t in "$input/bins/"*.gz
792024-04-20T01:05:55.704Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-04-20T01:05:55.704Z+ b=crucible-hammer.gz
812024-04-20T01:05:55.704Z+ b=crucible-hammer
822024-04-20T01:05:55.704Z+ gunzip
832024-04-20T01:05:57.766Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-04-20T01:05:57.791Z+ for t in "$input/bins/"*.gz
852024-04-20T01:05:57.791Z++ basename /input/build/work/bins/crutest.gz
862024-04-20T01:05:57.791Z+ b=crutest.gz
872024-04-20T01:05:57.791Z+ b=crutest
882024-04-20T01:05:57.791Z+ gunzip
892024-04-20T01:06:00.407Z+ chmod +x /var/tmp/bins/crutest
902024-04-20T01:06:00.433Z+ for t in "$input/bins/"*.gz
912024-04-20T01:06:00.433Z++ basename /input/build/work/bins/dsc.gz
922024-04-20T01:06:00.433Z+ b=dsc.gz
932024-04-20T01:06:00.433Z+ b=dsc
942024-04-20T01:06:00.433Z+ gunzip
952024-04-20T01:06:02.020Z+ chmod +x /var/tmp/bins/dsc
962024-04-20T01:06:02.049Z+ export BINDIR=/var/tmp/bins
972024-04-20T01:06:02.049Z+ BINDIR=/var/tmp/bins
982024-04-20T01:06:02.049Z+ banner test_up_unencrypted
992024-04-20T01:06:02.050Z
1002024-04-20T01:06:02.050Z ##### ###### #### ##### # # ##### # # # #
1012024-04-20T01:06:02.051Z # # # # # # # # # # ## #
1022024-04-20T01:06:02.051Z # ##### #### # # # # # # # # # #
1032024-04-20T01:06:02.051Z # # # # # # ##### # # # # #
1042024-04-20T01:06:02.051Z # # # # # # # # # # # ##
1052024-04-20T01:06:02.051Z # ###### #### # ####### #### # ####### #### # #
1062024-04-20T01:06:02.051Z
1072024-04-20T01:06:02.051Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1082024-04-20T01:06:02.051Z/input/build/work
1092024-04-20T01:06:02.100ZTurn off color for downstairs dump
1102024-04-20T01:06:02.100Zdsc output goes to /tmp/test_up/dsc-out.txt
1112024-04-20T01:06:02.100ZCreating three downstairs regions
1122024-04-20T01:06:02.274ZStarting three downstairs
1132024-04-20T01:06:07.282ZDisable automatic restart on all downstairs
1142024-04-20T01:06:07.369Z
1152024-04-20T01:06:07.370ZBegin tests, output goes to /tmp/test_up/test_up_out.txt
1162024-04-20T01:06:07.402ZRunning test: span
1172024-04-20T01:06:11.495ZCompleted test: span
1182024-04-20T01:06:16.505ZRunning test: big
1192024-04-20T01:06:20.718ZCompleted test: big
1202024-04-20T01:06:25.728ZRunning test: dep
1212024-04-20T01:06:40.447ZCompleted test: dep
1222024-04-20T01:06:45.456ZRunning test: balloon
1232024-04-20T01:06:59.914ZCompleted test: balloon
1242024-04-20T01:07:04.924ZRunning test: deactivate
1252024-04-20T01:07:59.157ZCompleted test: deactivate
1262024-04-20T01:08:04.168ZRunning hammer
1272024-04-20T01:08:13.779ZRun repair tests
1282024-04-20T01:08:13.869Z/var/tmp/bins/crutest fill -g 31 -q -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file
1292024-04-20T01:08:13.870Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.420608817Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175}
1302024-04-20T01:08:13.870ZCrucible runtime is spawned
1312024-04-20T01:08:13.870Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421272759Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175}
1322024-04-20T01:08:13.870Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421311897Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175}
1332024-04-20T01:08:13.870Z{"msg":"Crucible 99c9309c-5b70-42cd-8811-3df2575ebb7c has session id: 65d0879f-7c2b-4167-91ca-c3b1ef8b688a","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421340825Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1342024-04-20T01:08:13.870Z{"msg":"Upstairs opts: Upstairs UUID: 99c9309c-5b70-42cd-8811-3df2575ebb7c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421432417Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1352024-04-20T01:08:13.870Z{"msg":"Crucible stats registered with UUID: 99c9309c-5b70-42cd-8811-3df2575ebb7c","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421465934Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1362024-04-20T01:08:13.870Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421494641Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175}
1372024-04-20T01:08:13.870Z{"msg":"99c9309c-5b70-42cd-8811-3df2575ebb7c active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421521084Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1382024-04-20T01:08:13.870Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421546625Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"0","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1392024-04-20T01:08:13.870Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421576805Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"0","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1402024-04-20T01:08:13.870Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421631394Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"1","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1412024-04-20T01:08:13.870Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421651273Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"1","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1422024-04-20T01:08:13.870Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421670031Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"2","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1432024-04-20T01:08:13.870Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421703287Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"2","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1442024-04-20T01:08:13.870Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421721383Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1452024-04-20T01:08:13.870Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.421739008Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1462024-04-20T01:08:13.870Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.42183524Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1472024-04-20T01:08:13.870Z{"msg":"ds_connection connected from Ok(127.0.0.1:54427)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.422104999Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1482024-04-20T01:08:13.870Z{"msg":"ds_connection connected from Ok(127.0.0.1:33056)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.422599169Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1492024-04-20T01:08:13.870Z{"msg":"ds_connection connected from Ok(127.0.0.1:42726)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.422667966Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1502024-04-20T01:08:13.870Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.423054137Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"2","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1512024-04-20T01:08:13.871Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.423113085Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"2","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1522024-04-20T01:08:13.871Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.423476358Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"0","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1532024-04-20T01:08:13.871Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.4235372Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"0","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1542024-04-20T01:08:13.871Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.423740865Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"1","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1552024-04-20T01:08:13.871Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.423920162Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"1","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1562024-04-20T01:08:13.871Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.423997156Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"2","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1572024-04-20T01:08:13.871Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.424446093Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"0","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1582024-04-20T01:08:13.871Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.424482756Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"2","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1592024-04-20T01:08:13.871Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.424503246Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"2","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1602024-04-20T01:08:13.871Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.424539599Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1612024-04-20T01:08:13.871Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.424713044Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"1","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1622024-04-20T01:08:13.871Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.42485632Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"0","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1632024-04-20T01:08:13.871Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.424894806Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"0","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1642024-04-20T01:08:13.871Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.42491192Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1652024-04-20T01:08:13.871Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425092089Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"1","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1662024-04-20T01:08:13.871Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425130746Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"1","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1672024-04-20T01:08:13.871Z{"msg":"[0]R flush_numbers: [1538, 1538, 1539, 1539, 1538]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425150736Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1682024-04-20T01:08:13.871Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425168912Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1692024-04-20T01:08:13.871Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425199914Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1702024-04-20T01:08:13.871Z{"msg":"[1]R flush_numbers: [1538, 1538, 1539, 1539, 1538]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425216798Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1712024-04-20T01:08:13.871Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425232489Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1722024-04-20T01:08:13.871Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425262439Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1732024-04-20T01:08:13.871Z{"msg":"[2]R flush_numbers: [1538, 1538, 1539, 1539, 1538]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425279022Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1742024-04-20T01:08:13.871Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425295274Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1752024-04-20T01:08:13.872Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1762024-04-20T01:08:13.872Z,"time":"2024-04-20T01:08:13.425326456Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1772024-04-20T01:08:13.872Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425350213Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1782024-04-20T01:08:13.872Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425366295Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1792024-04-20T01:08:13.872Z{"msg":"Next flush: 1540","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425396005Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1802024-04-20T01:08:13.872Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425412958Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"":"downstairs","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1812024-04-20T01:08:13.872Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425447788Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1822024-04-20T01:08:13.872Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425463178Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"0","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1832024-04-20T01:08:13.872Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425480353Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"1","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1842024-04-20T01:08:13.872Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425510703Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"client":"2","":"downstairs","session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1852024-04-20T01:08:13.872Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425527016Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1862024-04-20T01:08:13.872Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425543969Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1872024-04-20T01:08:13.872Z{"msg":"99c9309c-5b70-42cd-8811-3df2575ebb7c is now active with session: 65d0879f-7c2b-4167-91ca-c3b1ef8b688a","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425559651Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1882024-04-20T01:08:13.872Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425588478Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175,"session_id":"65d0879f-7c2b-4167-91ca-c3b1ef8b688a"}
1892024-04-20T01:08:13.872Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:13.425603588Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175}
1902024-04-20T01:08:13.872ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
1912024-04-20T01:08:13.872ZFill test
1922024-04-20T01:08:13.897ZRead and Verify all blocks (0..50 range:false)
1932024-04-20T01:08:13.898ZWrote out file "/var/tmp/test_up/verify_file"
1942024-04-20T01:08:13.898ZCLIENT: Tests done. All submitted work has been ACK'd
1952024-04-20T01:08:13.898Z----------------------------------------------------------------
1962024-04-20T01:08:13.898Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
1972024-04-20T01:08:13.898ZStates: Active Active Active
1982024-04-20T01:08:13.898ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
1992024-04-20T01:08:13.898Z 3 Acked 1002 Read 50 Done Done Done false
2002024-04-20T01:08:13.898Z STATES DS:0 DS:1 DS:2 TOTAL
2012024-04-20T01:08:13.898Z New 0 0 0 0
2022024-04-20T01:08:13.898Z Sent 0 0 0 0
2032024-04-20T01:08:13.898Z Done 1 1 1 3
2042024-04-20T01:08:13.898Z Skipped 0 0 0 0
2052024-04-20T01:08:13.898Z Error 0 0 0 0
2062024-04-20T01:08:13.898ZLast Flush: 1001 1001 1001
2072024-04-20T01:08:13.898ZDownstairs last five completed: 1001 1000
2082024-04-20T01:08:13.898ZUpstairs last five completed: 3 2 1
2092024-04-20T01:08:13.898ZCLIENT: Up:0 ds:1 act:3
2102024-04-20T01:08:17.826Z----------------------------------------------------------------
2112024-04-20T01:08:17.826Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2122024-04-20T01:08:17.826ZDownstairs last five completed: 1003 1002 1001 1000
2132024-04-20T01:08:17.826ZUpstairs last five completed: 4 3 2 1
2142024-04-20T01:08:17.826ZCLIENT: Up:0 ds:0 act:3
2152024-04-20T01:08:17.826ZCLIENT: All crucible jobs finished, exiting program
2162024-04-20T01:08:17.857Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:08:17.446975909Z","hostname":"bmat-EVT22200007-00000f7a","pid":1175}
2172024-04-20T01:08:17.857ZRepair setup passed
2182024-04-20T01:08:17.857ZCopy the 8830 file
2192024-04-20T01:08:17.857Zcp -r /var/tmp/test_up/8830 /var/tmp/test_up/previous
2202024-04-20T01:08:17.857Z/var/tmp/bins/crutest repair -g 32 -q -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file --verify-in /var/tmp/test_up/verify_file
2212024-04-20T01:08:17.937Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.484435223Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181}
2222024-04-20T01:08:17.938Z{Crucible runtime is spawned
2232024-04-20T01:08:17.938Z"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.48506147Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181}
2242024-04-20T01:08:17.938Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.485084787Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181}
2252024-04-20T01:08:17.938Z{"msg":"Crucible 7d8d68ad-795f-4751-a254-14b68263d287 has session id: 63bf7886-b11d-4575-89c7-a3e11636f965","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.485106049Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2262024-04-20T01:08:17.938Z{"msg":"Upstairs opts: Upstairs UUID: 7d8d68ad-795f-4751-a254-14b68263d287, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.485148273Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2272024-04-20T01:08:17.939Z{"msg":"Crucible stats registered with UUID: 7d8d68ad-795f-4751-a254-14b68263d287","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.485169535Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2282024-04-20T01:08:17.939Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.485184315Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181}
2292024-04-20T01:08:17.939Z{"msg":"7d8d68ad-795f-4751-a254-14b68263d287 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.485689551Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2302024-04-20T01:08:17.939Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.48572902Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"0","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2312024-04-20T01:08:17.939Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.485748819Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"0","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2322024-04-20T01:08:17.939Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.485765573Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"1","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2332024-04-20T01:08:17.939Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.485782136Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"1","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2342024-04-20T01:08:17.940Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.485799069Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"2","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2352024-04-20T01:08:17.940Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.485830712Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"2","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2362024-04-20T01:08:17.940Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.485967425Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2372024-04-20T01:08:17.940Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.486016051Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2382024-04-20T01:08:17.940Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.486289368Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2392024-04-20T01:08:17.940Z{"msg":"ds_connection connected from Ok(127.0.0.1:42626)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.48637531Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2402024-04-20T01:08:17.940Z{"msg":"ds_connection connected from Ok(127.0.0.1:33733)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.486416081Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2412024-04-20T01:08:17.941Z{"msg":"ds_connection connected from Ok(127.0.0.1:60222)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.48643568Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2422024-04-20T01:08:17.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.487600149Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"0","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2432024-04-20T01:08:17.941Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.487650209Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"0","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2442024-04-20T01:08:17.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.487679958Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"1","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2452024-04-20T01:08:17.941Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.487707273Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"1","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2462024-04-20T01:08:17.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.487733745Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"2","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2472024-04-20T01:08:17.942Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.487775709Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"2","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2482024-04-20T01:08:17.942Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.488380594Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"1","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2492024-04-20T01:08:17.942Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.488430804Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"2","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2502024-04-20T01:08:17.942Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.488678217Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"0","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2512024-04-20T01:08:17.942Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489062467Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"2","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2522024-04-20T01:08:17.942Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489158598Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"2","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2532024-04-20T01:08:17.943Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489180923Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2542024-04-20T01:08:17.943Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489200392Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"0","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2552024-04-20T01:08:17.943Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489231894Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"0","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2562024-04-20T01:08:17.943Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.48924986Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"session_id":"Wait for a query_work_queue command to finish before sending IO
2572024-04-20T01:08:17.943Z63bf7886-b11d-4575-89c7-a3e11636f965"}
2582024-04-20T01:08:17.943Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489270592Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"1","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2592024-04-20T01:08:17.943Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489303367Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"1","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2602024-04-20T01:08:17.944Z{"msg":"[0]R flush_numbers: [1540, 1540, 1540, 1540, 1540]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489321784Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2612024-04-20T01:08:17.944Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489339679Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2622024-04-20T01:08:17.944Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489370781Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2632024-04-20T01:08:17.944Z{"msg":"[1]R flush_numbers: [1540, 1540, 1540, 1540, 1540]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489388546Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2642024-04-20T01:08:17.944Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.48940554Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2652024-04-20T01:08:17.944Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489421773Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2662024-04-20T01:08:17.945Z{"msg":"[2]R flush_numbers: [1540, 1540, 1540, 1540, 1540]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489440239Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2672024-04-20T01:08:17.945Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489458025Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2682024-04-20T01:08:17.945Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489488606Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2692024-04-20T01:08:17.945Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489504998Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2702024-04-20T01:08:17.945Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489534327Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2712024-04-20T01:08:17.945Z{"msg":"Next flush: 1541","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489552102Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2722024-04-20T01:08:17.945Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489567603Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"":"downstairs","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2732024-04-20T01:08:17.946Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489585599Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2742024-04-20T01:08:17.946Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489599968Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"0","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2752024-04-20T01:08:17.946ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2762024-04-20T01:08:17.946Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489624717Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"1","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2772024-04-20T01:08:17.946Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489655699Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"client":"2","":"downstairs","session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2782024-04-20T01:08:17.946Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489672382Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2792024-04-20T01:08:17.946Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489687452Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2802024-04-20T01:08:17.947Z{"msg":"7d8d68ad-795f-4751-a254-14b68263d287 is now active with session: 63bf7886-b11d-4575-89c7-a3e11636f965","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489704746Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2812024-04-20T01:08:17.947Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489719927Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181,"session_id":"63bf7886-b11d-4575-89c7-a3e11636f965"}
2822024-04-20T01:08:17.947Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.489735457Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181}
2832024-04-20T01:08:17.947ZLoading write count information from file "/var/tmp/test_up/verify_file"
2842024-04-20T01:08:17.947ZRun Repair workload
2852024-04-20T01:08:17.947Z01/10 Write block 1 len 4096 data: 2
2862024-04-20T01:08:17.947Z02/10 Write block 45 len 20480 data: 2 2 2 2 2
2872024-04-20T01:08:17.948Z03/10 Read block 40 len 20480
2882024-04-20T01:08:17.948Z04/10 Read block 32 len 20480
2892024-04-20T01:08:17.948Z05/10 Read block 7 len 28672
2902024-04-20T01:08:17.948Z06/10 Write block 25 len 20480 data: 2 2 2 2 2
2912024-04-20T01:08:17.948Z07/10 Write block 10 len 16384 data: 2 2 2 2
2922024-04-20T01:08:17.948Z08/10 Write block 37 len 16384 data: 2 2 2 2
2932024-04-20T01:08:17.948Z09/10 Read block 13 len 8192
2942024-04-20T01:08:17.949Z10/10 Read block 39 len 12288
2952024-04-20T01:08:17.949Z----------------------------------------------------------------
2962024-04-20T01:08:17.949Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:10
2972024-04-20T01:08:17.949ZStates: Active Active Active
2982024-04-20T01:08:17.949ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2992024-04-20T01:08:17.950Z 1 Acked 1000 Write 1 Done Done Done false
3002024-04-20T01:08:17.950Z 2 Acked 1001 Write 5 Done Done Done false
3012024-04-20T01:08:17.950Z 3 Acked 1002 Read 5 Done Done Done false
3022024-04-20T01:08:17.950Z 4 Acked 1003 Read 5 Done Done Done false
3032024-04-20T01:08:17.950Z 5 Acked 1004 Read 7 Done Done Done false
3042024-04-20T01:08:17.950Z 6 Acked 1005 Write 5 Done Done Done false
3052024-04-20T01:08:17.951Z 7 Acked 1006 Write 4 Done Done Done false
3062024-04-20T01:08:17.951Z 8 Acked 1007 Write 4 Done Done Done false
3072024-04-20T01:08:17.951Z 9 Acked 1008 Read 2 Done Done Done false
3082024-04-20T01:08:17.951Z 10 Acked 1009 Read 3 Done Sent Done false
3092024-04-20T01:08:17.951Z STATES DS:0 DS:1 DS:2 TOTAL
3102024-04-20T01:08:17.951Z New 0 0 0 0
3112024-04-20T01:08:17.951Z Sent 0 1 0 1
3122024-04-20T01:08:17.952Z Done 10 9 10 29
3132024-04-20T01:08:17.952Z Skipped 0 0 0 0
3142024-04-20T01:08:17.952Z Error 0 0 0 0
3152024-04-20T01:08:17.952ZLast Flush: 0 0 0
3162024-04-20T01:08:17.952ZDownstairs last five completed:
3172024-04-20T01:08:17.953ZUpstairs last five completed: 10 9 8 7 6
3182024-04-20T01:08:17.953Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:08:17.502546535Z","hostname":"bmat-EVT22200007-00000f7a","pid":1181}
3192024-04-20T01:08:17.953ZWrote out file "/var/tmp/test_up/verify_file"
3202024-04-20T01:08:17.983ZRepair part 1 passed
3212024-04-20T01:08:17.983Z
3222024-04-20T01:08:17.983ZKill the current downstairs
3232024-04-20T01:08:18.026Zrm -rf /var/tmp/test_up/8830
3242024-04-20T01:08:18.026ZNow put back the original so we have a mismatch
3252024-04-20T01:08:18.026Zmv /var/tmp/test_up/previous /var/tmp/test_up/8830
3262024-04-20T01:08:18.053ZRestart downstairs with old directory
3272024-04-20T01:08:18.116Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830
3282024-04-20T01:08:18.145ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3292024-04-20T01:08:18.145Z 0 000-009 31 31 31 1540 1540 1540 T T F <---
3302024-04-20T01:08:18.145Z 1 010-019 31 31 31 1540 1540 1540 T T F <---
3312024-04-20T01:08:18.145Z 2 020-029 31 31 31 1540 1540 1540 T T F <---
3322024-04-20T01:08:18.145Z 3 030-039 31 31 31 1540 1540 1540 T T F <---
3332024-04-20T01:08:18.146Z 4 040-049 31 31 31 1540 1540 1540 T T F <---
3342024-04-20T01:08:18.146ZMax gen: 31, Max flush: 1540
3352024-04-20T01:08:18.146ZError: Difference in extent metadata found!
3362024-04-20T01:08:18.146Zdump test found error as expected
3372024-04-20T01:08:18.146Z
3382024-04-20T01:08:18.147Z
3392024-04-20T01:08:18.147Z/var/tmp/bins/crutest deactivate --range -g 33 -q -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file --verify-in /var/tmp/test_up/verify_file
3402024-04-20T01:08:18.147Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.765248675Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188}
3412024-04-20T01:08:18.147ZCrucible runtime is spawned
3422024-04-20T01:08:18.181Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"b97e434eec561fa80f965a2f40ec87c83a3a439e\",\n git_commit_timestamp: \"2024-04-20T00:34:22.000000000Z\",\n git_branch: \"artemis/remove-reqwest-block-io\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.765800905Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188}
3432024-04-20T01:08:18.181Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.765864762Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188}
3442024-04-20T01:08:18.181Z{"msg":"Crucible 05b703e1-8383-4be2-a263-9d96b46ceeab has session id: ae8a0e28-db83-4239-bb5b-db91f7c3b39f","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.76589374Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3452024-04-20T01:08:18.182Z{"msg":"Upstairs opts: Upstairs UUID: 05b703e1-8383-4be2-a263-9d96b46ceeab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.765945823Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3462024-04-20T01:08:18.182Z{"msg":"Crucible stats registered with UUID: 05b703e1-8383-4be2-a263-9d96b46ceeab","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.765975953Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3472024-04-20T01:08:18.182Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766025422Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188}
3482024-04-20T01:08:18.182Z{"msg":"05b703e1-8383-4be2-a263-9d96b46ceeab active request set","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766050151Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3492024-04-20T01:08:18.182Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766095983Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"0","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3502024-04-20T01:08:18.183Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766127135Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"0","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3512024-04-20T01:08:18.183Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766176864Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3522024-04-20T01:08:18.183Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.7662054Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3532024-04-20T01:08:18.183Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766254398Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3542024-04-20T01:08:18.183Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766283887Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3552024-04-20T01:08:18.183Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766330369Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3562024-04-20T01:08:18.184Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766349568Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3572024-04-20T01:08:18.184Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766381271Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3582024-04-20T01:08:18.184Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:08:17.766589806Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3592024-04-20T01:08:18.184Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:08:17.766633613Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3602024-04-20T01:08:18.184Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:08:17.766654906Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3612024-04-20T01:08:18.184Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766672902Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3622024-04-20T01:08:18.185Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766692521Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3632024-04-20T01:08:18.185Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766778151Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3642024-04-20T01:08:18.185Z{"msg":"ds_connection connected from Ok(127.0.0.1:58832)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766805235Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3652024-04-20T01:08:18.185Z{"msg":"ds_connection connected from Ok(127.0.0.1:38426)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.766824253Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3662024-04-20T01:08:18.185Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.767890557Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"0","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3672024-04-20T01:08:18.185Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.767929635Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"0","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3682024-04-20T01:08:18.185Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.767948202Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3692024-04-20T01:08:18.185Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.767966979Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3702024-04-20T01:08:18.186Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.768758606Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3712024-04-20T01:08:18.186Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.76908799Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"0","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3722024-04-20T01:08:18.186Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.769465231Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3732024-04-20T01:08:18.186Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.769512425Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3742024-04-20T01:08:18.186Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.769533267Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3752024-04-20T01:08:18.186Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.769551643Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"0","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3762024-04-20T01:08:18.187Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.76958539Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"0","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3772024-04-20T01:08:18.187Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:17.769604238Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3782024-04-20T01:08:28.145Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.768801328Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3792024-04-20T01:08:28.236Z{"msg":"ds_connection connected from Ok(127.0.0.1:44707)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.769152648Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3802024-04-20T01:08:28.236Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.770212479Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3812024-04-20T01:08:28.236Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.770258611Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3822024-04-20T01:08:28.236Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.770901201Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3832024-04-20T01:08:28.236Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.771666355Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3842024-04-20T01:08:28.236Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.771706976Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3852024-04-20T01:08:28.236Z{"msg":"[0]R flush_numbers: [1540, 1540, 1540, 1540, 1540]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.771727005Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3862024-04-20T01:08:28.236Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.771748408Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3872024-04-20T01:08:28.237Z{"msg":"[0]R dirty: [true, true, true, true, true]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.771767696Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3882024-04-20T01:08:28.237Z{"msg":"[1]R flush_numbers: [1540, 1540, 1540, 1540, 1540]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.771801374Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3892024-04-20T01:08:28.237Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.771820622Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3902024-04-20T01:08:28.237Z{"msg":"[1]R dirty: [true, true, true, true, true]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.771840752Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3912024-04-20T01:08:28.237Z{"msg":"[2]R flush_numbers: [1540, 1540, 1540, 1540, 1540]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.771859249Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3922024-04-20T01:08:28.237Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.771891152Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3932024-04-20T01:08:28.237Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.77191001Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3942024-04-20T01:08:28.237Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.771975781Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3952024-04-20T01:08:28.237Z{"msg":"Generation requested: 33 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.771996853Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3962024-04-20T01:08:28.237Z{"msg":"Next flush: 1541","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772028506Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3972024-04-20T01:08:28.237Z{"msg":"Extents 0 dirty","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772046832Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3982024-04-20T01:08:28.237Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.77206553Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
3992024-04-20T01:08:28.237Z{"msg":"extent:0 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772102754Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4002024-04-20T01:08:28.237Z{"msg":"extent:0 flush: 1540 1540 1540 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772123886Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4012024-04-20T01:08:28.237Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772158294Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4022024-04-20T01:08:28.237Z{"msg":"extent:0 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772177944Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4032024-04-20T01:08:28.237Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772199346Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4042024-04-20T01:08:28.237Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772218895Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4052024-04-20T01:08:28.237Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772251831Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4062024-04-20T01:08:28.237Z{"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772272101Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4072024-04-20T01:08:28.237Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772292271Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4082024-04-20T01:08:28.237Z{"msg":"extent:1 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.77231165Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4092024-04-20T01:08:28.237Z{"msg":"extent:1 flush: 1540 1540 1540 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772345688Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4102024-04-20T01:08:28.237Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772365788Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4112024-04-20T01:08:28.238Z{"msg":"extent:1 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772398363Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4122024-04-20T01:08:28.238Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772418202Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4132024-04-20T01:08:28.238Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772451849Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4142024-04-20T01:08:28.238Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772471689Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4152024-04-20T01:08:28.238Z{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772490867Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4162024-04-20T01:08:28.238Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772523332Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4172024-04-20T01:08:28.238Z{"msg":"extent:2 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.77254282Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4182024-04-20T01:08:28.238Z{"msg":"extent:2 flush: 1540 1540 1540 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772577299Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4192024-04-20T01:08:28.238Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772598341Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4202024-04-20T01:08:28.238Z{"msg":"extent:2 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772631166Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4212024-04-20T01:08:28.238Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772650996Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4222024-04-20T01:08:28.238Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772672188Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4232024-04-20T01:08:28.238Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772691817Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4242024-04-20T01:08:28.238Z{"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772724262Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4252024-04-20T01:08:28.238Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772743821Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4262024-04-20T01:08:28.238Z{"msg":"extent:3 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772776556Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4272024-04-20T01:08:28.238Z{"msg":"extent:3 flush: 1540 1540 1540 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772797377Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4282024-04-20T01:08:28.238Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772817447Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4292024-04-20T01:08:28.238Z{"msg":"extent:3 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772849932Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4302024-04-20T01:08:28.238Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772870112Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4312024-04-20T01:08:28.238Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772903278Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4322024-04-20T01:08:28.238Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772922978Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4332024-04-20T01:08:28.238Z{"msg":"Extents 4 dirty","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772956434Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4342024-04-20T01:08:28.239Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772975182Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4352024-04-20T01:08:28.239Z{"msg":"extent:4 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.772996093Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4362024-04-20T01:08:28.239Z{"msg":"extent:4 flush: 1540 1540 1540 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.773032245Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4372024-04-20T01:08:28.239Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.773052686Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4382024-04-20T01:08:28.239Z{"msg":"extent:4 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.773086473Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4392024-04-20T01:08:28.239Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.773106694Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4402024-04-20T01:08:28.239Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.773126453Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4412024-04-20T01:08:28.239Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.77315997Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4422024-04-20T01:08:28.239Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.77318013Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"0","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4432024-04-20T01:08:28.239Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.773213336Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4442024-04-20T01:08:28.239Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.773231763Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4452024-04-20T01:08:28.239Z{"msg":"starting reconciliation 1db29a51-4d8a-4254-a7c7-27c3320b6ad6: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.773266081Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4462024-04-20T01:08:28.239Z{"msg":"Full repair list: {0: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 4: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 1: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 3: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 2: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.773294327Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4472024-04-20T01:08:28.239Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 0, client_id: ClientId(0), flush_number: 1541, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 0 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 0, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 0 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 4, client_id: ClientId(0), flush_number: 1541, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 4 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 4, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 4 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 1, client_id: ClientId(0), flush_number: 1541, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 1 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 1, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 1 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 3, client_id: ClientId(0), flush_number: 1541, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 3 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 3, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 3 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 2, client_id: ClientId(0), flush_number: 1541, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 2, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 2 }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.773448785Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4482024-04-20T01:08:28.239Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.774294282Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4492024-04-20T01:08:28.239Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.774397828Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4502024-04-20T01:08:28.264Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.857263775Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4512024-04-20T01:08:28.264Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.85732682Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4522024-04-20T01:08:28.289Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.909621218Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4532024-04-20T01:08:28.289Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.909708242Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4542024-04-20T01:08:28.335Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.958772854Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4552024-04-20T01:08:28.335Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:27.958877603Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4562024-04-20T01:08:28.385Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:28.009398518Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4572024-04-20T01:08:28.386Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:28.009461153Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4582024-04-20T01:08:28.433Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:28.056567549Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"":"downstairs","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4592024-04-20T01:08:28.433ZWait for a query_work_queue command to finish before sending IO
4602024-04-20T01:08:28.433Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:28.056646176Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"0","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4612024-04-20T01:08:28.433Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:28.056667939Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"1","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4622024-04-20T01:08:28.433Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:28.056686105Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"client":"2","":"downstairs","session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4632024-04-20T01:08:28.433Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:28.056719252Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4642024-04-20T01:08:28.458Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:28.056735985Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4652024-04-20T01:08:28.458Z{"msg":"05b703e1-8383-4be2-a263-9d96b46ceeab is now active with session: ae8a0e28-db83-4239-bb5b-db91f7c3b39f","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:28.056766325Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188,"session_id":"ae8a0e28-db83-4239-bb5b-db91f7c3b39f"}
4662024-04-20T01:08:28.458Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-04-20T01:08:28.05678393Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188}
4672024-04-20T01:08:28.458ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4682024-04-20T01:08:28.458ZLoading write count information from file "/var/tmp/test_up/verify_file"
4692024-04-20T01:08:28.458ZRead and Verify all blocks (0..50 range:true)
4702024-04-20T01:08:28.458ZWrote out file "/var/tmp/test_up/verify_file"
4712024-04-20T01:08:28.458ZVerify test completed
4722024-04-20T01:08:28.458ZWrote out file "/var/tmp/test_up/verify_file"
4732024-04-20T01:08:28.458ZCLIENT: Tests done. All submitted work has been ACK'd
4742024-04-20T01:08:28.458Z----------------------------------------------------------------
4752024-04-20T01:08:28.458Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
4762024-04-20T01:08:28.458ZStates: Active Active Active
4772024-04-20T01:08:28.458ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
4782024-04-20T01:08:28.458Z 1 Acked 1000 Read 50 Done Done Done false
4792024-04-20T01:08:28.459Z STATES DS:0 DS:1 DS:2 TOTAL
4802024-04-20T01:08:28.459Z New 0 0 0 0
4812024-04-20T01:08:28.459Z Sent 0 0 0 0
4822024-04-20T01:08:28.459Z Done 1 1 1 3
4832024-04-20T01:08:28.459Z Skipped 0 0 0 0
4842024-04-20T01:08:28.459Z Error 0 0 0 0
4852024-04-20T01:08:28.459ZLast Flush: 0 0 0
4862024-04-20T01:08:28.459ZDownstairs last five completed:
4872024-04-20T01:08:28.459ZUpstairs last five completed: 1
4882024-04-20T01:08:28.459ZCLIENT: Up:0 ds:1 act:3
4892024-04-20T01:08:32.442Z----------------------------------------------------------------
4902024-04-20T01:08:32.442Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
4912024-04-20T01:08:32.443ZDownstairs last five completed: 1001 1000
4922024-04-20T01:08:32.443ZUpstairs last five completed: 2 1
4932024-04-20T01:08:32.443ZCLIENT: Up:0 ds:0 act:3
4942024-04-20T01:08:32.443ZCLIENT: All crucible jobs finished, exiting program
4952024-04-20T01:08:32.469Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-20T01:08:32.066604874Z","hostname":"bmat-EVT22200007-00000f7a","pid":1188}
4962024-04-20T01:08:32.469ZRepair part 2 passed
4972024-04-20T01:08:32.469Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830
4982024-04-20T01:08:32.470ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4992024-04-20T01:08:32.470Z 0 000-009 32 32 32 1541 1541 1541 F F F
5002024-04-20T01:08:32.470Z 1 010-019 32 32 32 1541 1541 1541 F F F
5012024-04-20T01:08:32.470Z 2 020-029 32 32 32 1541 1541 1541 F F F
5022024-04-20T01:08:32.470Z 3 030-039 32 32 32 1541 1541 1541 F F F
5032024-04-20T01:08:32.470Z 4 040-049 32 32 32 1541 1541 1541 F F F
5042024-04-20T01:08:32.470ZMax gen: 32, Max flush: 1541
5052024-04-20T01:08:32.495Zdump test passed
5062024-04-20T01:08:32.495Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 -e 1
5072024-04-20T01:08:32.495Z Extent 1
5082024-04-20T01:08:32.495ZGEN 32 32 32
5092024-04-20T01:08:32.495ZFLUSH_ID 1541 1541 1541
5102024-04-20T01:08:32.495ZDIRTY
5112024-04-20T01:08:32.495Z
5122024-04-20T01:08:32.495ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5132024-04-20T01:08:32.495Z010 A A A A A A
5142024-04-20T01:08:32.495Z011 A A A A A A
5152024-04-20T01:08:32.495Z012 A A A A A A
5162024-04-20T01:08:32.495Z013 A A A A A A
5172024-04-20T01:08:32.495Z014 A A A A A A
5182024-04-20T01:08:32.495Z015 A A A A A A
5192024-04-20T01:08:32.496Z016 A A A A A A
5202024-04-20T01:08:32.496Z017 A A A A A A
5212024-04-20T01:08:32.521Z018 A A A A A A
5222024-04-20T01:08:32.521Z019 A A A A A A
5232024-04-20T01:08:32.521Zdump extent test passed
5242024-04-20T01:08:32.521Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 -b 20
5252024-04-20T01:08:32.521ZExtent 2 Block in extent 0 Actual block 20
5262024-04-20T01:08:32.521Z
5272024-04-20T01:08:32.521Z DATA SHA256 VER
5282024-04-20T01:08:32.521Z------ ---------------------------------------------------------------- ---
5292024-04-20T01:08:32.521Z 0 7c06d6ec4d4a970d4585f526f5400d8405c63d6f6b1424682e1fab6d3a776079 A
5302024-04-20T01:08:32.521Z 1 7c06d6ec4d4a970d4585f526f5400d8405c63d6f6b1424682e1fab6d3a776079 A
5312024-04-20T01:08:32.521Z 2 7c06d6ec4d4a970d4585f526f5400d8405c63d6f6b1424682e1fab6d3a776079 A
5322024-04-20T01:08:32.521Z
5332024-04-20T01:08:32.521ZNONCES 0 1 2 DIFF
5342024-04-20T01:08:32.521Z------ ------------------------ ------------------------ ------------------------ -----
5352024-04-20T01:08:32.521Z 0 <---
5362024-04-20T01:08:32.521Z
5372024-04-20T01:08:32.521Z TAGS 0 1 2 DIFF
5382024-04-20T01:08:32.521Z------ -------------------------------- -------------------------------- -------------------------------- -----
5392024-04-20T01:08:32.521Z 0 <---
5402024-04-20T01:08:32.521Z
5412024-04-20T01:08:32.522ZHASHES 0 1 2 DIFF
5422024-04-20T01:08:32.522Z------ ---------------- ---------------- ---------------- -----
5432024-04-20T01:08:32.522Z 0 a0d586716ee487f5 a0d586716ee487f5 a0d586716ee487f5
5442024-04-20T01:08:32.522Z
5452024-04-20T01:08:32.546Zdump block test passed
5462024-04-20T01:08:32.546ZUpstairs tests have completed, stopping all downstairs
5472024-04-20T01:08:32.594Z
5482024-04-20T01:08:32.594ZAll Tests have passed
5492024-04-20T01:08:32.619Z2:31 Test duration
5502024-04-20T01:08:32.619Z
5512024-04-20T01:08:32.620Zreal 2:30.572661849
5522024-04-20T01:08:32.620Zuser 36.789008744
5532024-04-20T01:08:32.620Zsys 10.426949822
5542024-04-20T01:08:32.620Ztrap 0.141121072
5552024-04-20T01:08:32.620Ztflt 0.023137979
5562024-04-20T01:08:32.620Zdflt 0.007222466
5572024-04-20T01:08:32.620Zkflt 0.000482529
5582024-04-20T01:08:32.620Zlock 33:46.172669933
5592024-04-20T01:08:32.620Zslp 4:51.445333421
5602024-04-20T01:08:32.620Zlat 14.965526937
5612024-04-20T01:08:32.620Zstop 0.030284070
5622024-04-20T01:08:32.620Zprocess exited: duration 159685 ms, exit code 0
 
5632024-04-20T01:08:32.697Zfound 5 output files
5642024-04-20T01:08:32.697Zuploading: /tmp/test_up/dsc-out.txt (4472 bytes)
5652024-04-20T01:08:33.796Zuploaded: /tmp/test_up/dsc-out.txt
5662024-04-20T01:08:33.796Zuploading: /tmp/test_up/test_up_out.txt (4457605 bytes)
5672024-04-20T01:08:36.894Zuploaded: /tmp/test_up/test_up_out.txt
5682024-04-20T01:08:36.894Zuploading: /tmp/test_up/dsc/downstairs-8810.txt (58483 bytes)
5692024-04-20T01:08:38.090Zuploaded: /tmp/test_up/dsc/downstairs-8810.txt
5702024-04-20T01:08:38.114Zuploading: /tmp/test_up/dsc/downstairs-8820.txt (51811 bytes)
5712024-04-20T01:08:39.281Zuploaded: /tmp/test_up/dsc/downstairs-8820.txt
5722024-04-20T01:08:39.306Zuploading: /tmp/test_up/dsc/downstairs-8830.txt (11091 bytes)
5732024-04-20T01:08:40.430Zuploaded: /tmp/test_up/dsc/downstairs-8830.txt