01HP3NZXBQ3FQ9E47DMQKZ5EC0: test-up-encrypted

BasicConfig {
    output_rules: [
        "/tmp/test_up/*.txt",
        "/tmp/test_up/dsc/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HP3P09PMJRQ5SCFDMEDM3FKQ

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-08T06:36:26.359Zjob dependencies complete; ready to run (waiting for 15 m 44 s)
22024-02-08T06:37:36.800Zjob assigned to worker 01HP3PX9HXET7CAA4SAMN28MST (queued for 1 m 10 s)
32024-02-08T06:37:37.006Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-08T06:37:58.483Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-08T06:37:58.483Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-08T06:38:22.802Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-08T06:38:22.803Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-08T06:38:43.031Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-08T06:38:43.031Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-08T06:38:50.530Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-08T06:38:50.534Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-08T06:38:50.545Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-08T06:38:50.546Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-08T06:38:50.561Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-08T06:38:50.561Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-08T06:38:50.579Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-08T06:38:50.579Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-08T06:38:50.595Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-08T06:38:50.595Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-08T06:38:50.613Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-08T06:38:50.613Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-08T06:38:50.628Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-08T06:38:50.628Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-08T06:38:50.642Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-08T06:38:50.646Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-08T06:39:04.340Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-08T06:39:04.340Zstarting task 0: "setup"
282024-02-08T06:39:04.583Z++ uname -s
292024-02-08T06:39:04.587Z+ kern=SunOS
302024-02-08T06:39:04.587Z+ case "$kern" in
312024-02-08T06:39:04.587Z+ groupadd -g 12345 build
322024-02-08T06:39:04.590Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-08T06:39:06.597Z+ zfs create -o mountpoint=/work rpool/work
342024-02-08T06:39:06.672Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-08T06:39:06.676Z+ home_fs=zfs
362024-02-08T06:39:06.676Z+ [[ zfs == autofs ]]
372024-02-08T06:39:06.677Z+ mkdir -p /home/build
382024-02-08T06:39:06.680Z+ chown build:build /home/build /work
392024-02-08T06:39:08.743Z+ chmod 0700 /home/build /work
402024-02-08T06:39:08.749Zprocess exited: duration 4357 ms, exit code 0
 
412024-02-08T06:39:08.757Zstarting task 1: "authentication"
422024-02-08T06:39:08.775Zprocess exited: duration 18 ms, exit code 0
 
432024-02-08T06:39:08.782Zstarting task 2: "build"
442024-02-08T06:39:08.787Z+ banner cores
452024-02-08T06:39:08.834Z
462024-02-08T06:39:08.834Z #### #### ##### ###### ####
472024-02-08T06:39:08.834Z # # # # # # # #
482024-02-08T06:39:08.834Z # # # # # ##### ####
492024-02-08T06:39:08.834Z # # # ##### # #
502024-02-08T06:39:08.834Z # # # # # # # # #
512024-02-08T06:39:08.834Z #### #### # # ###### ####
522024-02-08T06:39:08.834Z
532024-02-08T06:39:08.837Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-08T06:39:08.846Z+ echo 'input bins dir contains:'
552024-02-08T06:39:08.846Zinput bins dir contains:
562024-02-08T06:39:08.856Z+ ls -ltr /input/build/work/bins
572024-02-08T06:39:08.856Ztotal 763503
582024-02-08T06:39:08.856Z-rw-r--r-- 1 root root 122132441 Feb 8 06:37 crucible-downstairs.gz
592024-02-08T06:39:08.856Z-rw-r--r-- 1 root root 91406059 Feb 8 06:38 crucible-hammer.gz
602024-02-08T06:39:08.856Z-rw-r--r-- 1 root root 105100057 Feb 8 06:38 crutest.gz
612024-02-08T06:39:08.856Z-rw-r--r-- 1 root root 72023935 Feb 8 06:38 dsc.gz
622024-02-08T06:39:08.856Z+ banner unpack
632024-02-08T06:39:08.856Z
642024-02-08T06:39:08.856Z # # # # ##### ## #### # #
652024-02-08T06:39:08.856Z # # ## # # # # # # # # #
662024-02-08T06:39:08.856Z # # # # # # # # # # ####
672024-02-08T06:39:08.856Z # # # # # ##### ###### # # #
682024-02-08T06:39:08.856Z # # # ## # # # # # # #
692024-02-08T06:39:08.856Z #### # # # # # #### # #
702024-02-08T06:39:08.856Z
712024-02-08T06:39:08.857Z+ mkdir -p /var/tmp/bins
722024-02-08T06:39:08.863Z+ for t in "$input/bins/"*.gz
732024-02-08T06:39:08.863Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-02-08T06:39:08.863Z+ b=crucible-downstairs.gz
752024-02-08T06:39:08.863Z+ b=crucible-downstairs
762024-02-08T06:39:08.863Z+ gunzip
772024-02-08T06:39:11.717Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-02-08T06:39:11.720Z+ for t in "$input/bins/"*.gz
792024-02-08T06:39:11.723Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-02-08T06:39:11.724Z+ b=crucible-hammer.gz
812024-02-08T06:39:11.724Z+ b=crucible-hammer
822024-02-08T06:39:11.724Z+ gunzip
832024-02-08T06:39:13.812Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-02-08T06:39:13.816Z+ for t in "$input/bins/"*.gz
852024-02-08T06:39:13.816Z++ basename /input/build/work/bins/crutest.gz
862024-02-08T06:39:13.819Z+ b=crutest.gz
872024-02-08T06:39:13.819Z+ b=crutest
882024-02-08T06:39:13.819Z+ gunzip
892024-02-08T06:39:17.335Z+ chmod +x /var/tmp/bins/crutest
902024-02-08T06:39:17.416Z+ for t in "$input/bins/"*.gz
912024-02-08T06:39:17.416Z++ basename /input/build/work/bins/dsc.gz
922024-02-08T06:39:17.416Z+ b=dsc.gz
932024-02-08T06:39:17.416Z+ b=dsc
942024-02-08T06:39:17.416Z+ gunzip
952024-02-08T06:39:17.938Z+ chmod +x /var/tmp/bins/dsc
962024-02-08T06:39:17.941Z+ export BINDIR=/var/tmp/bins
972024-02-08T06:39:17.941Z+ BINDIR=/var/tmp/bins
982024-02-08T06:39:17.941Z+ banner test_up_encrypted
992024-02-08T06:39:17.945Z
1002024-02-08T06:39:17.945Z ##### ###### #### ##### # # ##### ###### # #
1012024-02-08T06:39:17.945Z # # # # # # # # # ## #
1022024-02-08T06:39:17.945Z # ##### #### # # # # # ##### # # #
1032024-02-08T06:39:17.945Z # # # # # # ##### # # # #
1042024-02-08T06:39:17.945Z # # # # # # # # # # ##
1052024-02-08T06:39:17.945Z # ###### #### # ####### #### # ####### ###### # #
1062024-02-08T06:39:17.945Z
1072024-02-08T06:39:17.945Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
1082024-02-08T06:39:18.031Z/input/build/work
1092024-02-08T06:39:18.039ZTurn off color for downstairs dump
1102024-02-08T06:39:18.236ZUpstairs using key: 9cpc8AvryyntZ0YBRNiCuvxpj/SNQJbg9h5yFW/TmXQ=
1112024-02-08T06:39:18.239Zdsc output goes to /tmp/test_up/dsc-out.txt
1122024-02-08T06:39:18.239ZCreating three downstairs regions
1132024-02-08T06:39:18.380ZStarting three downstairs
1142024-02-08T06:39:23.389ZDisable automatic restart on all downstairs
1152024-02-08T06:39:23.448Z
1162024-02-08T06:39:23.448ZBegin tests, output goes to /tmp/test_up/test_up_out.txt
1172024-02-08T06:39:23.452ZRunning test: span
1182024-02-08T06:39:27.508ZCompleted test: span
1192024-02-08T06:39:32.519ZRunning test: big
1202024-02-08T06:39:36.813ZCompleted test: big
1212024-02-08T06:39:41.819ZRunning test: dep
1222024-02-08T06:40:24.573ZCompleted test: dep
1232024-02-08T06:40:29.575ZRunning test: balloon
1242024-02-08T06:41:08.823ZCompleted test: balloon
1252024-02-08T06:41:13.830ZRunning test: deactivate
1262024-02-08T06:41:23.432ZCompleted test: deactivate
1272024-02-08T06:41:28.440ZRunning hammer
1282024-02-08T06:41:40.273ZRun repair tests
1292024-02-08T06:41:40.276Z/var/tmp/bins/crutest fill -g 31 -q --key 9cpc8AvryyntZ0YBRNiCuvxpj/SNQJbg9h5yFW/TmXQ= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file
1302024-02-08T06:41:40.287Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.848082511Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175}
1312024-02-08T06:41:40.287Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.848433318Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175}
1322024-02-08T06:41:40.287Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.848474725Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175}
1332024-02-08T06:41:40.287Z{"msg":"Crucible runtime is spawned
1342024-02-08T06:41:40.294ZCrucible 3e2c32fc-611c-4b3d-ad77-3f70c99298d3 has session id: bece93b8-5614-4f04-abbb-7536a45de58f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.848514003Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1352024-02-08T06:41:40.294Z{"msg":"Upstairs opts: Upstairs UUID: 3e2c32fc-611c-4b3d-ad77-3f70c99298d3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.848574228Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1362024-02-08T06:41:40.294Z{"msg":"Crucible stats registered with UUID: 3e2c32fc-611c-4b3d-ad77-3f70c99298d3","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.848600997Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1372024-02-08T06:41:40.294Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.848639294Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175}
1382024-02-08T06:41:40.294Z{"msg":"3e2c32fc-611c-4b3d-ad77-3f70c99298d3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.84885013Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1392024-02-08T06:41:40.294Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.848882428Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1402024-02-08T06:41:40.294Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.848901207Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1412024-02-08T06:41:40.294Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.848933565Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1422024-02-08T06:41:40.294Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.848960463Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1432024-02-08T06:41:40.295Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.848997991Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1442024-02-08T06:41:40.295Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.849026059Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1452024-02-08T06:41:40.295Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.849061746Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1462024-02-08T06:41:40.295Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.849094774Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1472024-02-08T06:41:40.295Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.849130892Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1482024-02-08T06:41:40.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.84930649Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1492024-02-08T06:41:40.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.849371616Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1502024-02-08T06:41:40.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.849470979Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1512024-02-08T06:41:40.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.850049211Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1522024-02-08T06:41:40.295Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.850084909Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1532024-02-08T06:41:40.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.850100818Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1542024-02-08T06:41:40.295Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.850131386Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1552024-02-08T06:41:40.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.850147815Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1562024-02-08T06:41:40.295Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.850174903Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1572024-02-08T06:41:40.295Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.850721957Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1582024-02-08T06:41:40.296Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.850778753Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1592024-02-08T06:41:40.296Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.850813841Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1602024-02-08T06:41:40.296Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851209774Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1612024-02-08T06:41:40.296Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851249202Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1622024-02-08T06:41:40.296Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851266411Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1632024-02-08T06:41:40.296Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.85128748Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1642024-02-08T06:41:40.296Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851301959Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1652024-02-08T06:41:40.296Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851315667Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1662024-02-08T06:41:40.296Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851335396Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1672024-02-08T06:41:40.296Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851349325Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1682024-02-08T06:41:40.296Z{"msg":"[0]R flush_numbers: [1660, 1660, 1660, 1660, 1660]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851369864Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1692024-02-08T06:41:40.296Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851383753Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1702024-02-08T06:41:40.296Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851409821Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1712024-02-08T06:41:40.296Z{"msg":"[1]R flush_numbers: [1660, 1660, 1660, 1660, 1660]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1722024-02-08T06:41:40.296Z,"time":"2024-02-08T06:41:39.85143123Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1732024-02-08T06:41:40.297Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851464298Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1742024-02-08T06:41:40.297Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851504235Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1752024-02-08T06:41:40.297Z{"msg":"[2]R flush_numbers: [1660, 1660, 1660, 1660, 1660]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851542493Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1762024-02-08T06:41:40.297Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.85158114Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1772024-02-08T06:41:40.297Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851617458Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1782024-02-08T06:41:40.297Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851655635Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1792024-02-08T06:41:40.297Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851693293Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1802024-02-08T06:41:40.297Z{"msg":"Next flush: 1661","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.8517305Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1812024-02-08T06:41:40.297Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851766908Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1822024-02-08T06:41:40.297Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851805475Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1832024-02-08T06:41:40.297Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
1842024-02-08T06:41:40.297Z,"time":"2024-02-08T06:41:39.851841943Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1852024-02-08T06:41:40.297Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Fill test
1862024-02-08T06:41:40.297Z,"time":"2024-02-08T06:41:39.851877701Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1872024-02-08T06:41:40.297Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851894629Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1882024-02-08T06:41:40.297Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851916168Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1892024-02-08T06:41:40.298Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851929287Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1902024-02-08T06:41:40.298Z{"msg":"3e2c32fc-611c-4b3d-ad77-3f70c99298d3 is now active with session: bece93b8-5614-4f04-abbb-7536a45de58f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851950346Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1912024-02-08T06:41:40.298Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851962875Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175,"session_id":"bece93b8-5614-4f04-abbb-7536a45de58f"}
1922024-02-08T06:41:40.298Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:39.851976834Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1175}
1932024-02-08T06:41:40.329ZRead and Verify all blocks (0..50 range:false)
1942024-02-08T06:41:40.369ZWrote out file "/var/tmp/test_up/verify_file"
1952024-02-08T06:41:40.372ZCLIENT: Tests done. All submitted work has been ACK'd
1962024-02-08T06:41:40.372Z----------------------------------------------------------------
1972024-02-08T06:41:40.372Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
1982024-02-08T06:41:40.372ZStates: Active Active Active
1992024-02-08T06:41:40.372ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2002024-02-08T06:41:40.372Z 3 Acked 1002 Read 50 Done Done Done false
2012024-02-08T06:41:40.372Z STATES DS:0 DS:1 DS:2 TOTAL
2022024-02-08T06:41:40.372Z New 0 0 0 0
2032024-02-08T06:41:40.372Z Sent 0 0 0 0
2042024-02-08T06:41:40.373Z Done 1 1 1 3
2052024-02-08T06:41:40.373Z Skipped 0 0 0 0
2062024-02-08T06:41:40.373Z Error 0 0 0 0
2072024-02-08T06:41:40.373ZLast Flush: 1001 1001 1001
2082024-02-08T06:41:40.373ZDownstairs last five completed: 1001 1000
2092024-02-08T06:41:40.373ZUpstairs last five completed: 3 2 1
2102024-02-08T06:41:40.373ZCLIENT: Up:0 ds:1 act:3
2112024-02-08T06:41:44.371Z----------------------------------------------------------------
2122024-02-08T06:41:44.372Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2132024-02-08T06:41:44.372ZDownstairs last five completed: 1003 1002 1001 1000
2142024-02-08T06:41:44.372ZUpstairs last five completed: 4 3 2 1
2152024-02-08T06:41:44.372ZCLIENT: Up:0 ds:0 act:3
2162024-02-08T06:41:44.372ZCLIENT: All crucible jobs finished, exiting program
2172024-02-08T06:41:44.379ZRepair setup passed
2182024-02-08T06:41:44.383ZCopy the 8830 file
2192024-02-08T06:41:44.386Zcp -r /var/tmp/test_up/8830 /var/tmp/test_up/previous
2202024-02-08T06:41:44.389Z/var/tmp/bins/crutest repair -g 32 -q --key 9cpc8AvryyntZ0YBRNiCuvxpj/SNQJbg9h5yFW/TmXQ= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file --verify-in /var/tmp/test_up/verify_file
2212024-02-08T06:41:44.399Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.960339976Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181}
2222024-02-08T06:41:44.405Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.960689523Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181}
2232024-02-08T06:41:44.405Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
2242024-02-08T06:41:44.405Z,"time":"2024-02-08T06:41:43.960740669Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181}
2252024-02-08T06:41:44.405Z{"msg":"Crucible 7da277be-e4a3-4313-bddd-59bf9513fee3 has session id: 99117a10-3918-4701-8fca-af05095816a7","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.960775257Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2262024-02-08T06:41:44.405Z{"msg":"Upstairs opts: Upstairs UUID: 7da277be-e4a3-4313-bddd-59bf9513fee3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.960828014Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2272024-02-08T06:41:44.405Z{"msg":"Crucible stats registered with UUID: 7da277be-e4a3-4313-bddd-59bf9513fee3","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.960855552Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2282024-02-08T06:41:44.405Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.96088414Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181}
2292024-02-08T06:41:44.405Z{"msg":"7da277be-e4a3-4313-bddd-59bf9513fee3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.960998342Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2302024-02-08T06:41:44.405Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.96104014Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"0","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2312024-02-08T06:41:44.405Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.961089536Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"0","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2322024-02-08T06:41:44.405Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.961128974Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"1","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2332024-02-08T06:41:44.406Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.96118723Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"1","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2342024-02-08T06:41:44.406Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.961237607Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"2","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2352024-02-08T06:41:44.406Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.961275324Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"2","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2362024-02-08T06:41:44.406Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.961417855Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2372024-02-08T06:41:44.406Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.961465172Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2382024-02-08T06:41:44.406Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.961498389Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2392024-02-08T06:41:44.406Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.961525417Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2402024-02-08T06:41:44.406Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.961558715Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2412024-02-08T06:41:44.406Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.961590033Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2422024-02-08T06:41:44.406Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.962216402Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"0","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2432024-02-08T06:41:44.406Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.962262099Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"0","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2442024-02-08T06:41:44.406Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.962305426Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"1","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2452024-02-08T06:41:44.406Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.962390051Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"1","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2462024-02-08T06:41:44.406Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.962454246Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"2","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2472024-02-08T06:41:44.406Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.962488804Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"2","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2482024-02-08T06:41:44.407Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.962966182Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"2","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2492024-02-08T06:41:44.407Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963116092Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"0","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2502024-02-08T06:41:44.407Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963171769Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"1","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2512024-02-08T06:41:44.407Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.96330329Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"2","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2522024-02-08T06:41:44.407Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963349327Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"2","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2532024-02-08T06:41:44.407Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963372026Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2542024-02-08T06:41:44.407Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963545324Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"1","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2552024-02-08T06:41:44.407Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963597201Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"1","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2562024-02-08T06:41:44.407Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963621819Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2572024-02-08T06:41:44.407Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963653907Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"0","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2582024-02-08T06:41:44.407Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963688635Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"0","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2592024-02-08T06:41:44.407Z{"msg":"[0]R flush_numbers: [1661, 1661, 1661, 1661, 1661]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963724502Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2602024-02-08T06:41:44.412Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963751421Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2612024-02-08T06:41:44.412Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
2622024-02-08T06:41:44.412Z,"level":30,"time":"2024-02-08T06:41:43.963792718Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2632024-02-08T06:41:44.412Z{"msg":"[1]R flush_numbers: [1661, 1661, 1661, 1661, 1661]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963834465Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2642024-02-08T06:41:44.412Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963872373Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2652024-02-08T06:41:44.412Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.96391232Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2662024-02-08T06:41:44.412Z{"msg":"[2]R flush_numbers: [1661, 1661, 1661, 1661, 1661]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963953877Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2672024-02-08T06:41:44.412Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.963981965Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2682024-02-08T06:41:44.412Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964018623Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2692024-02-08T06:41:44.412Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964047641Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2702024-02-08T06:41:44.412Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964083639Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2712024-02-08T06:41:44.412Z{"msg":"Next flush: 1662","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964120256Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2722024-02-08T06:41:44.413Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964149184Z","hostname":"Region: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2732024-02-08T06:41:44.413Zip-10-150-1-85.us-west-2.compute.internal","pid":1181,"":"downstairs","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2742024-02-08T06:41:44.413Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964187442Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2752024-02-08T06:41:44.413Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.96421233Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"0","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2762024-02-08T06:41:44.413Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964227539Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"1","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2772024-02-08T06:41:44.413Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964248058Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"client":"2","":"downstairs","session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2782024-02-08T06:41:44.413Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964262237Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2792024-02-08T06:41:44.413Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964282495Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2802024-02-08T06:41:44.413Z{"msg":"7da277be-e4a3-4313-bddd-59bf9513fee3 is now active with session: 99117a10-3918-4701-8fca-af05095816a7","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964296155Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2812024-02-08T06:41:44.413Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964314713Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181,"session_id":"99117a10-3918-4701-8fca-af05095816a7"}
2822024-02-08T06:41:44.413Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:43.964327623Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181}
2832024-02-08T06:41:44.413ZLoading write count information from file "/var/tmp/test_up/verify_file"
2842024-02-08T06:41:44.413ZRun Repair workload
2852024-02-08T06:41:44.413Z01/10 Write block 25 len 20480 data: 2 2 2 2 2
2862024-02-08T06:41:44.413Z02/10 Read block 16 len 24576
2872024-02-08T06:41:44.420Z03/10 Write block 14 len 40960 data: 2 2 2 2 2 2 2 2 2 2
2882024-02-08T06:41:44.424Z04/10 Write block 17 len 24576 data: 3 3 3 3 3 3
2892024-02-08T06:41:44.427Z05/10 Read block 43 len 4096
2902024-02-08T06:41:44.430Z06/10 Read block 6 len 8192
2912024-02-08T06:41:44.433Z07/10 Read block 7 len 4096
2922024-02-08T06:41:44.436Z08/10 Read block 27 len 40960
2932024-02-08T06:41:44.442Z09/10 Read block 20 len 8192
2942024-02-08T06:41:44.445Z10/10 Write block 10 len 32768 data: 2 2 2 2 3 3 3 4
2952024-02-08T06:41:44.452Z----------------------------------------------------------------
2962024-02-08T06:41:44.452Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:10
2972024-02-08T06:41:44.452ZStates: Active Active Active
2982024-02-08T06:41:44.452ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2992024-02-08T06:41:44.452Z 1 Acked 1000 Write 5 Done Done Done false
3002024-02-08T06:41:44.452Z 2 Acked 1001 Read 6 Done Done Done false
3012024-02-08T06:41:44.452Z 3 Acked 1002 Write 10 Done Done Done false
3022024-02-08T06:41:44.452Z 4 Acked 1003 Write 6 Done Done Done false
3032024-02-08T06:41:44.452Z 5 Acked 1004 Read 1 Done Done Done false
3042024-02-08T06:41:44.452Z 6 Acked 1005 Read 2 Done Done Done false
3052024-02-08T06:41:44.452Z 7 Acked 1006 Read 1 Done Done Done false
3062024-02-08T06:41:44.452Z 8 Acked 1007 Read 10 Done Done Done false
3072024-02-08T06:41:44.453Z 9 Acked 1008 Read 2 Done Done Done false
3082024-02-08T06:41:44.453Z 10 Acked 1009 Write 8 Sent Sent Sent false
3092024-02-08T06:41:44.453Z STATES DS:0 DS:1 DS:2 TOTAL
3102024-02-08T06:41:44.453Z New 0 0 0 0
3112024-02-08T06:41:44.453Z Sent 1 1 1 3
3122024-02-08T06:41:44.453Z Done 9 9 9 27
3132024-02-08T06:41:44.453Z Skipped 0 0 0 0
3142024-02-08T06:41:44.453Z Error 0 0 0 0
3152024-02-08T06:41:44.453ZLast Flush: 0 0 0
3162024-02-08T06:41:44.453ZDownstairs last five completed:
3172024-02-08T06:41:44.456ZUpstairs last five completed: 10 9 8 7 6
3182024-02-08T06:41:44.456Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:44.012865609Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1181}
3192024-02-08T06:41:44.457ZWrote out file "/var/tmp/test_up/verify_file"
3202024-02-08T06:41:44.457ZRepair part 1 passed
3212024-02-08T06:41:44.457Z
3222024-02-08T06:41:44.457ZKill the current downstairs
3232024-02-08T06:41:44.564Zrm -rf /var/tmp/test_up/8830
3242024-02-08T06:41:44.564ZNow put back the original so we have a mismatch
3252024-02-08T06:41:44.564Zmv /var/tmp/test_up/previous /var/tmp/test_up/8830
3262024-02-08T06:41:44.567ZRestart downstairs with old directory
3272024-02-08T06:41:44.622Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830
3282024-02-08T06:41:44.635ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3292024-02-08T06:41:44.636Z 0 000-009 31 31 31 1661 1661 1661 F F F
3302024-02-08T06:41:44.642Z 1 010-019 31 31 31 1661 1661 1661 T T F <---
3312024-02-08T06:41:44.642Z 2 020-029 31 31 31 1661 1661 1661 T T F <---
3322024-02-08T06:41:44.642Z 3 030-039 31 31 31 1661 1661 1661 F F F
3332024-02-08T06:41:44.642Z 4 040-049 31 31 31 1661 1661 1661 F F F
3342024-02-08T06:41:44.642ZMax gen: 31, Max flush: 1661
3352024-02-08T06:41:44.642ZError: Difference in extent metadata found!
3362024-02-08T06:41:44.642Zdump test found error as expected
3372024-02-08T06:41:44.642Z
3382024-02-08T06:41:44.642Z
3392024-02-08T06:41:44.642Z/var/tmp/bins/crutest deactivate --range -g 33 -q --key 9cpc8AvryyntZ0YBRNiCuvxpj/SNQJbg9h5yFW/TmXQ= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file --verify-in /var/tmp/test_up/verify_file
3402024-02-08T06:41:44.651Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.211644581Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188}
3412024-02-08T06:41:44.651Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.21197163Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188}
3422024-02-08T06:41:44.651Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212010157Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188}
3432024-02-08T06:41:44.651Z{"msg":"Crucible runtime is spawned
3442024-02-08T06:41:44.656ZCrucible 554b33dc-c156-4bfd-9b35-38f15e769668 has session id: fe40f975-7839-4bab-93c1-5cfbbd1d005b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212045285Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3452024-02-08T06:41:44.656Z{"msg":"Upstairs opts: Upstairs UUID: 554b33dc-c156-4bfd-9b35-38f15e769668, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212087422Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3462024-02-08T06:41:44.656Z{"msg":"Crucible stats registered with UUID: 554b33dc-c156-4bfd-9b35-38f15e769668","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212135859Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3472024-02-08T06:41:44.656Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212182226Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188}
3482024-02-08T06:41:44.656Z{"msg":"554b33dc-c156-4bfd-9b35-38f15e769668 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212237162Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3492024-02-08T06:41:44.656Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212284559Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3502024-02-08T06:41:44.656Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212320197Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3512024-02-08T06:41:44.657Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212351895Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3522024-02-08T06:41:44.657Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212371683Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3532024-02-08T06:41:44.657Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.21241796Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3542024-02-08T06:41:44.657Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212467077Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3552024-02-08T06:41:44.657Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212490085Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3562024-02-08T06:41:44.657Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212527823Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3572024-02-08T06:41:44.657Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212558861Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3582024-02-08T06:41:44.657Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:44.212660034Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3592024-02-08T06:41:44.657Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:44.212741699Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3602024-02-08T06:41:44.657Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212782706Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3612024-02-08T06:41:44.657Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212823183Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3622024-02-08T06:41:44.657Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:44.212853002Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3632024-02-08T06:41:44.657Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212868261Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3642024-02-08T06:41:44.657Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.212883169Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3652024-02-08T06:41:44.657Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.213041109Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3662024-02-08T06:41:44.658Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.213498889Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3672024-02-08T06:41:44.658Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.213535856Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3682024-02-08T06:41:44.658Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.213554935Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3692024-02-08T06:41:44.658Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.213596992Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3702024-02-08T06:41:44.658Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.214036053Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3712024-02-08T06:41:44.658Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.214150316Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3722024-02-08T06:41:44.658Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.214322494Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3732024-02-08T06:41:44.658Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.214375971Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3742024-02-08T06:41:44.658Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.214409519Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3752024-02-08T06:41:44.658Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.214444736Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3762024-02-08T06:41:44.658Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.214474444Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3772024-02-08T06:41:44.658Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.214489503Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3782024-02-08T06:41:45.654Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:45.21402346Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3792024-02-08T06:41:45.654Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:45.214102495Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3802024-02-08T06:41:45.654Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:45.214129004Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3812024-02-08T06:41:45.654Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:45.214149972Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3822024-02-08T06:41:45.654Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:45.214166181Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3832024-02-08T06:41:45.654Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:45.214190039Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3842024-02-08T06:41:45.654Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:45.214226147Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3852024-02-08T06:41:46.655Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:46.214850002Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3862024-02-08T06:41:46.655Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:46.214931216Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3872024-02-08T06:41:46.655Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:46.214962694Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3882024-02-08T06:41:46.655Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:46.214983973Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3892024-02-08T06:41:46.655Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:46.215015141Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3902024-02-08T06:41:46.659Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:46.215036509Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3912024-02-08T06:41:46.659Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:46.215202818Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3922024-02-08T06:41:47.656Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:47.215779047Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3932024-02-08T06:41:47.656Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:47.215862541Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3942024-02-08T06:41:47.656Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:47.215898679Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3952024-02-08T06:41:47.659Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:47.215934826Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3962024-02-08T06:41:47.659Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:47.215965854Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3972024-02-08T06:41:47.660Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:47.215996942Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3982024-02-08T06:41:47.660Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:47.2160338Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
3992024-02-08T06:41:48.658Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.217612412Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4002024-02-08T06:41:48.658Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.217694186Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4012024-02-08T06:41:48.665Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.218483084Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4022024-02-08T06:41:48.665Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.218520562Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4032024-02-08T06:41:48.665Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.218985931Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4042024-02-08T06:41:48.665Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219320709Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4052024-02-08T06:41:48.665Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219360806Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4062024-02-08T06:41:48.665Z{"msg":"[0]R flush_numbers: [1661, 1661, 1661, 1661, 1661]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219380345Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4072024-02-08T06:41:48.665Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219408663Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4082024-02-08T06:41:48.665Z{"msg":"[0]R dirty: [false, true, true, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219427112Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4092024-02-08T06:41:48.666Z{"msg":"[1]R flush_numbers: [1661, 1661, 1661, 1661, 1661]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.21945475Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4102024-02-08T06:41:48.666Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219473169Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4112024-02-08T06:41:48.666Z{"msg":"[1]R dirty: [false, true, true, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219498447Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4122024-02-08T06:41:48.666Z{"msg":"[2]R flush_numbers: [1661, 1661, 1661, 1661, 1661]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219516656Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4132024-02-08T06:41:48.666Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219541314Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4142024-02-08T06:41:48.666Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.21960491Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4152024-02-08T06:41:48.666Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219634878Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4162024-02-08T06:41:48.666Z{"msg":"Generation requested: 33 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219654117Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4172024-02-08T06:41:48.666Z{"msg":"Next flush: 1662","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219679995Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4182024-02-08T06:41:48.666Z{"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219698314Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4192024-02-08T06:41:48.667Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219727542Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4202024-02-08T06:41:48.667Z{"msg":"extent:1 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.21975483Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4212024-02-08T06:41:48.667Z{"msg":"extent:1 flush: 1661 1661 1661 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219775159Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4222024-02-08T06:41:48.667Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219801987Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4232024-02-08T06:41:48.667Z{"msg":"extent:1 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219821536Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4242024-02-08T06:41:48.667Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219848154Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4252024-02-08T06:41:48.667Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219867343Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4262024-02-08T06:41:48.667Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219894111Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4272024-02-08T06:41:48.667Z{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219921709Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4282024-02-08T06:41:48.667Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219955267Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4292024-02-08T06:41:48.668Z{"msg":"extent:2 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.219997054Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4302024-02-08T06:41:48.668Z{"msg":"extent:2 flush: 1661 1661 1661 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.220037052Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4312024-02-08T06:41:48.668Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.22006559Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4322024-02-08T06:41:48.668Z{"msg":"extent:2 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.220092708Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4332024-02-08T06:41:48.668Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.220120036Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4342024-02-08T06:41:48.668Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.220146164Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4352024-02-08T06:41:48.668Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.220172313Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4362024-02-08T06:41:48.668Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.220187732Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4372024-02-08T06:41:48.668Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.22020878Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4382024-02-08T06:41:48.668Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.220222579Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4392024-02-08T06:41:48.669Z{"msg":"Found 2 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.220243458Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4402024-02-08T06:41:48.669Z{"msg":"Full repair list: {2: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 1: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.220259287Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4412024-02-08T06:41:48.669Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 2, client_id: ClientId(0), flush_number: 1662, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 2, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 1, client_id: ClientId(0), flush_number: 1662, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 1 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 1, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 1 }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.220317823Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4422024-02-08T06:41:48.669Z{"msg":"sending flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.220342252Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4432024-02-08T06:41:48.669Z{"msg":"skipping flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.2203575Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4442024-02-08T06:41:48.669Z{"msg":"skipping flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.22037191Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4452024-02-08T06:41:48.669Z{"msg":"no action needed request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.221311248Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4462024-02-08T06:41:48.669Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.221349525Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4472024-02-08T06:41:48.669Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.221366714Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4482024-02-08T06:41:48.722Z{"msg":"sending flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.282454542Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4492024-02-08T06:41:48.722Z{"msg":"skipping flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.282526458Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4502024-02-08T06:41:48.723Z{"msg":"skipping flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.282550786Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4512024-02-08T06:41:48.726Z{"msg":"no action needed request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.283875279Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4522024-02-08T06:41:48.726Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.283918276Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4532024-02-08T06:41:48.726Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.283949124Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4542024-02-08T06:41:48.764Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
4552024-02-08T06:41:48.764Ztime":"2024-02-08T06:41:48.32446825Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"":"downstairs","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4562024-02-08T06:41:48.765Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.324536735Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"0","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4572024-02-08T06:41:48.768Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.324558964Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"1","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4582024-02-08T06:41:48.768Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.324589272Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"client":"2","":"downstairs","session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4592024-02-08T06:41:48.768Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.32461598Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4602024-02-08T06:41:48.768Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.324642908Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4612024-02-08T06:41:48.768Z{"msg":"554b33dc-c156-4bfd-9b35-38f15e769668 is now active with session: fe40f975-7839-4bab-93c1-5cfbbd1d005b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.324669716Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188,"session_id":"fe40f975-7839-4bab-93c1-5cfbbd1d005b"}
4622024-02-08T06:41:48.769Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:48.324698385Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188}
4632024-02-08T06:41:48.769ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4642024-02-08T06:41:48.769ZLoading write count information from file "/var/tmp/test_up/verify_file"
4652024-02-08T06:41:48.769ZRead and Verify all blocks (0..50 range:true)
4662024-02-08T06:41:48.801ZWrote out file "/var/tmp/test_up/verify_file"
4672024-02-08T06:41:48.802ZVerify test completed
4682024-02-08T06:41:48.805ZWrote out file "/var/tmp/test_up/verify_file"
4692024-02-08T06:41:48.806ZCLIENT: Tests done. All submitted work has been ACK'd
4702024-02-08T06:41:48.806Z----------------------------------------------------------------
4712024-02-08T06:41:48.806Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
4722024-02-08T06:41:48.806ZStates: Active Active Active
4732024-02-08T06:41:48.806ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
4742024-02-08T06:41:48.806Z 1 Acked 1000 Read 50 Done Done Done false
4752024-02-08T06:41:48.806Z STATES DS:0 DS:1 DS:2 TOTAL
4762024-02-08T06:41:48.806Z New 0 0 0 0
4772024-02-08T06:41:48.806Z Sent 0 0 0 0
4782024-02-08T06:41:48.807Z Done 1 1 1 3
4792024-02-08T06:41:48.807Z Skipped 0 0 0 0
4802024-02-08T06:41:48.807Z Error 0 0 0 0
4812024-02-08T06:41:48.807ZLast Flush: 0 0 0
4822024-02-08T06:41:48.807ZDownstairs last five completed:
4832024-02-08T06:41:48.807ZUpstairs last five completed: 1
4842024-02-08T06:41:48.807ZCLIENT: Up:0 ds:1 act:3
4852024-02-08T06:41:52.804Z----------------------------------------------------------------
4862024-02-08T06:41:52.804Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
4872024-02-08T06:41:52.804ZDownstairs last five completed: 1001 1000
4882024-02-08T06:41:52.804ZUpstairs last five completed: 2 1
4892024-02-08T06:41:52.804ZCLIENT: Up:0 ds:0 act:3
4902024-02-08T06:41:52.804ZCLIENT: All crucible jobs finished, exiting program
4912024-02-08T06:41:52.807Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:52.363944708Z","hostname":"ip-10-150-1-85.us-west-2.compute.internal","pid":1188}
4922024-02-08T06:41:52.810ZRepair part 2 passed
4932024-02-08T06:41:52.810Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830
4942024-02-08T06:41:52.822ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4952024-02-08T06:41:52.822Z 0 000-009 31 31 31 1661 1661 1661 F F F
4962024-02-08T06:41:52.822Z 1 010-019 32 32 32 1662 1662 1662 F F F
4972024-02-08T06:41:52.822Z 2 020-029 32 32 32 1662 1662 1662 F F F
4982024-02-08T06:41:52.822Z 3 030-039 31 31 31 1661 1661 1661 F F F
4992024-02-08T06:41:52.822Z 4 040-049 31 31 31 1661 1661 1661 F F F
5002024-02-08T06:41:52.822ZMax gen: 32, Max flush: 1662
5012024-02-08T06:41:52.825Zdump test passed
5022024-02-08T06:41:52.825Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 -e 1
5032024-02-08T06:41:52.837Z Extent 1
5042024-02-08T06:41:52.837ZGEN 32 32 32
5052024-02-08T06:41:52.837ZFLUSH_ID 1662 1662 1662
5062024-02-08T06:41:52.837ZDIRTY
5072024-02-08T06:41:52.837Z
5082024-02-08T06:41:52.837ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5092024-02-08T06:41:52.841Z010 A A A A A A
5102024-02-08T06:41:52.841Z011 A A A A A A
5112024-02-08T06:41:52.841Z012 A A A A A A
5122024-02-08T06:41:52.841Z013 A A A A A A
5132024-02-08T06:41:52.841Z014 A A A A A A
5142024-02-08T06:41:52.841Z015 A A A A A A
5152024-02-08T06:41:52.841Z016 A A A A A A
5162024-02-08T06:41:52.841Z017 A A A A A A
5172024-02-08T06:41:52.844Z018 A A A A A A
5182024-02-08T06:41:52.844Z019 A A A A A A
5192024-02-08T06:41:52.844Zdump extent test passed
5202024-02-08T06:41:52.845Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 -b 20
5212024-02-08T06:41:52.857ZExtent 2 Block in extent 0 Actual block 20
5222024-02-08T06:41:52.857Z
5232024-02-08T06:41:52.863Z DATA SHA256 VER
5242024-02-08T06:41:52.863Z------ ---------------------------------------------------------------- ---
5252024-02-08T06:41:52.864Z 0 a47ec001fe8ebe9f4f27c8261729468b14fa619c0fdc49dda592708ff4a28349 A
5262024-02-08T06:41:52.864Z 1 a47ec001fe8ebe9f4f27c8261729468b14fa619c0fdc49dda592708ff4a28349 A
5272024-02-08T06:41:52.864Z 2 a47ec001fe8ebe9f4f27c8261729468b14fa619c0fdc49dda592708ff4a28349 A
5282024-02-08T06:41:52.864Z
5292024-02-08T06:41:52.864ZNONCES 0 1 2 DIFF
5302024-02-08T06:41:52.864Z------ ------------------------ ------------------------ ------------------------ -----
5312024-02-08T06:41:52.864Z 0 fa003bf4c86c13dd50581b7d fa003bf4c86c13dd50581b7d fa003bf4c86c13dd50581b7d
5322024-02-08T06:41:52.864Z
5332024-02-08T06:41:52.864Z TAGS 0 1 2 DIFF
5342024-02-08T06:41:52.864Z------ -------------------------------- -------------------------------- -------------------------------- -----
5352024-02-08T06:41:52.864Z 0 d07a4bc0f2681527f6a483df9707d6dc d07a4bc0f2681527f6a483df9707d6dc d07a4bc0f2681527f6a483df9707d6dc
5362024-02-08T06:41:52.864Z
5372024-02-08T06:41:52.864ZHASHES 0 1 2 DIFF
5382024-02-08T06:41:52.864Z------ ---------------- ---------------- ---------------- -----
5392024-02-08T06:41:52.864Z 0 073b15b94c1f1260 073b15b94c1f1260 073b15b94c1f1260
5402024-02-08T06:41:52.865Z
5412024-02-08T06:41:52.868Zdump block test passed
5422024-02-08T06:41:52.868ZUpstairs tests have completed, stopping all downstairs
5432024-02-08T06:41:52.920Z
5442024-02-08T06:41:52.920ZAll Tests have passed
5452024-02-08T06:41:52.923Z2:35 Test duration
5462024-02-08T06:41:52.923Z
5472024-02-08T06:41:52.923Zreal 2:34.915297984
5482024-02-08T06:41:52.923Zuser 2:18.722910196
5492024-02-08T06:41:52.924Zsys 7.259164641
5502024-02-08T06:41:52.924Ztrap 0.116622306
5512024-02-08T06:41:52.924Ztflt 0.006988533
5522024-02-08T06:41:52.924Zdflt 0.005840704
5532024-02-08T06:41:52.924Zkflt 0.001129902
5542024-02-08T06:41:52.924Zlock 34:40.951950965
5552024-02-08T06:41:52.924Zslp 4:58.279495981
5562024-02-08T06:41:52.924Zlat 12.673302096
5572024-02-08T06:41:52.924Zstop 0.030068436
5582024-02-08T06:41:52.924Zprocess exited: duration 164126 ms, exit code 0
 
5592024-02-08T06:41:52.935Zfound 5 output files
5602024-02-08T06:41:52.935Zuploading: /tmp/test_up/dsc-out.txt (4484 bytes)
5612024-02-08T06:41:53.948Zuploaded: /tmp/test_up/dsc-out.txt
5622024-02-08T06:41:53.948Zuploading: /tmp/test_up/test_up_out.txt (4649584 bytes)
5632024-02-08T06:41:57.713Zuploaded: /tmp/test_up/test_up_out.txt
5642024-02-08T06:41:57.713Zuploading: /tmp/test_up/dsc/downstairs-8810.txt (52860 bytes)
5652024-02-08T06:41:58.731Zuploaded: /tmp/test_up/dsc/downstairs-8810.txt
5662024-02-08T06:41:58.735Zuploading: /tmp/test_up/dsc/downstairs-8820.txt (51273 bytes)
5672024-02-08T06:41:59.747Zuploaded: /tmp/test_up/dsc/downstairs-8820.txt
5682024-02-08T06:41:59.747Zuploading: /tmp/test_up/dsc/downstairs-8830.txt (7388 bytes)
5692024-02-08T06:42:00.758Zuploaded: /tmp/test_up/dsc/downstairs-8830.txt