01HKT6T8PXS4B7VQD8FPN98316: test-up-unencrypted

BasicConfig {
    output_rules: [
        "/tmp/test_up/*.txt",
        "/tmp/test_up/dsc/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HKT6TQKH0RT67B8AGTWX67MK

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-01-10T17:51:53.935Zjob dependencies complete; ready to run (waiting for 20 m 56 s)
22024-01-10T17:53:26.804Zjob assigned to worker 01HKT816F7YD77ZM5HQBNXCE67 (queued for 1 m 32 s)
32024-01-10T17:53:27.143Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-01-10T17:53:42.880Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-01-10T17:53:42.880Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-01-10T17:53:54.035Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-01-10T17:53:54.035Zdownloading input: /input/build/work/bins/crutest.gz
82024-01-10T17:54:08.026Zdownloaded input: /input/build/work/bins/crutest.gz
92024-01-10T17:54:08.026Zdownloading input: /input/build/work/bins/dsc.gz
102024-01-10T17:54:18.248Zdownloaded input: /input/build/work/bins/dsc.gz
112024-01-10T17:54:18.255Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-01-10T17:54:18.270Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-01-10T17:54:18.270Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-01-10T17:54:18.301Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-01-10T17:54:18.301Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-01-10T17:54:18.317Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-01-10T17:54:18.317Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-01-10T17:54:18.333Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-01-10T17:54:18.333Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-01-10T17:54:18.347Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-01-10T17:54:18.348Zdownloading input: /input/build/work/scripts/test_up.sh
222024-01-10T17:54:18.363Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-01-10T17:54:18.363Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-01-10T17:54:18.379Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-01-10T17:54:18.382Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-01-10T17:54:44.643Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-01-10T17:54:44.644Zstarting task 0: "setup"
282024-01-10T17:54:44.732Z++ uname -s
292024-01-10T17:54:44.735Z+ kern=SunOS
302024-01-10T17:54:44.735Z+ case "$kern" in
312024-01-10T17:54:44.735Z+ groupadd -g 12345 build
322024-01-10T17:54:44.738Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-01-10T17:54:46.839Z+ zfs create -o mountpoint=/work rpool/work
342024-01-10T17:54:46.904Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-01-10T17:54:46.908Z+ home_fs=zfs
362024-01-10T17:54:46.909Z+ [[ zfs == autofs ]]
372024-01-10T17:54:46.909Z+ mkdir -p /home/build
382024-01-10T17:54:46.912Z+ chown build:build /home/build /work
392024-01-10T17:54:48.913Z+ chmod 0700 /home/build /work
402024-01-10T17:54:48.916Zprocess exited: duration 4291 ms, exit code 0
 
412024-01-10T17:54:48.922Zstarting task 1: "authentication"
422024-01-10T17:54:48.948Zprocess exited: duration 25 ms, exit code 0
 
432024-01-10T17:54:48.954Zstarting task 2: "build"
442024-01-10T17:54:48.962Z+ banner cores
452024-01-10T17:54:48.997Z
462024-01-10T17:54:48.998Z #### #### ##### ###### ####
472024-01-10T17:54:48.998Z # # # # # # # #
482024-01-10T17:54:48.998Z # # # # # ##### ####
492024-01-10T17:54:48.999Z # # # ##### # #
502024-01-10T17:54:48.999Z # # # # # # # # #
512024-01-10T17:54:48.999Z #### #### # # ###### ####
522024-01-10T17:54:48.999Z
532024-01-10T17:54:49.002Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-01-10T17:54:49.014Z+ echo 'input bins dir contains:'
552024-01-10T17:54:49.014Zinput bins dir contains:
562024-01-10T17:54:49.018Z+ ls -ltr /input/build/work/bins
572024-01-10T17:54:49.018Ztotal 698594
582024-01-10T17:54:49.018Z-rw-r--r-- 1 root root 109456429 Jan 10 17:53 crucible-downstairs.gz
592024-01-10T17:54:49.018Z-rw-r--r-- 1 root root 85807289 Jan 10 17:53 crucible-hammer.gz
602024-01-10T17:54:49.018Z-rw-r--r-- 1 root root 93634688 Jan 10 17:54 crutest.gz
612024-01-10T17:54:49.018Z-rw-r--r-- 1 root root 68539139 Jan 10 17:54 dsc.gz
622024-01-10T17:54:49.018Z+ banner unpack
632024-01-10T17:54:49.018Z
642024-01-10T17:54:49.018Z # # # # ##### ## #### # #
652024-01-10T17:54:49.018Z # # ## # # # # # # # # #
662024-01-10T17:54:49.018Z # # # # # # # # # # ####
672024-01-10T17:54:49.018Z # # # # # ##### ###### # # #
682024-01-10T17:54:49.018Z # # # ## # # # # # # #
692024-01-10T17:54:49.018Z #### # # # # # #### # #
702024-01-10T17:54:49.018Z
712024-01-10T17:54:49.018Z+ mkdir -p /var/tmp/bins
722024-01-10T17:54:49.021Z+ for t in "$input/bins/"*.gz
732024-01-10T17:54:49.022Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-01-10T17:54:49.025Z+ b=crucible-downstairs.gz
752024-01-10T17:54:49.025Z+ b=crucible-downstairs
762024-01-10T17:54:49.025Z+ gunzip
772024-01-10T17:54:52.870Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-01-10T17:54:52.873Z+ for t in "$input/bins/"*.gz
792024-01-10T17:54:52.873Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-01-10T17:54:52.873Z+ b=crucible-hammer.gz
812024-01-10T17:54:52.873Z+ b=crucible-hammer
822024-01-10T17:54:52.873Z+ gunzip
832024-01-10T17:54:55.900Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-01-10T17:54:55.903Z+ for t in "$input/bins/"*.gz
852024-01-10T17:54:55.906Z++ basename /input/build/work/bins/crutest.gz
862024-01-10T17:54:55.906Z+ b=crutest.gz
872024-01-10T17:54:55.906Z+ b=crutest
882024-01-10T17:54:55.906Z+ gunzip
892024-01-10T17:54:59.138Z+ chmod +x /var/tmp/bins/crutest
902024-01-10T17:54:59.145Z+ for t in "$input/bins/"*.gz
912024-01-10T17:54:59.147Z++ basename /input/build/work/bins/dsc.gz
922024-01-10T17:54:59.147Z+ b=dsc.gz
932024-01-10T17:54:59.147Z+ b=dsc
942024-01-10T17:54:59.148Z+ gunzip
952024-01-10T17:55:01.495Z+ chmod +x /var/tmp/bins/dsc
962024-01-10T17:55:01.499Z+ export BINDIR=/var/tmp/bins
972024-01-10T17:55:01.499Z+ BINDIR=/var/tmp/bins
982024-01-10T17:55:01.499Z+ banner test_up_unencrypted
992024-01-10T17:55:01.502Z
1002024-01-10T17:55:01.502Z ##### ###### #### ##### # # ##### # # # #
1012024-01-10T17:55:01.502Z # # # # # # # # # # ## #
1022024-01-10T17:55:01.502Z # ##### #### # # # # # # # # # #
1032024-01-10T17:55:01.502Z # # # # # # ##### # # # # #
1042024-01-10T17:55:01.503Z # # # # # # # # # # # ##
1052024-01-10T17:55:01.503Z # ###### #### # ####### #### # ####### #### # #
1062024-01-10T17:55:01.503Z
1072024-01-10T17:55:01.503Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1082024-01-10T17:55:01.581Z/input/build/work
1092024-01-10T17:55:01.592ZTurn off color for downstairs dump
1102024-01-10T17:55:01.595Zdsc output goes to /tmp/test_up/dsc-out.txt
1112024-01-10T17:55:01.595ZCreating three downstairs regions
1122024-01-10T17:55:01.969ZStarting three downstairs
1132024-01-10T17:55:06.980ZDisable automatic restart on all downstairs
1142024-01-10T17:55:07.098Z
1152024-01-10T17:55:07.098ZBegin tests, output goes to /tmp/test_up/test_up_out.txt
1162024-01-10T17:55:07.103ZRunning test: span
1172024-01-10T17:55:11.193ZCompleted test: span
1182024-01-10T17:55:16.202ZRunning test: big
1192024-01-10T17:55:20.449ZCompleted test: big
1202024-01-10T17:55:25.457ZRunning test: dep
1212024-01-10T17:56:08.541ZCompleted test: dep
1222024-01-10T17:56:08.835ZRunning test: balloon
1232024-01-10T17:56:18.136ZCompleted test: balloon
1242024-01-10T17:56:23.144ZRunning test: deactivate
1252024-01-10T17:56:32.481ZCompleted test: deactivate
1262024-01-10T17:56:37.493ZRunning hammer
1272024-01-10T17:56:48.668ZRun repair tests
1282024-01-10T17:56:48.672Z/var/tmp/bins/crutest fill -g 31 -q -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file
1292024-01-10T17:56:48.687Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.688590517Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192}
1302024-01-10T17:56:48.687Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689060247Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192}
1312024-01-10T17:56:48.687Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689090154Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192}
1322024-01-10T17:56:48.692Z{"msg":"Crucible f1cfd237-e6c0-4857-bc97-ec062d3556de has session id: 31ea3151-7113-4452-88c0-05732ca1198b","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.68911137Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1332024-01-10T17:56:48.693Z{"msg":"Upstairs opts: Upstairs UUID: f1cfd237-e6c0-4857-bc97-ec062d3556de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689200638Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1342024-01-10T17:56:48.693Z{"msg":"Crucible stats registered with UUID: f1cfd237-e6c0-4857-bc97-ec062d3556de","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689247692Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1352024-01-10T17:56:48.694ZCrucible runtime is spawned
1362024-01-10T17:56:48.694Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689495981Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192}
1372024-01-10T17:56:48.694Z{"msg":"f1cfd237-e6c0-4857-bc97-ec062d3556de active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689661048Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1382024-01-10T17:56:48.694Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689715654Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"0","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1392024-01-10T17:56:48.694Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689742266Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"0","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1402024-01-10T17:56:48.694Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689764496Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"1","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1412024-01-10T17:56:48.694Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689818695Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"1","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1422024-01-10T17:56:48.694Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689842807Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"2","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1432024-01-10T17:56:48.697Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689865694Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"2","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1442024-01-10T17:56:48.697Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689887365Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1452024-01-10T17:56:48.697Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689910164Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1462024-01-10T17:56:48.698Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.689964797Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1472024-01-10T17:56:48.698Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.690203671Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1482024-01-10T17:56:48.698Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.690311454Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1492024-01-10T17:56:48.698Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.690354865Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1502024-01-10T17:56:48.698Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.691389901Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"1","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1512024-01-10T17:56:48.698Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.691461272Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"1","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1522024-01-10T17:56:48.698Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.691487253Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"0","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1532024-01-10T17:56:48.700Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.691510617Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"0","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1542024-01-10T17:56:48.701Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.691591116Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"2","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1552024-01-10T17:56:48.701Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.691654909Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"2","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1562024-01-10T17:56:48.701Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.692391886Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"0","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1572024-01-10T17:56:48.701Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.692468079Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"1","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1582024-01-10T17:56:48.701Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.692518593Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"2","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1592024-01-10T17:56:48.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.693011137Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"0","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1602024-01-10T17:56:48.707Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.693133357Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"0","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1612024-01-10T17:56:48.707Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.693228358Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1622024-01-10T17:56:48.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.693274332Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"1","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1632024-01-10T17:56:48.707Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.693345082Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"1","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1642024-01-10T17:56:48.707Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.6934207Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1652024-01-10T17:56:48.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.693489974Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"2","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1662024-01-10T17:56:48.708Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.693533534Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"2","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1672024-01-10T17:56:48.708Z{"msg":"[0]R flush_numbers: [1584, 1584, 1585, 1585, 1585]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.693607473Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO
1682024-01-10T17:56:48.708Z,"session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1692024-01-10T17:56:48.708Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.69365529Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1702024-01-10T17:56:48.708Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.693733026Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1712024-01-10T17:56:48.708Z{"msg":"[1]R flush_numbers: [1584, 1584, 1585, 1585, 1585]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.693805848Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1722024-01-10T17:56:48.708Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.693874091Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1732024-01-10T17:56:48.708Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.69392028Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1742024-01-10T17:56:48.710Z{"msg":"[2]R flush_numbers: [1584, 1584, 1585, 1585, 1585]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.69399215Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1752024-01-10T17:56:48.710Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.694067083Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1762024-01-10T17:56:48.710Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.694140564Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1772024-01-10T17:56:48.711Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.694209868Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1782024-01-10T17:56:48.711Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.694258969Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1792024-01-10T17:56:48.711Z{"msg":"Next flush: 1586","v":0,"name":"crucible","level":30,Region: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
1802024-01-10T17:56:48.711Z"time":"2024-01-10T17:56:48.6943009Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1812024-01-10T17:56:48.711Z{"msg":"All extents match","v":0,"name":"crucible","level":30Fill test
1822024-01-10T17:56:48.711Z,"time":"2024-01-10T17:56:48.6944597Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"":"downstairs","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1832024-01-10T17:56:48.711Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.694512234Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1842024-01-10T17:56:48.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.694534906Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"0","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1852024-01-10T17:56:48.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.694557848Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"1","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1862024-01-10T17:56:48.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.694579587Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"client":"2","":"downstairs","session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1872024-01-10T17:56:48.711Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.694601187Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1882024-01-10T17:56:48.711Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.694621297Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1892024-01-10T17:56:48.711Z{"msg":"f1cfd237-e6c0-4857-bc97-ec062d3556de is now active with session: 31ea3151-7113-4452-88c0-05732ca1198b","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.694674244Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1902024-01-10T17:56:48.712Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.69469797Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192,"session_id":"31ea3151-7113-4452-88c0-05732ca1198b"}
1912024-01-10T17:56:48.712Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:48.69471834Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1192}
1922024-01-10T17:56:48.716ZRead and Verify all blocks (0..50 range:false)
1932024-01-10T17:56:48.739ZWrote out file "/var/tmp/test_up/verify_file"
1942024-01-10T17:56:48.739ZCLIENT: Tests done. All submitted work has been ACK'd
1952024-01-10T17:56:48.887Z----------------------------------------------------------------
1962024-01-10T17:56:48.887Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
1972024-01-10T17:56:48.887ZStates: Active Active Active
1982024-01-10T17:56:48.887ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
1992024-01-10T17:56:48.887Z 3 Acked 1002 Read 50 Done Done Done false
2002024-01-10T17:56:48.887Z STATES DS:0 DS:1 DS:2 TOTAL
2012024-01-10T17:56:48.887Z New 0 0 0 0
2022024-01-10T17:56:48.887Z Sent 0 0 0 0
2032024-01-10T17:56:48.887Z Done 1 1 1 3
2042024-01-10T17:56:48.887Z Skipped 0 0 0 0
2052024-01-10T17:56:48.887Z Error 0 0 0 0
2062024-01-10T17:56:48.887ZLast Flush: 1001 1001 1001
2072024-01-10T17:56:48.887ZDownstairs last five completed: 1001 1000
2082024-01-10T17:56:48.887ZUpstairs last five completed: 3 2 1
2092024-01-10T17:56:48.887ZCLIENT: Up:0 ds:1 act:3
2102024-01-10T17:56:52.730Z----------------------------------------------------------------
2112024-01-10T17:56:52.730Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2122024-01-10T17:56:52.730ZDownstairs last five completed: 1003 1002 1001 1000
2132024-01-10T17:56:52.731ZUpstairs last five completed: 4 3 2 1
2142024-01-10T17:56:52.731ZCLIENT: Up:0 ds:0 act:3
2152024-01-10T17:56:52.731ZCLIENT: All crucible jobs finished, exiting program
2162024-01-10T17:56:52.742ZRepair setup passed
2172024-01-10T17:56:52.747ZCopy the 8830 file
2182024-01-10T17:56:52.749Zcp -r /var/tmp/test_up/8830 /var/tmp/test_up/previous
2192024-01-10T17:56:52.752Z/var/tmp/bins/crutest repair -g 32 -q -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file --verify-in /var/tmp/test_up/verify_file
2202024-01-10T17:56:52.770Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.772390423Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198}
2212024-01-10T17:56:52.770Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.772907177Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198}
2222024-01-10T17:56:52.775Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.772974625Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198}
2232024-01-10T17:56:52.775Z{"msg":"Crucible 43f4537c-09cb-4d3d-8494-10eca1d88240 has session id: 0a1eb8ea-af0e-4a46-848d-6d92c784162f","v":0,"name":"crucible","level":30Crucible runtime is spawned
2242024-01-10T17:56:52.775Z,"time":"2024-01-10T17:56:52.773057093Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2252024-01-10T17:56:52.775Z{"msg":"Upstairs opts: Upstairs UUID: 43f4537c-09cb-4d3d-8494-10eca1d88240, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.773160625Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2262024-01-10T17:56:52.775Z{"msg":"Crucible stats registered with UUID: 43f4537c-09cb-4d3d-8494-10eca1d88240","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.773230195Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2272024-01-10T17:56:52.775Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.773256715Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198}
2282024-01-10T17:56:52.776Z{"msg":"43f4537c-09cb-4d3d-8494-10eca1d88240 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.77343938Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2292024-01-10T17:56:52.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.773498645Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"0","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2302024-01-10T17:56:52.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.773525092Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"0","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2312024-01-10T17:56:52.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.773547631Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"1","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2322024-01-10T17:56:52.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.773590425Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"1","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2332024-01-10T17:56:52.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.773663743Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"2","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2342024-01-10T17:56:52.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.773699792Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"2","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2352024-01-10T17:56:52.776Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.773754849Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2362024-01-10T17:56:52.776Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.773795337Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2372024-01-10T17:56:52.777Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.773858335Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2382024-01-10T17:56:52.777Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.774060973Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2392024-01-10T17:56:52.777Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.77414496Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2402024-01-10T17:56:52.777Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.77473804Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2412024-01-10T17:56:52.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.775654641Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"1","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2422024-01-10T17:56:52.777Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.775723844Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"1","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2432024-01-10T17:56:52.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.775751073Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"2","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2442024-01-10T17:56:52.777Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.775820701Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"2","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2452024-01-10T17:56:52.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.775919939Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"0","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2462024-01-10T17:56:52.777Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.775947781Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"0","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2472024-01-10T17:56:52.783Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.776673437Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"1","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2482024-01-10T17:56:52.783Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.776754278Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"2","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2492024-01-10T17:56:52.783Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.776845172Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"0","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2502024-01-10T17:56:52.783Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.777499397Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"0","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2512024-01-10T17:56:52.783Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.777578273Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"0","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2522024-01-10T17:56:52.783Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.777656291Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2532024-01-10T17:56:52.783Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.777716452Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"1","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2542024-01-10T17:56:52.783Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.777743615Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"1","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2552024-01-10T17:56:52.783Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.777766655Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2562024-01-10T17:56:52.783Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.777807Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"2","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2572024-01-10T17:56:52.784Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.777840665Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"2","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2582024-01-10T17:56:52.784ZWait for a query_work_queue command to finish before sending IO
2592024-01-10T17:56:52.784Z{"msg":"[0]R flush_numbers: [1586, 1586, 1586, 1586, 1586]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.777912756Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2602024-01-10T17:56:52.784Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.777952891Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2612024-01-10T17:56:52.784Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.777977434Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2622024-01-10T17:56:52.784Z{"msg":"[1]R flush_numbers: [1586, 1586, 1586, 1586, 1586]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778038619Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2632024-01-10T17:56:52.784Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778117272Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2642024-01-10T17:56:52.784Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778162977Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2652024-01-10T17:56:52.784Z{"msg":"[2]R flush_numbers: [1586, 1586, 1586, 1586, 1586]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.77821646Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2662024-01-10T17:56:52.784Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778288339Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2672024-01-10T17:56:52.785Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778356303Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2682024-01-10T17:56:52.785Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778383595Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2692024-01-10T17:56:52.785Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778435238Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2702024-01-10T17:56:52.785Z{"msg":"Next flush: 1587","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.77850516Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2712024-01-10T17:56:52.785Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778535183Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"":"downstairs","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2722024-01-10T17:56:52.785Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778579641Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2732024-01-10T17:56:52.785Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778614953Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"0","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2742024-01-10T17:56:52.785Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778684759Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"1","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2752024-01-10T17:56:52.785Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778719464Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"client":"2","":"downstairs","session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2762024-01-10T17:56:52.785Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778742039Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2772024-01-10T17:56:52.786Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778762173Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2782024-01-10T17:56:52.786Z{"msg":"43f4537c-09cb-4d3d-8494-10eca1d88240 is now active with session: 0a1eb8ea-af0e-4a46-848d-6d92c784162f","v":0,"name":"crucible","level":30,"timeRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2792024-01-10T17:56:52.786Z":"2024-01-10T17:56:52.778855733Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2802024-01-10T17:56:52.786Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.778993772Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198,"session_id":"0a1eb8ea-af0e-4a46-848d-6d92c784162f"}
2812024-01-10T17:56:52.786Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:52.779070746Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1198}
2822024-01-10T17:56:52.786ZLoading write count information from file "/var/tmp/test_up/verify_file"
2832024-01-10T17:56:52.786ZRun Repair workload
2842024-01-10T17:56:52.786Z01/10 Write block 35 len 12288 data: 2 2 2
2852024-01-10T17:56:52.786Z02/10 Write block 5 len 12288 data: 2 2 2
2862024-01-10T17:56:52.786Z03/10 Read block 46 len 8192
2872024-01-10T17:56:52.789Z04/10 Write block 17 len 28672 data: 2 2 2 2 2 2 2
2882024-01-10T17:56:52.789Z05/10 Read block 22 len 28672
2892024-01-10T17:56:52.789Z06/10 Read block 0 len 32768
2902024-01-10T17:56:52.793Z07/10 Read block 9 len 20480
2912024-01-10T17:56:52.793Z08/10 Write block 17 len 4096 data: 3
2922024-01-10T17:56:52.793Z09/10 Read block 8 len 12288
2932024-01-10T17:56:52.796Z10/10 Read block 27 len 24576
2942024-01-10T17:56:52.799Z----------------------------------------------------------------
2952024-01-10T17:56:52.799Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:10
2962024-01-10T17:56:52.799ZStates: Active Active Active
2972024-01-10T17:56:52.799ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2982024-01-10T17:56:52.799Z 1 Acked 1000 Write 3 Done Done Done false
2992024-01-10T17:56:52.799Z 2 Acked 1001 Write 3 Done Done Done false
3002024-01-10T17:56:52.799Z 3 Acked 1002 Read 2 Done Done Done false
3012024-01-10T17:56:52.799Z 4 Acked 1003 Write 7 Done Done Done false
3022024-01-10T17:56:52.799Z 5 Acked 1004 Read 7 Done Done Done false
3032024-01-10T17:56:52.799Z 6 Acked 1005 Read 8 Done Done Done false
3042024-01-10T17:56:52.799Z 7 Acked 1006 Read 5 Done Done Done false
3052024-01-10T17:56:52.799Z 8 Acked 1007 Write 1 Done Done Done false
3062024-01-10T17:56:52.800Z 9 Acked 1008 Read 3 Done Done Done false
3072024-01-10T17:56:52.800Z 10 Acked 1009 Read 6 Done Done Done false
3082024-01-10T17:56:52.800Z STATES DS:0 DS:1 DS:2 TOTAL
3092024-01-10T17:56:52.800Z New 0 0 0 0
3102024-01-10T17:56:52.800Z Sent 0 0 0 0
3112024-01-10T17:56:52.800Z Done 10 10 10 30
3122024-01-10T17:56:52.800Z Skipped 0 0 0 0
3132024-01-10T17:56:52.800Z Error 0 0 0 0
3142024-01-10T17:56:52.800ZLast Flush: 0 0 0
3152024-01-10T17:56:52.800ZDownstairs last five completed:
3162024-01-10T17:56:52.800ZUpstairs last five completed: 10 9 8 7 6
3172024-01-10T17:56:52.800ZWrote out file "/var/tmp/test_up/verify_file"
3182024-01-10T17:56:52.803ZRepair part 1 passed
3192024-01-10T17:56:52.806Z
3202024-01-10T17:56:52.806ZKill the current downstairs
3212024-01-10T17:56:53.002Zrm -rf /var/tmp/test_up/8830
3222024-01-10T17:56:53.003ZNow put back the original so we have a mismatch
3232024-01-10T17:56:53.003Zmv /var/tmp/test_up/previous /var/tmp/test_up/8830
3242024-01-10T17:56:53.008ZRestart downstairs with old directory
3252024-01-10T17:56:53.108Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830
3262024-01-10T17:56:53.127ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3272024-01-10T17:56:53.131Z 0 000-009 31 31 31 1586 1586 1586 T T F <---
3282024-01-10T17:56:53.131Z 1 010-019 31 31 31 1586 1586 1586 T T F <---
3292024-01-10T17:56:53.131Z 2 020-029 31 31 31 1586 1586 1586 T T F <---
3302024-01-10T17:56:53.131Z 3 030-039 31 31 31 1586 1586 1586 T T F <---
3312024-01-10T17:56:53.131Z 4 040-049 31 31 31 1586 1586 1586 F F F
3322024-01-10T17:56:53.132ZMax gen: 31, Max flush: 1586
3332024-01-10T17:56:53.132ZError: Difference in extent metadata found!
3342024-01-10T17:56:53.132Zdump test found error as expected
3352024-01-10T17:56:53.132Z
3362024-01-10T17:56:53.132Z
3372024-01-10T17:56:53.132Z/var/tmp/bins/crutest deactivate --range -g 33 -q -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file --verify-in /var/tmp/test_up/verify_file
3382024-01-10T17:56:53.148Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.150122123Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205}
3392024-01-10T17:56:53.148Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.150643722Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205}
3402024-01-10T17:56:53.148Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.150719095Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205}
3412024-01-10T17:56:53.159Z{"msg":"Crucible ae69ecee-4893-4baf-a991-44d559ef10df has session id: 169dc596-1009-43db-8c6c-c33b1f00386c","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.150789644Z","hostname":"Crucible runtime is spawned
3422024-01-10T17:56:53.159Zip-10-150-1-73.us-west-2.compute.internal","pid":1205,"session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3432024-01-10T17:56:53.159Z{"msg":"Upstairs opts: Upstairs UUID: ae69ecee-4893-4baf-a991-44d559ef10df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.15086368Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3442024-01-10T17:56:53.160Z{"msg":"Crucible stats registered with UUID: ae69ecee-4893-4baf-a991-44d559ef10df","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.150965269Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3452024-01-10T17:56:53.160Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.150996939Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205}
3462024-01-10T17:56:53.160Z{"msg":"ae69ecee-4893-4baf-a991-44d559ef10df active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151040828Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3472024-01-10T17:56:53.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151063222Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3482024-01-10T17:56:53.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151111684Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3492024-01-10T17:56:53.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151135983Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3502024-01-10T17:56:53.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151158725Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3512024-01-10T17:56:53.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151181926Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3522024-01-10T17:56:53.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151204514Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3532024-01-10T17:56:53.161Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151252107Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3542024-01-10T17:56:53.161Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151299632Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3552024-01-10T17:56:53.161Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151324137Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3562024-01-10T17:56:53.161Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-01-10T17:56:53.151547743Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3572024-01-10T17:56:53.161Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151610442Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3582024-01-10T17:56:53.161Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-01-10T17:56:53.151663347Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3592024-01-10T17:56:53.161Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151708192Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3602024-01-10T17:56:53.161Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-01-10T17:56:53.151761717Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3612024-01-10T17:56:53.161Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151800483Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3622024-01-10T17:56:53.161Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.151883299Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3632024-01-10T17:56:53.161Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.152062024Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3642024-01-10T17:56:53.161Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.152924044Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3652024-01-10T17:56:53.161Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.152992489Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3662024-01-10T17:56:53.161Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.153034278Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3672024-01-10T17:56:53.161Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.153102851Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3682024-01-10T17:56:53.161Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.153834317Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3692024-01-10T17:56:53.165Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.153973926Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3702024-01-10T17:56:53.165Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.154581655Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3712024-01-10T17:56:53.165Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.154672086Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3722024-01-10T17:56:53.165Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.154757577Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3732024-01-10T17:56:53.165Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.154822631Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3742024-01-10T17:56:53.166Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.154871887Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3752024-01-10T17:56:53.166Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.154896223Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3762024-01-10T17:56:54.151Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:54.152827765Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3772024-01-10T17:56:54.151Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-01-10T17:56:54.152920795Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3782024-01-10T17:56:54.151Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-01-10T17:56:54.152951845Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3792024-01-10T17:56:54.151Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-01-10T17:56:54.15297864Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3802024-01-10T17:56:54.151Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:54.152999542Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3812024-01-10T17:56:54.154Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:54.153041999Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3822024-01-10T17:56:54.155Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:54.153101077Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3832024-01-10T17:56:55.152Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.154041606Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3842024-01-10T17:56:55.152Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.154220643Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3852024-01-10T17:56:55.158Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.155322283Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3862024-01-10T17:56:55.158Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.155382424Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3872024-01-10T17:56:55.158Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.156130257Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3882024-01-10T17:56:55.158Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.156622941Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3892024-01-10T17:56:55.158Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.156705507Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3902024-01-10T17:56:55.159Z{"msg":"[0]R flush_numbers: [1586, 1586, 1586, 1586, 1586]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.156740476Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3912024-01-10T17:56:55.159Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.156764497Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3922024-01-10T17:56:55.159Z{"msg":"[0]R dirty: [true, true, true, true, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.156786748Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3932024-01-10T17:56:55.159Z{"msg":"[1]R flush_numbers: [1586, 1586, 1586, 1586, 1586]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.156830128Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3942024-01-10T17:56:55.159Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.156854436Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3952024-01-10T17:56:55.160Z{"msg":"[1]R dirty: [true, true, true, true, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.156876406Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3962024-01-10T17:56:55.160Z{"msg":"[2]R flush_numbers: [1586, 1586, 1586, 1586, 1586]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.156899258Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3972024-01-10T17:56:55.160Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.156940903Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3982024-01-10T17:56:55.160Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.156964601Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
3992024-01-10T17:56:55.160Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157005012Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4002024-01-10T17:56:55.160Z{"msg":"Generation requested: 33 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157028698Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4012024-01-10T17:56:55.160Z{"msg":"Next flush: 1587","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157069258Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4022024-01-10T17:56:55.160Z{"msg":"Extents 0 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157093072Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4032024-01-10T17:56:55.160Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157135213Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4042024-01-10T17:56:55.160Z{"msg":"extent:0 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157160029Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4052024-01-10T17:56:55.161Z{"msg":"extent:0 flush: 1586 1586 1586 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157184414Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4062024-01-10T17:56:55.161Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157208186Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4072024-01-10T17:56:55.161Z{"msg":"extent:0 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157255936Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4082024-01-10T17:56:55.161Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157281345Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4092024-01-10T17:56:55.161Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157304694Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4102024-01-10T17:56:55.161Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157346534Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4112024-01-10T17:56:55.161Z{"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157372594Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4122024-01-10T17:56:55.161Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157398041Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4132024-01-10T17:56:55.161Z{"msg":"extent:1 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157440738Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4142024-01-10T17:56:55.161Z{"msg":"extent:1 flush: 1586 1586 1586 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157467106Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4152024-01-10T17:56:55.162Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157492124Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4162024-01-10T17:56:55.162Z{"msg":"extent:1 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157534319Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4172024-01-10T17:56:55.162Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157559424Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4182024-01-10T17:56:55.162Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157582759Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4192024-01-10T17:56:55.162Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157625313Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4202024-01-10T17:56:55.162Z{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157650312Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4212024-01-10T17:56:55.162Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157673401Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4222024-01-10T17:56:55.162Z{"msg":"extent:2 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157696259Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4232024-01-10T17:56:55.163Z{"msg":"extent:2 flush: 1586 1586 1586 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157721786Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4242024-01-10T17:56:55.163Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157746479Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4252024-01-10T17:56:55.163Z{"msg":"extent:2 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.15778814Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4262024-01-10T17:56:55.163Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157814105Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4272024-01-10T17:56:55.164Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157856234Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4282024-01-10T17:56:55.164Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157881026Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4292024-01-10T17:56:55.164Z{"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157922831Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4302024-01-10T17:56:55.164Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157947322Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4312024-01-10T17:56:55.164Z{"msg":"extent:3 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.157989351Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4322024-01-10T17:56:55.164Z{"msg":"extent:3 flush: 1586 1586 1586 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158015523Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4332024-01-10T17:56:55.164Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158059198Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4342024-01-10T17:56:55.164Z{"msg":"extent:3 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158084151Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4352024-01-10T17:56:55.164Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158107671Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4362024-01-10T17:56:55.164Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158149894Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4372024-01-10T17:56:55.188Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158175001Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4382024-01-10T17:56:55.188Z{"msg":"Transition from WaitQuorum to Repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158200319Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4392024-01-10T17:56:55.188Z{"msg":"Transition from WaitQuorum to Repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158222114Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4402024-01-10T17:56:55.188Z{"msg":"Transition from WaitQuorum to Repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158262365Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4412024-01-10T17:56:55.189Z{"msg":"Found 4 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158285277Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4422024-01-10T17:56:55.190Z{"msg":"Full repair list: {3: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 2: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 0: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 1: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158333931Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4432024-01-10T17:56:55.190Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 3, client_id: ClientId(0), flush_number: 1587, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 3 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 3, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 3 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 2, client_id: ClientId(0), flush_number: 1587, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 2, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 0, client_id: ClientId(0), flush_number: 1587, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 0 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 0, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 0 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 1, client_id: ClientId(0), flush_number: 1587, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 1 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 1, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 1 }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.1584824Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4442024-01-10T17:56:55.192Z{"msg":"sending flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158532691Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4452024-01-10T17:56:55.192Z{"msg":"skipping flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158557027Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4462024-01-10T17:56:55.194Z{"msg":"skipping flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.158578681Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4472024-01-10T17:56:55.194Z{"msg":"no action needed request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.160246365Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4482024-01-10T17:56:55.194Z{"msg":"sending repair request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.160307037Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4492024-01-10T17:56:55.194Z{"msg":"sending repair request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.160332695Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4502024-01-10T17:56:55.276Z{"msg":"sending flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.27465687Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4512024-01-10T17:56:55.276Z{"msg":"skipping flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.274746685Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4522024-01-10T17:56:55.276Z{"msg":"skipping flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.274774279Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4532024-01-10T17:56:55.279Z{"msg":"no action needed request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.276517365Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4542024-01-10T17:56:55.279Z{"msg":"sending repair request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.276581094Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4552024-01-10T17:56:55.280Z{"msg":"sending repair request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.276622044Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4562024-01-10T17:56:55.364Z{"msg":"sending flush request ReconciliationId(8)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.356459767Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4572024-01-10T17:56:55.364Z{"msg":"skipping flush request ReconciliationId(8)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.356545937Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4582024-01-10T17:56:55.364Z{"msg":"skipping flush request ReconciliationId(8)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.356573371Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4592024-01-10T17:56:55.368Z{"msg":"no action needed request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.35837863Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4602024-01-10T17:56:55.368Z{"msg":"sending repair request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.358446993Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4612024-01-10T17:56:55.368Z{"msg":"sending repair request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.358475768Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4622024-01-10T17:56:55.431Z{"msg":"sending flush request ReconciliationId(12)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.434008435Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4632024-01-10T17:56:55.432Z{"msg":"skipping flush request ReconciliationId(12)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.434094429Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4642024-01-10T17:56:55.432Z{"msg":"skipping flush request ReconciliationId(12)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.434121762Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4652024-01-10T17:56:55.435Z{"msg":"no action needed request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.436476894Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4662024-01-10T17:56:55.435Z{"msg":"sending repair request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.436544508Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4672024-01-10T17:56:55.435Z{"msg":"sending repair request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.436570771Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4682024-01-10T17:56:55.511Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.512568936Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"":"downstairs","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4692024-01-10T17:56:55.511Z{"msg":"Transition from Repair to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
4702024-01-10T17:56:55.511Z,"time":"2024-01-10T17:56:55.512670472Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"0","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4712024-01-10T17:56:55.511Z{"msg":"Transition from Repair to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.512700488Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"1","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4722024-01-10T17:56:55.514Z{"msg":"Transition from Repair to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.512746711Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"client":"2","":"downstairs","session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4732024-01-10T17:56:55.515Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.512790792Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4742024-01-10T17:56:55.517Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.512813178Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4752024-01-10T17:56:55.517Z{"msg":"ae69ecee-4893-4baf-a991-44d559ef10df is now active with session: 169dc596-1009-43db-8c6c-c33b1f00386c","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.512854923Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205,"session_id":"169dc596-1009-43db-8c6c-c33b1f00386c"}
4762024-01-10T17:56:55.517Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:55.512877437Z","hostname":"ip-10-150-1-73.us-west-2.compute.internal","pid":1205}
4772024-01-10T17:56:55.517ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4782024-01-10T17:56:55.517ZLoading write count information from file "/var/tmp/test_up/verify_file"
4792024-01-10T17:56:55.517ZRead and Verify all blocks (0..50 range:true)
4802024-01-10T17:56:55.531ZWrote out file "/var/tmp/test_up/verify_file"
4812024-01-10T17:56:55.531ZVerify test completed
4822024-01-10T17:56:55.534ZWrote out file "/var/tmp/test_up/verify_file"
4832024-01-10T17:56:55.534ZCLIENT: Tests done. All submitted work has been ACK'd
4842024-01-10T17:56:55.534Z----------------------------------------------------------------
4852024-01-10T17:56:55.534Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
4862024-01-10T17:56:55.535ZStates: Active Active Active
4872024-01-10T17:56:55.535ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
4882024-01-10T17:56:55.535Z 1 Acked 1000 Read 50 Done Done Done false
4892024-01-10T17:56:55.535Z STATES DS:0 DS:1 DS:2 TOTAL
4902024-01-10T17:56:55.535Z New 0 0 0 0
4912024-01-10T17:56:55.535Z Sent 0 0 0 0
4922024-01-10T17:56:55.535Z Done 1 1 1 3
4932024-01-10T17:56:55.535Z Skipped 0 0 0 0
4942024-01-10T17:56:55.535Z Error 0 0 0 0
4952024-01-10T17:56:55.536ZLast Flush: 0 0 0
4962024-01-10T17:56:55.536ZDownstairs last five completed:
4972024-01-10T17:56:55.536ZUpstairs last five completed: 1
4982024-01-10T17:56:55.536ZCLIENT: Up:0 ds:1 act:3
4992024-01-10T17:56:59.534Z----------------------------------------------------------------
5002024-01-10T17:56:59.534Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5012024-01-10T17:56:59.534ZDownstairs last five completed: 1001 1000
5022024-01-10T17:56:59.534ZUpstairs last five completed: 2 1
5032024-01-10T17:56:59.535ZCLIENT: Up:0 ds:0 act:3
5042024-01-10T17:56:59.535ZCLIENT: All crucible jobs finished, exiting program
5052024-01-10T17:56:59.539ZRepair part 2 passed
5062024-01-10T17:56:59.543Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830
5072024-01-10T17:56:59.560ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5082024-01-10T17:56:59.560Z 0 000-009 32 32 32 1587 1587 1587 F F F
5092024-01-10T17:56:59.560Z 1 010-019 32 32 32 1587 1587 1587 F F F
5102024-01-10T17:56:59.560Z 2 020-029 32 32 32 1587 1587 1587 F F F
5112024-01-10T17:56:59.560Z 3 030-039 32 32 32 1587 1587 1587 F F F
5122024-01-10T17:56:59.560Z 4 040-049 31 31 31 1586 1586 1586 F F F
5132024-01-10T17:56:59.564ZMax gen: 32, Max flush: 1587
5142024-01-10T17:56:59.564Zdump test passed
5152024-01-10T17:56:59.564Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 -e 1
5162024-01-10T17:56:59.580Z Extent 1
5172024-01-10T17:56:59.580ZGEN 32 32 32
5182024-01-10T17:56:59.580ZFLUSH_ID 1587 1587 1587
5192024-01-10T17:56:59.580ZDIRTY
5202024-01-10T17:56:59.581Z
5212024-01-10T17:56:59.584ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5222024-01-10T17:56:59.584Z010 A A A A A A
5232024-01-10T17:56:59.584Z011 A A A A A A
5242024-01-10T17:56:59.584Z012 A A A A A A
5252024-01-10T17:56:59.584Z013 A A A A A A
5262024-01-10T17:56:59.584Z014 A A A A A A
5272024-01-10T17:56:59.587Z015 A A A A A A
5282024-01-10T17:56:59.587Z016 A A A A A A
5292024-01-10T17:56:59.587Z017 A A A A A A
5302024-01-10T17:56:59.587Z018 A A A A A A
5312024-01-10T17:56:59.587Z019 A A A A A A
5322024-01-10T17:56:59.591Zdump extent test passed
5332024-01-10T17:56:59.594Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 -b 20
5342024-01-10T17:56:59.608ZExtent 2 Block in extent 0 Actual block 20
5352024-01-10T17:56:59.608Z
5362024-01-10T17:56:59.612Z DATA SHA256 VER
5372024-01-10T17:56:59.612Z------ ---------------------------------------------------------------- ---
5382024-01-10T17:56:59.612Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5392024-01-10T17:56:59.612Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5402024-01-10T17:56:59.618Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5412024-01-10T17:56:59.618Z
5422024-01-10T17:56:59.618ZNONCES 0 1 2 DIFF
5432024-01-10T17:56:59.618Z------ ------------------------ ------------------------ ------------------------ -----
5442024-01-10T17:56:59.618Z 0 <---
5452024-01-10T17:56:59.618Z
5462024-01-10T17:56:59.618Z TAGS 0 1 2 DIFF
5472024-01-10T17:56:59.618Z------ -------------------------------- -------------------------------- -------------------------------- -----
5482024-01-10T17:56:59.618Z 0 <---
5492024-01-10T17:56:59.618Z
5502024-01-10T17:56:59.618ZHASHES 0 1 2 DIFF
5512024-01-10T17:56:59.618Z------ ---------------- ---------------- ---------------- -----
5522024-01-10T17:56:59.618Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5532024-01-10T17:56:59.619Z
5542024-01-10T17:56:59.622Zdump block test passed
5552024-01-10T17:56:59.622ZUpstairs tests have completed, stopping all downstairs
5562024-01-10T17:56:59.717Z
5572024-01-10T17:56:59.717ZAll Tests have passed
5582024-01-10T17:56:59.720Z1:58 Test duration
5592024-01-10T17:56:59.720Z
5602024-01-10T17:56:59.720Zreal 1:58.152015829
5612024-01-10T17:56:59.720Zuser 1:08.457477147
5622024-01-10T17:56:59.720Zsys 16.507763508
5632024-01-10T17:56:59.720Ztrap 0.140902798
5642024-01-10T17:56:59.720Ztflt 0.007637838
5652024-01-10T17:56:59.720Zdflt 0.036294513
5662024-01-10T17:56:59.721Zkflt 0.001116291
5672024-01-10T17:56:59.721Zlock 24:25.376145533
5682024-01-10T17:56:59.721Zslp 3:40.614807427
5692024-01-10T17:56:59.721Zlat 13.411857856
5702024-01-10T17:56:59.721Zstop 0.030584236
5712024-01-10T17:56:59.721Zprocess exited: duration 130770 ms, exit code 0
 
5722024-01-10T17:56:59.733Zfound 5 output files
5732024-01-10T17:56:59.733Zuploading: /tmp/test_up/dsc-out.txt (4473 bytes)
5742024-01-10T17:57:00.748Zuploaded: /tmp/test_up/dsc-out.txt
5752024-01-10T17:57:00.748Zuploading: /tmp/test_up/test_up_out.txt (5317377 bytes)
5762024-01-10T17:57:02.463Zuploaded: /tmp/test_up/test_up_out.txt
5772024-01-10T17:57:02.463Zuploading: /tmp/test_up/dsc/downstairs-8810.txt (56140 bytes)
5782024-01-10T17:57:03.482Zuploaded: /tmp/test_up/dsc/downstairs-8810.txt
5792024-01-10T17:57:03.482Zuploading: /tmp/test_up/dsc/downstairs-8820.txt (52927 bytes)
5802024-01-10T17:57:04.500Zuploaded: /tmp/test_up/dsc/downstairs-8820.txt
5812024-01-10T17:57:04.500Zuploading: /tmp/test_up/dsc/downstairs-8830.txt (9710 bytes)
5822024-01-10T17:57:05.511Zuploaded: /tmp/test_up/dsc/downstairs-8830.txt