01J8TTN3WKEEH5Z1N8E1VFZH6G: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01J8TTNGZQ105CJBKFGBMVHKHY

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-09-27T23:03:10.453Zjob dependencies complete; ready to run (waiting for 15 m 53 s)
22024-09-27T23:04:20.858Zjob assigned to worker 01J8TVJW4S3S4T5CS22D0A62BC (queued for 1 m 10 s)
32024-09-27T23:04:30.402Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-09-27T23:04:31.511Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-09-27T23:04:31.511Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-09-27T23:04:32.586Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-09-27T23:04:32.586Zdownloading input: /input/build/work/bins/crutest.gz
82024-09-27T23:04:33.709Zdownloaded input: /input/build/work/bins/crutest.gz
92024-09-27T23:04:33.709Zdownloading input: /input/build/work/bins/dsc.gz
102024-09-27T23:04:34.434Zdownloaded input: /input/build/work/bins/dsc.gz
112024-09-27T23:04:34.436Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-09-27T23:04:34.454Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-09-27T23:04:34.454Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-09-27T23:04:34.473Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-09-27T23:04:34.473Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-09-27T23:04:34.494Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-09-27T23:04:34.494Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-09-27T23:04:34.514Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-09-27T23:04:34.514Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-09-27T23:04:34.533Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-09-27T23:04:34.533Zdownloading input: /input/build/work/scripts/test_up.sh
222024-09-27T23:04:34.551Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-09-27T23:04:34.551Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-09-27T23:04:34.569Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-09-27T23:04:34.572Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-09-27T23:04:35.337Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-09-27T23:04:35.337Zstarting task 0: "setup"
282024-09-27T23:04:35.346Z++ uname -s
292024-09-27T23:04:35.348Z+ kern=SunOS
302024-09-27T23:04:35.348Z+ case "$kern" in
312024-09-27T23:04:35.348Z+ groupadd -g 12345 build
322024-09-27T23:04:35.351Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-09-27T23:04:37.369Z+ zfs create -o mountpoint=/work rpool/work
342024-09-27T23:04:37.986Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-09-27T23:04:38.008Z+ home_fs=zfs
362024-09-27T23:04:38.008Z+ [[ zfs == autofs ]]
372024-09-27T23:04:38.008Z+ mkdir -p /home/build
382024-09-27T23:04:38.008Z+ chown build:build /home/build /work
392024-09-27T23:04:38.984Z+ chmod 0700 /home/build /work
402024-09-27T23:04:38.988Zprocess exited: duration 3650 ms, exit code 0
 
412024-09-27T23:04:38.993Zstarting task 1: "authentication"
422024-09-27T23:04:39.010Zprocess exited: duration 16 ms, exit code 0
 
432024-09-27T23:04:39.017Zstarting task 2: "build"
442024-09-27T23:04:39.020Z+ banner cores
452024-09-27T23:04:39.040Z
462024-09-27T23:04:39.040Z #### #### ##### ###### ####
472024-09-27T23:04:39.040Z # # # # # # # #
482024-09-27T23:04:39.040Z # # # # # ##### ####
492024-09-27T23:04:39.040Z # # # ##### # #
502024-09-27T23:04:39.040Z # # # # # # # # #
512024-09-27T23:04:39.040Z #### #### # # ###### ####
522024-09-27T23:04:39.040Z
532024-09-27T23:04:39.042Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-09-27T23:04:39.050Z+ echo 'input bins dir contains:'
552024-09-27T23:04:39.050Zinput bins dir contains:
562024-09-27T23:04:39.053Z+ ls -ltr /input/build/work/bins
572024-09-27T23:04:39.053Ztotal 771769
582024-09-27T23:04:39.053Z-rw-r--r-- 1 root root 110734273 Sep 27 23:04 crucible-downstairs.gz
592024-09-27T23:04:39.053Z-rw-r--r-- 1 root root 105947543 Sep 27 23:04 crucible-hammer.gz
602024-09-27T23:04:39.053Z-rw-r--r-- 1 root root 111407444 Sep 27 23:04 crutest.gz
612024-09-27T23:04:39.053Z-rw-r--r-- 1 root root 66799675 Sep 27 23:04 dsc.gz
622024-09-27T23:04:39.053Z+ banner unpack
632024-09-27T23:04:39.055Z
642024-09-27T23:04:39.055Z # # # # ##### ## #### # #
652024-09-27T23:04:39.055Z # # ## # # # # # # # # #
662024-09-27T23:04:39.055Z # # # # # # # # # # ####
672024-09-27T23:04:39.055Z # # # # # ##### ###### # # #
682024-09-27T23:04:39.055Z # # # ## # # # # # # #
692024-09-27T23:04:39.055Z #### # # # # # #### # #
702024-09-27T23:04:39.055Z
712024-09-27T23:04:39.055Z+ mkdir -p /var/tmp/bins
722024-09-27T23:04:39.058Z+ for t in "$input/bins/"*.gz
732024-09-27T23:04:39.058Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-09-27T23:04:39.061Z+ b=crucible-downstairs.gz
752024-09-27T23:04:39.061Z+ b=crucible-downstairs
762024-09-27T23:04:39.061Z+ gunzip
772024-09-27T23:04:41.347Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-09-27T23:04:41.349Z+ for t in "$input/bins/"*.gz
792024-09-27T23:04:41.352Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-09-27T23:04:41.352Z+ b=crucible-hammer.gz
812024-09-27T23:04:41.352Z+ b=crucible-hammer
822024-09-27T23:04:41.352Z+ gunzip
832024-09-27T23:04:43.570Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-09-27T23:04:43.572Z+ for t in "$input/bins/"*.gz
852024-09-27T23:04:43.574Z++ basename /input/build/work/bins/crutest.gz
862024-09-27T23:04:43.574Z+ b=crutest.gz
872024-09-27T23:04:43.574Z+ b=crutest
882024-09-27T23:04:43.574Z+ gunzip
892024-09-27T23:04:45.849Z+ chmod +x /var/tmp/bins/crutest
902024-09-27T23:04:45.855Z+ for t in "$input/bins/"*.gz
912024-09-27T23:04:45.859Z++ basename /input/build/work/bins/dsc.gz
922024-09-27T23:04:45.859Z+ b=dsc.gz
932024-09-27T23:04:45.859Z+ b=dsc
942024-09-27T23:04:45.859Z+ gunzip
952024-09-27T23:04:47.282Z+ chmod +x /var/tmp/bins/dsc
962024-09-27T23:04:47.285Z+ export BINDIR=/var/tmp/bins
972024-09-27T23:04:47.285Z+ BINDIR=/var/tmp/bins
982024-09-27T23:04:47.289Z+ jobpid=1139
992024-09-27T23:04:47.289Z+ echo 'Setup debug logging'
1002024-09-27T23:04:47.289ZSetup debug logging
1012024-09-27T23:04:47.289Z+ mkdir /tmp/debug
1022024-09-27T23:04:47.289Z+ sleep 7200
1032024-09-27T23:04:47.301Z+ psrinfo -v
1042024-09-27T23:04:47.301Z+ df -h
1052024-09-27T23:04:47.316Z+ prstat -d d -mLc 1
1062024-09-27T23:04:47.319Z+ iostat -T d -xn 1
1072024-09-27T23:04:47.319Z+ mpstat -T d 1
1082024-09-27T23:04:47.319Z+ vmstat -T d -p 1
1092024-09-27T23:04:47.321Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102024-09-27T23:04:47.322Z+ banner test_up_unencrypted
1112024-09-27T23:04:47.322Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1122024-09-27T23:04:47.324Z
1132024-09-27T23:04:47.324Z ##### ###### #### ##### # # ##### # # # #
1142024-09-27T23:04:47.324Z # # # # # # # # # # ## #
1152024-09-27T23:04:47.324Z # ##### #### # # # # # # # # # #
1162024-09-27T23:04:47.324Z # # # # # # ##### # # # # #
1172024-09-27T23:04:47.324Z # # # # # # # # # # # ##
1182024-09-27T23:04:47.324Z # ###### #### # ####### #### # ####### #### # #
1192024-09-27T23:04:47.324Z
1202024-09-27T23:04:47.324Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1212024-09-27T23:04:47.333Z/input/build/work
1222024-09-27T23:04:47.363ZTurn off color for downstairs dump
1232024-09-27T23:04:47.365Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1242024-09-27T23:04:47.365ZCreating three downstairs regions
1252024-09-27T23:04:47.763ZStarting three downstairs
1262024-09-27T23:04:52.770ZDisable automatic restart on all downstairs
1272024-09-27T23:04:52.842Z
1282024-09-27T23:04:52.842ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292024-09-27T23:04:52.845ZRunning test: span
1302024-09-27T23:04:56.986ZCompleted test: span
1312024-09-27T23:05:01.992ZRunning test: big
1322024-09-27T23:05:06.220ZCompleted test: big
1332024-09-27T23:05:11.227ZRunning test: dep
1342024-09-27T23:05:24.484ZCompleted test: dep
1352024-09-27T23:05:29.491ZRunning test: balloon
1362024-09-27T23:05:43.206ZCompleted test: balloon
1372024-09-27T23:05:48.213ZRunning test: deactivate
1382024-09-27T23:06:42.506ZCompleted test: deactivate
1392024-09-27T23:06:47.528ZRunning hammer
1402024-09-27T23:06:51.169ZRun repair tests
1412024-09-27T23:06:51.172Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422024-09-27T23:06:51.231Z{"msg":"use region info: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.38138707Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"task":"crutest"}
1432024-09-27T23:06:51.252Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.381941222Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"task":"crutest"}
1442024-09-27T23:06:51.252Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.383660456Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"task":"crutest"}
1452024-09-27T23:06:51.252Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.383827804Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218}
1462024-09-27T23:06:51.252Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"60f78286ad49cec90cdcc3e028e5e9c76d4fdc1d\",\n git_commit_timestamp: \"2024-09-27T22:46:32.000000000Z\",\n git_branch: \"alan/tests-use-dsc\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.383876541Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218}
1472024-09-27T23:06:51.253Z{"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.383903689Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218}
1482024-09-27T23:06:51.253Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.383923118Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218}
1492024-09-27T23:06:51.253Z{"msg":"Crucible ae763077-766a-41d5-a311-777da5e37697 has session id: e249589f-4c71-469c-a429-85d120e595a3","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.384303722Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1502024-09-27T23:06:51.253Z{"msg":"Upstairs opts: Upstairs UUID: ae763077-766a-41d5-a311-777da5e37697, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.38434352Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1512024-09-27T23:06:51.253Z{"msg":"Crucible stats registered with UUID: ae763077-766a-41d5-a311-777da5e37697","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.384361248Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1522024-09-27T23:06:51.285Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.417992648Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218}
1532024-09-27T23:06:51.285Z{"msg":"ae763077-766a-41d5-a311-777da5e37697 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.418053054Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1542024-09-27T23:06:51.285Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.418099431Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"0","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1552024-09-27T23:06:51.285Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.418167747Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"0","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1562024-09-27T23:06:51.285Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.418222953Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"1","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1572024-09-27T23:06:51.285Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.418281469Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"1","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1582024-09-27T23:06:51.286Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.418320336Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"2","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1592024-09-27T23:06:51.286Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.418369823Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"2","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1602024-09-27T23:06:51.286Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.41840532Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"io task","client":"2","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1612024-09-27T23:06:51.286Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.418439178Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"io task","client":"0","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1622024-09-27T23:06:51.286Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.418487025Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"io task","client":"1","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1632024-09-27T23:06:51.286Z{"msg":"ds_connection connected from Ok(127.0.0.1:38978)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.418531782Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"io task","client":"2","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1642024-09-27T23:06:51.286Z{"msg":"ds_connection connected from Ok(127.0.0.1:39463)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.41856142Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"io task","client":"0","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1652024-09-27T23:06:51.286Z{"msg":"ds_connection connected from Ok(127.0.0.1:44905)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.418589588Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"io task","client":"1","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1662024-09-27T23:06:51.286Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.419182308Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"2","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1672024-09-27T23:06:51.286Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.419233184Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"2","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1682024-09-27T23:06:51.286Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.419267552Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"0","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1692024-09-27T23:06:51.286Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.419303739Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"0","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1702024-09-27T23:06:51.286Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.419356056Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"1","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1712024-09-27T23:06:51.286Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.419395953Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"1","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1722024-09-27T23:06:51.286Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.41988084Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"1","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1732024-09-27T23:06:51.286Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.419951966Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"2","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1742024-09-27T23:06:51.286Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420047309Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"0","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1752024-09-27T23:06:51.286Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420295542Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"1","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1762024-09-27T23:06:51.286Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420338249Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"1","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1772024-09-27T23:06:51.286Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420359678Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1782024-09-27T23:06:51.286Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420455361Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"0","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1792024-09-27T23:06:51.286Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.42048365Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"0","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1802024-09-27T23:06:51.286Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420499449Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1812024-09-27T23:06:51.286Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420512048Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"2","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1822024-09-27T23:06:51.286Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420533466Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"2","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1832024-09-27T23:06:51.286Z{"msg":"[0]R flush_numbers: [1531, 1531, 1531, 1531, 1531]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420548455Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1842024-09-27T23:06:51.286Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420569464Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1852024-09-27T23:06:51.286Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420583463Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1862024-09-27T23:06:51.287Z{"msg":"[1]R flush_numbers: [1531, 1531, 1531, 1531, 1531]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420603601Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1872024-09-27T23:06:51.287Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.42061809Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1882024-09-27T23:06:51.287Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420632Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1892024-09-27T23:06:51.287ZWait for a query_work_queue command to finish before sending IO
1902024-09-27T23:06:51.287Z{"msg":"[2]R flush_numbers: [1531, 1531, 1531, 1531, 1531]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420720583Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1912024-09-27T23:06:51.287Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420785769Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1922024-09-27T23:06:51.287Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420817767Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1932024-09-27T23:06:51.287Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420832426Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1942024-09-27T23:06:51.287Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420858574Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1952024-09-27T23:06:51.287Z{"msg":"Next flush: 1532","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420874713Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1962024-09-27T23:06:51.287Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420889802Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"":"downstairs","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1972024-09-27T23:06:51.287Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.42091131Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1982024-09-27T23:06:51.287Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.42092498Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"0","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
1992024-09-27T23:06:51.287Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2002024-09-27T23:06:51.287Z,"time":"2024-09-27T23:06:50.420945738Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"1","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
2012024-09-27T23:06:51.287Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420962337Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"client":"2","":"downstairs","session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
2022024-09-27T23:06:51.287ZFill test
2032024-09-27T23:06:51.287Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420984925Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
2042024-09-27T23:06:51.287Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.420998185Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
2052024-09-27T23:06:51.287Z{"msg":"ae763077-766a-41d5-a311-777da5e37697 is now active with session: e249589f-4c71-469c-a429-85d120e595a3","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.421018913Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
2062024-09-27T23:06:51.287Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.421032122Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218,"session_id":"e249589f-4c71-469c-a429-85d120e595a3"}
2072024-09-27T23:06:51.300Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:50.421047291Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218}
2082024-09-27T23:06:51.300ZRead and Verify all blocks (0..50 range:false)
2092024-09-27T23:06:51.300ZWrote out file "/var/tmp/test_up-build/verify_file"
2102024-09-27T23:06:51.300ZCLIENT: Tests done. All submitted work has been ACK'd
2112024-09-27T23:06:51.300Z----------------------------------------------------------------
2122024-09-27T23:06:51.300Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2132024-09-27T23:06:51.300ZStates: Active Active Active
2142024-09-27T23:06:51.300ZGW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2152024-09-27T23:06:51.301Z 3 Acked 1002 Read 50 Done Done Done false
2162024-09-27T23:06:51.301Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-09-27T23:06:51.301Z Sent 0 0 0 0
2182024-09-27T23:06:51.301Z Done 1 1 1 3
2192024-09-27T23:06:51.301Z Skipped 0 0 0 0
2202024-09-27T23:06:51.301Z Error 0 0 0 0
2212024-09-27T23:06:51.301ZLast Flush: 1001 1001 1001
2222024-09-27T23:06:51.301ZDownstairs last five completed: 1001 1000
2232024-09-27T23:06:51.301ZUpstairs last five completed: 3 2 1
2242024-09-27T23:06:51.301ZCLIENT: Up:0 ds:1 act:3
2252024-09-27T23:06:55.289Z----------------------------------------------------------------
2262024-09-27T23:06:55.289Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2272024-09-27T23:06:55.289ZDownstairs last five completed: 1003 1002 1001 1000
2282024-09-27T23:06:55.289ZUpstairs last five completed: 4 3 2 1
2292024-09-27T23:06:55.289ZCLIENT: Up:0 ds:0 act:3
2302024-09-27T23:06:55.289ZCLIENT: All crucible jobs finished, exiting program
2312024-09-27T23:06:55.291Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T23:06:54.440778229Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1218}
2322024-09-27T23:06:55.300ZRepair setup passed
2332024-09-27T23:06:55.304ZCopy the region for /var/tmp/test_up-build/8830
2342024-09-27T23:06:55.307Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2352024-09-27T23:06:55.309Z/var/tmp/bins/crutest repair -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2362024-09-27T23:06:55.365Z{"msg":"use region info: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.516480757Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"task":"crutest"}
2372024-09-27T23:06:55.370Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.51703509Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"task":"crutest"}
2382024-09-27T23:06:55.370Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.518674609Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"task":"crutest"}
2392024-09-27T23:06:55.370Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.518847557Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224}
2402024-09-27T23:06:55.370Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"60f78286ad49cec90cdcc3e028e5e9c76d4fdc1d\",\n git_commit_timestamp: \"2024-09-27T22:46:32.000000000Z\",\n git_branch: \"alan/tests-use-dsc\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.518900143Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224}
2412024-09-27T23:06:55.370Z{"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.518938611Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224}
2422024-09-27T23:06:55.370Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.518967009Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224}
2432024-09-27T23:06:55.371Z{"msg":"Crucible e8e86cc8-d9e6-43e0-88ba-8bdeffce21ed has session id: 05fa253f-62c1-4bf6-a50c-3dc436dfc03c","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.519320905Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2442024-09-27T23:06:55.371Z{"msg":"Upstairs opts: Upstairs UUID: e8e86cc8-d9e6-43e0-88ba-8bdeffce21ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.51939045Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2452024-09-27T23:06:55.371Z{"msg":"Crucible stats registered with UUID: e8e86cc8-d9e6-43e0-88ba-8bdeffce21ed","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.519433687Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2462024-09-27T23:06:55.402Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.553743081Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224}
2472024-09-27T23:06:55.402Z{"msg":"e8e86cc8-d9e6-43e0-88ba-8bdeffce21ed active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.553803067Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2482024-09-27T23:06:55.404Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.553838465Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"0","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2492024-09-27T23:06:55.404Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.553932928Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"0","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2502024-09-27T23:06:55.404Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.554000554Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"1","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2512024-09-27T23:06:55.404Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.554034981Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"1","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2522024-09-27T23:06:55.404Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.554068359Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"2","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2532024-09-27T23:06:55.404Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.554115446Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"2","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2542024-09-27T23:06:55.404Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.554150234Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2552024-09-27T23:06:55.404Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.554317052Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2562024-09-27T23:06:55.404Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.554388877Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2572024-09-27T23:06:55.404Z{"msg":"ds_connection connected from Ok(127.0.0.1:58574)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.554446624Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2582024-09-27T23:06:55.404Z{"msg":"ds_connection connected from Ok(127.0.0.1:60639)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.554520399Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2592024-09-27T23:06:55.404Z{"msg":"ds_connection connected from Ok(127.0.0.1:38590)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.554555416Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2602024-09-27T23:06:55.405Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.555024874Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"1","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2612024-09-27T23:06:55.405Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.555062002Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"1","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2622024-09-27T23:06:55.405Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.555101709Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"0","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2632024-09-27T23:06:55.405Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.555125587Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"0","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2642024-09-27T23:06:55.405Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.555208732Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"2","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2652024-09-27T23:06:55.405Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.555274237Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"2","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2662024-09-27T23:06:55.405Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.555836239Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"1","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2672024-09-27T23:06:55.405Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.555874777Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"2","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2682024-09-27T23:06:55.405Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.555893446Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"0","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2692024-09-27T23:06:55.408Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556297188Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"0","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2702024-09-27T23:06:55.408Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556338345Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"0","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2712024-09-27T23:06:55.408Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556355744Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2722024-09-27T23:06:55.408Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556368673Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"2","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2732024-09-27T23:06:55.408Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556383162Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"2","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2742024-09-27T23:06:55.408Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556396641Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2752024-09-27T23:06:55.408Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.55641727Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"1","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2762024-09-27T23:06:55.408Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556432329Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"1","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2772024-09-27T23:06:55.408Z{"msg":"[0]R flush_numbers: [1532, 1532, 1532, 1532, 1532]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556470446Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2782024-09-27T23:06:55.408Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556485265Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2792024-09-27T23:06:55.408Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556499664Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2802024-09-27T23:06:55.408Z{"msg":"[1]R flush_numbers: [1532, 1532, 1532, 1532, 1532]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556513533Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2812024-09-27T23:06:55.408Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556541362Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2822024-09-27T23:06:55.408ZWait for a query_work_queue command to finish before sending IO
2832024-09-27T23:06:55.408Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556583389Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2842024-09-27T23:06:55.408Z{"msg":"[2]R flush_numbers: [1532, 1532, 1532, 1532, 1532]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556598968Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2852024-09-27T23:06:55.408Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556612837Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2862024-09-27T23:06:55.408Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556639015Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2872024-09-27T23:06:55.408Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556671053Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2882024-09-27T23:06:55.408Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556686462Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2892024-09-27T23:06:55.408Z{"msg":"Next flush: 1533","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556700161Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2902024-09-27T23:06:55.408Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556714Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"":"downstairs","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2912024-09-27T23:06:55.408Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556746518Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2922024-09-27T23:06:55.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556763287Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"0","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2932024-09-27T23:06:55.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556785755Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"1","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2942024-09-27T23:06:55.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556844281Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"client":"2","":"downstairs","session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2952024-09-27T23:06:55.408ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2962024-09-27T23:06:55.408Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556878779Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2972024-09-27T23:06:55.408Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556894238Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2982024-09-27T23:06:55.409Z{"msg":"e8e86cc8-d9e6-43e0-88ba-8bdeffce21ed is now active with session: 05fa253f-62c1-4bf6-a50c-3dc436dfc03c","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556915556Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
2992024-09-27T23:06:55.409Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556928925Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224,"session_id":"05fa253f-62c1-4bf6-a50c-3dc436dfc03c"}
3002024-09-27T23:06:55.409Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.556941994Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224}
3012024-09-27T23:06:55.409ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
3022024-09-27T23:06:55.409ZRun Repair workload
3032024-09-27T23:06:55.409Z01/10 Write block 31 len 16384 data: 2 2 2 2
3042024-09-27T23:06:55.409Z02/10 Write block 26 len 32768 data: 2 2 2 2 2 3 3 3
3052024-09-27T23:06:55.409Z03/10 Write block 5 len 36864 data: 2 2 2 2 2 2 2 2 2
3062024-09-27T23:06:55.409Z04/10 Write block 38 len 24576 data: 2 2 2 2 2 2
3072024-09-27T23:06:55.409Z05/10 Write block 34 len 36864 data: 3 2 2 2 3 3 3 3 3
3082024-09-27T23:06:55.414Z06/10 Write block 12 len 32768 data: 3 3 2 2 2 2 2 2
3092024-09-27T23:06:55.414Z07/10 Read block 14 len 16384
3102024-09-27T23:06:55.418Z08/10 Write block 5 len 12288 data: 3 3 3
3112024-09-27T23:06:55.418Z09/10 Read block 41 len 28672
3122024-09-27T23:06:55.418Z10/10 Read block 6 len 24576
3132024-09-27T23:06:55.421Z----------------------------------------------------------------
3142024-09-27T23:06:55.421Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:10
3152024-09-27T23:06:55.421ZStates: Active Active Active
3162024-09-27T23:06:55.421ZGW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3172024-09-27T23:06:55.421Z 1 Acked 1000 Write 4 Done Done Done false
3182024-09-27T23:06:55.421Z 2 Acked 1001 Write 8 Done Done Done false
3192024-09-27T23:06:55.421Z 3 Acked 1002 Write 9 Done Done Done false
3202024-09-27T23:06:55.421Z 4 Acked 1003 Write 6 Done Done Done false
3212024-09-27T23:06:55.421Z 5 Acked 1004 Write 9 Done Done Done false
3222024-09-27T23:06:55.421Z 6 Acked 1005 Write 8 Done Done Done false
3232024-09-27T23:06:55.421Z 7 Acked 1006 Read 4 Done Done Done false
3242024-09-27T23:06:55.421Z 8 Acked 1007 Write 3 Done Done Done false
3252024-09-27T23:06:55.421Z 9 Acked 1008 Read 7 Done Done Done false
3262024-09-27T23:06:55.421Z 10 Acked 1009 Read 6 Sent Done Sent false
3272024-09-27T23:06:55.421Z STATES DS:0 DS:1 DS:2 TOTAL
3282024-09-27T23:06:55.421Z Sent 1 0 1 2
3292024-09-27T23:06:55.421Z Done 9 10 9 28
3302024-09-27T23:06:55.421Z Skipped 0 0 0 0
3312024-09-27T23:06:55.421Z Error 0 0 0 0
3322024-09-27T23:06:55.421ZLast Flush: 0 0 0
3332024-09-27T23:06:55.421ZDownstairs last five completed:
3342024-09-27T23:06:55.421ZUpstairs last five completed: 10 9 8 7 6
3352024-09-27T23:06:55.421Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T23:06:54.568966749Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1224}
3362024-09-27T23:06:55.421ZWrote out file "/var/tmp/test_up-build/verify_file"
3372024-09-27T23:06:55.424ZRepair part 1 passed
3382024-09-27T23:06:55.424Z
3392024-09-27T23:06:55.424ZKill the current downstairs
3402024-09-27T23:06:55.532Zrm -rf /var/tmp/test_up-build/8830
3412024-09-27T23:06:55.532ZNow put back the original so we have a mismatch
3422024-09-27T23:06:55.532Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3432024-09-27T23:06:55.535ZRestart downstairs with old directory
3442024-09-27T23:06:55.588Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3452024-09-27T23:06:55.605ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3462024-09-27T23:06:55.608Z 0 000-009 31 31 31 1532 1532 1532 T T F <---
3472024-09-27T23:06:55.608Z 1 010-019 31 31 31 1532 1532 1532 T T F <---
3482024-09-27T23:06:55.608Z 2 020-029 31 31 31 1532 1532 1532 T T F <---
3492024-09-27T23:06:55.608Z 3 030-039 31 31 31 1532 1532 1532 T T F <---
3502024-09-27T23:06:55.608Z 4 040-049 31 31 31 1532 1532 1532 T T F <---
3512024-09-27T23:06:55.608ZMax gen: 31, Max flush: 1532
3522024-09-27T23:06:55.608ZError: Difference in extent metadata found!
3532024-09-27T23:06:55.608Zdump test found error as expected
3542024-09-27T23:06:55.611Z
3552024-09-27T23:06:55.611Z
3562024-09-27T23:06:55.611Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3572024-09-27T23:06:55.668Z{"msg":"use region info: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.817331192Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"task":"crutest"}
3582024-09-27T23:06:55.671Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.817894644Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"task":"crutest"}
3592024-09-27T23:06:55.672Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.819527213Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"task":"crutest"}
3602024-09-27T23:06:55.672Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.819695081Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231}
3612024-09-27T23:06:55.672Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"60f78286ad49cec90cdcc3e028e5e9c76d4fdc1d\",\n git_commit_timestamp: \"2024-09-27T22:46:32.000000000Z\",\n git_branch: \"alan/tests-use-dsc\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.819735569Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231}
3622024-09-27T23:06:55.672Z{"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.819758107Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231}
3632024-09-27T23:06:55.672Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.819774106Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231}
3642024-09-27T23:06:55.672Z{"msg":"Crucible 2caab520-18eb-4a6c-afdf-858339081238 has session id: 9b694fd4-c7c9-4d22-98af-61ab3819a441","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.820227775Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3652024-09-27T23:06:55.672Z{"msg":"Upstairs opts: Upstairs UUID: 2caab520-18eb-4a6c-afdf-858339081238, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.820285051Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3662024-09-27T23:06:55.672Z{"msg":"Crucible stats registered with UUID: 2caab520-18eb-4a6c-afdf-858339081238","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.82030898Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3672024-09-27T23:06:55.703Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854164705Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231}
3682024-09-27T23:06:55.703Z{"msg":"2caab520-18eb-4a6c-afdf-858339081238 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.85423112Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3692024-09-27T23:06:55.706Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854278937Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"0","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3702024-09-27T23:06:55.706Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854343462Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"0","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3712024-09-27T23:06:55.706Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854399529Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3722024-09-27T23:06:55.706Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854448795Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3732024-09-27T23:06:55.706Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854479823Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3742024-09-27T23:06:55.706Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854510661Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3752024-09-27T23:06:55.706Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854560458Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3762024-09-27T23:06:55.706Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854598665Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3772024-09-27T23:06:55.706Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854642572Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3782024-09-27T23:06:55.707Z{"msg":"ds_connection connected from Ok(127.0.0.1:55945)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854703648Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3792024-09-27T23:06:55.707Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-09-27T23:06:54.854785763Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3802024-09-27T23:06:55.707Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-09-27T23:06:54.854842489Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3812024-09-27T23:06:55.707Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-09-27T23:06:54.854872177Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3822024-09-27T23:06:55.707Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854894775Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3832024-09-27T23:06:55.707Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854919334Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3842024-09-27T23:06:55.707Z{"msg":"ds_connection connected from Ok(127.0.0.1:43518)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854934973Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3852024-09-27T23:06:55.707Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.854957511Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3862024-09-27T23:06:55.707Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.855306777Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3872024-09-27T23:06:55.707Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.855348245Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3882024-09-27T23:06:55.707Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.855377732Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"0","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3892024-09-27T23:06:55.707Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.85540727Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"0","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3902024-09-27T23:06:55.707Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.855963903Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3912024-09-27T23:06:55.707Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.85599865Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"0","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3922024-09-27T23:06:55.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.856378295Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"0","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3932024-09-27T23:06:55.707Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.856416482Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"0","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3942024-09-27T23:06:55.707Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.85644228Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3952024-09-27T23:06:55.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.856469838Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3962024-09-27T23:06:55.707Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.856494797Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3972024-09-27T23:06:55.707Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:06:54.856527414Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3982024-09-27T23:07:05.705Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.85568136Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
3992024-09-27T23:07:05.711Z{"msg":"ds_connection connected from Ok(127.0.0.1:47104)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.855862857Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4002024-09-27T23:07:05.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.85671374Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4012024-09-27T23:07:05.711Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.856761157Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4022024-09-27T23:07:05.711Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857322128Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4032024-09-27T23:07:05.711Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857710482Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4042024-09-27T23:07:05.711Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.85774154Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4052024-09-27T23:07:05.711Z{"msg":"[0]R flush_numbers: [1532, 1532, 1532, 1532, 1532]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857758609Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4062024-09-27T23:07:05.711Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857773428Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4072024-09-27T23:07:05.712Z{"msg":"[0]R dirty: [true, true, true, true, true]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857795076Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4082024-09-27T23:07:05.712Z{"msg":"[1]R flush_numbers: [1532, 1532, 1532, 1532, 1532]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857809765Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4092024-09-27T23:07:05.712Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857830864Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4102024-09-27T23:07:05.712Z{"msg":"[1]R dirty: [true, true, true, true, true]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857845283Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4112024-09-27T23:07:05.712Z{"msg":"[2]R flush_numbers: [1532, 1532, 1532, 1532, 1532]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857866631Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4122024-09-27T23:07:05.712Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857880581Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4132024-09-27T23:07:05.712Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857903649Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4142024-09-27T23:07:05.712Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857917698Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4152024-09-27T23:07:05.712Z{"msg":"Generation requested: 33 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857939167Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4162024-09-27T23:07:05.712Z{"msg":"Next flush: 1533","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857952726Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4172024-09-27T23:07:05.712Z{"msg":"Extents 0 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857972804Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4182024-09-27T23:07:05.712Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.857987393Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4192024-09-27T23:07:05.712Z{"msg":"extent:0 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858002022Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4202024-09-27T23:07:05.712Z{"msg":"extent:0 flush: 1532 1532 1532 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858025431Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4212024-09-27T23:07:05.712Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.85804076Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4222024-09-27T23:07:05.712Z{"msg":"extent:0 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858062168Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4232024-09-27T23:07:05.712Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858077167Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4242024-09-27T23:07:05.712Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858098226Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4252024-09-27T23:07:05.712Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858112995Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4262024-09-27T23:07:05.712Z{"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858141413Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4272024-09-27T23:07:05.712Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858164961Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4282024-09-27T23:07:05.712Z{"msg":"extent:1 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.85817991Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4292024-09-27T23:07:05.712Z{"msg":"extent:1 flush: 1532 1532 1532 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858202459Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4302024-09-27T23:07:05.712Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858217408Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4312024-09-27T23:07:05.712Z{"msg":"extent:1 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858242756Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4322024-09-27T23:07:05.713Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858270844Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4332024-09-27T23:07:05.713Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858298662Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4342024-09-27T23:07:05.713Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858343229Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4352024-09-27T23:07:05.713Z{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858383856Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4362024-09-27T23:07:05.713Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858419574Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4372024-09-27T23:07:05.713Z{"msg":"extent:2 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858452332Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4382024-09-27T23:07:05.713Z{"msg":"extent:2 flush: 1532 1532 1532 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.85847622Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4392024-09-27T23:07:05.713Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858491729Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4402024-09-27T23:07:05.713Z{"msg":"extent:2 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858511678Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4412024-09-27T23:07:05.713Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858536736Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4422024-09-27T23:07:05.713Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858569474Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4432024-09-27T23:07:05.713Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858586683Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4442024-09-27T23:07:05.713Z{"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858608811Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4452024-09-27T23:07:05.713Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.85862316Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4462024-09-27T23:07:05.713Z{"msg":"extent:3 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858644679Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4472024-09-27T23:07:05.713Z{"msg":"extent:3 flush: 1532 1532 1532 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858659928Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4482024-09-27T23:07:05.713Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858681066Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4492024-09-27T23:07:05.713Z{"msg":"extent:3 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858695725Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4502024-09-27T23:07:05.713Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858716604Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4512024-09-27T23:07:05.713Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858731113Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4522024-09-27T23:07:05.713Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858751951Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4532024-09-27T23:07:05.713Z{"msg":"Extents 4 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858766521Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4542024-09-27T23:07:05.713Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858786799Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4552024-09-27T23:07:05.713Z{"msg":"extent:4 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858801118Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4562024-09-27T23:07:05.714Z{"msg":"extent:4 flush: 1532 1532 1532 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858822627Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4572024-09-27T23:07:05.714Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858837656Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4582024-09-27T23:07:05.714Z{"msg":"extent:4 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858858614Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4592024-09-27T23:07:05.714Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858873453Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4602024-09-27T23:07:05.714Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858887892Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4612024-09-27T23:07:05.714Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858903101Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4622024-09-27T23:07:05.714Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.85891836Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"0","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4632024-09-27T23:07:05.714Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858939049Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4642024-09-27T23:07:05.714Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858953148Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4652024-09-27T23:07:05.714Z{"msg":"starting reconciliation fd18a7b6-9f8d-48f3-b2f8-884a90650e9c: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858974067Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4662024-09-27T23:07:05.714Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.858993575Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4672024-09-27T23:07:05.714Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1533, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1533, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1533, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1533, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1533, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.859083919Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4682024-09-27T23:07:05.714Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.859108007Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4692024-09-27T23:07:05.714Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.859124266Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4702024-09-27T23:07:05.718Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.859501891Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4712024-09-27T23:07:05.718Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.8598084Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4722024-09-27T23:07:05.718Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.859867606Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4732024-09-27T23:07:05.718Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.859913643Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4742024-09-27T23:07:05.722Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.872461282Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4752024-09-27T23:07:05.725Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.872902272Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4762024-09-27T23:07:05.725Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.87411568Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4772024-09-27T23:07:05.726Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.87440743Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4782024-09-27T23:07:05.726Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.874451837Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4792024-09-27T23:07:05.726Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.874474796Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4802024-09-27T23:07:05.733Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.882767863Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4812024-09-27T23:07:05.736Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.883207564Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4822024-09-27T23:07:05.736Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.884390864Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4832024-09-27T23:07:05.736Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.884688653Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4842024-09-27T23:07:05.736Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.88474174Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4852024-09-27T23:07:05.736Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.884783617Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4862024-09-27T23:07:05.743Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.893701262Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4872024-09-27T23:07:05.749Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.89417432Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4882024-09-27T23:07:05.749Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.895281115Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4892024-09-27T23:07:05.749Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.895565506Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4902024-09-27T23:07:05.749Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.895609743Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4912024-09-27T23:07:05.749Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.89565663Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4922024-09-27T23:07:05.754Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.904067959Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4932024-09-27T23:07:05.759Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.904437325Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4942024-09-27T23:07:05.759Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.90553574Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4952024-09-27T23:07:05.759Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.90583181Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4962024-09-27T23:07:05.759Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.905875507Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4972024-09-27T23:07:05.759Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.905912554Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4982024-09-27T23:07:05.766Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.914596026Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
4992024-09-27T23:07:05.771Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.914924483Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
5002024-09-27T23:07:05.771Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.914963041Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"0","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
5012024-09-27T23:07:05.771Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.914983799Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"1","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
5022024-09-27T23:07:05.771Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.915014048Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"client":"2","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
5032024-09-27T23:07:05.771Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.915044585Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
5042024-09-27T23:07:05.771Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.915061884Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
5052024-09-27T23:07:05.771ZWait for a query_work_queue command to finish before sending IO
5062024-09-27T23:07:05.771Z{"msg":"2caab520-18eb-4a6c-afdf-858339081238 is now active with session: 9b694fd4-c7c9-4d22-98af-61ab3819a441","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.915097832Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
5072024-09-27T23:07:05.771Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:04.915138669Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231}
5082024-09-27T23:07:05.772Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:07:04.915167727Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231,"reconcile":"fd18a7b6-9f8d-48f3-b2f8-884a90650e9c","":"downstairs","":"downstairs","session_id":"9b694fd4-c7c9-4d22-98af-61ab3819a441"}
5092024-09-27T23:07:05.772ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
5102024-09-27T23:07:05.772ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
5112024-09-27T23:07:05.772ZRead and Verify all blocks (0..50 range:true)
5122024-09-27T23:07:05.775ZWrote out file "/var/tmp/test_up-build/verify_file"
5132024-09-27T23:07:05.775ZVerify test completed
5142024-09-27T23:07:05.775ZWrote out file "/var/tmp/test_up-build/verify_file"
5152024-09-27T23:07:05.775ZCLIENT: Tests done. All submitted work has been ACK'd
5162024-09-27T23:07:05.775Z----------------------------------------------------------------
5172024-09-27T23:07:05.775Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5182024-09-27T23:07:05.775ZStates: Active Active Active
5192024-09-27T23:07:05.775ZGW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5202024-09-27T23:07:05.775Z 1 Acked 1000 Read 50 Done Done Done false
5212024-09-27T23:07:05.775Z STATES DS:0 DS:1 DS:2 TOTAL
5222024-09-27T23:07:05.775Z Sent 0 0 0 0
5232024-09-27T23:07:05.775Z Done 1 1 1 3
5242024-09-27T23:07:05.775Z Skipped 0 0 0 0
5252024-09-27T23:07:05.775Z Error 0 0 0 0
5262024-09-27T23:07:05.776ZLast Flush: 0 0 0
5272024-09-27T23:07:05.776ZDownstairs last five completed:
5282024-09-27T23:07:05.776ZUpstairs last five completed: 1
5292024-09-27T23:07:05.776ZCLIENT: Up:0 ds:1 act:3
5302024-09-27T23:07:09.773Z----------------------------------------------------------------
5312024-09-27T23:07:09.773Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5322024-09-27T23:07:09.773ZDownstairs last five completed: 1001 1000
5332024-09-27T23:07:09.773ZUpstairs last five completed: 2 1
5342024-09-27T23:07:09.773ZCLIENT: Up:0 ds:0 act:3
5352024-09-27T23:07:09.773ZCLIENT: All crucible jobs finished, exiting program
5362024-09-27T23:07:09.787Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T23:07:08.923445042Z","hostname":"w-01J8TVJW4S3S4T5CS22D0A62BC","pid":1231}
5372024-09-27T23:07:09.790ZRepair part 2 passed
5382024-09-27T23:07:09.790Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5392024-09-27T23:07:09.814ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5402024-09-27T23:07:09.815Z 0 000-009 32 32 32 1533 1533 1533 F F F
5412024-09-27T23:07:09.815Z 1 010-019 32 32 32 1533 1533 1533 F F F
5422024-09-27T23:07:09.815Z 2 020-029 32 32 32 1533 1533 1533 F F F
5432024-09-27T23:07:09.815Z 3 030-039 32 32 32 1533 1533 1533 F F F
5442024-09-27T23:07:09.815Z 4 040-049 32 32 32 1533 1533 1533 F F F
5452024-09-27T23:07:09.815ZMax gen: 32, Max flush: 1533
5462024-09-27T23:07:09.815Zdump test passed
5472024-09-27T23:07:09.815Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5482024-09-27T23:07:09.818Z Extent 1
5492024-09-27T23:07:09.819ZGEN 32 32 32
5502024-09-27T23:07:09.819ZFLUSH_ID 1533 1533 1533
5512024-09-27T23:07:09.819ZDIRTY
5522024-09-27T23:07:09.819Z
5532024-09-27T23:07:09.819ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5542024-09-27T23:07:09.819Z010 A A A A A A
5552024-09-27T23:07:09.819Z011 A A A A A A
5562024-09-27T23:07:09.822Z012 A A A A A A
5572024-09-27T23:07:09.822Z013 A A A A A A
5582024-09-27T23:07:09.822Z014 A A A A A A
5592024-09-27T23:07:09.829Z015 A A A A A A
5602024-09-27T23:07:09.829Z016 A A A A A A
5612024-09-27T23:07:09.841Z017 A A A A A A
5622024-09-27T23:07:09.842Z018 A A A A A A
5632024-09-27T23:07:09.842Z019 A A A A A A
5642024-09-27T23:07:09.842Zdump extent test passed
5652024-09-27T23:07:09.842Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5662024-09-27T23:07:09.849ZExtent 2 Block in extent 0 Actual block 20
5672024-09-27T23:07:09.849Z
5682024-09-27T23:07:09.849Z DATA SHA256 VER
5692024-09-27T23:07:09.849Z------ ---------------------------------------------------------------- ---
5702024-09-27T23:07:09.849Z 0 7c06d6ec4d4a970d4585f526f5400d8405c63d6f6b1424682e1fab6d3a776079 A
5712024-09-27T23:07:09.856Z 1 7c06d6ec4d4a970d4585f526f5400d8405c63d6f6b1424682e1fab6d3a776079 A
5722024-09-27T23:07:09.856Z 2 7c06d6ec4d4a970d4585f526f5400d8405c63d6f6b1424682e1fab6d3a776079 A
5732024-09-27T23:07:09.856Z
5742024-09-27T23:07:09.856ZNONCES 0 1 2 DIFF
5752024-09-27T23:07:09.856Z------ ------------------------ ------------------------ ------------------------ -----
5762024-09-27T23:07:09.856Z
5772024-09-27T23:07:09.856Z TAGS 0 1 2 DIFF
5782024-09-27T23:07:09.856Z------ -------------------------------- -------------------------------- -------------------------------- -----
5792024-09-27T23:07:09.856Z
5802024-09-27T23:07:09.856ZHASHES 0 1 2 DIFF
5812024-09-27T23:07:09.856Z------ ---------------- ---------------- ---------------- -----
5822024-09-27T23:07:09.856Z 0 a0d586716ee487f5 a0d586716ee487f5 a0d586716ee487f5
5832024-09-27T23:07:09.856Z
5842024-09-27T23:07:09.856Zdump block test passed
5852024-09-27T23:07:09.856ZUpstairs tests have completed, stopping all downstairs
5862024-09-27T23:07:17.854ZCreating four larger downstairs regions
5872024-09-27T23:07:17.932ZStarting four downstairs
5882024-09-27T23:07:20.362ZNow do the replace-reconcile test
5892024-09-27T23:10:03.871ZCompleted test: replace-reconcile
5902024-09-27T23:10:03.874ZNow do the replace-before-active test
5912024-09-27T23:12:45.468ZCompleted test: replace-before-active
5922024-09-27T23:12:45.471ZTests have completed, stopping all downstairs
5932024-09-27T23:12:45.526Z
5942024-09-27T23:12:45.526ZAll Tests have passed
5952024-09-27T23:12:45.529Z7:58 Test duration
5962024-09-27T23:12:45.529Z
5972024-09-27T23:12:45.529Zreal 7:58.157741122
5982024-09-27T23:12:45.529Zuser 1:06.618163923
5992024-09-27T23:12:45.529Zsys 11.687283562
6002024-09-27T23:12:45.529Ztrap 0.174218130
6012024-09-27T23:12:45.529Ztflt 0.015316077
6022024-09-27T23:12:45.529Zdflt 0.046496965
6032024-09-27T23:12:45.529Zkflt 0.000867279
6042024-09-27T23:12:45.529Zlock 5:49:58.467258770
6052024-09-27T23:12:45.529Zslp 25:04.067105301
6062024-09-27T23:12:45.529Zlat 10.652152263
6072024-09-27T23:12:45.529Zstop 0.127312818
6082024-09-27T23:12:45.529Z+ echo 'test-up-unencrypted ends'
6092024-09-27T23:12:45.529Ztest-up-unencrypted ends
6102024-09-27T23:12:50.532Zprocess exited: duration 486467 ms, exit code 0
6112024-09-27T23:12:50.532Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6122024-09-27T23:13:50.584Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6132024-09-27T23:13:50.584Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6142024-09-27T23:13:50.595Zfound 14 output files
6152024-09-27T23:13:50.595Zuploading: /tmp/test_up-build/dsc-out.txt (35717 bytes)
6162024-09-27T23:13:51.610Zuploaded: /tmp/test_up-build/dsc-out.txt
6172024-09-27T23:13:51.610Zuploading: /tmp/test_up-build/test_up_out.txt (10475542 bytes)
6182024-09-27T23:13:52.781Zuploaded: /tmp/test_up-build/test_up_out.txt
6192024-09-27T23:13:52.784Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (253694 bytes)
6202024-09-27T23:13:53.804Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6212024-09-27T23:13:53.805Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144633 bytes)
6222024-09-27T23:13:54.824Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6232024-09-27T23:13:54.824Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251066 bytes)
6242024-09-27T23:13:55.845Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6252024-09-27T23:13:55.845Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139420 bytes)
6262024-09-27T23:13:56.862Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6272024-09-27T23:13:56.862Zuploading: /tmp/debug/df.txt (1270 bytes)
6282024-09-27T23:13:57.869Zuploaded: /tmp/debug/df.txt
6292024-09-27T23:13:57.869Zuploading: /tmp/debug/dtrace.txt (514757 bytes)
6302024-09-27T23:13:58.899Zuploaded: /tmp/debug/dtrace.txt
6312024-09-27T23:13:58.899Zuploading: /tmp/debug/iostat.txt (164379 bytes)
6322024-09-27T23:13:58.913Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 164379 -> 166795
6332024-09-27T23:13:59.928Zuploaded: /tmp/debug/iostat.txt
6342024-09-27T23:13:59.928Zuploading: /tmp/debug/mpstat.txt (397665 bytes)
6352024-09-27T23:13:59.949Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 397665 -> 404244
6362024-09-27T23:14:00.957Zuploaded: /tmp/debug/mpstat.txt
6372024-09-27T23:14:00.958Zuploading: /tmp/debug/paging.txt (72338 bytes)
6382024-09-27T23:14:00.966Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 72338 -> 73666
6392024-09-27T23:14:01.972Zuploaded: /tmp/debug/paging.txt
6402024-09-27T23:14:01.973Zuploading: /tmp/debug/prstat.txt (749400 bytes)
6412024-09-27T23:14:02.004Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 749400 -> 763645
6422024-09-27T23:14:03.011Zuploaded: /tmp/debug/prstat.txt
6432024-09-27T23:14:03.014Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6442024-09-27T23:14:04.022Zuploaded: /tmp/debug/psrinfo.txt
6452024-09-27T23:14:04.022Zuploading: /tmp/debug/upstairs-info.txt (73786 bytes)
6462024-09-27T23:14:05.037Zuploaded: /tmp/debug/upstairs-info.txt