01KP3RYPCSMDDAZTR0TEH10N63: test-live-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/*.log",
        "%/tmp/debug/*",
        "/tmp/core.*",
        "/tmp/dsc/*.txt",
        "/tmp/dsc.tar",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KP3RZ5H7EF0QX7GGEYQBFB5Z

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-04-13T16:09:08.932Zjob dependencies complete; ready to run (waiting for 12 m 1 s)
22026-04-13T16:09:55.061Zjob assigned to worker 01KP3SN6ZGSM68FXXAFK932CAS [factory aws, i-047032a243fdbd6d8] (queued for 46 s)
32026-04-13T16:10:02.098Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-04-13T16:10:03.713Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-04-13T16:10:03.717Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-04-13T16:10:05.392Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-04-13T16:10:05.395Zdownloading input: /input/build/work/bins/crutest.gz
82026-04-13T16:10:06.997Zdownloaded input: /input/build/work/bins/crutest.gz
92026-04-13T16:10:06.997Zdownloading input: /input/build/work/bins/dsc.gz
102026-04-13T16:10:08.115Zdownloaded input: /input/build/work/bins/dsc.gz
112026-04-13T16:10:08.119Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-04-13T16:10:08.145Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-04-13T16:10:08.148Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-04-13T16:10:08.188Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-04-13T16:10:08.192Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-04-13T16:10:08.216Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-04-13T16:10:08.216Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-04-13T16:10:08.244Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-04-13T16:10:08.245Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-04-13T16:10:08.277Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-04-13T16:10:08.277Zdownloading input: /input/build/work/scripts/test_up.sh
222026-04-13T16:10:08.303Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-04-13T16:10:08.303Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-04-13T16:10:08.348Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-04-13T16:10:08.353Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-04-13T16:10:08.916Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-04-13T16:10:08.916Zstarting task 0: "setup"
282026-04-13T16:10:08.922Z++ uname -s
292026-04-13T16:10:08.926Z+ kern=SunOS
302026-04-13T16:10:08.926Z+ build_user=build
312026-04-13T16:10:08.926Z+ build_uid=12345
322026-04-13T16:10:08.926Z+ work_dir=/work
332026-04-13T16:10:08.926Z+ input_dir=/input
342026-04-13T16:10:08.926Z+ [[ 0 == 12345 ]]
352026-04-13T16:10:08.926Z+ case "$kern" in
362026-04-13T16:10:08.926Z+ groupadd -g 12345 build
372026-04-13T16:10:08.926Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-04-13T16:10:10.932Z+ zfs create -o mountpoint=/work rpool/work
392026-04-13T16:10:11.341Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-04-13T16:10:11.347Z+ home_fs=zfs
412026-04-13T16:10:11.347Z+ [[ zfs == autofs ]]
422026-04-13T16:10:11.347Z+ mkdir -p /home/build
432026-04-13T16:10:11.347Z+ chown build:build /home/build /work
442026-04-13T16:10:12.346Z+ chmod 0700 /home/build /work
452026-04-13T16:10:12.350Zprocess exited: duration 3429 ms, exit code 0
 
462026-04-13T16:10:12.362Zstarting task 1: "authentication"
472026-04-13T16:10:12.386Zprocess exited: duration 18 ms, exit code 0
 
482026-04-13T16:10:12.397Zstarting task 2: "build"
492026-04-13T16:10:12.400Z+ banner cores
502026-04-13T16:10:12.408Z
512026-04-13T16:10:12.408Z #### #### ##### ###### ####
522026-04-13T16:10:12.408Z # # # # # # # #
532026-04-13T16:10:12.408Z # # # # # ##### ####
542026-04-13T16:10:12.408Z # # # ##### # #
552026-04-13T16:10:12.408Z # # # # # # # # #
562026-04-13T16:10:12.408Z #### #### # # ###### ####
572026-04-13T16:10:12.408Z
582026-04-13T16:10:12.408Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-04-13T16:10:12.412Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
602026-04-13T16:10:12.412Z+ echo 'input bins dir contains:'
612026-04-13T16:10:12.412Zinput bins dir contains:
622026-04-13T16:10:12.412Z+ ls -ltr /input/build/work/bins
632026-04-13T16:10:12.415Ztotal 1036444
642026-04-13T16:10:12.415Z-rw-r--r-- 1 root root 157301608 Apr 13 16:10 crucible-downstairs.gz
652026-04-13T16:10:12.415Z-rw-r--r-- 1 root root 145686170 Apr 13 16:10 crucible-hammer.gz
662026-04-13T16:10:12.415Z-rw-r--r-- 1 root root 154840432 Apr 13 16:10 crutest.gz
672026-04-13T16:10:12.415Z-rw-r--r-- 1 root root 72474450 Apr 13 16:10 dsc.gz
682026-04-13T16:10:12.415Zinput script dir contains:
692026-04-13T16:10:12.415Z+ echo 'input script dir contains:'
702026-04-13T16:10:12.415Z+ ls -ltr /input/build/work/scripts
712026-04-13T16:10:12.415Ztotal 46
722026-04-13T16:10:12.415Z-rwxr-xr-x 1 root root 3056 Apr 13 16:10 perf-downstairs-tick.d
732026-04-13T16:10:12.416Z-rwxr-xr-x 1 root root 2127 Apr 13 16:10 test_ds.sh
742026-04-13T16:10:12.416Z-rwxr-xr-x 1 root root 4923 Apr 13 16:10 test_live_repair.sh
752026-04-13T16:10:12.416Z-rwxr-xr-x 1 root root 7419 Apr 13 16:10 test_repair.sh
762026-04-13T16:10:12.416Z-rwxr-xr-x 1 root root 4282 Apr 13 16:10 test_replay.sh
772026-04-13T16:10:12.416Z-rwxr-xr-x 1 root root 13887 Apr 13 16:10 test_up.sh
782026-04-13T16:10:12.416Z-rwxr-xr-x 1 root root 3241 Apr 13 16:10 upstairs_info.d
792026-04-13T16:10:12.416Z+ banner Unpack
802026-04-13T16:10:12.421Z# #
812026-04-13T16:10:12.421Z# # # # ##### ## #### # #
822026-04-13T16:10:12.421Z# # ## # # # # # # # # #
832026-04-13T16:10:12.422Z# # # # # # # # # # ####
842026-04-13T16:10:12.422Z# # # # # ##### ###### # # #
852026-04-13T16:10:12.422Z# # # ## # # # # # # #
862026-04-13T16:10:12.422Z ##### # # # # # #### # #
872026-04-13T16:10:12.422Z
882026-04-13T16:10:12.422Z+ mkdir -p /var/tmp/bins
892026-04-13T16:10:12.422Z+ for t in "$input/bins/"*.gz
902026-04-13T16:10:12.422Z++ basename /input/build/work/bins/crucible-downstairs.gz
912026-04-13T16:10:12.422Z+ b=crucible-downstairs.gz
922026-04-13T16:10:12.422Z+ b=crucible-downstairs
932026-04-13T16:10:12.422Z+ gunzip
942026-04-13T16:10:15.030Z+ chmod +x /var/tmp/bins/crucible-downstairs
952026-04-13T16:10:15.035Z+ for t in "$input/bins/"*.gz
962026-04-13T16:10:15.035Z++ basename /input/build/work/bins/crucible-hammer.gz
972026-04-13T16:10:15.035Z+ b=crucible-hammer.gz
982026-04-13T16:10:15.035Z+ b=crucible-hammer
992026-04-13T16:10:15.035Z+ gunzip
1002026-04-13T16:10:17.410Z+ chmod +x /var/tmp/bins/crucible-hammer
1012026-04-13T16:10:17.414Z+ for t in "$input/bins/"*.gz
1022026-04-13T16:10:17.414Z++ basename /input/build/work/bins/crutest.gz
1032026-04-13T16:10:17.417Z+ b=crutest.gz
1042026-04-13T16:10:17.417Z+ b=crutest
1052026-04-13T16:10:17.417Z+ gunzip
1062026-04-13T16:10:19.983Z+ chmod +x /var/tmp/bins/crutest
1072026-04-13T16:10:19.986Z+ for t in "$input/bins/"*.gz
1082026-04-13T16:10:19.986Z++ basename /input/build/work/bins/dsc.gz
1092026-04-13T16:10:19.986Z+ b=dsc.gz
1102026-04-13T16:10:19.986Z+ b=dsc
1112026-04-13T16:10:19.986Z+ gunzip
1122026-04-13T16:10:21.174Z+ chmod +x /var/tmp/bins/dsc
1132026-04-13T16:10:21.178Z+ export BINDIR=/var/tmp/bins
1142026-04-13T16:10:21.178Z+ BINDIR=/var/tmp/bins
1152026-04-13T16:10:21.178Z+ export RUST_BACKTRACE=1
1162026-04-13T16:10:21.178Z+ RUST_BACKTRACE=1
1172026-04-13T16:10:21.178Z+ echo 'BINDIR is /var/tmp/bins'
1182026-04-13T16:10:21.178ZBINDIR is /var/tmp/bins
1192026-04-13T16:10:21.178Z+ echo 'bindir contains:'
1202026-04-13T16:10:21.178Zbindir contains:
1212026-04-13T16:10:21.178Z+ ls -ltr /var/tmp/bins
1222026-04-13T16:10:21.178Ztotal 1109417
1232026-04-13T16:10:21.178Z-rwxr-xr-x 1 build build 578265752 Apr 13 16:10 crucible-downstairs
1242026-04-13T16:10:21.178Z-rwxr-xr-x 1 build build 534361384 Apr 13 16:10 crucible-hammer
1252026-04-13T16:10:21.178Z-rwxr-xr-x 1 build build 569490000 Apr 13 16:10 crutest
1262026-04-13T16:10:21.178Z-rwxr-xr-x 1 build build 255478144 Apr 13 16:10 dsc
1272026-04-13T16:10:21.178Z+ banner CreateDS
1282026-04-13T16:10:21.181Z ##### ###### #####
1292026-04-13T16:10:21.181Z# # ##### ###### ## ##### ###### # # # #
1302026-04-13T16:10:21.181Z# # # # # # # # # # #
1312026-04-13T16:10:21.181Z# # # ##### # # # ##### # # #####
1322026-04-13T16:10:21.181Z# ##### # ###### # # # # #
1332026-04-13T16:10:21.182Z# # # # # # # # # # # # #
1342026-04-13T16:10:21.182Z ##### # # ###### # # # ###### ###### #####
1352026-04-13T16:10:21.182Z
1362026-04-13T16:10:21.182Z+ echo /var/tmp/bins/dsc create --ds-bin /var/tmp/bins/crucible-downstairs --extent-size 4000 --extent-count 200 --region-count 4 --cleanup
1372026-04-13T16:10:21.182Z/var/tmp/bins/dsc create --ds-bin /var/tmp/bins/crucible-downstairs --extent-size 4000 --extent-count 200 --region-count 4 --cleanup
1382026-04-13T16:10:21.182Z+ /var/tmp/bins/dsc create --ds-bin /var/tmp/bins/crucible-downstairs --extent-size 4000 --extent-count 200 --region-count 4 --cleanup
1392026-04-13T16:10:21.186ZCreating region directory at: /var/tmp/dsc/region
1402026-04-13T16:10:21.189ZCreating dsc directory at: /tmp/dsc
1412026-04-13T16:10:21.319ZDownstairs region 0 created at /var/tmp/dsc/region/8810 in 0.13244234
1422026-04-13T16:10:21.382ZDownstairs region 1 created at /var/tmp/dsc/region/8820 in 0.0625445
1432026-04-13T16:10:21.444ZDownstairs region 2 created at /var/tmp/dsc/region/8830 in 0.062291875
1442026-04-13T16:10:21.506ZDownstairs region 3 created at /var/tmp/dsc/region/8840 in 0.061455313
1452026-04-13T16:10:21.506ZCreated 4 regions
1462026-04-13T16:10:21.509Z+ banner StartDS
1472026-04-13T16:10:21.512Z ##### ###### #####
1482026-04-13T16:10:21.512Z# # ##### ## ##### ##### # # # #
1492026-04-13T16:10:21.512Z# # # # # # # # # #
1502026-04-13T16:10:21.512Z ##### # # # # # # # # #####
1512026-04-13T16:10:21.512Z # # ###### ##### # # # #
1522026-04-13T16:10:21.512Z# # # # # # # # # # # #
1532026-04-13T16:10:21.512Z ##### # # # # # # ###### #####
1542026-04-13T16:10:21.512Z
1552026-04-13T16:10:21.512Z+ sleep 5
1562026-04-13T16:10:21.512Z+ /var/tmp/bins/dsc start --ds-bin /var/tmp/bins/crucible-downstairs --region-count 4
1572026-04-13T16:10:26.516Z++ pgrep dsc
1582026-04-13T16:10:26.520Z+ dsc_pid=1119
1592026-04-13T16:10:26.520Z+ [[ 1119 -eq 0 ]]
1602026-04-13T16:10:26.520Z+ ps -p 1119
1612026-04-13T16:10:26.524Z PID TTY TIME CMD
1622026-04-13T16:10:26.524Z 1119 ? 0:00 dsc
1632026-04-13T16:10:26.524Z+ echo 'Found dsc running, continue tests'
1642026-04-13T16:10:26.524ZFound dsc running, continue tests
1652026-04-13T16:10:26.524Z+ echo 'Setup self timeout'
1662026-04-13T16:10:26.524ZSetup self timeout
1672026-04-13T16:10:26.524Z+ jobpid=1090
1682026-04-13T16:10:26.524Z+ echo 'Setup debug logging'
1692026-04-13T16:10:26.524ZSetup debug logging
1702026-04-13T16:10:26.524Z+ mkdir /tmp/debug
1712026-04-13T16:10:26.524Z+ sleep 7200
1722026-04-13T16:10:26.524Z+ psrinfo -v
1732026-04-13T16:10:26.528Z+ df -h
1742026-04-13T16:10:26.700Z+ prstat -d d -mLc 1
1752026-04-13T16:10:27.106Z+ iostat -T d -xn 1
1762026-04-13T16:10:27.179Z+ mpstat -T d 1
1772026-04-13T16:10:27.212Z+ vmstat -T d -p 1
1782026-04-13T16:10:27.268Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1792026-04-13T16:10:27.385Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1802026-04-13T16:10:27.530Z+ banner LR
1812026-04-13T16:10:27.534Z# ######
1822026-04-13T16:10:27.534Z# # #
1832026-04-13T16:10:27.534Z# # #
1842026-04-13T16:10:27.534Z# ######
1852026-04-13T16:10:27.534Z# # #
1862026-04-13T16:10:27.534Z# # #
1872026-04-13T16:10:27.534Z####### # #
1882026-04-13T16:10:27.534Z
1892026-04-13T16:10:27.707Z+ ptime -m /var/tmp/bins/crutest replace -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --replacement 127.0.0.1:8840 -g 1 -c 10 --stable
1902026-04-13T16:10:28.732Z+ _exit_trap
1912026-04-13T16:10:28.732Z+ local status=1
1922026-04-13T16:10:28.732Z+ [[ 1 -eq 0 ]]
1932026-04-13T16:10:28.732Z+ set +o errexit
1942026-04-13T16:10:28.735Z+ set -o xtrace
1952026-04-13T16:10:28.735Z+ sleep 5
1962026-04-13T16:10:33.734Z+ banner evidence
1972026-04-13T16:10:33.738Z
1982026-04-13T16:10:33.738Z ###### # # # ##### ###### # # #### ######
1992026-04-13T16:10:33.738Z # # # # # # # ## # # # #
2002026-04-13T16:10:33.738Z ##### # # # # # ##### # # # # #####
2012026-04-13T16:10:33.738Z # # # # # # # # # # # #
2022026-04-13T16:10:33.738Z # # # # # # # # ## # # #
2032026-04-13T16:10:33.738Z ###### ## # ##### ###### # # #### ######
2042026-04-13T16:10:33.738Z
2052026-04-13T16:10:33.738Z++ ls '/tmp/core*'
2062026-04-13T16:10:33.741Z/tmp/core*: No such file or directory
2072026-04-13T16:10:33.741Z+ CORES=
2082026-04-13T16:10:33.741Z+ tar cf /tmp/dsc.tar /var/tmp/dsc/region
2092026-04-13T16:10:41.018Z+ echo 'Final region compare'
2102026-04-13T16:10:41.018ZFinal region compare
2112026-04-13T16:10:41.018Z+ /var/tmp/bins/crucible-downstairs dump -d /var/tmp/dsc/region/8810 -d /var/tmp/dsc/region/8820 -d /var/tmp/dsc/region/8830
2122026-04-13T16:10:41.063Z{"msg":"validated 200 extents on startup","v":0,"name":"crucible","level":30,"time":"2026-04-13T16:10:39.756206421Z","hostname":"w-01KP3SN6ZGSM68FXXAFK932CAS","pid":1146}
2132026-04-13T16:10:41.093Z{"msg":"validated 200 extents on startup","v":0,"name":"crucible","level":30,"time":"2026-04-13T16:10:39.787312619Z","hostname":"w-01KP3SN6ZGSM68FXXAFK932CAS","pid":1146}
2142026-04-13T16:10:41.124Z{"msg":"validated 200 extents on startup","v":0,"name":"crucible","level":30,"time":"2026-04-13T16:10:39.818180923Z","hostname":"w-01KP3SN6ZGSM68FXXAFK932CAS","pid":1146}
2152026-04-13T16:10:41.129ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2
2162026-04-13T16:10:41.129Z 0 000000-003999 0 0 0 0 0 0 F F F
2172026-04-13T16:10:41.129Z 1 004000-007999 0 0 0 0 0 0 F F F
2182026-04-13T16:10:41.129Z 2 008000-011999 0 0 0 0 0 0 F F F
2192026-04-13T16:10:41.129Z 3 012000-015999 0 0 0 0 0 0 F F F
2202026-04-13T16:10:41.129Z 4 016000-019999 0 0 0 0 0 0 F F F
2212026-04-13T16:10:41.129Z 5 020000-023999 0 0 0 0 0 0 F F F
2222026-04-13T16:10:41.129Z 6 024000-027999 0 0 0 0 0 0 F F F
2232026-04-13T16:10:41.129Z 7 028000-031999 0 0 0 0 0 0 F F F
2242026-04-13T16:10:41.129Z 8 032000-035999 0 0 0 0 0 0 F F F
2252026-04-13T16:10:41.129Z 9 036000-039999 0 0 0 0 0 0 F F F
2262026-04-13T16:10:41.129Z 10 040000-043999 0 0 0 0 0 0 F F F
2272026-04-13T16:10:41.129Z 11 044000-047999 0 0 0 0 0 0 F F F
2282026-04-13T16:10:41.129Z 12 048000-051999 0 0 0 0 0 0 F F F
2292026-04-13T16:10:41.129Z 13 052000-055999 0 0 0 0 0 0 F F F
2302026-04-13T16:10:41.129Z 14 056000-059999 0 0 0 0 0 0 F F F
2312026-04-13T16:10:41.129Z 15 060000-063999 0 0 0 0 0 0 F F F
2322026-04-13T16:10:41.129Z 16 064000-067999 0 0 0 0 0 0 F F F
2332026-04-13T16:10:41.129Z 17 068000-071999 0 0 0 0 0 0 F F F
2342026-04-13T16:10:41.129Z 18 072000-075999 0 0 0 0 0 0 F F F
2352026-04-13T16:10:41.129Z 19 076000-079999 0 0 0 0 0 0 F F F
2362026-04-13T16:10:41.129Z 20 080000-083999 0 0 0 0 0 0 F F F
2372026-04-13T16:10:41.129Z 21 084000-087999 0 0 0 0 0 0 F F F
2382026-04-13T16:10:41.129Z 22 088000-091999 0 0 0 0 0 0 F F F
2392026-04-13T16:10:41.129Z 23 092000-095999 0 0 0 0 0 0 F F F
2402026-04-13T16:10:41.129Z 24 096000-099999 0 0 0 0 0 0 F F F
2412026-04-13T16:10:41.129Z 25 100000-103999 0 0 0 0 0 0 F F F
2422026-04-13T16:10:41.129Z 26 104000-107999 0 0 0 0 0 0 F F F
2432026-04-13T16:10:41.129Z 27 108000-111999 0 0 0 0 0 0 F F F
2442026-04-13T16:10:41.129Z 28 112000-115999 0 0 0 0 0 0 F F F
2452026-04-13T16:10:41.130Z 29 116000-119999 0 0 0 0 0 0 F F F
2462026-04-13T16:10:41.130Z 30 120000-123999 0 0 0 0 0 0 F F F
2472026-04-13T16:10:41.130Z 31 124000-127999 0 0 0 0 0 0 F F F
2482026-04-13T16:10:41.130Z 32 128000-131999 0 0 0 0 0 0 F F F
2492026-04-13T16:10:41.130Z 33 132000-135999 0 0 0 0 0 0 F F F
2502026-04-13T16:10:41.130Z 34 136000-139999 0 0 0 0 0 0 F F F
2512026-04-13T16:10:41.130Z 35 140000-143999 0 0 0 0 0 0 F F F
2522026-04-13T16:10:41.130Z 36 144000-147999 0 0 0 0 0 0 F F F
2532026-04-13T16:10:41.130Z 37 148000-151999 0 0 0 0 0 0 F F F
2542026-04-13T16:10:41.130Z 38 152000-155999 0 0 0 0 0 0 F F F
2552026-04-13T16:10:41.130Z 39 156000-159999 0 0 0 0 0 0 F F F
2562026-04-13T16:10:41.130Z 40 160000-163999 0 0 0 0 0 0 F F F
2572026-04-13T16:10:41.130Z 41 164000-167999 0 0 0 0 0 0 F F F
2582026-04-13T16:10:41.130Z 42 168000-171999 0 0 0 0 0 0 F F F
2592026-04-13T16:10:41.130Z 43 172000-175999 0 0 0 0 0 0 F F F
2602026-04-13T16:10:41.130Z 44 176000-179999 0 0 0 0 0 0 F F F
2612026-04-13T16:10:41.130Z 45 180000-183999 0 0 0 0 0 0 F F F
2622026-04-13T16:10:41.130Z 46 184000-187999 0 0 0 0 0 0 F F F
2632026-04-13T16:10:41.130Z 47 188000-191999 0 0 0 0 0 0 F F F
2642026-04-13T16:10:41.130Z 48 192000-195999 0 0 0 0 0 0 F F F
2652026-04-13T16:10:41.130Z 49 196000-199999 0 0 0 0 0 0 F F F
2662026-04-13T16:10:41.130Z 50 200000-203999 0 0 0 0 0 0 F F F
2672026-04-13T16:10:41.130Z 51 204000-207999 0 0 0 0 0 0 F F F
2682026-04-13T16:10:41.130Z 52 208000-211999 0 0 0 0 0 0 F F F
2692026-04-13T16:10:41.130Z 53 212000-215999 0 0 0 0 0 0 F F F
2702026-04-13T16:10:41.130Z 54 216000-219999 0 0 0 0 0 0 F F F
2712026-04-13T16:10:41.130Z 55 220000-223999 0 0 0 0 0 0 F F F
2722026-04-13T16:10:41.130Z 56 224000-227999 0 0 0 0 0 0 F F F
2732026-04-13T16:10:41.130Z 57 228000-231999 0 0 0 0 0 0 F F F
2742026-04-13T16:10:41.130Z 58 232000-235999 0 0 0 0 0 0 F F F
2752026-04-13T16:10:41.130Z 59 236000-239999 0 0 0 0 0 0 F F F
2762026-04-13T16:10:41.130Z 60 240000-243999 0 0 0 0 0 0 F F F
2772026-04-13T16:10:41.130Z 61 244000-247999 0 0 0 0 0 0 F F F
2782026-04-13T16:10:41.130Z 62 248000-251999 0 0 0 0 0 0 F F F
2792026-04-13T16:10:41.130Z 63 252000-255999 0 0 0 0 0 0 F F F
2802026-04-13T16:10:41.130Z 64 256000-259999 0 0 0 0 0 0 F F F
2812026-04-13T16:10:41.130Z 65 260000-263999 0 0 0 0 0 0 F F F
2822026-04-13T16:10:41.130Z 66 264000-267999 0 0 0 0 0 0 F F F
2832026-04-13T16:10:41.130Z 67 268000-271999 0 0 0 0 0 0 F F F
2842026-04-13T16:10:41.131Z 68 272000-275999 0 0 0 0 0 0 F F F
2852026-04-13T16:10:41.131Z 69 276000-279999 0 0 0 0 0 0 F F F
2862026-04-13T16:10:41.131Z 70 280000-283999 0 0 0 0 0 0 F F F
2872026-04-13T16:10:41.131Z 71 284000-287999 0 0 0 0 0 0 F F F
2882026-04-13T16:10:41.131Z 72 288000-291999 0 0 0 0 0 0 F F F
2892026-04-13T16:10:41.131Z 73 292000-295999 0 0 0 0 0 0 F F F
2902026-04-13T16:10:41.131Z 74 296000-299999 0 0 0 0 0 0 F F F
2912026-04-13T16:10:41.131Z 75 300000-303999 0 0 0 0 0 0 F F F
2922026-04-13T16:10:41.131Z 76 304000-307999 0 0 0 0 0 0 F F F
2932026-04-13T16:10:41.131Z 77 308000-311999 0 0 0 0 0 0 F F F
2942026-04-13T16:10:41.131Z 78 312000-315999 0 0 0 0 0 0 F F F
2952026-04-13T16:10:41.131Z 79 316000-319999 0 0 0 0 0 0 F F F
2962026-04-13T16:10:41.131Z 80 320000-323999 0 0 0 0 0 0 F F F
2972026-04-13T16:10:41.131Z 81 324000-327999 0 0 0 0 0 0 F F F
2982026-04-13T16:10:41.131Z 82 328000-331999 0 0 0 0 0 0 F F F
2992026-04-13T16:10:41.131Z 83 332000-335999 0 0 0 0 0 0 F F F
3002026-04-13T16:10:41.131Z 84 336000-339999 0 0 0 0 0 0 F F F
3012026-04-13T16:10:41.131Z 85 340000-343999 0 0 0 0 0 0 F F F
3022026-04-13T16:10:41.131Z 86 344000-347999 0 0 0 0 0 0 F F F
3032026-04-13T16:10:41.131Z 87 348000-351999 0 0 0 0 0 0 F F F
3042026-04-13T16:10:41.131Z 88 352000-355999 0 0 0 0 0 0 F F F
3052026-04-13T16:10:41.131Z 89 356000-359999 0 0 0 0 0 0 F F F
3062026-04-13T16:10:41.131Z 90 360000-363999 0 0 0 0 0 0 F F F
3072026-04-13T16:10:41.131Z 91 364000-367999 0 0 0 0 0 0 F F F
3082026-04-13T16:10:41.131Z 92 368000-371999 0 0 0 0 0 0 F F F
3092026-04-13T16:10:41.131Z 93 372000-375999 0 0 0 0 0 0 F F F
3102026-04-13T16:10:41.131Z 94 376000-379999 0 0 0 0 0 0 F F F
3112026-04-13T16:10:41.131Z 95 380000-383999 0 0 0 0 0 0 F F F
3122026-04-13T16:10:41.131Z 96 384000-387999 0 0 0 0 0 0 F F F
3132026-04-13T16:10:41.131Z 97 388000-391999 0 0 0 0 0 0 F F F
3142026-04-13T16:10:41.131Z 98 392000-395999 0 0 0 0 0 0 F F F
3152026-04-13T16:10:41.136Z 99 396000-399999 0 0 0 0 0 0 F F F
3162026-04-13T16:10:41.136Z100 400000-403999 0 0 0 0 0 0 F F F
3172026-04-13T16:10:41.137Z101 404000-407999 0 0 0 0 0 0 F F F
3182026-04-13T16:10:41.137Z102 408000-411999 0 0 0 0 0 0 F F F
3192026-04-13T16:10:41.137Z103 412000-415999 0 0 0 0 0 0 F F F
3202026-04-13T16:10:41.137Z104 416000-419999 0 0 0 0 0 0 F F F
3212026-04-13T16:10:41.137Z105 420000-423999 0 0 0 0 0 0 F F F
3222026-04-13T16:10:41.137Z106 424000-427999 0 0 0 0 0 0 F F F
3232026-04-13T16:10:41.137Z107 428000-431999 0 0 0 0 0 0 F F F
3242026-04-13T16:10:41.137Z108 432000-435999 0 0 0 0 0 0 F F F
3252026-04-13T16:10:41.137Z109 436000-439999 0 0 0 0 0 0 F F F
3262026-04-13T16:10:41.137Z110 440000-443999 0 0 0 0 0 0 F F F
3272026-04-13T16:10:41.137Z111 444000-447999 0 0 0 0 0 0 F F F
3282026-04-13T16:10:41.137Z112 448000-451999 0 0 0 0 0 0 F F F
3292026-04-13T16:10:41.137Z113 452000-455999 0 0 0 0 0 0 F F F
3302026-04-13T16:10:41.137Z114 456000-459999 0 0 0 0 0 0 F F F
3312026-04-13T16:10:41.137Z115 460000-463999 0 0 0 0 0 0 F F F
3322026-04-13T16:10:41.137Z116 464000-467999 0 0 0 0 0 0 F F F
3332026-04-13T16:10:41.137Z117 468000-471999 0 0 0 0 0 0 F F F
3342026-04-13T16:10:41.137Z118 472000-475999 0 0 0 0 0 0 F F F
3352026-04-13T16:10:41.137Z119 476000-479999 0 0 0 0 0 0 F F F
3362026-04-13T16:10:41.137Z120 480000-483999 0 0 0 0 0 0 F F F
3372026-04-13T16:10:41.137Z121 484000-487999 0 0 0 0 0 0 F F F
3382026-04-13T16:10:41.137Z122 488000-491999 0 0 0 0 0 0 F F F
3392026-04-13T16:10:41.137Z123 492000-495999 0 0 0 0 0 0 F F F
3402026-04-13T16:10:41.137Z124 496000-499999 0 0 0 0 0 0 F F F
3412026-04-13T16:10:41.137Z125 500000-503999 0 0 0 0 0 0 F F F
3422026-04-13T16:10:41.137Z126 504000-507999 0 0 0 0 0 0 F F F
3432026-04-13T16:10:41.137Z127 508000-511999 0 0 0 0 0 0 F F F
3442026-04-13T16:10:41.137Z128 512000-515999 0 0 0 0 0 0 F F F
3452026-04-13T16:10:41.137Z129 516000-519999 0 0 0 0 0 0 F F F
3462026-04-13T16:10:41.137Z130 520000-523999 0 0 0 0 0 0 F F F
3472026-04-13T16:10:41.137Z131 524000-527999 0 0 0 0 0 0 F F F
3482026-04-13T16:10:41.137Z132 528000-531999 0 0 0 0 0 0 F F F
3492026-04-13T16:10:41.137Z133 532000-535999 0 0 0 0 0 0 F F F
3502026-04-13T16:10:41.137Z134 536000-539999 0 0 0 0 0 0 F F F
3512026-04-13T16:10:41.137Z135 540000-543999 0 0 0 0 0 0 F F F
3522026-04-13T16:10:41.137Z136 544000-547999 0 0 0 0 0 0 F F F
3532026-04-13T16:10:41.137Z137 548000-551999 0 0 0 0 0 0 F F F
3542026-04-13T16:10:41.137Z138 552000-555999 0 0 0 0 0 0 F F F
3552026-04-13T16:10:41.137Z139 556000-559999 0 0 0 0 0 0 F F F
3562026-04-13T16:10:41.138Z140 560000-563999 0 0 0 0 0 0 F F F
3572026-04-13T16:10:41.138Z141 564000-567999 0 0 0 0 0 0 F F F
3582026-04-13T16:10:41.138Z142 568000-571999 0 0 0 0 0 0 F F F
3592026-04-13T16:10:41.138Z143 572000-575999 0 0 0 0 0 0 F F F
3602026-04-13T16:10:41.138Z144 576000-579999 0 0 0 0 0 0 F F F
3612026-04-13T16:10:41.138Z145 580000-583999 0 0 0 0 0 0 F F F
3622026-04-13T16:10:41.138Z146 584000-587999 0 0 0 0 0 0 F F F
3632026-04-13T16:10:41.138Z147 588000-591999 0 0 0 0 0 0 F F F
3642026-04-13T16:10:41.138Z148 592000-595999 0 0 0 0 0 0 F F F
3652026-04-13T16:10:41.138Z149 596000-599999 0 0 0 0 0 0 F F F
3662026-04-13T16:10:41.138Z150 600000-603999 0 0 0 0 0 0 F F F
3672026-04-13T16:10:41.138Z151 604000-607999 0 0 0 0 0 0 F F F
3682026-04-13T16:10:41.138Z152 608000-611999 0 0 0 0 0 0 F F F
3692026-04-13T16:10:41.138Z153 612000-615999 0 0 0 0 0 0 F F F
3702026-04-13T16:10:41.138Z154 616000-619999 0 0 0 0 0 0 F F F
3712026-04-13T16:10:41.138Z155 620000-623999 0 0 0 0 0 0 F F F
3722026-04-13T16:10:41.138Z156 624000-627999 0 0 0 0 0 0 F F F
3732026-04-13T16:10:41.138Z157 628000-631999 0 0 0 0 0 0 F F F
3742026-04-13T16:10:41.138Z158 632000-635999 0 0 0 0 0 0 F F F
3752026-04-13T16:10:41.138Z159 636000-639999 0 0 0 0 0 0 F F F
3762026-04-13T16:10:41.138Z160 640000-643999 0 0 0 0 0 0 F F F
3772026-04-13T16:10:41.138Z161 644000-647999 0 0 0 0 0 0 F F F
3782026-04-13T16:10:41.138Z162 648000-651999 0 0 0 0 0 0 F F F
3792026-04-13T16:10:41.138Z163 652000-655999 0 0 0 0 0 0 F F F
3802026-04-13T16:10:41.138Z164 656000-659999 0 0 0 0 0 0 F F F
3812026-04-13T16:10:41.138Z165 660000-663999 0 0 0 0 0 0 F F F
3822026-04-13T16:10:41.138Z166 664000-667999 0 0 0 0 0 0 F F F
3832026-04-13T16:10:41.138Z167 668000-671999 0 0 0 0 0 0 F F F
3842026-04-13T16:10:41.138Z168 672000-675999 0 0 0 0 0 0 F F F
3852026-04-13T16:10:41.138Z169 676000-679999 0 0 0 0 0 0 F F F
3862026-04-13T16:10:41.138Z170 680000-683999 0 0 0 0 0 0 F F F
3872026-04-13T16:10:41.138Z171 684000-687999 0 0 0 0 0 0 F F F
3882026-04-13T16:10:41.138Z172 688000-691999 0 0 0 0 0 0 F F F
3892026-04-13T16:10:41.138Z173 692000-695999 0 0 0 0 0 0 F F F
3902026-04-13T16:10:41.138Z174 696000-699999 0 0 0 0 0 0 F F F
3912026-04-13T16:10:41.138Z175 700000-703999 0 0 0 0 0 0 F F F
3922026-04-13T16:10:41.138Z176 704000-707999 0 0 0 0 0 0 F F F
3932026-04-13T16:10:41.138Z177 708000-711999 0 0 0 0 0 0 F F F
3942026-04-13T16:10:41.138Z178 712000-715999 0 0 0 0 0 0 F F F
3952026-04-13T16:10:41.139Z179 716000-719999 0 0 0 0 0 0 F F F
3962026-04-13T16:10:41.139Z180 720000-723999 0 0 0 0 0 0 F F F
3972026-04-13T16:10:41.139Z181 724000-727999 0 0 0 0 0 0 F F F
3982026-04-13T16:10:41.139Z182 728000-731999 0 0 0 0 0 0 F F F
3992026-04-13T16:10:41.139Z183 732000-735999 0 0 0 0 0 0 F F F
4002026-04-13T16:10:41.139Z184 736000-739999 0 0 0 0 0 0 F F F
4012026-04-13T16:10:41.139Z185 740000-743999 0 0 0 0 0 0 F F F
4022026-04-13T16:10:41.139Z186 744000-747999 0 0 0 0 0 0 F F F
4032026-04-13T16:10:41.139Z187 748000-751999 0 0 0 0 0 0 F F F
4042026-04-13T16:10:41.139Z188 752000-755999 0 0 0 0 0 0 F F F
4052026-04-13T16:10:41.139Z189 756000-759999 0 0 0 0 0 0 F F F
4062026-04-13T16:10:41.139Z190 760000-763999 0 0 0 0 0 0 F F F
4072026-04-13T16:10:41.139Z191 764000-767999 0 0 0 0 0 0 F F F
4082026-04-13T16:10:41.139Z192 768000-771999 0 0 0 0 0 0 F F F
4092026-04-13T16:10:41.139Z193 772000-775999 0 0 0 0 0 0 F F F
4102026-04-13T16:10:41.139Z194 776000-779999 0 0 0 0 0 0 F F F
4112026-04-13T16:10:41.139Z195 780000-783999 0 0 0 0 0 0 F F F
4122026-04-13T16:10:41.139Z196 784000-787999 0 0 0 0 0 0 F F F
4132026-04-13T16:10:41.139Z197 788000-791999 0 0 0 0 0 0 F F F
4142026-04-13T16:10:41.139Z198 792000-795999 0 0 0 0 0 0 F F F
4152026-04-13T16:10:41.143Z199 796000-799999 0 0 0 0 0 0 F F F
4162026-04-13T16:10:41.143ZMax gen: 0, Max flush: 0
4172026-04-13T16:10:41.143Z+ exit 1
4182026-04-13T16:10:46.138Zprocess exited: duration 28730 ms, exit code 1
4192026-04-13T16:10:46.138Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
4202026-04-13T16:11:46.192Zexec warning: : stdout descriptor may be held open by a background process; giving up!
4212026-04-13T16:11:46.192Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
4222026-04-13T16:11:46.204Zfound 15 output files
4232026-04-13T16:11:46.205Zuploading: /tmp/crutest-replace.log (2094 bytes)
4242026-04-13T16:11:47.216Zuploaded: /tmp/crutest-replace.log
4252026-04-13T16:11:47.219Zuploading: /tmp/dsc.log (1318 bytes)
4262026-04-13T16:11:48.227Zuploaded: /tmp/dsc.log
4272026-04-13T16:11:48.227Zuploading: /tmp/debug/df.txt (1270 bytes)
4282026-04-13T16:11:49.237Zuploaded: /tmp/debug/df.txt
4292026-04-13T16:11:49.238Zuploading: /tmp/debug/dtrace.txt (266 bytes)
4302026-04-13T16:11:50.250Zuploaded: /tmp/debug/dtrace.txt
4312026-04-13T16:11:50.250Zuploading: /tmp/debug/iostat.txt (23842 bytes)
4322026-04-13T16:11:50.256Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 23842 -> 25034
4332026-04-13T16:11:51.264Zuploaded: /tmp/debug/iostat.txt
4342026-04-13T16:11:51.264Zuploading: /tmp/debug/mpstat.txt (57442 bytes)
4352026-04-13T16:11:51.272Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 57442 -> 61804
4362026-04-13T16:11:52.280Zuploaded: /tmp/debug/mpstat.txt
4372026-04-13T16:11:52.280Zuploading: /tmp/debug/paging.txt (10191 bytes)
4382026-04-13T16:11:52.286Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 10191 -> 11140
4392026-04-13T16:11:53.294Zuploaded: /tmp/debug/paging.txt
4402026-04-13T16:11:53.294Zuploading: /tmp/debug/prstat.txt (104498 bytes)
4412026-04-13T16:11:53.304Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 104498 -> 113551
4422026-04-13T16:11:54.314Zuploaded: /tmp/debug/prstat.txt
4432026-04-13T16:11:54.317Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
4442026-04-13T16:11:55.326Zuploaded: /tmp/debug/psrinfo.txt
4452026-04-13T16:11:55.326Zuploading: /tmp/debug/upstairs-info.txt (119 bytes)
4462026-04-13T16:11:56.337Zuploaded: /tmp/debug/upstairs-info.txt
4472026-04-13T16:11:56.337Zuploading: /tmp/dsc/downstairs-8810.txt (2667 bytes)
4482026-04-13T16:11:57.347Zuploaded: /tmp/dsc/downstairs-8810.txt
4492026-04-13T16:11:57.347Zuploading: /tmp/dsc/downstairs-8820.txt (2666 bytes)
4502026-04-13T16:11:58.358Zuploaded: /tmp/dsc/downstairs-8820.txt
4512026-04-13T16:11:58.361Zuploading: /tmp/dsc/downstairs-8830.txt (2666 bytes)
4522026-04-13T16:11:59.370Zuploaded: /tmp/dsc/downstairs-8830.txt
4532026-04-13T16:11:59.370Zuploading: /tmp/dsc/downstairs-8840.txt (2667 bytes)
4542026-04-13T16:12:00.381Zuploaded: /tmp/dsc/downstairs-8840.txt
4552026-04-13T16:12:00.382Zuploading: /tmp/dsc.tar (13415640576 bytes)
4562026-04-13T16:12:00.385Zupload error: file "/tmp/dsc.tar" is 13415640576 bytes in size, which is larger than the maximum output size of 5368709120 bytes and cannot be uploaded at this time.