01KP3RYPD3K4Y1B8M2MJHK0Z4E: test-repair

BasicConfig {
    output_rules: [
        "%/tmp/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KP3RZ5NG3T5V17SFGXQ78ZS6

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-04-13T16:09:08.936Zjob dependencies complete; ready to run (waiting for 12 m 1 s)
22026-04-13T16:10:04.088Zjob assigned to worker 01KP3SNCSGYQWA9J5F92JAM31N [factory aws, i-025923e6cf60133ec] (queued for 55 s)
32026-04-13T16:10:10.711Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-04-13T16:10:12.479Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-04-13T16:10:12.483Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-04-13T16:10:14.337Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-04-13T16:10:14.341Zdownloading input: /input/build/work/bins/crutest.gz
82026-04-13T16:10:16.226Zdownloaded input: /input/build/work/bins/crutest.gz
92026-04-13T16:10:16.230Zdownloading input: /input/build/work/bins/dsc.gz
102026-04-13T16:10:17.335Zdownloaded input: /input/build/work/bins/dsc.gz
112026-04-13T16:10:17.339Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-04-13T16:10:17.367Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-04-13T16:10:17.371Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-04-13T16:10:17.395Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-04-13T16:10:17.395Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-04-13T16:10:17.426Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-04-13T16:10:17.426Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-04-13T16:10:17.475Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-04-13T16:10:17.475Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-04-13T16:10:17.501Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-04-13T16:10:17.501Zdownloading input: /input/build/work/scripts/test_up.sh
222026-04-13T16:10:17.529Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-04-13T16:10:17.529Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-04-13T16:10:17.556Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-04-13T16:10:17.559Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-04-13T16:10:18.226Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-04-13T16:10:18.226Zstarting task 0: "setup"
282026-04-13T16:10:18.231Z++ uname -s
292026-04-13T16:10:18.235Z+ kern=SunOS
302026-04-13T16:10:18.235Z+ build_user=build
312026-04-13T16:10:18.235Z+ build_uid=12345
322026-04-13T16:10:18.235Z+ work_dir=/work
332026-04-13T16:10:18.235Z+ input_dir=/input
342026-04-13T16:10:18.235Z+ [[ 0 == 12345 ]]
352026-04-13T16:10:18.235Z+ case "$kern" in
362026-04-13T16:10:18.235Z+ groupadd -g 12345 build
372026-04-13T16:10:18.235Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-04-13T16:10:20.241Z+ zfs create -o mountpoint=/work rpool/work
392026-04-13T16:10:20.513Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-04-13T16:10:20.517Z+ home_fs=zfs
412026-04-13T16:10:20.517Z+ [[ zfs == autofs ]]
422026-04-13T16:10:20.517Z+ mkdir -p /home/build
432026-04-13T16:10:20.517Z+ chown build:build /home/build /work
442026-04-13T16:10:22.518Z+ chmod 0700 /home/build /work
452026-04-13T16:10:22.522Zprocess exited: duration 4294 ms, exit code 0
 
462026-04-13T16:10:22.529Zstarting task 1: "authentication"
472026-04-13T16:10:22.547Zprocess exited: duration 18 ms, exit code 0
 
482026-04-13T16:10:22.554Zstarting task 2: "build"
492026-04-13T16:10:22.558Z+ banner cores
502026-04-13T16:10:22.561Z
512026-04-13T16:10:22.561Z #### #### ##### ###### ####
522026-04-13T16:10:22.561Z # # # # # # # #
532026-04-13T16:10:22.561Z # # # # # ##### ####
542026-04-13T16:10:22.561Z # # # ##### # #
552026-04-13T16:10:22.561Z # # # # # # # # #
562026-04-13T16:10:22.561Z #### #### # # ###### ####
572026-04-13T16:10:22.561Z
582026-04-13T16:10:22.561Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-04-13T16:10:22.565Z+ echo 'input bins dir contains:'
602026-04-13T16:10:22.565Zinput bins dir contains:
612026-04-13T16:10:22.565Z+ ls -ltr /input/build/work/bins
622026-04-13T16:10:22.569Ztotal 1036444
632026-04-13T16:10:22.569Z-rw-r--r-- 1 root root 157301608 Apr 13 16:10 crucible-downstairs.gz
642026-04-13T16:10:22.569Z-rw-r--r-- 1 root root 145686170 Apr 13 16:10 crucible-hammer.gz
652026-04-13T16:10:22.569Z-rw-r--r-- 1 root root 154840432 Apr 13 16:10 crutest.gz
662026-04-13T16:10:22.569Z-rw-r--r-- 1 root root 72474450 Apr 13 16:10 dsc.gz
672026-04-13T16:10:22.569Z+ echo 'input script dir contains:'
682026-04-13T16:10:22.569Zinput script dir contains:
692026-04-13T16:10:22.569Z+ ls -ltr /input/build/work/scripts
702026-04-13T16:10:22.569Ztotal 46
712026-04-13T16:10:22.569Z-rw-r--r-- 1 root root 3056 Apr 13 16:10 perf-downstairs-tick.d
722026-04-13T16:10:22.569Z-rw-r--r-- 1 root root 2127 Apr 13 16:10 test_ds.sh
732026-04-13T16:10:22.569Z-rw-r--r-- 1 root root 4923 Apr 13 16:10 test_live_repair.sh
742026-04-13T16:10:22.569Z-rw-r--r-- 1 root root 7419 Apr 13 16:10 test_repair.sh
752026-04-13T16:10:22.569Z-rw-r--r-- 1 root root 4282 Apr 13 16:10 test_replay.sh
762026-04-13T16:10:22.569Z-rw-r--r-- 1 root root 13887 Apr 13 16:10 test_up.sh
772026-04-13T16:10:22.569Z-rw-r--r-- 1 root root 3241 Apr 13 16:10 upstairs_info.d
782026-04-13T16:10:22.569Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
792026-04-13T16:10:22.573Z+ echo ' chmod input script dir contains:'
802026-04-13T16:10:22.573Z chmod input script dir contains:
812026-04-13T16:10:22.573Z+ ls -ltr /input/build/work/scripts
822026-04-13T16:10:22.573Ztotal 46
832026-04-13T16:10:22.573Z-rwxr-xr-x 1 root root 3056 Apr 13 16:10 perf-downstairs-tick.d
842026-04-13T16:10:22.573Z-rwxr-xr-x 1 root root 2127 Apr 13 16:10 test_ds.sh
852026-04-13T16:10:22.573Z-rwxr-xr-x 1 root root 4923 Apr 13 16:10 test_live_repair.sh
862026-04-13T16:10:22.573Z-rwxr-xr-x 1 root root 7419 Apr 13 16:10 test_repair.sh
872026-04-13T16:10:22.573Z-rwxr-xr-x 1 root root 4282 Apr 13 16:10 test_replay.sh
882026-04-13T16:10:22.573Z-rwxr-xr-x 1 root root 13887 Apr 13 16:10 test_up.sh
892026-04-13T16:10:22.573Z-rwxr-xr-x 1 root root 3241 Apr 13 16:10 upstairs_info.d
902026-04-13T16:10:22.573Z+ banner unpack
912026-04-13T16:10:22.573Z
922026-04-13T16:10:22.573Z # # # # ##### ## #### # #
932026-04-13T16:10:22.573Z # # ## # # # # # # # # #
942026-04-13T16:10:22.573Z # # # # # # # # # # ####
952026-04-13T16:10:22.573Z # # # # # ##### ###### # # #
962026-04-13T16:10:22.573Z # # # ## # # # # # # #
972026-04-13T16:10:22.573Z #### # # # # # #### # #
982026-04-13T16:10:22.573Z
992026-04-13T16:10:22.577Z+ mkdir -p /var/tmp/bins
1002026-04-13T16:10:22.577Z+ for t in "$input/bins/"*.gz
1012026-04-13T16:10:22.577Z++ basename /input/build/work/bins/crucible-downstairs.gz
1022026-04-13T16:10:22.577Z+ b=crucible-downstairs.gz
1032026-04-13T16:10:22.577Z+ b=crucible-downstairs
1042026-04-13T16:10:22.577Z+ gunzip
1052026-04-13T16:10:25.211Z+ chmod +x /var/tmp/bins/crucible-downstairs
1062026-04-13T16:10:25.215Z+ for t in "$input/bins/"*.gz
1072026-04-13T16:10:25.215Z++ basename /input/build/work/bins/crucible-hammer.gz
1082026-04-13T16:10:25.215Z+ b=crucible-hammer.gz
1092026-04-13T16:10:25.215Z+ b=crucible-hammer
1102026-04-13T16:10:25.215Z+ gunzip
1112026-04-13T16:10:27.587Z+ chmod +x /var/tmp/bins/crucible-hammer
1122026-04-13T16:10:27.590Z+ for t in "$input/bins/"*.gz
1132026-04-13T16:10:27.590Z++ basename /input/build/work/bins/crutest.gz
1142026-04-13T16:10:27.594Z+ b=crutest.gz
1152026-04-13T16:10:27.594Z+ b=crutest
1162026-04-13T16:10:27.594Z+ gunzip
1172026-04-13T16:10:30.155Z+ chmod +x /var/tmp/bins/crutest
1182026-04-13T16:10:30.161Z+ for t in "$input/bins/"*.gz
1192026-04-13T16:10:30.161Z++ basename /input/build/work/bins/dsc.gz
1202026-04-13T16:10:30.161Z+ b=dsc.gz
1212026-04-13T16:10:30.161Z+ b=dsc
1222026-04-13T16:10:30.161Z+ gunzip
1232026-04-13T16:10:31.338Z+ chmod +x /var/tmp/bins/dsc
1242026-04-13T16:10:31.343Z+ export BINDIR=/var/tmp/bins
1252026-04-13T16:10:31.343Z+ BINDIR=/var/tmp/bins
1262026-04-13T16:10:31.343Z+ export RUST_BACKTRACE=1
1272026-04-13T16:10:31.343Z+ RUST_BACKTRACE=1
1282026-04-13T16:10:31.343Z+ echo 'Setup self timeout'
1292026-04-13T16:10:31.343ZSetup self timeout
1302026-04-13T16:10:31.343Z+ jobpid=1088
1312026-04-13T16:10:31.343Z+ echo 'Setup debug logging'
1322026-04-13T16:10:31.343ZSetup debug logging
1332026-04-13T16:10:31.343Z+ mkdir /tmp/debug
1342026-04-13T16:10:31.344Z+ sleep 7200
1352026-04-13T16:10:31.344Z+ psrinfo -v
1362026-04-13T16:10:31.348Z+ df -h
1372026-04-13T16:10:31.352Z+ prstat -d d -mLc 1
1382026-04-13T16:10:31.352Z+ iostat -T d -xn 1
1392026-04-13T16:10:31.352Z+ mpstat -T d 1
1402026-04-13T16:10:31.352Z+ vmstat -T d -p 1
1412026-04-13T16:10:31.352Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1422026-04-13T16:10:31.352Z+ banner repair
1432026-04-13T16:10:31.352Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1442026-04-13T16:10:31.355Z
1452026-04-13T16:10:31.355Z ##### ###### ##### ## # #####
1462026-04-13T16:10:31.355Z # # # # # # # # # #
1472026-04-13T16:10:31.356Z # # ##### # # # # # # #
1482026-04-13T16:10:31.356Z ##### # ##### ###### # #####
1492026-04-13T16:10:31.356Z # # # # # # # # #
1502026-04-13T16:10:31.356Z # # ###### # # # # # #
1512026-04-13T16:10:31.356Z
1522026-04-13T16:10:31.356Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1532026-04-13T16:37:11.160Z+ echo 'Test repair finished with 0'
1542026-04-13T16:37:11.160ZTest repair finished with 0
1552026-04-13T16:37:16.166Zprocess exited: duration 1608348 ms, exit code 0
1562026-04-13T16:37:16.166Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
1572026-04-13T16:38:16.177Zexec warning: : stdout descriptor may be held open by a background process; giving up!
1582026-04-13T16:38:16.177Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
1592026-04-13T16:38:16.192Zfound 9 output files
1602026-04-13T16:38:16.192Zuploading: /tmp/test-repair-out.txt (2575159 bytes)
1612026-04-13T16:38:17.231Zuploaded: /tmp/test-repair-out.txt
1622026-04-13T16:38:17.231Zuploading: /tmp/debug/df.txt (1270 bytes)
1632026-04-13T16:38:18.241Zuploaded: /tmp/debug/df.txt
1642026-04-13T16:38:18.241Zuploading: /tmp/debug/dtrace.txt (9009281 bytes)
1652026-04-13T16:38:19.312Zuploaded: /tmp/debug/dtrace.txt
1662026-04-13T16:38:19.312Zuploading: /tmp/debug/iostat.txt (496184 bytes)
1672026-04-13T16:38:19.328Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 496184 -> 497078
1682026-04-13T16:38:20.335Zuploaded: /tmp/debug/iostat.txt
1692026-04-13T16:38:20.335Zuploading: /tmp/debug/mpstat.txt (1210581 bytes)
1702026-04-13T16:38:20.357Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 1210581 -> 1213489
1712026-04-13T16:38:21.366Zuploaded: /tmp/debug/mpstat.txt
1722026-04-13T16:38:21.367Zuploading: /tmp/debug/paging.txt (214531 bytes)
1732026-04-13T16:38:21.379Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 214531 -> 215096
1742026-04-13T16:38:22.387Zuploaded: /tmp/debug/paging.txt
1752026-04-13T16:38:22.387Zuploading: /tmp/debug/prstat.txt (2252513 bytes)
1762026-04-13T16:38:22.416Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 2252513 -> 2260292
1772026-04-13T16:38:23.424Zuploaded: /tmp/debug/prstat.txt
1782026-04-13T16:38:23.424Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
1792026-04-13T16:38:24.436Zuploaded: /tmp/debug/psrinfo.txt
1802026-04-13T16:38:24.436Zuploading: /tmp/debug/upstairs-info.txt (191352 bytes)
1812026-04-13T16:38:25.454Zuploaded: /tmp/debug/upstairs-info.txt