01JBM7WNYBJJ5VR2HN1NM5X2E3: test-replay

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/*.log",
        "%/tmp/debug/*.txt",
        "%/tmp/dsc/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JBM7YMB5XNTM354F5VMMJJS9

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-11-01T16:26:07.401Zjob dependencies complete; ready to run (waiting for 14 m 16 s)
22024-11-01T16:49:59.068Zjob assigned to worker 01JBMA2V1ED74CMYQ1QEAX6KWF (queued for 23 m 51 s)
32024-11-01T16:50:06.866Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-11-01T16:50:08.123Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-11-01T16:50:08.123Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-11-01T16:50:10.076Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-11-01T16:50:10.076Zdownloading input: /input/build/work/bins/crutest.gz
82024-11-01T16:50:12.411Zdownloaded input: /input/build/work/bins/crutest.gz
92024-11-01T16:50:12.412Zdownloading input: /input/build/work/bins/dsc.gz
102024-11-01T16:50:13.123Zdownloaded input: /input/build/work/bins/dsc.gz
112024-11-01T16:50:13.126Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-11-01T16:50:13.143Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-11-01T16:50:13.143Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-11-01T16:50:13.160Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-11-01T16:50:13.160Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-11-01T16:50:13.180Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-11-01T16:50:13.180Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-11-01T16:50:13.198Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-11-01T16:50:13.198Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-11-01T16:50:13.217Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-11-01T16:50:13.217Zdownloading input: /input/build/work/scripts/test_up.sh
222024-11-01T16:50:13.240Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-11-01T16:50:13.240Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-11-01T16:50:13.274Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-11-01T16:50:13.279Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-11-01T16:50:14.196Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-11-01T16:50:14.196Zstarting task 0: "setup"
282024-11-01T16:50:14.207Z++ uname -s
292024-11-01T16:50:14.209Z+ kern=SunOS
302024-11-01T16:50:14.209Z+ case "$kern" in
312024-11-01T16:50:14.209Z+ [[ 0 == 12345 ]]
322024-11-01T16:50:14.209Z+ groupadd -g 12345 build
332024-11-01T16:50:14.212Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
342024-11-01T16:50:16.221Z+ zfs create -o mountpoint=/work rpool/work
352024-11-01T16:50:16.337Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
362024-11-01T16:50:16.340Z+ home_fs=zfs
372024-11-01T16:50:16.340Z+ [[ zfs == autofs ]]
382024-11-01T16:50:16.340Z+ mkdir -p /home/build
392024-11-01T16:50:16.343Z+ chown build:build /home/build /work
402024-11-01T16:50:18.344Z+ chmod 0700 /home/build /work
412024-11-01T16:50:18.346Zprocess exited: duration 4149 ms, exit code 0
 
422024-11-01T16:50:18.352Zstarting task 1: "authentication"
432024-11-01T16:50:18.370Zprocess exited: duration 17 ms, exit code 0
 
442024-11-01T16:50:18.375Zstarting task 2: "build"
452024-11-01T16:50:18.380Z+ banner cores
462024-11-01T16:50:18.412Z
472024-11-01T16:50:18.412Z #### #### ##### ###### ####
482024-11-01T16:50:18.412Z # # # # # # # #
492024-11-01T16:50:18.412Z # # # # # ##### ####
502024-11-01T16:50:18.412Z # # # ##### # #
512024-11-01T16:50:18.412Z # # # # # # # # #
522024-11-01T16:50:18.412Z #### #### # # ###### ####
532024-11-01T16:50:18.412Z
542024-11-01T16:50:18.415Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
552024-11-01T16:50:18.423Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
562024-11-01T16:50:18.426Z+ echo ' chmod input script dir contains:'
572024-11-01T16:50:18.429Z chmod input script dir contains:
582024-11-01T16:50:18.429Z+ ls -ltr /input/build/work/scripts
592024-11-01T16:50:18.429Ztotal 42
602024-11-01T16:50:18.429Z-rwxr-xr-x 1 root root 3056 Nov 1 16:50 perf-downstairs-tick.d
612024-11-01T16:50:18.429Z-rwxr-xr-x 1 root root 2127 Nov 1 16:50 test_ds.sh
622024-11-01T16:50:18.429Z-rwxr-xr-x 1 root root 4272 Nov 1 16:50 test_live_repair.sh
632024-11-01T16:50:18.429Z-rwxr-xr-x 1 root root 6724 Nov 1 16:50 test_repair.sh
642024-11-01T16:50:18.429Z-rwxr-xr-x 1 root root 3494 Nov 1 16:50 test_replay.sh
652024-11-01T16:50:18.429Z-rwxr-xr-x 1 root root 13887 Nov 1 16:50 test_up.sh
662024-11-01T16:50:18.429Z-rwxr-xr-x 1 root root 2407 Nov 1 16:50 upstairs_info.d
672024-11-01T16:50:18.429Z+ banner unpack
682024-11-01T16:50:18.432Z
692024-11-01T16:50:18.432Z # # # # ##### ## #### # #
702024-11-01T16:50:18.432Z # # ## # # # # # # # # #
712024-11-01T16:50:18.432Z # # # # # # # # # # ####
722024-11-01T16:50:18.432Z # # # # # ##### ###### # # #
732024-11-01T16:50:18.432Z # # # ## # # # # # # #
742024-11-01T16:50:18.432Z #### # # # # # #### # #
752024-11-01T16:50:18.432Z
762024-11-01T16:50:18.432Z+ mkdir -p /var/tmp/bins
772024-11-01T16:50:18.435Z+ for t in "$input/bins/"*.gz
782024-11-01T16:50:18.435Z++ basename /input/build/work/bins/crucible-downstairs.gz
792024-11-01T16:50:18.435Z+ b=crucible-downstairs.gz
802024-11-01T16:50:18.435Z+ b=crucible-downstairs
812024-11-01T16:50:18.435Z+ gunzip
822024-11-01T16:50:20.689Z+ chmod +x /var/tmp/bins/crucible-downstairs
832024-11-01T16:50:20.692Z+ for t in "$input/bins/"*.gz
842024-11-01T16:50:20.694Z++ basename /input/build/work/bins/crucible-hammer.gz
852024-11-01T16:50:20.694Z+ b=crucible-hammer.gz
862024-11-01T16:50:20.694Z+ b=crucible-hammer
872024-11-01T16:50:20.694Z+ gunzip
882024-11-01T16:50:22.858Z+ chmod +x /var/tmp/bins/crucible-hammer
892024-11-01T16:50:22.861Z+ for t in "$input/bins/"*.gz
902024-11-01T16:50:22.861Z++ basename /input/build/work/bins/crutest.gz
912024-11-01T16:50:22.864Z+ b=crutest.gz
922024-11-01T16:50:22.865Z+ b=crutest
932024-11-01T16:50:22.865Z+ gunzip
942024-11-01T16:50:25.189Z+ chmod +x /var/tmp/bins/crutest
952024-11-01T16:50:25.192Z+ for t in "$input/bins/"*.gz
962024-11-01T16:50:25.196Z++ basename /input/build/work/bins/dsc.gz
972024-11-01T16:50:25.196Z+ b=dsc.gz
982024-11-01T16:50:25.196Z+ b=dsc
992024-11-01T16:50:25.196Z+ gunzip
1002024-11-01T16:50:26.576Z+ chmod +x /var/tmp/bins/dsc
1012024-11-01T16:50:26.578Z+ export BINDIR=/var/tmp/bins
1022024-11-01T16:50:26.579Z+ BINDIR=/var/tmp/bins
1032024-11-01T16:50:26.579Z+ banner setup
1042024-11-01T16:50:26.581Z
1052024-11-01T16:50:26.582Z #### ###### ##### # # #####
1062024-11-01T16:50:26.582Z # # # # # # #
1072024-11-01T16:50:26.582Z #### ##### # # # # #
1082024-11-01T16:50:26.582Z # # # # # #####
1092024-11-01T16:50:26.582Z # # # # # # #
1102024-11-01T16:50:26.582Z #### ###### # #### #
1112024-11-01T16:50:26.582Z
1122024-11-01T16:50:26.582Z+ echo 'Setup self timeout'
1132024-11-01T16:50:26.582ZSetup self timeout
1142024-11-01T16:50:26.582Z+ jobpid=1137
1152024-11-01T16:50:26.582Z+ echo 'Setup debug logging'
1162024-11-01T16:50:26.582ZSetup debug logging
1172024-11-01T16:50:26.582Z+ mkdir /tmp/debug
1182024-11-01T16:50:26.582Z+ sleep 10800
1192024-11-01T16:50:26.585Z+ psrinfo -v
1202024-11-01T16:50:26.588Z+ df -h
1212024-11-01T16:50:26.611Z+ prstat -d d -mLc 1
1222024-11-01T16:50:26.614Z+ iostat -T d -xn 1
1232024-11-01T16:50:26.614Z+ mpstat -T d 1
1242024-11-01T16:50:26.614Z+ vmstat -T d -p 1
1252024-11-01T16:50:26.614Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1262024-11-01T16:50:26.617Z+ banner replay
1272024-11-01T16:50:26.617Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1282024-11-01T16:50:26.617Z
1292024-11-01T16:50:26.617Z ##### ###### ##### # ## # #
1302024-11-01T16:50:26.617Z # # # # # # # # # #
1312024-11-01T16:50:26.617Z # # ##### # # # # # #
1322024-11-01T16:50:26.617Z ##### # ##### # ###### #
1332024-11-01T16:50:26.617Z # # # # # # # #
1342024-11-01T16:50:26.617Z # # ###### # ###### # # #
1352024-11-01T16:50:26.617Z
1362024-11-01T16:50:26.617Z+ ptime -m bash /input/build/work/scripts/test_replay.sh
1372024-11-01T16:50:26.634Zstarting November 1, 2024 at 04:50:25 PM UTC
1382024-11-01T16:50:26.637ZTail /tmp/test_replay.log for test output
1392024-11-01T16:50:26.640ZCreating 3 downstairs regions
1402024-11-01T16:50:27.161ZStarting 3 downstairs
1412024-11-01T16:50:32.173ZRunning initial fill
1422024-11-01T16:50:32.917ZReplay loop starts now November 1, 2024 at 04:50:32 PM UTC
1432024-11-01T16:58:06.523ZReplays:30 time: 7:33
1442024-11-01T16:58:06.526ZDo final verify
1452024-11-01T16:58:16.119ZNovember 1, 2024 at 04:58:15 PM UTC Test ends with 0
1462024-11-01T16:58:16.122Z
1472024-11-01T16:58:16.122Zreal 7:49.412727029
1482024-11-01T16:58:16.122Zuser 24.582896174
1492024-11-01T16:58:16.122Zsys 6.777896818
1502024-11-01T16:58:16.122Ztrap 0.055657287
1512024-11-01T16:58:16.122Ztflt 0.007722707
1522024-11-01T16:58:16.122Zdflt 0.043495031
1532024-11-01T16:58:16.122Zkflt 0.001041090
1542024-11-01T16:58:16.122Zlock 12:15:04.493016559
1552024-11-01T16:58:16.122Zslp 42:18.552517358
1562024-11-01T16:58:16.122Zlat 3.560932527
1572024-11-01T16:58:16.122Zstop 0.557698367
1582024-11-01T16:58:21.125Zprocess exited: duration 477656 ms, exit code 0
1592024-11-01T16:58:21.126Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
1602024-11-01T16:59:21.174Zexec warning: : stdout descriptor may be held open by a background process; giving up!
1612024-11-01T16:59:21.174Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
1622024-11-01T16:59:21.189Zfound 13 output files
1632024-11-01T16:59:21.189Zuploading: /tmp/test_replay.log (2081200 bytes)
1642024-11-01T16:59:22.229Zuploaded: /tmp/test_replay.log
1652024-11-01T16:59:22.229Zuploading: /tmp/test_replay_verify.log (70184 bytes)
1662024-11-01T16:59:23.249Zuploaded: /tmp/test_replay_verify.log
1672024-11-01T16:59:23.249Zuploading: /tmp/debug/df.txt (1270 bytes)
1682024-11-01T16:59:24.258Zuploaded: /tmp/debug/df.txt
1692024-11-01T16:59:24.261Zuploading: /tmp/debug/iostat.txt (161043 bytes)
1702024-11-01T16:59:24.268Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 161043 -> 161946
1712024-11-01T16:59:25.274Zuploaded: /tmp/debug/iostat.txt
1722024-11-01T16:59:25.277Zuploading: /tmp/debug/mpstat.txt (390553 bytes)
1732024-11-01T16:59:25.287Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 390553 -> 393473
1742024-11-01T16:59:26.294Zuploaded: /tmp/debug/mpstat.txt
1752024-11-01T16:59:26.297Zuploading: /tmp/debug/paging.txt (70592 bytes)
1762024-11-01T16:59:26.302Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 70592 -> 71172
1772024-11-01T16:59:27.308Zuploaded: /tmp/debug/paging.txt
1782024-11-01T16:59:27.308Zuploading: /tmp/debug/perf.txt (1004267 bytes)
1792024-11-01T16:59:28.330Zuploaded: /tmp/debug/perf.txt
1802024-11-01T16:59:28.333Zuploading: /tmp/debug/prstat.txt (732792 bytes)
1812024-11-01T16:59:28.345Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 732792 -> 741845
1822024-11-01T16:59:29.353Zuploaded: /tmp/debug/prstat.txt
1832024-11-01T16:59:29.353Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
1842024-11-01T16:59:30.361Zuploaded: /tmp/debug/psrinfo.txt
1852024-11-01T16:59:30.363Zuploading: /tmp/debug/upinfo.txt (75524 bytes)
1862024-11-01T16:59:31.373Zuploaded: /tmp/debug/upinfo.txt
1872024-11-01T16:59:31.373Zuploading: /tmp/dsc/downstairs-8810.txt (7837 bytes)
1882024-11-01T16:59:32.382Zuploaded: /tmp/dsc/downstairs-8810.txt
1892024-11-01T16:59:32.382Zuploading: /tmp/dsc/downstairs-8820.txt (7837 bytes)
1902024-11-01T16:59:33.391Zuploaded: /tmp/dsc/downstairs-8820.txt
1912024-11-01T16:59:33.394Zuploading: /tmp/dsc/downstairs-8830.txt (7835 bytes)
1922024-11-01T16:59:34.403Zuploaded: /tmp/dsc/downstairs-8830.txt