01JMZ5521GF23D00EYGACJS9RA: test-replay

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/*.log",
        "%/tmp/debug/*.txt",
        "%/tmp/dsc/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JMZ55M5ACXBJPYGA5R84YRN5

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-02-25T20:32:15.821Zjob dependencies complete; ready to run (waiting for 2 h 16 m 6 s)
22025-02-25T20:46:09.820Zjob assigned to worker 01JMZDPGE8Y12NMX8R0HG7VCSW [factory aws, i-01792d73c278f4954] (queued for 13 m 53 s)
32025-02-25T20:46:17.742Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-02-25T20:46:18.924Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-02-25T20:46:18.924Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-02-25T20:46:19.988Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-02-25T20:46:19.990Zdownloading input: /input/build/work/bins/crutest.gz
82025-02-25T20:46:21.203Zdownloaded input: /input/build/work/bins/crutest.gz
92025-02-25T20:46:21.203Zdownloading input: /input/build/work/bins/dsc.gz
102025-02-25T20:46:21.875Zdownloaded input: /input/build/work/bins/dsc.gz
112025-02-25T20:46:21.880Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-02-25T20:46:21.897Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-02-25T20:46:21.899Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-02-25T20:46:21.914Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-02-25T20:46:21.914Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-02-25T20:46:21.938Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-02-25T20:46:21.938Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-02-25T20:46:21.958Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-02-25T20:46:21.959Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-02-25T20:46:21.976Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-02-25T20:46:21.976Zdownloading input: /input/build/work/scripts/test_up.sh
222025-02-25T20:46:21.992Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-02-25T20:46:21.992Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-02-25T20:46:22.008Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-02-25T20:46:22.011Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-02-25T20:46:22.667Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-02-25T20:46:22.667Zstarting task 0: "setup"
282025-02-25T20:46:22.675Z++ uname -s
292025-02-25T20:46:22.680Z+ kern=SunOS
302025-02-25T20:46:22.680Z+ build_user=build
312025-02-25T20:46:22.680Z+ build_uid=12345
322025-02-25T20:46:22.680Z+ work_dir=/work
332025-02-25T20:46:22.680Z+ input_dir=/input
342025-02-25T20:46:22.680Z+ [[ 0 == 12345 ]]
352025-02-25T20:46:22.680Z+ case "$kern" in
362025-02-25T20:46:22.680Z+ groupadd -g 12345 build
372025-02-25T20:46:22.682Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-02-25T20:46:24.688Z+ zfs create -o mountpoint=/work rpool/work
392025-02-25T20:46:24.961Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-02-25T20:46:24.964Z+ home_fs=zfs
412025-02-25T20:46:24.964Z+ [[ zfs == autofs ]]
422025-02-25T20:46:24.964Z+ mkdir -p /home/build
432025-02-25T20:46:24.967Z+ chown build:build /home/build /work
442025-02-25T20:46:26.968Z+ chmod 0700 /home/build /work
452025-02-25T20:46:26.971Zprocess exited: duration 4302 ms, exit code 0
 
462025-02-25T20:46:26.976Zstarting task 1: "authentication"
472025-02-25T20:46:26.994Zprocess exited: duration 18 ms, exit code 0
 
482025-02-25T20:46:27.003Zstarting task 2: "build"
492025-02-25T20:46:27.005Z+ banner cores
502025-02-25T20:46:27.008Z
512025-02-25T20:46:27.008Z #### #### ##### ###### ####
522025-02-25T20:46:27.008Z # # # # # # # #
532025-02-25T20:46:27.008Z # # # # # ##### ####
542025-02-25T20:46:27.008Z # # # ##### # #
552025-02-25T20:46:27.008Z # # # # # # # # #
562025-02-25T20:46:27.008Z #### #### # # ###### ####
572025-02-25T20:46:27.008Z
582025-02-25T20:46:27.011Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-02-25T20:46:27.019Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
602025-02-25T20:46:27.021Z+ echo ' chmod input script dir contains:'
612025-02-25T20:46:27.021Z chmod input script dir contains:
622025-02-25T20:46:27.021Z+ ls -ltr /input/build/work/scripts
632025-02-25T20:46:27.024Ztotal 43
642025-02-25T20:46:27.024Z-rwxr-xr-x 1 root root 3056 Feb 25 20:46 perf-downstairs-tick.d
652025-02-25T20:46:27.024Z-rwxr-xr-x 1 root root 2127 Feb 25 20:46 test_ds.sh
662025-02-25T20:46:27.024Z-rwxr-xr-x 1 root root 4272 Feb 25 20:46 test_live_repair.sh
672025-02-25T20:46:27.024Z-rwxr-xr-x 1 root root 6724 Feb 25 20:46 test_repair.sh
682025-02-25T20:46:27.024Z-rwxr-xr-x 1 root root 3494 Feb 25 20:46 test_replay.sh
692025-02-25T20:46:27.024Z-rwxr-xr-x 1 root root 13887 Feb 25 20:46 test_up.sh
702025-02-25T20:46:27.024Z-rwxr-xr-x 1 root root 2996 Feb 25 20:46 upstairs_info.d
712025-02-25T20:46:27.024Z+ banner unpack
722025-02-25T20:46:27.027Z
732025-02-25T20:46:27.027Z # # # # ##### ## #### # #
742025-02-25T20:46:27.027Z # # ## # # # # # # # # #
752025-02-25T20:46:27.027Z # # # # # # # # # # ####
762025-02-25T20:46:27.027Z # # # # # ##### ###### # # #
772025-02-25T20:46:27.027Z # # # ## # # # # # # #
782025-02-25T20:46:27.027Z #### # # # # # #### # #
792025-02-25T20:46:27.027Z
802025-02-25T20:46:27.027Z+ mkdir -p /var/tmp/bins
812025-02-25T20:46:27.027Z+ for t in "$input/bins/"*.gz
822025-02-25T20:46:27.030Z++ basename /input/build/work/bins/crucible-downstairs.gz
832025-02-25T20:46:27.030Z+ b=crucible-downstairs.gz
842025-02-25T20:46:27.030Z+ b=crucible-downstairs
852025-02-25T20:46:27.030Z+ gunzip
862025-02-25T20:46:29.455Z+ chmod +x /var/tmp/bins/crucible-downstairs
872025-02-25T20:46:29.458Z+ for t in "$input/bins/"*.gz
882025-02-25T20:46:29.461Z++ basename /input/build/work/bins/crucible-hammer.gz
892025-02-25T20:46:29.461Z+ b=crucible-hammer.gz
902025-02-25T20:46:29.461Z+ b=crucible-hammer
912025-02-25T20:46:29.461Z+ gunzip
922025-02-25T20:46:31.584Z+ chmod +x /var/tmp/bins/crucible-hammer
932025-02-25T20:46:31.586Z+ for t in "$input/bins/"*.gz
942025-02-25T20:46:31.589Z++ basename /input/build/work/bins/crutest.gz
952025-02-25T20:46:31.589Z+ b=crutest.gz
962025-02-25T20:46:31.590Z+ b=crutest
972025-02-25T20:46:31.590Z+ gunzip
982025-02-25T20:46:34.018Z+ chmod +x /var/tmp/bins/crutest
992025-02-25T20:46:34.020Z+ for t in "$input/bins/"*.gz
1002025-02-25T20:46:34.020Z++ basename /input/build/work/bins/dsc.gz
1012025-02-25T20:46:34.020Z+ b=dsc.gz
1022025-02-25T20:46:34.021Z+ b=dsc
1032025-02-25T20:46:34.021Z+ gunzip
1042025-02-25T20:46:35.364Z+ chmod +x /var/tmp/bins/dsc
1052025-02-25T20:46:35.367Z+ export BINDIR=/var/tmp/bins
1062025-02-25T20:46:35.367Z+ BINDIR=/var/tmp/bins
1072025-02-25T20:46:35.367Z+ export RUST_BACKTRACE=1
1082025-02-25T20:46:35.368Z+ RUST_BACKTRACE=1
1092025-02-25T20:46:35.368Z+ banner setup
1102025-02-25T20:46:35.371Z
1112025-02-25T20:46:35.371Z #### ###### ##### # # #####
1122025-02-25T20:46:35.371Z # # # # # # #
1132025-02-25T20:46:35.371Z #### ##### # # # # #
1142025-02-25T20:46:35.371Z # # # # # #####
1152025-02-25T20:46:35.371Z # # # # # # #
1162025-02-25T20:46:35.371Z #### ###### # #### #
1172025-02-25T20:46:35.371Z
1182025-02-25T20:46:35.371Z+ echo 'Setup self timeout'
1192025-02-25T20:46:35.371ZSetup self timeout
1202025-02-25T20:46:35.371Z+ jobpid=1145
1212025-02-25T20:46:35.371Z+ echo 'Setup debug logging'
1222025-02-25T20:46:35.371ZSetup debug logging
1232025-02-25T20:46:35.371Z+ mkdir /tmp/debug
1242025-02-25T20:46:35.371Z+ sleep 10800
1252025-02-25T20:46:35.375Z+ psrinfo -v
1262025-02-25T20:46:35.379Z+ df -h
1272025-02-25T20:46:35.384Z+ prstat -d d -mLc 1
1282025-02-25T20:46:35.386Z+ iostat -T d -xn 1
1292025-02-25T20:46:35.387Z+ mpstat -T d 1
1302025-02-25T20:46:35.387Z+ vmstat -T d -p 1
1312025-02-25T20:46:35.387Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1322025-02-25T20:46:35.387Z+ banner replay
1332025-02-25T20:46:35.387Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1342025-02-25T20:46:35.390Z
1352025-02-25T20:46:35.390Z ##### ###### ##### # ## # #
1362025-02-25T20:46:35.390Z # # # # # # # # # #
1372025-02-25T20:46:35.390Z # # ##### # # # # # #
1382025-02-25T20:46:35.390Z ##### # ##### # ###### #
1392025-02-25T20:46:35.390Z # # # # # # # #
1402025-02-25T20:46:35.390Z # # ###### # ###### # # #
1412025-02-25T20:46:35.390Z
1422025-02-25T20:46:35.390Z+ ptime -m bash /input/build/work/scripts/test_replay.sh
1432025-02-25T20:46:35.406Zstarting February 25, 2025 at 08:46:34 PM UTC
1442025-02-25T20:46:35.409ZTail /tmp/test_replay.log for test output
1452025-02-25T20:46:35.412ZCreating 3 downstairs regions
1462025-02-25T20:46:35.747ZStarting 3 downstairs
1472025-02-25T20:46:40.759ZRunning initial fill
1482025-02-25T20:46:45.483ZReplay loop starts now February 25, 2025 at 08:46:44 PM UTC
1492025-02-25T20:54:25.643ZReplays:30 time: 7:40
1502025-02-25T20:54:25.646ZDo final verify
1512025-02-25T20:54:35.195ZFebruary 25, 2025 at 08:54:34 PM UTC Test ends with 0
1522025-02-25T20:54:35.198Z
1532025-02-25T20:54:35.198Zreal 7:59.762299649
1542025-02-25T20:54:35.198Zuser 23.817414675
1552025-02-25T20:54:35.198Zsys 6.002895951
1562025-02-25T20:54:35.198Ztrap 0.047984684
1572025-02-25T20:54:35.198Ztflt 0.013173875
1582025-02-25T20:54:35.198Zdflt 0.014082750
1592025-02-25T20:54:35.198Zkflt 0.001579773
1602025-02-25T20:54:35.198Zlock 12:30:24.253160535
1612025-02-25T20:54:35.198Zslp 43:04.623798311
1622025-02-25T20:54:35.198Zlat 3.397267520
1632025-02-25T20:54:35.199Zstop 0.579883928
1642025-02-25T20:54:40.202Zprocess exited: duration 488153 ms, exit code 0
1652025-02-25T20:54:40.202Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
1662025-02-25T20:55:40.256Zexec warning: : stdout descriptor may be held open by a background process; giving up!
1672025-02-25T20:55:40.256Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
1682025-02-25T20:55:40.267Zfound 13 output files
1692025-02-25T20:55:40.269Zuploading: /tmp/test_replay.log (2100592 bytes)
1702025-02-25T20:55:41.340Zuploaded: /tmp/test_replay.log
1712025-02-25T20:55:41.341Zuploading: /tmp/test_replay_verify.log (70144 bytes)
1722025-02-25T20:55:42.378Zuploaded: /tmp/test_replay_verify.log
1732025-02-25T20:55:42.381Zuploading: /tmp/debug/df.txt (1270 bytes)
1742025-02-25T20:55:43.446Zuploaded: /tmp/debug/df.txt
1752025-02-25T20:55:43.446Zuploading: /tmp/debug/iostat.txt (164055 bytes)
1762025-02-25T20:55:43.493Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 164055 -> 165259
1772025-02-25T20:55:44.506Zuploaded: /tmp/debug/iostat.txt
1782025-02-25T20:55:44.509Zuploading: /tmp/debug/mpstat.txt (397852 bytes)
1792025-02-25T20:55:44.520Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 397852 -> 401502
1802025-02-25T20:55:45.528Zuploaded: /tmp/debug/mpstat.txt
1812025-02-25T20:55:45.531Zuploading: /tmp/debug/paging.txt (71910 bytes)
1822025-02-25T20:55:45.534Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 71910 -> 72764
1832025-02-25T20:55:46.540Zuploaded: /tmp/debug/paging.txt
1842025-02-25T20:55:46.541Zuploading: /tmp/debug/perf.txt (1017683 bytes)
1852025-02-25T20:55:47.559Zuploaded: /tmp/debug/perf.txt
1862025-02-25T20:55:47.559Zuploading: /tmp/debug/prstat.txt (743863 bytes)
1872025-02-25T20:55:47.573Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 743863 -> 754208
1882025-02-25T20:55:48.582Zuploaded: /tmp/debug/prstat.txt
1892025-02-25T20:55:48.582Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
1902025-02-25T20:55:49.610Zuploaded: /tmp/debug/psrinfo.txt
1912025-02-25T20:55:49.610Zuploading: /tmp/debug/upinfo.txt (53900 bytes)
1922025-02-25T20:55:50.620Zuploaded: /tmp/debug/upinfo.txt
1932025-02-25T20:55:50.623Zuploading: /tmp/dsc/downstairs-8810.txt (7848 bytes)
1942025-02-25T20:55:51.629Zuploaded: /tmp/dsc/downstairs-8810.txt
1952025-02-25T20:55:51.629Zuploading: /tmp/dsc/downstairs-8820.txt (7851 bytes)
1962025-02-25T20:55:52.637Zuploaded: /tmp/dsc/downstairs-8820.txt
1972025-02-25T20:55:52.637Zuploading: /tmp/dsc/downstairs-8830.txt (7846 bytes)
1982025-02-25T20:55:53.645Zuploaded: /tmp/dsc/downstairs-8830.txt