01J44ZV82T9BHMAYK0PV52B44F: test-replay

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/*.log",
        "%/tmp/debug/*.txt",
        "%/tmp/dsc/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01J44ZVHR32RH5ZMJEJZC151RJ

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-07-31T18:25:29.552Zjob dependencies complete; ready to run (waiting for 13 m 37 s)
22024-07-31T18:26:30.707Zjob assigned to worker 01J450MMQ8A7NJXX69R7BNFMMP (queued for 1 m 1 s)
32024-07-31T18:26:39.405Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-07-31T18:26:40.463Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-07-31T18:26:40.464Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-07-31T18:26:41.305Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-07-31T18:26:41.305Zdownloading input: /input/build/work/bins/crutest.gz
82024-07-31T18:26:42.641Zdownloaded input: /input/build/work/bins/crutest.gz
92024-07-31T18:26:42.641Zdownloading input: /input/build/work/bins/dsc.gz
102024-07-31T18:26:43.321Zdownloaded input: /input/build/work/bins/dsc.gz
112024-07-31T18:26:43.327Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-07-31T18:26:43.343Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-07-31T18:26:43.344Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-07-31T18:26:43.372Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-07-31T18:26:43.372Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-07-31T18:26:43.386Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-07-31T18:26:43.386Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-07-31T18:26:43.413Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-07-31T18:26:43.413Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-07-31T18:26:43.437Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-07-31T18:26:43.438Zdownloading input: /input/build/work/scripts/test_up.sh
222024-07-31T18:26:43.461Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-07-31T18:26:43.461Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-07-31T18:26:43.490Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-07-31T18:26:43.498Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-07-31T18:26:44.481Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-07-31T18:26:44.481Zstarting task 0: "setup"
282024-07-31T18:26:44.521Z++ uname -s
292024-07-31T18:26:44.526Z+ kern=SunOS
302024-07-31T18:26:44.526Z+ case "$kern" in
312024-07-31T18:26:44.526Z+ groupadd -g 12345 build
322024-07-31T18:26:44.526Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-07-31T18:26:46.537Z+ zfs create -o mountpoint=/work rpool/work
342024-07-31T18:26:46.959Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-07-31T18:26:46.962Z+ home_fs=zfs
362024-07-31T18:26:46.962Z+ [[ zfs == autofs ]]
372024-07-31T18:26:46.962Z+ mkdir -p /home/build
382024-07-31T18:26:46.962Z+ chown build:build /home/build /work
392024-07-31T18:26:48.965Z+ chmod 0700 /home/build /work
402024-07-31T18:26:48.968Zprocess exited: duration 4503 ms, exit code 0
 
412024-07-31T18:26:48.974Zstarting task 1: "authentication"
422024-07-31T18:26:48.992Zprocess exited: duration 17 ms, exit code 0
 
432024-07-31T18:26:48.999Zstarting task 2: "build"
442024-07-31T18:26:49.004Z+ banner cores
452024-07-31T18:26:49.023Z
462024-07-31T18:26:49.023Z #### #### ##### ###### ####
472024-07-31T18:26:49.023Z # # # # # # # #
482024-07-31T18:26:49.023Z # # # # # ##### ####
492024-07-31T18:26:49.023Z # # # ##### # #
502024-07-31T18:26:49.023Z # # # # # # # # #
512024-07-31T18:26:49.023Z #### #### # # ###### ####
522024-07-31T18:26:49.023Z
532024-07-31T18:26:49.026Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-07-31T18:26:49.034Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
552024-07-31T18:26:49.039Z+ echo ' chmod input script dir contains:'
562024-07-31T18:26:49.039Z chmod input script dir contains:
572024-07-31T18:26:49.039Z+ ls -ltr /input/build/work/scripts
582024-07-31T18:26:49.043Ztotal 40
592024-07-31T18:26:49.043Z-rwxr-xr-x 1 root root 3056 Jul 31 18:26 perf-downstairs-tick.d
602024-07-31T18:26:49.043Z-rwxr-xr-x 1 root root 2102 Jul 31 18:26 test_ds.sh
612024-07-31T18:26:49.043Z-rwxr-xr-x 1 root root 4086 Jul 31 18:26 test_live_repair.sh
622024-07-31T18:26:49.043Z-rwxr-xr-x 1 root root 6671 Jul 31 18:26 test_repair.sh
632024-07-31T18:26:49.043Z-rwxr-xr-x 1 root root 3029 Jul 31 18:26 test_replay.sh
642024-07-31T18:26:49.043Z-rwxr-xr-x 1 root root 11934 Jul 31 18:26 test_up.sh
652024-07-31T18:26:49.043Z-rwxr-xr-x 1 root root 2750 Jul 31 18:26 upstairs_info.d
662024-07-31T18:26:49.043Z+ banner unpack
672024-07-31T18:26:49.043Z
682024-07-31T18:26:49.043Z # # # # ##### ## #### # #
692024-07-31T18:26:49.043Z # # ## # # # # # # # # #
702024-07-31T18:26:49.043Z # # # # # # # # # # ####
712024-07-31T18:26:49.043Z # # # # # ##### ###### # # #
722024-07-31T18:26:49.043Z # # # ## # # # # # # #
732024-07-31T18:26:49.043Z #### # # # # # #### # #
742024-07-31T18:26:49.043Z
752024-07-31T18:26:49.043Z+ mkdir -p /var/tmp/bins
762024-07-31T18:26:49.047Z+ for t in "$input/bins/"*.gz
772024-07-31T18:26:49.047Z++ basename /input/build/work/bins/crucible-downstairs.gz
782024-07-31T18:26:49.047Z+ b=crucible-downstairs.gz
792024-07-31T18:26:49.048Z+ b=crucible-downstairs
802024-07-31T18:26:49.048Z+ gunzip
812024-07-31T18:26:51.255Z+ chmod +x /var/tmp/bins/crucible-downstairs
822024-07-31T18:26:51.258Z+ for t in "$input/bins/"*.gz
832024-07-31T18:26:51.260Z++ basename /input/build/work/bins/crucible-hammer.gz
842024-07-31T18:26:51.260Z+ b=crucible-hammer.gz
852024-07-31T18:26:51.260Z+ b=crucible-hammer
862024-07-31T18:26:51.261Z+ gunzip
872024-07-31T18:26:53.001Z+ chmod +x /var/tmp/bins/crucible-hammer
882024-07-31T18:26:53.005Z+ for t in "$input/bins/"*.gz
892024-07-31T18:26:53.005Z++ basename /input/build/work/bins/crutest.gz
902024-07-31T18:26:53.005Z+ b=crutest.gz
912024-07-31T18:26:53.005Z+ b=crutest
922024-07-31T18:26:53.005Z+ gunzip
932024-07-31T18:26:55.170Z+ chmod +x /var/tmp/bins/crutest
942024-07-31T18:26:55.173Z+ for t in "$input/bins/"*.gz
952024-07-31T18:26:55.176Z++ basename /input/build/work/bins/dsc.gz
962024-07-31T18:26:55.176Z+ b=dsc.gz
972024-07-31T18:26:55.176Z+ b=dsc
982024-07-31T18:26:55.176Z+ gunzip
992024-07-31T18:26:56.564Z+ chmod +x /var/tmp/bins/dsc
1002024-07-31T18:26:56.567Z+ export BINDIR=/var/tmp/bins
1012024-07-31T18:26:56.567Z+ BINDIR=/var/tmp/bins
1022024-07-31T18:26:56.567Z+ banner setup
1032024-07-31T18:26:56.570Z
1042024-07-31T18:26:56.570Z #### ###### ##### # # #####
1052024-07-31T18:26:56.570Z # # # # # # #
1062024-07-31T18:26:56.570Z #### ##### # # # # #
1072024-07-31T18:26:56.570Z # # # # # #####
1082024-07-31T18:26:56.570Z # # # # # # #
1092024-07-31T18:26:56.570Z #### ###### # #### #
1102024-07-31T18:26:56.570Z
1112024-07-31T18:26:56.571Z+ echo 'Setup self timeout'
1122024-07-31T18:26:56.571ZSetup self timeout
1132024-07-31T18:26:56.571Z+ jobpid=1142
1142024-07-31T18:26:56.571Z+ echo 'Setup debug logging'
1152024-07-31T18:26:56.571ZSetup debug logging
1162024-07-31T18:26:56.571Z+ mkdir /tmp/debug
1172024-07-31T18:26:56.571Z+ sleep 10800
1182024-07-31T18:26:56.574Z+ psrinfo -v
1192024-07-31T18:26:56.576Z+ df -h
1202024-07-31T18:26:56.582Z+ prstat -d d -mLc 1
1212024-07-31T18:26:56.585Z+ iostat -T d -xn 1
1222024-07-31T18:26:56.585Z+ mpstat -T d 1
1232024-07-31T18:26:56.585Z+ vmstat -T d -p 1
1242024-07-31T18:26:56.588Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1252024-07-31T18:26:56.588Z+ banner replay
1262024-07-31T18:26:56.588Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1272024-07-31T18:26:56.588Z
1282024-07-31T18:26:56.588Z ##### ###### ##### # ## # #
1292024-07-31T18:26:56.588Z # # # # # # # # # #
1302024-07-31T18:26:56.588Z # # ##### # # # # # #
1312024-07-31T18:26:56.588Z ##### # ##### # ###### #
1322024-07-31T18:26:56.588Z # # # # # # # #
1332024-07-31T18:26:56.588Z # # ###### # ###### # # #
1342024-07-31T18:26:56.588Z
1352024-07-31T18:26:56.588Z+ ptime -m bash /input/build/work/scripts/test_replay.sh
1362024-07-31T18:26:56.633Zstarting July 31, 2024 at 06:26:55 PM UTC
1372024-07-31T18:26:56.636ZTail /tmp/test_replay.log for test output
1382024-07-31T18:26:56.639ZCreating downstairs regions
1392024-07-31T18:26:57.125ZStarting downstairs
1402024-07-31T18:27:02.152ZRunning initial fill
1412024-07-31T18:27:06.788ZReplay loop starts now July 31, 2024 at 06:27:05 PM UTC
1422024-07-31T18:34:18.619ZReplays:30 time: 7:12
1432024-07-31T18:34:18.623ZDo final verify
1442024-07-31T18:34:28.090ZJuly 31, 2024 at 06:34:26 PM UTC Test ends with 0
1452024-07-31T18:34:28.093Z
1462024-07-31T18:34:28.093Zreal 7:31.401614149
1472024-07-31T18:34:28.093Zuser 28.165464291
1482024-07-31T18:34:28.093Zsys 8.655295697
1492024-07-31T18:34:28.093Ztrap 0.070474405
1502024-07-31T18:34:28.093Ztflt 0.018540234
1512024-07-31T18:34:28.093Zdflt 0.015757385
1522024-07-31T18:34:28.093Zkflt 0.001375932
1532024-07-31T18:34:28.093Zlock 11:45:35.444876721
1542024-07-31T18:34:28.093Zslp 41:12.669331035
1552024-07-31T18:34:28.093Zlat 5.623215977
1562024-07-31T18:34:28.093Zstop 0.549992270
1572024-07-31T18:34:33.097Zprocess exited: duration 459021 ms, exit code 0
1582024-07-31T18:34:33.097Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
1592024-07-31T18:35:33.144Zexec warning: : stdout descriptor may be held open by a background process; giving up!
1602024-07-31T18:35:33.144Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
1612024-07-31T18:35:33.156Zfound 12 output files
1622024-07-31T18:35:33.156Zuploading: /tmp/test_replay.log (2124954 bytes)
1632024-07-31T18:35:34.201Zuploaded: /tmp/test_replay.log
1642024-07-31T18:35:34.206Zuploading: /tmp/debug/df.txt (1270 bytes)
1652024-07-31T18:35:35.213Zuploaded: /tmp/debug/df.txt
1662024-07-31T18:35:35.216Zuploading: /tmp/debug/iostat.txt (153560 bytes)
1672024-07-31T18:35:35.222Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 153560 -> 154154
1682024-07-31T18:35:36.228Zuploaded: /tmp/debug/iostat.txt
1692024-07-31T18:35:36.231Zuploading: /tmp/debug/mpstat.txt (375343 bytes)
1702024-07-31T18:35:36.241Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 375343 -> 377521
1712024-07-31T18:35:37.248Zuploaded: /tmp/debug/mpstat.txt
1722024-07-31T18:35:37.248Zuploading: /tmp/debug/paging.txt (66120 bytes)
1732024-07-31T18:35:37.256Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 66120 -> 66726
1742024-07-31T18:35:38.263Zuploaded: /tmp/debug/paging.txt
1752024-07-31T18:35:38.263Zuploading: /tmp/debug/perf.txt (979028 bytes)
1762024-07-31T18:35:39.285Zuploaded: /tmp/debug/perf.txt
1772024-07-31T18:35:39.285Zuploading: /tmp/debug/prstat.txt (707632 bytes)
1782024-07-31T18:35:39.307Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 707632 -> 715385
1792024-07-31T18:35:40.314Zuploaded: /tmp/debug/prstat.txt
1802024-07-31T18:35:40.314Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
1812024-07-31T18:35:41.324Zuploaded: /tmp/debug/psrinfo.txt
1822024-07-31T18:35:41.324Zuploading: /tmp/debug/upinfo.txt (81420 bytes)
1832024-07-31T18:35:42.337Zuploaded: /tmp/debug/upinfo.txt
1842024-07-31T18:35:42.337Zuploading: /tmp/dsc/downstairs-8810.txt (7817 bytes)
1852024-07-31T18:35:43.347Zuploaded: /tmp/dsc/downstairs-8810.txt
1862024-07-31T18:35:43.347Zuploading: /tmp/dsc/downstairs-8820.txt (7812 bytes)
1872024-07-31T18:35:44.361Zuploaded: /tmp/dsc/downstairs-8820.txt
1882024-07-31T18:35:44.364Zuploading: /tmp/dsc/downstairs-8830.txt (7814 bytes)
1892024-07-31T18:35:45.370Zuploaded: /tmp/dsc/downstairs-8830.txt