01HCKKW0XS4H876EJ907GNRDQF: test-live-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/*.log",
        "/tmp/core.*",
        "/tmp/dsc/*.txt",
        "/tmp/dsc.tar",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HCKKY5Y002CD2GQZEH04AYTV

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12023-10-13T04:38:17.696Zjob dependencies complete; ready to run (waiting for 22 m 49 s)
22023-10-13T04:39:27.943Zjob assigned to worker 01HCKN8883A1G23P0SQV009ERB (queued for 1 m 10 s)
32023-10-13T04:39:28.436Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42023-10-13T04:39:52.405Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52023-10-13T04:39:52.415Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62023-10-13T04:40:12.524Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72023-10-13T04:40:12.544Zdownloading input: /input/build/work/bins/crutest.gz
82023-10-13T04:40:37.262Zdownloaded input: /input/build/work/bins/crutest.gz
92023-10-13T04:40:37.269Zdownloading input: /input/build/work/bins/dsc.gz
102023-10-13T04:40:50.762Zdownloaded input: /input/build/work/bins/dsc.gz
112023-10-13T04:40:50.781Zdownloading input: /input/build/work/scripts/test_ds.sh
122023-10-13T04:40:50.798Zdownloaded input: /input/build/work/scripts/test_ds.sh
132023-10-13T04:40:50.814Zdownloading input: /input/build/work/scripts/test_live_repair.sh
142023-10-13T04:40:50.827Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
152023-10-13T04:40:50.844Zdownloading input: /input/build/work/scripts/test_repair.sh
162023-10-13T04:40:50.858Zdownloaded input: /input/build/work/scripts/test_repair.sh
172023-10-13T04:40:50.872Zdownloading input: /input/build/work/scripts/test_replay.sh
182023-10-13T04:40:50.946Zdownloaded input: /input/build/work/scripts/test_replay.sh
192023-10-13T04:40:50.960Zdownloading input: /input/build/work/scripts/test_up.sh
202023-10-13T04:40:51.043Zdownloaded input: /input/build/work/scripts/test_up.sh
212023-10-13T04:40:51.057Zdownloading input: /input/build/tmp/cargo-test-out.log
222023-10-13T04:40:52.163Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
232023-10-13T04:40:52.168Zstarting task 0: "setup"
242023-10-13T04:40:52.392Z++ uname -s
252023-10-13T04:40:52.413Z+ kern=SunOS
262023-10-13T04:40:52.428Z+ case "$kern" in
272023-10-13T04:40:52.437Z+ groupadd -g 12345 build
282023-10-13T04:40:52.447Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
292023-10-13T04:40:54.397Z+ zfs create -o mountpoint=/work rpool/work
302023-10-13T04:40:54.667Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
312023-10-13T04:40:54.675Z+ home_fs=zfs
322023-10-13T04:40:54.681Z+ [[ zfs == autofs ]]
332023-10-13T04:40:54.689Z+ mkdir -p /home/build
342023-10-13T04:40:54.699Z+ chown build:build /home/build /work
352023-10-13T04:40:56.671Z+ chmod 0700 /home/build /work
362023-10-13T04:40:56.678Zprocess exited: duration 4496 ms, exit code 0
 
372023-10-13T04:40:56.692Zstarting task 1: "authentication"
382023-10-13T04:40:56.723Zprocess exited: duration 26 ms, exit code 0
 
392023-10-13T04:40:56.731Zstarting task 2: "build"
402023-10-13T04:40:56.745Z+ banner cores
412023-10-13T04:40:57.282Z
422023-10-13T04:40:57.293Z #### #### ##### ###### ####
432023-10-13T04:40:57.300Z # # # # # # # #
442023-10-13T04:40:57.312Z # # # # # ##### ####
452023-10-13T04:40:57.326Z # # # ##### # #
462023-10-13T04:40:57.338Z # # # # # # # # #
472023-10-13T04:40:57.764Z #### #### # # ###### ####
482023-10-13T04:40:57.779Z
492023-10-13T04:40:58.203Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
502023-10-13T04:40:58.217Z+ pfexec chmod +x /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh
512023-10-13T04:40:58.227Z+ echo 'input bins dir contains:'
522023-10-13T04:40:58.662Zinput bins dir contains:
532023-10-13T04:40:58.669Z+ ls -ltr /input/build/work/bins
542023-10-13T04:40:59.097Ztotal 697495
552023-10-13T04:40:59.112Z-rw-r--r-- 1 root root 107725299 Oct 13 04:39 crucible-downstairs.gz
562023-10-13T04:40:59.119Z-rw-r--r-- 1 root root 85722441 Oct 13 04:40 crucible-hammer.gz
572023-10-13T04:40:59.132Z-rw-r--r-- 1 root root 95189178 Oct 13 04:40 crutest.gz
582023-10-13T04:40:59.141Z-rw-r--r-- 1 root root 68233986 Oct 13 04:40 dsc.gz
592023-10-13T04:40:59.154Z+ echo 'input script dir contains:'
602023-10-13T04:40:59.168Zinput script dir contains:
612023-10-13T04:40:59.185Z+ ls -ltr /input/build/work/scripts
622023-10-13T04:40:59.200Ztotal 32
632023-10-13T04:40:59.220Z-rwxr-xr-x 1 root root 939 Oct 13 04:40 test_ds.sh
642023-10-13T04:40:59.251Z-rwxr-xr-x 1 root root 3746 Oct 13 04:40 test_live_repair.sh
652023-10-13T04:40:59.272Z-rwxr-xr-x 1 root root 6416 Oct 13 04:40 test_repair.sh
662023-10-13T04:40:59.288Z-rwxr-xr-x 1 root root 2969 Oct 13 04:40 test_replay.sh
672023-10-13T04:40:59.310Z-rwxr-xr-x 1 root root 9311 Oct 13 04:40 test_up.sh
682023-10-13T04:40:59.732Z+ banner Unpack
692023-10-13T04:40:59.745Z# #
702023-10-13T04:41:00.167Z# # # # ##### ## #### # #
712023-10-13T04:41:00.600Z# # ## # # # # # # # # #
722023-10-13T04:41:00.620Z# # # # # # # # # # ####
732023-10-13T04:41:00.637Z# # # # # ##### ###### # # #
742023-10-13T04:41:00.653Z# # # ## # # # # # # #
752023-10-13T04:41:00.673Z ##### # # # # # #### # #
762023-10-13T04:41:01.092Z
772023-10-13T04:41:01.101Z+ mkdir -p /var/tmp/bins
782023-10-13T04:41:01.110Z+ for t in "$input/bins/"*.gz
792023-10-13T04:41:01.121Z++ basename /input/build/work/bins/crucible-downstairs.gz
802023-10-13T04:41:01.138Z+ b=crucible-downstairs.gz
812023-10-13T04:41:01.150Z+ b=crucible-downstairs
822023-10-13T04:41:01.160Z+ gunzip
832023-10-13T04:41:01.171Z+ chmod +x /var/tmp/bins/crucible-downstairs
842023-10-13T04:41:01.181Z+ for t in "$input/bins/"*.gz
852023-10-13T04:41:01.192Z++ basename /input/build/work/bins/crucible-hammer.gz
862023-10-13T04:41:01.209Z+ b=crucible-hammer.gz
872023-10-13T04:41:01.222Z+ b=crucible-hammer
882023-10-13T04:41:01.234Z+ gunzip
892023-10-13T04:41:03.689Z+ chmod +x /var/tmp/bins/crucible-hammer
902023-10-13T04:41:03.746Z+ for t in "$input/bins/"*.gz
912023-10-13T04:41:03.776Z++ basename /input/build/work/bins/crutest.gz
922023-10-13T04:41:03.800Z+ b=crutest.gz
932023-10-13T04:41:04.237Z+ b=crutest
942023-10-13T04:41:04.673Z+ gunzip
952023-10-13T04:41:06.962Z+ chmod +x /var/tmp/bins/crutest
962023-10-13T04:41:06.975Z+ for t in "$input/bins/"*.gz
972023-10-13T04:41:06.993Z++ basename /input/build/work/bins/dsc.gz
982023-10-13T04:41:07.013Z+ b=dsc.gz
992023-10-13T04:41:07.021Z+ b=dsc
1002023-10-13T04:41:07.025Z+ gunzip
1012023-10-13T04:41:09.341Z+ chmod +x /var/tmp/bins/dsc
1022023-10-13T04:41:09.349Z+ export BINDIR=/var/tmp/bins
1032023-10-13T04:41:09.356Z+ BINDIR=/var/tmp/bins
1042023-10-13T04:41:09.360Z+ echo 'BINDIR is /var/tmp/bins'
1052023-10-13T04:41:09.367ZBINDIR is /var/tmp/bins
1062023-10-13T04:41:09.372Z+ echo 'bindir contains:'
1072023-10-13T04:41:09.376Zbindir contains:
1082023-10-13T04:41:09.384Z+ ls -ltr /var/tmp/bins
1092023-10-13T04:41:09.391Ztotal 702268
1102023-10-13T04:41:09.396Z-rwxr-xr-x 1 build build 481779120 Oct 13 04:40 crucible-downstairs
1112023-10-13T04:41:09.407Z-rwxr-xr-x 1 build build 374855496 Oct 13 04:41 crucible-hammer
1122023-10-13T04:41:09.420Z-rwxr-xr-x 1 build build 412880624 Oct 13 04:41 crutest
1132023-10-13T04:41:09.438Z-rwxr-xr-x 1 build build 298093480 Oct 13 04:41 dsc
1142023-10-13T04:41:09.453Z+ banner CreateDS
1152023-10-13T04:41:09.476Z ##### ###### #####
1162023-10-13T04:41:09.489Z# # ##### ###### ## ##### ###### # # # #
1172023-10-13T04:41:09.508Z# # # # # # # # # # #
1182023-10-13T04:41:09.520Z# # # ##### # # # ##### # # #####
1192023-10-13T04:41:09.533Z# ##### # ###### # # # # #
1202023-10-13T04:41:09.550Z# # # # # # # # # # # # #
1212023-10-13T04:41:09.563Z ##### # # ###### # # # ###### ###### #####
1222023-10-13T04:41:09.577Z
1232023-10-13T04:41:09.606Z+ echo /var/tmp/bins/dsc create --ds-bin /var/tmp/bins/crucible-downstairs --region-count 4 --cleanup
1242023-10-13T04:41:09.616Z/var/tmp/bins/dsc create --ds-bin /var/tmp/bins/crucible-downstairs --region-count 4 --cleanup
1252023-10-13T04:41:09.633Z+ /var/tmp/bins/dsc create --ds-bin /var/tmp/bins/crucible-downstairs --region-count 4 --cleanup
1262023-10-13T04:41:09.645ZCreating region directory at: /var/tmp/dsc/region
1272023-10-13T04:41:09.660ZCreating dsc directory at: /tmp/dsc
1282023-10-13T04:41:09.693ZDownstairs region 0 created at /var/tmp/dsc/region/8810 in 0.31319597
1292023-10-13T04:41:09.804ZDownstairs region 1 created at /var/tmp/dsc/region/8820 in 0.0893429
1302023-10-13T04:41:10.239ZDownstairs region 2 created at /var/tmp/dsc/region/8830 in 0.08485477
1312023-10-13T04:41:10.660ZDownstairs region 3 created at /var/tmp/dsc/region/8840 in 0.07347387
1322023-10-13T04:41:11.093ZRegion set with 4 regions was created
1332023-10-13T04:41:11.098Z+ banner StartDS
1342023-10-13T04:41:11.105Z ##### ###### #####
1352023-10-13T04:41:11.530Z# # ##### ## ##### ##### # # # #
1362023-10-13T04:41:11.565Z# # # # # # # # # #
1372023-10-13T04:41:11.582Z ##### # # # # # # # # #####
1382023-10-13T04:41:11.597Z # # ###### ##### # # # #
1392023-10-13T04:41:11.607Z# # # # # # # # # # # #
1402023-10-13T04:41:13.292Z ##### # # # # # # ###### #####
1412023-10-13T04:41:13.829Z
1422023-10-13T04:41:13.843Z+ sleep 5
1432023-10-13T04:41:13.859Z+ /var/tmp/bins/dsc start --ds-bin /var/tmp/bins/crucible-downstairs --region-count 4
1442023-10-13T04:41:14.952Z++ pgrep dsc
1452023-10-13T04:41:14.959Z+ dsc_pid=1154
1462023-10-13T04:41:14.974Z+ [[ 1154 -eq 0 ]]
1472023-10-13T04:41:14.989Z+ ps -p 1154
1482023-10-13T04:41:15.006Z PID TTY TIME CMD
1492023-10-13T04:41:15.019Z 1154 ? 0:00 dsc
1502023-10-13T04:41:15.035Z+ echo 'Found dsc running, continue tests'
1512023-10-13T04:41:15.056ZFound dsc running, continue tests
1522023-10-13T04:41:15.068Z+ banner LR
1532023-10-13T04:41:15.078Z# ######
1542023-10-13T04:41:15.093Z# # #
1552023-10-13T04:41:15.132Z# # #
1562023-10-13T04:41:15.145Z# ######
1572023-10-13T04:41:15.156Z# # #
1582023-10-13T04:41:15.167Z# # #
1592023-10-13T04:41:15.181Z####### # #
1602023-10-13T04:41:15.192Z
1612023-10-13T04:41:15.204Z+ ptime -m /var/tmp/bins/crutest replace -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --replacement 127.0.0.1:8840 -g 1 -c 10 --stable
1622023-10-13T04:43:13.259Z+ banner StopDSC
1632023-10-13T04:43:13.272Z ##### ###### ##### #####
1642023-10-13T04:43:13.287Z# # ##### #### ##### # # # # # #
1652023-10-13T04:43:13.304Z# # # # # # # # # #
1662023-10-13T04:43:13.732Z ##### # # # # # # # ##### #
1672023-10-13T04:43:13.752Z # # # # ##### # # # #
1682023-10-13T04:43:13.766Z# # # # # # # # # # # #
1692023-10-13T04:43:13.773Z ##### # #### # ###### ##### #####
1702023-10-13T04:43:13.788Z
1712023-10-13T04:43:13.800Z+ /var/tmp/bins/dsc cmd shutdown
1722023-10-13T04:43:13.804Z+ banner WaitStop
1732023-10-13T04:43:13.809Z# # #####
1742023-10-13T04:43:13.817Z# # # ## # ##### # # ##### #### #####
1752023-10-13T04:43:13.823Z# # # # # # # # # # # # #
1762023-10-13T04:43:13.834Z# # # # # # # ##### # # # # #
1772023-10-13T04:43:13.842Z# # # ###### # # # # # # #####
1782023-10-13T04:43:13.849Z# # # # # # # # # # # # #
1792023-10-13T04:43:13.860Z ## ## # # # # ##### # #### #
1802023-10-13T04:43:13.865Z
1812023-10-13T04:43:13.870Z+ wait 1154
1822023-10-13T04:43:14.376Z+ _exit_trap
1832023-10-13T04:43:14.384Z+ local status=0
1842023-10-13T04:43:14.393Z+ [[ 0 -eq 0 ]]
1852023-10-13T04:43:14.400Z+ exit 0
1862023-10-13T04:43:14.412Zprocess exited: duration 137640 ms, exit code 0
 
1872023-10-13T04:43:14.496Zfound 6 output files
1882023-10-13T04:43:14.530Zuploading: /tmp/crutest-replace.log (1417630 bytes)
1892023-10-13T04:43:15.703Zuploaded: /tmp/crutest-replace.log
1902023-10-13T04:43:15.710Zuploading: /tmp/dsc.log (1991 bytes)
1912023-10-13T04:43:16.749Zuploaded: /tmp/dsc.log
1922023-10-13T04:43:16.757Zuploading: /tmp/dsc/downstairs-8810.txt (130009 bytes)
1932023-10-13T04:43:17.808Zuploaded: /tmp/dsc/downstairs-8810.txt
1942023-10-13T04:43:17.826Zuploading: /tmp/dsc/downstairs-8820.txt (88043 bytes)
1952023-10-13T04:43:18.969Zuploaded: /tmp/dsc/downstairs-8820.txt
1962023-10-13T04:43:18.981Zuploading: /tmp/dsc/downstairs-8830.txt (109301 bytes)
1972023-10-13T04:43:20.026Zuploaded: /tmp/dsc/downstairs-8830.txt
1982023-10-13T04:43:20.035Zuploading: /tmp/dsc/downstairs-8840.txt (107125 bytes)
1992023-10-13T04:43:21.072Zuploaded: /tmp/dsc/downstairs-8840.txt