01HAZHJ54D212ENN8MBAH9Z5RH: test-live-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/*.log",
        "/tmp/core.*",
        "/tmp/dsc/*.txt",
        "/tmp/dsc.tar",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HAZHJETFH0SBMS33Y8V5G4QS

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12023-09-22T23:21:23.319Zjob dependencies complete; ready to run (waiting for 27 m 46 s)
22023-09-22T23:22:15.418Zjob assigned to worker 01HAZK5HYFY4J0MY7H9A294JJY (queued for 52 s)
32023-09-22T23:22:15.802Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42023-09-22T23:22:37.867Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52023-09-22T23:22:37.872Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62023-09-22T23:22:50.658Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72023-09-22T23:22:50.662Zdownloading input: /input/build/work/bins/crutest.gz
82023-09-22T23:23:15.410Zdownloaded input: /input/build/work/bins/crutest.gz
92023-09-22T23:23:15.413Zdownloading input: /input/build/work/bins/dsc.gz
102023-09-22T23:23:24.013Zdownloaded input: /input/build/work/bins/dsc.gz
112023-09-22T23:23:24.016Zdownloading input: /input/build/work/scripts/test_ds.sh
122023-09-22T23:23:24.027Zdownloaded input: /input/build/work/scripts/test_ds.sh
132023-09-22T23:23:24.030Zdownloading input: /input/build/work/scripts/test_live_repair.sh
142023-09-22T23:23:24.040Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
152023-09-22T23:23:24.043Zdownloading input: /input/build/work/scripts/test_repair.sh
162023-09-22T23:23:24.056Zdownloaded input: /input/build/work/scripts/test_repair.sh
172023-09-22T23:23:24.059Zdownloading input: /input/build/work/scripts/test_replay.sh
182023-09-22T23:23:24.090Zdownloaded input: /input/build/work/scripts/test_replay.sh
192023-09-22T23:23:24.093Zdownloading input: /input/build/work/scripts/test_up.sh
202023-09-22T23:23:24.103Zdownloaded input: /input/build/work/scripts/test_up.sh
212023-09-22T23:23:24.106Zdownloading input: /input/build/tmp/cargo-test-out.log
222023-09-22T23:23:25.171Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
232023-09-22T23:23:25.176Zstarting task 0: "setup"
242023-09-22T23:23:25.305Z++ uname -s
252023-09-22T23:23:25.312Z+ kern=SunOS
262023-09-22T23:23:25.317Z+ case "$kern" in
272023-09-22T23:23:25.323Z+ groupadd -g 12345 build
282023-09-22T23:23:25.330Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
292023-09-22T23:23:27.327Z+ zfs create -o mountpoint=/work rpool/work
302023-09-22T23:23:27.687Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
312023-09-22T23:23:27.701Z+ home_fs=zfs
322023-09-22T23:23:27.713Z+ [[ zfs == autofs ]]
332023-09-22T23:23:27.723Z+ mkdir -p /home/build
342023-09-22T23:23:27.735Z+ chown build:build /home/build /work
352023-09-22T23:23:28.687Z+ chmod 0700 /home/build /work
362023-09-22T23:23:28.690Zprocess exited: duration 3510 ms, exit code 0
 
372023-09-22T23:23:28.696Zstarting task 1: "authentication"
382023-09-22T23:23:28.724Zprocess exited: duration 24 ms, exit code 0
 
392023-09-22T23:23:28.730Zstarting task 2: "build"
402023-09-22T23:23:28.740Z+ banner cores
412023-09-22T23:23:28.777Z
422023-09-22T23:23:28.780Z #### #### ##### ###### ####
432023-09-22T23:23:28.783Z # # # # # # # #
442023-09-22T23:23:28.786Z # # # # # ##### ####
452023-09-22T23:23:28.789Z # # # ##### # #
462023-09-22T23:23:28.792Z # # # # # # # # #
472023-09-22T23:23:28.794Z #### #### # # ###### ####
482023-09-22T23:23:28.798Z
492023-09-22T23:23:28.801Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
502023-09-22T23:23:28.804Z+ pfexec chmod +x /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh
512023-09-22T23:23:28.807Z+ echo 'input bins dir contains:'
522023-09-22T23:23:28.830Zinput bins dir contains:
532023-09-22T23:23:28.833Z+ ls -ltr /input/build/work/bins
542023-09-22T23:23:28.836Ztotal 703421
552023-09-22T23:23:28.838Z-rw-r--r-- 1 root root 108909782 Sep 22 23:22 crucible-downstairs.gz
562023-09-22T23:23:28.841Z-rw-r--r-- 1 root root 86315868 Sep 22 23:22 crucible-hammer.gz
572023-09-22T23:23:28.844Z-rw-r--r-- 1 root root 96159197 Sep 22 23:23 crutest.gz
582023-09-22T23:23:28.847Z-rw-r--r-- 1 root root 68523274 Sep 22 23:23 dsc.gz
592023-09-22T23:23:28.850Z+ echo 'input script dir contains:'
602023-09-22T23:23:28.852Zinput script dir contains:
612023-09-22T23:23:28.855Z+ ls -ltr /input/build/work/scripts
622023-09-22T23:23:28.858Ztotal 32
632023-09-22T23:23:28.861Z-rwxr-xr-x 1 root root 939 Sep 22 23:23 test_ds.sh
642023-09-22T23:23:28.863Z-rwxr-xr-x 1 root root 3746 Sep 22 23:23 test_live_repair.sh
652023-09-22T23:23:28.866Z-rwxr-xr-x 1 root root 6416 Sep 22 23:23 test_repair.sh
662023-09-22T23:23:28.869Z-rwxr-xr-x 1 root root 2969 Sep 22 23:23 test_replay.sh
672023-09-22T23:23:28.872Z-rwxr-xr-x 1 root root 9311 Sep 22 23:23 test_up.sh
682023-09-22T23:23:28.875Z+ banner Unpack
692023-09-22T23:23:28.878Z# #
702023-09-22T23:23:28.881Z# # # # ##### ## #### # #
712023-09-22T23:23:28.883Z# # ## # # # # # # # # #
722023-09-22T23:23:28.886Z# # # # # # # # # # ####
732023-09-22T23:23:28.889Z# # # # # ##### ###### # # #
742023-09-22T23:23:28.893Z# # # ## # # # # # # #
752023-09-22T23:23:28.900Z ##### # # # # # #### # #
762023-09-22T23:23:28.908Z
772023-09-22T23:23:28.918Z+ mkdir -p /var/tmp/bins
782023-09-22T23:23:28.927Z+ for t in "$input/bins/"*.gz
792023-09-22T23:23:28.938Z++ basename /input/build/work/bins/crucible-downstairs.gz
802023-09-22T23:23:28.946Z+ b=crucible-downstairs.gz
812023-09-22T23:23:28.953Z+ b=crucible-downstairs
822023-09-22T23:23:28.957Z+ gunzip
832023-09-22T23:23:32.665Z+ chmod +x /var/tmp/bins/crucible-downstairs
842023-09-22T23:23:32.669Z+ for t in "$input/bins/"*.gz
852023-09-22T23:23:32.672Z++ basename /input/build/work/bins/crucible-hammer.gz
862023-09-22T23:23:32.675Z+ b=crucible-hammer.gz
872023-09-22T23:23:32.678Z+ b=crucible-hammer
882023-09-22T23:23:32.681Z+ gunzip
892023-09-22T23:23:35.646Z+ chmod +x /var/tmp/bins/crucible-hammer
902023-09-22T23:23:35.649Z+ for t in "$input/bins/"*.gz
912023-09-22T23:23:35.651Z++ basename /input/build/work/bins/crutest.gz
922023-09-22T23:23:35.654Z+ b=crutest.gz
932023-09-22T23:23:35.657Z+ b=crutest
942023-09-22T23:23:35.659Z+ gunzip
952023-09-22T23:23:38.997Z+ chmod +x /var/tmp/bins/crutest
962023-09-22T23:23:39.000Z+ for t in "$input/bins/"*.gz
972023-09-22T23:23:39.003Z++ basename /input/build/work/bins/dsc.gz
982023-09-22T23:23:39.006Z+ b=dsc.gz
992023-09-22T23:23:39.008Z+ b=dsc
1002023-09-22T23:23:39.011Z+ gunzip
1012023-09-22T23:23:41.328Z+ chmod +x /var/tmp/bins/dsc
1022023-09-22T23:23:41.332Z+ export BINDIR=/var/tmp/bins
1032023-09-22T23:23:41.335Z+ BINDIR=/var/tmp/bins
1042023-09-22T23:23:41.338Z+ echo 'BINDIR is /var/tmp/bins'
1052023-09-22T23:23:41.340ZBINDIR is /var/tmp/bins
1062023-09-22T23:23:41.343Z+ echo 'bindir contains:'
1072023-09-22T23:23:41.346Zbindir contains:
1082023-09-22T23:23:41.349Z+ ls -ltr /var/tmp/bins
1092023-09-22T23:23:41.352Ztotal 814333
1102023-09-22T23:23:41.354Z-rwxr-xr-x 1 build build 485759344 Sep 22 23:23 crucible-downstairs
1112023-09-22T23:23:41.357Z-rwxr-xr-x 1 build build 376521528 Sep 22 23:23 crucible-hammer
1122023-09-22T23:23:41.360Z-rwxr-xr-x 1 build build 415889520 Sep 22 23:23 crutest
1132023-09-22T23:23:41.363Z-rwxr-xr-x 1 build build 299073424 Sep 22 23:23 dsc
1142023-09-22T23:23:41.387Z+ banner CreateDS
1152023-09-22T23:23:41.390Z ##### ###### #####
1162023-09-22T23:23:41.392Z# # ##### ###### ## ##### ###### # # # #
1172023-09-22T23:23:41.395Z# # # # # # # # # # #
1182023-09-22T23:23:41.397Z# # # ##### # # # ##### # # #####
1192023-09-22T23:23:41.400Z# ##### # ###### # # # # #
1202023-09-22T23:23:41.403Z# # # # # # # # # # # # #
1212023-09-22T23:23:41.406Z ##### # # ###### # # # ###### ###### #####
1222023-09-22T23:23:41.408Z
1232023-09-22T23:23:41.411Z+ echo /var/tmp/bins/dsc create --ds-bin /var/tmp/bins/crucible-downstairs --region-count 4 --cleanup
1242023-09-22T23:23:41.414Z/var/tmp/bins/dsc create --ds-bin /var/tmp/bins/crucible-downstairs --region-count 4 --cleanup
1252023-09-22T23:23:41.416Z+ /var/tmp/bins/dsc create --ds-bin /var/tmp/bins/crucible-downstairs --region-count 4 --cleanup
1262023-09-22T23:23:41.419ZCreating region directory at: /var/tmp/dsc/region
1272023-09-22T23:23:41.422ZCreating dsc directory at: /tmp/dsc
1282023-09-22T23:23:41.618ZDownstairs region 0 created at /var/tmp/dsc/region/8810 in 0.25805306
1292023-09-22T23:23:41.669ZDownstairs region 1 created at /var/tmp/dsc/region/8820 in 0.050465107
1302023-09-22T23:23:41.719ZDownstairs region 2 created at /var/tmp/dsc/region/8830 in 0.050475303
1312023-09-22T23:23:41.770ZDownstairs region 3 created at /var/tmp/dsc/region/8840 in 0.051190797
1322023-09-22T23:23:41.773ZRegion set with 4 regions was created
1332023-09-22T23:23:41.776Z+ banner StartDS
1342023-09-22T23:23:41.779Z ##### ###### #####
1352023-09-22T23:23:41.782Z# # ##### ## ##### ##### # # # #
1362023-09-22T23:23:41.788Z# # # # # # # # # #
1372023-09-22T23:23:41.791Z ##### # # # # # # # # #####
1382023-09-22T23:23:41.794Z # # ###### ##### # # # #
1392023-09-22T23:23:41.797Z# # # # # # # # # # # #
1402023-09-22T23:23:41.800Z ##### # # # # # # ###### #####
1412023-09-22T23:23:41.804Z
1422023-09-22T23:23:41.807Z+ sleep 5
1432023-09-22T23:23:41.809Z+ /var/tmp/bins/dsc start --ds-bin /var/tmp/bins/crucible-downstairs --region-count 4
1442023-09-22T23:23:46.787Z++ pgrep dsc
1452023-09-22T23:23:46.795Z+ dsc_pid=1154
1462023-09-22T23:23:46.799Z+ [[ 1154 -eq 0 ]]
1472023-09-22T23:23:46.808Z+ ps -p 1154
1482023-09-22T23:23:46.828Z PID TTY TIME CMD
1492023-09-22T23:23:46.833Z 1154 ? 0:00 dsc
1502023-09-22T23:23:46.838Z+ echo 'Found dsc running, continue tests'
1512023-09-22T23:23:46.844ZFound dsc running, continue tests
1522023-09-22T23:23:46.849Z+ banner LR
1532023-09-22T23:23:46.855Z# ######
1542023-09-22T23:23:46.864Z# # #
1552023-09-22T23:23:46.874Z# # #
1562023-09-22T23:23:46.884Z# ######
1572023-09-22T23:23:46.905Z# # #
1582023-09-22T23:23:46.913Z# # #
1592023-09-22T23:23:46.919Z####### # #
1602023-09-22T23:23:46.925Z
1612023-09-22T23:23:46.931Z+ ptime -m /var/tmp/bins/crutest replace -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --replacement 127.0.0.1:8840 -g 1 -c 10 --stable
1622023-09-22T23:25:52.996Z+ banner StopDSC
1632023-09-22T23:25:52.999Z ##### ###### ##### #####
1642023-09-22T23:25:53.002Z# # ##### #### ##### # # # # # #
1652023-09-22T23:25:53.005Z# # # # # # # # # #
1662023-09-22T23:25:53.008Z ##### # # # # # # # ##### #
1672023-09-22T23:25:53.011Z # # # # ##### # # # #
1682023-09-22T23:25:53.014Z# # # # # # # # # # # #
1692023-09-22T23:25:53.017Z ##### # #### # ###### ##### #####
1702023-09-22T23:25:53.020Z
1712023-09-22T23:25:53.023Z+ /var/tmp/bins/dsc cmd shutdown
1722023-09-22T23:25:53.109Z+ banner WaitStop
1732023-09-22T23:25:53.112Z# # #####
1742023-09-22T23:25:53.116Z# # # ## # ##### # # ##### #### #####
1752023-09-22T23:25:53.118Z# # # # # # # # # # # # #
1762023-09-22T23:25:53.121Z# # # # # # # ##### # # # # #
1772023-09-22T23:25:53.124Z# # # ###### # # # # # # #####
1782023-09-22T23:25:53.127Z# # # # # # # # # # # # #
1792023-09-22T23:25:53.130Z ## ## # # # # ##### # #### #
1802023-09-22T23:25:53.133Z
1812023-09-22T23:25:53.135Z+ wait 1154
1822023-09-22T23:25:54.108Z+ _exit_trap
1832023-09-22T23:25:54.111Z+ local status=0
1842023-09-22T23:25:54.114Z+ [[ 0 -eq 0 ]]
1852023-09-22T23:25:54.117Z+ exit 0
1862023-09-22T23:25:54.120Zprocess exited: duration 145381 ms, exit code 0
 
1872023-09-22T23:25:54.125Zfound 6 output files
1882023-09-22T23:25:54.135Zuploading: /tmp/crutest-replace.log (1406779 bytes)
1892023-09-22T23:25:56.390Zuploaded: /tmp/crutest-replace.log
1902023-09-22T23:25:56.397Zuploading: /tmp/dsc.log (1991 bytes)
1912023-09-22T23:25:57.406Zuploaded: /tmp/dsc.log
1922023-09-22T23:25:57.409Zuploading: /tmp/dsc/downstairs-8810.txt (169308 bytes)
1932023-09-22T23:25:58.422Zuploaded: /tmp/dsc/downstairs-8810.txt
1942023-09-22T23:25:58.424Zuploading: /tmp/dsc/downstairs-8820.txt (126340 bytes)
1952023-09-22T23:25:59.437Zuploaded: /tmp/dsc/downstairs-8820.txt
1962023-09-22T23:25:59.440Zuploading: /tmp/dsc/downstairs-8830.txt (150794 bytes)
1972023-09-22T23:26:00.454Zuploaded: /tmp/dsc/downstairs-8830.txt
1982023-09-22T23:26:00.457Zuploading: /tmp/dsc/downstairs-8840.txt (150005 bytes)
1992023-09-22T23:26:01.474Zuploaded: /tmp/dsc/downstairs-8840.txt