|
|
|
1 | 2024-08-05T19:07:57.800Z | job dependencies complete; ready to run (waiting for 19 m 59 s) |
2 | 2024-08-05T19:08:02.840Z | job assigned to worker 01J4HZ05AKZNW4W8HBSH083TZY (queued for 5 s) |
3 | 2024-08-05T19:08:10.279Z | downloading input: /input/build/work/bins/crucible-downstairs.gz |
4 | 2024-08-05T19:08:11.532Z | downloaded input: /input/build/work/bins/crucible-downstairs.gz |
5 | 2024-08-05T19:08:11.532Z | downloading input: /input/build/work/bins/crucible-hammer.gz |
6 | 2024-08-05T19:08:12.398Z | downloaded input: /input/build/work/bins/crucible-hammer.gz |
7 | 2024-08-05T19:08:12.398Z | downloading input: /input/build/work/bins/crutest.gz |
8 | 2024-08-05T19:08:13.453Z | downloaded input: /input/build/work/bins/crutest.gz |
9 | 2024-08-05T19:08:13.454Z | downloading input: /input/build/work/bins/dsc.gz |
10 | 2024-08-05T19:08:14.301Z | downloaded input: /input/build/work/bins/dsc.gz |
11 | 2024-08-05T19:08:14.305Z | downloading input: /input/build/work/scripts/perf-downstairs-tick.d |
12 | 2024-08-05T19:08:14.322Z | downloaded input: /input/build/work/scripts/perf-downstairs-tick.d |
13 | 2024-08-05T19:08:14.322Z | downloading input: /input/build/work/scripts/test_ds.sh |
14 | 2024-08-05T19:08:14.343Z | downloaded input: /input/build/work/scripts/test_ds.sh |
15 | 2024-08-05T19:08:14.343Z | downloading input: /input/build/work/scripts/test_live_repair.sh |
16 | 2024-08-05T19:08:14.363Z | downloaded input: /input/build/work/scripts/test_live_repair.sh |
17 | 2024-08-05T19:08:14.363Z | downloading input: /input/build/work/scripts/test_repair.sh |
18 | 2024-08-05T19:08:14.383Z | downloaded input: /input/build/work/scripts/test_repair.sh |
19 | 2024-08-05T19:08:14.384Z | downloading input: /input/build/work/scripts/test_replay.sh |
20 | 2024-08-05T19:08:14.404Z | downloaded input: /input/build/work/scripts/test_replay.sh |
21 | 2024-08-05T19:08:14.405Z | downloading input: /input/build/work/scripts/test_up.sh |
22 | 2024-08-05T19:08:14.425Z | downloaded input: /input/build/work/scripts/test_up.sh |
23 | 2024-08-05T19:08:14.425Z | downloading input: /input/build/work/scripts/upstairs_info.d |
24 | 2024-08-05T19:08:14.446Z | downloaded input: /input/build/work/scripts/upstairs_info.d |
25 | 2024-08-05T19:08:14.449Z | downloading input: /input/build/tmp/cargo-test-out.log |
26 | 2024-08-05T19:08:15.346Z | downloaded input: /input/build/tmp/cargo-test-out.log |
|
27 | 2024-08-05T19:08:15.346Z | starting task 0: "setup" |
28 | 2024-08-05T19:08:15.402Z | ++ uname -s |
29 | 2024-08-05T19:08:15.405Z | + kern=SunOS |
30 | 2024-08-05T19:08:15.405Z | + case "$kern" in |
31 | 2024-08-05T19:08:15.405Z | + groupadd -g 12345 build |
32 | 2024-08-05T19:08:15.405Z | + useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build |
33 | 2024-08-05T19:08:17.414Z | + zfs create -o mountpoint=/work rpool/work |
34 | 2024-08-05T19:08:17.626Z | ++ awk '$2 == "/home" { print $3 }' /etc/mnttab |
35 | 2024-08-05T19:08:17.629Z | + home_fs=zfs |
36 | 2024-08-05T19:08:17.629Z | + [[ zfs == autofs ]] |
37 | 2024-08-05T19:08:17.629Z | + mkdir -p /home/build |
38 | 2024-08-05T19:08:17.632Z | + chown build:build /home/build /work |
39 | 2024-08-05T19:08:19.632Z | + chmod 0700 /home/build /work |
40 | 2024-08-05T19:08:19.636Z | process exited: duration 4288 ms, exit code 0 |
|
41 | 2024-08-05T19:08:19.643Z | starting task 1: "authentication" |
42 | 2024-08-05T19:08:19.660Z | process exited: duration 16 ms, exit code 0 |
|
43 | 2024-08-05T19:08:19.667Z | starting task 2: "build" |
44 | 2024-08-05T19:08:19.670Z | + banner cores |
45 | 2024-08-05T19:08:19.690Z | |
46 | 2024-08-05T19:08:19.690Z | #### #### ##### ###### #### |
47 | 2024-08-05T19:08:19.690Z | # # # # # # # # |
48 | 2024-08-05T19:08:19.690Z | # # # # # ##### #### |
49 | 2024-08-05T19:08:19.690Z | # # # ##### # # |
50 | 2024-08-05T19:08:19.690Z | # # # # # # # # # |
51 | 2024-08-05T19:08:19.690Z | #### #### # # ###### #### |
52 | 2024-08-05T19:08:19.690Z | |
53 | 2024-08-05T19:08:19.693Z | + pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid |
54 | 2024-08-05T19:08:19.700Z | + pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d |
55 | 2024-08-05T19:08:19.703Z | + echo 'input bins dir contains:' |
56 | 2024-08-05T19:08:19.703Z | input bins dir contains: |
57 | 2024-08-05T19:08:19.703Z | + ls -ltr /input/build/work/bins |
58 | 2024-08-05T19:08:19.706Z | total 697383 |
59 | 2024-08-05T19:08:19.706Z | -rw-r--r-- 1 root root 105756823 Aug 5 19:08 crucible-downstairs.gz |
60 | 2024-08-05T19:08:19.706Z | -rw-r--r-- 1 root root 83355941 Aug 5 19:08 crucible-hammer.gz |
61 | 2024-08-05T19:08:19.706Z | -rw-r--r-- 1 root root 103298177 Aug 5 19:08 crutest.gz |
62 | 2024-08-05T19:08:19.706Z | -rw-r--r-- 1 root root 64414616 Aug 5 19:08 dsc.gz |
63 | 2024-08-05T19:08:19.706Z | + echo 'input script dir contains:' |
64 | 2024-08-05T19:08:19.706Z | input script dir contains: |
65 | 2024-08-05T19:08:19.706Z | + ls -ltr /input/build/work/scripts |
66 | 2024-08-05T19:08:19.709Z | total 40 |
67 | 2024-08-05T19:08:19.709Z | -rwxr-xr-x 1 root root 3056 Aug 5 19:08 perf-downstairs-tick.d |
68 | 2024-08-05T19:08:19.709Z | -rwxr-xr-x 1 root root 2102 Aug 5 19:08 test_ds.sh |
69 | 2024-08-05T19:08:19.709Z | -rwxr-xr-x 1 root root 4086 Aug 5 19:08 test_live_repair.sh |
70 | 2024-08-05T19:08:19.709Z | -rwxr-xr-x 1 root root 6671 Aug 5 19:08 test_repair.sh |
71 | 2024-08-05T19:08:19.709Z | -rwxr-xr-x 1 root root 3029 Aug 5 19:08 test_replay.sh |
72 | 2024-08-05T19:08:19.709Z | -rwxr-xr-x 1 root root 11934 Aug 5 19:08 test_up.sh |
73 | 2024-08-05T19:08:19.709Z | -rwxr-xr-x 1 root root 2750 Aug 5 19:08 upstairs_info.d |
74 | 2024-08-05T19:08:19.709Z | + banner Unpack |
75 | 2024-08-05T19:08:19.709Z | # # |
76 | 2024-08-05T19:08:19.709Z | # # # # ##### ## #### # # |
77 | 2024-08-05T19:08:19.709Z | # # ## # # # # # # # # # |
78 | 2024-08-05T19:08:19.709Z | # # # # # # # # # # #### |
79 | 2024-08-05T19:08:19.709Z | # # # # # ##### ###### # # # |
80 | 2024-08-05T19:08:19.709Z | # # # ## # # # # # # # |
81 | 2024-08-05T19:08:19.709Z | ##### # # # # # #### # # |
82 | 2024-08-05T19:08:19.709Z | |
83 | 2024-08-05T19:08:19.709Z | + mkdir -p /var/tmp/bins |
84 | 2024-08-05T19:08:19.712Z | + for t in "$input/bins/"*.gz |
85 | 2024-08-05T19:08:19.712Z | ++ basename /input/build/work/bins/crucible-downstairs.gz |
86 | 2024-08-05T19:08:19.712Z | + b=crucible-downstairs.gz |
87 | 2024-08-05T19:08:19.715Z | + b=crucible-downstairs |
88 | 2024-08-05T19:08:19.715Z | + gunzip |
89 | 2024-08-05T19:08:21.925Z | + chmod +x /var/tmp/bins/crucible-downstairs |
90 | 2024-08-05T19:08:21.928Z | + for t in "$input/bins/"*.gz |
91 | 2024-08-05T19:08:21.930Z | ++ basename /input/build/work/bins/crucible-hammer.gz |
92 | 2024-08-05T19:08:21.931Z | + b=crucible-hammer.gz |
93 | 2024-08-05T19:08:21.931Z | + b=crucible-hammer |
94 | 2024-08-05T19:08:21.931Z | + gunzip |
95 | 2024-08-05T19:08:23.686Z | + chmod +x /var/tmp/bins/crucible-hammer |
96 | 2024-08-05T19:08:23.689Z | + for t in "$input/bins/"*.gz |
97 | 2024-08-05T19:08:23.689Z | ++ basename /input/build/work/bins/crutest.gz |
98 | 2024-08-05T19:08:23.692Z | + b=crutest.gz |
99 | 2024-08-05T19:08:23.692Z | + b=crutest |
100 | 2024-08-05T19:08:23.692Z | + gunzip |
101 | 2024-08-05T19:08:25.874Z | + chmod +x /var/tmp/bins/crutest |
102 | 2024-08-05T19:08:25.878Z | + for t in "$input/bins/"*.gz |
103 | 2024-08-05T19:08:25.878Z | ++ basename /input/build/work/bins/dsc.gz |
104 | 2024-08-05T19:08:25.878Z | + b=dsc.gz |
105 | 2024-08-05T19:08:25.878Z | + b=dsc |
106 | 2024-08-05T19:08:25.879Z | + gunzip |
107 | 2024-08-05T19:08:27.274Z | + chmod +x /var/tmp/bins/dsc |
108 | 2024-08-05T19:08:27.277Z | + export BINDIR=/var/tmp/bins |
109 | 2024-08-05T19:08:27.278Z | + BINDIR=/var/tmp/bins |
110 | 2024-08-05T19:08:27.278Z | + echo 'BINDIR is /var/tmp/bins' |
111 | 2024-08-05T19:08:27.278Z | BINDIR is /var/tmp/bins |
112 | 2024-08-05T19:08:27.278Z | + echo 'bindir contains:' |
113 | 2024-08-05T19:08:27.278Z | bindir contains: |
114 | 2024-08-05T19:08:27.278Z | + ls -ltr /var/tmp/bins |
115 | 2024-08-05T19:08:27.281Z | total 671561 |
116 | 2024-08-05T19:08:27.281Z | -rwxr-xr-x 1 build build 401915544 Aug 5 19:08 crucible-downstairs |
117 | 2024-08-05T19:08:27.281Z | -rwxr-xr-x 1 build build 312479504 Aug 5 19:08 crucible-hammer |
118 | 2024-08-05T19:08:27.281Z | -rwxr-xr-x 1 build build 390929528 Aug 5 19:08 crutest |
119 | 2024-08-05T19:08:27.281Z | -rwxr-xr-x 1 build build 240974112 Aug 5 19:08 dsc |
120 | 2024-08-05T19:08:27.281Z | + banner CreateDS |
121 | 2024-08-05T19:08:27.281Z | ##### ###### ##### |
122 | 2024-08-05T19:08:27.281Z | # # ##### ###### ## ##### ###### # # # # |
123 | 2024-08-05T19:08:27.281Z | # # # # # # # # # # # |
124 | 2024-08-05T19:08:27.281Z | # # # ##### # # # ##### # # ##### |
125 | 2024-08-05T19:08:27.281Z | # ##### # ###### # # # # # |
126 | 2024-08-05T19:08:27.281Z | # # # # # # # # # # # # # |
127 | 2024-08-05T19:08:27.281Z | ##### # # ###### # # # ###### ###### ##### |
128 | 2024-08-05T19:08:27.281Z | |
129 | 2024-08-05T19:08:27.281Z | + echo /var/tmp/bins/dsc create --ds-bin /var/tmp/bins/crucible-downstairs --extent-size 4000 --extent-count 200 --region-count 4 --cleanup |
130 | 2024-08-05T19:08:27.281Z | /var/tmp/bins/dsc create --ds-bin /var/tmp/bins/crucible-downstairs --extent-size 4000 --extent-count 200 --region-count 4 --cleanup |
131 | 2024-08-05T19:08:27.281Z | + /var/tmp/bins/dsc create --ds-bin /var/tmp/bins/crucible-downstairs --extent-size 4000 --extent-count 200 --region-count 4 --cleanup |
132 | 2024-08-05T19:08:27.297Z | Creating region directory at: /var/tmp/dsc/region |
133 | 2024-08-05T19:08:27.300Z | Creating dsc directory at: /tmp/dsc |
134 | 2024-08-05T19:08:28.130Z | Downstairs region 0 created at /var/tmp/dsc/region/8810 in 0.83007544 |
135 | 2024-08-05T19:08:28.340Z | Downstairs region 1 created at /var/tmp/dsc/region/8820 in 0.21179943 |
136 | 2024-08-05T19:08:28.555Z | Downstairs region 2 created at /var/tmp/dsc/region/8830 in 0.21577673 |
137 | 2024-08-05T19:08:28.788Z | Downstairs region 3 created at /var/tmp/dsc/region/8840 in 0.2331287 |
138 | 2024-08-05T19:08:28.789Z | Region set with 4 regions was created |
139 | 2024-08-05T19:08:28.792Z | + banner StartDS |
140 | 2024-08-05T19:08:28.795Z | ##### ###### ##### |
141 | 2024-08-05T19:08:28.795Z | # # ##### ## ##### ##### # # # # |
142 | 2024-08-05T19:08:28.795Z | # # # # # # # # # # |
143 | 2024-08-05T19:08:28.795Z | ##### # # # # # # # # ##### |
144 | 2024-08-05T19:08:28.795Z | # # ###### ##### # # # # |
145 | 2024-08-05T19:08:28.795Z | # # # # # # # # # # # # |
146 | 2024-08-05T19:08:28.795Z | ##### # # # # # # ###### ##### |
147 | 2024-08-05T19:08:28.795Z | |
148 | 2024-08-05T19:08:28.795Z | + sleep 5 |
149 | 2024-08-05T19:08:28.796Z | + /var/tmp/bins/dsc start --ds-bin /var/tmp/bins/crucible-downstairs --region-count 4 |
150 | 2024-08-05T19:08:33.800Z | ++ pgrep dsc |
151 | 2024-08-05T19:08:33.803Z | + dsc_pid=1169 |
152 | 2024-08-05T19:08:33.803Z | + [[ 1169 -eq 0 ]] |
153 | 2024-08-05T19:08:33.803Z | + ps -p 1169 |
154 | 2024-08-05T19:08:33.820Z | PID TTY TIME CMD |
155 | 2024-08-05T19:08:33.820Z | 1169 ? 0:00 dsc |
156 | 2024-08-05T19:08:33.825Z | + echo 'Found dsc running, continue tests' |
157 | 2024-08-05T19:08:33.825Z | Found dsc running, continue tests |
158 | 2024-08-05T19:08:33.825Z | + echo 'Setup self timeout' |
159 | 2024-08-05T19:08:33.825Z | Setup self timeout |
160 | 2024-08-05T19:08:33.825Z | + jobpid=1140 |
161 | 2024-08-05T19:08:33.825Z | + echo 'Setup debug logging' |
162 | 2024-08-05T19:08:33.825Z | + mkdir /tmp/debug |
163 | 2024-08-05T19:08:33.825Z | Setup debug logging |
164 | 2024-08-05T19:08:33.825Z | + sleep 7200 |
165 | 2024-08-05T19:08:33.825Z | + psrinfo -v |
166 | 2024-08-05T19:08:33.829Z | + df -h |
167 | 2024-08-05T19:08:33.833Z | + prstat -d d -mLc 1 |
168 | 2024-08-05T19:08:33.835Z | + iostat -T d -xn 1 |
169 | 2024-08-05T19:08:33.835Z | + mpstat -T d 1 |
170 | 2024-08-05T19:08:33.835Z | + vmstat -T d -p 1 |
171 | 2024-08-05T19:08:33.838Z | + pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d |
172 | 2024-08-05T19:08:33.838Z | + banner LR |
173 | 2024-08-05T19:08:33.838Z | + pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d |
174 | 2024-08-05T19:08:33.841Z | # ###### |
175 | 2024-08-05T19:08:33.841Z | # # # |
176 | 2024-08-05T19:08:33.841Z | # # # |
177 | 2024-08-05T19:08:33.841Z | # ###### |
178 | 2024-08-05T19:08:33.841Z | # # # |
179 | 2024-08-05T19:08:33.841Z | # # # |
180 | 2024-08-05T19:08:33.841Z | ####### # # |
181 | 2024-08-05T19:08:33.841Z | |
182 | 2024-08-05T19:08:33.841Z | + ptime -m /var/tmp/bins/crutest replace -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --replacement 127.0.0.1:8840 -g 1 -c 10 --stable |
183 | 2024-08-05T19:18:54.439Z | + banner StopDSC |
184 | 2024-08-05T19:18:54.442Z | ##### ###### ##### ##### |
185 | 2024-08-05T19:18:54.442Z | # # ##### #### ##### # # # # # # |
186 | 2024-08-05T19:18:54.442Z | # # # # # # # # # # |
187 | 2024-08-05T19:18:54.442Z | ##### # # # # # # # ##### # |
188 | 2024-08-05T19:18:54.442Z | # # # # ##### # # # # |
189 | 2024-08-05T19:18:54.442Z | # # # # # # # # # # # # |
190 | 2024-08-05T19:18:54.442Z | ##### # #### # ###### ##### ##### |
191 | 2024-08-05T19:18:54.442Z | |
192 | 2024-08-05T19:18:54.442Z | + /var/tmp/bins/dsc cmd shutdown |
193 | 2024-08-05T19:18:54.520Z | + banner WaitStop |
194 | 2024-08-05T19:18:54.523Z | # # ##### |
195 | 2024-08-05T19:18:54.523Z | # # # ## # ##### # # ##### #### ##### |
196 | 2024-08-05T19:18:54.523Z | # # # # # # # # # # # # # |
197 | 2024-08-05T19:18:54.523Z | # # # # # # # ##### # # # # # |
198 | 2024-08-05T19:18:54.523Z | # # # ###### # # # # # # ##### |
199 | 2024-08-05T19:18:54.523Z | # # # # # # # # # # # # # |
200 | 2024-08-05T19:18:54.523Z | ## ## # # # # ##### # #### # |
201 | 2024-08-05T19:18:54.523Z | |
202 | 2024-08-05T19:18:54.523Z | + wait 1169 |
203 | 2024-08-05T19:18:55.519Z | + _exit_trap |
204 | 2024-08-05T19:18:55.519Z | + local status=0 |
205 | 2024-08-05T19:18:55.522Z | + [[ 0 -eq 0 ]] |
206 | 2024-08-05T19:18:55.522Z | + exit 0 |
207 | 2024-08-05T19:19:00.525Z | process exited: duration 635856 ms, exit code 0 |
208 | 2024-08-05T19:19:00.525Z | exec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close |
209 | 2024-08-05T19:20:00.531Z | exec warning: : stdout descriptor may be held open by a background process; giving up! |
210 | 2024-08-05T19:20:00.531Z | exec warning: : stderr descriptor may be held open by a background process; giving up! |
|
211 | 2024-08-05T19:20:00.544Z | found 14 output files |
212 | 2024-08-05T19:20:00.622Z | uploading: /tmp/crutest-replace.log (16160417 bytes) |
213 | 2024-08-05T19:20:01.809Z | uploaded: /tmp/crutest-replace.log |
214 | 2024-08-05T19:20:01.809Z | uploading: /tmp/dsc.log (1991 bytes) |
215 | 2024-08-05T19:20:02.818Z | uploaded: /tmp/dsc.log |
216 | 2024-08-05T19:20:02.818Z | uploading: /tmp/debug/df.txt (1270 bytes) |
217 | 2024-08-05T19:20:03.826Z | uploaded: /tmp/debug/df.txt |
218 | 2024-08-05T19:20:03.826Z | uploading: /tmp/debug/dtrace.txt (391934 bytes) |
219 | 2024-08-05T19:20:04.853Z | uploaded: /tmp/debug/dtrace.txt |
220 | 2024-08-05T19:20:04.856Z | uploading: /tmp/debug/iostat.txt (207652 bytes) |
221 | 2024-08-05T19:20:04.870Z | upload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 207652 -> 209147 |
222 | 2024-08-05T19:20:05.876Z | uploaded: /tmp/debug/iostat.txt |
223 | 2024-08-05T19:20:05.877Z | uploading: /tmp/debug/mpstat.txt (501320 bytes) |
224 | 2024-08-05T19:20:05.902Z | upload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 501320 -> 505688 |
225 | 2024-08-05T19:20:06.910Z | uploaded: /tmp/debug/mpstat.txt |
226 | 2024-08-05T19:20:06.910Z | uploading: /tmp/debug/paging.txt (89220 bytes) |
227 | 2024-08-05T19:20:06.920Z | upload warning: file "/tmp/debug/paging.txt" changed size mid upload: 89220 -> 90176 |
228 | 2024-08-05T19:20:07.926Z | uploaded: /tmp/debug/paging.txt |
229 | 2024-08-05T19:20:07.929Z | uploading: /tmp/debug/prstat.txt (1112594 bytes) |
230 | 2024-08-05T19:20:07.967Z | upload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1112594 -> 1121631 |
231 | 2024-08-05T19:20:08.974Z | uploaded: /tmp/debug/prstat.txt |
232 | 2024-08-05T19:20:08.974Z | uploading: /tmp/debug/psrinfo.txt (1528 bytes) |
233 | 2024-08-05T19:20:09.984Z | uploaded: /tmp/debug/psrinfo.txt |
234 | 2024-08-05T19:20:09.984Z | uploading: /tmp/debug/upstairs-info.txt (114873 bytes) |
235 | 2024-08-05T19:20:11.001Z | uploaded: /tmp/debug/upstairs-info.txt |
236 | 2024-08-05T19:20:11.001Z | uploading: /tmp/dsc/downstairs-8810.txt (1461640 bytes) |
237 | 2024-08-05T19:20:12.053Z | uploaded: /tmp/dsc/downstairs-8810.txt |
238 | 2024-08-05T19:20:12.053Z | uploading: /tmp/dsc/downstairs-8820.txt (976130 bytes) |
239 | 2024-08-05T19:20:13.095Z | uploaded: /tmp/dsc/downstairs-8820.txt |
240 | 2024-08-05T19:20:13.096Z | uploading: /tmp/dsc/downstairs-8830.txt (1194366 bytes) |
241 | 2024-08-05T19:20:14.141Z | uploaded: /tmp/dsc/downstairs-8830.txt |
242 | 2024-08-05T19:20:14.141Z | uploading: /tmp/dsc/downstairs-8840.txt (1240825 bytes) |
243 | 2024-08-05T19:20:15.188Z | uploaded: /tmp/dsc/downstairs-8840.txt |