|
|
|
1 | 2024-10-15T16:08:24.897Z | job dependencies complete; ready to run (waiting for 15 m 28 s) |
2 | 2024-10-15T16:09:33.191Z | job assigned to worker 01JA8F0KPR8Q5NQMZ4R04P05A7 (queued for 1 m 8 s) |
3 | 2024-10-15T16:09:40.646Z | downloading input: /input/build/work/bins/crucible-downstairs.gz |
4 | 2024-10-15T16:09:41.791Z | downloaded input: /input/build/work/bins/crucible-downstairs.gz |
5 | 2024-10-15T16:09:41.791Z | downloading input: /input/build/work/bins/crucible-hammer.gz |
6 | 2024-10-15T16:09:42.891Z | downloaded input: /input/build/work/bins/crucible-hammer.gz |
7 | 2024-10-15T16:09:42.892Z | downloading input: /input/build/work/bins/crutest.gz |
8 | 2024-10-15T16:09:44.052Z | downloaded input: /input/build/work/bins/crutest.gz |
9 | 2024-10-15T16:09:44.052Z | downloading input: /input/build/work/bins/dsc.gz |
10 | 2024-10-15T16:09:44.749Z | downloaded input: /input/build/work/bins/dsc.gz |
11 | 2024-10-15T16:09:44.752Z | downloading input: /input/build/work/scripts/perf-downstairs-tick.d |
12 | 2024-10-15T16:09:44.767Z | downloaded input: /input/build/work/scripts/perf-downstairs-tick.d |
13 | 2024-10-15T16:09:44.767Z | downloading input: /input/build/work/scripts/test_ds.sh |
14 | 2024-10-15T16:09:44.786Z | downloaded input: /input/build/work/scripts/test_ds.sh |
15 | 2024-10-15T16:09:44.786Z | downloading input: /input/build/work/scripts/test_live_repair.sh |
16 | 2024-10-15T16:09:44.804Z | downloaded input: /input/build/work/scripts/test_live_repair.sh |
17 | 2024-10-15T16:09:44.804Z | downloading input: /input/build/work/scripts/test_repair.sh |
18 | 2024-10-15T16:09:44.823Z | downloaded input: /input/build/work/scripts/test_repair.sh |
19 | 2024-10-15T16:09:44.823Z | downloading input: /input/build/work/scripts/test_replay.sh |
20 | 2024-10-15T16:09:44.840Z | downloaded input: /input/build/work/scripts/test_replay.sh |
21 | 2024-10-15T16:09:44.843Z | downloading input: /input/build/work/scripts/test_up.sh |
22 | 2024-10-15T16:09:44.859Z | downloaded input: /input/build/work/scripts/test_up.sh |
23 | 2024-10-15T16:09:44.860Z | downloading input: /input/build/work/scripts/upstairs_info.d |
24 | 2024-10-15T16:09:44.878Z | downloaded input: /input/build/work/scripts/upstairs_info.d |
25 | 2024-10-15T16:09:44.880Z | downloading input: /input/build/tmp/cargo-test-out.log |
26 | 2024-10-15T16:09:45.647Z | downloaded input: /input/build/tmp/cargo-test-out.log |
|
27 | 2024-10-15T16:09:45.647Z | starting task 0: "setup" |
28 | 2024-10-15T16:09:45.658Z | ++ uname -s |
29 | 2024-10-15T16:09:45.661Z | + kern=SunOS |
30 | 2024-10-15T16:09:45.661Z | + case "$kern" in |
31 | 2024-10-15T16:09:45.661Z | + groupadd -g 12345 build |
32 | 2024-10-15T16:09:45.664Z | + useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build |
33 | 2024-10-15T16:09:47.673Z | + zfs create -o mountpoint=/work rpool/work |
34 | 2024-10-15T16:09:48.252Z | ++ awk '$2 == "/home" { print $3 }' /etc/mnttab |
35 | 2024-10-15T16:09:48.254Z | + home_fs=zfs |
36 | 2024-10-15T16:09:48.254Z | + [[ zfs == autofs ]] |
37 | 2024-10-15T16:09:48.254Z | + mkdir -p /home/build |
38 | 2024-10-15T16:09:48.257Z | + chown build:build /home/build /work |
39 | 2024-10-15T16:09:49.263Z | + chmod 0700 /home/build /work |
40 | 2024-10-15T16:09:49.267Z | process exited: duration 3613 ms, exit code 0 |
|
41 | 2024-10-15T16:09:49.273Z | starting task 1: "authentication" |
42 | 2024-10-15T16:09:49.290Z | process exited: duration 17 ms, exit code 0 |
|
43 | 2024-10-15T16:09:49.299Z | starting task 2: "build" |
44 | 2024-10-15T16:09:49.304Z | + banner cores |
45 | 2024-10-15T16:09:49.325Z | |
46 | 2024-10-15T16:09:49.325Z | #### #### ##### ###### #### |
47 | 2024-10-15T16:09:49.325Z | # # # # # # # # |
48 | 2024-10-15T16:09:49.325Z | # # # # # ##### #### |
49 | 2024-10-15T16:09:49.325Z | # # # ##### # # |
50 | 2024-10-15T16:09:49.325Z | # # # # # # # # # |
51 | 2024-10-15T16:09:49.325Z | #### #### # # ###### #### |
52 | 2024-10-15T16:09:49.325Z | |
53 | 2024-10-15T16:09:49.328Z | + pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid |
54 | 2024-10-15T16:09:49.335Z | + pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d |
55 | 2024-10-15T16:09:49.338Z | + echo ' chmod input script dir contains:' |
56 | 2024-10-15T16:09:49.338Z | chmod input script dir contains: |
57 | 2024-10-15T16:09:49.338Z | + ls -ltr /input/build/work/scripts |
58 | 2024-10-15T16:09:49.341Z | total 42 |
59 | 2024-10-15T16:09:49.341Z | -rwxr-xr-x 1 root root 3056 Oct 15 16:09 perf-downstairs-tick.d |
60 | 2024-10-15T16:09:49.341Z | -rwxr-xr-x 1 root root 2127 Oct 15 16:09 test_ds.sh |
61 | 2024-10-15T16:09:49.341Z | -rwxr-xr-x 1 root root 4272 Oct 15 16:09 test_live_repair.sh |
62 | 2024-10-15T16:09:49.341Z | -rwxr-xr-x 1 root root 6724 Oct 15 16:09 test_repair.sh |
63 | 2024-10-15T16:09:49.341Z | -rwxr-xr-x 1 root root 3494 Oct 15 16:09 test_replay.sh |
64 | 2024-10-15T16:09:49.341Z | -rwxr-xr-x 1 root root 13887 Oct 15 16:09 test_up.sh |
65 | 2024-10-15T16:09:49.341Z | -rwxr-xr-x 1 root root 2407 Oct 15 16:09 upstairs_info.d |
66 | 2024-10-15T16:09:49.341Z | + banner unpack |
67 | 2024-10-15T16:09:49.343Z | |
68 | 2024-10-15T16:09:49.343Z | # # # # ##### ## #### # # |
69 | 2024-10-15T16:09:49.344Z | # # ## # # # # # # # # # |
70 | 2024-10-15T16:09:49.344Z | # # # # # # # # # # #### |
71 | 2024-10-15T16:09:49.344Z | # # # # # ##### ###### # # # |
72 | 2024-10-15T16:09:49.344Z | # # # ## # # # # # # # |
73 | 2024-10-15T16:09:49.344Z | #### # # # # # #### # # |
74 | 2024-10-15T16:09:49.344Z | |
75 | 2024-10-15T16:09:49.344Z | + mkdir -p /var/tmp/bins |
76 | 2024-10-15T16:09:49.347Z | + for t in "$input/bins/"*.gz |
77 | 2024-10-15T16:09:49.347Z | ++ basename /input/build/work/bins/crucible-downstairs.gz |
78 | 2024-10-15T16:09:49.347Z | + b=crucible-downstairs.gz |
79 | 2024-10-15T16:09:49.347Z | + b=crucible-downstairs |
80 | 2024-10-15T16:09:49.347Z | + gunzip |
81 | 2024-10-15T16:09:51.733Z | + chmod +x /var/tmp/bins/crucible-downstairs |
82 | 2024-10-15T16:09:51.736Z | + for t in "$input/bins/"*.gz |
83 | 2024-10-15T16:09:51.736Z | ++ basename /input/build/work/bins/crucible-hammer.gz |
84 | 2024-10-15T16:09:51.739Z | + b=crucible-hammer.gz |
85 | 2024-10-15T16:09:51.739Z | + b=crucible-hammer |
86 | 2024-10-15T16:09:51.739Z | + gunzip |
87 | 2024-10-15T16:09:54.054Z | + chmod +x /var/tmp/bins/crucible-hammer |
88 | 2024-10-15T16:09:54.056Z | + for t in "$input/bins/"*.gz |
89 | 2024-10-15T16:09:54.056Z | ++ basename /input/build/work/bins/crutest.gz |
90 | 2024-10-15T16:09:54.056Z | + b=crutest.gz |
91 | 2024-10-15T16:09:54.056Z | + b=crutest |
92 | 2024-10-15T16:09:54.056Z | + gunzip |
93 | 2024-10-15T16:09:56.428Z | + chmod +x /var/tmp/bins/crutest |
94 | 2024-10-15T16:09:56.431Z | + for t in "$input/bins/"*.gz |
95 | 2024-10-15T16:09:56.433Z | ++ basename /input/build/work/bins/dsc.gz |
96 | 2024-10-15T16:09:56.433Z | + b=dsc.gz |
97 | 2024-10-15T16:09:56.433Z | + b=dsc |
98 | 2024-10-15T16:09:56.433Z | + gunzip |
99 | 2024-10-15T16:09:57.908Z | + chmod +x /var/tmp/bins/dsc |
100 | 2024-10-15T16:09:57.910Z | + export BINDIR=/var/tmp/bins |
101 | 2024-10-15T16:09:57.910Z | + BINDIR=/var/tmp/bins |
102 | 2024-10-15T16:09:57.910Z | + banner setup |
103 | 2024-10-15T16:09:57.913Z | |
104 | 2024-10-15T16:09:57.913Z | #### ###### ##### # # ##### |
105 | 2024-10-15T16:09:57.913Z | # # # # # # # |
106 | 2024-10-15T16:09:57.913Z | #### ##### # # # # # |
107 | 2024-10-15T16:09:57.913Z | # # # # # ##### |
108 | 2024-10-15T16:09:57.913Z | # # # # # # # |
109 | 2024-10-15T16:09:57.913Z | #### ###### # #### # |
110 | 2024-10-15T16:09:57.913Z | |
111 | 2024-10-15T16:09:57.913Z | + echo 'Setup self timeout' |
112 | 2024-10-15T16:09:57.913Z | Setup self timeout |
113 | 2024-10-15T16:09:57.913Z | + jobpid=1137 |
114 | 2024-10-15T16:09:57.916Z | + echo 'Setup debug logging' |
115 | 2024-10-15T16:09:57.916Z | Setup debug logging |
116 | 2024-10-15T16:09:57.916Z | + mkdir /tmp/debug |
117 | 2024-10-15T16:09:57.916Z | + sleep 10800 |
118 | 2024-10-15T16:09:57.916Z | + psrinfo -v |
119 | 2024-10-15T16:09:57.919Z | + df -h |
120 | 2024-10-15T16:09:57.945Z | + prstat -d d -mLc 1 |
121 | 2024-10-15T16:09:57.947Z | + iostat -T d -xn 1 |
122 | 2024-10-15T16:09:57.947Z | + mpstat -T d 1 |
123 | 2024-10-15T16:09:57.947Z | + vmstat -T d -p 1 |
124 | 2024-10-15T16:09:57.950Z | + pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d |
125 | 2024-10-15T16:09:57.950Z | + banner replay |
126 | 2024-10-15T16:09:57.950Z | + pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d |
127 | 2024-10-15T16:09:57.952Z | |
128 | 2024-10-15T16:09:57.952Z | ##### ###### ##### # ## # # |
129 | 2024-10-15T16:09:57.952Z | # # # # # # # # # # |
130 | 2024-10-15T16:09:57.952Z | # # ##### # # # # # # |
131 | 2024-10-15T16:09:57.952Z | ##### # ##### # ###### # |
132 | 2024-10-15T16:09:57.952Z | # # # # # # # # |
133 | 2024-10-15T16:09:57.952Z | # # ###### # ###### # # # |
134 | 2024-10-15T16:09:57.952Z | |
135 | 2024-10-15T16:09:57.952Z | + ptime -m bash /input/build/work/scripts/test_replay.sh |
136 | 2024-10-15T16:09:57.968Z | starting October 15, 2024 at 04:09:57 PM UTC |
137 | 2024-10-15T16:09:57.970Z | Tail /tmp/test_replay.log for test output |
138 | 2024-10-15T16:09:57.973Z | Creating 3 downstairs regions |
139 | 2024-10-15T16:09:58.524Z | Starting 3 downstairs |
140 | 2024-10-15T16:10:03.536Z | Running initial fill |
141 | 2024-10-15T16:10:04.269Z | Replay loop starts now October 15, 2024 at 04:10:03 PM UTC |
142 | 2024-10-15T16:17:13.715Z | Replays:30 time: 7:09 |
143 | 2024-10-15T16:17:13.718Z | Do final verify |
144 | 2024-10-15T16:17:23.300Z | October 15, 2024 at 04:17:22 PM UTC Test ends with 0 |
145 | 2024-10-15T16:17:23.303Z | |
146 | 2024-10-15T16:17:23.303Z | real 7:25.256798990 |
147 | 2024-10-15T16:17:23.303Z | user 24.637625323 |
148 | 2024-10-15T16:17:23.303Z | sys 6.623917024 |
149 | 2024-10-15T16:17:23.303Z | trap 0.053990265 |
150 | 2024-10-15T16:17:23.303Z | tflt 0.012581292 |
151 | 2024-10-15T16:17:23.303Z | dflt 0.043427302 |
152 | 2024-10-15T16:17:23.303Z | kflt 0.001236519 |
153 | 2024-10-15T16:17:23.303Z | lock 11:42:47.017952195 |
154 | 2024-10-15T16:17:23.303Z | slp 40:17.491787536 |
155 | 2024-10-15T16:17:23.303Z | lat 3.506511424 |
156 | 2024-10-15T16:17:23.303Z | stop 0.562992275 |
157 | 2024-10-15T16:17:28.308Z | process exited: duration 453912 ms, exit code 0 |
158 | 2024-10-15T16:17:28.308Z | exec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close |
159 | 2024-10-15T16:18:28.364Z | exec warning: : stdout descriptor may be held open by a background process; giving up! |
160 | 2024-10-15T16:18:28.364Z | exec warning: : stderr descriptor may be held open by a background process; giving up! |
|
161 | 2024-10-15T16:18:28.376Z | found 13 output files |
162 | 2024-10-15T16:18:28.376Z | uploading: /tmp/test_replay.log (2863786 bytes) |
163 | 2024-10-15T16:18:29.440Z | uploaded: /tmp/test_replay.log |
164 | 2024-10-15T16:18:29.440Z | uploading: /tmp/test_replay_verify.log (70120 bytes) |
165 | 2024-10-15T16:18:30.453Z | uploaded: /tmp/test_replay_verify.log |
166 | 2024-10-15T16:18:30.453Z | uploading: /tmp/debug/df.txt (1270 bytes) |
167 | 2024-10-15T16:18:31.461Z | uploaded: /tmp/debug/df.txt |
168 | 2024-10-15T16:18:31.461Z | uploading: /tmp/debug/iostat.txt (153307 bytes) |
169 | 2024-10-15T16:18:31.474Z | upload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 153307 -> 154207 |
170 | 2024-10-15T16:18:32.480Z | uploaded: /tmp/debug/iostat.txt |
171 | 2024-10-15T16:18:32.480Z | uploading: /tmp/debug/mpstat.txt (372521 bytes) |
172 | 2024-10-15T16:18:32.501Z | upload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 372521 -> 375437 |
173 | 2024-10-15T16:18:33.509Z | uploaded: /tmp/debug/mpstat.txt |
174 | 2024-10-15T16:18:33.511Z | uploading: /tmp/debug/paging.txt (66981 bytes) |
175 | 2024-10-15T16:18:33.517Z | upload warning: file "/tmp/debug/paging.txt" changed size mid upload: 66981 -> 67556 |
176 | 2024-10-15T16:18:34.524Z | uploaded: /tmp/debug/paging.txt |
177 | 2024-10-15T16:18:34.526Z | uploading: /tmp/debug/perf.txt (1021754 bytes) |
178 | 2024-10-15T16:18:35.566Z | uploaded: /tmp/debug/perf.txt |
179 | 2024-10-15T16:18:35.568Z | uploading: /tmp/debug/prstat.txt (700006 bytes) |
180 | 2024-10-15T16:18:35.597Z | upload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 700006 -> 709045 |
181 | 2024-10-15T16:18:36.603Z | uploaded: /tmp/debug/prstat.txt |
182 | 2024-10-15T16:18:36.605Z | uploading: /tmp/debug/psrinfo.txt (1528 bytes) |
183 | 2024-10-15T16:18:37.612Z | uploaded: /tmp/debug/psrinfo.txt |
184 | 2024-10-15T16:18:37.614Z | uploading: /tmp/debug/upinfo.txt (71574 bytes) |
185 | 2024-10-15T16:18:38.625Z | uploaded: /tmp/debug/upinfo.txt |
186 | 2024-10-15T16:18:38.625Z | uploading: /tmp/dsc/downstairs-8810.txt (62738 bytes) |
187 | 2024-10-15T16:18:39.637Z | uploaded: /tmp/dsc/downstairs-8810.txt |
188 | 2024-10-15T16:18:39.637Z | uploading: /tmp/dsc/downstairs-8820.txt (62929 bytes) |
189 | 2024-10-15T16:18:40.651Z | uploaded: /tmp/dsc/downstairs-8820.txt |
190 | 2024-10-15T16:18:40.655Z | uploading: /tmp/dsc/downstairs-8830.txt (62952 bytes) |
191 | 2024-10-15T16:18:41.667Z | uploaded: /tmp/dsc/downstairs-8830.txt |