Debian Codegraph @

tools/testing/reporting/scripts/runAllTests

raw
1 #!/bin/bash
2
3 # Licensed to the Apache Software Foundation (ASF) under one or more
4 # contributor license agreements. See the NOTICE file distributed with
5 # this work for additional information regarding copyright ownership.
6 # The ASF licenses this file to you under the Apache License, Version 2.0
7 # (the "License"); you may not use this file except in compliance with
8 # the License. You may obtain a copy of the License at
9 #
10 # http://www.apache.org/licenses/LICENSE-2.0
11 #
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17
18
19 GETHOSTNAME="host"
20 # GETHOSTNAME="echo"
21 # Options:
22 # VERBOSE="-Dverbose=true"
23 # VERBOSE="-Dverbose=false"
24 VERBOSE=""
25 # You should NOT need to modify anything below.
26 #---------------------------------------------
27 TOOLDIR="${DERBYDIR}/${BRANCH_DIR}/tools/testing/reporting/scripts"
28 . ${TOOLDIR}/env
29
30 SEND_UPDATE_EXECUTE_REVISION_MISMATCH_MSG="true"
31 SVNREVUSED="-"
32
33 SANDBOX=`basename ${DERBYDIR}`
34
35 move_log(){
36 TESTDIRl="$1"
37 TESTHOSTl="$2"
38 OSPLATFROMl="$3"
39 HWPROCl="$4"
40 startTimel="$5"
41 TESTLOGDIRl="$6"
42 ## Following may need modification.
43 # EXCLUDEl="-x \*.dat" # Does not seem to work.
44 EXCLUDEl="-x@/${DERBYDIR}/testing/ExcludeFromZip"
45 # EXCLUDEl=""
46 echo "*************************************************************"
47 echo "mkdir -p ${TESTLOGDIRl}/${OSPLATFROMl}_${HWPROCl}"
48 echo "zip -r -q -9 ${TESTLOGDIRl}/${OSPLATFROMl}_${HWPROCl}/${startTimel}.zip . ${EXCLUDEl}"
49 mkdir -p ${TESTLOGDIRl}/${OSPLATFROMl}_${HWPROCl} && \
50 chmod go+rx ${TESTLOGDIRl}/${OSPLATFROMl}_${HWPROCl} && \
51 zip -r -q -9 ${TESTLOGDIRl}/${OSPLATFROMl}_${HWPROCl}/${startTimel}.zip . ${EXCLUDEl} && \
52 cd ../ && \
53 # Wait: keep until starting next testrun.rm -rf ${OSPLATFROMl}_${HWPROCl} && \
54 echo "Successfully moved logfiles to ${TESTLOGDIRl}/${OSPLATFROMl}_${HWPROCl}/${startTimel}.zip"
55 chmod go+r ${TESTLOGDIRl}/${OSPLATFROMl}_${HWPROCl}/${startTimel}.zip
56 }
57 # move_log END
58
59 if [ "$1" == "" ]
60 then
61 echo "Usage: $0 fileListingTestsToRun fullPathTestLogDir"
62 echo " Derby db files and test logs are moved to fullPathTestLogDir after test."
63 echo "E.g. Unix: runAllTests testing/testlist ~/Apache/Derby/testing/testlog"
64 echo " Win: ~/bin/runAllTests.cygwin testing/testlist /log/huge3/derbytester/DerbyTestLog"
65 echo " Rerun: cd ..../Derby/adhoc/"
66 echo " runAllTests ../testing/testlistReRun ~/Apache/Derby/testing/testlogEmbeddedReRun"
67 exit
68 fi
69 if [ "$2" == "" ]
70 then
71 echo "Usage: $0 $1 testLogDir"
72 echo " Derby db files and test logs are moved to fullPathTestLogDir after test."
73 exit
74 fi
75
76 TESTLISTFILE=$1
77 TESTLOGDIR=$2
78
79 if [ ! -e $TESTLISTFILE ]
80 then
81 echo "$TESTLISTFILE does not exist."
82 exit
83 fi
84
85 USEDERBYNET=""
86 if [ "$3" != "" ]
87 then
88 USEDERBYNET="-Dframework=DerbyNet"
89 fi
90
91 HASFULLPATH=`echo "$TESTLOGDIR" | sed -e 's|^/|HASFULLPATH /|' | gawk '{ print $1 }'`
92 #DEBUG echo "${HASFULLPATH}"
93 if [ "${HASFULLPATH}" != "HASFULLPATH" ]
94 then
95 echo "Directory for test logs must be full path. You supplied '"$TESTLOGDIR"'"
96 exit
97 fi
98 if [ ! -d $TESTLOGDIR ]
99 then
100 echo "$TESTLOGDIR does not exist."
101 exit
102 fi
103 if [ ! -w $TESTLOGDIR ]
104 then
105 echo "$TESTLOGDIR is not writable."
106 exit
107 fi
108
109 SUITES=`gawk '{ print $1 }' ${TESTLISTFILE} | grep -v "^#"`
110 TESTHOST=`uname -n`
111 TESTHOST=`${GETHOSTNAME} ${TESTHOST} | awk '{print $1}'`
112
113 KERNEL=`uname -s`
114 KERN_REV=`uname -r`
115 OSPLATFROM="${KERNEL}-${KERN_REV}"
116 # OSPLATFROM="${KERNEL}"
117 HWNAME=`uname -m`
118 PROCNAME=`uname -p`
119 HWPROC="${HWNAME}-${PROCNAME}"
120 HWPLATFORM=`uname -i`
121
122 TESTLOG_PLATF_DIR="${TESTLOGDIR}/${OSPLATFROM}_${HWPROC}"
123
124 #DEBUG echo "Log files will be zipped to $TESTLOG_PLATF_DIR/"
125
126 # echo "mkdir -p ${TESTLOG_PLATF_DIR}"
127 mkdir -p ${TESTLOG_PLATF_DIR}
128 chmod go+rx ${TESTLOG_PLATF_DIR}
129
130 TESTDIR="${TESTEXECUTIONDIR}/${OSPLATFROM}_${HWPROC}"
131
132 # We keep old results until next test run is started, so now is the time to:
133 rm -rf ${TESTDIR}
134
135 # echo "mkdir -p ${TESTDIR}"
136 mkdir -p ${TESTDIR}
137 chmod go+rx ${TESTDIR}
138 if [ ! -w $TESTDIR ]
139 then
140 echo "$TESTDIR is not writable."
141 exit
142 fi
143 cd ${TESTDIR}
144
145 #DEBUG pwd
146
147 echo "Tests are executed on ${TESTHOST} in ${TESTDIR}"
148 echo "${TESTHOST}: ${OSPLATFROM}_${HWPROC} [${HWPLATFORM}]" > OS_HW
149 echo "ANT_HOME: ${ANT_HOME}" >> OS_HW
150 echo "JAVA_HOME: ${JAVA_HOME}" >> OS_HW
151 ant -version >> OS_HW
152 java -version 2>> OS_HW
153
154 echo "cp ${BUILDLOGDIR}/lastBuildInfo buildInfo"
155 cp ${BUILDLOGDIR}/lastBuildInfo buildInfo
156
157 REVISION=`head -2 buildInfo | tail -1 | gawk '{ print $3 }'`
158 REPOSITORY=`head -2 buildInfo | tail -1 | gawk '{ print $6 }'`
159 BUILTOS=`head -1 buildInfo | gawk '{ print $3 }'`
160 BUILTOSKERN=`head -1 buildInfo | gawk '{ print $4 }'`
161 BUILTDATE=`head -1 buildInfo | gawk '{ print $8" "$9 }'`
162 BUILTINFO=`grep "^Built on" buildInfo`
163
164 # echo "REVISION: ${REVISION} REPOSITORY: ${REPOSITORY} BUILTOS: ${BUILTOS} BUILTOSKERN: ${BUILTOSKERN} BUILTDATE: ${BUILTDATE} BUILTINFO: ${BUILTINFO}"
165
166 timeId="${REVISION}"
167 startTime=`date +%Y-%m-%d" "%H:%M:%S`
168
169 SUITESLOG="${TESTLOG_PLATF_DIR}/${timeId}.csv"
170 JAVAINFO="${TESTLOG_PLATF_DIR}/${timeId}.javainfo"
171 CURRSUITESLOG="${TESTLOG_PLATF_DIR}/current.csv"
172 PREVSUITESLOG="${TESTLOG_PLATF_DIR}/previous.csv"
173 LOG="${TESTLOG_PLATF_DIR}/${timeId}.txt"
174 cp ${CURRSUITESLOG} ${PREVSUITESLOG}
175 echo -n "Starting Derby tests on ${TESTHOST} [${OSPLATFROM}_${HWPROC} [${HWPLATFORM}]] at "
176 echo -n -e "${TESTHOST}\t${KERNEL}\t${KERN_REV}\t${HWNAME}\t${PROCNAME}\t${HWPLATFORM}\t${REVISION}\t${REPOSITORY}\t" > ${CURRSUITESLOG}
177 echo ${startTime}
178 echo -e ${startTime} >> ${CURRSUITESLOG}
179 echo "${BUILTINFO}" >> ${CURRSUITESLOG}
180 echo "" >> ${CURRSUITESLOG}
181
182 # Start without knowing....
183 echo "-" > ${JAVAINFO}
184 echo "-" >> ${JAVAINFO}
185 chmod go+r ${JAVAINFO}
186
187 ${TOOLDIR}/system > ${TESTLOG_PLATF_DIR}/${REVISION}-system
188 ${TOOLDIR}/cpu_mem > ${TESTLOG_PLATF_DIR}/${REVISION}-cpu_mem
189 chmod go+r ${TESTLOG_PLATF_DIR}/${REVISION}-system
190 chmod go+r ${TESTLOG_PLATF_DIR}/${REVISION}-cpu_mem
191
192 # echo "CLASSPATH = ${CLASSPATH}"
193
194 HEADER="Tests \tPassed \tFailed \tSkippd \tDuration"
195 echo -e "Suite \t${HEADER}" >> ${CURRSUITESLOG}
196
197 # export CLASSPATH=`cygpath -wp $CLASSPATH`
198 # echo "CLASSPATH: $CLASSPATH"
199
200 # echo ${SUITES}
201 echo "" > ${LOG}
202 JAVAVENDOR=""
203 for suite in ${SUITES}
204 do
205 if [ "${suite}" == "exit" ]
206 then
207 echo "Exit: found ${suite}"
208 echo -e "\n\n\n" >> ${CURRSUITESLOG}
209 endTime=`date +%Y-%m-%d" "%H:%M:%S`
210 echo -e "Started:\t${startTime}\tEnd:\t${endTime}\tHost:\t${TESTHOST}" >> ${CURRSUITESLOG}
211 # cat buildInfo >> ${CURRSUITESLOG}
212 echo "Started: ${startTime} End: ${endTime} ${TESTHOST}"
213 cp -f ${CURRSUITESLOG} ${SUITESLOG}
214 chmod go+r ${SUITESLOG}
215 move_log ${TESTDIR} ${TESTHOST} ${OSPLATFROM} ${HWPROC} ${timeId} ${TESTLOGDIR}
216
217
218 SUBJECT="Regression Test: ${SANDBOX} ${timeId}/${SVNREVUSED} - ${OSPLATFROM}_${HWPROC} - ${TESTHOST} - ${BUILTOS} ${BUILTOSKERN} ${BUILTDATE}"
219 FILELIST="${SUITESLOG}"
220 ### ${TOOLDIR}/textMail "${SUBJECT}" ${MAILTOEXT} "${FILELIST}"
221
222 exit
223 fi
224 suiteStart=`date +%Y-%m-%d" "%H:%M:%S`
225 echo
226 echo "------------ ${suite} "
227 echo "${suiteStart} ------------ ${suite} " >> ${LOG}
228
229 # Remove possible leftovers:
230 rm -rf ${suite}
231 # (re)Create the suite dir:
232 mkdir -p ${suite}
233 chmod go+rx ${suite}
234 cd ${suite}
235
236 ### VMPID=`${TOOLDIR}/beginVmStat vmstat.log`
237
238 date +%H:%M:%S > ${TESTLOG_PLATF_DIR}/suiteStart.txt
239 # ---------------
240 # The test suite is run here:
241 JUNIT=`echo ${suite} | grep org.apache.derbyTesting`
242 if [ "${JUNIT}" == "" ]
243 then
244 echo "java ${VERBOSE} ${USEDERBYNET} org.apache.derbyTesting.functionTests.harness.RunSuite ${suite}"
245 echo -n "Start: ${suiteStart} End: "
246 java ${VERBOSE} ${USEDERBYNET} org.apache.derbyTesting.functionTests.harness.RunSuite ${suite} > out.log 2>out.err
247 else
248 echo "java junit.textui.TestRunner ${suite}"
249 echo -n "Start: ${suiteStart} End: "
250 java junit.textui.TestRunner ${suite} > out.log 2>out.err
251 java org.apache.derby.tools.sysinfo > ${suite}_report.txt # To have sysinfo for junit tests
252 fi
253 # ---------------
254 echo "" > ${TESTLOG_PLATF_DIR}/suiteStart.txt
255
256 ### ${TOOLDIR}/endVmStat ${VMPID} vmstat.log
257 date +%Y-%m-%d" "%H:%M:%S
258 # date +%Y-%m-%d" "%H:%M:%S >> suite.log
259
260 if [ "${SEND_UPDATE_EXECUTE_REVISION_MISMATCH_MSG}" = "true" ]
261 then
262 SVNREVUSED=`grep ".jar] " ${suite}_report.txt | head -1 | gawk '{ print $NF }' | sed -e 's/(//' | sed -e 's/)//'`
263
264 if [ "${SVNREVUSED}" != "${REVISION}" ]
265 then
266 SUBJECT="Regression Test: ${SANDBOX} Expected revision ${REVISION}, found ${SVNREVUSED} - ${OSPLATFROM}_${HWPROC} - ${TESTHOST} - ${BUILTDATE}"
267 FILELIST="" # JVM1.6 produces a LARGE _report.txt file. "${suite}_report.txt"
268 ### UNCOMMENT TO SEND MAIL. ${TOOLDIR}/textMail "${SUBJECT}" ${MAILTOEXT} "${FILELIST}"
269 ### UNCOMMENT TO SEND MAIL. ${TOOLDIR}/textMail "${SUBJECT}" ${MAILTOINT} "${FILELIST}"
270 SEND_UPDATE_EXECUTE_REVISION_MISMATCH_MSG="false"
271 else
272 SUBJECT="Regression Test: ${SANDBOX} ${REVISION}/${SVNREVUSED} - ${OSPLATFROM}_${HWPROC} - ${TESTHOST} - ${BUILTDATE}"
273 FILELIST="${suite}_report.txt"
274 ### UNCOMMENT TO SEND MAIL. ${TOOLDIR}/textMail "${SUBJECT}" ${MAILTOEXT} "${FILELIST}"
275 ### UNCOMMENT TO SEND MAIL. ${TOOLDIR}/textMail "${SUBJECT}" ${MAILTOINT} "${FILELIST}"
276 SEND_UPDATE_EXECUTE_REVISION_MISMATCH_MSG="false"
277
278 fi
279
280 fi
281
282
283 TESTSRUN="UNKNOWN"
284 PASS="UNKNOWN"
285 FAIL="UNKNOWN"
286 SKIPPED="UNKNOWN"
287 DURATION="UNKNOWN"
288 if [ "${JUNIT}" == "" ]
289 then
290 TESTSRUN=`grep "Tests Run" ${suite}_report.txt | gawk '{ print $1 }'`
291 PASS=`grep "% Pass" ${suite}_report.txt | gawk '{ print $3 }' | sed -e 's|(||'`
292 FAIL=`grep "% Fail" ${suite}_report.txt | gawk '{ print $3 }' | sed -e 's|(||'`
293 SKIPPED=`grep "Suites skipped" ${suite}_report.txt | gawk '{ print $1 }'`
294 DURATION=`grep "Test Run Duration:" ${suite}_report.txt | gawk '{ print $4 }'`
295 else
296 OK=`tail -3 out.log | grep "OK ("`
297 FAILURES=`tail -3 out.log | grep "FAILURES"`
298 if [ "${OK}" != "" ]
299 then
300 TESTSRUN=`tail -3 out.log | grep "OK ("| gawk '{ print $2 }' | sed -e s/\(//`
301 PASS=${TESTSRUN}
302 FAIL=0
303 SKIPPED=0 # NA for JUnit tests
304 TIME=`tail -5 out.log | grep "^Time: " ` # seconds as: 1,498.864
305 SECSONLY=`echo ${TIME} | gawk '{ print $2 }' | gawk -F. '{ print $1 }' | sed -e s/,//`
306 DURATION=`${TOOLDIR}/fromSeconds ${SECSONLY}`
307 elif [ "${FAILURES}" != "" ]
308 then
309 SUMMARY=`tail -3 out.log | grep "^Tests run: "`
310 TESTSRUN=`echo ${SUMMARY} | gawk '{ print $3 }' | sed -e s/,//`
311 FAILES=`echo ${SUMMARY} | gawk '{ print $5 }' | sed -e s/,//` # Assertion in test failed
312 ERRS=`echo ${SUMMARY} | gawk '{ print $7 }'` # Unhandled error, exception not catched...
313 let " PASS = ${TESTSRUN} - ${FAILES} - ${ERRS} " # FIXME: may get PASS less than 0....
314 FAIL="F:${FAILES},E:${ERRS}"
315 SKIPPED=0 # NA for JUnit tests
316 TIME=`grep "^Time: " out.log` # seconds as: 1,498.864
317 SECSONLY=`echo ${TIME} | gawk '{ print $2 }' | gawk -F. '{ print $1 }' | sed -e s/,//`
318 DURATION=`${TOOLDIR}/fromSeconds ${SECSONLY}`
319 grep "^.*) " out.log > ${suite}_fail.txt
320 fi
321 cat out.log >> ${suite}_report.txt
322 echo "------------------------------" >> ${suite}_report.txt
323 cat out.err >> ${suite}_report.txt
324 fi
325 MSG="${TESTSRUN} \t${PASS} \t${FAIL} \t${SKIPPED} \t${DURATION}"
326 TEXT="${HEADER}\n${MSG}"
327 echo -e "${TEXT}"
328 echo -e "${TEXT}" > suite.log
329 echo -e "${TEXT}" >> ${LOG}
330
331 if [ "${JAVAVENDOR}" == "" ]
332 then
333 JAVAVERSION=`grep '(build ' ../OS_HW | head -1 | gawk -F\( '{ print $NF }' | gawk '{ print $2 }' | sed -e s/\)//`
334 JAVAVENDOR=`grep "Java Vendor:" ${suite}_report.txt | sed -e 's/Java Vendor://'`
335 fi
336 echo ${JAVAVENDOR} > ${JAVAINFO}
337 echo ${JAVAVERSION} >> ${JAVAINFO}
338
339
340 if [ "${FAIL}" != "0" ]
341 then
342 FAILLOG="${TESTLOG_PLATF_DIR}/${timeId}-${suite}_fail.txt"
343 DIFFLOG="${TESTLOG_PLATF_DIR}/${timeId}-${suite}_diff.txt"
344
345 echo "${suite} fail *************************************************************" > ${FAILLOG}
346 cat ${suite}_fail.txt >> ${FAILLOG}
347 echo "${suite} fail *************************************************************" > ${DIFFLOG}
348 cat ${suite}_fail.txt >> ${DIFFLOG}
349 echo "" >> ${DIFFLOG}
350 echo "${suite} diff -------------------------------------------------------------" >> ${DIFFLOG}
351 cat ${suite}_report.txt >> ${DIFFLOG}
352 chmod go+r ${DIFFLOG} ${FAILLOG}
353 fi
354 if [ ${SKIPPED} != 0 ]
355 then
356 SKIPLOG="${TESTLOG_PLATF_DIR}/${timeId}-${suite}_skipped.txt"
357 cat ${suite}_skip.txt > ${SKIPLOG}
358 fi
359
360 cd ../
361 echo -e "${suite} \t${MSG}" >> ${CURRSUITESLOG}
362
363 #----------------
364 # Make it avalable as soon as possible:
365 cp -f ${CURRSUITESLOG} ${SUITESLOG}
366 chmod go+r ${SUITESLOG}
367 #----------------
368
369 echo "==========================================================="
370
371 done
372 echo -e "\n\n\n" >> ${CURRSUITESLOG}
373 endTime=`date +%Y-%m-%d" "%H:%M:%S`
374 echo -e "Started:\t${startTime}\tEnd:\t${endTime}\tHost:\t${TESTHOST}" >> ${CURRSUITESLOG}
375 # cat buildInfo >> ${CURRSUITESLOG}
376 echo "Started: ${startTime} End: ${endTime} ${TESTHOST}"
377 cp -f ${CURRSUITESLOG} ${SUITESLOG}
378 chmod go+r ${SUITESLOG}
379 move_log ${TESTDIR} ${TESTHOST} ${OSPLATFROM} ${HWPROC} ${timeId} ${TESTLOGDIR}
380
381 SUBJECT="Regression Test: ${SANDBOX} ${timeId}/${SVNREVUSED} - ${OSPLATFROM}_${HWPROC} - ${TESTHOST} - ${BUILTOS} ${BUILTOSKERN} ${BUILTDATE}"
382 FILELIST="${SUITESLOG}"
383 ### UNCOMMENT TO SEND MAIL. ${TOOLDIR}/textMail "${SUBJECT}" ${MAILTOINT} "${FILELIST}"