blob: cc8ffd6b918b47f3d8771da1543861b220b8d3c3 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
|
#!/usr/bin/env bash
#
# This script executes PAL tests from the specified build location.
#
if [ $# -lt 1 -o $# -gt 3 ]
then
echo "Usage..."
echo "runpaltests.sh <path to root build directory> [<path to temp folder for PAL tests>]"
echo
echo "For example:"
echo "runpaltests.sh /projectk/build/debug"
echo
exit 1
fi
echo
echo "***** Testing PAL *****"
echo
# Store the location of the root of build directory
BUILD_ROOD_DIR=$1
# Create path to the compiled PAL tets in the build directory
PAL_TEST_BUILD=$BUILD_ROOD_DIR/src/pal/tests/palsuite
echo Running PAL tests from $PAL_TEST_BUILD
# Create absolute path to the file that contains a list of PAL tests to execute.
# This file is located next to this script in the source tree
RELATIVE_PATH_TO_PAL_TESTS=$0
# Remove the name of this script from the path
RELATIVE_PATH_TO_PAL_TESTS=${RELATIVE_PATH_TO_PAL_TESTS%/*.*}
# Change current directory to the location of this script
cd $RELATIVE_PATH_TO_PAL_TESTS
# Environment variable PWD contains absolute path to the current folder
# so use it to create absolute path to the file with a list of tests.
PAL_TEST_LIST=$PWD/paltestlist.txt
# Change current directory back to the original location
cd $OLDPWD
echo The list of PAL tests to run will be read from $PAL_TEST_LIST
# Create the test output root directory
mkdir -p /tmp/PalTestOutput
if [ ! -d /tmp/PalTestOutput ]; then
rm -f -r /tmp/PalTestOutput
mkdir -p /tmp/PalTestOutput
fi
# Determine the folder to use for PAL test output during the run, and the folder where output files were requested to be copied.
# First check if the output folder was passed as a parameter to the script. It is supposed be the second parameter so check if
# we have more than 1 argument.
PAL_TEST_OUTPUT_DIR=/tmp/PalTestOutput/default
if [ $# -gt 1 ]
then
COPY_TO_TEST_OUTPUT_DIR=$2
else
COPY_TO_TEST_OUTPUT_DIR=$PAL_TEST_OUTPUT_DIR
fi
# Determine the folder to use for PAL test output during the run
if [ "$COPY_TO_TEST_OUTPUT_DIR" != "$PAL_TEST_OUTPUT_DIR" ]; then
# Output files were requested to be copied to a specific folder. In this mode, we need to support parallel runs of PAL tests
# on the same machine. Make a unique temp folder for working output inside /tmp/PalTestOutput.
PAL_TEST_OUTPUT_DIR=$(mktemp -d /tmp/PalTestOutput/tmp.XXXXXXXX)
fi
echo PAL tests will store their temporary files and output in $PAL_TEST_OUTPUT_DIR.
if [ "$COPY_TO_TEST_OUTPUT_DIR" != "$PAL_TEST_OUTPUT_DIR" ]; then
echo Output files will be copied to $COPY_TO_TEST_OUTPUT_DIR at the end.
fi
# Path to a file that will contains a list PAL tests that failed during the test run.
PAL_FAILED_TEST_LIST=$PAL_TEST_OUTPUT_DIR/palfailedtests.txt
# Path to a file that will contain the XUnit style test result for Jenkins
# We use a temp file as at the end we have to prepend with the number of tests
# and failures
PAL_XUNIT_TEST_LIST_TMP=$PAL_TEST_OUTPUT_DIR/pal_tests.xml.tmp
PAL_XUNIT_TEST_LIST=$PAL_TEST_OUTPUT_DIR/pal_tests.xml
# Capturing stdout and stderr
PAL_OUT_FILE=$PAL_TEST_OUTPUT_DIR/pal_test_out
# Remove and recreate the temporary test output directory, and the directory where output files were requested to be copied.
if [ "$COPY_TO_TEST_OUTPUT_DIR" == "$PAL_TEST_OUTPUT_DIR" ]; then
if [ -e $PAL_TEST_OUTPUT_DIR ]; then
rm -f -r $PAL_TEST_OUTPUT_DIR
fi
mkdir -p $PAL_TEST_OUTPUT_DIR
else
# No need to recreate the temp output directory, as mktemp would have created a unique empty directory
if [ -e $COPY_TO_TEST_OUTPUT_DIR ]; then
rm -f -r $COPY_TO_TEST_OUTPUT_DIR
fi
mkdir -p $COPY_TO_TEST_OUTPUT_DIR
if [ ! -d $COPY_TO_TEST_OUTPUT_DIR ]; then
echo Failed to create $COPY_TO_TEST_OUTPUT_DIR.
COPY_TO_TEST_OUTPUT_DIR=$PAL_TEST_OUTPUT_DIR
fi
fi
cd $PAL_TEST_OUTPUT_DIR
echo
echo "Running tests..."
echo
NUMBER_OF_PASSED_TESTS=0
NUMBER_OF_FAILED_TESTS=0
# Read PAL tests names from the $PAL_TEST_LIST file and run them one by one.
while read TEST_NAME
do
# Remove stdout/stderr file if it exists
rm -f $PAL_OUT_FILE
# Create a folder with the test name, and use that as the working directory for the test. Many PAL tests don't clean up after
# themselves and may leave files/directories around, but even to handle test failures that result in a dirty state, run each
# test in its own folder.
TEST_WORKING_DIR=$(basename $TEST_NAME)
if [ -e $TEST_WORKING_DIR ]; then
rm -f -r $TEST_WORKING_DIR
fi
mkdir $TEST_WORKING_DIR
cd $TEST_WORKING_DIR
# Create path to a test executable to run
TEST_COMMAND="$PAL_TEST_BUILD/$TEST_NAME"
echo -n .
# Redirect to temp file
$TEST_COMMAND 2>&1 | tee ${PAL_OUT_FILE} ; ( exit ${PIPESTATUS[0]} )
# Get exit code of the test process.
TEST_EXIT_CODE=$?
# Change back to the output directory, and remove the test's working directory if it's empty
cd $PAL_TEST_OUTPUT_DIR
rmdir $TEST_WORKING_DIR 2>/dev/null
TEST_XUNIT_NAME=$(dirname $TEST_NAME)
TEST_XUNIT_CLASSNAME=$(dirname $TEST_XUNIT_NAME)
TEST_XUNIT_NAME=${TEST_XUNIT_NAME#*/}
TEST_XUNIT_NAME=${TEST_XUNIT_NAME#*/}
TEST_XUNIT_NAME=$(echo $TEST_XUNIT_NAME | tr / .)
TEST_XUNIT_CLASSNAME=$(echo $TEST_XUNIT_CLASSNAME | tr / .)
echo -n "<test name=\"$TEST_XUNIT_CLASSNAME.$TEST_XUNIT_NAME\" type=\"$TEST_XUNIT_CLASSNAME\" method=\"$TEST_XUNIT_NAME\" result=\"" >> $PAL_XUNIT_TEST_LIST_TMP
# If the exit code is 0 then the test passed, otherwise record a failure.
if [ "$TEST_EXIT_CODE" -eq "0" ]; then
NUMBER_OF_PASSED_TESTS=$(($NUMBER_OF_PASSED_TESTS + 1))
echo "Pass\" />" >> $PAL_XUNIT_TEST_LIST_TMP
else
echo "Fail\" >" >> $PAL_XUNIT_TEST_LIST_TMP
echo "<failure exception-type=\"Exit code: $TEST_EXIT_CODE\">" >> $PAL_XUNIT_TEST_LIST_TMP
echo "<message><![CDATA[$(cat $PAL_OUT_FILE)]]></message>" >> $PAL_XUNIT_TEST_LIST_TMP
echo "</failure>" >> $PAL_XUNIT_TEST_LIST_TMP
echo "</test>" >> $PAL_XUNIT_TEST_LIST_TMP
FAILED_TEST="$TEST_NAME. Exit code: $TEST_EXIT_CODE"
echo
echo FAILED: $FAILED_TEST
echo
# Store the name of the failed test in the list of failed tests.
echo $FAILED_TEST >> $PAL_FAILED_TEST_LIST
NUMBER_OF_FAILED_TESTS=$(($NUMBER_OF_FAILED_TESTS + 1))
fi
done < $PAL_TEST_LIST
# We are done running tests.
echo
echo Finished running PAL tests.
echo
# Finish XUnit file, output to finished file with the number of failures, tests etc
NUMBER_OF_TESTS=$(($NUMBER_OF_PASSED_TESTS + $NUMBER_OF_FAILED_TESTS))
XUNIT_SUFFIX="</collection>\n"
XUNIT_SUFFIX+="</assembly>\n"
XUNIT_SUFFIX+="</assemblies>"
XUNIT_PREFIX="<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
XUNIT_PREFIX+="<assemblies>\n"
XUNIT_PREFIX+="<assembly name=\"PAL\" total=\"$NUMBER_OF_TESTS\" passed=\"$NUMBER_OF_PASSED_TESTS\" failed=\"$NUMBER_OF_FAILED_TESTS\" skipped=\"0\">\n"
XUNIT_PREFIX+="<collection total=\"$NUMBER_OF_TESTS\" passed=\"$NUMBER_OF_PASSED_TESTS\" failed=\"$NUMBER_OF_FAILED_TESTS\" skipped=\"0\" name=\"palsuite\">"
printf "$XUNIT_SUFFIX" >> $PAL_XUNIT_TEST_LIST_TMP
printf "$XUNIT_PREFIX" | cat - $PAL_XUNIT_TEST_LIST_TMP > $PAL_XUNIT_TEST_LIST
# If there were tests failures then print the list of failed tests
if [ $NUMBER_OF_FAILED_TESTS -gt "0" ]; then
echo "The following test(s) failed:"
while read FAILED_TEST_NAME
do
echo $FAILED_TEST_NAME
done < $PAL_FAILED_TEST_LIST
echo
fi
echo PAL Test Results:
echo " Passed: $NUMBER_OF_PASSED_TESTS"
echo " Failed: $NUMBER_OF_FAILED_TESTS"
echo
if [ "$COPY_TO_TEST_OUTPUT_DIR" != "$PAL_TEST_OUTPUT_DIR" ]; then
mv -f $PAL_TEST_OUTPUT_DIR/* $COPY_TO_TEST_OUTPUT_DIR/
rm -f -r $PAL_TEST_OUTPUT_DIR
echo Copied PAL test output files to $COPY_TO_TEST_OUTPUT_DIR.
fi
# Set exit code to be equal to the number PAL tests that have failed.
# Exit code 0 indicates success.
exit $NUMBER_OF_FAILED_TESTS
|