| 1 |
\documentclass{beamer} |
| 2 |
|
| 3 |
% You can also use a 16:9 aspect ratio: |
| 4 |
%\documentclass[aspectratio=169]{beamer} |
| 5 |
\usetheme{TACC16} |
| 6 |
|
| 7 |
% It's possible to move the footer to the right: |
| 8 |
%\usetheme[rightfooter]{TACC16} |
| 9 |
|
| 10 |
\begin{document} |
| 11 |
\title[Lmod]{Lmod Testing System} |
| 12 |
\author{Robert McLay} |
| 13 |
\date{March. 1, 2022} |
| 14 |
|
| 15 |
% page 1 |
| 16 |
\frame{\titlepage} |
| 17 |
|
| 18 |
|
| 19 |
% page 2 |
| 20 |
\begin{frame}{Lmod Testing System} |
| 21 |
\center{\includegraphics[width=.9\textwidth]{[email protected]}} |
| 22 |
\begin{itemize} |
| 23 |
\item Testing philosophy in Lmod |
| 24 |
\item Goals of testing Lmod |
| 25 |
\item Hermes/tm basic operations |
| 26 |
\item Details of how an Lmod test works |
| 27 |
\item Future Topics |
| 28 |
\end{itemize} |
| 29 |
\end{frame} |
| 30 |
|
| 31 |
% page 3 |
| 32 |
\begin{frame}{Alternative story} |
| 33 |
\begin{itemize} |
| 34 |
\item How I used a testing tool I already had |
| 35 |
\item How to shoehorn the Lmod testing to use \texttt{tm} |
| 36 |
\item Why I'm using a system testing method, not unit tests. |
| 37 |
\item System tests came first for me, unit testing later. |
| 38 |
\item Lmod uses some unit tests as well. |
| 39 |
\end{itemize} |
| 40 |
\end{frame} |
| 41 |
|
| 42 |
% page 4 |
| 43 |
\begin{rame}{Testing philosophy in Lmod} |
| 44 |
\begin{itemize} |
| 45 |
\item Lmod's success relies heavily on the testing system. |
| 46 |
\item Passing all the tests usually means a new version can be released. |
| 47 |
\item I don't think that anyone is using it beside Lmod (But it is |
| 48 |
very useful) |
| 49 |
\item My philosophy is to test features in general |
| 50 |
\item Not to setup a torture test |
| 51 |
\item No way I can test every possible scenario. |
| 52 |
\item My imagination is not that good. |
| 53 |
\end{itemize} |
| 54 |
\end{frame} |
| 55 |
|
| 56 |
|
| 57 |
% page 5 |
| 58 |
\begin{frame}{Goals of testing Lmod} |
| 59 |
\begin{itemize} |
| 60 |
\item Test various features of Lmod. |
| 61 |
\item New feature won't break old features. |
| 62 |
\item Test Lmod on Linux/MacOS, Lua 5.1 to 5.4 |
| 63 |
\item Make development of Lmod easier. |
| 64 |
\item Add tests of new bugs $\Rightarrow$ Don't repeat them! |
| 65 |
\end{itemize} |
| 66 |
\end{frame} |
| 67 |
|
| 68 |
% page 6 |
| 69 |
\begin{frame}{It is hard to test everything} |
| 70 |
\begin{itemize} |
| 71 |
\item Testing Old data with new versions(Collections, spiderT.lua) |
| 72 |
\item One test (end2end) builds Lmod and tests the built version |
| 73 |
\item All other tests use the source code directly |
| 74 |
\item Special hacks to use configuration options. |
| 75 |
\item Environment variable are checked NOT configuration options |
| 76 |
\end{itemize} |
| 77 |
\end{frame} |
| 78 |
|
| 79 |
% page 7 |
| 80 |
\begin{frame}{Hermes/tm Testing system} |
| 81 |
\begin{itemize} |
| 82 |
\item Hermes is a group of tools to help with testing |
| 83 |
\item \texttt{tm} is the testing manager program. |
| 84 |
\item The main function of \texttt{tm} is to select tests and run them. |
| 85 |
\item Each test is independent! |
| 86 |
\item \texttt{tm} knows \emph{nothing} about what is being tested. |
| 87 |
\item Must tell if test passed via special file (Lua file named |
| 88 |
t1.results) |
| 89 |
\item Three kinds of results |
| 90 |
\begin{enumerate} |
| 91 |
\item Passed: All steps passed |
| 92 |
\item Failed: Did not produce a t1.results file |
| 93 |
\item Diffed: Produced diffs between gold files and test |
| 94 |
result files. |
| 95 |
\end{enumerate} |
| 96 |
\end{itemize} |
| 97 |
\end{frame} |
| 98 |
|
| 99 |
% page 8 |
| 100 |
\begin{frame}{\texttt{tm} flow} |
| 101 |
\begin{itemize} |
| 102 |
\item \texttt{tm} searches for tests from the current directory |
| 103 |
down |
| 104 |
\item It is looking for files with the *.tdesc extension (testDir) |
| 105 |
\item Once all tests have been selected, it runs them all |
| 106 |
\item For each test directory a sub-dir tree is created. |
| 107 |
\item Typically: t1/$<$\$TARG$>$-$<$date\_time$>$-$<$uname |
| 108 |
-s$>$-$<$arch$>$-$<$test\_name$>$ |
| 109 |
\item The above dir is the outputDir |
| 110 |
\item The test is run in \$outputDir |
| 111 |
\item The generated bash test script is named {\color{blue} t1.script} |
| 112 |
\item The log of the run is {\color{blue} t1.log} |
| 113 |
\item The results file are {\color{blue} t1.result} and {\color{blue} t1.runtime} |
| 114 |
\end{itemize} |
| 115 |
\end{frame} |
| 116 |
|
| 117 |
% page 9 |
| 118 |
\begin{frame}{Every project using \texttt{tm} must have an \emph{acceptance} tool} |
| 119 |
\begin{itemize} |
| 120 |
\item There must be an automatic way to decide a test passed. |
| 121 |
\item A numerical code can use an $L^2$ norm. |
| 122 |
\item The new answer can be different but close w/ numerical codes. |
| 123 |
\item Lmod use diff on stdout and stderr between gold and test |
| 124 |
results |
| 125 |
\item Filtering is required to deal with OS and file location |
| 126 |
differences |
| 127 |
\item To pass the filtered result {\color{blue} \emph{must}} be |
| 128 |
the same. |
| 129 |
\item This is a major pain but it has been worth the effort. |
| 130 |
\end{itemize} |
| 131 |
\end{frame} |
| 132 |
|
| 133 |
% page 10 |
| 134 |
\begin{frame}{Test files (*.tdesc)} |
| 135 |
\begin{itemize} |
| 136 |
\item The testDescript is a table describing the the test |
| 137 |
\item Some special parameters are: |
| 138 |
\begin{enumerate} |
| 139 |
\item \$(testDir): where the *.tdesc is located |
| 140 |
\item \$(projectDir): where Hermes.db is located (top of the |
| 141 |
project) |
| 142 |
\item \$(outputDir): where the test is run |
| 143 |
\item \$(resultFn): The name of the results lua file. |
| 144 |
\end{enumerate} |
| 145 |
\end{itemize} |
| 146 |
\end{frame} |
| 147 |
|
| 148 |
% page 11 |
| 149 |
\begin{frame}{Lmod tests} |
| 150 |
\begin{itemize} |
| 151 |
\item Uptil now this talk has been about \texttt{tm} |
| 152 |
\item Now lets talk about Lmod tests: |
| 153 |
\begin{itemize} |
| 154 |
\item Each test contains multiple steps |
| 155 |
\item Each step generates \_stderr.\#\#\# and \_stdout.\#\#\# files |
| 156 |
\item These are combined and filtered into err.txt and out.txt |
| 157 |
\item These file are compared with the gold files in \$testDir |
| 158 |
\item Result file is generated. |
| 159 |
\item To pass all steps must be the same! |
| 160 |
\end{itemize} |
| 161 |
\end{itemize} |
| 162 |
\end{frame} |
| 163 |
|
| 164 |
% page 12 |
| 165 |
\begin{frame}[fragile] |
| 166 |
\frametitle{\texttt{extension.tdesc}} |
| 167 |
{\tiny |
| 168 |
\begin{semiverbatim} |
| 169 |
local testName = "extensions" |
| 170 |
testdescript = \{ |
| 171 |
keywords = \{testName \}, |
| 172 |
active = true, |
| 173 |
testName = testName, |
| 174 |
|
| 175 |
runScript = [[ |
| 176 |
. $(projectDir)/rt/common\_funcs.sh |
| 177 |
unsetMT; initStdEnvVars |
| 178 |
export MODULEPATH\_ROOT=$(testDir)/mf |
| 179 |
export MODULEPATH=$MODULEPATH\_ROOT/Core |
| 180 |
rm -rf \_stderr.* \_stdout.* err.* out.* .lmod.d |
| 181 |
|
| 182 |
runLmod --version # 1 |
| 183 |
runLmod avail # 2 |
| 184 |
|
| 185 |
# combine \_stdout.[0-9][0-9][0-9] -> \_stdout.orig |
| 186 |
# cleanup \_stdout.orig -> out.txt |
| 187 |
|
| 188 |
# combine \_stderr.[0-9][0-9][0-9] -> \_stderr.orig |
| 189 |
# cleanup \_stderr.orig -> err.txt |
| 190 |
|
| 191 |
wrapperDiff --csv results.csv $(testDir)/out.txt out.txt |
| 192 |
wrapperDiff --csv results.csv $(testDir)/err.txt err.txt |
| 193 |
testFinish -r $(resultFn) -t $(runtimeFn) results.csv |
| 194 |
]], |
| 195 |
tests = \{ |
| 196 |
\{ id='t1'\}, |
| 197 |
\}, |
| 198 |
\} |
| 199 |
\end{semiverbatim} |
| 200 |
} |
| 201 |
\end{frame} |
| 202 |
%$ |
| 203 |
% page 13 |
| 204 |
\begin{frame}{\$(projectDir)/rt/common\_funcs.sh} |
| 205 |
\begin{itemize} |
| 206 |
\item Common bash shell functions are in this file |
| 207 |
\item runLmod: runs the Lmod command |
| 208 |
\item runBase: base command (explained later) |
| 209 |
\item cleanup: Makes output generic (canonical?) |
| 210 |
\item initStdEnvVars: set standard env vars, cleans up my env |
| 211 |
\item unsetMT: remove moduletable from env |
| 212 |
\end{itemize} |
| 213 |
\end{frame} |
| 214 |
|
| 215 |
% page 14 |
| 216 |
\begin{frame}[fragile] |
| 217 |
\frametitle{\texttt{runLmod}} |
| 218 |
{\tiny |
| 219 |
\begin{semiverbatim} |
| 220 |
runLmod () |
| 221 |
\{ |
| 222 |
runBase \$LUA\_EXEC \$projectDir/src/lmod.in.lua bash --regression\_testing "\$@" |
| 223 |
eval "`cat \_stdout.\$NUM`" |
| 224 |
\} |
| 225 |
\end{semiverbatim} |
| 226 |
} |
| 227 |
\end{frame} |
| 228 |
|
| 229 |
% page 15 |
| 230 |
\begin{frame}[fragile] |
| 231 |
\frametitle{\texttt{runBase}} |
| 232 |
{\tiny |
| 233 |
\begin{semiverbatim} |
| 234 |
runBase () |
| 235 |
\{ |
| 236 |
COUNT=\$((\$COUNT + 1)) |
| 237 |
numStep=\$((\$numStep+1)) |
| 238 |
NUM=`printf "\%03d" \$numStep` |
| 239 |
echo "===========================" > \_stderr.\$NUM |
| 240 |
echo "step \$COUNT" >> \_stderr.\$NUM |
| 241 |
echo "\$@" >> \_stderr.\$NUM |
| 242 |
echo "===========================" >> \_stderr.\$NUM |
| 243 |
|
| 244 |
echo "===========================" > \_stdout.\$NUM |
| 245 |
echo "step \$COUNT" >> \_stdout.\$NUM |
| 246 |
echo "\$@" >> \_stdout.\$NUM |
| 247 |
echo "===========================" >> \_stdout.\$NUM |
| 248 |
|
| 249 |
numStep=\$((\$numStep+1)) |
| 250 |
NUM=`printf "\%03d" \$numStep` |
| 251 |
"\$@" > \_stdout.\$NUM 2>> \_stderr.\$NUM |
| 252 |
\} |
| 253 |
\end{semiverbatim} |
| 254 |
} |
| 255 |
\end{frame} |
| 256 |
|
| 257 |
|
| 258 |
% page 16 |
| 259 |
\begin{frame}[fragile] |
| 260 |
\frametitle{Cleanup for stderr} |
| 261 |
{\tiny |
| 262 |
\begin{semiverbatim} |
| 263 |
cat \_stderr.[0-9][0-9][0-9] > \_stderr.orig |
| 264 |
cleanUp \_stderr.orig err.txt |
| 265 |
\end{semiverbatim} |
| 266 |
} |
| 267 |
\begin{itemize} |
| 268 |
\item Combine all stderr files into \_stderr.orig |
| 269 |
\item Use the cleanup shell function to canonicalize err.txt output |
| 270 |
\end{itemize} |
| 271 |
\end{frame} |
| 272 |
|
| 273 |
% page 17 |
| 274 |
\begin{frame}[fragile] |
| 275 |
\frametitle{Cleanup for stdout} |
| 276 |
{\tiny |
| 277 |
\begin{semiverbatim} |
| 278 |
cat \_stdout.[0-9][0-9][0-9] > \_stdout.orig |
| 279 |
joinBase64Results -bash \_stdout.orig \_stdout.new |
| 280 |
cleanUp \_stdout.new out.txt |
| 281 |
\end{semiverbatim} |
| 282 |
} |
| 283 |
\begin{itemize} |
| 284 |
\item Combine all stdout files into \_stdout.orig |
| 285 |
\item Convert all base64 text into regular text |
| 286 |
\item Use the cleanup shell function to canonicalize out.txt output |
| 287 |
\end{itemize} |
| 288 |
\end{frame} |
| 289 |
|
| 290 |
% page 18 |
| 291 |
\begin{frame}[fragile] |
| 292 |
\frametitle{Cleanup script} |
| 293 |
{\tiny |
| 294 |
\begin{semiverbatim} |
| 295 |
\end{semiverbatim} |
| 296 |
} |
| 297 |
\begin{itemize} |
| 298 |
\item converts local path names into ``ProjectDIR'' |
| 299 |
\item converts path to lua or sha1 to generic names |
| 300 |
\item Cleans up error msgs |
| 301 |
\item And many other fixes. |
| 302 |
\end{itemize} |
| 303 |
\end{frame} |
| 304 |
|
| 305 |
% page 19 |
| 306 |
\begin{frame}[fragile] |
| 307 |
\frametitle{Cleanup script (II)} |
| 308 |
{\tiny |
| 309 |
\begin{semiverbatim} |
| 310 |
{\color{blue} \_stderr.orig}: |
| 311 |
=========================== |
| 312 |
step 8 |
| 313 |
/opt/apps/lua/lua/bin/lua /Users/mclay/w/lmod/src/lmod.in.lua bash --rtesting -t avail |
| 314 |
=========================== |
| 315 |
/Users/mclay/w/lmod/rt/avail/mf/Core: |
| 316 |
PrgEnv |
| 317 |
admin/ |
| 318 |
admin/admin-1.0 |
| 319 |
|
| 320 |
{\color{blue} err.txt}: |
| 321 |
=========================== |
| 322 |
step 8 |
| 323 |
lua ProjectDIR/src/lmod.in.lua bash --rtesting -t avail |
| 324 |
=========================== |
| 325 |
ProjectDIR/rt/avail/mf/Core: |
| 326 |
PrgEnv |
| 327 |
admin/ |
| 328 |
admin/admin-1.0 |
| 329 |
\end{semiverbatim} |
| 330 |
} |
| 331 |
\begin{itemize} |
| 332 |
\item Cleanup: \_stderr.orig $\Rightarrow$ err.txt |
| 333 |
\end{itemize} |
| 334 |
\end{frame} |
| 335 |
|
| 336 |
% page 20 |
| 337 |
\begin{frame}[fragile] |
| 338 |
\frametitle{Deciding if a test passes} |
| 339 |
{\tiny |
| 340 |
\begin{semiverbatim} |
| 341 |
rm -f results.csv |
| 342 |
wrapperDiff --csv results.csv \$(testDir)/out.txt out.txt |
| 343 |
wrapperDiff --csv results.csv \$(testDir)/err.txt err.txt |
| 344 |
testFinish -r \$(resultFn) -t \$(runtimeFn) results.csv |
| 345 |
\end{semiverbatim} |
| 346 |
} |
| 347 |
\begin{itemize} |
| 348 |
\item \texttt{wrapperDiff} is a hermes tool that runs diff and |
| 349 |
generates a csv file (results.csv) |
| 350 |
\item It also removes the Lmod version info from err.txt |
| 351 |
\item \texttt{testFinish} is another hermes tool that converts |
| 352 |
results.csv into \$resultFn |
| 353 |
\item Then \texttt{tm} reads \$resultFn to decide if this test passes |
| 354 |
\end{itemize} |
| 355 |
\end{frame} |
| 356 |
|
| 357 |
% page 21 |
| 358 |
\begin{frame}{Future Topics} |
| 359 |
\begin{itemize} |
| 360 |
\item Write one new test. |
| 361 |
\item Explain how Mname object converts names into a filename. |
| 362 |
\item More internals of Lmod? |
| 363 |
\end{itemize} |
| 364 |
\end{frame} |
| 365 |
|
| 366 |
\end{document} |