Debian Codegraph @

my_docs/22/lmod_zoom_mtg_2022_03_01/presentation.tex

raw
1 \documentclass{beamer}
2
3 % You can also use a 16:9 aspect ratio:
4 %\documentclass[aspectratio=169]{beamer}
5 \usetheme{TACC16}
6
7 % It's possible to move the footer to the right:
8 %\usetheme[rightfooter]{TACC16}
9
10 \begin{document}
11 \title[Lmod]{Lmod Testing System}
12 \author{Robert McLay}
13 \date{March. 1, 2022}
14
15 % page 1
16 \frame{\titlepage}
17
18
19 % page 2
20 \begin{frame}{Lmod Testing System}
21 \center{\includegraphics[width=.9\textwidth]{[email protected]}}
22 \begin{itemize}
23 \item Testing philosophy in Lmod
24 \item Goals of testing Lmod
25 \item Hermes/tm basic operations
26 \item Details of how an Lmod test works
27 \item Future Topics
28 \end{itemize}
29 \end{frame}
30
31 % page 3
32 \begin{frame}{Alternative story}
33 \begin{itemize}
34 \item How I used a testing tool I already had
35 \item How to shoehorn the Lmod testing to use \texttt{tm}
36 \item Why I'm using a system testing method, not unit tests.
37 \item System tests came first for me, unit testing later.
38 \item Lmod uses some unit tests as well.
39 \end{itemize}
40 \end{frame}
41
42 % page 4
43 \begin{rame}{Testing philosophy in Lmod}
44 \begin{itemize}
45 \item Lmod's success relies heavily on the testing system.
46 \item Passing all the tests usually means a new version can be released.
47 \item I don't think that anyone is using it beside Lmod (But it is
48 very useful)
49 \item My philosophy is to test features in general
50 \item Not to setup a torture test
51 \item No way I can test every possible scenario.
52 \item My imagination is not that good.
53 \end{itemize}
54 \end{frame}
55
56
57 % page 5
58 \begin{frame}{Goals of testing Lmod}
59 \begin{itemize}
60 \item Test various features of Lmod.
61 \item New feature won't break old features.
62 \item Test Lmod on Linux/MacOS, Lua 5.1 to 5.4
63 \item Make development of Lmod easier.
64 \item Add tests of new bugs $\Rightarrow$ Don't repeat them!
65 \end{itemize}
66 \end{frame}
67
68 % page 6
69 \begin{frame}{It is hard to test everything}
70 \begin{itemize}
71 \item Testing Old data with new versions(Collections, spiderT.lua)
72 \item One test (end2end) builds Lmod and tests the built version
73 \item All other tests use the source code directly
74 \item Special hacks to use configuration options.
75 \item Environment variable are checked NOT configuration options
76 \end{itemize}
77 \end{frame}
78
79 % page 7
80 \begin{frame}{Hermes/tm Testing system}
81 \begin{itemize}
82 \item Hermes is a group of tools to help with testing
83 \item \texttt{tm} is the testing manager program.
84 \item The main function of \texttt{tm} is to select tests and run them.
85 \item Each test is independent!
86 \item \texttt{tm} knows \emph{nothing} about what is being tested.
87 \item Must tell if test passed via special file (Lua file named
88 t1.results)
89 \item Three kinds of results
90 \begin{enumerate}
91 \item Passed: All steps passed
92 \item Failed: Did not produce a t1.results file
93 \item Diffed: Produced diffs between gold files and test
94 result files.
95 \end{enumerate}
96 \end{itemize}
97 \end{frame}
98
99 % page 8
100 \begin{frame}{\texttt{tm} flow}
101 \begin{itemize}
102 \item \texttt{tm} searches for tests from the current directory
103 down
104 \item It is looking for files with the *.tdesc extension (testDir)
105 \item Once all tests have been selected, it runs them all
106 \item For each test directory a sub-dir tree is created.
107 \item Typically: t1/$<$\$TARG$>$-$<$date\_time$>$-$<$uname
108 -s$>$-$<$arch$>$-$<$test\_name$>$
109 \item The above dir is the outputDir
110 \item The test is run in \$outputDir
111 \item The generated bash test script is named {\color{blue} t1.script}
112 \item The log of the run is {\color{blue} t1.log}
113 \item The results file are {\color{blue} t1.result} and {\color{blue} t1.runtime}
114 \end{itemize}
115 \end{frame}
116
117 % page 9
118 \begin{frame}{Every project using \texttt{tm} must have an \emph{acceptance} tool}
119 \begin{itemize}
120 \item There must be an automatic way to decide a test passed.
121 \item A numerical code can use an $L^2$ norm.
122 \item The new answer can be different but close w/ numerical codes.
123 \item Lmod use diff on stdout and stderr between gold and test
124 results
125 \item Filtering is required to deal with OS and file location
126 differences
127 \item To pass the filtered result {\color{blue} \emph{must}} be
128 the same.
129 \item This is a major pain but it has been worth the effort.
130 \end{itemize}
131 \end{frame}
132
133 % page 10
134 \begin{frame}{Test files (*.tdesc)}
135 \begin{itemize}
136 \item The testDescript is a table describing the the test
137 \item Some special parameters are:
138 \begin{enumerate}
139 \item \$(testDir): where the *.tdesc is located
140 \item \$(projectDir): where Hermes.db is located (top of the
141 project)
142 \item \$(outputDir): where the test is run
143 \item \$(resultFn): The name of the results lua file.
144 \end{enumerate}
145 \end{itemize}
146 \end{frame}
147
148 % page 11
149 \begin{frame}{Lmod tests}
150 \begin{itemize}
151 \item Uptil now this talk has been about \texttt{tm}
152 \item Now lets talk about Lmod tests:
153 \begin{itemize}
154 \item Each test contains multiple steps
155 \item Each step generates \_stderr.\#\#\# and \_stdout.\#\#\# files
156 \item These are combined and filtered into err.txt and out.txt
157 \item These file are compared with the gold files in \$testDir
158 \item Result file is generated.
159 \item To pass all steps must be the same!
160 \end{itemize}
161 \end{itemize}
162 \end{frame}
163
164 % page 12
165 \begin{frame}[fragile]
166 \frametitle{\texttt{extension.tdesc}}
167 {\tiny
168 \begin{semiverbatim}
169 local testName = "extensions"
170 testdescript = \{
171 keywords = \{testName \},
172 active = true,
173 testName = testName,
174
175 runScript = [[
176 . $(projectDir)/rt/common\_funcs.sh
177 unsetMT; initStdEnvVars
178 export MODULEPATH\_ROOT=$(testDir)/mf
179 export MODULEPATH=$MODULEPATH\_ROOT/Core
180 rm -rf \_stderr.* \_stdout.* err.* out.* .lmod.d
181
182 runLmod --version # 1
183 runLmod avail # 2
184
185 # combine \_stdout.[0-9][0-9][0-9] -> \_stdout.orig
186 # cleanup \_stdout.orig -> out.txt
187
188 # combine \_stderr.[0-9][0-9][0-9] -> \_stderr.orig
189 # cleanup \_stderr.orig -> err.txt
190
191 wrapperDiff --csv results.csv $(testDir)/out.txt out.txt
192 wrapperDiff --csv results.csv $(testDir)/err.txt err.txt
193 testFinish -r $(resultFn) -t $(runtimeFn) results.csv
194 ]],
195 tests = \{
196 \{ id='t1'\},
197 \},
198 \}
199 \end{semiverbatim}
200 }
201 \end{frame}
202 %$
203 % page 13
204 \begin{frame}{\$(projectDir)/rt/common\_funcs.sh}
205 \begin{itemize}
206 \item Common bash shell functions are in this file
207 \item runLmod: runs the Lmod command
208 \item runBase: base command (explained later)
209 \item cleanup: Makes output generic (canonical?)
210 \item initStdEnvVars: set standard env vars, cleans up my env
211 \item unsetMT: remove moduletable from env
212 \end{itemize}
213 \end{frame}
214
215 % page 14
216 \begin{frame}[fragile]
217 \frametitle{\texttt{runLmod}}
218 {\tiny
219 \begin{semiverbatim}
220 runLmod ()
221 \{
222 runBase \$LUA\_EXEC \$projectDir/src/lmod.in.lua bash --regression\_testing "\$@"
223 eval "`cat \_stdout.\$NUM`"
224 \}
225 \end{semiverbatim}
226 }
227 \end{frame}
228
229 % page 15
230 \begin{frame}[fragile]
231 \frametitle{\texttt{runBase}}
232 {\tiny
233 \begin{semiverbatim}
234 runBase ()
235 \{
236 COUNT=\$((\$COUNT + 1))
237 numStep=\$((\$numStep+1))
238 NUM=`printf "\%03d" \$numStep`
239 echo "===========================" > \_stderr.\$NUM
240 echo "step \$COUNT" >> \_stderr.\$NUM
241 echo "\$@" >> \_stderr.\$NUM
242 echo "===========================" >> \_stderr.\$NUM
243
244 echo "===========================" > \_stdout.\$NUM
245 echo "step \$COUNT" >> \_stdout.\$NUM
246 echo "\$@" >> \_stdout.\$NUM
247 echo "===========================" >> \_stdout.\$NUM
248
249 numStep=\$((\$numStep+1))
250 NUM=`printf "\%03d" \$numStep`
251 "\$@" > \_stdout.\$NUM 2>> \_stderr.\$NUM
252 \}
253 \end{semiverbatim}
254 }
255 \end{frame}
256
257
258 % page 16
259 \begin{frame}[fragile]
260 \frametitle{Cleanup for stderr}
261 {\tiny
262 \begin{semiverbatim}
263 cat \_stderr.[0-9][0-9][0-9] > \_stderr.orig
264 cleanUp \_stderr.orig err.txt
265 \end{semiverbatim}
266 }
267 \begin{itemize}
268 \item Combine all stderr files into \_stderr.orig
269 \item Use the cleanup shell function to canonicalize err.txt output
270 \end{itemize}
271 \end{frame}
272
273 % page 17
274 \begin{frame}[fragile]
275 \frametitle{Cleanup for stdout}
276 {\tiny
277 \begin{semiverbatim}
278 cat \_stdout.[0-9][0-9][0-9] > \_stdout.orig
279 joinBase64Results -bash \_stdout.orig \_stdout.new
280 cleanUp \_stdout.new out.txt
281 \end{semiverbatim}
282 }
283 \begin{itemize}
284 \item Combine all stdout files into \_stdout.orig
285 \item Convert all base64 text into regular text
286 \item Use the cleanup shell function to canonicalize out.txt output
287 \end{itemize}
288 \end{frame}
289
290 % page 18
291 \begin{frame}[fragile]
292 \frametitle{Cleanup script}
293 {\tiny
294 \begin{semiverbatim}
295 \end{semiverbatim}
296 }
297 \begin{itemize}
298 \item converts local path names into ``ProjectDIR''
299 \item converts path to lua or sha1 to generic names
300 \item Cleans up error msgs
301 \item And many other fixes.
302 \end{itemize}
303 \end{frame}
304
305 % page 19
306 \begin{frame}[fragile]
307 \frametitle{Cleanup script (II)}
308 {\tiny
309 \begin{semiverbatim}
310 {\color{blue} \_stderr.orig}:
311 ===========================
312 step 8
313 /opt/apps/lua/lua/bin/lua /Users/mclay/w/lmod/src/lmod.in.lua bash --rtesting -t avail
314 ===========================
315 /Users/mclay/w/lmod/rt/avail/mf/Core:
316 PrgEnv
317 admin/
318 admin/admin-1.0
319
320 {\color{blue} err.txt}:
321 ===========================
322 step 8
323 lua ProjectDIR/src/lmod.in.lua bash --rtesting -t avail
324 ===========================
325 ProjectDIR/rt/avail/mf/Core:
326 PrgEnv
327 admin/
328 admin/admin-1.0
329 \end{semiverbatim}
330 }
331 \begin{itemize}
332 \item Cleanup: \_stderr.orig $\Rightarrow$ err.txt
333 \end{itemize}
334 \end{frame}
335
336 % page 20
337 \begin{frame}[fragile]
338 \frametitle{Deciding if a test passes}
339 {\tiny
340 \begin{semiverbatim}
341 rm -f results.csv
342 wrapperDiff --csv results.csv \$(testDir)/out.txt out.txt
343 wrapperDiff --csv results.csv \$(testDir)/err.txt err.txt
344 testFinish -r \$(resultFn) -t \$(runtimeFn) results.csv
345 \end{semiverbatim}
346 }
347 \begin{itemize}
348 \item \texttt{wrapperDiff} is a hermes tool that runs diff and
349 generates a csv file (results.csv)
350 \item It also removes the Lmod version info from err.txt
351 \item \texttt{testFinish} is another hermes tool that converts
352 results.csv into \$resultFn
353 \item Then \texttt{tm} reads \$resultFn to decide if this test passes
354 \end{itemize}
355 \end{frame}
356
357 % page 21
358 \begin{frame}{Future Topics}
359 \begin{itemize}
360 \item Write one new test.
361 \item Explain how Mname object converts names into a filename.
362 \item More internals of Lmod?
363 \end{itemize}
364 \end{frame}
365
366 \end{document}