Debian Codegraph @

my_docs/22/lmod_zoom_mtg_2022_05_03/presentation.tex

raw
1 \documentclass{beamer}
2
3 % You can also use a 16:9 aspect ratio:
4 %\documentclass[aspectratio=169]{beamer}
5 \usetheme{TACC16}
6
7 % It's possible to move the footer to the right:
8 %\usetheme[rightfooter]{TACC16}
9
10 %% page
11 %\begin{frame}{}
12 % \begin{itemize}
13 % \item
14 % \end{itemize}
15 %\end{frame}
16 %
17 %% page
18 %\begin{frame}[fragile]
19 % \frametitle{}
20 % {\tiny
21 % \begin{semiverbatim}
22 % \end{semiverbatim}
23 %}
24 % \begin{itemize}
25 % \item
26 % \end{itemize}
27 %
28 %\end{frame}
29
30 \begin{document}
31 \title[Lmod]{The complicated story about TCL break}
32 \author{Robert McLay}
33 \date{May. 3, 2022}
34
35 % page 1
36 \frame{\titlepage}
37
38
39 % page 2
40 \begin{frame}{Outline}
41 \center{\includegraphics[width=.9\textwidth]{[email protected]}}
42 \begin{itemize}
43 \item Let's talk about TCL break (and LmodBreak)
44 \item Lmod didn't really support TCL break at all until Lmod 8.6
45 (really Lmod 8.7)
46 \item Years ago mailing list question: support for break
47 \item Lmod 6 and below could not support break
48 \item Why?
49 \end{itemize}
50 \end{frame}
51
52 % page 3
53 \begin{frame}[fragile]
54 \frametitle{Reminder: How Lmod works}
55 \begin{itemize}
56 \item In order to have a command effect the current shell
57 \item A simple module command for bash is given below
58 \item The \$LMOD\_CMD command generate shell commands as text
59 \item The eval "..." evaluate the text to change the current shell
60 \item For the rest of this talk: focus on what \$LMOD\_CMD produces
61 \end{itemize}
62 {\small
63 \begin{semiverbatim}
64 module () \{ eval "\$(\$LMOD\_CMD bash "\$@")"; \}
65 \end{semiverbatim}
66 }
67 \end{frame}
68
69 % page 4
70 \begin{frame}[fragile]
71 \frametitle{Reminder: How Lmod TCL processing works}
72 \begin{itemize}
73 \item Internally Lmod knows when a file is a TCL modulefile
74 \item No *.lua extension $\Rightarrow$ TCL modulefile
75 \item The program tcl2lua.tcl is called to process the tcl
76 \item It converts TCL modulefile into Lua with Lmod module
77 commands
78 \end{itemize}
79 {\tiny
80 \begin{semiverbatim}
81 setenv FOO bar \Rightarrow setenv("FOO","bar")
82 prepend-path PATH /prgm/bin \Rightarrow prepend_path("PATH","/prgm/bin")
83 break \Rightarrow LmodBreak() -- Only for bare breaks
84 \end{semiverbatim}
85 }
86 \end{frame}
87
88 % page 5
89 \begin{frame}[fragile]
90 \frametitle{TCL Break}
91 {\small
92 \begin{semiverbatim}
93 for \{set i 0\} \{\$i < 5\} \{incr i\} \{
94 puts stderr "\$i"
95 if \{ \$i == 3 \} \{
96 break # This breaks out of the loop
97 \}
98 \}
99 break # This causes the modulefile
100 # to stop being processed.
101 \end{semiverbatim}
102 }
103 \end{frame}
104
105 % page 6
106 \begin{frame}{Why was TCL break such a problem for Lmod?}
107 \begin{itemize}
108 \item TCL break stops processing the current module
109 \item It ignores any changes in a module that has a break
110 \item But it keeps all other modules loaded.
111 \item {\color{blue} \texttt{module load A B C D}}
112 \item Where C has a break
113 \item Then A B are loaded but C and D are not.
114 \end{itemize}
115 \end{frame}
116
117 % page 7
118 \begin{frame}{LmodError is different}
119 \begin{itemize}
120 \item {\color{blue} \texttt{module load A B C D}}
121 \item Where C has an LmodError()
122 \item No modules are loaded.
123 \end{itemize}
124 \end{frame}
125
126 % page 8
127 \begin{frame}{Lmod waits to produce output}
128 \begin{itemize}
129 \item When loading several modules, Lmod waits
130 \item All module actions are completed internally
131 \item Then Lmod generates shell command output.
132 \item Lmod 6 and earlier wouldn't know what changes to ignore when
133 processing a break.
134 \item Lmod produces either an error or environment changes
135 not both.
136 \end{itemize}
137 \end{frame}
138
139 % page 9
140 \begin{frame}{Lmod 7+ was a complete re-write of Lmod}
141 \begin{itemize}
142 \item It was needed to support Name/Version/Version (N/V/V) modulefiles
143 \item Before Lmod only supported N/V or C/N/V
144 \item Lmod 7+ now has a FrameStk (AKA the stack-frame)
145 \item The FrameStk contains a stack of the environment var table (varT) and the
146 module table (mt)
147 \end{itemize}
148 \end{frame}
149
150 % page 10
151 \begin{frame}{FrameStk: varT and mt}
152 \begin{itemize}
153 \item The table varT contains key-value pairs that represent the
154 new env. var values
155 \item The table mt is the module table containing the currently
156 loaded modules among other things
157 \item The Module Table is stored in the environment via
158 \$\_ModuleTable001\_ etc.
159 \end{itemize}
160 \end{frame}
161
162 % page 11
163 \begin{frame}[fragile]
164 \frametitle{assignment versus deepcopy() in Lua}
165 {\small
166 \begin{semiverbatim}
167 a = \{\}
168 a[1] = "foo"
169 b = a
170 b[1] = "bar"
171 print(a[1]) -> ``bar'' not ``foo''
172 \end{semiverbatim}
173 }
174 \begin{itemize}
175 \item Lua tries to be efficent
176 \item It just copies reference
177 \item As shown above.
178 \item Lmod provides deepcopy() function.
179 \item This creates a new table
180 \end{itemize}
181 \end{frame}
182
183 % page 12
184 \begin{frame}{FrameStk}
185 \begin{itemize}
186 \item Before each module: Deep Copy copies the previous varT and mt to top of
187 FrameStk.
188 \item Each evaluation of modulefile is updated on the top of the
189 FrameStk
190 \item When the current modulefile evaluation is completed
191 \item The FrameStk is pop'ed
192 \item The previous stack values are replaced with current
193 \end{itemize}
194 \end{frame}
195
196 % page 13
197 \begin{frame}{FrameStk implications}
198 \begin{itemize}
199 \item Cannot trust local values of mt
200 \item Lmod constantly has to refresh mt:
201 \item {\color{blue}mt = frameStk:mt()}
202 \item Because a module load might have updated it.
203 \end{itemize}
204 \end{frame}
205
206 % page 14
207 \begin{frame}[fragile]
208 \frametitle{LmodBreak or TCL break}
209 \begin{itemize}
210 \item If LmodBreak() is called, the current module changes are
211 ignored
212 \item LmodBreak() causes the previous values to be current
213 \item FrameStk:pop() pops the stack.
214 \item The FrameStk code is shown below:
215 \end{itemize}
216 {\tiny
217 \begin{semiverbatim}
218 function M.LmodBreak(self)
219 local stack = self.\_\_stack
220 local count = self.\_\_count
221 stack[count].mt = deepcopy(stack[count-1].mt)
222 stack[count].varT = deepcopy(stack[count-1].varT)
223 end
224
225 function M.pop(self)
226 local stack = self.\_\_stack
227 local count = self.\_\_count
228 stack[count-1].mt = stack[count].mt
229 stack[count-1].varT = stack[count].varT
230 stack[count] = nil
231 self.__count = count - 1
232 end
233 \end{semiverbatim}
234 }
235 \end{frame}
236
237 % page 15
238 \begin{frame}{Support for TCL break}
239 \begin{itemize}
240 \item Lmod 8.6+ added support LmodBreak()
241 \item Lmod 8.6+ added support a bare TCL break
242 \item Lmod 8.7+ added support for regular break and bare break
243 \end{itemize}
244 \end{frame}
245
246 % page 16
247 \begin{frame}[fragile]
248 \frametitle{TCL Break strangeness}
249 {\small
250 \begin{semiverbatim}
251 for \{set i 0\} \{\$i < 5\} \{incr i\} \{
252 puts stderr "\$i"
253 if \{ \$i == 3 \} \{
254 break # This breaks out of the loop
255 \}
256 \}
257 break # This causes the modulefile
258 # to stop being processed.
259 \end{semiverbatim}
260 }
261 \begin{itemize}
262 \item TCL treats a bare break as an error
263 \item Tmod 3, 4 and 5 catch the error
264 \item Lmod 8.7+ now catch the error too!
265 \end{itemize}
266 \end{frame}
267
268 % page 17
269 \begin{frame}[fragile]
270 \frametitle{To support regular and bare break in TCL in tcl2lua.tcl}
271 {\tiny
272 \begin{semiverbatim}
273 set sourceFailed [catch \{source \$ModulesCurrentModulefile \} errorMsg] # (1)
274 set returnval 0
275 if \{ $g\_help && [info procs "ModulesHelp"] == "ModulesHelp" \} \{
276 # handle module help
277 ...
278 \}
279 if \{\$sourceFailed\} \{
280 if \{ \$sourceFailed == 3 || \$errorMsg == \{invoked "break" outside of a loop\}\} \{
281 set returnVal 1
282 myBreak
283 showResults
284 return \$returnVal
285 \}
286 reportError \$errorMsg
287 set returnVal 1
288 \}
289 showResults
290 return $returnVal
291 \end{semiverbatim}
292 }
293 \begin{itemize}
294 \item line 1 evaluate the TCL modulefile
295 \item \$sourceFailed will be non-zero for TCL errors
296 \item \$sourceFailed == 3 means a bare break has been found.
297 \end{itemize}
298 \end{frame}
299
300 % page 18
301 \begin{frame}[fragile]
302 \frametitle{What happens when?}
303 {\tiny
304 \begin{semiverbatim}
305 % cat C.lua
306 load("X","Y")
307 LmodBreak()
308
309 % module load A B C D
310 % module list
311 Currently Loaded Modules:
312 1) A 2) B
313 \end{semiverbatim}
314 }
315 \begin{itemize}
316 \item Module A and B are loaded internally
317 \item When loading C, modules X and Y are loaded internally
318 \item When LmodBreak() is encounter, processing of C stops
319 \item Also the effects of X and Y are ignored.
320 \end{itemize}
321 \end{frame}
322
323 % page 19
324 \begin{frame}[fragile]
325 \frametitle{Lmod 8.6.15 could create an endless loop}
326 {\tiny
327 \begin{semiverbatim}
328 % cat foo3/1.0
329 #%Module
330 catch \{set foo \$env(FOO)\}
331 if \{ [info exists foo] \} \{
332 puts stderr "already set"
333 break
334 \}
335 setenv FOO "just me"
336
337 % module load foo3/1.0; module load foo3/1.0
338 already set
339 already set
340 ...continues until crtl+C ...
341 \end{semiverbatim}
342 }
343 \begin{itemize}
344 \item Loading foo3/1.0 twice causes an endless loop
345 \item Why?
346 \item The second load forces foo3/1.0 to unload (which it can't)
347 \item Lmod then tries to re-load foo3/1.0 which causes the unload etc.
348 \end{itemize}
349
350 \end{frame}
351
352 % page 20
353 \begin{frame}{LmodBreak() is a no-op on unload}
354 \begin{itemize}
355 \item A bare TCL break becomes an LmodBreak() when translated.
356 \item LmodBreak() does nothing during unload.
357 \item This prevents the endless loop shown above. (Lmod 8.7+)
358 \end{itemize}
359 \end{frame}
360
361 % page 21
362 \begin{frame}{Conclusions}
363 \begin{itemize}
364 \item Implementing break is trickier then you might think.
365 \item Lmod now can support bare breaks finally in Lmod 8.7
366 \item The FrameStk is the price to be paid to support break.
367 \end{itemize}
368 \end{frame}
369
370 % page 22
371 \begin{frame}{Future Topics}
372 \begin{itemize}
373 \item Next Meeting: June 7th 9:30 US Central (14:30 UTC)
374 \item Show how Lmod processes a module load command, stepping
375 through the codebase.
376 \item Suggestions?
377 \end{itemize}
378 \end{frame}
379
380 \end{document}