Debian Codegraph @

my_docs/22/lmod_zoom_mtg_2022_09_06/presentation.tex

raw
1 \documentclass{beamer}
2
3 % You can also use a 16:9 aspect ratio:
4 %\documentclass[aspectratio=169]{beamer}
5 \usetheme{TACC16}
6
7 % It's possible to move the footer to the right:
8 %\usetheme[rightfooter]{TACC16}
9
10 %% page
11 %\begin{frame}{}
12 % \begin{itemize}
13 % \item
14 % \end{itemize}
15 %\end{frame}
16 %
17 %% page
18 %\begin{frame}[fragile]
19 % \frametitle{}
20 % {\tiny
21 % \begin{semiverbatim}
22 % \end{semiverbatim}
23 %}
24 % \begin{itemize}
25 % \item
26 % \end{itemize}
27 %
28 %\end{frame}
29
30 \begin{document}
31 \title[Lmod]{How Lmod Loads a Modulefile, Part 2}
32 \author{Robert McLay}
33 \date{Sept. 6, 2022}
34
35 % page 1
36 \frame{\titlepage}
37
38
39 % page 2
40 \begin{frame}{Outline}
41 \center{\includegraphics[width=.9\textwidth]{[email protected]}}
42 \begin{itemize}
43 \item Part 1 took us to Master:load() and loadModuleFile(t)
44 \item This part will start with Master:load() and loadModuleFile(t)
45 \item This talk will be about handing off your modulefile to the
46 Lua interpreter and Lmod routines.
47 \end{itemize}
48 \end{frame}
49
50 % page 3
51 \begin{frame}[fragile]
52 \frametitle{Outline (II)}
53 \begin{itemize}
54 \item \texttt{main()}: How main() parses the command line
55 \item \texttt{...}
56 \item \texttt{Master:load(mA)}: Where the heavy lifting is done
57 \item \texttt{loadModuleFile()}: Where the modulefile is actually evaluated
58 \end{itemize}
59 \end{frame}
60
61 % page 4
62 \begin{frame}[fragile]
63 \frametitle{\texttt{Master:load()}: getting ready to call loadModuleFile(t)}
64 \begin{itemize}
65 \item src: src/Master.lua
66 \item There is no duplicate sn so we can load the modulefile {\color{blue}\emph{foo}}.
67 \end{itemize}
68 {\tiny
69 \begin{semiverbatim}
70 elseif (fn) then
71 frameStk:push(mname)
72 mt = frameStk:mt()
73 mt:add(mname,"pending")
74 local status = loadModuleFile\{file = fn, shell = shellNm,
75 mList = mList, reportErr = true\}
76 -- part 2
77
78 \end{semiverbatim}
79 }
80 \end{frame}
81
82 % page 5
83 \begin{frame}[fragile]
84 \frametitle{After loading modulefile}
85 {\tiny
86 \begin{semiverbatim}
87 -- Now fn is loaded
88
89 mt = frameStk:mt() -- Why?
90
91 -- A modulefile could the same named module over top of the current modulefile
92 -- Say modulefile abc/2.0 loads abc/.cpu/2.0. Then in the case of abc/2.0
93 -- the filename won't match.
94 if (mt:fn(sn) == fn and status) then
95 mt:setStatus(sn, "active")
96 hook.apply("load",\{fn = mname:fn(), modFullName =
97 mname:fullName(), mname = mname\})
98 end
99 frameStk:pop()
100 loaded = true
101 \end{semiverbatim}
102 }
103 \end{frame}
104
105 % page 6
106 \begin{frame}{Notes about loadModuleFile(t) function}
107 \begin{itemize}
108 \item Almost all module commands will eval one or more modules
109 \item This means that loadModuleFile(t) will be called.
110 \item Does not: module list, module -t avail
111 \item Does: ~~~~module load, module unload, module help, module whatis, ...
112 \item Even if your site uses a system spider cache, you will re-eval modulefiles that change \$MODULEPATH (New)
113 \end{itemize}
114 \end{frame}
115
116 % page 7
117 \begin{frame}[fragile]
118 \frametitle{Let's load the modulefile w/ \texttt{loadModuleFile(t)}}
119 {\tiny
120 \begin{semiverbatim}
121 function loadModuleFile(t)
122 ----------------------------------------------------------------------
123 -- Don't complain if t.file has disappeared when mode() == "unload"
124 ----------------------------------------------------------------------
125 -- Check for infinite loop on mode() == "load"
126 ----------------------------------------------------------------------
127 -- Read modulefile into string "whole"
128
129 if (t.ext == ".lua") then
130 -- Read complete Lua modulefile into "whole"
131 else
132 -- Convert TCL modulefile into Lua
133 end
134
135 ----------------------------------------------------------------------
136 -- Use sandbox to evaluate modulefile
137
138 ----------------------------------------------------------------------
139 -- Report any errors and error out
140
141 ----------------------------------------------------------------------
142 -- Mark lmodBrk is LmodBreak() is called inside moduleFile
143 return not lmodBrk
144 end
145 \end{semiverbatim}
146 }
147 \begin{itemize}
148 \item Note that we passed an anonymous table in Master.lua.
149 \item The result is that there is a single argument "t" to the function.
150 \end{itemize}
151 \end{frame}
152
153 % page 8
154 \begin{frame}[fragile]
155 \frametitle{Reading entire modulefile}
156 {\tiny
157 \begin{semiverbatim}
158 if (t.ext == ".lua") then
159 -- Read in lua module file into a [[whole]] string.
160 local f = io.open(t.file)
161 if (f) then
162 whole = f:read("*all")
163 f:close()
164 end
165 \end{semiverbatim}
166 }
167 \end{frame}
168
169 % page 9
170 \begin{frame}{Converting TCL to Lua}
171 \begin{itemize}
172 \item This will be another talk. But TL; DR
173 \item The TCL interpreter run on the TCL modulefile
174 \item But \texttt{setenv FOO BAR} is converted to the string
175 \texttt{setenv("FOO","BAR")}
176 \item \texttt{prepend-path PATH /...} $\Rightarrow$
177 \texttt{prepend\_path("PATH","/..."}
178 \item etc
179 \end{itemize}
180 \end{frame}
181
182 % page 10
183 \begin{frame}[fragile]
184 \frametitle{TCL conversion example}
185 {\tiny
186 \begin{semiverbatim}
187 ---------- Input: -----------------
188
189 global env
190 proc ModulesHelp { } {
191 puts stderr "Help message..."
192 }
193 set modulepath\_root \$env(MODULEPATH\_ROOT)
194 set moduleshome "\$modulepath\_root/TACC"
195
196 module load Linux
197 module try-add cluster
198 module load TACC-paths
199
200 ---------- Output: -----------------
201
202 load("Linux")
203 try_load("cluster")
204 load("TACC-paths")
205 \end{semiverbatim}
206 }
207 \end{frame}
208
209
210 % page 11
211 \begin{frame}{What is the sandbox?}
212 \begin{itemize}
213 \item Lua offers the ability to evaluate a string containing Lua
214 code
215 \item \emph{WITH} the possibility of a limited choice of
216 functions.
217 \item In other words, Modulefile evaluation has a limited set of functions.
218 \item They cannot call internal routines Lmod functions.
219 \item I did this once testing Lmod and realized the problem.
220 \item Sites may run their own special function
221 \item BUT: They must be registered with the sandbox
222 \item Sites can use the SitePackage.lua or
223 /etc/lmod/lmod\_config.lua to register their own functions
224 \item Note that the sandbox controls what the modulefiles can call
225 \item Once inside however all functions can be called.
226 \end{itemize}
227 \end{frame}
228
229 % page 12
230 \begin{frame}[fragile]
231 \frametitle{status, msg = sandbox\_run(whole)}
232 {\tiny
233 \begin{semiverbatim}
234 local function l\_run5\_2(untrusted\_code)
235 local untrusted\_function, message = load(untrusted\_code, nil, 't', sandbox\_env)
236 if not untrusted\_function then return nil, message end
237 return pcall(untrusted\_function)
238 end
239
240 --------------------------------------------------------------------------
241 -- Define two version: Lua 5.1 or 5.2. It is likely that
242 -- The 5.2 version will be good for many new versions of
243 -- Lua but time will only tell.
244 sandbox\_run = (\_VERSION == "Lua 5.1") and l\_run5\_1 or l\_run5\_2
245 \end{semiverbatim}
246 }
247 \end{frame}
248
249
250 % page 13
251 \begin{frame}[fragile]
252 \frametitle{What is sandbox\_env (src/sandbox.lua?}
253 {\tiny
254 \begin{semiverbatim}
255 local sandbox\_env = \{
256 assert = assert,
257 loadfile = loadfile,
258 require = require,
259 ipairs = ipairs,
260 next = next,
261 pairs = pairs,
262 pcall = pcall,
263 tonumber = tonumber,
264 tostring = tostring,
265 type = type,
266 load = load\_module,
267 load\_any = load\_any,
268
269 --- PATH functions ---
270 prepend\_path = prepend\_path,
271 append\_path = append\_path,
272 remove\_path = remove\_path,
273
274 --- Set Environment functions ----
275 setenv = setenv,
276 pushenv = pushenv,
277 unsetenv = unsetenv,
278
279 ...
280 \}
281 \end{semiverbatim}
282 }
283 \begin{itemize}
284 \item Modulefiles can only execute functions in sandbox\_env
285 \end{itemize}
286 \end{frame}
287
288 % page 14
289 \begin{frame}[fragile]
290 \frametitle{Modulefile Evaluation: Who is in charge?}
291 \begin{itemize}
292 \item Once \texttt{sandbox\_run()} is called then Lua is in charge
293 \item Not Lmod!
294 \item All normal Lua statements are run by Lua.
295 \item Lmod never sees if () stmts etc.
296 \item Lmod code is called when executing setenv(), prepend\_path() etc.
297 \end{itemize}
298 \end{frame}
299
300 % page 15
301 \begin{frame}[fragile]
302 \frametitle{Example: StdEnv.lua}
303 {\tiny
304 \begin{semiverbatim}
305 local hroot = pathJoin(os.getenv("HOME") or "","myModules")
306 local userDir = pathJoin(hroot,"Core")
307 if (isDir(userDir)) then
308 prepend_path("MODULEPATH",userDir)
309 end
310 haveDynamicMPATH()
311 \end{semiverbatim}
312 }
313 \begin{itemize}
314 \item Recent change in Lmod 8.7.4: Dynamic Spider Cache
315 \item Lmod remember modules that change \texttt{\$MODULEPATH}
316 \item If $\sim$/myModules doesn't exist then Lmod will never know
317 that this module could change \texttt{\$MODULEPATH}
318 \item That is why this module needs \texttt{haveDynamicMPATH()}.
319 \end{itemize}
320
321 \end{frame}
322
323 % page 16
324 \begin{frame}[fragile]
325 \frametitle{Suppose your modulefile calls \texttt{setenv()}}
326 \begin{itemize}
327 \item Module command functions are implemented in src/modfuncs.lua
328 \item Below is the modfunc.lua function setenv()
329 \item So the arguments are check to be strings and/or numbers with l\_validateArgsWithValue()
330 \item Then the start of the work happens with mcp:setenv(...)
331 \end{itemize}
332 {\tiny
333 \begin{semiverbatim}
334 function setenv(...)
335 dbg.start\{"setenv(",l_concatTbl({...},", "),")"\}
336 if (not l_validateArgsWithValue("setenv",...)) then return end
337
338 mcp:setenv(...)
339 dbg.fini("setenv")
340 return
341 end
342 \end{semiverbatim}
343 }
344 \end{frame}
345
346 % page 17
347 \begin{frame}[fragile]
348 \frametitle{\texttt{mcp:setenv() what is it gonna do?}}
349 \begin{itemize}
350 \item \texttt{mode() == "load"} $\Rightarrow$ MasterControl:setenv()
351 \item \texttt{mode() == "unload"} $\Rightarrow$ MasterControl:unsetenv()
352 \item \texttt{mode() == "show"} $\Rightarrow$ MC\_Show:setenv()
353 \item \texttt{mode() == "refresh"} $\Rightarrow$ MasterControl:quiet()
354 \item Note that there is NOT if block making these choices
355 \item It depends on which object type \texttt{mcp} was build as.
356 \end{itemize}
357 \end{frame}
358
359 % page 18
360 \begin{frame}{There are ten different types of MasterControl classes}
361 \begin{itemize}
362 \item MC\_Access.lua $\Rightarrow$ whatis, help
363 \item MC\_CheckSyntax.lua $\Rightarrow$ for checking the syntax of a modulefile.
364 \item MC\_ComputeHash lua $\Rightarrow$ Computing the hash value used in user collections.
365 \item MC\_DependencyCk.lua $\Rightarrow$ reloading modules to report any problems with the dependencies.
366 \item MC\_Load.lua $\Rightarrow$ loading modules
367 \item MC\_Mgrload $\Rightarrow$ the loading method when loading a collection (ignore load() type functions inside modulefiles)
368 \item MC\_Refresh $\Rightarrow$ reload all currently loaded modules but only run set\_alias() set\_shell\_function() and nothing else
369 \item MC\_Show $\Rightarrow$ Just print out commands
370 \item MC\_Spider $\Rightarrow$ How to process modules when in spider mode
371 \item MC\_Unload $\Rightarrow$ How Lmod unloads modules (setenv $\Rightarrow$ unsetenv, prepend\_path() $\Rightarrow$ remove\_path() etc)
372 \end{itemize}
373 \end{frame}
374
375 % page 19
376 \begin{frame}[fragile]
377 \frametitle{MasterControl:setenv()}
378 {\tiny
379 \begin{semiverbatim}
380 function M.setenv(self, name, value, respect)
381 dbg.start\{"MasterControl:setenv(\"",name,"\", \"",value,"\", \"",
382 respect,"\")"\}
383 l_check_for_valid_name("setenv",name)
384
385 if (value == nil) then
386 LmodError{msg="e_Missing_Value", func = "setenv", name = name}
387 end
388
389 if (respect and getenv(name)) then
390 dbg.print{"Respecting old value"}
391
392 dbg.fini("MasterControl:setenv")
393 return
394 end
395
396 local frameStk = FrameStk:singleton()
397 local varT = frameStk:varT()
398 if (varT[name] == nil) then
399 varT[name] = Var:new(name)
400 end
401 varT[name]:set(tostring(value))
402 dbg.fini("MasterControl:setenv")
403 end
404 \end{semiverbatim}
405 }
406 \end{frame}
407
408 % page 20
409 \begin{frame}[fragile]
410 \frametitle{Lmod changing your Environment in lmod.in.lua}
411 {\tiny
412 \begin{semiverbatim}
413 -- Output all newly created path and variables.
414 Shell:expand(varT)
415
416 -- Expand any execute\{\} cmds
417 if (Shell:real\_shell())then
418 Exec:exec():expand()
419 end
420 \end{semiverbatim}
421 }
422 \begin{itemize}
423 \item Assuming no errors
424 \item Then Lmod prints out the changes to your env.
425 \item With the above code.
426 \item All changes are in \texttt{varT}
427 \end{itemize}
428
429 \end{frame}
430
431 % page 21
432 \begin{frame}[fragile]
433 \frametitle{\texttt{module unload foo}}
434 {\tiny
435 \begin{semiverbatim}
436 function M.unsetenv(self, name, value, respect)
437 name = name:trim()
438 dbg.start\{"MasterControl:unsetenv(\"",name,"\", \"",value,"\")"\}
439
440 l_check_for_valid_name("unsetenv",name)
441
442 if (respect and getenv(name) ~= value) then
443 dbg.print{"Respecting old value"}
444 dbg.fini("MasterControl:unsetenv")
445 return
446 end
447
448 local frameStk = FrameStk:singleton()
449 local varT = frameStk:varT()
450 if (varT[name] == nil) then
451 varT[name] = Var:new(name)
452 end
453 varT[name]:unset()
454
455 -- Unset stack variable if it exists
456 local stackName = l_createStackName(name)
457 if (varT[stackName]) then
458 varT[name]:unset()
459 end
460 dbg.fini("MasterControl:unsetenv")
461 end
462 \end{semiverbatim}
463 }
464 \begin{itemize}
465 \item Here we unsetenv() by calling varT[name]:unset()
466 \end{itemize}
467
468 \end{frame}
469
470 % page 22
471 \begin{frame}[fragile]
472 \frametitle{\texttt{module show foo}}
473 {\tiny
474 \begin{semiverbatim}
475 function M.setenv(self, name,value)
476 l_ShowCmd("setenv", name, value)
477 end
478 \end{semiverbatim}
479 }
480 \begin{itemize}
481 \item In src/MC\_Show.lua the setenv command is printed.
482 \end{itemize}
483
484 \end{frame}
485
486 % page 23
487 \begin{frame}{Next time}
488 \begin{itemize}
489 \item How the translation from TCL to Lua is handled
490 \item Why it is not perfect.
491 \end{itemize}
492 \end{frame}
493
494 % page 24
495 \begin{frame}{Conclusions}
496 \begin{itemize}
497 \item Lmod provide a way to evaluate your modulefile.
498 \item It does so through the sandbox\_run() function
499 \item And hands off the evaluation to Lua.
500 \end{itemize}
501 \end{frame}
502
503 % page 25
504 \begin{frame}{Future Topics}
505 \begin{itemize}
506 \item Next Meeting: October 4th 9:30 US Central (14:30 UTC)
507 \item How the translation from TCL to Lua is handled
508 \end{itemize}
509 \end{frame}
510
511 \end{document}