Debian Codegraph @

my_docs/22/lmod_zoom_mtg_2022_06_07/presentation.tex

raw
1 \documentclass{beamer}
2
3 % You can also use a 16:9 aspect ratio:
4 %\documentclass[aspectratio=169]{beamer}
5 \usetheme{TACC16}
6
7 % It's possible to move the footer to the right:
8 %\usetheme[rightfooter]{TACC16}
9
10 %% page
11 %\begin{frame}{}
12 % \begin{itemize}
13 % \item
14 % \end{itemize}
15 %\end{frame}
16 %
17 %% page
18 %\begin{frame}[fragile]
19 % \frametitle{}
20 % {\tiny
21 % \begin{semiverbatim}
22 % \end{semiverbatim}
23 %}
24 % \begin{itemize}
25 % \item
26 % \end{itemize}
27 %
28 %\end{frame}
29
30 \begin{document}
31 \title[Lmod]{How Lmod Loads a Modulefile, Part 1}
32 \author{Robert McLay}
33 \date{June. 7, 2022}
34
35 % page 1
36 \frame{\titlepage}
37
38
39 % page 2
40 \begin{frame}{Outline}
41 \center{\includegraphics[width=.9\textwidth]{[email protected]}}
42 \begin{itemize}
43 \item Talk: How Lmod finds a modulefile to load
44 \item Next Time: Part 2: How Lmod actually evaluates a modulefile
45 \item There is a surprising amount to talk about before the
46 modulefile gets evaluated.
47 \item We will be talking about a user requested load.
48 \item A module requested load is slightly different.
49 \end{itemize}
50 \end{frame}
51
52 % page 3
53 \begin{frame}[fragile]
54 \frametitle{Outline (II)}
55 \begin{itemize}
56 \item \texttt{main()}: How main() parses the command line
57 \item \texttt{Load\_Usr()}: Starts working your modules to load
58 \item \texttt{l\_usrLoad()}: Splits modules to loads or unloads
59 \item \texttt{MName Class}: Maps module name to a filename
60 \item \texttt{mcp}: Object to control kind of evaluation
61 \item \texttt{MasterControl:load(mA)}: We are loading a modulefile
62 \item \texttt{Master:load(mA)}: Where the heavy lifting is done
63 \item \texttt{loadModuleFile()}: Where the modulefile is actually evaluated
64 \end{itemize}
65 \end{frame}
66
67 % page 4
68 \begin{frame}[fragile]
69 \frametitle{\texttt{main()}: parsing command line}
70 \begin{itemize}
71 \item main() is located in lmod.in.lua (installed as lmod)
72 \item The command is: {\color{blue}\emph{module load foo}}
73 \item To parse Lmod uses the 2nd argument w/o a minus: {\color{blue}\emph{load}}
74 \item Lmod searches \texttt{lmodCmdA} to map string to command
75 \end{itemize}
76 \end{frame}
77
78 % page 5
79 \begin{frame}{A side note on Lmod Coding Conventions}
80 \begin{itemize}
81 \item A variable w/ a trailing ``A'' means that an array
82 \item A variable w/ a trailing ``T'' means that an table (or dictionary)
83 \item A variable w/ a trailing ``Tbl'' means that an table (or dictionary)
84 \item A routine with a l\_name is a local function (file scope)
85 \item Class Name are in CamelCase
86 \end{itemize}
87 \end{frame}
88
89 % page 6
90 \begin{frame}[fragile]
91 \frametitle{lmodCmdA}
92 {\tiny
93 \begin{semiverbatim}
94 local lmodCmdA = \{
95 \{cmd = 'add', min = 2, action = loadTbl \},
96 \{cmd = 'avail', min = 2, action = availTbl \},
97 \{cmd = 'isloaded', min = 3, action = isLoadedTbl \},
98 \{cmd = 'is_loaded', min = 4, action = isLoadedTbl \},
99 \{cmd = 'is-loaded', min = 4, action = isLoadedTbl \},
100 \{cmd = 'load', min = 2, action = loadTbl \},
101 ...
102 \}
103 local loadTbl = \{ name = "load", cmd = Load_Usr \}
104 \end{semiverbatim}
105 \begin{itemize}
106 \item So ``load'' matches ``load'' with more than 2 chars
107 \end{itemize}
108 }
109 \end{frame}
110
111 % page 7
112 \begin{frame}[fragile]
113 \frametitle{Going from lmodCmdA to Load\_Usr()}
114 \begin{itemize}
115 \item Sets cmdT to loadTbl
116 \item Sets cmdName to cmdT.name (forces standard name not user
117 command name)
118 \item cmdT.cmd(unpack(masterTbl.pargs)) $\Rightarrow$ Load\_Usr
119 \end{itemize}
120
121 \end{frame}
122
123 % page 8
124 \begin{frame}[fragile]
125 \frametitle{Calling Load\_Usr()}
126 \begin{itemize}
127 \item All functions implementing user commands are in src/cmdfunc.lua
128 \end{itemize}
129
130 {\tiny
131 \begin{semiverbatim}
132 function Load\_Try(...)
133 dbg.start\{"Load\_Try(",concatTbl(\{...\},", "),")"\}
134 local check\_must\_load = false
135 local argA = pack(...)
136 l\_usrLoad(argA, check\_must\_load)
137 dbg.fini("Load\_Try")
138 end
139
140 function Load\_Usr(...)
141 dbg.start\{"Load\_Usr(",concatTbl(\{...\},", "),")"\}
142 local check\_must\_load = true
143 local argA = pack(...)
144 l\_usrLoad(argA, check\_must\_load)
145 dbg.fini("Load\_Usr")
146 end
147 \end{semiverbatim}
148 }
149 \end{frame}
150
151 % page 9
152 \begin{frame}[fragile]
153 \frametitle{l\_usrLoad(argA, check\_must\_load)}
154 \begin{itemize}
155 \item Split argA into loads in lA, unloads in uA (-foo)
156 \item Both uA and lA are an array of MName objects.
157 \item unload modules in uA
158 \item {\color{blue} \texttt{lA[\#lA+1] = MName:new("load",module\_name)}}
159 \item {\color{blue} \texttt{mcp:load\_usr(lA)}}
160 \item src: src/cmdfunc.lua
161 \end{itemize}
162 \end{frame}
163
164 % page 10
165 \begin{frame}[fragile]
166 \frametitle{MName class: Module Name class}
167 {\tiny
168 \begin{semiverbatim}
169 \end{semiverbatim}
170 }
171 \begin{itemize}
172 \item Maps name (``foo'' or ``foo/1.1'') to filename
173 \item There are two kinds of searching ``load'' or ``mt''
174 \item Load: must search file system.
175 \item mt: filename is in moduletable
176 \item Evaluation must be lazy or just-in-time
177 \item Software hierarchy means that
178 \item {\color{blue}\texttt{module load gcc mpich}}
179 \item mpich might not be in \$MODULEPATH until after gcc is loaded
180 \item src: src/MName.lua
181 \end{itemize}
182
183 \end{frame}
184
185 % page 11
186 \begin{frame}{MName key concepts}
187 \begin{itemize}
188 \item \textbf{userName}: name on the command line
189 \item It might be gcc or gcc/9.3.0
190 \item \textbf{sn}: the shortName or a name without a version
191 \item \textbf{fullName}: The full name of the module (sn/version)
192 \item Examples:
193 \begin{enumerate}
194 \item gcc/9.3.0 (sn: gcc, N/V)
195 \item gcc/x86\_64/9.3.0 (sn: gcc, N/V/V)
196 \item compiler/gcc/9.3.0 (sn: compiler/gcc, C/N/V)
197 \item compiler/gcc/x86\_64/9.3.0 (sn: compiler/gcc, N/V/V)
198 \end{enumerate}
199 \end{itemize}
200 \end{frame}
201
202 % page 12
203 \begin{frame}[fragile]
204 \frametitle{mcp and MasterControl class}
205 \begin{itemize}
206 \item MasterControl class is what controls whether a "load" in a
207 modulefile is a load or unload
208 \item mcp is a global variable that is built to be in a mode()
209 like load, unload, spider, etc.
210 \item We talked about this in an earlier presentation.
211 \end{itemize}
212 \end{frame}
213
214 % page 13
215 \begin{frame}[fragile]
216 \frametitle{MasterControl:load(mA)}
217 {\tiny
218 \begin{semiverbatim}
219 function M.load(self, mA)
220 local master = Master:singleton()
221 local a = master:load(mA)
222
223 if (not quiet()) then
224 self:registerAdminMsg(mA) -- nag msg registration.
225 end
226 return a
227 end
228 \end{semiverbatim}
229 }
230 \begin{itemize}
231 \item MasterControl functions call Master Functions to do the work.
232 \item src: src/MasterControl.lua
233 \end{itemize}
234 \end{frame}
235
236 % page 14
237 \begin{frame}[fragile]
238 \frametitle{Master:load(mA)}
239 {\tiny
240 \begin{semiverbatim}
241 function M.load(mA)
242 for i = 1, #mA do
243 repeat
244 mname = mA[i]
245 sn = mname:sn() -- shortName
246 fn = mname:fn() -- file name
247 -- if blank sn -> pushModule (might have to wait until
248 -- compiler or mpi is loaded.
249 -- and break (really continue)
250
251 if (mt:have(sn,"active)) then
252 -- Block 1: Check for previously loaded module with same sn
253
254 elseif (fn) then
255 -- Block 2: Load modulefile
256
257 -- Check for family stack (e.g. compiler, mpi etc)
258 if (mcp.processFamilyStack(fullName)) then
259 -- Suppose gcc is loaded and it was "replaced" by intel
260 -- unload gcc and reload intel
261 end
262 until true
263 end
264
265 -- Reload every module if change in MODULEPATH.
266
267 -- load any modules on module stack
268 end
269 \end{semiverbatim}
270 }
271 \begin{itemize}
272 \item This is where the heavy lifting is done.
273 \item src: src/Master.lua
274 \end{itemize}
275
276 \end{frame}
277
278 % page 15
279 \begin{frame}[fragile]
280 \frametitle{Block 1: Check for previous loaded module w/same sn}
281 {\tiny
282 \begin{semiverbatim}
283 if (mt:have(sn,"active)) then
284 -- if disable_same_name_autoswap -> error out
285 -- Otherwise: unload previous module
286 local mcp_old = mcp
287 local mcp = MCP
288 unload_internal\{MName:new("mt",sn)\}
289 mname:reset() -- force a new lazyEval
290 local status = mcp:load_usr\{mname\}
291 mcp = mcp_old
292 \end{semiverbatim}
293 }
294 \begin{itemize}
295 \item Here we guarantee the right mcp
296 \item Unload the old module
297 \item Recursively mcp:load\_usr\{mname\}
298 \end{itemize}
299 \end{frame}
300
301 % page 16
302 \begin{frame}[fragile]
303 \frametitle{Block 2: Load modulefile}
304 {\tiny
305 \begin{semiverbatim}
306 elseif (fn) then
307 frameStk:push(mname)
308 mt = frameStk:mt()
309 mt:add(mname,"pending")
310 local status = loadModuleFile\{file = fn, shell = shellNm,
311 mList = mList, reportErr = true\}
312 mt = frameStk:mt()
313 mt:setStatus(sn,"active")
314 frameStk:pop()
315 loaded = true
316 end
317 \end{semiverbatim}
318 }
319 \end{frame}
320
321 % page 17
322 \begin{frame}[fragile]
323 \frametitle{loadModuleFile(t)}
324 \begin{itemize}
325 \item This is where Lmod handle either *.lua files or TCL Modulefiles
326 \item Once either read in as a block (for *.lua) or converted (TCL
327 modulefile $\Rightarrow$ Lua)
328 \item src: src/loadModulefile.lua
329 \end{itemize}
330 {\tiny
331 \begin{semiverbatim}
332 -- Use the sandbox to evaluate modulefile text.
333 if (whole) then
334 status, msg = sandbox_run(whole)
335 else
336 status = nil
337 msg = "Empty or non-existent file"
338 end
339 -- report any errors
340 \end{semiverbatim}
341 }
342
343 \end{frame}
344
345 % page 18
346 \begin{frame}{Next time}
347 \begin{itemize}
348 \item What is a sandbox and how does it work?
349 \item Why I want a sandbox?
350 \item Next time handing control to modulefile
351 \end{itemize}
352 \end{frame}
353
354
355
356 % page 19
357 \begin{frame}{Conclusions}
358 \begin{itemize}
359 \item It takes a lot to get to the point where Lmod is evaluating
360 your modulefile.
361 \item Lmod uses several ``classes'' to manage the loading of a
362 module
363 \item Plus a couple of Design Patterns such as Singletons
364 \end{itemize}
365 \end{frame}
366
367 % page 20
368 \begin{frame}{Future Topics}
369 \begin{itemize}
370 \item Next Meeting: July 5th 9:30 US Central (14:30 UTC)
371 \item What happens from the loadModulefile(t).
372 \item This is where Lmod hands off control to the user.
373 \end{itemize}
374 \end{frame}
375
376 \end{document}