Debian Codegraph @

my_docs/22/lmod_zoom_mtg_2022_01_11/presentation.tex

raw
1 \documentclass{beamer}
2
3 % You can also use a 16:9 aspect ratio:
4 %\documentclass[aspectratio=169]{beamer}
5 \usetheme{TACC16}
6
7 % It's possible to move the footer to the right:
8 %\usetheme[rightfooter]{TACC16}
9
10 \begin{document}
11 \title[Lmod]{How Collections work \& ml spider output}
12 \author{Robert McLay}
13 \date{Jan. 11, 2022}
14
15 % page 1
16 \frame{\titlepage}
17
18
19 % page 2
20 \begin{frame}{Collections}
21 \center{\includegraphics[width=.9\textwidth]{[email protected]}}
22 \begin{itemize}
23 \item What: A saved list of modules that can be restored
24 \item Rules on what gets loaded.
25 \item How collections work
26 \item General Principle: Restoring collection is the same as by hand
27 \item Conclusions
28 \item Future Topics
29 \end{itemize}
30 \end{frame}
31
32 % page 3
33 \begin{frame}{Basics}
34 \begin{itemize}
35 \item As many named collections as you like.
36 \item A collection named ``default'' overrides site default
37 (assuming correct startup setup)
38 \item Collection replace current modules not an addition.
39 \item \texttt{module load foo/1.1} always load \texttt{foo/1.1}
40 independent of current default.
41 \item \texttt{module load foo} always restores the default
42 \texttt{foo}
43 \item Unless LMOD\_PIN\_VERSION=yes is set.
44 \item In this case Lmod restore module version when stored.
45 \end{itemize}
46 \end{frame}
47
48
49 % page 4
50 \begin{frame}{Story: How collections started.}
51 \begin{itemize}
52 \item A colleague, Bill Barth, asked if there could be a way to
53 save the current list of modules.
54 \item The default modules were not what I wanted.
55 \item This idea sounded simple to implement but ...
56 \item There be dragons in that idea.
57 \item It took more than a year to work out the ``right way'' to do it.
58 \end{itemize}
59 \end{frame}
60
61 % page 5
62 \begin{frame}{Original Implementation for Collections}
63 \begin{itemize}
64 \item Save the module table into a file ($\sim$/.lmod.d/default)
65 \item Restore steps:
66 \begin{enumerate}
67 \item Purge ALL modules (including sticky ones)
68 \item set \$MODULEPATH to one stored in collection
69 \item Loop over list of modules in collection (They are in
70 load order)
71 \item Remove module not in the list {\color{red} (DRAGONS!!)}
72 \end{enumerate}
73 \item This does work right in certain cases.
74 \end{itemize}
75 \end{frame}
76
77 % page 6
78 \begin{frame}{Why this does not work}
79 \begin{itemize}
80 \item Assume simple 4 module system: Meta, intel, impi, openmpi
81 \item Default module: Meta (which loads intel, impi)
82 \item User collection: Meta, intel, openmpi
83 \item Assume no family() functions in use
84 \end{itemize}
85 \end{frame}
86
87 % page 7
88 \begin{frame}[fragile]
89 \frametitle{Why this does not work (II)}
90 {\small
91 \begin{semiverbatim}
92 User does this:
93 \$ ml purge; ml Meta; ml -impi openmpi; ml save
94 Original Collection impl:
95 3a) load Meta -> load Meta, intel impi
96 3b) load intel (again)
97 3c) load openmpi
98 4a) unload impi
99 \end{semiverbatim}
100 }
101 \end{frame}
102
103 % page 8
104 \begin{frame}{Why this does not work (III)}
105 \begin{itemize}
106 \item At our site, both impi and openmpi set \$MPI\_MODE
107 \item \$MPI\_MODE is used in our local mpirun command
108 \item Step 4a unload impi which unset \$MPI\_MODE
109 \item Disaster!!! $\Rightarrow$ User cannot launch mpi programs.
110 \end{itemize}
111 \end{frame}
112
113 % page 9
114 \begin{frame}{Why this does not work (IV)}
115 \begin{itemize}
116 \item If any modules share an env. var. $\Rightarrow$ TROUBLE!
117 \item The problem is that setenv() is not pushenv()
118 \item Thought about making all setenv() work like pushenv()
119 \item Ultimately came up with a different design.
120 \item It took several iterations to get here.
121 \end{itemize}
122 \end{frame}
123
124 % page 10
125 \begin{frame}{Current collection restore implementation}
126 \begin{enumerate}
127 \item Purge ALL modules
128 \item Load all modules in list order BUT ignore load() like
129 functions inside a modulefile.
130 \end{enumerate}
131 \end{frame}
132
133 % page 11
134 \begin{frame}[fragile]
135 \frametitle{How does this help?}
136 {\small
137 \begin{semiverbatim}
138
139 3a) load Meta
140 -> load Meta only (ignore load("intel", "impi"))
141 3b) load intel
142 3c) load openmpi
143
144 => no modules to unload!!
145 \end{semiverbatim}
146 }
147 \end{frame}
148
149 % page 12
150 \begin{frame}{What are the drawbacks to the solution?}
151 \begin{itemize}
152 \item This works fine until ``Meta'' gets another ``load()''
153 \item In collections, all load() are ignored,
154 the user doesn't get the same modules
155 \item Solution: error out when modules get new load() or changes
156 via prepend\_path() or append\_path() to \$MODULEPATH.
157 \end{itemize}
158 \end{frame}
159
160 % page 13
161 \begin{frame}{Saving modules in collections}
162 \begin{itemize}
163 \item Saving causes a ``show'' for each module
164 \item But only for load() like functions and changes to \$MODULEPATH
165 \item All other Lmod functions are ignored.
166 \item A sha1 of the resulting string is computed and saved.
167 \end{itemize}
168 \end{frame}
169
170 % page 14
171 \begin{frame}{Restoring a collection}
172 \begin{enumerate}
173 \item Lmod purges all modules
174 \item set \$MODULEPATH to one stored in collection
175 \item Loop over list of modules in collection (They are in
176 load order)
177 \item Compute sha1sum of each module with only load() and changes to
178 \$MODULEPATH shown
179 \item Compare sha1sum value with stored value in collection
180 \item Error out if sha1sum values do not match.
181 \end{enumerate}
182 \end{frame}
183
184 % page 15
185 \begin{frame}{Drawbacks to this solution}
186 \begin{itemize}
187 \item Most modules are null strings
188 \item But Meta, compiler and mpi modules changes can trigger
189 invalid collection $\Rightarrow$ rebuild collection message.
190 \end{itemize}
191 \end{frame}
192
193 % page 16
194 \begin{frame}{Issue 388: Forcibly loading a collection}
195 \begin{itemize}
196 \item The drawback to users is that their job might die when
197 a ``Meta'' module changes
198 \item Issue 338 (2018) requested that a collection be loaded anyway.
199 \item I won't do it because it won't be the same modules.
200 \end{itemize}
201 \end{frame}
202
203 % page 17
204 \begin{frame}{Conclusions}
205 \begin{itemize}
206 \item Collections provide a convenient way to group modules together
207 \item Rather than users creating their own ``Meta'' modules
208 \item Collection have to be rebuilt as ``Meta'' modules change.
209 \end{itemize}
210 \end{frame}
211
212 % page 18
213 \begin{frame}{New Topic: Issue 551: Handling Descriptions: ml spider GROMACS/2019}
214 \begin{itemize}
215 \item Level 2 output shows module help and description
216 \item Lmod picks one modules's help and description.
217 \item Might be confusing when there are GPU and CPU versions.
218 \item Lmod is only going to pick one.
219 \item Comments?
220 \end{itemize}
221 \end{frame}
222
223 % page 19
224 \begin{frame}{Future Topics}
225 \begin{itemize}
226 \item Lmod Testing System?
227 \item Explain how to pass module info to hooks(Issue 552)
228 \item More internals of Lmod?
229 \item Guest Presentation of special issues?
230 \end{itemize}
231 \end{frame}
232
233 \end{document}