Debian Codegraph @

texmf-dist/source/latex/latex2pydata/latex2pydata.dtx

raw
1 % \iffalse meta-comment
2 % !TEX TS-program = pdflatexmk
3 %
4 % Copyright (C) 2023-2026 by Geoffrey M. Poore <[email protected]>
5 % ---------------------------------------------------------------------------
6 % This work may be distributed and/or modified under the
7 % conditions of the LaTeX Project Public License, either version 1.3c
8 % of this license or (at your option) any later version.
9 % The latest version of this license is in
10 % http://www.latex-project.org/lppl.txt
11 % and version 1.3c or later is part of all distributions of LaTeX
12 % version 2008/05/04 or later.
13 %
14 % This work has the LPPL maintenance status `maintained'.
15 %
16 % The Current Maintainer of this work is Geoffrey M. Poore.
17 %
18 % This work consists of the files latex2pydata.dtx and latex2pydata.ins
19 % and the derived filebase latex2pydata.sty.
20 %
21 % \fi
22 %
23 % \iffalse
24 %<*driver>
25 \ProvidesFile{latex2pydata.dtx}
26 %</driver>
27 %<package>\NeedsTeXFormat{LaTeX2e}[1999/12/01]
28 %<package>\ProvidesPackage{latex2pydata}
29 %<*package>
30 [2026/02/25 v0.7.0 latex2pydata - write data to file in Python literal format]
31 %</package>
32 %
33 %<*driver>
34 \documentclass{ltxdoc}
35
36 \makeatletter
37
38 \usepackage[T1]{fontenc}
39 \usepackage[utf8]{inputenc}
40 \usepackage{lmodern}
41 \usepackage{fourier}
42 \usepackage{microtype}
43
44 \usepackage[svgnames]{xcolor}
45 \usepackage{upquote}
46 % The typesetting for macrocode doesn't use \@noligs, which upquote modifies.
47 % So apply the upquote fix to \verbatim@nolig@list as well, which is in macrocode.
48 \begingroup
49 \catcode`'=\active
50 \catcode``=\active
51 \g@addto@macro\verbatim@nolig@list{%
52 \let'\textquotesingle
53 \let`\textasciigrave
54 \ifx\encodingdefault\upquote@OTone
55 \ifx\ttdefault\upquote@cmtt
56 \def'{\char13 }%
57 \def`{\char18 }%
58 \fi\fi}
59 \endgroup
60 \usepackage{fvextra}
61 \usepackage{latex2pydata}
62
63 \usepackage{graphicx}
64
65 \usepackage{amsmath, amssymb}
66
67 \usepackage{environ}
68
69 \usepackage{tcolorbox}
70 \tcbuselibrary{listings}
71 \tcbset{verbatim ignore percent}
72
73 \usepackage{hyperref}
74 \hypersetup{
75 pdftitle=The latex2pydata package: write key-value data to file in Python literal format,
76 pdfauthor=Geoffrey M. Poore,
77 pdfsubject={latex2pydata LaTeX package manual},
78 colorlinks=true,
79 allcolors=ForestGreen,
80 }
81 \usepackage{cleveref}
82
83 \let\code\Verb
84 \def\meta#1{\mbox{\ensuremath{\langle}\textit{\ttfamily#1}\ensuremath{\rangle}}}
85
86 % A more robust \cmd
87 \let\cmd\Verb
88
89 % Create a short verbatim pipe that handles quotation marks properly
90 \begingroup
91 \catcode`\|=\active
92 \gdef\pipe@active@verbatim{%
93 \begingroup
94 \let\do\@makeother\dospecials
95 \catcode`\|=\active
96 \catcode`\`=\active
97 \catcode`\'=\active
98 \catcode`\<=\active
99 \catcode`\>=\active
100 \catcode`\-=\active
101 \catcode`\,=\active
102 \catcode`\ =\active
103 \pipe@active@verbatim@i}
104 \gdef\pipe@active@verbatim@i#1|{%
105 \endgroup
106 \begingroup
107 \def\FV@SV@pipe@active@verbatim{%
108 \FV@Gobble
109 \expandafter\FV@ProcessLine\expandafter{#1}}%
110 %\let\FV@BeginVBox\relax
111 %\let\FV@EndVBox\relax
112 %\def\FV@BProcessLine##1{\FancyVerbFormatLine{##1}}%
113 \BUseVerbatim{pipe@active@verbatim}%
114 \endgroup}
115 \AtBeginDocument{\let|\pipe@active@verbatim}
116 \endgroup
117
118 \newcommand\pkg[1]{\textsf{\mbox{#1}}}
119
120 \def\MacroFont{%
121 \fontencoding\encodingdefault%
122 \fontfamily\ttdefault%
123 \fontseries\mddefault%
124 \fontshape\updefault%
125 \small}
126
127 \def\PrintMacroName#1{{\strut\MacroFont\color{DarkGreen}\footnotesize\string #1\ }}
128
129 \def\PrintDescribeMacro#1{\strut\MacroFont\textcolor{DarkGreen}{\string #1\ }}
130 \let\PrintDescribeEnv\PrintDescribeMacro
131 %\let\PrintMacroName\PrintDescribeMacro
132 \let\PrintEnvName\PrintDescribeEnv
133
134 \def\theCodelineNo{\textcolor{DarkGreen}{\sffamily\scriptsize{\arabic{CodelineNo}}}}
135
136
137 \def\DescMacro{%
138 \begingroup\makeatletter\DescMacro@i}
139 \def\DescMacro@i#1{%
140 \endgroup
141 \DescMacro@ii#1\FV@Sentinel
142 }
143 \def\DescMacro@ii#1#2\FV@Sentinel{%
144 \@nameuse{SpecialMacroIndex}{#1}%
145 \vspace{1em}%
146 \begingroup
147 \ttfamily
148 \large
149 \setlength{\parindent}{0pt}%
150 \hspace{-3em}{\bfseries\color{DarkGreen}\detokenize{#1}}{\normalsize#2}%
151 \endgroup
152 \par\vspace{0.5em}\noindent\ignorespaces
153 }
154 \def\DescEnv#1{%
155 \@nameuse{SpecialEnvIndex}{#1}%
156 \vspace{1em}%
157 \begingroup
158 \ttfamily
159 \large
160 \setlength{\parindent}{0pt}%
161 \hspace{-3em}{\bfseries\color{DarkGreen}\detokenize{#1}}\space%
162 \endgroup
163 (\textit{env.})
164 \par\vspace{0.5em}\noindent\ignorespaces
165 }
166
167 \let\orig@footnote\footnote
168 \renewcommand{\footnote}{%
169 \begingroup
170 \let\do\@makeother
171 \dospecials
172 \catcode`\{=1
173 \catcode`\}=2
174 \new@footnote}
175 \newcommand{\new@footnote}[1]{%
176 \endgroup
177 \orig@footnote{\scantokens{#1}}}
178
179 \def\printopt#1(#2) (#3){%
180 \vspace{0.1in}%
181 \leavevmode%
182 \marginpar{\raggedleft\texttt{\small\textcolor{DarkGreen}{#1}}\ }%
183 \kern-\parindent\textsf{(#2)}\hfill(default: \texttt{#3})\\}
184
185 \newenvironment{optionlist}%
186 {%
187 ~\par\vspace{-14pt}%
188 \def\pipechar{|}
189 \let\|\pipechar
190 \newcommand*\optionlistnext{}%
191 \renewcommand*\item[1][]{%
192 \optionlistnext%
193 \renewcommand*\optionlistnext{\par}%
194 \printopt##1%
195 \ignorespaces}}
196 {%
197 \par}
198
199 \newenvironment{example}
200 {\VerbatimEnvironment
201 \begin{VerbatimOut}[gobble=4]{example.out}}
202 {\end{VerbatimOut}%
203 \vspace{1ex}%
204 \setlength{\parindent}{0pt}%
205 \setlength{\fboxsep}{1em}%
206 \fcolorbox{DarkGreen}{white}{\begin{minipage}{0.5\linewidth}%
207 \VerbatimInput{example.out}%
208 \end{minipage}%
209 \hspace{0.025\linewidth}%
210 {\color{DarkGreen}\vrule}%
211 \hspace{0.025\linewidth}%
212 \begin{minipage}{0.4\linewidth}%
213 \input{example.out}%
214 \end{minipage}%
215 }\vspace{1ex}}
216
217 \newenvironment{longexample}
218 {\VerbatimEnvironment
219 \begin{VerbatimOut}[gobble=4]{example.out}}
220 {\end{VerbatimOut}%
221 \vspace{1ex}%
222 \setlength{\parindent}{0pt}%
223 \setlength{\fboxsep}{1em}%
224 \fcolorbox{DarkGreen}{white}{\begin{minipage}{0.94\linewidth}%
225 \VerbatimInput{example.out}%
226 {\color{DarkGreen}\hrulefill}
227 \setlength{\fboxsep}{3pt}%
228 \input{example.out}%
229 \end{minipage}%
230 }\vspace{1ex}}
231
232 \CustomVerbatimEnvironment{VerbatimVerbatim}{Verbatim}{}
233
234 \edef\hashchar{\string#}
235
236 \begingroup
237 \catcode`\#=12\relax
238 \gdef\astliteval{\href{https://docs.python.org/3/library/ast.html#ast.literal_eval}{\code{ast.literal_eval()}}}
239 \endgroup
240 \def\pydatapy{\href{https://github.com/gpoore/latex2pydata/tree/main/python}{\pkg{latex2pydata} Python package}}
241 \def\fvextra{\href{https://github.com/gpoore/fvextra/}{\pkg{fvextra}}}
242 \def\fancyvrb{\href{https://ctan.org/pkg/fancyvrb}{\pkg{fancyvrb}}}
243
244 %\EnableCrossrefs
245 %\CodelineIndex
246 %\RecordChanges
247
248 \makeatother
249 \begin{document}
250 \DocInput{latex2pydata.dtx}
251 %\PrintChanges
252 %\PrintIndex
253 \end{document}
254 %</driver>
255 % \fi
256 %
257 %
258 % \DoNotIndex{\newcommand,\newenvironment}
259 % \DoNotIndex{\#,\$,\%,\&,\@,\\,\{,\},\^,\_,\~,\ }
260 % \DoNotIndex{\@ne}
261 % \DoNotIndex{\advance,\begingroup,\catcode,\closein}
262 % \DoNotIndex{\closeout,\day,\def,\edef,\else,\empty,\endgroup}
263 % \DoNotIndex{\begin,\end,\bgroup,\egroup}
264 %
265 % \providecommand*{\url}{\texttt}
266 % \newcommand{\pydata}{\pkg{latex2pydata}}
267 % \GetFileInfo{latex2pydata.dtx}
268 %
269 % \title{\vspace{-0.5in}The \pydata\ package}
270 % \author{Geoffrey M.\ Poore \\ \href{mailto://[email protected]}{\texttt{[email protected]}} \\ \href{https://github.com/gpoore/latex2pydata/tree/main/latex}{\Verb{github.com/gpoore/latex2pydata/tree/main/latex}}}
271 % \date{\fileversion~from \filedate}
272 %
273 % \maketitle
274 %
275 % \begin{abstract}
276 % \noindent\pydata\ is a \LaTeX\ package for writing data to file using \href{https://docs.python.org/3/reference/lexical_analysis.html#literals}{Python literal syntax}. The data may then be loaded safely in Python using the \astliteval\ function or the \pydatapy.
277 %
278 % \vspace{1.25in}
279 % \noindent The original development of this package was funded by a \href{https://tug.org/tc/devfund/grants.html}{\TeX\ Development Fund grant} from the \href{https://tug.org/}{\TeX\ Users Group}. \pydata\ is part of the 2023 grant for improvements to the \href{https://github.com/gpoore/minted/}{\pkg{minted}} package.
280 % \end{abstract}
281 %
282 %
283 % \pagebreak
284 % \begingroup
285 % \makeatletter
286 % ^^A https://tex.stackexchange.com/a/45165/10742
287 % \patchcmd{\@dottedtocline}
288 % {\rightskip\@tocrmarg}
289 % {\rightskip\@tocrmarg plus 4em \hyphenpenalty\@M}
290 % {}{}
291 % \makeatother
292 % \tableofcontents
293 % \endgroup
294 % \pagebreak
295 %
296 %
297 % \section{Introduction}
298 %
299 % The \pydata\ package is designed for passing data from \LaTeX\ into Python. It writes data to file using \href{https://docs.python.org/3/reference/lexical_analysis.html#literals}{Python literal syntax}. The data may then be loaded safely in Python using the \astliteval\ function or the \pydatapy.
300 %
301 % The data that \pydata\ writes to file can take two forms. The top-level data structure can be configured as a Python dict. This is appropriate for representing a single \LaTeX\ command or environment. The top-level data structure can also be configured as a list of dicts. This is useful for representing a sequence of \LaTeX\ commands or environments. In both cases, all keys and values within dicts are written to file as Python string literals. Thus, the overall data is |dict[str, str]| or |list[dict[str, str]]|. This does not limit the data types that can be passed from LaTeX to Python, however. When data is loaded, the included schema functionality makes it possible to convert string values into other Python data types such as dicts, lists, sets, bools, and numbers.
302 %
303 % The data is suitable for direct loading in Python with \astliteval. It is also possible to load data with the \pydatapy, which serves as a wrapper for \code{ast.literal_eval()}. The Python package requires all keys to match the regex \code{[A-Za-z_][0-9A-Za-z_]*}. Periods in keys are interpreted as key paths and indicate sub-dicts. For example, the key path |main.sub| represents a key |main| in the main dict that maps to a sub-dict containing a key |sub|. This makes it convenient to represent nested dicts.
304 %
305 % \pkg{latex2pydata} optionally supports writing metadata to file, including basic schema definitions for values. When the \pydatapy\ loads data with a schema definition for a given value, the value is initially loaded as a string, which is the verbatim text sent from \LaTeX. Then this string is evaluated with \code{ast.literal_eval()}. An error is raised if this process does not result in an object with the data type specified in the schema.
306 %
307 %
308 %
309 % \section{Example}
310 %
311 %\begin{tcblisting}{}
312 %\pydatasetfilename{\jobname.pydata}
313 %\pydatawritedictopen
314 %\pydatawritekeyvalue{key}{value with "quote" and \backslash\ ...}
315 %\pydatawritedictclose
316 %\pydataclosefilename{\jobname.pydata}
317 %\VerbatimInput{\jobname.pydata}
318 %\end{tcblisting}
319 %
320 %
321 %
322 % \section{Design considerations}
323 %
324 % \pydata\ is intended for use with Python. Python literal syntax was chosen instead of \href{https://www.json.org/json-en.html}{JSON} or another data format because it provides simpler compatibility with \LaTeX.
325 % \begin{itemize}
326 % \item It must be possible to serialize the contents of a \LaTeX\ environment verbatim. Python literal syntax supports multi-line string literals, so this is straightforward: write an opening multi-line string delimiter to file, write the environment contents a line at a time (backslash-escaping any delimiter characters), and finally write a closing multi-line string delimiter. Meanwhile, JSON requires that all literal newlines in strings be replaced with ``\code{\n}''. The naive \LaTeX\ implementation of this would be to accumulate the entire environment contents verbatim within a single macro and then perform newline substitutions. For long environment contents, this can lead to buffer memory errors (\LaTeX's |buf_size|). It should be possible to avoid this, but only with more creative algorithms that bring additional complexity.
327 % \item Python literal syntax only requires that the backslash plus the string delimiter be escaped within strings. JSON has the additional requirement that command characters be escaped.
328 % \end{itemize}
329 %
330 % \pkg{latex2pydata} is designed for use with Python and there are no plans to add additional data formats for use with other languages. Choosing Python literal syntax does make \pkg{latex2pydata} less compatible with other programming languages than JSON or some other formats would be. However, the only data structures used are |dict[str, str]| and |list[dict[str, str]]|. It should be straightforward to implement a parser for this subset of Python literal syntax in other languages.
331 %
332 % Data structures are limited to |dict[str, str]| and |list[dict[str, str]]| because the objective is to minimize the potential for errors during serialization and deserialization. These are simple enough data structures that basic checking for incomplete or malformed data is possible on the \LaTeX\ side during writing or buffering. More complex data types, such as floating point numbers or deeply nested dicts, would be difficult to validate on the \LaTeX\ side, so invalid values would tend to result in parse errors during deserialization in Python. The current approach still allows for a broad variety of data types via a schema, with the advantage that it can be easier to give useful error messages during schema validation than during deserialization parsing.
333 %
334 %
335 %
336 % \section{Usage}
337 %
338 % Load the package as usual: |\usepackage{latex2pydata}|. There are no package options.
339 %
340 %
341 % \subsection[Errors]{\textcolor{red}{Errors}}
342 %
343 % Most \LaTeX\ packages handle errors based on the |-interaction| and |-halt-on-error| command-line options, plus |\interactionmode| and associated macros. With the common |-interaction=nonstopmode|, \LaTeX\ will continue after most errors except some related to missing external files.
344 %
345 % \pydata\ is designed to force \LaTeX\ to exit immediately after any \pydata\ errors. \pydata\ is designed for serializing data to file, typically so that an external program (restricted or unrestricted shell escape, or otherwise) can process the data and potentially generate output intended for \LaTeX. Data that is known to be incomplete or malformed should not be passed to external programs, particularly via shell escape.
346 %
347 % When \pydata\ forces \LaTeX\ to exit immediately, there will typically be a message similar to ``\Verb[breaklines]{! Emergency stop [...] cannot \read from terminal in nonstop modes}.'' This is due to the mechanism that \pydata\ uses to force \LaTeX\ to exit. To debug, go back further up the log to find the \pydata\ error message that caused exiting.
348 %
349 %
350 % \subsection{File handling}
351 %
352 % All file handling commands operate globally (|\global|, |\gdef|, etc.).
353 %
354 % \DescMacro{\pydatasetfilehandle\marg{filehandle}}
355 % Configure writing to file using an existing file handle created with |\newwrite|. This allows manual management of the file handle. For example:
356 % \begin{Verbatim}[gobble=2]
357 % \newwrite\testdata
358 % \immediate\openout\testdata=\jobname.pydata\relax
359 % \pydatasetfilehandle{\testdata}
360 % ...
361 % \pydatareleasefilehandle{\testdata}
362 % \immediate\closeout\testdata
363 % \end{Verbatim}
364 %
365 % To switch from one file handle to another, simply use |\pydatasetfilehandle| with the new file handle. When the file handle is no longer in use, |\pydatareleasefilehandle| is recommended (but not required) to remove references to the file handle and perform basic checking for incomplete or malformed data written to file.
366 %
367 % |\pydatasetfilehandle| sets the file handle globally.
368 %
369 % \DescMacro{\pydatareleasefilehandle\marg{filehandle}}
370 % When a file handle is no longer needed, remove references to it. Also perform basic checking for incomplete or malformed data written to file.
371 %
372 % This should only be used once per opened file, after all data has been written, just before the file is closed. It is not needed when switching from one file handle to another when both files remain open; in that case, only |\pydatasetfilehandle| is needed. If |\pydatareleasefilehandle| is used before all data is written, or it is used multiple times while writing to the same file, then it is no longer possible to detect incomplete or malformed data.
373 %
374 % \DescMacro{\pydatasetfilename\marg{filename}}
375 % Configure a file for writing based on filename, opening the file if necessary. For example:
376 % \begin{Verbatim}[gobble=2]
377 % \pydatasetfilename{\jobname.pydata}
378 % \end{Verbatim}
379 % This is not designed for manual management of the file handle. The file does not have to be closed manually since this will happen automatically at the end of the document. However, using |\pydataclosefilename|\marg{filename} is recommended since it closes the file immediately and also performs basic checking for incomplete or malformed data written to file.
380 %
381 % To switch from one file to another, simply use |\pydatasetfilename| with the new filename. When the file is no longer in use, |\pydataclosefilename| is recommended.
382 %
383 % |\pydatasetfilename| sets the filename globally.
384 %
385 % Implementation note: This automatically creates the necessary file handles with |\newwrite|. File handles are automatically reused when files are closed, so that the total number of file handles created is never more than the maximum number of files open simultaneously. This minimizes the potential for ``\Verb[breaklines]{No more room for a new \write}'' errors.
386 %
387 % \DescMacro{\pydataclosefilename\marg{filename}}
388 % Close a file previously opened with |\pydatasetfilename|. Also perform basic checking for incomplete or malformed data written to file.
389 %
390 %
391 % \subsection{Metadata}
392 %
393 % \pkg{latex2pydata} optionally supports writing metadata to file, including basic schema definitions for values. When data is loaded with the \pydatapy, the schema is used to perform type conversion and type checking. When a schema definition exists for a given value, the value is initially loaded as a string, and then it is evaluated with \astliteval. An error is raised if this process does not result in an object with the data type specified in the schema.
394 %
395 % \DescMacro{\pydatasetschemamissing\marg{missing behavior}}
396 % This determines how the schema is processed when the schema is missing definitions for one or more key-value pairs. Options for \meta{missing behavior}:
397 % \begin{itemize}
398 % \item |error| (default): If a schema is defined then a complete schema is required. That is, a schema definition must exist for all key-value pairs or an error is raised.
399 % \item |verbatim|: The schema is enforced for all key-value pairs for which it is defined, and any other key-value pairs are kept with string values. These string values are the raw verbatim text passed from \LaTeX.
400 % \item |evalany|: The schema is enforced for all key-value pairs for which it is defined, and any other key-value pairs have the value evaluated with \astliteval, with all value data types being permitted. Because all values without a schema definition are evaluated, any string values without a schema definition must be quoted and escaped as Python strings on the \LaTeX\ side.
401 % \end{itemize}
402 %
403 % \DescMacro{\pydatasetschemakeytype\marg{key}\marg{value type}}
404 % Define a key's schema. For example, |\pydatasetschemakeytype{key}{int}|. The value is initially loaded as a string, and then this is evaluated with \astliteval. An error is raised if this process does not result in an object with the specified data type. \meta{value type} should be a standard Python type annotation, such as |list[int]| or |dict[str, float]|. To keep a string value received from \LaTeX\ verbatim without any evaluation, use the special |verbatim| type.
405 %
406 % The following scalar data types are supported: |bool|, |bytes|, |float|, |int|, |None|, |str|, and |tuple|. The following collection types are supported: |dict|, |list|, and |set|. |Any| is supported for scalars and for collections (subscripting |Any[...]| is not supported for collections). There is also a |verbatim| data type that is defined specifically for \pkg{latex2pydata}. This keeps the string data received from \LaTeX\ verbatim, without any interpretation by \astliteval.
407 %
408 % See the \pydatapy\ documentation for more details.
409 %
410 % \DescMacro{\pydataclearschema}
411 % Delete the existing schema. If the schema is not deleted, it can be reused across multiple output files.
412 %
413 % \DescMacro{\pydatawritemeta}
414 % Write metadata, including schema, to a file previously configured with |\pydatasetfilename| or |\pydatasetfilehandle|. Metadata must always be the first thing written to file, before any data.
415 %
416 % \DescMacro{\pydataclearmeta}
417 % Clear all metadata. This includes deleting the schema and resetting schema missing behavior to the default.
418 %
419 %
420 %
421 % \subsection{Writing list and dict delimiters}
422 % The overall data structure, before any schema is applied by the \pydatapy, can be either |list[dict[str, str]]| or |dict[str, str]|. This determines which data collection delimiters are needed.
423 %
424 % Delimiters are written to the file previously configured via |\pydatasetfilehandle| or |\pydatasetfilename|.
425 %
426 % \DescMacro{\pydatawritedictopen}
427 % Write an opening dict delimiter |{| to file.
428 %
429 % \DescMacro{\pydatawritedictclose}
430 % Write a closing dict delimiter |}| to file.
431 %
432 % \DescMacro{\pydatawritelistopen}
433 % Write an opening list delimiter |[| to file.
434 %
435 % \DescMacro{\pydatawritelistclose}
436 % Write a closing list delimiter |]| to file.
437 %
438 %
439 % \subsection{Writing keys and values}
440 % All keys must be single-line strings of text without a newline. Both single-line and multi-line values are supported. Keys and values are written to the file previously configured via |\pydatasetfilehandle| or |\pydatasetfilename|.
441 %
442 % Commands for writing keys and values may read these keys and values in one of two ways.
443 % \begin{itemize}
444 % \item Commands whose names contain |key| or |value| read these arguments verbatim, as described below.
445 % \item Commands whose names contain |edefkey| or |edefvalue| read these arguments normally, then expand the arguments via |\edef|, and finally interpret the result as verbatim text.
446 % \end{itemize}
447 %
448 % The \pydata\ commands that read keys and values verbatim have some limitations. When these commands are used inside other commands, they use macros from \fvextra\ to attempt to interpret their arguments as verbatim. However, there are limitations in this case because the arguments are already tokenized:
449 % \begin{itemize}
450 % \item |#| and |%| cannot be used.
451 % \item Curly braces are only allowed in pairs.
452 % \item Multiple adjacent spaces will be collapsed into a single space.
453 % \item Be careful with backslashes. A backslash that is followed by one or more ASCII letters will cause a following space to be lost, if the space is not immediately followed by an ASCII letter.
454 % \item A single |^| is fine, but |^^| will serve as an escape sequence for an ASCII command character.
455 % \end{itemize}
456 % When the \pydata\ commands are used inside other commands that pass their arguments to the \pydata\ commands, it may be best to avoid these limitations by defining the other commands to read their arguments verbatim. Consider using the \href{https://ctan.org/pkg/xparse}{\pkg{xparse}} package. It is also possible to use |\FVExtraReadVArg| from \fvextra; for an example, see the implementation of |\pydatawritekey|.
457 %
458 % Because the \pydata\ commands treat keys and values as verbatim, any desired macro expansion must be performed before passing the keys and values to the \pydata\ commands.
459 %
460 % \DescMacro{\pydatawritekey\marg{key}}
461 % Write a key to file.
462 %
463 % \DescMacro{\pydatawritevalue\marg{value}}
464 % Write a single-line value to file.
465 %
466 % \DescMacro{\pydatawritekeyvalue\marg{key}\marg{value}}
467 % Write a key and a single-line value to file simultaneously.
468 %
469 % \DescMacro{\pydatawritekeyedefvalue\marg{key}\marg{value}}
470 % Write a key and a single-line value to file simultaneously. The value is expanded via |\edef| before being interpreted as verbatim text and then written.
471 %
472 % \DescEnv{pydatawritemlvalue}
473 % Write a multi-line value to file.
474 %
475 % This environment uses \fvextra\ and \fancyvrb\ internally to capture the environment contents verbatim. If a new environment is defined as a wrapper for |pydatawritemlvalue|, then |\VerbatimEnvironment| must be used at the beginning of the new environment definition. This configures \fancyvrb\ to find the end of the new environment correctly.
476 %
477 % \DescMacro{\pydatawritemlvalueopen}
478 %
479 % \DescMacro{\pydatawritemlvalueline\marg{line}}
480 %
481 % \DescMacro{\pydatawritemlvalueclose}
482 % These commands allow writing a multi-line value to file one line at a time. \meta{line} is interpreted verbatim.
483 %
484 %
485 %
486 % \subsection{Buffer}
487 %
488 % Key-value data can be written to file once a dict is opened with |\pydatawritedictopen|. It is also possible to accumulate key-value data in a ``buffer.'' This is convenient when the data serves as input to an external program that generates cached content. Buffered data can be hashed in memory without being written to file, so the existence of cached content can be checked efficiently.
489 %
490 % A buffer consists of a sequence of macros of the form |\|\meta{buffername}|line|\meta{n}, where each line of data corresponds to a macro and \meta{n} is an integer greater than or equal to one (one-based indexing). The length of the buffer is stored in the macro |\|\meta{buffername}|length|. Buffers are limited to containing comma-separated key-value data, without any opening or closing dict delimiters |{}|.
491 %
492 % All buffer commands that set the buffer or modify the buffer operate globally (|\global|, |\gdef|, etc.).
493 %
494 %
495 % \subsubsection{Creating and deleting buffers}
496 %
497 % \DescMacro{\pydatasetbuffername\marg{buffername}}
498 % Initialize a new buffer if \meta{buffername} has not been used previously, and configure all buffer operations to use \meta{buffername}.
499 %
500 % \meta{buffername} is used as a base name for creating the buffer line macros of the form |\|\meta{buffername}|line|\meta{n} and the buffer length macro |\|\meta{buffername}|length|.
501 %
502 % \DescMacro{\pydataclearbuffername\marg{buffername}}
503 % Delete the specified buffer. |\let| all line macros |\|\meta{buffername}|line|\meta{n} to an undefined macro, and set the length macro |\|\meta{buffername}|length| to zero.
504 %
505 %
506 % \subsubsection{Special buffer operations}
507 %
508 % \DescMacro{\pydatabuffermdfivesum}
509 % Calculate the MD5 hash of the current buffer, using |\str_mdfive_hash:e|. This is fully expandable. For example:
510 % \begin{Verbatim}[gobble=2]
511 % \edef\hash{\pydatabuffermdfivesum}
512 % \end{Verbatim}
513 %
514 % \DescMacro{\pydatawritebuffer}
515 % Write the current buffer to the file previously configured via |\pydatasetfilename| or |\pydatasetfilehandle|.
516 %
517 % Writing the buffer does not modify the buffer in any way or delete it. To delete the buffer after writing, use |\pydataclearbuffername|.
518 %
519 %
520 % \subsubsection{Buffering keys and values}
521 % All keys must be single-line strings of text without a newline. Both single-line and multi-line values are supported. Keys and values are appended to the buffer previously configured via |\pydatasetbuffername|.
522 %
523 % The \pydata\ commands read keys and values verbatim. Like the commands for writing keys and values, the commands for buffering keys and values have limitations when used inside other commands.
524 %
525 % \DescMacro{\pydatabufferkey\marg{key}}
526 % Append a key to the buffer.
527 %
528 % \DescMacro{\pydatabuffervalue\marg{value}}
529 % Append a single-line value to the buffer.
530 %
531 % \DescMacro{\pydatabufferkeyvalue\marg{key}\marg{value}}
532 % Append a key and a single-line value to the buffer simultaneously.
533 %
534 % \DescMacro{\pydatabufferkeyedefvalue\marg{key}\marg{value}}
535 % Append a key and a single-line value to the buffer simultaneously. The value is expanded via |\edef| before being interpreted as verbatim text and then buffered.
536 %
537 % \DescEnv{pydatabuffermlvalue}
538 % Append a multi-line value to the buffer.
539 %
540 % This environment uses \fvextra\ and \fancyvrb\ internally to capture the environment contents verbatim. If a new environment is defined as a wrapper for |pydatabuffermlvalue|, then |\VerbatimEnvironment| must be used at the beginning of the new environment definition. This configures \fancyvrb\ to find the end of the new environment correctly.
541 %
542 % \DescMacro{\pydatabuffermlvalueopen}
543 %
544 % \DescMacro{\pydatabuffermlvalueline\marg{line}}
545 %
546 % \DescMacro{\pydatabuffermlvalueclose}
547 % These commands allow buffering a multi-line value one line at a time. \meta{line} is interpreted verbatim.
548 %
549 %
550 %
551 %
552 % \StopEventually{\PrintIndex}
553 %
554 % \section{Implementation}
555 %
556 % \iffalse
557 %<*package>
558 % \fi
559 %
560 %
561 %
562 % \subsection{Exception handling}
563 % \begin{macro}{\pydata@error}
564 % Shortcut for error message. The |\batchmode\read -1 to \pydata@exitnow| forces an immediate exit with ``\Verb[breaklines]{! Emergency stop [...] cannot \read from terminal in nonstop modes}.'' Due to the potentially critical nature of written or buffered data, any errors in assembling the data should be treated as fatal.
565 % \begin{macrocode}
566 \def\pydata@error#1{%
567 \PackageError{latex2pydata}{#1}{}%
568 \batchmode\read -1 to \pydata@exitnow}
569 % \end{macrocode}
570 % \end{macro}
571 %
572 % \begin{macro}{\pydata@warning}
573 % Shortcut for warning message.
574 % \begin{macrocode}
575 \def\pydata@warning#1{%
576 \PackageWarning{latex2pydata}{#1}}
577 % \end{macrocode}
578 % \end{macro}
579 %
580 %
581 %
582 % \subsection{Required packages}
583 % \begin{macrocode}
584 \RequirePackage{etoolbox}
585 \RequirePackage{fvextra}
586 \IfPackageAtLeastTF{fvextra}{2026/02/25}%
587 {}{\pydata@error{package fvextra is outdated; upgrade to the latest version}}
588 % \end{macrocode}
589 %
590 %
591 %
592 % \subsection{Util}
593 %
594 % \begin{macro}{\pydata@empty}
595 % Empty macro.
596 % \begin{macrocode}
597 \def\pydata@empty{}
598 % \end{macrocode}
599 % \end{macro}
600 %
601 %
602 % \begin{macro}{\pydata@newglobalbool, \pydata@provideglobalbool}
603 % Variants of \pkg{etoolbox}'s |\newbool| and |\providebool| that create bools whose state is always global. When these global bools are used with |\setbool|, |\booltrue|, or |\boolfalse|, the global state is updated regardless of whether the command is prefixed with |\global|. These use a global variant of \LaTeX's |\newif| internally.
604 % \begin{macrocode}
605 \def\pydata@gnewif#1{%
606 \count@\escapechar
607 \escapechar\m@ne
608 \global\let#1\iffalse
609 \pydata@gif#1\iftrue
610 \pydata@gif#1\iffalse
611 \escapechar\count@}
612 \def\pydata@gif#1#2{%
613 \expandafter\gdef\csname
614 \expandafter\@gobbletwo\string#1\expandafter\@gobbletwo\string#2\endcsname
615 {\global\let#1#2}}
616 \newrobustcmd*{\pydata@newglobalbool}[1]{%
617 \begingroup
618 \let\newif\pydata@gnewif
619 \newbool{#1}%
620 \endgroup}
621 \newrobustcmd*{\pydata@provideglobalbool}[1]{%
622 \begingroup
623 \let\newif\pydata@gnewif
624 \providebool{#1}%
625 \endgroup}
626 % \end{macrocode}
627 % \end{macro}
628 %
629 %
630 %
631 % \subsection{State}
632 %
633 % Track state of writing data and of buffering data. Notice that bools for tracking state are a special, custom variant that is always global.
634 %
635 % \begin{macro}{pydata@canwrite}
636 % Whether data can be written. False if a file handle has not been set or if the top-level data structure has been closed.
637 % \begin{macrocode}
638 \pydata@newglobalbool{pydata@canwrite}
639 % \end{macrocode}
640 % \end{macro}
641 %
642 % \begin{macro}{pydata@hasmeta}
643 % Whether metadata was written. Metadata is a \code{dict[str, str | dict[str, str]]}.
644 % \begin{macrocode}
645 \pydata@newglobalbool{pydata@hasmeta}
646 % \end{macrocode}
647 % \end{macro}
648 %
649 % \begin{macro}{pydata@topexists}
650 % Whether the top-level data structure has been configured. The top-level data structure can be a list or a dict. The overall data structure must be either |dict[str, str]| or |list[dict[str, str]]|.
651 % \begin{macrocode}
652 \pydata@newglobalbool{pydata@topexists}
653 % \end{macrocode}
654 % \end{macro}
655 %
656 % \begin{macro}{pydata@topislist}
657 % Whether the top-level data structure is a list.
658 % \begin{macrocode}
659 \pydata@newglobalbool{pydata@topislist}
660 % \end{macrocode}
661 % \end{macro}
662 %
663 % \begin{macro}{pydata@indict}
664 % Whether a dict has been opened.
665 % \begin{macrocode}
666 \pydata@newglobalbool{pydata@indict}
667 % \end{macrocode}
668 % \end{macro}
669 %
670 % \begin{macro}{pydata@haskey}
671 % Whether a key has been written (waiting for a value).
672 % \begin{macrocode}
673 \pydata@newglobalbool{pydata@haskey}
674 % \end{macrocode}
675 % \end{macro}
676 %
677 % \begin{macro}{\pydata@fhstartstate, \pydata@fhstopstate, \pydata@fhresetstate}
678 % Start and stop state tracking for a file handle (|\newwrite|), or reset state after writing is complete. Each file handle has its own set of state bools of the form |pydata@|\meta{boolname}|@|\meta{fh}. When a file handle is in use, the values of these bools are copied into the |pydata@|\meta{boolname} bools; when the file handle is no longer in use, |pydata@|\meta{boolname} values are copied back into |pydata@|\meta{boolname}|@|\meta{fh}.
679 % \begin{macrocode}
680 \def\pydata@fhstartstate#1{%
681 \expandafter\pydata@fhstartstate@i\expandafter{\number#1}}
682 \newbool{pydata@fhnewstate}
683 \def\pydata@fhstartstate@i#1{%
684 \ifcsname ifpydata@canwrite@#1\endcsname
685 \boolfalse{pydata@fhnewstate}%
686 \else
687 \booltrue{pydata@fhnewstate}%
688 \fi
689 \def\do##1{%
690 \pydata@provideglobalbool{pydata@##1@#1}%
691 \ifbool{pydata@##1@#1}{\booltrue{pydata@##1}}{\boolfalse{pydata@##1}}}%
692 \docsvlist{canwrite, hasmeta, topexists, topislist, indict, haskey}%
693 \ifbool{pydata@fhnewstate}%
694 {\booltrue{pydata@canwrite}}{}%
695 \ifbool{pydata@fhisreleased@#1}%
696 {\boolfalse{pydata@fhisreleased@#1}\booltrue{pydata@canwrite}}{}}
697 \def\pydata@fhstopstate#1{%
698 \expandafter\pydata@fhstopstate@i\expandafter{\number#1}}
699 \def\pydata@fhstopstate@i#1{%
700 \ifcsname ifpydata@canwrite@#1\endcsname
701 \def\do##1{%
702 \ifbool{pydata@##1}{\booltrue{pydata@##1@#1}}{\boolfalse{pydata@##1@#1}}%
703 \boolfalse{pydata@##1}}%
704 \docsvlist{canwrite, hasmeta, topexists, topislist, indict, haskey}%
705 \fi}
706 \def\pydata@fhresetstate#1{%
707 \expandafter\pydata@fhresetstate@i\expandafter{\number#1}}
708 \def\pydata@fhresetstate@i#1{%
709 \def\do##1{%
710 \boolfalse{pydata@##1@#1}}%
711 \docsvlist{canwrite, hasmeta, topexists, topislist, indict, haskey}}
712 % \end{macrocode}
713 % \end{macro}
714 %
715 % \begin{macro}{pydata@bufferhaskey}
716 % Whether a key has been added to the buffer (waiting for a value).
717 %
718 % If multiple buffers are in use, all buffers use the same |pydata@bufferhaskey|. Inconsistent state is avoided by requiring that |\pydatasetbuffername| can only be invoked when |pydata@bufferhaskey| is false.
719 % \begin{macrocode}
720 \pydata@newglobalbool{pydata@bufferhaskey}
721 % \end{macrocode}
722 % \end{macro}
723 %
724 %
725 %
726 % \subsection{File handle}
727 %
728 % \begin{macro}{\pydata@filehandle}
729 % File handle for writing data.
730 % \begin{macrocode}
731 \let\pydata@filehandle\relax
732 % \end{macrocode}
733 % \end{macro}
734 %
735 % \begin{macro}{\pydata@checkfilehandle}
736 % Check whether file handle has been set.
737 % \begin{macrocode}
738 \def\pydata@checkfilehandle{%
739 \ifx\pydata@filehandle\relax
740 \pydata@error{Undefined file handle; use \string\pydatasetfilehandle}%
741 \fi}
742 % \end{macrocode}
743 % \end{macro}
744 %
745 % \begin{macro}{\pydatasetfilehandle, \pydatareleasefilehandle}
746 % Set and release file handle. Release isn't strictly required, but it is necessary for basic data checking on the \LaTeX\ side.
747 % \begin{macrocode}
748 \def\pydatasetfilehandle#1{%
749 \if\relax\detokenize{#1}\relax
750 \pydata@error{Missing file handle}%
751 \fi
752 \ifx\pydata@filehandle\relax
753 \else\ifx\pydata@filehandle#1\relax
754 \else
755 \pydata@fhstopstate{\pydata@filehandle}%
756 \fi\fi
757 \ifx\pydata@filehandle#1\relax
758 \else
759 \global\let\pydata@filehandle#1\relax
760 \pydata@provideglobalbool{pydata@fhisreleased@\number#1}%
761 \pydata@fhstartstate{#1}%
762 \fi}
763 \def\pydatareleasefilehandle#1{%
764 \ifcsname ifpydata@canwrite@\number#1\endcsname
765 \else
766 \pydata@error{Unknown file handle #1}%
767 \fi
768 \ifx\pydata@filehandle#1\relax
769 \pydata@fhstopstate{#1}%
770 \global\let\pydata@filehandle\relax
771 \fi
772 \ifbool{pydata@canwrite@\number#1}%
773 {\ifbool{pydata@haskey@\number#1}%
774 {\pydata@error{Incomplete data: key is waiting for value}}{}%
775 \ifbool{pydata@indict@\number#1}%
776 {\pydata@error{Incomplete data: dict is not closed}}{}%
777 \ifbool{pydata@topislist@\number#1}%
778 {\pydata@error{Incomplete data: list is not closed}}{}}%
779 {}%
780 \pydata@fhresetstate{#1}%
781 \booltrue{pydata@fhisreleased@\number#1}}
782 % \end{macrocode}
783 % \end{macro}
784 %
785 % \begin{macro}{\pydatasetfilename, \pydataclosefilename}
786 % Shortcut for invoking |\newwrite| and then passing the file handle to |\pydatasetfilehandle|. File handles are global. If the close macro is not invoked, then basic data checking on the \LaTeX\ side will not be performed. However, \TeX\ will \href{https://tex.stackexchange.com/a/337291}{automatically close open writes at the end of the compile}.
787 %
788 % File handles created by |\newwrite| are collected in a file handle ``pool'' and then reused when possible to minimize the potential for ``\Verb[breaklines]{No more room for a new \write}'' errors.
789 % \begin{macrocode}
790 \def\pydata@fhpoolsize{0}
791 \def\pydatasetfilename#1{%
792 \if\relax\detokenize{#1}\relax
793 \pydata@error{Missing filename}%
794 \fi
795 \ifcsname pydata@filenamefh@#1\endcsname
796 \expandafter\let\expandafter\pydata@fhtmp
797 \csname pydata@filenamefh@#1\endcsname
798 \expandafter\let\expandafter\pydata@fhpoolindextmp
799 \csname pydata@filenamefhpoolindex@#1\endcsname
800 \else
801 \def\pydata@fhpoolindex{0}%
802 \loop\unless\ifnum\pydata@fhpoolindex=\pydata@fhpoolsize\relax
803 \ifbool{pydata@fileisopen@\pydata@fhpoolindex}%
804 {}%
805 {\expandafter\let\expandafter\pydata@fhtmp
806 \csname pydata@fh@\pydata@fhpoolindex\endcsname
807 \let\pydata@fhpoolindextmp\pydata@fhpoolindex
808 \expandafter\global\expandafter
809 \let\csname pydata@filenamefh@#1\endcsname\pydata@fhtmp
810 \expandafter\global\expandafter
811 \let\csname pydata@filenamefhpoolindex@#1\endcsname\pydata@fhpoolindextmp
812 \let\pydata@fhpoolindex\pydata@fhpoolsize}%
813 \repeat
814 \let\pydata@fhpoolindex\pydata@undefined
815 \ifcsname pydata@filenamefh@#1\endcsname
816 \else
817 \expandafter\newwrite\csname pydata@fh@\pydata@fhpoolsize\endcsname
818 \pydata@newglobalbool{pydata@fileisopen@\pydata@fhpoolsize}%
819 \expandafter\let\expandafter\pydata@fhtmp
820 \csname pydata@fh@\pydata@fhpoolsize\endcsname
821 \expandafter\global\expandafter
822 \let\csname pydata@filenamefh@#1\endcsname\pydata@fhtmp
823 \let\pydata@fhpoolindextmp\pydata@fhpoolsize
824 \expandafter\global\expandafter
825 \let\csname pydata@filenamefhpoolindex@#1\endcsname\pydata@fhpoolindextmp
826 \xdef\pydata@fhpoolsize{\the\numexpr\pydata@fhpoolsize+1\relax}%
827 \fi
828 \fi
829 \ifbool{pydata@fileisopen@\pydata@fhpoolindextmp}%
830 {}%
831 {\immediate\openout\pydata@fhtmp=#1\relax
832 \booltrue{pydata@fileisopen@\pydata@fhpoolindextmp}}%
833 \pydatasetfilehandle{\pydata@fhtmp}%
834 \let\pydata@fhtmp\pydata@undefined
835 \let\pydata@fhpoolindextmp\pydata@undefined}
836 \def\pydataclosefilename#1{%
837 \ifcsname pydata@filenamefh@#1\endcsname
838 \expandafter\let\expandafter\pydata@fhtmp
839 \csname pydata@filenamefh@#1\endcsname
840 \expandafter\let\expandafter\pydata@fhpoolindextmp
841 \csname pydata@filenamefhpoolindex@#1\endcsname
842 \pydatareleasefilehandle{\pydata@fhtmp}%
843 \immediate\closeout\pydata@fhtmp
844 \boolfalse{pydata@fileisopen@\pydata@fhpoolindextmp}%
845 \expandafter\global\expandafter
846 \let\csname pydata@filenamefh@#1\endcsname\pydata@undefined
847 \expandafter\global\expandafter
848 \let\csname pydata@filenamefhpoolindex@#1\endcsname\pydata@undefined
849 \let\pydata@fhtmp\pydata@undefined
850 \let\pydata@fhpoolindextmp\pydata@undefined
851 \else
852 \pydata@error{Unknown file name "#1"}%
853 \fi}
854 % \end{macrocode}
855 % \end{macro}
856 %
857 %
858 %
859 % \subsection{Buffer}
860 %
861 % Key-value data can be written directly to file once a dict is opened. It is also possible to accumulate key-value data in a ``buffer.'' This is convenient when the data serves as input to an external program that generates cached content. Buffered data can be hashed in memory without being written to file, so the existence of cached content can be checked efficiently.
862 %
863 % The buffer consists of a sequence of macros of the form |\<buffer_name>line<n>|, where each line of data corresponds to a macro and |<n>| is an integer greater than or equal to one. The length of the buffer is stored in the macro |\<buffer_name>length|. The buffer includes comma-separated key-value data, without any opening or closing dict delimiters |{}|.
864 %
865 % \begin{macro}{\pydata@bufferindex}
866 % Macro for looping through buffers.
867 % \begin{macrocode}
868 \def\pydata@bufferindex{0}
869 % \end{macrocode}
870 % \end{macro}
871 %
872 % \begin{macro}{\pydatasetbuffername, \pydata@buffername, \pydata@bufferlinename, \pydata@bufferlengthname, \pydata@bufferlengthmacro}
873 % Set the buffer base name and create a corresponding length macro if it does not exist.
874 % \begin{macrocode}
875 \def\pydatasetbuffername#1{%
876 \ifbool{pydata@bufferhaskey}%
877 {\pydata@error{Cannot change buffers when a buffered key is waiting for a value}}%
878 {}%
879 \gdef\pydata@buffername{#1}%
880 \gdef\pydata@bufferlinename{#1line}%
881 \gdef\pydata@bufferlengthname{#1length}%
882 \ifcsname\pydata@bufferlengthname\endcsname
883 \else
884 \expandafter\gdef\csname\pydata@bufferlengthname\endcsname{0}%
885 \fi
886 \expandafter\gdef\expandafter\pydata@bufferlengthmacro\expandafter{%
887 \csname\pydata@bufferlengthname\endcsname}}
888 \pydatasetbuffername{pydata@defaultbuffer}
889 % \end{macrocode}
890 % \end{macro}
891 %
892 % \begin{macro}{\pydatawritebuffer}
893 % Write existing buffer macros to file handle.
894 % \begin{macrocode}
895 \def\pydatawritebuffer{%
896 \ifnum\pydata@bufferlengthmacro<1\relax
897 \pydata@error{Cannot write empty buffer}%
898 \fi
899 \pydata@checkfilehandle
900 \ifbool{pydata@indict}{}{\pydata@error{Cannot write buffer unless in a dict}}%
901 \ifbool{pydata@haskey}%
902 {\pydata@error{Cannot write buffer when file has a key waiting for a value}}{}%
903 \ifbool{pydata@bufferhaskey}%
904 {\pydata@error{Cannot write buffer when a buffered key is waiting for a value}}{}%
905 \gdef\pydata@bufferindex{1}%
906 \loop\unless\ifnum\pydata@bufferindex>\pydata@bufferlengthmacro\relax
907 \immediate\write\pydata@filehandle{%
908 \csname\pydata@bufferlinename\pydata@bufferindex\endcsname}%
909 \xdef\pydata@bufferindex{\the\numexpr\pydata@bufferindex+1\relax}%
910 \repeat
911 \gdef\pydata@bufferindex{0}}
912 % \end{macrocode}
913 % \end{macro}
914 %
915 %
916 % \begin{macro}{\pydataclearbuffername}
917 % Delete the buffer: |\let| all line macros to an undefined macro, and set length to zero.
918 % \begin{macrocode}
919 \def\pydataclearbuffername#1{%
920 \def\pydata@clearbuffername{#1}%
921 \ifcsname#1length\endcsname
922 \else
923 \pydata@error{Buffer #1 does not exist}%
924 \fi
925 \gdef\pydata@bufferindex{1}%
926 \loop\unless\ifnum\pydata@bufferindex>\csname#1length\endcsname\relax
927 \expandafter\global\expandafter\let
928 \csname#1line\pydata@bufferindex\endcsname\pydata@undefined
929 \xdef\pydata@bufferindex{\the\numexpr\pydata@bufferindex+1\relax}%
930 \repeat
931 \expandafter\gdef\csname#1length\endcsname{0}%
932 \gdef\pydata@bufferindex{0}%
933 \ifx\pydata@clearbuffername\pydata@buffername
934 \boolfalse{pydata@bufferhaskey}%
935 \fi}
936 % \end{macrocode}
937 % \end{macro}
938 %
939 %
940 % \begin{macro}{\pydatabuffermdfivesum}
941 % Calculate buffer MD5.
942 % \begin{macrocode}
943 \def\pydatabuffermdfivesum{%
944 \csname str_mdfive_hash:e\endcsname{%
945 \ifnum\pydata@bufferlengthmacro<1
946 \expandafter\@firstoftwo
947 \else
948 \expandafter\@secondoftwo
949 \fi
950 {}{\pydatabuffermdfivesum@i{1}}}}
951 \def\pydatabuffermdfivesum@i#1{%
952 \csname\pydata@bufferlinename#1\endcsname^^J%
953 \ifnum\pydata@bufferlengthmacro=#1
954 \expandafter\@gobble
955 \else
956 \expandafter\@firstofone
957 \fi
958 {\expandafter\pydatabuffermdfivesum@i\expandafter{\the\numexpr#1+1\relax}}}
959 % \end{macrocode}
960 % \end{macro}
961 %
962 %
963 %
964 % \subsection{String processing}
965 %
966 % Ensure correct catcode for double quotation mark, which will be used for delimiting all Python string literals.
967 % \begin{macrocode}
968 \begingroup
969 \catcode`\"=12\relax
970 % \end{macrocode}
971 %
972 %
973 % \begin{macro}{\pydata@escstrtext}
974 % Escape string text by replacing |\| with |\\| and |"| with |\"|. Any text that requires expansion must be expanded prior to escaping. The string text is processed with |\detokenize| to ensure catcodes and prepare it for writing. This is redundant in cases where text has already been processed with |\FVExtraDetokenizeVArg|.
975 % \begin{macrocode}
976 \begingroup
977 \catcode`\!=0
978 !catcode`!\=12
979 !gdef!pydata@escstrtext#1{%
980 !expandafter!pydata@escstrtext@i!detokenize{#1}\!FV@Sentinel}
981 !gdef!pydata@escstrtext@i#1\#2!FV@Sentinel{%
982 !if!relax!detokenize{#2}!relax
983 !expandafter!@firstoftwo
984 !else
985 !expandafter!@secondoftwo
986 !fi
987 {!pydata@escstrtext@ii#1"!FV@Sentinel}%
988 {!pydata@escstrtext@ii#1\\"!FV@Sentinel!pydata@escstrtext@i#2!FV@Sentinel}}
989 !gdef!pydata@escstrtext@ii#1"#2!FV@Sentinel{%
990 !if!relax!detokenize{#2}!relax
991 !expandafter!@firstoftwo
992 !else
993 !expandafter!@secondoftwo
994 !fi
995 {#1}%
996 {#1\"!pydata@escstrtext@ii#2!FV@Sentinel}}
997 !endgroup
998 % \end{macrocode}
999 % \end{macro}
1000 %
1001 %
1002 % \begin{macro}{\pydata@quotestr}
1003 % Escape a string then quote it with |"|.
1004 % \begin{macrocode}
1005 \gdef\pydata@quotestr#1{%
1006 "\pydata@escstrtext{#1}"}
1007 % \end{macrocode}
1008 % \end{macro}
1009 %
1010 %
1011 % \begin{macro}{\pydata@mlstropen, \pydata@mlstrclose}
1012 % Multi-line string delimiters. The opening delimiter has a trailing backslash to prevent the string from starting with a newline.
1013 % \begin{macrocode}
1014 \begingroup
1015 \catcode`\!=0
1016 !catcode`!\=12
1017 !gdef!pydata@mlstropen{"""\}
1018 !gdef!pydata@mlstrclose{"""}
1019 !endgroup
1020 % \end{macrocode}
1021 % \end{macro}
1022 %
1023 %
1024 % \noindent End |"| catcode.
1025 % \begin{macrocode}
1026 \endgroup
1027 % \end{macrocode}
1028 %
1029 %
1030 %
1031 % \subsection{Metadata}
1032 %
1033 % \begin{macro}{\pydata@schema}
1034 % Macro storing key-value schema data.
1035 % \begin{macrocode}
1036 \def\pydata@schema{}
1037 % \end{macrocode}
1038 % \end{macro}
1039 %
1040 % \begin{macro}{\pydatasetschemamissing, \pydata@schemamissing}
1041 % Define behavior for missing key-value pairs in a schema.
1042 % \begin{macrocode}
1043 \let\pydata@schemamissing@error\relax
1044 \let\pydata@schemamissing@verbatim\relax
1045 \let\pydata@schemamissing@evalany\relax
1046 \def\pydatasetschemamissing#1{%
1047 \ifcsname pydata@schemamissing@\detokenize{#1}\endcsname
1048 \else
1049 \pydata@error{Invalid schema missing setting #1}%
1050 \fi
1051 \gdef\pydata@schemamissing{#1}}
1052 \pydatasetschemamissing{error}
1053 % \end{macrocode}
1054 % \end{macro}
1055 %
1056 % \begin{macro}{\pydatasetschemakeytype}
1057 % Define a key's schema. For example, |\pydatasetschemakeytype{key}{int}|.
1058 % \begin{macrocode}
1059 \begingroup
1060 \catcode`\:=12\relax
1061 \catcode`\,=12\relax
1062 \gdef\pydatasetschemakeytype#1#2{%
1063 \ifbool{pydata@hasmeta}{\pydata@error{Must create schema before writing metadata}}{}%
1064 \ifbool{pydata@topexists}{\pydata@error{Must create schema before writing data}}{}%
1065 \expandafter\def\expandafter\pydata@schema\expandafter{%
1066 \pydata@schema\pydata@quotestr{#1}: \pydata@quotestr{#2}, }}
1067 \endgroup
1068 % \end{macrocode}
1069 % \end{macro}
1070 %
1071 % \begin{macro}{\pydataclearschema}
1072 % Delete existing schema. This isn't done automatically upon writing so that a schema can be defined and then reused.
1073 % \begin{macrocode}
1074 \def\pydataclearschema{%
1075 \gdef\pydata@schema{}}
1076 % \end{macrocode}
1077 % \end{macro}
1078 %
1079 %
1080 % \begin{macro}{\pydataclearmeta}
1081 % Delete existing metadata. This isn't done automatically upon writing so that metadata can be defined and then reused.
1082 % \begin{macrocode}
1083 \def\pydataclearmeta{%
1084 \pydatasetschemamissing{error}%
1085 \pydataclearschema}
1086 % \end{macrocode}
1087 % \end{macro}
1088 %
1089 % \begin{macro}{\pydatawritemeta}
1090 % Write metadata to file, including any schema.
1091 % \begin{macrocode}
1092 \begingroup
1093 \catcode`\:=12\relax
1094 \catcode`\#=12\relax
1095 \catcode`\,=12\relax
1096 \gdef\pydatawritemeta{%
1097 \ifbool{pydata@canwrite}%
1098 {}{\pydata@error{Data was already written; cannot write metadata}}%
1099 \ifbool{pydata@hasmeta}{\pydata@error{Already wrote metadata}}{}%
1100 \ifbool{pydata@topexists}{\pydata@error{Must write metadata before writing data}}{}%
1101 \edef\pydata@meta@exp{%
1102 # latex2pydata metadata:
1103 \@charlb
1104 \pydata@quotestr{schema_missing}:
1105 \expandafter\pydata@quotestr\expandafter{\pydata@schemamissing},
1106 \pydata@quotestr{schema}:
1107 \ifx\pydata@schema\pydata@empty
1108 \expandafter\@firstoftwo
1109 \else
1110 \expandafter\@secondoftwo
1111 \fi
1112 {None}{\@charlb\pydata@schema\@charrb},
1113 \@charrb}%
1114 \immediate\write\pydata@filehandle{\pydata@meta@exp}%
1115 \booltrue{pydata@hasmeta}}
1116 \endgroup
1117 % \end{macrocode}
1118 % \end{macro}
1119 %
1120 %
1121 %
1122 % \subsection{Collection delimiters}
1123 %
1124 % \begin{macro}{\pydatawritelistopen, \pydatawritelistclose}
1125 % Write list delimiters. These are only used when the top-level data structure is a list: |list[dict[str, str]]|.
1126 % \begin{macrocode}
1127 \begingroup
1128 \catcode`\[=12\relax
1129 \catcode`\]=12\relax
1130 \gdef\pydatawritelistopen{%
1131 \pydata@checkfilehandle
1132 \ifbool{pydata@canwrite}%
1133 {}{\pydata@error{Data structure is closed; cannot write delim}}%
1134 \ifbool{pydata@topexists}%
1135 {\pydata@error{Top-level data structure already exists}}{}%
1136 \immediate\write\pydata@filehandle{[}%
1137 \booltrue{pydata@topexists}%
1138 \booltrue{pydata@topislist}}
1139 \gdef\pydatawritelistclose{%
1140 \ifbool{pydata@topexists}%
1141 {}{\pydata@error{No data structure is open; cannot write delim}}%
1142 \ifbool{pydata@topislist}%
1143 {}{\pydata@error{Top-level data structure is not a list}}%
1144 \ifbool{pydata@haskey}%
1145 {\pydata@error{Cannot close data structure when key is waiting for value}}{}%
1146 \immediate\write\pydata@filehandle{]}%
1147 \boolfalse{pydata@topexists}%
1148 \boolfalse{pydata@topislist}%
1149 \boolfalse{pydata@hasmeta}%
1150 \boolfalse{pydata@canwrite}}
1151 \endgroup
1152 % \end{macrocode}
1153 % \end{macro}
1154 %
1155 %
1156 % \begin{macro}{\pydatawritedictopen, \pydatawritedictclose}
1157 % Write dict delimiters. These are not the top-level data structure for |list[dict[str, str]]| but are the top-level data structure for |dict[str, str]|.
1158 % \begin{macrocode}
1159 \begingroup
1160 \catcode`\,=12\relax
1161 \gdef\pydatawritedictopen{%
1162 \ifbool{pydata@topislist}%
1163 {\ifbool{pydata@indict}{\pydata@error{Already in a dict; cannot nest}}{}%
1164 \immediate\write\pydata@filehandle{\@charlb}%
1165 \booltrue{pydata@indict}}%
1166 {\pydata@checkfilehandle
1167 \ifbool{pydata@canwrite}%
1168 {}{\pydata@error{Data structure is closed; cannot write delim}}%
1169 \ifbool{pydata@topexists}%
1170 {\pydata@error{Top-level data structure already exists}}{}%
1171 \immediate\write\pydata@filehandle{\@charlb}%
1172 \booltrue{pydata@topexists}%
1173 \booltrue{pydata@indict}}}
1174 \gdef\pydatawritedictclose{%
1175 \ifbool{pydata@indict}{}{\pydata@error{No dict is open; cannot write delim}}%
1176 \ifbool{pydata@haskey}%
1177 {\pydata@error{Cannot close data structure when key is waiting for value}}{}%
1178 \ifbool{pydata@topislist}%
1179 {\immediate\write\pydata@filehandle{\@charrb,}%
1180 \boolfalse{pydata@indict}}%
1181 {\immediate\write\pydata@filehandle{\@charrb}%
1182 \boolfalse{pydata@indict}%
1183 \boolfalse{pydata@topexists}%
1184 \boolfalse{pydata@hasmeta}%
1185 \boolfalse{pydata@canwrite}}}
1186 \endgroup
1187 % \end{macrocode}
1188 % \end{macro}
1189 %
1190 %
1191 %
1192 % \subsection{Keys and values}
1193 %
1194 % \begin{macro}{\pydatawritekey, \pydatabufferkey}
1195 % Write key to file or append it to the buffer.
1196 % \begin{macrocode}
1197 \begingroup
1198 \catcode`\:=12\relax
1199 \gdef\pydatawritekey{%
1200 \FVExtraReadVArg{\FVExtraDetokenizeVArg{\pydatawritekey@i}}}
1201 \gdef\pydatawritekey@i#1{%
1202 \ifbool{pydata@indict}{}{\pydata@error{Cannot write a key unless in a dict}}%
1203 \ifbool{pydata@haskey}{\pydata@error{Cannot write a key when waiting for a value}}{}%
1204 \immediate\write\pydata@filehandle{%
1205 \pydata@quotestr{#1}:%
1206 }%
1207 \booltrue{pydata@haskey}}
1208 \gdef\pydatabufferkey{%
1209 \FVExtraReadVArg{\FVExtraDetokenizeVArg{\pydatabufferkey@i}}}
1210 \gdef\pydatabufferkey@i#1{%
1211 \ifbool{pydata@bufferhaskey}%
1212 {\pydata@error{Cannot buffer a key when waiting for a value}}{}%
1213 \expandafter\xdef\pydata@bufferlengthmacro{%
1214 \the\numexpr\pydata@bufferlengthmacro+1\relax}%
1215 \expandafter\xdef\csname\pydata@bufferlinename\pydata@bufferlengthmacro\endcsname{%
1216 \pydata@quotestr{#1}:%
1217 }%
1218 \booltrue{pydata@bufferhaskey}}
1219 \endgroup
1220 % \end{macrocode}
1221 % \end{macro}
1222 %
1223 % \begin{macro}{\pydatawritevalue, \pydatabuffervalue}
1224 % Write a value to file or append it to the buffer.
1225 % \begin{macrocode}
1226 \begingroup
1227 \catcode`\,=12\relax
1228 \gdef\pydatawritevalue{%
1229 \FVExtraReadVArg{\FVExtraDetokenizeVArg{\pydatawritevalue@i}}}
1230 \gdef\pydatawritevalue@i#1{%
1231 \ifbool{pydata@haskey}{}{\pydata@error{Cannot write value when waiting for a key}}%
1232 \immediate\write\pydata@filehandle{%
1233 \pydata@quotestr{#1},%
1234 }%
1235 \boolfalse{pydata@haskey}}
1236 \gdef\pydatabuffervalue{%
1237 \FVExtraReadVArg{\FVExtraDetokenizeVArg{\pydatabuffervalue@i}}}
1238 \gdef\pydatabuffervalue@i#1{%
1239 \ifbool{pydata@bufferhaskey}%
1240 {}{\pydata@error{Cannot buffer value when waiting for a key}}%
1241 \expandafter\xdef\pydata@bufferlengthmacro{%
1242 \the\numexpr\pydata@bufferlengthmacro+1\relax}%
1243 \expandafter\xdef\csname\pydata@bufferlinename\pydata@bufferlengthmacro\endcsname{%
1244 \pydata@quotestr{#1},%
1245 }%
1246 \boolfalse{pydata@bufferhaskey}}
1247 \endgroup
1248 % \end{macrocode}
1249 % \end{macro}
1250 %
1251 % \begin{macro}{\pydatawritekeyvalue, \pydatawritekeyedefvalue, \pydatabufferkeyvalue, \pydatabufferkeyedefvalue}
1252 % Write a key and a single-line value to file simultaneously, or append them to the buffer.
1253 % \begin{macrocode}
1254 \begingroup
1255 \catcode`\:=12\relax
1256 \catcode`\,=12\relax
1257 \gdef\pydatawritekeyvalue{%
1258 \FVExtraReadVArg{\FVExtraDetokenizeVArg{\pydatawritekeyvalue@i}}}
1259 \gdef\pydatawritekeyvalue@i#1{%
1260 \FVExtraReadVArg{\FVExtraDetokenizeVArg{\pydatawritekeyvalue@ii{#1}}}}
1261 \gdef\pydatawritekeyvalue@ii#1#2{%
1262 \ifbool{pydata@indict}{}{\pydata@error{Cannot write a key unless in a dict}}%
1263 \ifbool{pydata@haskey}{\pydata@error{Cannot write a key when waiting for a value}}{}%
1264 \immediate\write\pydata@filehandle{%
1265 \pydata@quotestr{#1}: \pydata@quotestr{#2},%
1266 }}
1267 \gdef\pydatawritekeyedefvalue{%
1268 \FVExtraReadVArg{\FVExtraDetokenizeVArg{\pydatawritekeyedefvalue@i}}}
1269 \gdef\pydatawritekeyedefvalue@i#1#2{%
1270 \edef\pydata@tmp{#2}%
1271 \expandafter\pydatawritekeyedefvalue@ii\expandafter{\pydata@tmp}{#1}}
1272 \gdef\pydatawritekeyedefvalue@ii#1#2{%
1273 \FVExtraDetokenizeVArg{\pydatawritekeyvalue@ii{#2}}{#1}}
1274 \gdef\pydatabufferkeyvalue{%
1275 \FVExtraReadVArg{\FVExtraDetokenizeVArg{\pydatabufferkeyvalue@i}}}
1276 \gdef\pydatabufferkeyvalue@i#1{%
1277 \FVExtraReadVArg{\FVExtraDetokenizeVArg{\pydatabufferkeyvalue@ii{#1}}}}
1278 \gdef\pydatabufferkeyvalue@ii#1#2{%
1279 \ifbool{pydata@bufferhaskey}%
1280 {\pydata@error{Cannot buffer a key when waiting for a value}}{}%
1281 \expandafter\xdef\pydata@bufferlengthmacro{%
1282 \the\numexpr\pydata@bufferlengthmacro+1\relax}%
1283 \expandafter\xdef\csname\pydata@bufferlinename\pydata@bufferlengthmacro\endcsname{%
1284 \pydata@quotestr{#1}: \pydata@quotestr{#2},%
1285 }}
1286 \gdef\pydatabufferkeyedefvalue{%
1287 \FVExtraReadVArg{\FVExtraDetokenizeVArg{\pydatabufferkeyedefvalue@i}}}
1288 \gdef\pydatabufferkeyedefvalue@i#1#2{%
1289 \edef\pydata@tmp{#2}%
1290 \expandafter\pydatabufferkeyedefvalue@ii\expandafter{\pydata@tmp}{#1}}
1291 \gdef\pydatabufferkeyedefvalue@ii#1#2{%
1292 \FVExtraDetokenizeVArg{\pydatabufferkeyvalue@ii{#2}}{#1}}
1293 \endgroup
1294 % \end{macrocode}
1295 % \end{macro}
1296 %
1297 %
1298 % \begin{macro}{\pydatawritemlvalueopen, \pydatawritemlvalueline, \pydatawritemlvalueclose, \pydatabuffermlvalueopen, \pydatabuffermlvalueline, \pydatabuffermlvalueclose}
1299 % Write a line of a multi-line value to file or append it to the buffer. Write the end delimiter of the value to file or append it to the buffer.
1300 % \begin{macrocode}
1301 \begingroup
1302 \catcode`\,=12\relax
1303 \gdef\pydatawritemlvalueopen{%
1304 \ifbool{pydata@haskey}{}{\pydata@error{Cannot write value when waiting for a key}}%
1305 \immediate\write\pydata@filehandle{%
1306 \pydata@mlstropen
1307 }}
1308 \gdef\pydatawritemlvalueline#1{%
1309 \ifbool{pydata@haskey}{}{\pydata@error{Cannot write value when waiting for a key}}%
1310 \immediate\write\pydata@filehandle{%
1311 \pydata@escstrtext{#1}%
1312 }}
1313 \gdef\pydatawritemlvalueclose{%
1314 \ifbool{pydata@haskey}{}{\pydata@error{Cannot write value when waiting for a key}}%
1315 \immediate\write\pydata@filehandle{%
1316 \pydata@mlstrclose,%
1317 }%
1318 \boolfalse{pydata@haskey}}
1319 \gdef\pydatabuffermlvalueopen{%
1320 \ifbool{pydata@bufferhaskey}%
1321 {}{\pydata@error{Cannot buffer value when waiting for a key}}%
1322 \expandafter\xdef\pydata@bufferlengthmacro{%
1323 \the\numexpr\pydata@bufferlengthmacro+1\relax}%
1324 \expandafter\xdef\csname\pydata@bufferlinename\pydata@bufferlengthmacro\endcsname{%
1325 \pydata@mlstropen
1326 }}
1327 \gdef\pydatabuffermlvalueline#1{%
1328 \ifbool{pydata@bufferhaskey}%
1329 {}{\pydata@error{Cannot buffer value when waiting for a key}}%
1330 \expandafter\xdef\pydata@bufferlengthmacro{%
1331 \the\numexpr\pydata@bufferlengthmacro+1\relax}%
1332 \expandafter\xdef\csname\pydata@bufferlinename\pydata@bufferlengthmacro\endcsname{%
1333 \pydata@escstrtext{#1}%
1334 }}
1335 \gdef\pydatabuffermlvalueclose{%
1336 \ifbool{pydata@bufferhaskey}%
1337 {}{\pydata@error{Cannot buffer value when waiting for a key}}%
1338 \expandafter\xdef\pydata@bufferlengthmacro{%
1339 \the\numexpr\pydata@bufferlengthmacro+1\relax}%
1340 \expandafter\xdef\csname\pydata@bufferlinename\pydata@bufferlengthmacro\endcsname{%
1341 \pydata@mlstrclose,%
1342 }%
1343 \boolfalse{pydata@bufferhaskey}}
1344 \endgroup
1345 % \end{macrocode}
1346 % |*start| and |*end| variants for backward compatibility with versions before v0.5.0.
1347 % \begin{macrocode}
1348 \let\pydatawritemlvaluestart\pydatawritemlvalueopen
1349 \let\pydatawritemlvalueend\pydatawritemlvalueclose
1350 \let\pydatabuffermlvaluestart\pydatabuffermlvalueopen
1351 \let\pydatabuffermlvalueend\pydatabuffermlvalueclose
1352 % \end{macrocode}
1353 % \end{macro}
1354 %
1355 % \begin{macro}{pydatawritemlvalue}
1356 % \begin{macrocode}
1357 \newenvironment{pydatawritemlvalue}%
1358 {\VerbatimEnvironment
1359 \pydatawritemlvalueopen
1360 \begin{VerbatimWrite}[writer=\pydatawritemlvalueline]}%
1361 {\end{VerbatimWrite}}
1362 \AfterEndEnvironment{pydatawritemlvalue}{\pydatawritemlvalueclose}
1363 % \end{macrocode}
1364 % \end{macro}
1365 %
1366 % \begin{macro}{pydatabuffermlvalue}
1367 % \begin{macrocode}
1368 \newenvironment{pydatabuffermlvalue}%
1369 {\VerbatimEnvironment
1370 \pydatabuffermlvalueopen
1371 \begin{VerbatimBuffer}[bufferer=\pydatabuffermlvalueline]}%
1372 {\end{VerbatimBuffer}%
1373 \pydatabuffermlvalueclose}
1374 % \end{macrocode}
1375 % \end{macro}
1376 %
1377 %
1378 %
1379 % \iffalse
1380 %</package>
1381 % \fi
1382 %% \Finale
1383 \endinput