Debian Codegraph @

texmf-dist/doc/generic/enctex/encdoc-e.tex

raw
1 % EncTeX documentation
2 %%%%%%%%%%%%%%%%%%%%%%
3 % Petr Olsak
4
5 % see encdoce.tex for Czech version of this documentation
6
7
8 {\nopagenumbers} % use the plain format (the command: "tex encdoc-e")
9
10 \def\Red{}
11 \def\Black{}
12 \def\Blue{}
13 \def\Green{}
14 \def\beglink#1{}
15 \def\endlink{}
16 \def\aimlink#1{}
17 \def\urllink#1{{\tt#1}}
18 \def\urllinkk#1#2{{\tt#1}}
19
20 \ifx\pdfoutput\undefined\else %%%% pdfTeX is used %%%%%%%%%%
21 \ifnum\pdfoutput>0
22
23 \ifx\pdfannotlink\undefined % Verze pdfTeXu >= 14
24 \let\pdfannotlink=\pdfstartlink
25 \fi
26
27 \def\beglink#1{%
28 \Green \pdfstartlink height9pt depth3pt
29 attr{/Border[0 0 0]} goto name{#1}\relax}
30 \def\endlink{\pdfendlink\Black}
31
32 \def\aimlink#1{%
33 \expandafter\ifx \csname aim:#1\endcsname \relax
34 \expandafter\gdef \csname aim:#1\endcsname {}%
35 \vbox to0pt{\vss\hbox{\pdfdest name{#1} fith}\kern15pt}%
36 \fi
37 }
38 \def\urllinkk#1#2{\pdfannotlink height 10pt depth 3pt
39 user{/Border[0 0 0]/Subtype/Link/A << /Type/Action/S/URI/URI(#2)>>}\relax
40 \Green{\tt #1}\Black\pdfendlink}
41 \def\urllink#1{\urllinkk{#1}{#1}}
42
43 \def\pdfsetcmykcolor#1{\special{PDF:#1 k}}
44 \def\Red{\leavevmode\pdfsetcmykcolor{0.1 0.9 0.9 0}}
45 \def\Black{\leavevmode\pdfsetcmykcolor{0 0 0 1}}
46 \def\Green{\leavevmode\pdfsetcmykcolor{0.9 0.1 0.9 .3}}
47 \def\Blue{\leavevmode\pdfsetcmykcolor{0.9 0.9 0.1 0}}
48
49 \pdfcompresslevel=9
50 \pdfinfo{/Author (Petr Olsak)
51 /CreationDate (Feb. 2003)
52 /ModDate (Jun. 2004)
53 /Creator (TeX)
54 /Producer (pdfTeX)
55 /Title (encTeX)
56 /Subject (Documentation)
57 /Keywords (TeX, fonts)
58 }
59
60 \fi\fi %%%%%%%%%%%%%%%%%%%%%%%%%%%% End of pdfTeX macros %%%%%
61
62
63 \font\titulfont=\fontname\tenbf\space scaled\magstep4
64 \font\bigbf=\fontname\tenbf\space scaled\magstep1
65
66 \parindent=12pt
67
68 \newcount\secnum \newcount\subsecnum
69
70 \def\kap #1\par{\ifnum\secnum>0 \goodbreak\fi\removelastskip
71 \vskip2\baselineskip
72 \subsecnum=0 \advance\secnum by1
73 \noindent{\bigbf\llap{\the\secnum.\quad}#1}\par\nobreak\medskip}
74 \def\sec #1\par{\removelastskip\bigskip
75 \advance\subsecnum by1
76 \noindent{\bf \llap{\aimlink{\the\secnum.\the\subsecnum}%
77 \the\secnum.\the\subsecnum.\quad}#1}\par\nobreak\medskip}
78 \def\title #1\par{\vglue2\baselineskip
79 \centerline{\titulfont #1}\vskip2\baselineskip}
80 \def\date #1\par{\bigskip#1\par}
81 \def\author #1\par{\bigskip#1\par}
82
83 \catcode`<=13
84 \def<{\hbox\bgroup\catcode`_=\active\relax\skoba}
85 \def\skoba #1>{$\langle$\it#1\/$\rangle$\egroup}
86 \def\,{\thinspace}
87 {\catcode`_=\active \gdef_{\_}}
88
89 {\obeyspaces \gdef\activespace{\obeyspaces\let =\ }}
90 \def\setverb{\def\do##1{\catcode`##1=12}\dospecials}
91 \def\begtt{\medskip\bgroup \setverb \activespace
92 \catcode`\"=12 \catcode`_=13
93 \def\par##1{\endgraf\ifx##1\par\leavevmode\fi ##1}
94 \obeylines \startverb}
95 {\catcode`\|=0 \catcode`\\=12
96 |gdef|startverb#1\endtt{|tt#1|egroup|medskip|testpar}}
97 \long\def\testpar#1{\ifx\par#1\else\noindent\fi#1}
98
99 \catcode`"=13
100 \def"{\leavevmode\hbox\bgroup\setverb\activespace\tt\readverb}
101 \def\readverb #1"{#1\egroup}
102
103 \def\begitems{\medskip\bgroup \catcode`*=13 }
104 {\catcode`*=13 \gdef*{\item{$\bullet$}}}
105 \def\enditems{\medskip\egroup}
106
107
108 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
109
110 {\nopagenumbers
111
112 \vglue 4cm
113
114 \centerline {\titulfont Enc\TeX}
115 \vskip2cm
116 \centerline {\titulfont The Extension of \TeX{}}
117 \medskip
118 \centerline {\titulfont For Input Re-encoding}
119
120 \vskip3cm
121
122 \centerline{\bigbf Petr Ol\v s\'ak}
123
124 \vskip2cm
125
126 \centerline{\urllinkk{www.olsak.net/enctex.html}{http://www.olsak.net/enctex.html}}
127
128 \vfil
129
130 \centerline {\bigbf This text documents the version Feb. 2003 and Jun. 2004}
131
132 \vskip 4cm
133 \break
134
135 \null \vfil
136
137 This package is free software; you can redistribute it and/or modify
138 it under the terms of the GNU General Public License as published by
139 the Free Software Foundation; either version 2 of the License, or
140 (at your option) any later version.
141
142 This package is available on
143 \urllink{ftp://math.feld.cvut.cz/pub/olsak/enctex/}.
144
145 This program is distributed in the hope that it will be useful,
146 but WITHOUT ANY WARRANTY; without even the implied warranty of
147 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
148 GNU General Public License for more details.
149
150 You should have received a copy of the GNU General Public License
151 along with this program; if not, write to the Free Software
152 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
153
154 \bigskip
155 \copyright\quad 1997, 2002, 2003, 2004\enspace RNDr. Petr Ol\v s\'ak
156
157 \bigskip
158 \TeX{} is trademark of the American Mathematical Society.
159
160 \bigskip
161 The author of the \TeX{} is professor Donald Knuth. The \TeX{}
162 is a free software with the specific license. See the documentation
163 of~\TeX.
164
165 \bigskip
166 The original version of the enc\TeX{} documentation (in Czech language) is in
167 "encdoc.tex" file.
168
169 P\accent23uvodn\'\i{} \v cesk\'a dokumentace je v souboru "encdoc.tex".
170
171 \break
172 }
173
174 \kap The basic information
175 %%%%%%%%%%%%%%%%%%%%%
176
177
178 The enc\TeX{} package is a little extension of \TeX{}. You can install
179 it from source files of \TeX{} by changing the "tex.ch" file in your
180 distribution. The patch to "tex.ch" file for "web2c" distribution is
181 supported.
182
183 The enc\TeX{} is backward compatible with the original \TeX{}. It adds
184 ten new primitives by which you can set or read the conversion
185 tables used by input processor of \TeX{} or used during output to the
186 terminal, log and "\write" files. These tables are stored to the
187 format files thus, they are reinitialized to the same state as in time
188 of "\dump" command when the format file is read.
189
190 This extension is fully tested and it passes the TRIP test with only
191 two differences:
192
193 \begitems
194 * The banner is different
195 * The number of ``multiletter control sequences'' is
196 greater by ten.
197 \enditems
198
199
200 \sec The installation
201 % ----------------
202
203 For install instructions of enc\TeX{} -- read the "INSTALL.eng" file.
204
205 \sec Versions
206 % --------
207
208 I released the first version of enc\TeX{} in 1997. This version
209 was able to do the byte to byte conversion only by xord and xchr
210 vector and to assign the characters as ``printable'' (the "\xordcode",
211 "\xchrcode" and "\xprncode" primitives).
212
213 The TCX tables were renoved in 1998. These tables do the same work as
214 enc\TeX{} 1997 thus I closed my support of enc\TeX{}. The problem
215 with the missing support of UTF-8 encoding on input processor caused
216 the reinterpretation of my old decision: I implemented the UTF-8
217 support to the enc\TeX{} in December 2002 and I propagate enc\TeX{} again.
218
219 The new version
220 ``Feb.~2003'' adds seven new primitives: "\mubyte",
221 "\endmubyte", "\mubytein", "\mubyteout", "\mubytelog", "\specialout"
222 and "\noconvert".
223 They give a possibility to set the conversion from UTF-8 encoded files.
224
225 The version ``Jun.~2004'' fixes a little bugs only.
226 No new features are added.
227 See "enctex.patch-jun2004" file for more information
228 about these bugs.
229
230 Of course, the new version is backward compatible with the old one
231 from 1997. I don't plan any new big changes. If a little change will
232 be done then {\it the backward compatibility with the previous version
233 of enc\TeX{} is guaranteed by me}.
234
235 \sec Enc\TeX{} in web2c distribution, TCX tables
236 % -------------------------------------------
237
238 If your web2c distribution implements enc\TeX{} then you can
239 initialize it by the "-enc" option in command line.
240 You have to use this option during ini\TeX{} because enc\TeX{} stores
241 its primitives and its data to the format file. When the format is
242 used, the enc\TeX{} is initialized from format file automatically and
243 you need not use the "-enc" option again. If you are using a format
244 without enc\TeX{} initialized in it and you write "-enc" option then
245 the warning is printed and this option is ignored.
246
247 The TCX tables ("-translate-file" option) are working with the same
248 xord and xchr vector as enc\TeX{} in web2c distribution. This implies
249 the following little conflicts: If enc\TeX{} is used together with TCX
250 table then TCX table may re-write the initial values of "\xordcode",
251 "\xchrcode" and "\xprncode". These initial values are documented in
252 section~2.2. If these values are stored in format by enc\TeX{} and TCX
253 table is used together with such format then the values from format
254 can be re-written by TCX table too. On the other hand, you can use the
255 "\xordcode", "\xchrcode" and "\xprncode" primitives for reading or
256 saving of these values after TCX table initialization without
257 problems.
258
259 \sec The \TeX{} license
260 % --------------------
261
262 The enc\TeX{} adds the new primitives into \TeX{} so, we cannot call
263 the resulting program by name \TeX. On the other hand D.~Knuth assumed
264 that \TeX{} internals are filtered always from system dependences.
265 This was a reason why he implemented xord/xchr vectors in \TeX.
266 D.~Knuth assumed that the parameters of filter from system
267 dependences is set at source code level. Enc\TeX{} only moves this
268 setting from source code level to the runtime level. This is nothing
269 new: the \TeX{} memory parameters are possible to set at runtime in
270 modern \TeX{} distributions too. You can set the conversion tables
271 depend on your system. Then you can say "\let\xordcode=\undefined"
272 etc.~(the same for other enc\TeX{} primitives) and you can do "\dump".
273 The format has the conversion tables stored by the system
274 specifications and the user cannot do any more changes. The using of
275 this format acts the same as the using of the original \TeX{}.
276
277 I think that the second line on the terminal and log file is
278 sufficient information about the fact that the program is
279 a modified version of \TeX. I think that if the UTF-8 encoding will be
280 used more common then there is no another way than to modify the input
281 processor of \TeX{} otherwise the 8bit \TeX{} will dead in short time.
282
283 It is important to say that enc\TeX{} has the same default behavior as
284 the original \TeX{} if the new primitives are never used.
285
286 IMHO, the new web2c \TeX{} is not exactly the \TeX{} too because you can
287 change its behavior by writing "%&" at the first line of the document.
288 This feature is not documented in {\it Computers \& Typesetting\/}
289 series.
290
291
292 \kap The byte per byte conversion
293 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
294
295 \sec The xord and xchr vectors
296 % -------------------------
297
298 All text inputs into \TeX{} are mapped by xord vector in input
299 preprocessor (the eyes in \TeX{}book terminology). If the character
300 has the code $x$ in your system, the same character has the code
301 $y$ = xord[x] in \TeX.
302
303 All text outputs from \TeX{} to terminal, log file and files managed
304 by "\write" primitive are filtered by xchr vector and by
305 ``printability'' feature of the character. If the character with code
306 $y$ is not ``printable'', then it outputs by "^^code" notation
307 (documented in \TeX{}book, page 45). If the character with code
308 $y$ is ``printable'' then the output code of this character on
309 terminal and text files is $z$ = xchr[$y$].
310
311 \sec The new primitives with the access to the xord and xchr vectors
312 % --------------------------------------------------------------
313
314 The enc\TeX{} extension introduces three new primitives with the same
315 syntax as "\lccode":
316
317 \begitems
318 * "\xordcode" $i$ ... is xord[$i$]
319 * "\xchrcode" $i$ ... is xchr[$i$]
320 * the character with the code $i$ is ``printable''
321 (not "^notation" on terminal and the log is used)
322 iff \hfil\break
323 \hbox{("\xprncode" $i > 0$ )} or \hbox{( $i\in \{32,...,126\}$ )}.
324 \enditems
325
326 All setting to "\xordcode", "\xchrcode" and "\xprncode" are possible
327 in 0...255 range and are {\it global} every time. It means
328 that the setting inside group are global and it is irrelevant
329 if you write "\global" prefix or you do not.
330
331 The initial values at ini\TeX{} state of the mentioned vectors are:
332
333 \begitems
334 * "\xordcode" $i = i $ for $ i \in \{128...255\}$,
335 * "\xchrcode" $i = i $ for $ i \in \{128...255\}$,
336 * "\xprncode" $i = 0 $ for $ i \in \{0...31, 127...255\}$,
337 * "\xprncode" $i = 1 $ for $ i \in \{32...126\}$.
338 \enditems
339
340 The "\xordcode" $i$ and "\xchrcode" $i$ for $i \in \{0...127\}$ are
341 system dependent, but on systems with ASCII encoding holds:
342 "\xordcode"~$i = i$, "\xchrcode"~$i = i$.
343
344
345 \kap The multi-byte conversion
346 %%%%%%%%%%%%%%%%%%%%%%%%
347
348 Since version Dec 2002, the enc\TeX{} is able to convert more bytes to
349 one byte or control sequence on input processor level. This ``one byte''
350 is converted back to the original ``more bytes'' when "\write" is
351 processed or \TeX{} outputs to the terminal or log file.
352 The main reason of this extension of \TeX{} is to serve to work with the
353 UTF-8 encoded input files: we need to assign the "\catcode"s,
354 "\uccode"s etc.~to the letters in our alphabet but some letters are
355 encoded in two bytes in UTF-8. The enc\TeX{} is able to map other codes
356 from UTF-8 to control sequences thus, the number of UTF-8 codes
357 from input file examined by \TeX{} is unlimited.
358
359 There are five new primitives to manage the conversion:
360 "\mubytein", "\mubyteout", "\mubytelog", "\mubyte", "\endmubyte".
361 The "\mubytein", "\mubyteout" and "\mubytelog" are integer registers with
362 zero value by default: it means that no conversion is processed even
363 if the conversion table (created by "\mubyte", "\endmubyte") is non
364 empty. If "\mubytein" is positive then the conversion on input processor
365 level is performed by the conversion table. If "\mubyteout"
366 is positive then the conversion for output to the "\write" files
367 is activated by the same conversion table. If "\mubytelog" is positive
368 then the output conversion is activated for log file and terminal
369 output.
370
371 The conversion table is empty by default and you can add the
372 new line into this table by the couple of "\mubyte", "\endmubyte"
373 primitives:
374
375 \begtt
376 \mubyte <first_token><one_optional_space><optional_prefix><byte_sequence>\endmubyte
377 \endtt
378
379 Each <byte_sequence> will be converted to the <first_token> at input
380 processor level. There are two possibilities for <first_token>: it may
381 be a character or a control sequence. If the <first_token> is a
382 character then the catcode of it is ignored and the <first_token> is
383 interpreted as a <byte>. This <byte> is converted back to the
384 <byte_sequence> in "\write" files, log file and terminal.
385
386 If the <first_token> is a control sequence then the <byte_sequence>
387 will be converted to this control sequence of the ``one token''
388 form at input processor level. It means that the
389 token processor never changes this control sequence.
390 The token processor stays in
391 middle line state after this control sequence is scanned.
392 If {\catcode`\<=12 "\mubyteout<2"} then the output to the "\write"
393 files is not converted back to the <byte_sequence> and
394 the control sequence is expanded as usual.
395 If "\mubyteout>=2" then the control sequence declared by "\mubyte" is
396 converted back to the <byte_sequence> in "\write" parameters.
397 This works only if the control sequence is not expanded.
398 It means that the control sequence have to be non expandable or
399 it have to be marked by "\noexpand". If "\mubyteout>=3" then
400 enc\TeX{} suppresses the expansion of control
401 sequences declared by "\mubyte" automatically. See section~3.7 for
402 more details.
403
404 The control sequences are never converted back to <byte_sequence>
405 in log file and on the terminal output.
406
407 The syntax and the meaning of <optional_prefix> will be explained in
408 section~3.4.
409
410 \sec The conversion table manipulation
411 % ---------------------------------
412
413 The data are stored into conversion table as a global assignment. On
414 the other hand the assignment to "\mubytein", "\mubyteout" and
415 "\mubytelog" registers are local as usually.
416
417 The "\mubyte", "\endmubyte" primitives work very similar as a
418 well known "\csname", "\endcsname" pair. The difference is that
419 the <first_token> is not expanded and that this token can be followed
420 by <one_optional_space> (after expansion). The <byte_sequence> is
421 scanned with the full expansion. If the other non expandable control
422 sequence than "\endmubyte" occurs during this process then the error
423 message is printed:
424
425 \begtt
426 ! Missing \endmubyte inserted.
427 \begtt
428
429 The "\mubyte" is not performed on the expand processor level: it is a
430 assign primitive. If you write
431
432 \begtt
433 \edef\a{\mubyte X ABC\endmubyte}
434 \endtt
435 %
436 then the macro "\a" includes the "\mubyte X ABC\endmubyte" tokens.
437
438 Examples:
439
440 \begtt
441 \mubyte ^^c1 ^^c3^^81\endmubyte % \'A
442 \mubyte ^^e1 ^^c3^^a1\endmubyte % \'a
443 % etc. -- the UTF8 implementation
444
445 \mubyte \endash ^^c4^^f6\endmubyte % the mapping to the control sequence
446 \mubyte \integral INT\endmubyte % the illustrative example, see below
447
448 \mubytein=1 \mubyteout=1 % conversions are activated here
449
450 \def\endash {--} % this is good definition for \write files too
451 \def\integral {\ifmmode \int\else $\int$\fi}
452 \endtt
453
454 We have written more spaces (or tabs) in <one optional space> in
455 this example because these characters have the catcode of the space
456 and the token processor converts them to right <one optional space>.
457
458 The word ``"INTEGRAL"'' is converted to the token "\integral" followed
459 by the letters ``"EGRAL"'' if the example code is used.
460 The text ``"INT something"'' is converted to the token "\integral"
461 followed by space and the word ``"something"''. You can write the
462 following constructions: "\defINT{something}", "\let INT=\foo", etc.
463 After "\show INT" we get:
464
465 \begtt
466 > \integral=macro:
467 ->\ifmmode \int \else $\int $\fi .
468 l.18 \show INT
469 \endtt
470 %
471 and "\string INT" expands to the text: "\integral".
472
473 Assume the "INT" declaration from the previous example and assume that
474 you write "\INT". What happens? Strictly speaking, the empty control
475 sequence ("\csname\endcsname") followed by "\integral" control
476 sequence would be the output from the token processor. But there is an
477 exception in enc\TeX{} because to avoid the confusion with the empty control
478 sequences. The "\INT" produces only the control sequence "\integral",
479 the backslash is ignored in this situation. The token processor stays
480 in middle state after "\INT" is scanned, the letter can follow
481 immediately.
482
483 \sec The features of the conversion process
484 % --------------------------------------
485
486 The input is converted immediately after "\mubytein" is set to the
487 positive value; it means the conversion may start at the same line
488 where the "\mubytein" setting occurs.
489
490 The <byte_sequence> is converted only if the whole <byte_sequence> is
491 included in the one line. The "\endlinechar" character can be the last
492 part of the <byte_sequence>.
493
494 The sequence "^^c3^^81" is not converted to the letter \'A even if the
495 code from the example was used. The reason is that the "^^" conversion
496 is done in token processor after the "\mubyte" conversion.
497
498 The "\xordcode" conversion is performed before "\mubyte" conversion
499 in input side and the "\xchrcode" conversion is done after "\mubyte"
500 conversion during output to the files or to the terminal. The
501 following diagram shows the sequence of the conversions:
502
503 \begtt
504 input text -> \xordcode -> \endlinechar appended ->
505 \mubyte -> token processor -> expand processor ...
506 \write argument -> expand processor -> \mubyte -> \xchrcode -> output
507 \endtt
508
509 The <byte_sequence> is not converted to the "^^" form during
510 output to the file even if the "\xprncode" of the bytes from
511 <byte_sequence> is zero. The <byte_sequence> is not converted again
512 even if there exist a character in it which is normally
513 converted by another rule in conversion table.
514
515 Let exist two or more <byte_sequences> in the conversion table which are
516 equal or which have the same begin part and one sequence is a
517 subsequence of the second. Then the conversion in input processor is
518 done by maximal possible <byte_sequence>. This feature was implemented
519 in version Feb.~2003. Example:
520
521 \begtt
522 \mubyte X A\endmubyte
523 \mubyte Y ABC\endmubyte
524 \mubyte \foo ABCD\endmubyte
525 \endtt
526
527 The letter A is normally converted to X in this example, but if the
528 BC letters immediately follow then ABC is converted to Y with the
529 exception ABCD which is converted to "\foo".
530
531 The order of "\mubyte" settings in this example has no significance.
532
533 If the same <byte_sequences> are used by "\mubyte" records then the
534 last one has a precedence and the previous records are cleared.
535
536
537 \sec The conversion to log file and to the terminal
538 % ----------------------------------------------
539
540 The output to terminal and to log file is not converted if
541 "\mubytelog" is zero. If the "\xprncode" of the character is zero then
542 the character is printed in "^^A" or "^^bc" form. If the "\mubytelog"
543 is positive then the characters stored in conversion table are
544 converted to the <byte_sequence> and the bytes from these
545 <byte_sequence> are never converted to "^^" form.
546 On the other hand, control sequences are keept unchanged
547 in log and in terminal even if the "\mubytelog" is positive.
548
549 The conversion is switched on or off by "\mubytelog" value for both:
550 terminal and log file. You cannot separate these outputs. It means that
551 (for example) the conversion to log and no conversion to terminal
552 is not possible.
553
554 There exists a special part of terminal and log output: if the complete
555 line from input is reprinted (for example when the context of an error is
556 shown). We call this situation as ``line-reprinting'' for the purpose
557 of the following text.
558
559 If the "\mubytein" is zero then line-reprinting works as usual in
560 standard \TeX{}. If "\mubytein" is positive and "\mubytelog" is zero
561 then line-reprinting includes the output from the input processor of
562 enc\TeX{}. It means that control sequences generated by input
563 processor can be shown here even if they don't actually present in the
564 input line. If both "\mubytein" and "\mubytlog" are positive then
565 line-reprinting works without any multi-byte conversion, only xord is
566 used immediately followed by xchr. No "^^A" form is used in this
567 situation. Note that the error messages can be somewhat peculiar when
568 "\mubytein" and "\mubytlog" are positive:
569
570 \begtt
571 \mubyte \cmd ABC\endmubyte \let\cmd=\undefined
572 \mubytein=1 \mubytelog=1
573 This is test of ABC and another text.
574 \endtt
575 %
576 We get the following message:
577
578 \begtt
579 ! Undefined control sequence.
580 l.3 This is test of ABC
581 and another text.
582 ?
583 \endtt
584 %
585 The "\show ABC" can say you more information:
586
587 \begtt
588 > \cmd=undefined.
589 l.3 \show ABC
590 \endtt
591
592
593 \sec Clearing records from the conversion table
594 % ------------------------------------------
595
596 There exists only the chance to clear all records which begin with
597 the same first byte of <byte_sequence>. This is done by the command
598 "\mubyte <char> <char>\endmubyte". For example
599
600 \begtt
601 \mubyte A A\endmubyte
602 \endtt
603 %
604 clears all <byte_sequences> from conversion table which begin with the
605 character A. The following code clears all conversion table:
606
607 \catcode`<=12
608 \begtt
609 {\catcode`\^^@=12
610 \gdef\clearmubytes{\bgroup \count255=1
611 \loop \uccode`X=\count255
612 \uppercase{\mubyte XX\endmubyte}%
613 \advance\count255 by1
614 \ifnum\count255<256 \repeat
615 \mubyte ^^@^^@\endmubyte
616 \egroup}
617 }
618 \clearmubytes
619 \endtt
620 \catcode`<=13
621
622
623
624 \sec Input and output sides of the conversion table
625 % ----------------------------------------------
626
627 The conversion table consists from two independent parts: input side
628 used by input processor and output side used during "\write" or
629 printing to the log and terminal. You can save the record only to one
630 of this parts by using the nonempty <optional_prefix>.
631 If the <optional_prefix> is empty then the same record is stored
632 twice: into input and output sides. If <optional_prefix> is a token of
633 catcode~8 (usually the "_" character) then the record is stored only
634 into input side . If <optional_prefix> is a pair of tokens catcode~8
635 (usually "__") then the record is stored only into output side.
636
637 If the optional prefix has a form of "__" then the following
638 <byte_sequence> can be empty. Enc\TeX{} removes the record
639 corresponding to the <first_token> from output side in such situation.
640
641 The macro code for clearing the conversion table from previous section
642 clears all records from input side but only the records concerned to
643 the <first_token> in ``one byte form'' from output side. You can
644 remove the record concerned to control sequence from output side
645 only by "\mubyte \foo __\endmubyte".
646
647 \sec Inserted control sequences
648 % --------------------------
649
650 If the <first_token> is the control sequence and the <optional_prefix>
651 is one token of catcode~6 (usually the "#" character)
652 followed by <number>
653 then the <number> bytes are kept by input
654 processor (it means they are no converted again)
655 and the declared control sequence is inserted before
656 <byte_sequence>. The example:
657
658 \begtt
659 \def\abc{ABC}
660 \mubyte X BC\endmubyte \mubytein=1
661 \mubyte \foo #3 \abc\endmubyte ABC is converted to \foo ABC
662 \mubyte \foo #1 \abc\endmubyte ABC is converted to \foo AX
663 \endtt
664
665 The <number> has the same syntax as <number> from \TeX{}book.
666 It means that ``one optional space'' can work as a separator of
667 digit(s). See the previous example.
668
669 If <number> is zero then the control sequence is inserted and the
670 whole <byte_sequence> is unchanged. This has the same effect as if the
671 <number> equals to the length of the <byte_sequence>.
672
673 The <number> is accepted only in the range 0 to 50. The negative
674 <numbers> are silently interpreted as zero and the numbers greater
675 than 50 mean that the rest of the converted line will be unchanged
676 by input processor.
677
678 More practical example follows. Note, that the <number> is greater
679 than the length of the <byte_sequence> here.
680
681 \begtt
682 \mubyte \warntwobytes #2^^c3\endmubyte
683 \mubyte \warntwobytes #2^^c4\endmubyte
684 \mubyte \warntwobytes #2^^c5\endmubyte
685 % atd...
686 \def\warntwobytes #1#2{\message{WARNING: the UTF8 code:
687 \noconvert#1\noconvert#2 is not defined i my macros.}}
688 \endtt
689
690 The new primitive "\noconvert" is used in this example
691 (see chapter~5). The similar code is used in the file
692 "utf8unkn.tex".
693
694
695 \sec The virtual mark of line begin
696 % ------------------------------
697
698 If "\mubytein>0" and if the first byte in <byte_sequence> is equal to
699 "\endlinechar" (it means <byte_sequence> has a format
700 <endlinechar><rest>)
701 then input processor checks the matching of the
702 <rest> with the begin of every line. If it matches then the given
703 conversion is done. The example:
704
705 \begtt
706 \bgroup \uccode`X=\endlinechar \uppercase{\gdef\echar{X}}\egroup
707 \mubyte \fooB \echar ABC\endmubyte % ABC matches at begin of line
708 \mubyte \fooE ABC\echar \endmubyte % ABC matches at end of line
709 \mubyte \fooW \spce\space ABC\space \endmubyte
710 % ABC matches as a word with spaces before and after
711 \mubyte \foo #\echar XYZ\endmubyte %
712 % if XYZ is at begin of line the \foo is inserted before them
713 \endtt
714
715 \sec The suppression of the expansion in write parameters
716 % ----------------------------------------------------
717
718 If you need to convert the control sequences back to its
719 <byte_sequences> then the expansion of such control sequences is not
720 welcome. You can suppress the expansion by "\let\macro=\relax" before
721 "\write" starts the expansion of its parameter. But "\write" works
722 asynchronously in most situations and you can manipulate with hundreds
723 or thousands control sequences declared as UTF-8 codes. The enc\TeX{}
724 serves a simple tool to solve this problem: If "\mubyteout>=3" then
725 enc\TeX{} gives the "\relax" meaning to each control sequence declared
726 in output side of the conversion table before the "\write" starts its
727 expansion and it returns back these control sequences to their original
728 meaning immediately after "\write" finish its work.
729 Example:
730
731 \begtt
732 \mubyte \foo ABC\endmubyte \def\foo{macro body}
733 \mubyteout=2
734 \immediate\write16{testwrite: \foo} % prints "testwrite: macro body"
735 \immediate\write16{testwrite: \noexpand\foo} % prints "testwrite: ABC"
736 \mubyteout=3
737 \immediate\write16{testwrite: \foo} % prints "testwrite: ABC"
738 \message{testmessage: \foo} % prints "testmessage: macro body"
739 \message{testmessage: \noexpand\foo} % prints "testmessage: \foo"
740 \edef\a{testedef: \foo} % expands to macro body
741 \foo % expands to macro body
742 \immediate\write16{\meaning\foo} % prints "\relax"
743 \message{\meaning\foo} % prints "macro:->macro body"
744 \endtt
745
746 Note the difference between "\message" and "\immediate\write16".
747 The control sequences in "\message" parameter are always expanded and
748 never converted to the <byte_sequence>.
749
750 You can set the ``noexpand'' flag (for "\write" parameters only)
751 to any <control_sequence> and you need not declare the <byte_sequence>
752 for it. Write "\mubyte <control_sequence> \relax \endmubyte" for this
753 purposes. This has the same effect as
754 "\mubyte" <control_sequence>" __\string" <control_sequence>"\space\endmubyte",
755 but this second solution is more memory consuming because \TeX{} has
756 to store the <byte_sequence> as a string to the pool.
757
758 You can write your own macros which expand to one code in normal
759 situation and to different code in write parameters. The declaration
760 of "\writeparameter" control sequence is recommended:
761
762 \begtt
763 \mubyte \writeparameter \relax \endmubyte \def\writeparameter{}
764 \def\mymacro{\ifx\writeparameter\relax THIS CODE IS USED IN WRITE.
765 \else THIS CODE IS USED IN NORMAL EXPANSION.\fi}
766 \endtt
767
768
769
770 \sec The asynchronous write command and the mubyteout value
771 % ------------------------------------------------------
772
773 If you don't use "\immediate" then the "\write" command first gets its
774 parameter but it expands and prints this parameter at another
775 time. The "\write" command stores the actual value of the "\mubyteout"
776 register when it gets its parameter. This value is used late when
777 parameter is expanded and written to the file.
778
779 This feature gives the possibility to write to more files, first (for
780 table of contents, for example) is written with conversion to UTF-8
781 and another files are written without this conversion, because (for
782 example) this file is an input for a program which cannot read the
783 UTF-8 encoding. You can try:
784
785 \begtt
786 \newwrite\tocfile \newwrite\indexfile
787 \immediate\openout\tocfile=\jobname.toc
788 \immediate\openout\indexfile=\jobname.idx
789 \mubyteout=3
790 \write\tocfile{this parameter will be converted to UTF-8}
791 {\mubyteout=0 \write\indexfile{this parameter stay unchanged}}
792 \write\tocfile{this parameter will be converted to UTF-8}
793 \end % now, all three writes are actually done
794 \endtt
795
796 \sec Summary of the mubyteout values
797 % -------------------------------
798
799 Apart from the values 0, 1, 2 and 3, you can set the "\mubyteout"
800 register to the value $-1$ or $-2$. The summary table of meanings of
801 these values follows:
802
803 \vbox{
804 \begtt
805 \mubyteout <byte>-><byte_sequence> <cs_name>-><byte_sequence> noexpanding
806 ----------------------------------------------------------------------------
807 0 off off off
808 1 on off off
809 2 on on off
810 3 on on on
811 -1 on off on
812 -2 off off on
813 \endtt
814 \par}
815
816 If the <byte>"->"<byte_sequence> conversion is on then all texts
817 written to the "\write" files, log file and to the terminal are converted.
818 On the other hand, the <cs_name>"->"<byte_sequence> conversion and the
819 noexpanding are related only to the "\write" arguments (and the
820 "\special" arguments, see the following chapter).
821
822
823
824 \kap The arguments of the special primitive
825 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
826
827 The plain texts in non-english languages can occur in
828 "\special" arguments. The PDF-outlines are a good
829 example of this situation. May be,
830 you need to save these arguments in UTF-8 encoding.
831 The enc\TeX{} gives the possibility to do it.
832
833 The argument of "\special" is processed by the value of the integer
834 primitive register "\specialout". This register is introduced by
835 enc\TeX{} and its default value is zero.
836
837 \begitems
838 * "\specialout=0" -- no conversion, the same as in the original \TeX.
839 * "\specialout=1" -- only the xchr conversion.
840 * "\specialout=2" -- only the "\mubyteout" conversion.
841 * "\specialout=3" -- the "\mubyteout" conversion followed by
842 the xchr conversion.
843 \enditems
844
845 The "\special" primitive expands its argument immediately. If
846 "\specialout" is 2 or 3 then the expansion is done by "\mubyteout"
847 value in the same manner as during the "\write" expansion.
848 Moreover, "\special" saves the current values of "\specialout" and
849 "\mubyteout" registers to its memory and use them at the time of the
850 output to the "dvi" file.
851
852
853 \kap The noconvert primitive
854 %%%%%%%%%%%%%%%%%%%%%%%
855
856 The "\noconvert" primitive is introduced by enc\TeX. This primitive
857 suppress the possible conversion of the following character or the
858 control sequence. More exactly: the "\noconvert" is non expandable
859 primitive and does nothing in typesetting output
860 (the same as "\relax"). If this primitive is used in "\message" or
861 "\errmessage" argument then the control sequence of this primitive is
862 never printed and the following character is not converted to
863 <byte_sequence> even if the "\mubytelog" is positive and the character
864 is recorded in output side of conversion table.
865
866 The primitive "\noconvert" does the same in "\write" and "\special"
867 parameters. Moreover, if a control sequence follows then this
868 control sequence is normally printed even if the "\mubyteout" is positive
869 and this control sequence is recorded in output side of conversion
870 table.
871
872 The "\noconvert\noconvert" yields to one "\noconvert" in the output.
873
874 The "\noconvert" primitive is normally printed to the log and to the
875 terminal in the another situation than "\message" and "\errmessage"
876 parameters. For example when "\tracing..." is used.
877
878
879 \kap Summary of the enc\TeX{}'s primitives
880 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
881
882 \begitems
883 * "\mubyte" --- new record to the conversion table, see chapter~3.
884 * "\endmubyte" --- the "\mubyte" separator.
885 * "\mubytein" --- integer register. 0: multi-byte input conversion
886 suppressed, 1 and more: multi-byte input conversion
887 activated.
888 * "\mubyteout" --- integer register, the level of the output conversion
889 in "\write" and "\special" parameters, see~3.10.
890 * "\mubytelog" --- integer register, 0: multi-byte output conversion
891 to log and terminal suppressed, 1 and more:
892 multi-byte output conversion activated.
893 * "\specialout" --- integer register, the mode of the "\special"
894 parameter conversion, see chapter~4.
895 * "\noconvert" --- similar as "\noexpand", but for conversion process.
896 See~chapter~5.
897 * "\xordcode" --- access to xord vector, see 2.1.
898 * "\xchrcode" --- access to xchr vector, see 2.1.
899 * "\xprncode" --- access to ``printability'' vector, see 2.2.
900 \enditems
901
902 The summary of <optional_prefixes> in "\mubyte" primitive follows. The
903 character ``"#"'' means the token of category~6 and ``"_"'' means the token of
904 category~7 here.
905
906 \begitems
907 * no prefix --- records to the input and output side of the conversion
908 table.
909 * "_" --- records only to the input side of the conversion table.
910 * "__" --- records only to the output side of the conversion table.
911 * "#<number>" --- records to the input side of the conversion table,
912 the control sequence will be inserted and <number>
913 bytes will be kept without conversion again.
914 * "\relax" --- the control sequence will be not expand in "\write"
915 parameters.
916 \enditems
917
918 More prefixes may be implemented in next versions of enc\TeX. The
919 prefix has its first character different from catcode 10, 11 or 12.
920 This rule will be kept in next versions thus it is sufficient to use
921 the first character of <byte_sequence> with this category if no prefix
922 is needed.
923
924
925
926 \kap The macro files
927 %%%%%%%%%%%%%%%
928
929 The enc\TeX{} package includes some encoding tables inputted by
930 "\input" during format generation. These tables support encodings
931 widely used in Czech texts. The more information about these macro
932 files are in comments of these files and in the Czech version of the
933 documentation.
934
935
936
937 \end
938
939
940
941
942
943
944
945
946
947