| 1 |
% \iffalse meta-comment |
| 2 |
% |
| 3 |
%% Copyright (C) 2020--2021 by Marcel Krueger |
| 4 |
%% |
| 5 |
%% This file may be distributed and/or modified under the |
| 6 |
%% conditions of the LaTeX Project Public License, either |
| 7 |
%% version 1.3 of this license or (at your option) any later |
| 8 |
%% version. The latest version of this license is in: |
| 9 |
%% |
| 10 |
%% http://www.latex-project.org/lppl.txt |
| 11 |
%% |
| 12 |
%% and version 1.3 or later is part of all distributions of |
| 13 |
%% LaTeX version 2005/12/01 or later. |
| 14 |
% |
| 15 |
%<*batch> |
| 16 |
%<*gobble> |
| 17 |
\ifx\jobname\relax\let\documentclass\undefined\fi |
| 18 |
\ifx\documentclass\undefined |
| 19 |
\csname fi\endcsname |
| 20 |
%</gobble> |
| 21 |
\input docstrip.tex |
| 22 |
\keepsilent |
| 23 |
\generate{% |
| 24 |
\file{inputnormalization.sty}{\from{inputnormalization.dtx}{package}} |
| 25 |
\file{inputnormalization.tex}{\from{inputnormalization.dtx}{tex-package}}} |
| 26 |
\endbatchfile |
| 27 |
%</batch> |
| 28 |
%<*gobble> |
| 29 |
\fi |
| 30 |
\expandafter\ifx\csname @currname\endcsname\empty |
| 31 |
\csname fi\endcsname |
| 32 |
%</gobble> |
| 33 |
%<*driver> |
| 34 |
\RequirePackage{inputnormalization} |
| 35 |
\documentclass{article} |
| 36 |
\usepackage{csquotes,doc,metalogo,hyperref,luacolor,tikzducks,pict2e} |
| 37 |
\RecordChanges |
| 38 |
\MakeShortVerb\| |
| 39 |
\begin{document} |
| 40 |
\DocInput{inputnormalization.dtx} |
| 41 |
\PrintIndex |
| 42 |
\PrintChanges |
| 43 |
\end{document} |
| 44 |
%</driver> |
| 45 |
%<*gobble> |
| 46 |
\fi |
| 47 |
%</gobble> |
| 48 |
% \fi |
| 49 |
% |
| 50 |
% \GetFileInfo{inputnormalization.sty} |
| 51 |
% \title{The inputnormalization package\thanks{This document |
| 52 |
% corresponds to inputnormalization~\fileversion, dated~\filedate.}} |
| 53 |
% \author{Marcel Kr\"uger \\ |
| 54 |
% \href{mailto:[email protected]}{[email protected]}} |
| 55 |
% |
| 56 |
% \maketitle |
| 57 |
% Add support for normalising input files for LuaTeX and provide a common interface for \LuaTeX\ and \XeTeX. |
| 58 |
% |
| 59 |
% \section{Motivation} |
| 60 |
% Modern \TeX\ engines like \XeTeX\ or \LuaTeX\ natively accept Unicode input. |
| 61 |
% Unicode is a rather special encoding since many characters can be encoded in |
| 62 |
% different ways which are officially considered equivalent. This can sometimes |
| 63 |
% lead to surprising behavior since many parts of \TeX\ are not aware of this |
| 64 |
% equivalences and therefore treats different encodings as different strings. |
| 65 |
% This can show itself during rendering when some the same text might appear |
| 66 |
% in different ways depending on the input, but it might also show itself in |
| 67 |
% macro or option names: When e.g.\ non-English macro names are in use, the |
| 68 |
% different encodings of the same name can name different macros, leading to |
| 69 |
% hard to understand and solve errors. |
| 70 |
% |
| 71 |
% Unicode defines a mechanism to solve such issues: It defines the normalization |
| 72 |
% forms NFC and NFD. When text is normalized to one of these forms, then two |
| 73 |
% equivalent strings are always encoded in the same way, leading to unique names |
| 74 |
% and consistent rendering. |
| 75 |
% |
| 76 |
% This package provides a uniform way to enable input normalization to either |
| 77 |
% of these forms in both \XeTeX\ and \LuaTeX. |
| 78 |
% |
| 79 |
% \section{Usage} |
| 80 |
% Just loading the package is enough to enable NFC normalization. This is the |
| 81 |
% right option for almost all users: |
| 82 |
% |
| 83 |
% \begin{verbatim} |
| 84 |
% \documentclass{article} |
| 85 |
% \usepackage{inputnormalization} |
| 86 |
% \begin{document} |
| 87 |
% Everything here gets normalized before it's processed by \TeX. |
| 88 |
% \end{document} |
| 89 |
% \end{verbatim} |
| 90 |
% |
| 91 |
% If you are a plain \LuaTeX/\XeTeX\ user, you can use |
| 92 |
% |
| 93 |
% \begin{verbatim} |
| 94 |
% \input inputnormalization |
| 95 |
% Everything here gets normalized before it's processed by \TeX. |
| 96 |
% \bye |
| 97 |
% \end{verbatim} |
| 98 |
% instead. |
| 99 |
% |
| 100 |
% \section{Advanced usage} |
| 101 |
% In addition to enabling NFC normalization by default, the package makes |
| 102 |
% \verb|\Uinputnormalization| available as a cross engine version of |
| 103 |
% \verb|\XeTeXinputnormalization| to make the normalization controllable. |
| 104 |
% See the \XeTeX\ documentation for detailed usage. E.g.\ you could write |
| 105 |
% \begin{verbatim} |
| 106 |
% \documentclass{article} |
| 107 |
% \usepackage{inputnormalization} |
| 108 |
% \begin{document} |
| 109 |
% Everything here gets normalized to NFC before it's processed by \TeX. |
| 110 |
% |
| 111 |
% \Uinputnormalization=0 |
| 112 |
% Now normalization is disabled. |
| 113 |
% |
| 114 |
% \Uinputnormalization=2 |
| 115 |
% Here we normalize to NFD instead. |
| 116 |
% \end{document} |
| 117 |
% \end{verbatim} |
| 118 |
% |
| 119 |
% \paragraph{Warning:} It is almost never a good idea to use different kinds |
| 120 |
% of normalization in the same document, therefore you should set one kind of |
| 121 |
% normalization directly after loading the package and not modify it afterwards. |
| 122 |
% |
| 123 |
% Additionally NFC works much better in a \TeX\ context than NFD, so you should |
| 124 |
% not set this at all unless you know exactly what you are doing. |
| 125 |
% |
| 126 |
% \StopEventually{} |
| 127 |
% \section{The implementation} |
| 128 |
% \changes{0.0.1}{2020-05-17}{Initial release} |
| 129 |
% \iffalse |
| 130 |
%<*package|tex-package> |
| 131 |
% \fi |
| 132 |
% \begin{macrocode} |
| 133 |
%<*package> |
| 134 |
\NeedsTeXFormat{LaTeX2e} |
| 135 |
\ProvidesPackage |
| 136 |
{inputnormalization} |
| 137 |
[2021/07/05 v0.2 Unicode input normalization] |
| 138 |
%</package> |
| 139 |
% \end{macrocode} |
| 140 |
% Only \LuaTeX\ and \XeTeX\ are supported. |
| 141 |
% For other engines we show an error. |
| 142 |
% \begin{macrocode} |
| 143 |
\ifx\directlua\undefined |
| 144 |
\ifx\XeTeXinputnormalization\undefined |
| 145 |
% \end{macrocode} |
| 146 |
% \iffalse |
| 147 |
%<*gobble> |
| 148 |
\iffalse |
| 149 |
%</gobble> |
| 150 |
% \fi \fi |
| 151 |
% \begin{macrocode} |
| 152 |
%<*tex-package> |
| 153 |
\begingroup |
| 154 |
\ifx\PackageError\undefined |
| 155 |
\def\PackageError#1#2#3{\errhelp{#3}\errmessage{#1: #2}} |
| 156 |
\fi |
| 157 |
%</tex-package> |
| 158 |
% \end{macrocode} |
| 159 |
% \iffalse \iffalse |
| 160 |
%<*gobble> |
| 161 |
\fi |
| 162 |
%</gobble> |
| 163 |
% \fi |
| 164 |
% \begin{macrocode} |
| 165 |
\PackageError{inputnormalization}{LuaTeX or XeTeX required}% |
| 166 |
{inputnormalization requires LuaTeX or XeTeX. |
| 167 |
Maybe you forgot to switch the engine in your editor?} |
| 168 |
% \end{macrocode} |
| 169 |
% \iffalse |
| 170 |
%<*gobble> |
| 171 |
\iffalse |
| 172 |
%</gobble> |
| 173 |
% \fi \fi |
| 174 |
% \begin{macrocode} |
| 175 |
%<*tex-package> |
| 176 |
\endgroup |
| 177 |
%</tex-package> |
| 178 |
% \end{macrocode} |
| 179 |
% \iffalse \iffalse |
| 180 |
%<*gobble> |
| 181 |
\fi |
| 182 |
%</gobble> |
| 183 |
% \fi |
| 184 |
% \begin{macrocode} |
| 185 |
\else |
| 186 |
% \end{macrocode} |
| 187 |
% First deal with \XeTeX: Define |\Uinputnormalization| as an alias for |\XeTeXinputnormalization|. |
| 188 |
% Make sure that \texttt{ltluatex} is loaded. |
| 189 |
% \begin{macrocode} |
| 190 |
\let\Uinputnormalization\XeTeXinputnormalization |
| 191 |
\fi |
| 192 |
\else |
| 193 |
% \end{macrocode} |
| 194 |
% In \LuaTeX\ we emulate |\Uinputnormalization| using a |process_input_buffer| callback. |
| 195 |
% First ensure that |ltluatex| is loaded to have proper callback handling: |
| 196 |
% \begin{macrocode} |
| 197 |
\ifx\newluafunction\@undefined |
| 198 |
\input ltluatex |
| 199 |
\fi |
| 200 |
% \end{macrocode} |
| 201 |
% We need a integer register to control the normalization and then the actual implementation of the callback. |
| 202 |
% Nothing particularly interesting is happening here, the actual normalization is handled by \texttt{lua-uni-algos}. |
| 203 |
% \begin{macrocode} |
| 204 |
\newcount\Uinputnormalization |
| 205 |
\directlua{ |
| 206 |
local getcount = tex.getcount |
| 207 |
local function ident(buf) return buf end |
| 208 |
local uni_normalize = require'lua-uni-normalize' |
| 209 |
local normalize = {[0] = ident, uni_normalize.NFC, uni_normalize.NFD} |
| 210 |
luatexbase.add_to_callback('process_input_buffer', function(buf) |
| 211 |
return normalize[getcount(\the\allocationnumber)](buf) |
| 212 |
end, 'inputnormalization') |
| 213 |
} |
| 214 |
\fi |
| 215 |
% \end{macrocode} |
| 216 |
% \changes{0.2}{2021-07-05}{Enable NFC by default} |
| 217 |
% Finally we enable NFC normalization as a reasonable default: |
| 218 |
% \begin{macrocode} |
| 219 |
\Uinputnormalization=1 |
| 220 |
\endinput |
| 221 |
% \end{macrocode} |
| 222 |
% \iffalse |
| 223 |
%</package|tex-package> |
| 224 |
% \fi |
| 225 |
% \Finale |