Debian Codegraph @

podofo/staging/BuildPredefinedCMaps.cpp

raw
1 #include <iostream>
2 #include <filesystem>
3
4 #include <podofo/private/PdfDeclarationsPrivate.h>
5 #include <podofo/podofo.h>
6 #include <podofo/private/PdfFilterFactory.h>
7
8 namespace fs = std::filesystem;
9 using namespace std;
10 using namespace PoDoFo;
11
12 /**
13 * Helper tool to build predefined CMap resources for PoDoFo
14 * See ISO 32000-2:2020 "Table 116 — Predefined CJK CMap names"
15 */
16
17 struct Context
18 {
19 struct
20 {
21 unordered_set<string> Registries;
22 unordered_set<string> Orderings;
23 unordered_set<int> Supplements;
24 } Catalog;
25 vector<pair<string, string>> Maps;
26 };
27
28 static void handleCMapFolder(const fs::path& path, const unordered_set<string>& knowCmaps, Context& context, bool skipAddToContext);
29
30 static unique_ptr<FileStreamDevice> s_Stream;
31
32 int main()
33 {
34 Context context;
35
36 unordered_set<string> knowCmaps = {
37 // Predefined CMaps
38 "83pv-RKSJ-H",
39 "90ms-RKSJ-H",
40 "90ms-RKSJ-V",
41 "90msp-RKSJ-H",
42 "90msp-RKSJ-V",
43 "90pv-RKSJ-H",
44 "Add-RKSJ-H",
45 "Add-RKSJ-V",
46 "B5pc-H",
47 "B5pc-V",
48 "CNS-EUC-H",
49 "CNS-EUC-V",
50 "ETen-B5-H",
51 "ETen-B5-V",
52 "ETenms-B5-H",
53 "ETenms-B5-V",
54 "EUC-H",
55 "EUC-V",
56 "Ext-RKSJ-H",
57 "Ext-RKSJ-V",
58 "GB-EUC-H",
59 "GB-EUC-V",
60 "GBK-EUC-H",
61 "GBK-EUC-V",
62 "GBK2K-H",
63 "GBK2K-V",
64 "GBKp-EUC-H",
65 "GBKp-EUC-V",
66 "GBpc-EUC-H",
67 "GBpc-EUC-V",
68 "H",
69 "HKscs-B5-H",
70 "HKscs-B5-V",
71 "KSC-EUC-H",
72 "KSC-EUC-V",
73 "KSCms-UHC-H",
74 "KSCms-UHC-HW-H",
75 "KSCms-UHC-HW-V",
76 "KSCms-UHC-V",
77 "KSCpc-EUC-H",
78 "UniCNS-UCS2-H",
79 "UniCNS-UCS2-V",
80 "UniCNS-UTF16-H",
81 "UniCNS-UTF16-V",
82 "UniGB-UCS2-H",
83 "UniGB-UCS2-V",
84 "UniGB-UTF16-H",
85 "UniGB-UTF16-V",
86 "UniJIS-UCS2-H",
87 "UniJIS-UCS2-HW-H",
88 "UniJIS-UCS2-HW-V",
89 "UniJIS-UCS2-V",
90 "UniJIS-UTF16-H",
91 "UniJIS-UTF16-V",
92 "UniKS-UCS2-H",
93 "UniKS-UCS2-V",
94 "UniKS-UTF16-H",
95 "UniKS-UTF16-V",
96 "V",
97 // ToUnicode maps
98 "Adobe-CNS1-UCS2",
99 "Adobe-GB1-UCS2",
100 "Adobe-Japan1-UCS2",
101 "Adobe-Korea1-UCS2",
102 };
103
104 auto currentPath = fs::current_path();
105 auto mainPodofoSrcPath = currentPath.parent_path().parent_path().parent_path().parent_path().parent_path()
106 / "src" / "podofo" / "main";
107
108 // Find a repository that has both "cmap-resources" (https://github.com/adobe-type-tools/cmap-resources)
109 // and "mapping-resources-pdf" (https://github.com/adobe-type-tools/mapping-resources-pdf)
110 auto adobeTypeToolsRepoPath = utls::GetEnvironmentVariable("ADOBE_TYPE_TOOLS_REPOSITORY");
111 if (adobeTypeToolsRepoPath.empty())
112 throw runtime_error("ADOBE_TYPE_TOOLS_REPOSITORY environment variable is missing");
113
114 s_Stream.reset(new FileStreamDevice((mainPodofoSrcPath / "PdfEncodingMapFactory_PredefinedCMaps.cpp").u8string(), FileMode::Create));
115 s_Stream->Write(R"(// This file was generated. DO NOT EDIT!
116 /**
117 * SPDX-FileCopyrightText: (C) 2024 Francesco Pretto <[email protected]>
118 * SPDX-License-Identifier: LGPL-2.0-or-later
119 * SPDX-License-Identifier: MPL-2.0
120 */
121 #include <podofo/private/PdfDeclarationsPrivate.h>
122 #include "PdfEncodingMapFactory.h"
123 #include <podofo/private/PdfFilterFactory.h>
124
125 using namespace std;
126 using namespace PoDoFo;
127
128 // NOTE: The mappings in this file were generated using the
129 // "staging/BuildPredefinedCMaps.cpp" script
130
131 static void buildMappings(const string_view& serialized, CodeUnitMap& mappings, CodeUnitRanges& ranges);
132
133 namespace
134 {
135 using MapGetter = std::add_pointer<const PdfCMapEncodingConstPtr&()>::type;
136 }
137 )");
138
139 s_Stream->Write("\nnamespace PoDoFo\n");
140 s_Stream->Write("{\n");
141 s_Stream->Write(" class PdfCMapEncodingFactory\n");
142 s_Stream->Write(" {\n");
143 s_Stream->Write(" public:\n");
144 handleCMapFolder(fs::u8path(adobeTypeToolsRepoPath) / "cmap-resources", knowCmaps, context, true);
145 handleCMapFolder(fs::u8path(adobeTypeToolsRepoPath) / "mapping-resources-pdf", knowCmaps, context, false);
146 s_Stream->Write(" };\n");
147 s_Stream->Write("}\n");
148 s_Stream->Write("\nstatic unordered_map<string_view, MapGetter> s_PredefinedCMaps = {\n");
149 for (auto& map : context.Maps)
150 {
151 s_Stream->Write(" { \"");
152 s_Stream->Write(map.first);
153 s_Stream->Write("\", &PdfCMapEncodingFactory::");
154 s_Stream->Write(map.second);
155 s_Stream->Write(" },\n");
156 }
157 s_Stream->Write("};\n");
158 s_Stream->Write(R"(
159 PdfCMapEncodingConstPtr PdfEncodingMapFactory::GetPredefinedCMap(const string_view& cmapName)
160 {
161 auto found = s_PredefinedCMaps.find(cmapName);
162 if (found == s_PredefinedCMaps.end())
163 return nullptr;
164 else
165 return found->second();
166 }
167
168 const PdfCMapEncoding& PdfEncodingMapFactory::GetPredefinedCMapInstance(const string_view& cmapName)
169 {
170 auto found = s_PredefinedCMaps.find(cmapName);
171 if (found == s_PredefinedCMaps.end())
172 PODOFO_RAISE_ERROR_INFO(PdfErrorCode::InvalidEncoding, "Could not find a cmap with a CMap name {}", cmapName);
173
174 return *found->second();
175 }
176
177 unsigned readCode(InputStream& stream, unsigned char codeSize)
178 {
179 switch (codeSize)
180 {
181 case 1:
182 {
183 return (unsigned char)stream.ReadChar();
184 }
185 case 2:
186 {
187 uint16_t code;
188 utls::ReadUInt16BE(stream, code);
189 return code;
190 }
191 case 3:
192 {
193 uint24_t code;
194 utls::ReadUInt24BE(stream, code);
195 return code;
196 }
197 case 4:
198 {
199 uint32_t code;
200 utls::ReadUInt32BE(stream, code);
201 return code;
202 }
203 default:
204 PODOFO_RAISE_ERROR(PdfErrorCode::InternalLogic);
205 }
206 }
207
208 void readMapping(InputStream& stream, CodeUnitMap& mappings, vector<codepoint>& temp)
209 {
210 unsigned char codeSize = (unsigned char)stream.ReadChar();
211 unsigned code = readCode(stream, codeSize);
212 unsigned char copdePointsSize = (unsigned char)stream.ReadChar();
213 temp.resize(copdePointsSize);
214 uint32_t cp;
215 for (unsigned char i = 0; i < copdePointsSize; i++)
216 {
217 utls::ReadUInt32BE(stream, cp);
218 temp[i] = cp;
219 }
220
221 mappings[PdfCharCode(code, codeSize)] = CodePointSpan(temp);
222 }
223
224 void readRange(InputStream& stream, CodeUnitRanges& ranges, vector<codepoint>& temp)
225 {
226 unsigned char codeSize = (unsigned char)stream.ReadChar();
227 unsigned code = readCode(stream, codeSize);
228
229 uint16_t rangeSize;
230 utls::ReadUInt16BE(stream, rangeSize);
231
232 unsigned char copdePointsSize = (unsigned char)stream.ReadChar();
233 temp.resize(copdePointsSize);
234 uint32_t cp;
235 for (unsigned char i = 0; i < copdePointsSize; i++)
236 {
237 utls::ReadUInt32BE(stream, cp);
238 temp[i] = cp;
239 }
240
241 ranges.insert(CodeUnitRange{ PdfCharCode(code, codeSize), rangeSize, CodePointSpan(temp) });
242 }
243
244 void buildMappings(const string_view& compressed, CodeUnitMap& mappings, CodeUnitRanges& ranges)
245 {
246 auto filter = PdfFilterFactory::Create(PdfFilterType::FlateDecode);
247 charbuff serialized;
248 filter->DecodeTo(serialized, compressed);
249
250 SpanStreamDevice stream(serialized);
251 uint16_t size;
252 utls::ReadUInt16BE(stream, size);
253 mappings.reserve(size);
254 vector<codepoint> temp;
255 for (unsigned i = 0; i < size; i++)
256 readMapping(stream, mappings, temp);
257
258 utls::ReadUInt16BE(stream, size);
259 for (unsigned i = 0; i < size; i++)
260 readRange(stream, ranges, temp);
261 }
262 )");
263
264 cout << "Registries: [" << endl;
265 for (auto& registry : context.Catalog.Registries)
266 cout << " " << registry << "," << endl;
267 cout << "]" << endl;
268 cout << "Orderings:" << endl;
269 for (auto& ordering : context.Catalog.Orderings)
270 cout << " " << ordering << "," << endl;
271 cout << "]" << endl;
272 cout << "Supplements:" << endl;
273 for (auto& supplement : context.Catalog.Supplements)
274 cout << " " << supplement << "," << endl;
275 cout << "]" << endl;
276 return 0;
277 }
278
279 void serializeCode(OutputStream& stream, unsigned code, unsigned char codeSize)
280 {
281 switch (codeSize)
282 {
283 case 1:
284 stream.Write((char)(uint8_t)code);
285 break;
286 case 2:
287 utls::WriteUInt16BE(stream, (uint16_t)code);
288 break;
289 case 3:
290 utls::WriteUInt24BE(stream, (uint24_t)code);
291 break;
292 case 4:
293 utls::WriteUInt32BE(stream, code);
294 break;
295 default:
296 throw runtime_error("Invalid code size");
297 }
298 }
299
300 void writeString(const string_view& view)
301 {
302 const int max_line_length = 16;
303 std::string line;
304 line.push_back('"');
305
306 for (size_t i = 0; i < view.size(); i++)
307 {
308 // Convert character to octal and append to the line
309 line += fmt::format("\\{:03o}", static_cast<unsigned char>(view[i]));
310
311 // If the line reaches the maximum length or it's the last character, write to the output
312 if ((i + 1) % max_line_length == 0 || i == view.size() - 1)
313 {
314 line.append("\"\n");
315 s_Stream->Write(line);
316 line.clear();
317 line.push_back('"');
318 }
319 }
320 }
321
322 void write(const PdfCharCodeMap& map)
323 {
324 string serialized;
325 ContainerStreamDevice stream(serialized);
326
327 auto& mappings = map.GetMappings();
328 assert(mappings.size() <= 65535);
329 utls::WriteUInt16BE(stream, (uint16_t)mappings.size());
330 for (auto& pair : mappings)
331 {
332 stream.Write((char)pair.first.CodeSpaceSize);
333 serializeCode(stream, pair.first.Code, pair.first.CodeSpaceSize);
334 assert(pair.second.size() <= 255);
335 stream.Write((char)(uint8_t)pair.second.size());
336 for (unsigned i = 0; i < pair.second.size(); i++)
337 utls::WriteUInt32BE(stream, pair.second[i]);
338 }
339
340 auto& ranges = map.GetRanges();
341 assert(ranges.size() <= 65535);
342 utls::WriteUInt16BE(stream, (uint16_t)ranges.size());
343 for (auto& range : ranges)
344 {
345 stream.Write((char)range.SrcCodeLo.CodeSpaceSize);
346 serializeCode(stream, range.SrcCodeLo.Code, range.SrcCodeLo.CodeSpaceSize);
347 assert(range.Size <= 65535);
348 utls::WriteUInt16BE(stream, (uint16_t)range.Size);
349 assert(range.DstCodeLo.size() <= 255);
350 stream.Write((char)(uint8_t)range.DstCodeLo.size());
351 for (unsigned i = 0; i < range.DstCodeLo.size(); i++)
352 utls::WriteUInt32BE(stream, range.DstCodeLo[i]);
353 }
354
355 auto filter = PdfFilterFactory::Create(PdfFilterType::FlateDecode);
356 charbuff compressed;
357 filter->EncodeTo(compressed, serialized);
358 writeString(compressed);
359 }
360
361 void write(const PdfCMapEncoding& encoding, Context& context)
362 {
363 s_Stream->Write(" static const PdfCMapEncodingConstPtr& ");
364 auto& info = encoding.GetCIDSystemInfo();
365 auto& map = encoding.GetCharMap();
366 auto& limits = encoding.GetLimits();
367 auto& mapLimits = map.GetLimits();
368 auto name = encoding.GetName().GetString();
369 string methodName = "Get_";
370 methodName.append(name);
371 std::replace(methodName.begin(), methodName.end(), '-', '_');
372 s_Stream->Write(methodName);
373 s_Stream->Write("()\n");
374 s_Stream->Write(" {\n");
375 s_Stream->Write(" static constexpr const char serialized[] =\n");
376 write(map);
377 s_Stream->Write(";\n");
378 s_Stream->Write(" static struct Init\n");
379 s_Stream->Write(" {\n");
380 s_Stream->Write(" Init()\n");
381 s_Stream->Write(" {\n");
382 s_Stream->Write(" CodeUnitMap mappings;\n");
383 s_Stream->Write(" CodeUnitRanges ranges;\n");
384 s_Stream->Write(" buildMappings(string_view(serialized, std::size(serialized) - 1), mappings, ranges);\n");
385 s_Stream->Write(" map.reset(new PdfCMapEncoding(PdfCharCodeMap(\n");
386 s_Stream->Write(fmt::format(" std::move(mappings), std::move(ranges), PdfEncodingLimits({}, {}, PdfCharCode({}, {}), PdfCharCode({}, {}))),\n",
387 mapLimits.MinCodeSize, mapLimits.MaxCodeSize, mapLimits.FirstChar.Code, mapLimits.FirstChar.CodeSpaceSize, mapLimits.LastChar.Code, mapLimits.LastChar.CodeSpaceSize));
388 s_Stream->Write(fmt::format(R"( true, "{}"_n, PdfCIDSystemInfo{{ "{}", "{}", {} }}, {}, PdfEncodingLimits({}, {}, PdfCharCode({}, {}), PdfCharCode({}, {}))));)",
389 name, info.Registry.GetString(), info.Ordering.GetString(), info.Supplement, (encoding.GetWMode() == PdfWModeKind::Vertical ? 1 : 0),
390 limits.MinCodeSize, limits.MaxCodeSize, limits.FirstChar.Code, limits.FirstChar.CodeSpaceSize, limits.LastChar.Code, limits.LastChar.CodeSpaceSize));
391 s_Stream->Write("\n }\n");
392 s_Stream->Write(" PdfCMapEncodingConstPtr map;\n");
393 s_Stream->Write(" } init;\n");
394 s_Stream->Write(" return init.map;\n");
395 s_Stream->Write(" }\n\n");
396 context.Maps.push_back({ (string)name, methodName });
397 }
398
399 void handleCMapFolder(const fs::path& path, const unordered_set<string>& knowCmaps, Context& context, bool addToCatalog)
400 {
401 // Iterate through the directory entries
402 for (const auto& outerFolder : fs::directory_iterator(path))
403 {
404 if (outerFolder.path().u8string().find(".git", 0) != string::npos)
405 continue;
406
407 if (outerFolder.path().u8string().find("deprecated", 0) != string::npos)
408 continue;
409
410 if (fs::is_directory(outerFolder.status()))
411 {
412 auto cmapfolder = outerFolder.path() / "CMap";
413 const fs::path* actualPath;
414 if (fs::exists(cmapfolder))
415 actualPath = &cmapfolder;
416 else
417 actualPath = &outerFolder.path();
418
419 for (auto& entry : fs::directory_iterator(*actualPath))
420 {
421 auto& entryPath = entry.path();
422 if (knowCmaps.find(entryPath.filename().u8string()) == knowCmaps.end())
423 continue;
424
425 cout << entryPath.u8string() << endl;
426 auto cmap = PdfCMapEncoding::Parse(entryPath.u8string());
427 cout << "Name: " << cmap.GetName().GetString()
428 << ", Ordering: " << cmap.GetCIDSystemInfo().Registry.GetString()
429 << ", Ordering: " << cmap.GetCIDSystemInfo().Ordering.GetString()
430 << ", Supplement: " << cmap.GetCIDSystemInfo().Supplement
431 << ", WMode: " << (cmap.GetWMode() == PdfWModeKind::Horizontal ? "H" : "V") << endl;
432
433 write(cmap, context);
434
435 if (addToCatalog)
436 {
437 context.Catalog.Registries.insert((string)cmap.GetCIDSystemInfo().Registry);
438 context.Catalog.Orderings.insert((string)cmap.GetCIDSystemInfo().Ordering);
439 context.Catalog.Supplements.insert(cmap.GetCIDSystemInfo().Supplement);
440 }
441 }
442 }
443 }
444 }