Debian Codegraph @

unikey/charset.cpp

raw
1 /*
2 * SPDX-FileCopyrightText: 1998-2002 Pham Kim Long <[email protected]>
3 *
4 * SPDX-License-Identifier: GPL-2.0-or-later
5 */
6
7 #include <ctype.h>
8 #include <memory.h>
9 #include <search.h>
10 #include <stddef.h>
11 #include <stdlib.h>
12
13 #include "charset.h"
14 #include "data.h"
15
16 int LoVowel['z' - 'a' + 1];
17 int HiVowel['Z' - 'A' + 1];
18
19 #define IS_VOWEL(x) \
20 ((x >= 'a' && x <= 'z' && LoVowel[x - 'a']) || \
21 (x >= 'A' && x <= 'Z' && HiVowel[x - 'A']))
22
23 SingleByteCharset *SgCharsets[CONV_TOTAL_SINGLE_CHARSETS];
24 DoubleByteCharset *DbCharsets[CONV_TOTAL_DOUBLE_CHARSETS];
25
26 DllExport CVnCharsetLib VnCharsetLibObj;
27
28 //////////////////////////////////////////////////////
29 // Generic VnCharset class
30 //////////////////////////////////////////////////////
31 int VnCharset::elementSize() { return 1; }
32
33 //-------------------------------------------
34 int VnInternalCharset::nextInput(ByteInStream &is, StdVnChar &stdChar,
35 int &bytesRead) {
36 if (!is.getNextDW(stdChar)) {
37 bytesRead = 0;
38 return 0;
39 }
40 bytesRead = sizeof(UKDWORD);
41 return 1;
42 }
43
44 //-------------------------------------------
45 int VnInternalCharset::putChar(ByteOutStream &os, StdVnChar stdChar,
46 int &outLen) {
47 outLen = sizeof(StdVnChar);
48 os.putW((UKWORD)stdChar);
49 return os.putW((UKWORD)(stdChar >> (sizeof(UKWORD) * 8)));
50 }
51
52 //-------------------------------------------
53 int VnInternalCharset::elementSize() { return 4; }
54
55 //-------------------------------------------
56 SingleByteCharset::SingleByteCharset(unsigned char *vnChars) {
57 int i;
58 m_vnChars = vnChars;
59 memset(m_stdMap, 0, 256 * sizeof(UKWORD));
60 for (i = 0; i < TOTAL_VNCHARS; i++) {
61 if (vnChars[i] != 0 &&
62 (i == TOTAL_VNCHARS - 1 || vnChars[i] != vnChars[i + 1]))
63 m_stdMap[vnChars[i]] = i + 1;
64 }
65 }
66
67 //-------------------------------------------
68 int SingleByteCharset::nextInput(ByteInStream &is, StdVnChar &stdChar,
69 int &bytesRead) {
70 unsigned char ch;
71 if (!is.getNext(ch)) {
72 bytesRead = 0;
73 return 0;
74 }
75
76 stdChar = (m_stdMap[ch]) ? (VnStdCharOffset + m_stdMap[ch] - 1) : ch;
77 bytesRead = 1;
78 return 1;
79 }
80
81 //-------------------------------------------
82 int SingleByteCharset::putChar(ByteOutStream &os, StdVnChar stdChar,
83 int &outLen) {
84 int ret;
85 unsigned char ch;
86 if (stdChar >= VnStdCharOffset) {
87 outLen = 1;
88 ch = m_vnChars[stdChar - VnStdCharOffset];
89 if (ch == 0)
90 ch = (stdChar == StdStartQuote)
91 ? PadStartQuote
92 : ((stdChar == StdEndQuote)
93 ? PadEndQuote
94 : ((stdChar == StdEllipsis) ? PadEllipsis
95 : PadChar));
96 ret = os.putB(ch);
97 } else {
98 if (stdChar > 255 || m_stdMap[stdChar]) {
99 // this character is missing in the charset
100 // output padding character
101 outLen = 1;
102 ret = os.putB(PadChar);
103 } else {
104 outLen = 1;
105 ret = os.putB((UKBYTE)stdChar);
106 }
107 }
108 return ret;
109 }
110
111 //-------------------------------------------
112 int wideCharCompare(const void *ele1, const void *ele2) {
113 UKWORD ch1 = LOWORD(*((UKDWORD *)ele1));
114 UKWORD ch2 = LOWORD(*((UKDWORD *)ele2));
115 return (ch1 == ch2) ? 0 : ((ch1 > ch2) ? 1 : -1);
116 }
117
118 //-------------------------------------------
119 UnicodeCharset::UnicodeCharset(UnicodeChar *vnChars) {
120 UKDWORD i;
121 m_toUnicode = vnChars;
122 for (i = 0; i < TOTAL_VNCHARS; i++)
123 m_vnChars[i] = (i << 16) + vnChars[i]; // high word is used for index
124 qsort(m_vnChars, TOTAL_VNCHARS, sizeof(UKDWORD), wideCharCompare);
125 }
126
127 //-------------------------------------------
128 int UnicodeCharset::nextInput(ByteInStream &is, StdVnChar &stdChar,
129 int &bytesRead) {
130 UnicodeChar uniCh;
131 if (!is.getNextW(uniCh)) {
132 bytesRead = 0;
133 return 0;
134 }
135 bytesRead = sizeof(UnicodeChar);
136 UKDWORD key = uniCh;
137 UKDWORD *pChar = (UKDWORD *)bsearch(&key, m_vnChars, TOTAL_VNCHARS,
138 sizeof(UKDWORD), wideCharCompare);
139 if (pChar)
140 stdChar = VnStdCharOffset + HIWORD(*pChar);
141 else
142 stdChar = uniCh;
143 return 1;
144 }
145
146 //-------------------------------------------
147 int UnicodeCharset::putChar(ByteOutStream &os, StdVnChar stdChar, int &outLen) {
148 outLen = sizeof(UnicodeChar);
149 return os.putW((stdChar >= VnStdCharOffset)
150 ? m_toUnicode[stdChar - VnStdCharOffset]
151 : (UnicodeChar)stdChar);
152 }
153
154 //-------------------------------------------
155 int UnicodeCharset::elementSize() { return 2; }
156
157 ////////////////////////////////////////
158 // Unicode decomposed
159 ////////////////////////////////////////
160 //-------------------------------------------
161 int uniCompInfoCompare(const void *ele1, const void *ele2) {
162 UKDWORD ch1 = ((UniCompCharInfo *)ele1)->compChar;
163 UKDWORD ch2 = ((UniCompCharInfo *)ele2)->compChar;
164 return (ch1 == ch2) ? 0 : ((ch1 > ch2) ? 1 : -1);
165 }
166
167 UnicodeCompCharset::UnicodeCompCharset(UnicodeChar *uniChars,
168 UKDWORD *uniCompChars) {
169 int i, k;
170 m_uniCompChars = uniCompChars;
171 m_totalChars = 0;
172 for (i = 0; i < TOTAL_VNCHARS; i++) {
173 m_info[i].compChar = uniCompChars[i];
174 m_info[i].stdIndex = i;
175 m_totalChars++;
176 }
177
178 for (k = 0, i = TOTAL_VNCHARS; k < TOTAL_VNCHARS; k++)
179 if (uniChars[k] != uniCompChars[k]) {
180 m_info[i].compChar = uniChars[k];
181 m_info[i].stdIndex = k;
182 m_totalChars++;
183 i++;
184 }
185
186 qsort(m_info, m_totalChars, sizeof(UniCompCharInfo), uniCompInfoCompare);
187 }
188
189 //---------------------------------------------
190 int UnicodeCompCharset::nextInput(ByteInStream &is, StdVnChar &stdChar,
191 int &bytesRead) {
192 // read first char
193
194 UniCompCharInfo key;
195 UKWORD w;
196 if (!is.getNextW(w)) {
197 bytesRead = 0;
198 return 0;
199 }
200 key.compChar = w;
201 bytesRead = 2;
202
203 UniCompCharInfo *pInfo =
204 (UniCompCharInfo *)bsearch(&key, m_info, m_totalChars,
205 sizeof(UniCompCharInfo), uniCompInfoCompare);
206 if (!pInfo)
207 stdChar = key.compChar;
208 else {
209 stdChar = pInfo->stdIndex + VnStdCharOffset;
210 if (is.peekNextW(w)) {
211 UKDWORD hi = w;
212 if (hi > 0) {
213 key.compChar += hi << 16;
214 pInfo = (UniCompCharInfo *)bsearch(&key, m_info, m_totalChars,
215 sizeof(UniCompCharInfo),
216 uniCompInfoCompare);
217 if (pInfo) {
218 stdChar = pInfo->stdIndex + VnStdCharOffset;
219 bytesRead += 2;
220 is.getNextW(w);
221 }
222 }
223 }
224 }
225 return 1;
226 }
227
228 //---------------------------------------------
229 int UnicodeCompCharset::putChar(ByteOutStream &os, StdVnChar stdChar,
230 int &outLen) {
231 int ret;
232 if (stdChar >= VnStdCharOffset) {
233 UKDWORD uniCompCh = m_uniCompChars[stdChar - VnStdCharOffset];
234 UKWORD lo = LOWORD(uniCompCh);
235 UKWORD hi = HIWORD(uniCompCh);
236 outLen = 2;
237 ret = os.putW(lo);
238 if (hi > 0) {
239 outLen += 2;
240 ret = os.putW(hi);
241 }
242 } else {
243 outLen = 2;
244 ret = os.putW((UKWORD)stdChar);
245 }
246 return ret;
247 }
248
249 //-------------------------------------------
250 int UnicodeCompCharset::elementSize() { return 2; }
251
252 ////////////////////////////////
253 // Unicode UTF-8 //
254 ////////////////////////////////
255 int UnicodeUTF8Charset::nextInput(ByteInStream &is, StdVnChar &stdChar,
256 int &bytesRead) {
257 UKWORD w1, w2, w3;
258 UKBYTE first, second, third;
259 UnicodeChar uniCh;
260
261 bytesRead = 0;
262 if (!is.getNext(first))
263 return 0;
264 bytesRead = 1;
265
266 if (first < 0x80)
267 uniCh = first; // 1-byte sequence
268 else if ((first & 0xE0) == 0xC0) {
269 // 2-byte sequence
270 if (!is.peekNext(second))
271 return 0;
272 if ((second & 0xC0) != 0x80) {
273 stdChar = INVALID_STD_CHAR;
274 return 1;
275 }
276 is.getNext(second);
277 bytesRead = 2;
278 w1 = first;
279 w2 = second;
280 uniCh = ((w1 & 0x001F) << 6) | (w2 & 0x3F);
281 } else if ((first & 0xF0) == 0xE0) {
282 // 3-byte sequence
283 if (!is.peekNext(second))
284 return 0;
285 if ((second & 0xC0) != 0x80) {
286 stdChar = INVALID_STD_CHAR;
287 return 1;
288 }
289 is.getNext(second);
290 bytesRead = 2;
291 if (!is.peekNext(third))
292 return 0;
293 if ((third & 0xC0) != 0x80) {
294 stdChar = INVALID_STD_CHAR;
295 return 1;
296 }
297 is.getNext(third);
298 bytesRead = 3;
299 w1 = first;
300 w2 = second;
301 w3 = third;
302 uniCh = ((w1 & 0x000F) << 12) | ((w2 & 0x003F) << 6) | (w3 & 0x003F);
303 } else {
304 stdChar = INVALID_STD_CHAR;
305 return 1;
306 }
307
308 // translate to StdVnChar
309 UKDWORD key = uniCh;
310 UKDWORD *pChar = (UKDWORD *)bsearch(&key, m_vnChars, TOTAL_VNCHARS,
311 sizeof(UKDWORD), wideCharCompare);
312 if (pChar)
313 stdChar = VnStdCharOffset + HIWORD(*pChar);
314 else
315 stdChar = uniCh;
316 return 1;
317 }
318
319 //-------------------------------------------
320 int UnicodeUTF8Charset::putChar(ByteOutStream &os, StdVnChar stdChar,
321 int &outLen) {
322 UnicodeChar uChar = (stdChar < VnStdCharOffset)
323 ? (UnicodeChar)stdChar
324 : m_toUnicode[stdChar - VnStdCharOffset];
325 int ret;
326 if (uChar < 0x0080) {
327 outLen = 1;
328 ret = os.putB((UKBYTE)uChar);
329 } else if (uChar < 0x0800) {
330 outLen = 2;
331 os.putB(0xC0 | (UKBYTE)(uChar >> 6));
332 ret = os.putB(0x80 | (UKBYTE)(uChar & 0x003F));
333 } else {
334 outLen = 3;
335 os.putB(0xE0 | (UKBYTE)(uChar >> 12));
336 os.putB(0x80 | (UKBYTE)((uChar >> 6) & 0x003F));
337 ret = os.putB(0x80 | (UKBYTE)(uChar & 0x003F));
338 }
339 return ret;
340 }
341
342 ////////////////////////////////////////
343 // Unicode character reference &#D; //
344 ////////////////////////////////////////
345 int hexDigitValue(unsigned char digit) {
346 if (digit >= 'a' && digit <= 'f')
347 return digit - 'a' + 10;
348 if (digit >= 'A' && digit <= 'F')
349 return digit - 'A' + 10;
350 if (digit >= '0' && digit <= '9')
351 return digit - '0';
352 return 0;
353 }
354
355 //--------------------------------------
356 int UnicodeRefCharset::nextInput(ByteInStream &is, StdVnChar &stdChar,
357 int &bytesRead) {
358 unsigned char ch;
359 UnicodeChar uniCh;
360 bytesRead = 0;
361 if (!is.getNext(ch))
362 return 0;
363 bytesRead = 1;
364 uniCh = ch;
365 if (ch == '&') {
366 if (is.peekNext(ch) && ch == '#') {
367 is.getNext(ch);
368 bytesRead++;
369 if (!is.eos()) {
370 is.peekNext(ch);
371 if (ch != 'x' && ch != 'X') {
372 UKWORD code = 0;
373 int digits = 0;
374 while (is.peekNext(ch) && isdigit(ch) && digits < 5) {
375 is.getNext(ch);
376 bytesRead++;
377 code = code * 10 + (ch - '0');
378 digits++;
379 }
380 if (is.peekNext(ch) && ch == ';') {
381 is.getNext(ch);
382 bytesRead++;
383 uniCh = code;
384 }
385 } else {
386 is.getNext(ch);
387 bytesRead++;
388 UKWORD code = 0;
389 int digits = 0;
390 while (is.peekNext(ch) && isxdigit(ch) && digits < 4) {
391 is.getNext(ch);
392 bytesRead++;
393 code = (code << 4) + hexDigitValue(ch);
394 digits++;
395 }
396 if (is.peekNext(ch) && ch == ';') {
397 is.getNext(ch);
398 bytesRead++;
399 uniCh = code;
400 }
401 } // hex digits
402 }
403 }
404 }
405
406 // translate to StdVnChar
407 UKDWORD key = uniCh;
408 UKDWORD *pChar = (UKDWORD *)bsearch(&key, m_vnChars, TOTAL_VNCHARS,
409 sizeof(UKDWORD), wideCharCompare);
410 if (pChar)
411 stdChar = VnStdCharOffset + HIWORD(*pChar);
412 else
413 stdChar = uniCh;
414 return 1;
415 }
416
417 //--------------------------------
418 int UnicodeRefCharset::putChar(ByteOutStream &os, StdVnChar stdChar,
419 int &outLen) {
420 UnicodeChar uChar = (stdChar < VnStdCharOffset)
421 ? (UnicodeChar)stdChar
422 : m_toUnicode[stdChar - VnStdCharOffset];
423 int ret;
424 if (uChar < 128) {
425 outLen = 1;
426 ret = os.putB((UKBYTE)uChar);
427 } else {
428 outLen = 2;
429 os.putB((UKBYTE)'&');
430 os.putB((UKBYTE)'#');
431
432 int i, digit, prev, base;
433 prev = 0;
434 base = 10000;
435 for (i = 0; i < 5; i++) {
436 digit = uChar / base;
437 if (digit || prev) {
438 prev = 1;
439 outLen++;
440 os.putB('0' + (unsigned char)digit);
441 }
442 uChar %= base;
443 base /= 10;
444 }
445 ret = os.putB((UKBYTE)';');
446 outLen++;
447 }
448 return ret;
449 }
450
451 #define HEX_DIGIT(x) ((x < 10) ? ('0' + x) : ('A' + x - 10))
452
453 //--------------------------------
454 int UnicodeHexCharset::putChar(ByteOutStream &os, StdVnChar stdChar,
455 int &outLen) {
456 UnicodeChar uChar = (stdChar < VnStdCharOffset)
457 ? (UnicodeChar)stdChar
458 : m_toUnicode[stdChar - VnStdCharOffset];
459 int ret;
460 if (uChar < 256) {
461 outLen = 1;
462 ret = os.putB((UKBYTE)uChar);
463 } else {
464 outLen = 3;
465 os.putB('&');
466 os.putB('#');
467 os.putB('x');
468
469 int i, digit;
470 int prev = 0;
471 int shifts = 12;
472
473 for (i = 0; i < 4; i++) {
474 digit = ((uChar >> shifts) & 0x000F);
475 if (digit > 0 || prev) {
476 prev = 1;
477 outLen++;
478 os.putB((UKBYTE)HEX_DIGIT(digit));
479 }
480 shifts -= 4;
481 }
482 ret = os.putB(';');
483 outLen++;
484 }
485 return ret;
486 }
487
488 /////////////////////////////////
489 // Class UnicodeCStringCharset /
490 /////////////////////////////////
491 void UnicodeCStringCharset::startInput() { m_prevIsHex = 0; }
492
493 //----------------------------------------
494 int UnicodeCStringCharset::nextInput(ByteInStream &is, StdVnChar &stdChar,
495 int &bytesRead) {
496 unsigned char ch;
497 UnicodeChar uniCh;
498 bytesRead = 0;
499 if (!is.getNext(ch))
500 return 0;
501 bytesRead = 1;
502 uniCh = ch;
503 if (ch == '\\') {
504 if (is.peekNext(ch) && (ch == 'x' || ch == 'X')) {
505 is.getNext(ch);
506 bytesRead++;
507 UKWORD code = 0;
508 int digits = 0;
509 while (is.peekNext(ch) && isxdigit(ch) && digits < 4) {
510 is.getNext(ch);
511 bytesRead++;
512 code = (code << 4) + hexDigitValue(ch);
513 digits++;
514 }
515 uniCh = code;
516 }
517 }
518
519 // translate to StdVnChar
520 UKDWORD key = uniCh;
521 UKDWORD *pChar = (UKDWORD *)bsearch(&key, m_vnChars, TOTAL_VNCHARS,
522 sizeof(UKDWORD), wideCharCompare);
523 if (pChar)
524 stdChar = VnStdCharOffset + HIWORD(*pChar);
525 else
526 stdChar = uniCh;
527 return 1;
528 }
529
530 //------------------------------------
531 int UnicodeCStringCharset::putChar(ByteOutStream &os, StdVnChar stdChar,
532 int &outLen) {
533 UnicodeChar uChar = (stdChar < VnStdCharOffset)
534 ? (UnicodeChar)stdChar
535 : m_toUnicode[stdChar - VnStdCharOffset];
536 int ret;
537 if (uChar < 128 && !isxdigit(uChar) && uChar != 'x' && uChar != 'X') {
538 outLen = 1;
539 ret = os.putB((UKBYTE)uChar);
540 } else {
541 outLen = 2;
542 os.putB('\\');
543 os.putB('x');
544
545 int i, digit;
546 int prev = 0;
547 int shifts = 12;
548
549 for (i = 0; i < 4; i++) {
550 digit = ((uChar >> shifts) & 0x000F);
551 if (digit > 0 || prev) {
552 prev = 1;
553 outLen++;
554 os.putB((UKBYTE)HEX_DIGIT(digit));
555 }
556 shifts -= 4;
557 }
558 ret = os.isOK();
559 m_prevIsHex = 1;
560 }
561 return ret;
562 }
563
564 /////////////////////////////////
565 // Double-byte charsets //
566 /////////////////////////////////
567 DoubleByteCharset::DoubleByteCharset(UKWORD *vnChars) {
568 m_toDoubleChar = vnChars;
569 memset(m_stdMap, 0, 256 * sizeof(UKWORD));
570 for (int i = 0; i < TOTAL_VNCHARS; i++) {
571 if (vnChars[i] >> 8) // a 2-byte character
572 m_stdMap[vnChars[i] >> 8] = 0xFFFF; // INVALID_STD_CHAR;
573 else if (m_stdMap[vnChars[i]] == 0)
574 m_stdMap[vnChars[i]] = i + 1;
575 m_vnChars[i] =
576 (i << 16) + vnChars[i]; // high word is used for StdChar index
577 }
578 qsort(m_vnChars, TOTAL_VNCHARS, sizeof(UKDWORD), wideCharCompare);
579 }
580
581 //---------------------------------------------
582 int DoubleByteCharset::nextInput(ByteInStream &is, StdVnChar &stdChar,
583 int &bytesRead) {
584 unsigned char ch;
585
586 // read first byte
587 bytesRead = 0;
588 if (!is.getNext(ch))
589 return 0;
590 bytesRead = 1;
591 stdChar = m_stdMap[ch];
592 if (stdChar == 0)
593 stdChar = ch;
594 else if (stdChar == 0xFFFF)
595 stdChar = INVALID_STD_CHAR;
596 else {
597 stdChar += VnStdCharOffset - 1;
598 UKBYTE hi;
599 if (is.peekNext(hi) && hi > 0) {
600 // test if a double-byte character is encountered
601 UKDWORD key = MAKEWORD(ch, hi);
602 UKDWORD *pChar =
603 (UKDWORD *)bsearch(&key, m_vnChars, TOTAL_VNCHARS,
604 sizeof(UKDWORD), wideCharCompare);
605 if (pChar) {
606 stdChar = VnStdCharOffset + HIWORD(*pChar);
607 bytesRead = 2;
608 is.getNext(hi);
609 }
610 }
611 }
612 return 1;
613 }
614
615 //---------------------------------------------
616 int DoubleByteCharset::putChar(ByteOutStream &os, StdVnChar stdChar,
617 int &outLen) {
618 int ret;
619 if (stdChar >= VnStdCharOffset) {
620 UKWORD wCh = m_toDoubleChar[stdChar - VnStdCharOffset];
621
622 if (wCh & 0xFF00) {
623 outLen = 2;
624 os.putB((UKBYTE)(wCh & 0x00FF));
625 ret = os.putB((UKBYTE)(wCh >> 8));
626 } else {
627 unsigned char b = (unsigned char)wCh;
628 if (m_stdMap[b] == 0xFFFF)
629 b = PadChar;
630 outLen = 1;
631 ret = os.putB(b);
632 }
633 /*
634 outLen = 1;
635 ret = os.putB((UKBYTE)(wCh & 0x00FF));
636 if (wCh & 0xFF00) {
637 outLen = 2;
638 ret = os.putB((UKBYTE)(wCh >> 8));
639 }
640 */
641 } else {
642 if (stdChar > 255 || m_stdMap[stdChar]) {
643 outLen = 1;
644 ret = os.putB((UKBYTE)PadChar);
645 } else {
646 outLen = 1;
647 ret = os.putB((UKBYTE)stdChar);
648 }
649 }
650 return ret;
651 }
652
653 /////////////////////////////////////////////
654 // Class: VIQRCharset //
655 /////////////////////////////////////////////
656
657 unsigned char VIQRTones[] = {'\'', '`', '?', '~', '.'};
658
659 const char *VIQREscapes[] = {
660 "://", "/", "@", "mailto:", "email:", "news:", "www", "ftp"};
661
662 const int VIQREscCount = sizeof(VIQREscapes) / sizeof(char *);
663
664 VIQRCharset::VIQRCharset(UKDWORD *vnChars) {
665 memset(m_stdMap, 0, 256 * sizeof(UKWORD));
666 int i;
667 UKDWORD dw;
668 m_vnChars = vnChars;
669 for (i = 0; i < TOTAL_VNCHARS; i++) {
670 dw = m_vnChars[i];
671 if (!(dw & 0xffffff00)) { // single byte
672 // ch = (unsigned char)(dw & 0xff);
673 m_stdMap[dw] = i + 256;
674 }
675 }
676
677 // set offset from base characters according to tone marks
678 m_stdMap[(unsigned char)'\''] = 2;
679 m_stdMap[(unsigned char)'`'] = 4;
680 m_stdMap[(unsigned char)'?'] = 6;
681 m_stdMap[(unsigned char)'~'] = 8;
682 m_stdMap[(unsigned char)'.'] = 10;
683 m_stdMap[(unsigned char)'^'] = 12;
684
685 m_stdMap[(unsigned char)'('] = 24;
686 m_stdMap[(unsigned char)'+'] = 26;
687 m_stdMap[(unsigned char)'*'] = 26;
688 }
689
690 //---------------------------------------------------
691 void VIQRCharset::startInput() {
692 m_suspicious = 0;
693 m_atWordBeginning = 1;
694 m_gotTone = 0;
695 m_escAll = 0;
696 if (VnCharsetLibObj.m_options.viqrEsc)
697 VnCharsetLibObj.m_VIQREscPatterns.reset();
698 }
699
700 //---------------------------------------------------
701 int VIQRCharset::nextInput(ByteInStream &is, StdVnChar &stdChar,
702 int &bytesRead) {
703 unsigned char ch1;
704 bytesRead = 0;
705
706 if (!is.getNext(ch1))
707 return 0;
708 bytesRead = 1;
709 stdChar = m_stdMap[ch1];
710
711 if (VnCharsetLibObj.m_options.viqrEsc) {
712 if (VnCharsetLibObj.m_VIQREscPatterns.foundAtNextChar(ch1) != -1) {
713 m_escAll = 1;
714 }
715 }
716
717 if (m_escAll && (ch1 == ' ' || ch1 == '\t' || ch1 == '\r' || ch1 == '\n'))
718 m_escAll = 0;
719
720 if (ch1 == '\\') {
721 // ecape character , try to read next
722 if (!is.getNext(ch1)) {
723 bytesRead++;
724 stdChar = m_stdMap[ch1];
725 }
726 }
727
728 if (stdChar < 256) {
729 stdChar = ch1;
730 } else if (!m_escAll && !is.eos()) {
731 // try to read the next byte
732 unsigned char ch2;
733 is.peekNext(ch2);
734 unsigned char upper = toupper(ch1);
735 if ((!VnCharsetLibObj.m_options.smartViqr || m_atWordBeginning) &&
736 upper == 'D' && (ch2 == 'd' || ch2 == 'D')) {
737 is.getNext(ch2);
738 bytesRead++;
739 stdChar += 2; // dd is 2 positions after d.
740 } else {
741 StdVnChar index = m_stdMap[ch2];
742
743 int cond;
744 if (m_suspicious) {
745 cond =
746 IS_VOWEL(ch1) &&
747 (index == 2 || index == 4 ||
748 index == 8 || // not accepting ? . in suspicious mode
749 (index == 12 &&
750 (upper == 'A' || upper == 'E' || upper == 'O')) ||
751 (m_stdMap[ch2] == 24 && upper == 'A') ||
752 (m_stdMap[ch2] == 26 && (upper == 'O' || upper == 'U')));
753 if (cond)
754 m_suspicious = 0;
755 } else
756 cond =
757 IS_VOWEL(ch1) &&
758 ((index <= 10 && index > 0 &&
759 (!m_gotTone || (index != 6 && index != 10))) ||
760 (index == 12 &&
761 (upper == 'A' || upper == 'E' || upper == 'O')) ||
762 (m_stdMap[ch2] == 24 && upper == 'A') ||
763 (m_stdMap[ch2] == 26 && (upper == 'O' || upper == 'U')));
764
765 if (cond) {
766 if (index > 0)
767 m_gotTone =
768 1; // we have a tone/breve/hook in the current word
769
770 // ok, take this byte
771 is.getNext(ch2);
772 bytesRead++;
773 int offset = m_stdMap[ch2];
774 if (offset == 26)
775 offset = 24;
776 if (offset == 24 && (ch1 == 'u' || ch1 == 'U'))
777 offset = 12;
778 stdChar += offset;
779 // check next byte
780 if (is.peekNext(ch2)) {
781 if (index > 10 && m_stdMap[ch2] > 0 &&
782 m_stdMap[ch2] <= 10) {
783 // ok, take one more byte
784 is.getNext(ch2);
785 bytesRead++;
786 stdChar += m_stdMap[ch2];
787 }
788 }
789 }
790 }
791 }
792 m_atWordBeginning = (stdChar < 256);
793 if (stdChar < 256) {
794 m_gotTone =
795 0; // reset this flag because we are at the beginning of a new word
796 }
797
798 // adjust stdChar
799 if (stdChar >= 256)
800 stdChar += VnStdCharOffset - 256;
801 return 1;
802 }
803
804 //---------------------------------------------------
805 void VIQRCharset::startOutput() {
806 m_escapeBowl = 0;
807 m_escapeRoof = 0;
808 m_escapeHook = 0;
809 m_escapeTone = 0;
810 m_noOutEsc = 0;
811 VnCharsetLibObj.m_VIQROutEscPatterns.reset();
812 }
813
814 //---------------------------------------------------
815 int VIQRCharset::putChar(ByteOutStream &os, StdVnChar stdChar, int &outLen) {
816 int ret;
817 UKBYTE b;
818 if (stdChar >= VnStdCharOffset) {
819 outLen = 1;
820 UKDWORD dw = m_vnChars[stdChar - VnStdCharOffset];
821
822 unsigned char first = (unsigned char)dw;
823 unsigned char firstUpper = toupper(first);
824
825 b = (UKBYTE)dw;
826 ret = os.putB(b);
827 if (VnCharsetLibObj.m_VIQROutEscPatterns.foundAtNextChar(b) != -1)
828 m_noOutEsc = 1;
829
830 if (m_noOutEsc && (b == ' ' || b == '\t' || b == '\r' || b == '\n'))
831 m_noOutEsc = 0;
832
833 if (dw & 0x0000FF00) {
834 // second byte is present
835 unsigned char second = (UKBYTE)(dw >> 8);
836 outLen++;
837 ret = os.putB(second);
838
839 if (dw & 0x00FF0000) {
840 // third byte is present
841 outLen++;
842 ret = os.putB((UKBYTE)(dw >> 16));
843 m_escapeTone = 0;
844 } else {
845 UKWORD index = m_stdMap[second];
846 m_escapeTone = (index == 12 || index == 24 || index == 26);
847 }
848
849 VnCharsetLibObj.m_VIQROutEscPatterns.reset();
850
851 m_escapeBowl = 0;
852 m_escapeHook = 0;
853 m_escapeRoof = 0;
854 } else {
855 m_escapeTone = IS_VOWEL(first);
856 m_escapeBowl = (firstUpper == 'A');
857 m_escapeHook = (firstUpper == 'U' || firstUpper == 'O');
858 m_escapeRoof =
859 (firstUpper == 'A' || firstUpper == 'E' || firstUpper == 'O');
860 }
861 } else {
862 if (stdChar > 255) {
863 outLen = 1;
864 ret = os.putB((UKBYTE)PadChar);
865 if (VnCharsetLibObj.m_VIQROutEscPatterns.foundAtNextChar(
866 (UKBYTE)PadChar) != -1)
867 m_noOutEsc = 1;
868 } else {
869 outLen = 1;
870 UKWORD index = m_stdMap[stdChar];
871 if (!VnCharsetLibObj.m_options.viqrMixed && !m_noOutEsc &&
872 (stdChar == '\\' ||
873 (index > 0 && index <= 10 && m_escapeTone) ||
874 (index == 12 && m_escapeRoof) ||
875 (index == 24 && m_escapeBowl) ||
876 (index == 26 && m_escapeHook))) {
877 //(m_stdMap[stdChar] > 0 && m_stdMap[stdChar] <= 26)) {
878 // tone mark, needs an escape character
879 outLen++;
880 ret = os.putB('\\');
881 if (VnCharsetLibObj.m_VIQROutEscPatterns.foundAtNextChar(
882 '\\') != -1)
883 m_noOutEsc = 1;
884 }
885 b = (UKBYTE)stdChar;
886 ret = os.putB(b);
887 if (VnCharsetLibObj.m_VIQROutEscPatterns.foundAtNextChar(b) != -1)
888 m_noOutEsc = 1;
889 if (m_noOutEsc && (b == ' ' || b == '\t' || b == '\r' || b == '\n'))
890 m_noOutEsc = 0;
891 }
892 // reset escape marks
893 m_escapeBowl = 0;
894 m_escapeRoof = 0;
895 m_escapeHook = 0;
896 m_escapeTone = 0;
897 }
898 return ret;
899 }
900
901 /////////////////////////////////////////////
902 // Class: UTF8VIQRCharset //
903 /////////////////////////////////////////////
904
905 //-----------------------------------------
906 UTF8VIQRCharset::UTF8VIQRCharset(UnicodeUTF8Charset *pUtf, VIQRCharset *pViqr) {
907 m_pUtf = pUtf;
908 m_pViqr = pViqr;
909 }
910
911 //-----------------------------------------
912 void UTF8VIQRCharset::startInput() {
913 m_pUtf->startInput();
914 m_pViqr->startInput();
915 }
916
917 //-----------------------------------------
918 void UTF8VIQRCharset::startOutput() {
919 m_pUtf->startOutput();
920 m_pViqr->startOutput();
921 }
922
923 //-----------------------------------------
924 int UTF8VIQRCharset::nextInput(ByteInStream &is, StdVnChar &stdChar,
925 int &bytesRead) {
926 UKBYTE ch;
927
928 if (!is.peekNext(ch))
929 return 0;
930
931 if (ch > 0xBF && ch < 0xFE) {
932 m_pViqr->startInput(); // just to reset the VIQR object state
933 m_pViqr->m_suspicious = 1;
934 return m_pUtf->nextInput(is, stdChar, bytesRead);
935 }
936
937 return m_pViqr->nextInput(is, stdChar, bytesRead);
938 }
939
940 //-----------------------------------------
941 int UTF8VIQRCharset::putChar(ByteOutStream &os, StdVnChar stdChar,
942 int &outLen) {
943 return m_pViqr->putChar(os, stdChar, outLen);
944 }
945
946 //-----------------------------------------
947 CVnCharsetLib::CVnCharsetLib() {
948 unsigned char ch;
949 for (ch = 'a'; ch < 'z'; ch++)
950 LoVowel[ch - 'a'] = 0;
951 LoVowel['a' - 'a'] = 1;
952 LoVowel['e' - 'a'] = 1;
953 LoVowel['i' - 'a'] = 1;
954 LoVowel['o' - 'a'] = 1;
955 LoVowel['u' - 'a'] = 1;
956 LoVowel['y' - 'a'] = 1;
957
958 for (ch = 'A'; ch < 'Z'; ch++)
959 HiVowel[ch - 'A'] = 0;
960 HiVowel['A' - 'A'] = 1;
961 HiVowel['E' - 'A'] = 1;
962 HiVowel['I' - 'A'] = 1;
963 HiVowel['O' - 'A'] = 1;
964 HiVowel['U' - 'A'] = 1;
965 HiVowel['Y' - 'A'] = 1;
966
967 m_pUniCharset = NULL;
968 m_pUniCompCharset = NULL;
969 m_pUniUTF8 = NULL;
970 m_pUniRef = NULL;
971 m_pUniHex = NULL;
972 m_pVIQRCharObj = NULL;
973 m_pUVIQRCharObj = NULL;
974 m_pWinCP1258 = NULL;
975 m_pVnIntCharset = NULL;
976
977 int i;
978 for (i = 0; i < CONV_TOTAL_SINGLE_CHARSETS; i++)
979 m_sgCharsets[i] = NULL;
980
981 for (i = 0; i < CONV_TOTAL_DOUBLE_CHARSETS; i++)
982 m_dbCharsets[i] = NULL;
983
984 VnConvResetOptions(&m_options);
985 m_VIQREscPatterns.init((char **)VIQREscapes, VIQREscCount);
986 m_VIQROutEscPatterns.init((char **)VIQREscapes, VIQREscCount);
987 }
988
989 //-----------------------------------------
990 CVnCharsetLib::~CVnCharsetLib() {
991 if (m_pUniCharset)
992 delete m_pUniCharset;
993 if (m_pUniUTF8)
994 delete m_pUniUTF8;
995 if (m_pUniRef)
996 delete m_pUniRef;
997 if (m_pUniHex)
998 delete m_pUniHex;
999 if (m_pVIQRCharObj)
1000 delete m_pVIQRCharObj;
1001 if (m_pUVIQRCharObj)
1002 delete m_pUVIQRCharObj;
1003 if (m_pWinCP1258)
1004 delete m_pWinCP1258;
1005 if (m_pUniCString)
1006 delete m_pUniCString;
1007 if (m_pVnIntCharset)
1008 delete m_pVnIntCharset;
1009
1010 int i;
1011 for (i = 0; i < CONV_TOTAL_SINGLE_CHARSETS; i++)
1012 if (m_sgCharsets[i])
1013 delete m_sgCharsets[i];
1014
1015 for (i = 0; i < CONV_TOTAL_DOUBLE_CHARSETS; i++)
1016 if (m_dbCharsets[i])
1017 delete m_dbCharsets[i];
1018 }
1019
1020 //-----------------------------------------
1021 VnCharset *CVnCharsetLib::getVnCharset(int charsetIdx) {
1022 switch (charsetIdx) {
1023
1024 case CONV_CHARSET_UNICODE:
1025 if (m_pUniCharset == NULL)
1026 m_pUniCharset = new UnicodeCharset(UnicodeTable);
1027 return m_pUniCharset;
1028 case CONV_CHARSET_UNIDECOMPOSED:
1029 if (m_pUniCompCharset == NULL)
1030 m_pUniCompCharset =
1031 new UnicodeCompCharset(UnicodeTable, UnicodeComposite);
1032 return m_pUniCompCharset;
1033 case CONV_CHARSET_UNIUTF8:
1034 case CONV_CHARSET_XUTF8:
1035 if (m_pUniUTF8 == NULL)
1036 m_pUniUTF8 = new UnicodeUTF8Charset(UnicodeTable);
1037 return m_pUniUTF8;
1038
1039 case CONV_CHARSET_UNIREF:
1040 if (m_pUniRef == NULL)
1041 m_pUniRef = new UnicodeRefCharset(UnicodeTable);
1042 return m_pUniRef;
1043
1044 case CONV_CHARSET_UNIREF_HEX:
1045 if (m_pUniHex == NULL)
1046 m_pUniHex = new UnicodeHexCharset(UnicodeTable);
1047 return m_pUniHex;
1048
1049 case CONV_CHARSET_UNI_CSTRING:
1050 if (m_pUniCString == NULL)
1051 m_pUniCString = new UnicodeCStringCharset(UnicodeTable);
1052 return m_pUniCString;
1053
1054 case CONV_CHARSET_WINCP1258:
1055 if (m_pWinCP1258 == NULL)
1056 m_pWinCP1258 = new WinCP1258Charset(WinCP1258, WinCP1258Pre);
1057 return m_pWinCP1258;
1058
1059 case CONV_CHARSET_VIQR:
1060 if (m_pVIQRCharObj == NULL)
1061 m_pVIQRCharObj = new VIQRCharset(VIQRTable);
1062 return m_pVIQRCharObj;
1063
1064 case CONV_CHARSET_VNSTANDARD:
1065 if (m_pVnIntCharset == NULL)
1066 m_pVnIntCharset = new VnInternalCharset();
1067 return m_pVnIntCharset;
1068
1069 case CONV_CHARSET_UTF8VIQR:
1070 if (m_pUVIQRCharObj == NULL) {
1071 if (m_pVIQRCharObj == NULL)
1072 m_pVIQRCharObj = new VIQRCharset(VIQRTable);
1073
1074 if (m_pUniUTF8 == NULL)
1075 m_pUniUTF8 = new UnicodeUTF8Charset(UnicodeTable);
1076 m_pUVIQRCharObj = new UTF8VIQRCharset(m_pUniUTF8, m_pVIQRCharObj);
1077 }
1078 return m_pUVIQRCharObj;
1079
1080 default:
1081 if (IS_SINGLE_BYTE_CHARSET(charsetIdx)) {
1082 int i = charsetIdx - CONV_CHARSET_TCVN3;
1083 if (m_sgCharsets[i] == NULL)
1084 m_sgCharsets[i] = new SingleByteCharset(SingleByteTables[i]);
1085 return m_sgCharsets[i];
1086 } else if (IS_DOUBLE_BYTE_CHARSET(charsetIdx)) {
1087 int i = charsetIdx - CONV_CHARSET_VNIWIN;
1088 if (m_dbCharsets[i] == NULL)
1089 m_dbCharsets[i] = new DoubleByteCharset(DoubleByteTables[i]);
1090 return m_dbCharsets[i];
1091 }
1092 }
1093 return NULL;
1094 }
1095
1096 //-------------------------------------------------
1097 DllExport void VnConvSetOptions(VnConvOptions *pOptions) {
1098 VnCharsetLibObj.m_options = *pOptions;
1099 }
1100
1101 //-------------------------------------------------
1102 DllExport void VnConvGetOptions(VnConvOptions *pOptions) {
1103 *pOptions = VnCharsetLibObj.m_options;
1104 }
1105
1106 //-------------------------------------------------
1107 DllExport void VnConvResetOptions(VnConvOptions *pOptions) {
1108 pOptions->viqrEsc = 1;
1109 pOptions->viqrMixed = 0;
1110 pOptions->toUpper = 0;
1111 pOptions->toLower = 0;
1112 pOptions->removeTone = 0;
1113 pOptions->smartViqr = 1;
1114 }
1115
1116 /////////////////////////////////////////////
1117 // Class WinCP1258Charset
1118 /////////////////////////////////////////////
1119 WinCP1258Charset::WinCP1258Charset(UKWORD *compositeChars,
1120 UKWORD *precomposedChars) {
1121 int i, k;
1122 m_toDoubleChar = compositeChars;
1123 memset(m_stdMap, 0, 256 * sizeof(UKWORD));
1124
1125 // encode composite chars
1126 for (i = 0; i < TOTAL_VNCHARS; i++) {
1127 if (compositeChars[i] >> 8) // a 2-byte character
1128 m_stdMap[compositeChars[i] >> 8] = 0xFFFF; // INVALID_STD_CHAR;
1129 else if (m_stdMap[compositeChars[i]] == 0)
1130 m_stdMap[compositeChars[i]] = i + 1;
1131
1132 m_vnChars[i] = (i << 16) +
1133 compositeChars[i]; // high word is used for StdChar index
1134 }
1135
1136 m_totalChars = TOTAL_VNCHARS;
1137
1138 // add precomposed chars to the table
1139 for (k = 0, i = TOTAL_VNCHARS; k < TOTAL_VNCHARS; k++)
1140 if (precomposedChars[k] != compositeChars[k]) {
1141 if (precomposedChars[k] >> 8) // a 2-byte character
1142 m_stdMap[precomposedChars[k] >> 8] =
1143 0xFFFF; // INVALID_STD_CHAR;
1144 else if (m_stdMap[precomposedChars[k]] == 0)
1145 m_stdMap[precomposedChars[k]] = k + 1;
1146
1147 m_vnChars[i] = (k << 16) + precomposedChars[k];
1148 m_totalChars++;
1149 i++;
1150 }
1151
1152 qsort(m_vnChars, m_totalChars, sizeof(UKDWORD), wideCharCompare);
1153 }
1154
1155 //---------------------------------------------------------------------
1156 // This fuction is basically the same as that of DoubleByteCharset
1157 // with m_totalChars is used instead of constant TOTAL_VNCHARS
1158 //---------------------------------------------------------------------
1159 int WinCP1258Charset::nextInput(ByteInStream &is, StdVnChar &stdChar,
1160 int &bytesRead) {
1161 unsigned char ch;
1162
1163 // read first byte
1164 bytesRead = 0;
1165 if (!is.getNext(ch))
1166 return 0;
1167 bytesRead = 1;
1168 stdChar = m_stdMap[ch];
1169 if (stdChar == 0)
1170 stdChar = ch;
1171 else if (stdChar == 0xFFFF)
1172 stdChar = INVALID_STD_CHAR;
1173 else {
1174 stdChar += VnStdCharOffset - 1;
1175 UKBYTE hi;
1176 if (is.peekNext(hi) && hi > 0) {
1177 // test if a double-byte character is encountered
1178 UKDWORD key = MAKEWORD(ch, hi);
1179 UKDWORD *pChar =
1180 (UKDWORD *)bsearch(&key, m_vnChars, m_totalChars,
1181 sizeof(UKDWORD), wideCharCompare);
1182 if (pChar) {
1183 stdChar = VnStdCharOffset + HIWORD(*pChar);
1184 bytesRead = 2;
1185 is.getNext(hi);
1186 }
1187 }
1188 }
1189 return 1;
1190 }
1191
1192 //---------------------------------------------------------------------
1193 // This fuction is exactly the same as that of DoubleByteCharset
1194 //---------------------------------------------------------------------
1195 int WinCP1258Charset::putChar(ByteOutStream &os, StdVnChar stdChar,
1196 int &outLen) {
1197 int ret;
1198 if (stdChar >= VnStdCharOffset) {
1199 UKWORD wCh = m_toDoubleChar[stdChar - VnStdCharOffset];
1200
1201 if (wCh & 0xFF00) {
1202 outLen = 2;
1203 os.putB((UKBYTE)(wCh & 0x00FF));
1204 ret = os.putB((UKBYTE)(wCh >> 8));
1205 } else {
1206 unsigned char b = (unsigned char)wCh;
1207 if (m_stdMap[b] == 0xFFFF)
1208 b = PadChar;
1209 outLen = 1;
1210 ret = os.putB(b);
1211 }
1212 } else {
1213 if (stdChar > 255 || m_stdMap[stdChar]) {
1214 outLen = 1;
1215 ret = os.putB((UKBYTE)PadChar);
1216 } else {
1217 outLen = 1;
1218 ret = os.putB((UKBYTE)stdChar);
1219 }
1220 }
1221 return ret;
1222 }
1223
1224 #define IS_ODD(x) (x & 1)
1225 #define IS_EVEN(x) (!(x & 1))
1226
1227 StdVnChar StdVnToUpper(StdVnChar ch) {
1228 if (ch >= VnStdCharOffset && ch < (VnStdCharOffset + TOTAL_ALPHA_VNCHARS) &&
1229 IS_ODD(ch))
1230 ch -= 1;
1231 return ch;
1232 }
1233
1234 //----------------------------------------
1235 StdVnChar StdVnToLower(StdVnChar ch) {
1236 if (ch >= VnStdCharOffset && ch < (VnStdCharOffset + TOTAL_ALPHA_VNCHARS) &&
1237 IS_EVEN(ch))
1238 ch += 1;
1239 return ch;
1240 }
1241
1242 //----------------------------------------
1243 StdVnChar StdVnGetRoot(StdVnChar ch) {
1244 if (ch >= VnStdCharOffset && ch < VnStdCharOffset + TOTAL_VNCHARS)
1245 ch = VnStdCharOffset + StdVnRootChar[ch - VnStdCharOffset];
1246 return ch;
1247 }