Debian Codegraph @

src/af/xap/xp/xap_EncodingManager.cpp

raw
1 /* -*- mode: C++; tab-width: 4; c-basic-offset: 4; -*- */
2
3 /* AbiSource Application Framework
4 * Copyright (C) 2000
5 * Orignially by Vlad Harchev <[email protected]>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
20 * 02110-1301 USA.
21 */
22
23 #include "xap_EncodingManager.h"
24 #include "ut_debugmsg.h"
25 #include "ut_string.h"
26 #include "ut_string_class.h"
27 #include "ut_assert.h"
28 #include "ut_bijection.h"
29 #include "ut_iconv.h"
30
31 #include <stdio.h>
32 #include <string.h>
33
34 static UT_iconv_t iconv_handle_N2U = UT_ICONV_INVALID;
35 static UT_iconv_t iconv_handle_U2N = UT_ICONV_INVALID;
36 static UT_iconv_t iconv_handle_U2Latin1 = UT_ICONV_INVALID;
37 static UT_iconv_t iconv_handle_U2Win = UT_ICONV_INVALID;
38 static UT_iconv_t iconv_handle_Win2U = UT_ICONV_INVALID;
39
40 XAP_EncodingManager* XAP_EncodingManager::_instance = NULL;
41
42 /*!
43 * Returns the native encoding, no matter what
44 *
45 * If the OS supports a system locale and a user locale,
46 * this will be the user locale's encoding.
47 */
48 const char* XAP_EncodingManager::getNativeEncodingName() const
49 {
50 return "ISO-8859-1"; /* this will definitely work*/
51 }
52
53 /*!
54 * Returns the system's underlying native encoding, no matter what
55 *
56 * If the OS supports a system locale and a user locale,
57 * this will be the system locale's encoding.
58 */
59 const char* XAP_EncodingManager::getNativeSystemEncodingName() const
60 {
61 return getNativeEncodingName();
62 }
63
64 /*!
65 * Returns the native 8-bit encoding
66 *
67 * Always returns an 8-bit encoding, even when the native
68 * encoding is wide char like UCS-2 on Windows NT.
69 * This includes multibyte encodings such as UTF-8 and CJK encodings.
70 * Any encoding which inludes the ASCII range is OK.
71 */
72 const char* XAP_EncodingManager::getNative8BitEncodingName() const
73 {
74 return getNativeEncodingName();
75 }
76
77 /*!
78 * Returns the native non-Unicode encoding
79 *
80 * Always returns a non-Unicode encoding, even when the native
81 * encoding is Unicode like UCS-2 on Windows NT or UTF-8 on *nix.
82 */
83 const char* XAP_EncodingManager::getNativeNonUnicodeEncodingName() const
84 {
85 return getNativeEncodingName();
86 }
87
88 /*!
89 * Returns the native Unicode encoding
90 *
91 * Typically UTF-8 on *nix and UCS-2 on Windows NT
92 */
93 const char* XAP_EncodingManager::getNativeUnicodeEncodingName() const
94 {
95 return "UTF-8"; /* this will definitely work*/
96 }
97
98 static const char * UCS2BEName = 0;
99 static const char * UCS2LEName = 0;
100 static const char * UCS4BEName = 0;
101 static const char * UCS4LEName = 0;
102
103 /*!
104 * Returns the name this system uses for UCS-2, big endian
105 *
106 * UCS-2BE is standard but some older iconvs use UCS-2-BE
107 */
108 const char* XAP_EncodingManager::getUCS2BEName() const
109 {
110 return UCS2BEName;
111 }
112
113 /*!
114 * Returns the name this system uses for UCS-2, little endian
115 *
116 * UCS-2LE is standard but some older iconvs use UCS-2-LE
117 */
118 const char* XAP_EncodingManager::getUCS2LEName() const
119 {
120 return UCS2LEName;
121 }
122
123 /*!
124 * Returns the name this system uses for UCS-4, big endian
125 *
126 * UCS-4BE is standard
127 */
128 const char* XAP_EncodingManager::getUCS4BEName() const
129 {
130 return UCS4BEName;
131 }
132
133 /*!
134 * Returns the name this system uses for UCS-4, little endian
135 *
136 * UCS-4LE is standard
137 */
138 const char* XAP_EncodingManager::getUCS4LEName() const
139 {
140 return UCS4LEName;
141 }
142
143 XAP_EncodingManager::~XAP_EncodingManager()
144 {
145 UT_iconv_close(iconv_handle_N2U);
146 UT_iconv_close(iconv_handle_U2N);
147 UT_iconv_close(iconv_handle_U2Latin1);
148 UT_iconv_close(iconv_handle_U2Win);
149 UT_iconv_close(iconv_handle_Win2U);
150 }
151
152 XAP_EncodingManager::XAP_EncodingManager() { }
153
154 void XAP_EncodingManager::Delete_instance()
155 {
156 delete _instance;
157 _instance = NULL;
158 }
159
160 const char* XAP_EncodingManager::getLanguageISOName() const
161 {
162 return "en";
163 }
164
165 const char* XAP_EncodingManager::getLanguageISOTerritory() const
166 {
167 return NULL;
168 }
169
170 // TODO Do we need an equivalent function which can return
171 // TODO U+FFFD "REPLACEMENT CHARACTER" or U+25A0 "BLACK SQUARE"
172 // TODO for translating into Unicode?
173 char XAP_EncodingManager::fallbackChar(UT_UCSChar) const
174 {
175 return '?';
176 }
177
178 /* in fact this method should provide full implementation */
179
180 UT_uint32 XAP_EncodingManager::approximate(char* out,UT_uint32 max_length,UT_UCSChar c) const
181 {
182 if (max_length==0)
183 return 0;
184 if (max_length==1)
185 {
186 switch (c)
187 {
188 case 0x201d:
189 case 0x201c:
190 *out = '"'; return 1;
191 default:
192 return 0;
193 }
194 }
195 else
196 {
197 /*
198 this case can't happen with current code, so there is no
199 proper implementation.
200 */
201 }
202 return 0;
203 }
204
205 UT_UCSChar XAP_EncodingManager::nativeToU(UT_UCSChar c) const
206 {
207 UT_UCSChar ret = try_nativeToU(c);
208 return ret ? ret : (UT_UCSChar)fallbackChar(c);
209 }
210
211 UT_UCSChar XAP_EncodingManager::UToNative(UT_UCSChar c) const
212 {
213 UT_UCSChar ret = try_UToNative(c);
214 if (!ret || ret>0xff)
215 {
216 char repl;
217 int repl_len = approximate(&repl,1,c);
218 return repl_len == 1? repl : fallbackChar(c);
219 }
220 else
221 return ret;
222 }
223
224 UT_UCSChar XAP_EncodingManager::WindowsToU(UT_UCSChar c) const
225 {
226 UT_UCSChar ret = try_WindowsToU(c);
227 return ret ? ret : (UT_UCSChar)fallbackChar(c);
228 }
229
230
231 UT_UCSChar XAP_EncodingManager::UToWindows(UT_UCSChar c) const
232 {
233 UT_UCSChar ret = try_UToWindows(c);
234 return ret && ret<=0xff ? ret : (UT_UCSChar)fallbackChar(c);
235 }
236
237
238 const char* XAP_EncodingManager::strToNative(const char* in,const char* charset, bool bReverse, bool bUseSysEncoding) const
239 {
240 static char buf[500];
241 return strToNative(in, charset, buf, sizeof(buf), bReverse, bUseSysEncoding);
242 }
243
244 const char* XAP_EncodingManager::strToNative(const char* in, const char* charset, char* buf, int bufsz,
245 bool bReverse, bool bUseSysEncoding) const
246 {
247 if (!charset || !*charset || !in || !*in || !buf)
248 return in; /*won't translate*/
249 #if 1
250 // TODO this gets around the fact that gtk 1.2 cannot handle UTF-8 input
251 // when we move to gtk 2 this original branch needs to be enabled
252 UT_iconv_t iconv_handle;
253 if (!bReverse)
254 iconv_handle = UT_iconv_open(
255 bUseSysEncoding ? getNativeSystemEncodingName() : getNativeEncodingName(), charset);
256 else
257 iconv_handle = UT_iconv_open(
258 charset, bUseSysEncoding ? getNativeSystemEncodingName() : getNativeEncodingName());
259
260 #else
261 UT_iconv_t iconv_handle;
262 const char * pNative = bUseSysEncoding ? getNativeSystemEncodingName() : getNativeEncodingName();
263
264 if(!g_ascii_strcasecmp(pNative, "UTF-8"))
265 pNative = getNativeNonUnicodeEncodingName();
266
267 iconv_handle = UT_iconv_open(pNative, charset);
268 xxx_UT_DEBUGMSG(("xap_EncodingManager::strToNative: pNative %s, iconv_handle 0x%x\n",pNative, iconv_handle));
269 #endif
270
271 if (!UT_iconv_isValid(iconv_handle))
272 return in;
273
274 const char* inptr = in;
275 char* outptr = buf;
276 size_t inbytes = strlen(in);
277 size_t outbytes = bufsz;
278 size_t donecnt = UT_iconv(iconv_handle, &inptr, &inbytes, &outptr, &outbytes);
279 const char* retstr = in;
280
281 if (donecnt != (size_t) -1 && inbytes == 0)
282 {
283 retstr = buf;
284 buf[bufsz - outbytes] = '\0';/*for sure*/
285 }
286
287 UT_iconv_close(iconv_handle);
288 return retstr;
289 }
290
291 int XAP_EncodingManager::XAP_XML_UnknownEncodingHandler(void* /*encodingHandlerData*/,
292 const gchar *name,
293 XML_Encoding *info)
294 {
295 #ifndef HAVE_EXPAT
296 UT_UNUSED(name);
297 UT_UNUSED(info);
298 return 0;
299 #else
300 /*this is used by code that reads xml using expat*/
301 if (get_instance()->cjk_locale())
302 return 0;/*this handler doesn't support multibyte encodings*/
303 UT_iconv_t iconv_handle = UT_iconv_open("UCS-2",name);
304 if (!UT_iconv_isValid(iconv_handle))
305 return 0;
306 info->convert = NULL;
307 info->release = NULL;
308 {
309 char ibuf[1],obuf[2];
310 for(int i=0;i<256;++i)
311 {
312 size_t ibuflen = 1, obuflen=2;
313 const char* iptr = ibuf;
314 char* optr = obuf;
315 ibuf[0] = static_cast<unsigned char>(i);
316 size_t donecnt = UT_iconv(iconv_handle,&iptr,&ibuflen,&optr,&obuflen);
317 if (donecnt!=(size_t)-1 && ibuflen==0)
318 {
319 unsigned short uval;
320 unsigned short b0 = static_cast<unsigned char>(obuf[swap_stou]);
321 unsigned short b1 = static_cast<unsigned char>(obuf[!swap_stou]);
322 uval = b0 | (b1<<8);
323 info->map[i] = static_cast<unsigned int>(uval);
324 }
325 else
326 info->map[i] = -1;/* malformed character. Such cases exist - e.g. 0x98 in cp1251*/
327
328 }
329 }
330 UT_iconv_close(iconv_handle);
331 return 1;
332 #endif
333 }
334
335 extern "C" { char *wvLIDToCodePageConverter(unsigned short lid); }
336
337 static void init_values(const XAP_EncodingManager* that)
338 {
339 const char * ucs4i = ucs4Internal ();
340 const char * naten = that->getNativeEncodingName ();
341
342 iconv_handle_N2U = UT_iconv_open (ucs4i, naten);
343 if (!UT_iconv_isValid(iconv_handle_N2U))
344 {
345 UT_DEBUGMSG(("WARNING: UT_iconv_open(%s,%s) failed!\n",ucs4i,naten));
346 }
347 iconv_handle_U2N = UT_iconv_open (naten, ucs4i);
348 if (!UT_iconv_isValid(iconv_handle_U2N))
349 {
350 UT_DEBUGMSG(("WARNING: UT_iconv_open(%s,%s) failed!\n",naten,ucs4i));
351 }
352 iconv_handle_U2Latin1 = UT_iconv_open ("ISO-8859-1", ucs4i);
353 if (!UT_iconv_isValid(iconv_handle_U2Latin1))
354 {
355 UT_DEBUGMSG(("WARNING: UT_iconv_open(ISO-8859-1,%s) failed!\n",ucs4i));
356 }
357
358 char* winencname = wvLIDToCodePageConverter(that->getWinLanguageCode());
359 iconv_handle_Win2U = UT_iconv_open(ucs4Internal(),winencname);
360 iconv_handle_U2Win = UT_iconv_open(winencname,ucs4Internal());
361 }
362
363
364 static UT_UCSChar try_CToU(UT_UCSChar c,UT_iconv_t iconv_handle)
365 {
366 if (!UT_iconv_isValid (iconv_handle)) return 0;
367 UT_iconv_reset (iconv_handle);
368
369 /* We don't support multibyte chars yet. wcstombcs should be used.
370 * NOTE: if so, then we shouldn't be using this function at all! [TODO: ugh.]
371 */
372 if (c > 255)
373 {
374 UT_DEBUGMSG(("WARNING: character code %x received, substituting 'E'\n", c));
375 c = 'E'; // was return 0, but that can be dangerous...
376 }
377
378 char ibuf[1];
379 char obuf[4];
380 size_t ibuflen = 1;
381 size_t obuflen = 4;
382 const char * iptr = ibuf;
383 char * optr = obuf;
384
385 ibuf[0] = static_cast<unsigned char>(c);
386
387 size_t donecnt = UT_iconv(iconv_handle,&iptr,&ibuflen,&optr,&obuflen);
388
389 UT_UCSChar uval = 0;
390
391 if (donecnt!=(size_t)-1 && ibuflen==0)
392 {
393 if (XAP_EncodingManager::swap_stou)
394 {
395 uval = static_cast<UT_UCSChar>(static_cast<unsigned char>(obuf[3]));
396 uval = static_cast<UT_UCSChar>(static_cast<unsigned char>(obuf[2])) | (uval<<8);
397 uval = static_cast<UT_UCSChar>(static_cast<unsigned char>(obuf[1])) | (uval<<8);
398 uval = static_cast<UT_UCSChar>(static_cast<unsigned char>(obuf[0])) | (uval<<8);
399 }
400 else
401 {
402 uval = static_cast<UT_UCSChar>(static_cast<unsigned char>(obuf[0]));
403 uval = static_cast<UT_UCSChar>(static_cast<unsigned char>(obuf[1])) | (uval<<8);
404 uval = static_cast<UT_UCSChar>(static_cast<unsigned char>(obuf[2])) | (uval<<8);
405 uval = static_cast<UT_UCSChar>(static_cast<unsigned char>(obuf[3])) | (uval<<8);
406 }
407 }
408 return uval;
409 }
410
411 static UT_UCSChar try_UToC(UT_UCSChar c,UT_iconv_t iconv_handle)
412 {
413 if (!UT_iconv_isValid(iconv_handle)) return 0;
414 UT_iconv_reset (iconv_handle);
415
416 char ibuf[4];
417 char obuf[6];
418 size_t ibuflen = 4;
419 size_t obuflen = 6;
420 const char * iptr = ibuf;
421 char * optr = obuf;
422
423 if (XAP_EncodingManager::swap_utos)
424 {
425 ibuf[0] = static_cast<char>((c ) & 0xff);
426 ibuf[1] = static_cast<char>((c >> 8) & 0xff);
427 ibuf[2] = static_cast<char>((c >> 16) & 0xff);
428 ibuf[3] = static_cast<char>((c >> 24) & 0xff);
429 }
430 else
431 {
432 ibuf[0] = static_cast<char>((c >> 24) & 0xff);
433 ibuf[1] = static_cast<char>((c >> 16) & 0xff);
434 ibuf[2] = static_cast<char>((c >> 8) & 0xff);
435 ibuf[3] = static_cast<char>((c ) & 0xff);
436 }
437
438 size_t donecnt = UT_iconv (iconv_handle,&iptr,&ibuflen,&optr,&obuflen);
439
440 UT_UCSChar byte = 0;
441
442 if (donecnt!=(size_t)-1 && ibuflen==0)
443 {
444 if (obuflen != 5) // grr... [TODO: ugh.]
445 {
446 UT_DEBUGMSG(("WARNING: character code %x received, substituting 'E'\n", c));
447 byte = static_cast<UT_UCSChar>(static_cast<unsigned char>('E'));
448 }
449 else byte = static_cast<UT_UCSChar>(static_cast<unsigned char>(obuf[0]));
450 }
451 return byte;
452 }
453
454 UT_UCSChar XAP_EncodingManager::try_nativeToU(UT_UCSChar c) const
455 {
456 return try_CToU(c,iconv_handle_N2U);
457 }
458
459 UT_UCSChar XAP_EncodingManager::try_UToNative(UT_UCSChar c) const
460 {
461 return try_UToC(c,iconv_handle_U2N);
462 }
463
464 UT_UCSChar XAP_EncodingManager::try_UToLatin1(UT_UCSChar c) const
465 {
466 return try_UToC(c,iconv_handle_U2Latin1);
467 }
468
469 UT_UCSChar XAP_EncodingManager::try_WindowsToU(UT_UCSChar c) const
470 {
471 return try_CToU(c,iconv_handle_Win2U);
472 }
473
474 UT_UCSChar XAP_EncodingManager::try_UToWindows(UT_UCSChar c) const
475 {
476 return try_UToC(c,iconv_handle_U2Win);
477 }
478
479 /*reverse map*/
480 struct _rmap
481 {
482 const char* value;//it can't be NULL for non-special entries like last or first
483 const char** keys;//NULL-teminated array of strings
484 };
485
486 static const char* search_rmap(const _rmap* m,const char* key,bool* is_default = NULL)
487 {
488 const _rmap* cur = m+1;
489 if (is_default)
490 *is_default = false;
491 for(;cur->value;++cur)
492 {
493 if (!cur->keys)
494 {
495 if (!g_ascii_strcasecmp(cur->value,key))
496 return cur->value;
497 else
498 continue;
499 };
500 const char** curkey = cur->keys;
501 for(;*curkey;++curkey)
502 if (!g_ascii_strcasecmp(*curkey,key))
503 return cur->value;
504 }
505 if (is_default)
506 *is_default = true;
507 return m->value;
508 }
509
510 static const char* search_rmap_with_opt_suffix(const _rmap* m,const char* key,const char* fallback_key=NULL,const char* fallback_key_final=NULL)
511 {
512 bool is_default;
513 const char* value = search_rmap(m,key,&is_default);
514 if (!is_default || !fallback_key)
515 return value;
516 return search_rmap_with_opt_suffix(m,fallback_key,fallback_key_final);
517 }
518
519
520 struct _map
521 {
522 const char* key;
523 const char* value;
524 };
525 static const char* search_map(const _map* m,const char* key,bool* is_default = NULL)
526 {
527 const _map* cur = m+1;
528 if (is_default)
529 *is_default = false;
530 for(;cur->key;++cur)
531 if (!g_ascii_strcasecmp(cur->key,key))
532 return cur->value;
533 if (is_default)
534 *is_default = true;
535 return m->value;
536 }
537
538 static const char* search_map_with_opt_suffix(const _map* m,const char* key,const char* fallback_key=NULL,const char* fallback_key_final=NULL)
539 {
540 bool is_default;
541 const char* value = search_map(m,key,&is_default);
542 if (!is_default || !fallback_key)
543 return value;
544 return search_map_with_opt_suffix(m,fallback_key,fallback_key_final);
545 }
546
547 /* ************************* here begin tables *************************/
548
549 /* this array describes mapping form current encoding to Tex's encoding name.
550 The 1st entry is default value.
551 If the 'keys' field is NULL, then the 'value' of that entry should be used
552 (in the same case) if searched key matches the value.
553 */
554 static const char* texenc_iso88595[] = { "iso8859-5", NULL };
555 static const _rmap native_tex_enc_map[]=
556 {
557 { NULL, NULL }, /* the 1st item tells default value */
558 { "koi8-r", NULL }, { "koi8-ru", NULL },
559 { "koi8-u", NULL }, { "cp1251", NULL }, { "cp866", NULL },
560 { "iso88595", texenc_iso88595},
561 { NULL, NULL } /*last entry has NULL as 'value'*/
562 };
563
564
565 static const _map langcode_to_babelarg[]=
566 {
567 {NULL,NULL},
568 {"ru","english,russian"},
569
570 /* I'm not sure that this is correct, but my teTeX 0.9.17 works only
571 this way (i.e. only with "russian" in the middle) - hvv */
572 {"uk","english,russian,ukrainian"},
573
574 /* I'm not sure again - my teTeX 0.9.17 doesn't know 'byelorussian'
575 language - hvv */
576 {"be","english,russian"},
577 {NULL,NULL}
578 };
579
580
581 /*
582 Mapping from langcode to windows charset code.
583 See wingdi.h for charset codes windows defines, macros *_CHARSET (e.g.
584 RUSSIAN_CHARSET).
585 */
586 static const char* wincharsetcode_ru[]= /* russian charset */
587 { "ru", "be", "uk" , NULL };
588 static const char* wincharsetcode_el[]= /* greek charset*/
589 { "el", NULL };
590
591 static const char* wincharsetcode_tr[]= /* turkish charset*/
592 { "tr", NULL };
593
594 static const char* wincharsetcode_vi[]= /* vietnamese charset*/
595 { "vi", NULL };
596
597 static const char* wincharsetcode_th[]= /* thai charset*/
598 { "th", NULL };
599
600 /*I'm not sure that charset code is the same for Big5 and GB2312.
601 Tested with GB2312 only.
602 */
603 static const char* wincharsetcode_zh_GB2312[]= /* chinese*/
604 { "zh_CN.GB2312", "zh_CN.GBK", "zh_CN.GB18030", NULL };
605
606 static const char* wincharsetcode_zh_BIG5[]= /* chinese*/
607 { "zh_TW.BIG5", "zh_HK.BIG5-HKSCS", NULL };
608
609 static const _rmap langcode_to_wincharsetcode[]=
610 {
611 {"0", NULL }, /* default value - ansi charset*/
612 {"204",wincharsetcode_ru},
613 {"161",wincharsetcode_el},
614 {"162",wincharsetcode_tr},
615 {"163",wincharsetcode_vi},
616 {"222",wincharsetcode_th},
617 {"134",wincharsetcode_zh_GB2312},
618 {"136",wincharsetcode_zh_BIG5},
619 {NULL, NULL}
620 };
621
622 static const UT_Bijection::pair_data zh_TW_big5[]=
623 {
624 /*
625 This data was constructed from the HJ's patch for support of Big5 to
626 AW-0.7.10 - VH
627 */
628 {"song","\xe5\xae\x8b\xe4\xbd\x93"},
629 {"fangsong","\xe4\xbb\xbf\xe5\xae\x8b"},
630 {"hei","\xe9\xbb\x91\xe4\xbd\x93"},
631 {"kai","\xe6\xa5\xb7\xe4\xbd\x93"},
632 {NULL,NULL}
633 };
634
635 static const char* zh_TW_big5_keys[]=
636 { "zh_TW.BIG5", NULL };
637
638 static const _rmap cjk_word_fontname_mapping_data[]=
639 {
640 {NULL, NULL},
641 {reinterpret_cast<const char*>(zh_TW_big5),zh_TW_big5_keys},
642 {NULL, NULL}
643 };
644
645
646 /*all CJK language codes should be listed here to be marked as CJK*/
647 static const char* cjk_languages[]=
648 { "zh", "ja", "ko", NULL };
649
650 static const _rmap langcode_to_cjk[]=
651 {
652 {"0", NULL}, /* default value - non-CJK environment */
653 {"1",cjk_languages},
654 {NULL, NULL}
655 };
656
657
658
659 /*
660 This table is useful since some iconv implementations don't know some cpNNNN
661 charsets but under some different name.
662 */
663 static const _map MSCodepagename_to_charset_name_map[]=
664 {
665 /*key, value*/
666 { NULL, NULL },
667
668 // { "CP0", ?? }, // ANSI code page
669 // { "CP1", ?? }, // OEM code page
670 // { "CP2", ?? }, // Macintosh code page
671 // { "CP3", ?? }, // The current thread's ANSI code page (Win2k)
672
673 // { "CP42", ?? }, // Symbol code page (Win2k)
674
675 // { "CP037", ?? }, // EBCDIC
676
677 { "CP437", "CP437" }, // MS-DOS United States
678
679 // { "CP500", ?? }, // EBCDIC "500V1"
680
681 { "CP708", "ASMO-708" }, // Arabic (ASMO 708)
682 // { "CP709", ?? }, // Arabic (ASMO 449+, BCON V4)
683 // { "CP710", ?? }, // Arabic (Transparent Arabic)
684 // { "CP720", ?? }, // Arabic (Transparent ASMO)
685
686 { "CP737", "CP737" }, // Greek (formerly 437G)
687 { "CP775", "CP775" }, // Baltic
688 { "CP850", "CP850" }, // MS-DOS Multilingual (Latin I)
689 { "CP852", "CP852" }, // MS-DOS Slavic (Latin II)
690 { "CP855", "CP855" }, // IBM Cyrillic (primarily Russian)
691 { "CP857", "CP857" }, // IBM Turkish
692 { "CP860", "CP861" }, // MS-DOS Portuguese
693 { "CP861", "CP861" }, // MS-DOS Icelandic
694 { "CP862", "CP862" }, // Hebrew
695 { "CP863", "CP863" }, // MS-DOS Canadian-French
696 { "CP864", "CP864" }, // Arabic
697 { "CP865", "CP865" }, // MS-DOS Nordic
698 { "CP866", "CP866" }, // MS-DOS Russian
699 { "CP869", "CP869" }, // IBM Modern Greek
700 { "CP874", "CP874" }, // Thai
701 // { "CP875", ?? }, // EBCDIC
702
703 { "CP932", "SJIS" }, // Japanese
704 { "CP936", "GBK" }, // Chinese (PRC, Singapore)
705 { "CP949", "CP949" }, // Korean
706 { "CP950", "BIG5" }, // Chinese (Taiwan; Hong Kong SAR, PRC)
707
708 // { "CP1026", ?? }, // EBCDIC
709
710 { "CP1200", "UCS-2" }, // Unicode (BMP of ISO 10646) // ??
711
712 { "CP1250", "CP1250" }, // Windows 3.1 Eastern European
713 { "CP1251", "CP1251" }, // Windows 3.1 Cyrillic
714 { "CP1252", "CP1252" }, // Windows 3.1 US (ANSI)
715 { "CP1253", "CP1253" }, // Windows 3.1 Greek
716 { "CP1254", "CP1254" }, // Windows 3.1 Turkish
717 { "CP1255", "CP1255" }, // Hebrew
718 { "CP1256", "CP1256" }, // Arabic
719 { "CP1257", "CP1257" }, // Baltic
720
721 { "CP1361", "JOHAB" }, // Korean (Johab)
722
723 { "CP10000", "MACINTOSH" }, // Macintosh Roman (alias name can be MACROMAN,
724 // but iconv from glibc seems to not have clue
725 { "CP10001", "SJIS" }, // Macintosh Japanese // ??
726 { "CP10006", "MACGREEK" }, // Macintosh Greek I
727 { "CP10007", "MACCYRILLIC" }, // Macintosh Cyrillic
728 { "CP10029", "MACCENTRALEUROPE" }, // Macintosh Latin 2
729 { "CP10079", "MACICELAND" }, // Macintosh Icelandic
730 { "CP10081", "MACTURKISH" }, // Macintosh Turkish
731
732 { "CP65000", "UTF-7" }, // Translate using UTF-7 (Win2k, NT 4.0) // ??
733 { "CP65001", "UTF-8" }, // Translate using UTF-8 (Win2k, NT 4.0) // ??
734
735 { NULL, NULL }
736 };
737
738 /*
739 This table is only concern CJK RTF part. It is a reverse table of
740 MSCodepagename_to_charset_name_map. Iconv doesn't know some cpNNNN,
741 but M$Word knows.
742 */
743 static const _map charset_name_to_MSCodepagename_map[]=
744 {
745 /*key,value*/
746 {NULL,NULL},
747 // libiconv also lists "SHIFT_JIS", "SHIFT-JIS", "MS_KANJI", "csShiftJIS"
748 {"SJIS","CP932"},
749 {"GB2312","CP936"},
750 {"GBK","CP936"},
751 {"GB18030","CP936"},
752 {"BIG5","CP950"},
753 {"BIG5-HKSCS","CP950"},
754 {"JOHAB","CP1361"},
755 {NULL,NULL}
756 };
757
758 /*warning: 0x400 won't be added to the values in this table. */
759 static const _map langcode_to_winlangcode[]=
760 {
761 /*key, value*/
762 {NULL, NULL},
763 {"zh_CN.GB2312", "0x804"},
764 {"zh_CN.GBK", "0x804"},
765 {"zh_CN.GB18030", "0x804"},
766 {"zh_CN.UTF-8", "0x804"},
767 {"zh_HK.BIG5-HKSCS", "0x404"},
768 {"zh_HK.UTF-8", "0x404"},
769 {"zh_TW.BIG5", "0x404"},
770 {"zh_TW.UTF-8", "0x404"},
771 {NULL, NULL}
772 };
773
774 #undef v
775 #define v(x) #x
776 static const char* non_cjk_fontsizes[]=
777 {
778 /* this is a list of sizes AW 0.7.11 had */
779 v(8),v(9),v(10),v(11),v(12),v(14),v(16),v(18),v(20),v(22),v(24),v(26),
780 v(28),v(36),v(48),v(72),NULL
781 };
782
783 static const char* cjk_fontsizes[]=
784 {
785 /* this list of font sizes was in HJ's Big5 patch to AW-0.7.10 */
786 v(5),v(5.5),v(6.5),v(7.5),v(9),v(10.5),v(12),v(14),v(15),v(16),v(18),
787 v(22),v(24),v(26),v(36),v(42),NULL
788 };
789 #undef v
790
791 /* This structure is built from
792 http://www.unicode.org/unicode/onlinedat/languages.html
793 using lynx, sed and hands of VH.
794 */
795 const XAP_SmartQuoteStyle XAP_EncodingManager::smartQuoteStyles[] =
796 {
797 { UCS_LDBLQUOTE, UCS_RDBLQUOTE }, // 0 English double quotes
798 { UCS_LQUOTE, UCS_RQUOTE }, // 1 English single quotes
799 { ((UT_UCSChar)0x00ab), ((UT_UCSChar)0x00bb) }, // 2 French double
800 { ((UT_UCSChar)0x00bb), ((UT_UCSChar)0x00ab) }, // 3 Danish double
801 { ((UT_UCSChar)0x00bb), ((UT_UCSChar)0x00bb) }, // 4
802 { UCS_LQUOTE, ((UT_UCSChar)0x201a) }, // 5
803 { UCS_RQUOTE, UCS_RQUOTE }, // 6
804 { ((UT_UCSChar)0x201a), UCS_LQUOTE }, // 7 German single
805 { ((UT_UCSChar)0x201a), UCS_RQUOTE }, // 8
806 { UCS_LDBLQUOTE, ((UT_UCSChar)0x201e) }, // 9
807 { UCS_RDBLQUOTE, UCS_RDBLQUOTE }, // 10
808 { ((UT_UCSChar)0x201e), UCS_RDBLQUOTE }, // 11
809 { ((UT_UCSChar)0x201e), UCS_LDBLQUOTE }, // 12 German double
810 { ((UT_UCSChar)0x2039), ((UT_UCSChar)0x203a) }, // 13 French single
811 { ((UT_UCSChar)0x203a), ((UT_UCSChar)0x2039) }, // 14 reverse French single
812 { ((UT_UCSChar)0x300c), ((UT_UCSChar)0x300d) }, // 15 Dark corner bracket
813 { ((UT_UCSChar)0x300e), ((UT_UCSChar)0x300f) }, // 16 White corner bracket
814 { UCS_RDBLQUOTE, UCS_LDBLQUOTE }, // 17 - Same as English, but RTL for Hebrew
815 { '\"', '\"' }, // 18 - ASCII double quote
816 { '\'', '\'' }, // 19 - ASCII single quote
817 { 0, 0 } // End of the list
818 };
819
820 /* This structure is built from
821 http://www.unicode.org/unicode/onlinedat/languages.html
822 using lynx, sed and hands of VH.
823 */
824 const XAP_LangInfo XAP_EncodingManager::langinfo[] =
825 {
826 {{ "Abkhazian", "ab", "", "", "", "", }, 0, 1 },
827 {{ "Afar", "aa", "", "", "", "", }, 0, 1 },
828 {{ "Afrikaans", "af", "", "0x0036", "", "", }, 12, 8 },
829 {{ "Albanian", "sq", "", "0x001c", "langAlbanian", "36", }, 2, 13 },
830 {{ "Amharic", "am", "", "", "langAmharic", "85", }, 0, 1 },
831 {{ "Arabic", "ar", "", "0x0001", "langArabic", "12", }, 0, 1 },
832 {{ "Armenian", "hy", "", "", "langArmenian", "51", }, 0, 1 },
833 {{ "Assamese", "as", "", "", "langAssamese", "68", }, 0, 1 },
834 {{ "Aymara", "ay", "", "", "langAymara", "134", }, 0, 1 },
835 {{ "Azerbaijani", "az", "", "", "langAzerbaijani(Latin)", "49", }, 0, 1 },
836 {{ "Azerbaijani", "az", "", "", "langAzerbaijanAr(Arabic)", "50", }, 0, 1 },
837 {{ "Bashkir", "ba", "", "", "", "", }, 0, 1 },
838 {{ "Basque", "eu", "", "0x002d", "langBasque", "129", }, 0, 1 },
839 {{ "Bengali", "bn", "", "", "langBengali", "67", }, 0, 1 },
840 {{ "Bangla", "bn", "", "", "langBengali", "67", }, 0, 1 },
841 {{ "Bhutani", "dz", "", "", "langDzongkha", "137", }, 0, 1 },
842 {{ "Bihari", "bh", "", "", "", "", }, 0, 1 },
843 {{ "Bislama", "bi", "", "", "", "", }, 0, 1 },
844 {{ "Breton", "br", "", "", "langBreton", "142", }, 0, 1 },
845 {{ "Bulgarian", "bg", "", "0x0002", "langBulgarian", "44", }, 11, 1 },
846 {{ "Burmese", "my", "", "", "langBurmese", "77", }, 0, 1 },
847 {{ "Byelorussian", "be", "", "0x0023", "langByelorussian", "46", }, 2, 1 },
848 {{ "Cambodian", "km", "", "", "langKhmer", "78", }, 0, 1 },
849 {{ "Catalan", "ca", "", "0x0003", "langCatalan", "130", }, 2, 0 },
850 {{ "Chewa", "", "", "", "langChewa", "92", }, 0, 1 },
851 {{ "Chinese", "zh", "", "0x0004", "langTradChinese", "19", }, 15, 16 },
852 {{ "Chinese", "zh", "", "0x0004", "langSimpChinese", "33", }, 0, 1 },
853 {{ "Cornish", "kw", "", "", "", "", }, 0, 1 }, // Jordi 19/10/2002
854 {{ "Corsican", "co", "", "", "", "", }, 0, 1 },
855 {{ "Croatian", "hr", "", "0x001a", "langCroatian", "18", }, 11, 1 },
856 {{ "Czech", "cs", "", "0x0005", "langCzech", "38", }, 11, 7 },
857 {{ "Danish", "da", "", "0x0006", "langDanish", "7", }, 3, 14 },
858 {{ "Dutch", "nl", "", "0x0013", "langDutch", "4", }, 12, 8 },
859 {{ "English", "en", "", "0x0009", "langEnglish", "0", }, 0, 1 },
860 {{ "Esperanto", "eo", "", "", "langEsperanto", "94", }, 0, 1 },
861 {{ "Estonian", "et", "", "0x0025", "langEstonian", "27", }, 11, 19 },
862 {{ "Faeroese", "fo", "", "0x0038", "langFaeroese", "30", }, 0, 1 },
863 {{ "Farsi", "fa", "", "0x0029", "langFarsi", "31", }, 0, 1 },
864 {{ "Farsi", "fa", "", "0x0029", "langPersian", "31", }, 0, 1 },
865 {{ "Fiji", "fj", "", "", "", "", }, 0, 1 },
866 {{ "Finnish", "fi", "", "0x000b", "langFinnish", "13", }, 10, 6 },
867 {{ "Flemish", "", "", "", "langFlemish", "34", }, 0, 1 },
868 {{ "French", "fr", "", "0x000c", "langFrench", "1", }, 2, 1 },
869 {{ "French", "fr", "CH", "0x000c", "langFrench", "1", }, 2, 13 },
870 {{ "Frisian", "fy", "", "", "", "", }, 0, 1 },
871 {{ "Galician", "gl", "", "", "", "", }, 0, 1 },
872 {{ "Galla", "", "", "", "langGalla", "87", }, 0, 1 },
873 {{ "Georgian", "ka", "", "", "langGeorgian", "52", }, 0, 1 },
874 {{ "German", "de", "", "0x0007", "langGerman", "2", }, 12, 7 },
875 {{ "German", "de", "CH", "0x0007", "langGerman", "2", }, 2, 13 }, // Swiss
876 {{ "Greek", "el", "", "0x0008", "langGreek", "14", }, 2, 19 },
877 {{ "Greenlandic", "kl", "", "", "", "", }, 0, 1 },
878 {{ "Guarani", "gn", "", "", "langGuarani", "133", }, 0, 1 },
879 {{ "Gujarati", "gu", "", "", "langGujarati", "69", }, 0, 1 },
880 {{ "Hausa", "ha", "", "", "", "", }, 0, 1 },
881 {{ "Hebrew", "he", "", "0x000d", "langHebrew", "10", }, 17, 1 },
882 {{ "Hebrew", "iw", "", "0x000d", "langHebrew", "10", }, 17, 1 },
883 {{ "Hindi", "hi", "", "0x0039", "langHindi", "21", }, 0, 1 },
884 {{ "Hungarian", "hu", "", "0x000e", "langHungarian", "26", }, 12, 3 },
885 {{ "Icelandic", "is", "", "0x000f", "langIcelandic", "15", }, 11, 7 },
886 {{ "Indonesian", "in", "", "0x0021", "langIndonesian", "81", }, 0, 1 },
887 {{ "Indonesian", "id", "", "0x0021", "langIndonesian", "81", }, 0, 1 },
888 {{ "Interlingua", "ia", "", "", "", "", }, 0, 1 },
889 {{ "Interlingue", "ie", "", "", "", "", }, 0, 1 },
890 {{ "Inuktitut", "iu", "", "", "langInuktitut", "143", }, 0, 1 },
891 {{ "Inupiak", "ik", "", "", "", "", }, 0, 1 },
892 {{ "Irish", "ga", "", "", "langIrish", "35", }, 0, 1 },
893 {{ "Italian", "it", "", "0x0010", "langItalian", "3", }, 2, 19 },
894 {{ "Italian", "it", "CH", "0x0010", "langItalian", "3", }, 2, 13 },
895 {{ "Japanese", "ja", "", "0x0011", "langJapanese", "11", }, 15, 16 },
896 {{ "Javanese", "jw", "", "", "langJavaneseRom", "138", }, 0, 1 },
897 {{ "Kannada", "kn", "", "", "langKannada", "73", }, 0, 1 },
898 {{ "Kashmiri", "ks", "", "", "langKashmiri", "61", }, 0, 1 },
899 {{ "Kazakh", "kk", "", "", "langKazakh", "48", }, 0, 1 },
900 {{ "Kinyarwanda", "rw", "", "", "", "", }, 0, 1 },
901 {{ "Kirghiz", "ky", "", "", "langKirghiz", "54", }, 0, 1 },
902 {{ "Kirundi", "rn", "", "", "", "", }, 0, 1 },
903 {{ "Korean", "ko", "", "0x0012", "langKorean", "23", }, 0, 1 },
904 {{ "Kurdish", "ku", "", "", "langKurdish", "60", }, 0, 1 },
905 {{ "Laothian", "lo", "", "", "langLao", "79", }, 0, 1 },
906 {{ "Lappish", "", "", "", "langLappish", "29", }, 0, 1 },
907 {{ "Lappish", "", "", "", "langSaamisk", "29", }, 0, 1 },
908 {{ "Latin", "la", "", "", "langLatin", "131", }, 0, 1 },
909 {{ "Latvian", "lv", "", "0x0026", "langLatvian", "28", }, 2, 12 },
910 {{ "Lettish", "lv", "", "0x0026", "langLatvian", "28", }, 0, 1 },
911 {{ "Lingala", "ln", "", "", "", "", }, 0, 1 },
912 {{ "Lithuanian", "lt", "", "0x0027", "langLithuanian", "24", }, 11, 7 },
913 {{ "Macedonian", "mk", "", "0x002f", "langMacedonian", "43", }, 0, 1 },
914 {{ "Malagasy", "mg", "", "", "langMalagasy", "93", }, 0, 1 },
915 {{ "Malay", "ms", "", "0x003e", "langMalayRoman(Latin)", "83", }, 0, 1 },
916 {{ "Malay", "ms", "", "0x003e", "langMalayArabic(Arabic)", "84", }, 0, 1 },
917 {{ "Malayalam", "ml", "", "", "langMalayalam", "72", }, 0, 1 },
918 {{ "Maltese", "mt", "", "", "langMaltese", "16", }, 0, 1 },
919 {{ "Manx Gaelic", "gv", "", "", "langGailck", "141", }, 0, 1 },
920 {{ "Maori", "mi", "", "", "", "", }, 0, 1 },
921 {{ "Marathi", "mr", "", "", "langMarathi", "66", }, 0, 1 },
922 {{ "Marshallese", "mh", "", "", "", "", }, 0, 1 },
923 {{ "Moldavian", "mo", "", "", "langMoldavian", "53", }, 0, 1 },
924 {{ "Mongolian", "mn", "", "", "langMongolian(Mongolian)", "57", }, 0, 1 },
925 {{ "Mongolian", "mn", "", "", "langMongolianCyr(Cyrillic)", "58", }, 0, 1 },
926 {{ "Nauru", "na", "", "", "", "", }, 0, 1 },
927 {{ "Nepali", "ne", "", "", "langNepali", "64", }, 0, 1 },
928 {{ "Norwegian", "no", "", "0x0014", "langNorwegian", "9", }, 2, 1 },
929 {{ "Occitan", "oc", "", "", "", "", }, 0, 1 },
930 {{ "Oriya", "or", "", "", "langOriya", "71", }, 0, 1 },
931 {{ "Oromo", "om", "", "", "langOromo", "87", }, 0, 1 },
932 {{ "Afan", "om", "", "", "langOromo", "87", }, 0, 1 },
933 {{ "Pashto", "ps", "", "", "langPashto", "59", }, 0, 1 },
934 {{ "Pushto", "ps", "", "", "langPashto", "59", }, 0, 1 },
935 {{ "Polish", "pl", "", "0x0015", "langPolish", "25", }, 12, 8 },
936 {{ "Portuguese", "pt", "", "0x0016", "langPortuguese", "8", }, 2, 0 },
937 {{ "Portuguese", "pt", "BR", "0x0016", "langPortuguese", "8", }, 0, 1 },
938 {{ "Punjabi", "pa", "", "", "langPunjabi", "70", }, 0, 1 },
939 {{ "Quechua", "qu", "", "", "langQuechua", "132", }, 0, 1 },
940 {{ "Rhaeto-Romance", "rm", "", "", "", "", }, 0, 1 },
941 {{ "Romanian", "ro", "", "0x0018", "langRomanian", "37", }, 12, 2 },
942 {{ "Ruanda", "", "", "", "langRuanda", "90", }, 0, 1 },
943 {{ "Rundi", "", "", "", "langRundi", "91", }, 0, 1 },
944 {{ "Russian", "ru", "", "0x0019", "langRussian", "32", }, 2, 11 },
945 {{ "Samoan", "sm", "", "", "", "", }, 0, 1 },
946 {{ "Sangro", "sg", "", "", "", "", }, 0, 1 },
947 {{ "Sanskrit", "sa", "", "", "langSanskrit", "65", }, 0, 1 },
948 {{ "Sardinian", "sc", "", "", "", "", }, 0, 1 }, // Jordi 19/10/2002
949 {{ "Scots Gaelic", "gd", "", "", "langGaidhlig", "140", }, 0, 1 },
950 {{ "Serbian", "sr", "", "0x001a", "langSerbian", "42", }, 11, 7 },
951 {{ "Serbo-Croatian", "sh", "", "", "", "", }, 11, 7 },
952 {{ "Sesotho", "st", "", "", "", "", }, 0, 1 },
953 {{ "Setswana", "tn", "", "", "", "", }, 0, 1 },
954 {{ "Shona", "sn", "", "", "", "", }, 0, 1 },
955 {{ "Sindhi", "sd", "", "", "langSindhi", "62", }, 0, 1 },
956 {{ "Singhalese", "si", "", "", "langSinhalese", "76", }, 0, 1 },
957 {{ "Siswati", "ss", "", "", "", "", }, 0, 1 },
958 {{ "Slovak", "sk", "", "0x001b", "langSlovak", "39", }, 11, 7 },
959 {{ "Slovenian", "sl", "", "0x0024", "langSlovenian", "40", }, 11, 7 },
960 {{ "Somali", "so", "", "", "langSomali", "88", }, 0, 1 },
961 {{ "Spanish", "es", "", "0x000a", "langSpanish", "6", }, 2, 0 },
962 {{ "Sundanese", "su", "", "", "langSundaneseRom", "139", }, 0, 1 },
963 {{ "Swahili", "sw", "", "0x0041", "langSwahili", "89", }, 0, 1 },
964 {{ "Swedish", "sv", "", "0x001d", "langSwedish", "5", }, 10, 6 },
965 {{ "Tagalog", "tl", "", "", "langTagalog", "82", }, 0, 1 },
966 {{ "Tajik", "tg", "", "", "langTajiki", "55", }, 0, 1 },
967 {{ "Tamil", "ta", "", "", "langTamil", "74", }, 0, 1 },
968 {{ "Tatar", "tt", "", "", "langTatar", "135", }, 0, 1 },
969 {{ "Telugu", "te", "", "", "langTelugu", "75", }, 0, 1 },
970 {{ "Thai", "th", "", "0x001e", "langThai", "22", }, 0, 1 },
971 {{ "Tibetan", "bo", "", "", "langTibetan", "63", }, 0, 1 },
972 {{ "Tigrinya", "ti", "", "", "langTigrinya", "86", }, 0, 1 },
973 {{ "Tonga", "to", "", "", "", "", }, 0, 1 },
974 {{ "Tsonga", "ts", "", "", "", "", }, 0, 1 },
975 {{ "Turkish", "tr", "", "0x001f", "langTurkish", "17", }, 2, 13 },
976 {{ "Turkmen", "tk", "", "", "langTurkmen", "56", }, 0, 1 },
977 {{ "Twi", "tw", "", "", "", "", }, 0, 1 },
978 {{ "Uighur", "ug", "", "", "langUighur", "136", }, 0, 1 },
979 {{ "Ukrainian", "uk", "", "0x0022", "langUkrainian", "45", }, 2, 19 },
980 {{ "Urdu", "ur", "", "0x0020", "langUrdu", "20", }, 0, 1 },
981 {{ "Uzbek", "uz", "", "", "langUzbek", "47", }, 0, 1 },
982 {{ "Vietnamese", "vi", "", "0x002a", "langVietnamese", "80", }, 0, 1 },
983 {{ "Volap�k", "vo", "", "", "", "", }, 0, 1 },
984 {{ "Welsh", "cy", "", "", "langWelsh", "128", }, 1, 0 },
985 {{ "Wolof", "wo", "", "", "", "", }, 0, 1 },
986 {{ "Xhosa", "xh", "", "", "", "", }, 0, 1 },
987 {{ "Yiddish", "ji", "", "", "langYiddish", "41", }, 17, 1 },
988 {{ "Yiddish", "yi", "", "", "langYiddish", "41", }, 17, 1 },
989 {{ "Yoruba", "yo", "", "", "", "", }, 0, 1 },
990 {{ "Zulu", "zu", "", "", "", "", }, 0, 1 },
991 {{ NULL, "", "", "", "", "" , }, 0, 1 },
992 };
993
994 /*
995 * Line Breaking tables
996 */
997
998
999 enum EUniCat {
1000 NONATOMIC=0,
1001 ATOMIC=1,
1002 PUNCNOEND=2,
1003 PUNCNOSTART=3,
1004 PUNCFORCE=4,
1005 UNKNOWN=5
1006 };
1007 // Prototype.
1008 static enum EUniCat categoriseUniChar(UT_UCS4Char c);
1009
1010 struct SCatRange {
1011 UT_UCS4Char start;
1012 UT_UCS4Char end;
1013 enum EUniCat cat;
1014 };
1015
1016 /*
1017 * This table categorises all known Unicode characters.
1018 * The entries are inclusive ranges which must be in numerical order.
1019 *
1020 * Defaults should be provided by access functions for unknown characters.
1021 *
1022 * Description of categories:
1023 *
1024 * NONATOMIC - The character does not form an atomic (i.e. non-breakable)
1025 * unit. For example, English characters are of this type,
1026 * since several are needed to make a word.
1027 *
1028 * ATOMIC - The character does form an atomic unit. For example, Chinese
1029 * characters are of this type; although several characters
1030 * combine to make a word, it is legal to break between any two
1031 * Chinese characters.
1032 *
1033 * PUNCNOEND - This is punctuation which must not be allow to end a line
1034 * (unless forced by PUNCFORCE); for example, an open bracket.
1035 * In both English and Chinese you cannot break before the
1036 * '(' in "(word)".
1037 *
1038 * PUNCNOSTART - Like PUNCNOSTART, but for punctuation which cannot start a
1039 * line, e.g: ) , :
1040 *
1041 * PUNCFORCE - Punctuation which it is always possible to break before and
1042 * after regardless of what the surrounding characters are. This
1043 * is for punctuation that acts like a space.
1044 *
1045 */
1046 struct SCatRange UniCharCats[] = {
1047 {0x20, 0x20, PUNCFORCE}, // Space
1048 {0x21, 0x21, PUNCNOSTART}, // !
1049 {0x22, 0x27, NONATOMIC}, // "#$%&'
1050 {0x28, 0x28, PUNCNOEND}, // (
1051 {0x29, 0x29, PUNCNOSTART}, // )
1052 {0x2a, 0x2b, NONATOMIC}, // *+
1053 {0x2c, 0x2e, PUNCNOSTART}, // ,-.
1054 {0x2f, 0x2f, NONATOMIC}, // /
1055 {0x30, 0x39, NONATOMIC}, // Western numerals.
1056 {0x3a, 0x3b, PUNCNOSTART}, // :;
1057 {0x3c, 0x3c, PUNCNOEND}, // <
1058 {0x3d, 0x3d, NONATOMIC}, // =
1059 {0x3e, 0x3f, PUNCNOSTART}, // >?
1060 {0x40, 0x40, NONATOMIC}, // @
1061 {0x41, 0x5a, NONATOMIC}, // Western A-Z.
1062 {0x5b, 0x5b, PUNCNOEND}, // [
1063 {0x5c, 0x5c, NONATOMIC}, // "\"
1064 {0x5d, 0x5d, PUNCNOSTART}, // ]
1065 {0x5e, 0x60, NONATOMIC}, // ^_`
1066 {0x61, 0x7a, NONATOMIC}, // Western a-z.
1067 {0x7b, 0x7b, PUNCNOEND}, // {
1068 {0x7c, 0x7c, NONATOMIC}, // "\"
1069 {0x7d, 0x7d, PUNCNOSTART}, // }
1070 {0x7e, 0x7e, NONATOMIC}, // ~
1071
1072 /*
1073 Korean Hangul Jamo
1074 Korean line breaking is similar to English. However, I'm not
1075 sure if these phonetic elements can occur on their own and, if
1076 they can, what the breaking rules should be.
1077 */
1078 {0x1100, 0x11ff, NONATOMIC}, // Korean phonetic elements.
1079
1080 /* General punctuation */
1081 {0x2002, 0x2003, PUNCFORCE}, // en-space, em-space
1082 {0x2010, 0x2010, PUNCNOSTART}, // Hyphen
1083 {0x2013, 0x2014, PUNCFORCE}, // en-dash, em-dash
1084 {0x2018, 0x2018, PUNCNOEND}, // Open single quote
1085 {0x2019, 0x2019, NONATOMIC}, // Close single quote (usually used as ')
1086 {0x201c, 0x201c, PUNCNOEND}, // Open double quote
1087 {0x201d, 0x201d, PUNCNOSTART}, // Close double quote
1088 {0x2020, 0x2021, PUNCNOSTART}, // Long cross, double dagger
1089 {0x2026, 0x2026, PUNCNOSTART}, // ...
1090 {0x2030, 0x2031, PUNCNOSTART}, // Per mille/10000 signs
1091 {0x2032, 0x2034, PUNCNOSTART}, // Single, double, triple primes
1092
1093 /* CJK Blocks */
1094
1095 {0x3002, 0x3002, PUNCNOSTART}, // Ideographic full stop
1096 {0x3008, 0x3008, PUNCNOEND}, // CJK <
1097 {0x3009, 0x3009, PUNCNOSTART}, // CJK >
1098 {0x300a, 0x300a, PUNCNOEND}, // CJK <<
1099 {0x300b, 0x300b, PUNCNOSTART}, // CJK >>
1100 {0x300c, 0x300c, PUNCNOEND}, // CJK quote open
1101 {0x300d, 0x300d, PUNCNOSTART}, // CJK quote close
1102 {0x300e, 0x300e, PUNCNOEND}, // CJK thick quote open
1103 {0x300f, 0x300f, PUNCNOSTART}, // CJK thick quote close
1104 {0x3010, 0x3010, PUNCNOEND}, // CJK |(
1105 {0x3011, 0x3011, PUNCNOSTART}, // CJK )|
1106 {0x3014, 0x3014, PUNCNOEND}, // CJK [
1107 {0x3015, 0x3015, PUNCNOSTART}, // CJK ]
1108 {0x3016, 0x3016, PUNCNOEND}, // CJK [(
1109 {0x3017, 0x3017, PUNCNOSTART}, // CJK )]
1110 {0x3018, 0x3018, PUNCNOEND}, // CJK [[
1111 {0x3019, 0x3019, PUNCNOSTART}, // CJK ]]
1112 {0x301a, 0x301a, PUNCNOEND}, // CJK ]|
1113 {0x301b, 0x301b, PUNCNOSTART}, // CJK |[
1114 {0x301d, 0x301d, PUNCNOEND}, // CJK ``
1115 {0x301e, 0x301e, PUNCNOSTART}, // CJK ''
1116
1117 /* Hangul Compatibility Jamo */
1118 {0x3130, 0x318f, NONATOMIC},
1119
1120 /* More CJK blocks */
1121 {0x3200, 0x32ff, ATOMIC}, // Enclosed CJK Letters and Months
1122 {0x3300, 0x33ff, ATOMIC}, // CJK Compatibility
1123 {0x3400, 0x34ff, ATOMIC}, // CJK Unified Ideographs Ext. A
1124 {0x4e00, 0x9faf, ATOMIC}, // CJK Unified Ideographs
1125
1126 /* Hangul Syllabuls */
1127 {0xac00, 0xd7af, NONATOMIC},
1128
1129 /* Another CJK block */
1130 {0xf900, 0xfaff, ATOMIC}, // CJK Compatibility Ideographs
1131
1132 /* Halfwidth and Fullwidth Forms. */
1133 {0xff01, 0xff01, PUNCNOSTART}, // !
1134 {0xff02, 0xff02, ATOMIC}, // "
1135 {0xff05, 0xff05, PUNCNOSTART}, // %
1136 {0xff06, 0xff07, ATOMIC}, // &'
1137 {0xff08, 0xff08, PUNCNOEND}, // (
1138 {0xff09, 0xff09, PUNCNOSTART}, // )
1139 {0xff0a, 0xff0b, ATOMIC}, // *+
1140 {0xff0c, 0xff0c, PUNCNOSTART}, // ,
1141 {0xff0d, 0xff0d, ATOMIC}, // -
1142 {0xff0e, 0xff0e, PUNCNOSTART}, // .
1143 {0xff0f, 0xff0f, ATOMIC}, // /
1144 {0xff10, 0xff19, ATOMIC}, // Numerals
1145 {0xff1a, 0xff1b, PUNCNOSTART}, // :;
1146 {0xff1c, 0xff1c, PUNCNOEND}, // <
1147 {0xff1d, 0xff1d, ATOMIC}, // =
1148 {0xff1e, 0xff1e, PUNCNOSTART}, // >
1149 {0xff1f, 0xff1f, PUNCNOSTART}, // ?
1150 {0xff20, 0xff20, ATOMIC}, // @
1151 {0xff21, 0xff3a, ATOMIC}, // A-Z
1152 {0xff3b, 0xff3b, PUNCNOEND}, // [
1153 {0xff3c, 0xff3c, ATOMIC}, // "\"
1154 {0xff3d, 0xff3d, PUNCNOSTART}, // ]
1155 {0xff3e, 0xff5a, ATOMIC}, // ^_`a-z
1156 {0xff5b, 0xff5b, PUNCNOEND}, // {
1157 {0xff5c, 0xff5c, ATOMIC}, // |
1158 {0xff5d, 0xff5d, PUNCNOSTART}, // }
1159 {0xff5e, 0xff5e, ATOMIC}, // ~
1160 {0xff61, 0xff61, PUNCNOSTART}, // Ideographic full stop
1161 {0xff62, 0xff62, PUNCNOEND}, // Halfwidth left corner bracket
1162 {0xff63, 0xff63, PUNCNOSTART}, // Halfwidth right corner bracket
1163 {0xff64, 0xff64, PUNCNOSTART}, // Halfwidth ideographic comma
1164 {0xffe0, 0xffe0, PUNCNOEND}, // Fullwidth Cent sign
1165 {0xffe1, 0xffe1, PUNCNOEND}, // Fullwidth Pound sign
1166 {0xffe5, 0xffe5, PUNCNOEND}, // Fullwidth Yen sign
1167
1168 /* More CJK blocks. */
1169
1170 {0x20000, 0x2a6df, ATOMIC}, // CJK Unified Ideographs Ext. B
1171 {0x2f800, 0x2fa1f, ATOMIC}, // CJK Compatibility Ideographs Sup.
1172 {0,0,ATOMIC}
1173 };
1174
1175 /*
1176 * Boolean rules for whether a line break is allowed between all possible
1177 * combinations of two categories.
1178 */
1179 static bool blineBreakRules[] = {
1180 // 2nd char: NONATOMIC, ATOMIC, PUNCNOEND, PUNCNOSTART, PUNCFORCE
1181 /* 1st char */
1182 /* NONATOMIC */ false, true, false, false, true,
1183 /* ATOMIC */ true, true, true, false, true,
1184 /* PUNCNOEND */ false, false, false, false, true,
1185 /* PUNCNOSTART */ true, true, true, false, true,
1186 /* PUNCFORCE */ true, true, true, true, true};
1187
1188 /* ************************* here end tables *************************/
1189
1190 const XAP_LangInfo* XAP_EncodingManager::findLangInfo(const char* key,XAP_LangInfo::fieldidx idx)
1191 {
1192 if (idx > XAP_LangInfo::max_idx)
1193 return NULL;
1194 const XAP_LangInfo* cur = langinfo;
1195 for(; cur->fields[0]; ++cur)
1196 if (!g_ascii_strcasecmp(cur->fields[idx],key))
1197 return cur;
1198 return NULL;
1199 }
1200
1201 const XAP_LangInfo* XAP_EncodingManager::findLangInfoByLocale(const char* locale)
1202 {
1203 if (!locale)
1204 return NULL;
1205
1206 std::string strISOName(locale, 2);
1207 std::string strCountryCode;
1208
1209 if (strlen(locale) == 5)
1210 {
1211 strCountryCode = (locale + 3);
1212 }
1213
1214 const XAP_LangInfo* cur = langinfo;
1215 const XAP_LangInfo* temp = NULL;
1216 for(; cur->fields[0]; ++cur)
1217 {
1218 if (strISOName == cur->fields[XAP_LangInfo::isoshortname_idx])
1219 {
1220 if (! *cur->fields[XAP_LangInfo::countrycode_idx])
1221 {
1222 if (strCountryCode.empty())
1223 return cur;
1224 else
1225 temp = cur; // store, just in case
1226 }
1227 else if (strCountryCode == cur->fields[XAP_LangInfo::countrycode_idx])
1228 {
1229 return cur;
1230 }
1231 }
1232 }
1233
1234 if (temp != NULL)
1235 {
1236 return temp;
1237 }
1238
1239 return NULL;
1240 }
1241
1242 bool XAP_EncodingManager::swap_utos = false;
1243 bool XAP_EncodingManager::swap_stou = false;
1244
1245 UT_Bijection XAP_EncodingManager::fontsizes_mapping;
1246
1247 void XAP_EncodingManager::initialize()
1248 {
1249 const char* isocode = getLanguageISOName(),
1250 *terrname = getLanguageISOTerritory(),
1251 *enc = getNativeEncodingName();
1252
1253 // UCS-2 Encoding Names
1254 static const char * (szUCS2BENames[]) = {
1255 "UTF-16BE", // superset
1256 "UTF-16-BE", // my guess
1257 "UCS-2BE", // preferred
1258 "UCS-2-BE", // older libiconv
1259 "UNICODEBIG", // older glibc
1260 "UNICODE-1-1", // in libiconv source
1261 0 };
1262 static const char * (szUCS2LENames[]) = {
1263 "UTF-16LE", // superset
1264 "UTF-16-LE", // my guess
1265 "UCS-2LE", // preferred
1266 "UCS-2-LE", // older libiconv
1267 "UNICODELITTLE", // older glibc
1268 0 };
1269
1270 // UCS-4 Encoding Names
1271 static const char * (szUCS4BENames[]) = {
1272 "UCS-4BE", // preferred
1273 "UCS-4-BE", // older libiconv (??)
1274 0 };
1275 static const char * (szUCS4LENames[]) = {
1276 "UCS-4LE", // preferred
1277 "UCS-4-LE", // older libiconv (??)
1278 0 };
1279
1280 const char ** p = 0;
1281 UT_iconv_t iconv_handle = UT_ICONV_INVALID;
1282
1283 for (p = szUCS2BENames; *p; ++p)
1284 {
1285 if ((iconv_handle = UT_iconv_open(*p,*p)) != UT_ICONV_INVALID)
1286 {
1287 UT_iconv_close(iconv_handle);
1288 UCS2BEName = *p;
1289 break;
1290 }
1291 }
1292 for (p = szUCS2LENames; *p; ++p)
1293 {
1294 if ((iconv_handle = UT_iconv_open(*p,*p)) != UT_ICONV_INVALID)
1295 {
1296 UT_iconv_close(iconv_handle);
1297 UCS2LEName = *p;
1298 break;
1299 }
1300 }
1301 if (UCS2BEName) {
1302 UT_DEBUGMSG(("This iconv supports UCS-2BE as \"%s\"\n",UCS2BEName));
1303 }
1304 else {
1305 UT_DEBUGMSG(("This iconv does not support UCS-2BE!\n"));
1306 }
1307 if (UCS2LEName) {
1308 UT_DEBUGMSG(("This iconv supports UCS-2LE as \"%s\"\n",UCS2LEName));
1309 }
1310 else {
1311 UT_DEBUGMSG(("This iconv does not support UCS-2LE!\n"));
1312 }
1313
1314 for (p = szUCS4BENames; *p; ++p)
1315 {
1316 if ((iconv_handle = UT_iconv_open(*p,*p)) != UT_ICONV_INVALID)
1317 {
1318 UT_iconv_close(iconv_handle);
1319 UCS4BEName = *p;
1320 break;
1321 }
1322 }
1323 for (p = szUCS4LENames; *p; ++p)
1324 {
1325 if ((iconv_handle = UT_iconv_open(*p,*p)) != UT_ICONV_INVALID)
1326 {
1327 UT_iconv_close(iconv_handle);
1328 UCS4LEName = *p;
1329 break;
1330 }
1331 }
1332 if (UCS4BEName) {
1333 UT_DEBUGMSG(("This iconv supports UCS-4BE as \"%s\"\n",UCS4BEName));
1334 }
1335 else {
1336 UT_DEBUGMSG(("This iconv does not support UCS-4BE!\n"));
1337 }
1338 if (UCS4LEName) {
1339 UT_DEBUGMSG(("This iconv supports UCS-4LE as \"%s\"\n",UCS4LEName));
1340 }
1341 else {
1342 UT_DEBUGMSG(("This iconv does not support UCS-4LE!\n"));
1343 }
1344
1345 if(!g_ascii_strcasecmp(enc, "UTF-8") || !g_ascii_strcasecmp(enc, "UTF8")
1346 || !g_ascii_strcasecmp(enc, "UTF-16") || !g_ascii_strcasecmp(enc, "UTF16")
1347 || !g_ascii_strcasecmp(enc, "UCS-2") || !g_ascii_strcasecmp(enc, "UCS2"))
1348 m_bIsUnicodeLocale = true;
1349 else
1350 m_bIsUnicodeLocale = false;
1351
1352 #define SEARCH_PARAMS fulllocname, langandterr, isocode
1353 char fulllocname[40],langandterr[40];
1354 if (terrname) {
1355 sprintf(langandterr,"%s_%s",isocode,terrname);
1356 sprintf(fulllocname,"%s_%s.%s",isocode,terrname,enc);
1357 }
1358 else {
1359 strncpy(langandterr,isocode,sizeof(langandterr)-1);
1360 langandterr[sizeof(langandterr)-1] = '\0';
1361 sprintf(fulllocname,"%s.%s",isocode,enc);
1362 }
1363 const char* NativeTexEncodingName = search_rmap_with_opt_suffix(native_tex_enc_map,enc);
1364 const char* NativeBabelArgument = search_map_with_opt_suffix(langcode_to_babelarg,SEARCH_PARAMS);
1365 {
1366 const char* str = search_rmap_with_opt_suffix(langcode_to_wincharsetcode,SEARCH_PARAMS);
1367 WinCharsetCode = str ? atoi(str) : 0;
1368 }
1369 {
1370 const XAP_LangInfo* found = findLangInfo(getLanguageISOName(),XAP_LangInfo::isoshortname_idx);
1371 const char* str;
1372 WinLanguageCode = 0;
1373 if (found && *(str=found->fields[XAP_LangInfo::winlangcode_idx]))
1374 {
1375 int val;
1376 if (sscanf(str,"%i",&val)==1)
1377 WinLanguageCode = 0x400 + val;
1378 }
1379 str = search_map_with_opt_suffix(langcode_to_winlangcode,SEARCH_PARAMS);
1380 if (str) {
1381 int val;
1382 if (sscanf(str,"%i",&val)==1)
1383 WinLanguageCode = val;
1384 };
1385 }
1386 {
1387 const char* str = search_rmap_with_opt_suffix(langcode_to_cjk,SEARCH_PARAMS);
1388 is_cjk_ = *str == '1';
1389 }
1390 {
1391 if (cjk_locale()) {
1392 /* CJK guys should do something similar to 'else' branch */
1393 TexPrologue = " ";
1394 } else {
1395 char buf[500];
1396 int len = 0;
1397 if (NativeTexEncodingName)
1398 len += sprintf(buf+len,"\\usepackage[%s]{inputenc}\n",NativeTexEncodingName);
1399 if (NativeBabelArgument)
1400 len += sprintf(buf+len,"\\usepackage[%s]{babel}\n",NativeBabelArgument);
1401 TexPrologue = len ? g_strdup(buf) : " ";
1402 };
1403 }
1404 {
1405 fontsizes_mapping.clear();
1406 const char** fontsizes = cjk_locale() ? cjk_fontsizes : non_cjk_fontsizes;
1407 for(const char** cur=fontsizes; *cur; ++cur)
1408 {
1409 UT_String buf;// ( " " );
1410 buf += *cur;
1411 // buf += " ";
1412 fontsizes_mapping.add(*cur, buf.c_str());
1413 }
1414 }
1415
1416 init_values(this); /*do this unconditionally! */
1417 {
1418 swap_utos = swap_stou = false;
1419 swap_utos = UToNative(0x20) != 0x20;
1420 swap_stou = nativeToU(0x20) != 0x20;
1421
1422 XAP_EncodingManager__swap_stou = swap_stou;
1423 XAP_EncodingManager__swap_utos = swap_utos;
1424 }
1425 }
1426
1427 int XAP_EncodingManager__swap_stou,XAP_EncodingManager__swap_utos;
1428
1429 /*
1430 I'm not sure whether any non-cjk language doesn't make distinction
1431 between upper and lower case of the letter, but let's be prepared.
1432 TODO Arabic, Hebrew, Thai, Lao, all Indic scripts
1433 */
1434 bool XAP_EncodingManager::single_case() const { return cjk_locale(); }
1435
1436 bool XAP_EncodingManager::is_cjk_letter(UT_UCSChar c) const
1437 {
1438 if (!cjk_locale())
1439 return 0;
1440 return (c>0xff);
1441 }
1442
1443 bool XAP_EncodingManager::noncjk_letters(const UT_UCSChar* str,int len) const
1444 {
1445 if (!cjk_locale())
1446 return 1;
1447 for(int i=0;i<len;++i) {
1448 if (is_cjk_letter(str[i]))
1449 return 0;
1450 };
1451 return 1;
1452 }
1453
1454 /*
1455 * Returns true or false depending on whether a break between c[0] and c[1]
1456 * is permissible.
1457 */
1458 bool XAP_EncodingManager::canBreakBetween(const UT_UCS4Char c[2]) const
1459 {
1460 UT_uint8 rule;
1461
1462 // CJK special case: Can't break between two em dashes.
1463 if (c[0] == UCS_EM_DASH && c[1] == UCS_EM_DASH)
1464 return false;
1465
1466 // Finnish special case: opening '' quote (#13037)
1467 if (c[0] == UCS_RDBLQUOTE && categoriseUniChar(c[1]) == NONATOMIC)
1468 return false;
1469
1470 // Find rule number based on character categories.
1471 rule = categoriseUniChar(c[0]) * 5 + categoriseUniChar(c[1]);
1472
1473 // Return corresponding answer.
1474 return blineBreakRules[rule];
1475 }
1476
1477
1478 const char* XAP_EncodingManager::getTexPrologue() const
1479 {
1480 return TexPrologue;
1481 }
1482
1483 // Warning:
1484 // This code forces us to use "GB2312", "BIG5", etc instead
1485 // of "CP936", "CP950", etc even when our iconv supports
1486 // the "CPxxx" form and the encodings differ.
1487 // Be sure this is what you want if you call this function.
1488 const char* XAP_EncodingManager::charsetFromCodepage(int lid) const
1489 {
1490 static char buf[100];
1491 sprintf(buf,"CP%d",lid);
1492 char* cpname = buf;
1493 bool is_default;
1494 const char* ret = search_map(MSCodepagename_to_charset_name_map,cpname,&is_default);
1495 return is_default ? cpname : ret;
1496 }
1497
1498 const char* XAP_EncodingManager::CodepageFromCharset(const char *charset) const
1499 {
1500 bool is_default;
1501 const char* ret = search_map(charset_name_to_MSCodepagename_map,charset,&is_default);
1502 UT_DEBUGMSG(("Belcon:in XAP_EncodingManager::CodepageFromCharset,charset=%s,ret=%s,is_default=%d\n",charset,ret,is_default));
1503 return is_default ? charset : ret;
1504 }
1505
1506 const char* XAP_EncodingManager::WindowsCharsetName() const
1507 {
1508 char* cpname = wvLIDToCodePageConverter(getWinLanguageCode());
1509 bool is_default;
1510 const char* ret = search_map(MSCodepagename_to_charset_name_map,cpname,&is_default);
1511 return is_default ? cpname : ret;
1512 }
1513
1514 UT_uint32 XAP_EncodingManager::getWinLanguageCode() const
1515 {
1516 return WinLanguageCode;
1517 }
1518
1519 UT_uint32 XAP_EncodingManager::getWinCharsetCode() const
1520 {
1521 return WinCharsetCode;
1522 }
1523
1524 void XAP_EncodingManager::describe()
1525 {
1526 UT_DEBUGMSG(("EncodingManager reports the following:\n"
1527 " LanguageISOName is %s, LanguageISOTerritory is %s\n"
1528 " NativeEncodingName is %s, NativeSystemEncodingName is %s,\n"
1529 " Native8BitEncodingName is %s, NativeNonUnicodeEncodingName is %s,\n"
1530 " NativeUnicodeEncodingName is %s,\n"
1531 " fallbackchar is '%c'\n"
1532 " TexPrologue follows:\n"
1533 "---8<--------------\n"
1534 "%s"
1535 "--->8--------------\n"
1536
1537 " WinLanguageCode is 0x%04x, WinCharsetCode is %d\n"
1538 " cjk_locale %d, swap_utos %d, swap_stou %d\n",
1539 getLanguageISOName(), getLanguageISOTerritory() ? getLanguageISOTerritory() : "NULL",
1540 getNativeEncodingName(),getNativeSystemEncodingName(),
1541 getNative8BitEncodingName(),getNativeNonUnicodeEncodingName(),
1542 getNativeUnicodeEncodingName(),
1543 fallbackChar(1072),
1544 getTexPrologue(),
1545 getWinLanguageCode(), getWinCharsetCode(),
1546 int(cjk_locale()), int(swap_utos),int(swap_stou)
1547 ));
1548 UT_ASSERT( UT_iconv_isValid(iconv_handle_N2U) && UT_iconv_isValid(iconv_handle_U2N) );
1549 }
1550
1551
1552 /*
1553 This one returns NULL-terminated vector of strings in static buffers (i.e.
1554 don't try to g_free anything). On next call, filled data will be lost.
1555 returns the following strings surrounded by prefix and suffix:
1556 if (!skip_fallback)
1557 "";
1558 "%s" XAP_E..M..::instance->getLanguageISOName()
1559 "%s" XAP_E..M..::getNativeEncodingName()
1560 "%s-%s" XAP_E..M..::getLanguageISOName(),XAP_E..M..::getLanguageISOTerritory()
1561 "%s-%s.%s" XAP_E..M..::getLanguageISOName(), \
1562 XAP_E..M..::getLanguageISOTerritory(), XAP_E..M..::getNativeEncodingName()
1563
1564 */
1565 const char** localeinfo_combinations(const char* prefix,const char* suffix,const char* sep, bool skip_fallback)
1566 {
1567 //_DEBUGMSG(("locale combinations: prefix %s, suffix %s, sep %s\n", prefix,suffix,sep));
1568 static UT_String buf[5];
1569 static const char *ptrs[6];
1570
1571 for (size_t i = 1; i < 5; i++)
1572 buf[i] = prefix;
1573
1574 int idx = 0;
1575 if (!skip_fallback)
1576 {
1577 buf[idx] = prefix;
1578
1579 if(suffix && *suffix) // do not append nothing
1580 buf[idx++] += suffix;
1581 else idx++;
1582 }
1583
1584 UT_String lang (XAP_EncodingManager::get_instance()->getLanguageISOName());
1585 UT_String territory (XAP_EncodingManager::get_instance()->getLanguageISOTerritory());
1586 UT_String enc (XAP_EncodingManager::get_instance()->getNativeEncodingName());
1587
1588 buf[idx] += sep;
1589 buf[idx] += lang;
1590 if(suffix && *suffix)
1591 buf[idx++] += suffix;
1592 else idx++;
1593
1594 buf[idx] += sep;
1595 buf[idx] += enc;
1596 if(suffix && *suffix)
1597 buf[idx++] += suffix;
1598 else idx++;
1599
1600 buf[idx] += sep;
1601 buf[idx] += lang;
1602 buf[idx] += '-';
1603 buf[idx] += territory;
1604 if(suffix && *suffix)
1605 buf[idx++] += suffix;
1606 else idx++;
1607
1608 buf[idx] += sep;
1609 buf[idx] += lang;
1610 buf[idx] += '-';
1611 buf[idx] += territory;
1612 buf[idx] += '.';
1613 buf[idx] += enc;
1614 if(suffix && *suffix)
1615 buf[idx++] += suffix;
1616
1617 for (size_t j = 0; j < 5; ++j)
1618 ptrs[j] = buf[j].c_str();
1619 ptrs[5] = 0;
1620
1621 return ptrs;
1622 }
1623
1624 /* pspell hack */
1625 extern "C" {
1626 const char * xap_encoding_manager_get_language_iso_name(void)
1627 {
1628 return XAP_EncodingManager::get_instance()->getLanguageISOName();
1629 }
1630
1631 }
1632
1633
1634 /*!
1635 * Return the line breaking catagory that "c" belongs to.
1636 *
1637 * Note: For performance reasons this function assumes that the entries
1638 * in UniCharCats are in NUMERICAL ORDER starting with the smallest
1639 * value.
1640 */
1641
1642 static int s_compare_unichar_cats(const void * pC, const void *puc)
1643 {
1644 UT_UCS4Char c = *((UT_UCS4Char*)pC);
1645 SCatRange * pUC = (SCatRange*) puc;
1646
1647 if(c < pUC->start)
1648 return -1;
1649 else if(c > pUC->end)
1650 return 1;
1651
1652 return 0;
1653 }
1654
1655 static enum EUniCat categoriseUniChar(UT_UCS4Char c) {
1656 enum EUniCat cat=UNKNOWN;
1657
1658 // use linear search for the bottom (western part of the table, and bsearch for the
1659 // rest
1660
1661 SCatRange * pUC = (SCatRange*)bsearch(&c, UniCharCats, G_N_ELEMENTS(UniCharCats), sizeof(SCatRange),
1662 s_compare_unichar_cats);
1663
1664 if(pUC)
1665 cat = pUC->cat;
1666
1667 /*
1668 * Crude defaults:
1669 *
1670 * If the character is not listed then assume it's nonatomic (like western
1671 * letters) for all code blocks below "Armenian". If it belongs to the
1672 * "Armenian" block or above we assume CJK like atomic letters.
1673 *
1674 * I have extended that up to 0x0800, so that Hebrew, Arabic and Syriac mostly work
1675 * (see bug 9792); this might need some fine tuning though.
1676 *
1677 * This is not sensible, but it should at least mean that Greek, Cyrillic,
1678 * maybe Korean, Chinese and maybe Japanese get handled OK.
1679 */
1680 if (cat == UNKNOWN)
1681 {
1682 if (c < 0x0800)
1683 cat = NONATOMIC;
1684 else
1685 cat = ATOMIC;
1686 }
1687 return cat;
1688 }
1689