Debian Codegraph @

src/char.c

raw
1 /*
2 * char.c - character and character set operations
3 *
4 * Copyright (c) 2000-2024 Shiro Kawai <[email protected]>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the authors nor the names of its contributors
18 * may be used to endorse or promote products derived from this
19 * software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
27 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include <ctype.h>
35 #define LIBGAUCHE_BODY
36 #include "gauche.h"
37 #include "gauche/char_attr.h"
38 #include "gauche/priv/configP.h"
39 #include "gauche/priv/charP.h"
40 #include "gauche/priv/vectorP.h"
41
42 static ScmObj predef_sets[SCM_CHAR_SET_NUM_PREDEFINED_SETS];
43 static ScmObj predef_sets_complement[SCM_CHAR_SET_NUM_PREDEFINED_SETS];
44
45 #include "char_attr.c" /* generated tables */
46
47 /*=======================================================================
48 * Character functions
49 */
50
51 ScmObj Scm_CharEncodingName(void)
52 {
53 return SCM_INTERN(SCM_CHAR_ENCODING_NAME);
54 }
55
56 /* includes encoding-specific auxiliary functions */
57 #define SCM_CHAR_ENCODING_BODY
58 #include "gauche/char_utf_8.h"
59
60 const char **Scm_SupportedCharacterEncodings(void)
61 {
62 return supportedCharacterEncodings;
63 }
64
65 int Scm_SupportedCharacterEncodingP(const char *encoding)
66 {
67 const char **cs = supportedCharacterEncodings;
68 for (;*cs;cs++) {
69 const char *p = *cs;
70 const char *q = encoding;
71 for (;*p && *q; p++, q++) {
72 if (tolower(*p) != tolower(*q)) break;
73 }
74 if (*p == '\0' && *q == '\0') return TRUE;
75 }
76 return FALSE;
77 }
78
79 /* '0' -> 0, 'a' -> 10, etc.
80 Radix is assumed in the range [2, 36] if non-extended,
81 [2, 10] if extended.
82 'Extended' means we recognize not only ASCII but all Nd characters.
83 */
84 int Scm_DigitToInt(ScmChar ch, int radix, int extended)
85 {
86 if (ch < '0') return -1;
87 if (radix < 2) return -1;
88 if (radix <= 10) {
89 if (ch < '0' + radix) return (ch - '0');
90 } else {
91 if (ch <= '9') return (ch - '0');
92 if (ch < 'A') return -1;
93 if (ch < 'A' + radix - 10) return (ch - 'A' + 10);
94 if (ch < 'a') return -1;
95 if (ch < 'a' + radix - 10) return (ch - 'a' + 10);
96 }
97 if (extended && ch > 0x80 && radix <= 10) {
98 ScmChar ucschar = Scm_CharToUcs(ch);
99 int val = ucs_digit_value(ucschar);
100 if (val < 0 || val >= radix) return -1;
101 return val;
102 } else {
103 return -1;
104 }
105 }
106
107 ScmChar Scm_IntToDigit(int n, int radix, int basechar1, int basechar2)
108 {
109 if (n < 0) return SCM_CHAR_INVALID;
110 if (basechar1 == 0) basechar1 = '0';
111 if (basechar2 == 0) basechar2 = 'a';
112 if (radix <= 10) {
113 if (n < radix) return (ScmChar)(n + basechar1);
114 else return SCM_CHAR_INVALID;
115 } else {
116 if (n < 10) return (ScmChar)(n + basechar1);
117 if (n < radix) return (ScmChar)(n - 10 + basechar2);
118 else return SCM_CHAR_INVALID;
119 }
120 }
121
122 /*
123 * Convert UCS4 code <-> character
124 */
125
126 ScmChar Scm_UcsToChar(int n)
127 {
128 if (n < 0) Scm_Error("bad character code: %d", n);
129 return (ScmChar)n;
130 }
131
132 int Scm_CharToUcs(ScmChar ch)
133 {
134 if (ch == SCM_CHAR_INVALID) Scm_Error("bad character");
135 return (int)ch;
136 }
137
138 /*
139 * Charcter classification for lexical parsing
140 */
141
142 /* Table of initial 128 bytes of ASCII characters to dispatch for
143 special meanings. */
144
145 enum {
146 INITIAL = 1<<0, /* <initial> */
147 SUBSEQUENT = 1<<1, /* <subsequent> */
148 SIGN_SUBSEQUENT = 1<<2, /* <sign subsequent> */
149 DELIMITER = 1<<3, /* <delimiter> */
150 GAUCHE_DELIMITER = 1<<4, /* Gauche-extended delimiter */
151 };
152
153 static const unsigned char ctypes[128] = {
154 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
155 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
156 /*SPC*/ DELIMITER,
157 /* ! */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
158 /* " */ DELIMITER,
159 /* # */ 0,
160 /* $ */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
161 /* % */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
162 /* & */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
163 /* ' */ GAUCHE_DELIMITER,
164 /* ( */ DELIMITER,
165 /* ) */ DELIMITER,
166 /* * */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
167 /* + */ SUBSEQUENT|SIGN_SUBSEQUENT,
168 /* , */ GAUCHE_DELIMITER,
169 /* - */ SUBSEQUENT|SIGN_SUBSEQUENT,
170 /* . */ SUBSEQUENT,
171 /* / */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
172
173 /* 0 */ SUBSEQUENT,
174 /* 1 */ SUBSEQUENT,
175 /* 2 */ SUBSEQUENT,
176 /* 3 */ SUBSEQUENT,
177 /* 4 */ SUBSEQUENT,
178 /* 5 */ SUBSEQUENT,
179 /* 6 */ SUBSEQUENT,
180 /* 7 */ SUBSEQUENT,
181 /* 8 */ SUBSEQUENT,
182 /* 9 */ SUBSEQUENT,
183 /* : */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
184 /* ; */ DELIMITER,
185 /* < */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
186 /* = */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
187 /* > */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
188 /* ? */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
189
190 /* @ */ SUBSEQUENT|SIGN_SUBSEQUENT,
191 /* A */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
192 /* B */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
193 /* C */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
194 /* D */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
195 /* E */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
196 /* F */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
197 /* G */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
198 /* H */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
199 /* I */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
200 /* J */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
201 /* K */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
202 /* L */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
203 /* M */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
204 /* N */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
205 /* O */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
206
207 /* P */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
208 /* Q */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
209 /* R */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
210 /* S */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
211 /* T */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
212 /* U */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
213 /* V */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
214 /* W */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
215 /* X */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
216 /* Y */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
217 /* Z */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
218 /* [ */ GAUCHE_DELIMITER,
219 /* \ */ 0,
220 /* ] */ GAUCHE_DELIMITER,
221 /* ^ */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
222 /* _ */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
223
224 /* ` */ GAUCHE_DELIMITER,
225 /* a */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
226 /* b */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
227 /* c */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
228 /* d */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
229 /* e */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
230 /* f */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
231 /* g */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
232 /* h */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
233 /* i */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
234 /* j */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
235 /* k */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
236 /* l */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
237 /* m */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
238 /* n */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
239 /* o */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
240
241 /* p */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
242 /* q */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
243 /* r */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
244 /* s */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
245 /* t */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
246 /* u */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
247 /* v */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
248 /* w */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
249 /* x */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
250 /* y */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
251 /* z */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
252 /* { */ GAUCHE_DELIMITER,
253 /* | */ DELIMITER,
254 /* } */ GAUCHE_DELIMITER,
255 /* ~ */ INITIAL|SUBSEQUENT|SIGN_SUBSEQUENT,
256 /*DEL*/ 0,
257 };
258
259 int Scm_CharLexerCategoryP(ScmChar c, ScmCharLexerCategory category)
260 {
261 if (c < 128) {
262 switch (category) {
263 case SCM_CHAR_INITIAL:
264 return !!(ctypes[c] & INITIAL);
265 case SCM_CHAR_SUBSEQUENT:
266 return !!(ctypes[c] & SUBSEQUENT);
267 case SCM_CHAR_SIGN_SUBSEQUENT:
268 return !!(ctypes[c] & SIGN_SUBSEQUENT);
269 }
270 }
271 if (c == 0x200c || c == 0x200d) {
272 /* can be INITIAL, SUBSEQUENT and SIGN_SUBSEQUENT */
273 return TRUE;
274 }
275 switch (Scm_CharGeneralCategory(c)) {
276 case SCM_CHAR_CATEGORY_Lu:
277 case SCM_CHAR_CATEGORY_Ll:
278 case SCM_CHAR_CATEGORY_Lt:
279 case SCM_CHAR_CATEGORY_Lm:
280 case SCM_CHAR_CATEGORY_Lo:
281 case SCM_CHAR_CATEGORY_Mn:
282 case SCM_CHAR_CATEGORY_Nl:
283 case SCM_CHAR_CATEGORY_No:
284 case SCM_CHAR_CATEGORY_Pd:
285 case SCM_CHAR_CATEGORY_Pc:
286 case SCM_CHAR_CATEGORY_Po:
287 case SCM_CHAR_CATEGORY_Sc:
288 case SCM_CHAR_CATEGORY_Sm:
289 case SCM_CHAR_CATEGORY_Sk:
290 case SCM_CHAR_CATEGORY_So:
291 case SCM_CHAR_CATEGORY_Co:
292 /* can be INITIAL, SUBSEQUENT and SIGN_SUBSEQUENT */
293 return TRUE;
294 case SCM_CHAR_CATEGORY_Nd:
295 case SCM_CHAR_CATEGORY_Mc:
296 case SCM_CHAR_CATEGORY_Me:
297 return (category != SCM_CHAR_INITIAL);
298 default: return FALSE;
299 }
300 }
301
302
303 /*=======================================================================
304 * Character set (cf. SRFI-14)
305 */
306 /* NB: operations on charset are not very optimized, for I don't see
307 * the immediate needs to do so, except Scm_CharSetContains.
308 */
309
310 static void charset_print(ScmObj obj, ScmPort *out, ScmWriteContext*);
311 static int charset_compare(ScmObj x, ScmObj y, int equalp);
312 SCM_DEFINE_BUILTIN_CLASS(Scm_CharSetClass,
313 charset_print, charset_compare, NULL, NULL,
314 SCM_CLASS_COLLECTION_CPL);
315
316 #define MASK_ISSET(cs, ch) SCM_BITS_TEST(cs->small, ch)
317 #define MASK_SET(cs, ch) SCM_BITS_SET(cs->small, ch)
318 #define MASK_RESET(cs, ch) SCM_BITS_RESET(cs->small, ch)
319
320 static inline void check_mutable(ScmCharSet *cs)
321 {
322 if (SCM_CHAR_SET_IMMUTABLE_P(cs))
323 Scm_Error("Char set is immutable: %S", cs);
324 }
325
326 static inline void set_large(ScmCharSet *cs, int flag)
327 {
328 if (flag) {
329 cs->flags |= SCM_CHAR_SET_LARGE;
330 } else {
331 cs->flags &= ~(SCM_CHAR_SET_LARGE);
332 }
333 }
334
335 /*----------------------------------------------------------------------
336 * Printer
337 */
338
339 static void charset_print_ch(ScmPort *out, ScmChar ch, int firstp)
340 {
341 if (ch != 0 && ch < 0x80
342 && (strchr("[]-\\", ch) != NULL || (ch == '^' && firstp))) {
343 Scm_Printf(out, "\\%C", ch);
344 } else {
345 switch (Scm_CharGeneralCategory(ch)) {
346 case SCM_CHAR_CATEGORY_Mn:
347 case SCM_CHAR_CATEGORY_Mc:
348 case SCM_CHAR_CATEGORY_Me:
349 case SCM_CHAR_CATEGORY_Cc:
350 case SCM_CHAR_CATEGORY_Cf:
351 case SCM_CHAR_CATEGORY_Cs:
352 case SCM_CHAR_CATEGORY_Co:
353 case SCM_CHAR_CATEGORY_Cn:
354 if (ch < 0x10000) Scm_Printf(out, "\\u%04lx", ch);
355 else Scm_Printf(out, "\\U%08lx", ch);
356 break;
357 default:
358 Scm_Putc(ch, out);
359 }
360 }
361 }
362
363 static void charset_print(ScmObj obj, ScmPort *out,
364 ScmWriteContext *ctx SCM_UNUSED)
365 {
366 int prev, code, first = TRUE;
367 ScmCharSet *cs = SCM_CHAR_SET(obj);
368
369 Scm_Printf(out, "#[");
370 for (prev = -1, code = 0; code < SCM_CHAR_SET_SMALL_CHARS; code++) {
371 if (MASK_ISSET(cs, code) && prev < 0) {
372 charset_print_ch(out, code, first);
373 prev = code;
374 first = FALSE;
375 }
376 else if (!MASK_ISSET(cs, code) && prev >= 0) {
377 if (code - prev > 1) {
378 if (code - prev > 2) Scm_Printf(out, "-");
379 charset_print_ch(out, code-1, FALSE);
380 }
381 prev = -1;
382 }
383 }
384 if (prev >= 0) {
385 if (code - prev > 1) {
386 if (prev < 0x7e) Scm_Printf(out, "-");
387 charset_print_ch(out, code-1, FALSE);
388 }
389 }
390
391 if (cs->flags & SCM_CHAR_SET_IMMUTABLE) {
392 const uint32_t *v = cs->large.frozen.vec;
393 for (ScmSize i = 0; i < cs->large.frozen.size; i += 2) {
394 charset_print_ch(out, (int)v[i], FALSE);
395 if (v[i] != v[i+1]) {
396 if (v[i+1] - v[i] > 2) Scm_Printf(out, "-");
397 charset_print_ch(out, (int)v[i+1], FALSE);
398 }
399 }
400 } else {
401 ScmTreeIter iter;
402 ScmDictEntry *e;
403 Scm_TreeIterInit(&iter, &cs->large.tree, NULL);
404 while ((e = Scm_TreeIterNext(&iter)) != NULL) {
405 charset_print_ch(out, (int)e->key, FALSE);
406 if (e->value != e->key) {
407 if (e->value - e->key > 2) Scm_Printf(out, "-");
408 charset_print_ch(out, (int)e->value, FALSE);
409 }
410 }
411 }
412 Scm_Printf(out, "]");
413 }
414
415 /*-----------------------------------------------------------------
416 * Iterators for large char set
417 */
418 typedef struct cs_iter_rec {
419 ScmCharSet *cs;
420 int end;
421 union {
422 ScmTreeIter ti;
423 ScmSize vi;
424 } iter;
425 } cs_iter;
426
427 static void cs_iter_init(cs_iter *ci, ScmCharSet *cs)
428 {
429 ci->cs = cs;
430 if (!SCM_CHAR_SET_LARGE_P(cs)) {
431 ci->end = TRUE;
432 } else {
433 ci->end = FALSE;
434 if (SCM_CHAR_SET_IMMUTABLE_P(cs)) {
435 ci->iter.vi = 0;
436 } else {
437 Scm_TreeIterInit(&ci->iter.ti, &cs->large.tree, NULL);
438 }
439 }
440 }
441
442 /* returns FALSE if already exhausted */
443 static int cs_iter_next(cs_iter *ci,
444 ScmChar *from /*out*/,
445 ScmChar *to /*out*/)
446 {
447 if (ci->end) return FALSE;
448 if (SCM_CHAR_SET_IMMUTABLE_P(ci->cs)) {
449 if (ci->iter.vi >= ci->cs->large.frozen.size) {
450 ci->end = TRUE;
451 return FALSE;
452 } else {
453 *from = (ScmChar)ci->cs->large.frozen.vec[ci->iter.vi];
454 *to = (ScmChar)ci->cs->large.frozen.vec[ci->iter.vi+1];
455 ci->iter.vi += 2;
456 return TRUE;
457 }
458 } else {
459 ScmDictEntry *e = Scm_TreeIterNext(&ci->iter.ti);
460 if (e == NULL) {
461 ci->end = TRUE;
462 return FALSE;
463 } else {
464 *from = (ScmChar)e->key;
465 *to = (ScmChar)e->value;
466 return TRUE;
467 }
468 }
469 }
470
471 /*-----------------------------------------------------------------
472 * Constructors
473 */
474 static int cmp(ScmTreeCore *tc SCM_UNUSED, intptr_t a, intptr_t b)
475 {
476 if (a > b) return 1;
477 if (a < b) return -1;
478 return 0;
479 }
480
481 static ScmCharSet *make_charset(void)
482 {
483 ScmCharSet *cs = SCM_NEW(ScmCharSet);
484 SCM_SET_CLASS(cs, SCM_CLASS_CHARSET);
485 Scm_BitsFill(cs->small, 0, SCM_CHAR_SET_SMALL_CHARS, 0);
486 Scm_TreeCoreInit(&cs->large.tree, cmp, NULL);
487 cs->flags = 0; /* small & mutable by default */
488 return cs;
489 }
490
491 ScmObj Scm_MakeEmptyCharSet(void)
492 {
493 return SCM_OBJ(make_charset());
494 }
495
496 /* This is mainly for precompiled module. */
497 ScmObj Scm_MakeImmutableCharSet(const ScmBits *small,
498 const uint32_t *vec,
499 size_t size)
500 {
501 SCM_ASSERT(size % 2 == 0);
502 ScmCharSet *cs = SCM_NEW(ScmCharSet);
503 SCM_SET_CLASS(cs, SCM_CLASS_CHARSET);
504 cs->flags |= SCM_CHAR_SET_IMMUTABLE;
505 memcpy(cs->small, small, sizeof(cs->small));
506 if (vec != NULL && size > 0) {
507 set_large(cs, TRUE);
508 if ((cs->large.frozen.size = size) == 2) {
509 cs->large.frozen.ivec[0] = vec[0];
510 cs->large.frozen.ivec[1] = vec[1];
511 cs->large.frozen.vec = cs->large.frozen.ivec;
512 } else {
513 cs->large.frozen.vec = vec;
514 }
515 }
516 return SCM_OBJ(cs);
517 }
518
519 ScmObj Scm_CharSetCopy(ScmCharSet *src)
520 {
521 ScmCharSet *dst = make_charset();
522 Scm_BitsCopyX(dst->small, 0, src->small, 0, SCM_CHAR_SET_SMALL_CHARS);
523 set_large(dst, SCM_CHAR_SET_LARGE_P(src));
524 if (SCM_CHAR_SET_IMMUTABLE_P(src)) {
525 /* The destination is mutable */
526 const uint32_t *vec = src->large.frozen.vec;
527 for (ScmSize k = 0; k < src->large.frozen.size; k += 2) {
528 ScmDictEntry *e = Scm_TreeCoreSearch(&dst->large.tree,
529 vec[k], SCM_DICT_CREATE);
530 e->value = vec[k+1];
531 }
532 } else {
533 Scm_TreeCoreCopy(&dst->large.tree, &src->large.tree);
534 }
535 return SCM_OBJ(dst);
536 }
537
538 /* Creates flat searched vector to be used for immutable charset.
539 SRC must be a mutable charset.
540 The caller must provide uint32_t[2] buffer for ivec. */
541 static uint32_t *char_set_freeze_vec(ScmCharSet *src,
542 uint32_t *ivec,
543 ScmSize *size /*out*/)
544 {
545 SCM_ASSERT(!SCM_CHAR_SET_IMMUTABLE_P(src));
546 size_t s = (size_t)Scm_TreeCoreNumEntries(&src->large.tree) * 2;
547 uint32_t *v = (s == 2)? ivec : SCM_NEW_ATOMIC_ARRAY(uint32_t, s);
548
549 cs_iter iter;
550 cs_iter_init(&iter, src);
551 ScmChar lo, hi;
552 for (size_t k = 0; cs_iter_next(&iter, &lo, &hi); k += 2) {
553 SCM_ASSERT(k < s);
554 v[k] = (uint32_t)lo;
555 v[k+1] = (uint32_t)hi;
556 }
557 *size = s;
558 return v;
559 }
560
561 ScmObj Scm_CharSetFreeze(ScmCharSet *src)
562 {
563 if (SCM_CHAR_SET_IMMUTABLE_P(src)) return SCM_OBJ(src);
564 ScmCharSet *dst = make_charset();
565 Scm_BitsCopyX(dst->small, 0, src->small, 0, SCM_CHAR_SET_SMALL_CHARS);
566
567 dst->flags |= SCM_CHAR_SET_IMMUTABLE;
568 if (SCM_CHAR_SET_LARGE_P(src)) {
569 set_large(dst, TRUE);
570 dst->large.frozen.vec = char_set_freeze_vec(src,
571 dst->large.frozen.ivec,
572 &dst->large.frozen.size);
573 } else {
574 dst->large.frozen.vec = NULL;
575 dst->large.frozen.size = 0;
576 }
577 return SCM_OBJ(dst);
578 }
579
580 ScmObj Scm_CharSetFreezeX(ScmCharSet *src)
581 {
582 if (SCM_CHAR_SET_IMMUTABLE_P(src)) return SCM_OBJ(src);
583 if (SCM_CHAR_SET_LARGE_P(src)) {
584 ScmSize s;
585 uint32_t iv[2];
586 uint32_t *v = char_set_freeze_vec(src, iv, &s);
587 src->large.frozen.size = s;
588 if (s == 2) {
589 src->large.frozen.vec = src->large.frozen.ivec;
590 src->large.frozen.ivec[0] = iv[0];
591 src->large.frozen.ivec[1] = iv[1];
592 } else {
593 src->large.frozen.vec = v;
594 }
595 }
596 src->flags |= SCM_CHAR_SET_IMMUTABLE;
597 return SCM_OBJ(src);
598 }
599
600 /*-----------------------------------------------------------------
601 * Comparison
602 */
603 static int charset_compare(ScmObj x, ScmObj y, int equalp)
604 {
605 ScmCharSet *xx = SCM_CHAR_SET(x);
606 ScmCharSet *yy = SCM_CHAR_SET(y);
607
608 if (equalp) {
609 return (Scm_CharSetEq(xx, yy)? 0 : 1);
610 } else {
611 if (Scm_CharSetEq(xx, yy)) return 0;
612 if (Scm_CharSetLE(xx, yy)) return -1;
613 if (Scm_CharSetLE(yy, xx)) return 1;
614 Scm_Error("cannot compare char-sets: %S vs %S", x, y);
615 return 0; /* dummy */
616 }
617 }
618
619 int Scm_CharSetEq(ScmCharSet *x, ScmCharSet *y)
620 {
621 if (!Scm_BitsEqual(x->small, y->small, 0, SCM_CHAR_SET_SMALL_CHARS))
622 return FALSE;
623 if (!SCM_CHAR_SET_IMMUTABLE_P(x) && !SCM_CHAR_SET_IMMUTABLE_P(y)) {
624 /* shortcut */
625 return Scm_TreeCoreEq(&x->large.tree, &y->large.tree);
626 } else {
627 cs_iter xi, yi;
628 cs_iter_init(&xi, x);
629 cs_iter_init(&yi, y);
630 ScmChar xl = 0, xh = 0, yl = 0, yh = 0;
631 for (;;) {
632 int xr = cs_iter_next(&xi, &xl, &xh);
633 int yr = cs_iter_next(&yi, &yl, &yh);
634 if (xr == FALSE && yr == FALSE) return TRUE;
635 if (!(xr && yr)) return FALSE;
636 if (!(xl == yl && xh == yh)) return FALSE;
637 }
638 }
639 }
640
641 /* See if cs contains the range [lo,hi] in large char range. */
642 static int cs_contains_range(ScmCharSet *s, ScmChar lo, ScmChar hi)
643 {
644 if (!SCM_CHAR_SET_LARGE_P(s)) return FALSE;
645 /* We can have two cases.
646 *
647 * Case 1:
648 * lo<---------->hi
649 * ye<----------------->
650 * Case 2:
651 * lo<---------->hi
652 * yl<------------------->
653 */
654 if (SCM_CHAR_SET_IMMUTABLE_P(s)) {
655 size_t ye, yl;
656 ye = Scm_BinarySearchU32(s->large.frozen.vec, s->large.frozen.size,
657 (uint32_t)lo, 1, &yl, NULL);
658 if (ye != (size_t)-1) { /* case 1 */
659 if (s->large.frozen.vec[ye+1] < (unsigned)hi) return FALSE;
660 } else if (yl != (size_t)-1) { /* case 2 */
661 if (s->large.frozen.vec[yl+1] < (unsigned)hi) return FALSE;
662 } else {
663 return FALSE;
664 }
665 } else {
666 ScmDictEntry *ye, *yl, *yh;
667 ye = Scm_TreeCoreClosestEntries(&s->large.tree, lo, &yl, &yh);
668 if (ye) { /* case 1 */
669 if (ye->value < hi) return FALSE;
670 } else if (yl) { /* case 2 */
671 if (yl->value < hi) return FALSE;
672 } else {
673 return FALSE;
674 }
675 }
676 return TRUE;
677 }
678
679 /* whether x <= y */
680 int Scm_CharSetLE(ScmCharSet *x, ScmCharSet *y)
681 {
682 if (!Scm_BitsIncludes(y->small, x->small, 0, SCM_CHAR_SET_SMALL_CHARS))
683 return FALSE;
684
685 cs_iter xi;
686 cs_iter_init(&xi, x);
687 ScmChar lo, hi;
688 while (cs_iter_next(&xi, &lo, &hi)) {
689 if (!cs_contains_range(y, lo, hi)) return FALSE;
690 }
691 return TRUE;
692 }
693
694 /*-----------------------------------------------------------------
695 * Modification
696 * We reject immutable set at the top, so that we only deal with treemap.
697 */
698
699 ScmObj Scm_CharSetAddRange(ScmCharSet *cs, ScmChar from, ScmChar to)
700 {
701 check_mutable(cs);
702
703 ScmDictEntry *e, *lo, *hi;
704
705 if (to < from) return SCM_OBJ(cs);
706 if (from < SCM_CHAR_SET_SMALL_CHARS) {
707 if (to < SCM_CHAR_SET_SMALL_CHARS) {
708 Scm_BitsFill(cs->small, (int)from, (int)to+1, TRUE);
709 return SCM_OBJ(cs);
710 }
711 Scm_BitsFill(cs->small, (int)from, SCM_CHAR_SET_SMALL_CHARS, TRUE);
712 from = SCM_CHAR_SET_SMALL_CHARS;
713 }
714
715 set_large(cs, TRUE);
716
717 /* Let e have the lower bound. */
718 e = Scm_TreeCoreClosestEntries(&cs->large.tree, from, &lo, &hi);
719 if (!e) {
720 if (!lo || lo->value < from-1) {
721 e = Scm_TreeCoreSearch(&cs->large.tree, from, SCM_DICT_CREATE);
722 } else {
723 e = lo;
724 }
725 }
726 /* Set up the upper bound.
727 NB: if e is a new entry, e->value is 0. */
728 if (e->value >= to) return SCM_OBJ(cs);
729
730 hi = e;
731 while ((hi = Scm_TreeCoreNextEntry(&cs->large.tree, hi->key)) != NULL) {
732 if (hi->key > to+1) {
733 e->value = to;
734 return SCM_OBJ(cs);
735 }
736 Scm_TreeCoreSearch(&cs->large.tree, hi->key, SCM_DICT_DELETE);
737 if (hi->value > to) {
738 e->value = hi->value;
739 return SCM_OBJ(cs);
740 }
741 }
742 e->value = to;
743 return SCM_OBJ(cs);
744 }
745
746 ScmObj Scm_CharSetAdd(ScmCharSet *dst, ScmCharSet *src)
747 {
748 check_mutable(dst);
749
750 if (dst == src) return SCM_OBJ(dst); /* precaution */
751
752 if (SCM_CHAR_SET_LARGE_P(src)) {
753 set_large(dst, TRUE);
754 }
755
756 ScmTreeIter iter;
757 ScmDictEntry *e;
758 Scm_BitsOperate(dst->small, SCM_BIT_IOR, dst->small, src->small,
759 0, SCM_CHAR_SET_SMALL_CHARS);
760 if (SCM_CHAR_SET_IMMUTABLE_P(src)) {
761 ScmSize k;
762 for (k = 0; k < src->large.frozen.size; k += 2) {
763 Scm_CharSetAddRange(dst,
764 SCM_CHAR(src->large.frozen.vec[k]),
765 SCM_CHAR(src->large.frozen.vec[k+1]));
766 }
767 } else {
768 Scm_TreeIterInit(&iter, &src->large.tree, NULL);
769 while ((e = Scm_TreeIterNext(&iter)) != NULL) {
770 Scm_CharSetAddRange(dst, SCM_CHAR(e->key), SCM_CHAR(e->value));
771 }
772 }
773 return SCM_OBJ(dst);
774 }
775
776 ScmObj Scm_CharSetComplement(ScmCharSet *cs)
777 {
778 check_mutable(cs);
779
780 ScmDictEntry *e, *n;
781
782 Scm_BitsOperate(cs->small, SCM_BIT_NOT1, cs->small, NULL,
783 0, SCM_CHAR_SET_SMALL_CHARS);
784 int last = SCM_CHAR_SET_SMALL_CHARS-1;
785 int largep = FALSE;
786 /* we can't use treeiter, since we modify the tree while traversing it. */
787 while ((e = Scm_TreeCoreNextEntry(&cs->large.tree, last)) != NULL) {
788 Scm_TreeCoreSearch(&cs->large.tree, e->key, SCM_DICT_DELETE);
789 if (last < e->key-1) {
790 n = Scm_TreeCoreSearch(&cs->large.tree, last+1, SCM_DICT_CREATE);
791 n->value = e->key-1;
792 largep = TRUE;
793 }
794 last = (int)e->value;
795 }
796 if (last < SCM_CHAR_MAX) {
797 n = Scm_TreeCoreSearch(&cs->large.tree, last+1, SCM_DICT_CREATE);
798 n->value = SCM_CHAR_MAX;
799 largep = TRUE;
800 }
801 set_large(cs, largep);
802 return SCM_OBJ(cs);
803 }
804
805 /* Make CS case-insensitive. */
806 ScmObj Scm_CharSetCaseFold(ScmCharSet *cs)
807 {
808 check_mutable(cs);
809
810 for (int ch='a'; ch<='z'; ch++) {
811 if (MASK_ISSET(cs, ch) || MASK_ISSET(cs, (ch-('a'-'A')))) {
812 MASK_SET(cs, ch);
813 MASK_SET(cs, (ch-('a'-'A')));
814 }
815 }
816
817 ScmTreeIter iter;
818 ScmDictEntry *e;
819 Scm_TreeIterInit(&iter, &cs->large.tree, NULL);
820 while ((e = Scm_TreeIterNext(&iter)) != NULL) {
821 for (ScmChar c = e->key; c <= e->value; c++) {
822 ScmChar uch = Scm_CharUpcase(c);
823 ScmChar lch = Scm_CharDowncase(c);
824 Scm_CharSetAddRange(cs, uch, uch);
825 Scm_CharSetAddRange(cs, lch, lch);
826 }
827 }
828 return SCM_OBJ(cs);
829 }
830
831 /*-----------------------------------------------------------------
832 * Query
833 */
834
835 int Scm_CharSetContains(ScmCharSet *cs, ScmChar c)
836 {
837 if (c < 0) return FALSE;
838 if (c < SCM_CHAR_SET_SMALL_CHARS) return MASK_ISSET(cs, c);
839 else if (SCM_CHAR_SET_IMMUTABLE_P(cs)) {
840 if (cs->large.frozen.size == 2) {
841 /* shortcut */
842 return (c >= (ScmChar)cs->large.frozen.ivec[0]
843 && c <= (ScmChar)cs->large.frozen.ivec[1]);
844 } else {
845 size_t lo;
846 size_t k = Scm_BinarySearchU32(cs->large.frozen.vec,
847 cs->large.frozen.size,
848 (uint32_t)c,
849 1, &lo, NULL);
850 if ((k != (size_t)-1)
851 || (lo != (size_t)-1 && (unsigned)c <= cs->large.frozen.vec[lo+1]))
852 return TRUE;
853 else
854 return FALSE;
855 }
856 } else {
857 ScmDictEntry *e, *l, *h;
858 e = Scm_TreeCoreClosestEntries(&cs->large.tree, (int)c, &l, &h);
859 if (e || (l && l->value >= c)) return TRUE;
860 else return FALSE;
861 }
862 }
863
864 /*-----------------------------------------------------------------
865 * Inspection
866 */
867
868 /* returns a list of ranges contained in the charset */
869 ScmObj Scm_CharSetRanges(ScmCharSet *cs)
870 {
871 ScmObj h = SCM_NIL, t = SCM_NIL;
872 int ind, begin = 0, prev = FALSE;
873
874 for (ind = 0; ind < SCM_CHAR_SET_SMALL_CHARS; ind++) {
875 int bit = MASK_ISSET(cs, ind);
876 if (!prev && bit) begin = ind;
877 if (prev && !bit) {
878 ScmObj cell = Scm_Cons(SCM_MAKE_INT(begin), SCM_MAKE_INT(ind-1));
879 SCM_APPEND1(h, t, cell);
880 }
881 prev = bit;
882 }
883 if (prev) {
884 ScmObj cell = Scm_Cons(SCM_MAKE_INT(begin), SCM_MAKE_INT(ind-1));
885 SCM_APPEND1(h, t, cell);
886 }
887
888 cs_iter iter;
889 cs_iter_init(&iter, cs);
890 ScmChar lo, hi;
891 while (cs_iter_next(&iter, &lo, &hi)) {
892 ScmObj cell = Scm_Cons(SCM_MAKE_INT(lo), SCM_MAKE_INT(hi));
893 SCM_APPEND1(h, t, cell);
894 }
895 return h;
896 }
897
898 void Scm_CharSetDump(ScmCharSet *cs, ScmPort *port)
899 {
900 Scm_Printf(port, "CharSet %p%s\nmask:",
901 cs,
902 SCM_CHAR_SET_IMMUTABLE_P(cs) ? " (frozen)" : "");
903 for (int i=0; i<SCM_BITS_NUM_WORDS(SCM_CHAR_SET_SMALL_CHARS); i++) {
904 #if SIZEOF_LONG == 4
905 Scm_Printf(port, "[%08lx]", cs->small[i]);
906 #else
907 Scm_Printf(port, "[%016lx]", cs->small[i]);
908 #endif
909 }
910 Scm_Printf(port, "\nranges:");
911 cs_iter iter;
912 cs_iter_init(&iter, cs);
913 ScmChar lo, hi;
914 while (cs_iter_next(&iter, &lo, &hi)) {
915 Scm_Printf(port, " %x-%x", lo, hi);
916 }
917 Scm_Printf(port, "\n");
918 }
919
920 /*-----------------------------------------------------------------
921 * Reader
922 */
923
924 /* Parse regexp-style character set specification (e.g. [a-zA-Z]).
925 Assumes the opening bracket is already read.
926 Always return a fresh charset, that can be modified afterwards.
927
928 If the input syntax is invalid, either signals an error or returns
929 #f, depending error_p flag.
930
931 If bracket_syntax is TRUE, the first closing bracket ']' in the
932 charset (except the complimenting caret) is taken as a literal
933 character, instead of terminating the charset. It should be TRUE
934 during reading the regexp syntax for compatibility to POSIX regexp.
935
936 If complement_p is not NULL, the location get a boolean value of
937 whether complement character (caret in the beginning) appeared or not.
938 In that case, the returned charset is not complemented. */
939
940 static ScmObj read_predef_charset(ScmPort*, int);
941
942 ScmObj Scm_CharSetRead(ScmPort *input, int *complement_p,
943 int error_p, int bracket_syntax)
944 {
945 #define REAL_BEGIN 1
946 #define CARET_BEGIN 2
947 ScmCharSet *set = make_charset();
948 int begin = REAL_BEGIN;
949 int complement = FALSE; /* Flag for the initial ^ */
950 int inrange = FALSE; /* The range '-' is being read */
951 ScmChar lastchar = SCM_CHAR_INVALID; /* The char before '-' range */
952 const char *prefetched = NULL; /* \x notation requires lookahead. After
953 it reads extra characters, this points
954 to them. The pointed string is guaranteed
955 to have only hexadecimal characters. */
956 ScmDString buf; /* Save read characters for error message */
957 Scm_DStringInit(&buf);
958 int ch;
959
960 for (;;) {
961 if (prefetched) {
962 ch = *prefetched++;
963 if (*prefetched == '\0') prefetched = NULL;
964 } else {
965 ch = Scm_Getc(input);
966 if (ch == EOF) goto err;
967 }
968
969 Scm_DStringPutc(&buf, ch);
970
971 if (begin == REAL_BEGIN && ch == '^') {
972 complement = TRUE;
973 begin = CARET_BEGIN;
974 continue;
975 }
976
977 ScmObj moreset;
978 switch (ch) {
979 case '^':
980 if (begin == REAL_BEGIN) {
981 complement = TRUE;
982 begin = CARET_BEGIN;
983 continue;
984 } else {
985 goto ordchar;
986 }
987 case ']':
988 if (begin && bracket_syntax) goto ordchar;
989 else break;
990 case '-':
991 if (begin || inrange) goto ordchar;
992 inrange = TRUE;
993 begin = FALSE;
994 continue;
995 case '\\':
996 ch = Scm_Getc(input);
997 if (ch == EOF) goto err;
998 Scm_DStringPutc(&buf, ch);
999 switch (ch) {
1000 case 'a': ch = 7; goto ordchar;
1001 case 'b': ch = 8; goto ordchar;
1002 case 'n': ch = '\n'; goto ordchar;
1003 case 'r': ch = '\r'; goto ordchar;
1004 case 't': ch = '\t'; goto ordchar;
1005 case 'f': ch = '\f'; goto ordchar;
1006 case 'e': ch = 0x1b; goto ordchar;
1007 case 'x': case 'u': case 'U': {
1008 ScmDString xbuf;
1009 Scm_DStringInit(&xbuf);
1010 ScmObj mode = Scm_GetPortReaderLexicalMode(input);
1011 ScmObj z = Scm_ReadXdigitsFromPort(input, ch, mode, FALSE, &xbuf);
1012 if (SCM_STRINGP(z)) {
1013 /* parse failure. z contains the prefetched string */
1014 Scm_DStringAdd(&buf, SCM_STRING(z));
1015 goto err;
1016 }
1017 /* xbuf contains the character that was hex-encoded,
1018 plus any hex digits that are prefetched. */
1019 Scm_DStringPutc(&xbuf, '\0');
1020 const char *cp = Scm_DStringPeek(&xbuf, NULL, NULL);
1021 SCM_CHAR_GET(cp, ch);
1022 cp += SCM_CHAR_NFOLLOWS(*cp)+1;
1023 if (*cp != '\0') {
1024 prefetched = cp;
1025 }
1026 goto ordchar;
1027 }
1028 case 'd':
1029 moreset = Scm_GetStandardCharSet(SCM_CHAR_SET_ASCII_DIGIT);
1030 goto addset;
1031 case 'D':
1032 moreset = Scm_GetStandardCharSet(-SCM_CHAR_SET_ASCII_DIGIT);
1033 goto addset;
1034 case 's':
1035 moreset = Scm_GetStandardCharSet(SCM_CHAR_SET_ASCII_WHITESPACE);
1036 goto addset;
1037 case 'S':
1038 moreset = Scm_GetStandardCharSet(-SCM_CHAR_SET_ASCII_WHITESPACE);
1039 goto addset;
1040 case 'w':
1041 moreset = Scm_GetStandardCharSet(SCM_CHAR_SET_ASCII_WORD);
1042 goto addset;
1043 case 'W':
1044 moreset = Scm_GetStandardCharSet(-SCM_CHAR_SET_ASCII_WORD);
1045 goto addset;
1046 case 'p': case 'P':
1047 moreset = Scm_GetStandardCharSet(Scm_CharSetParseCategory(input, ch));
1048 goto addset;
1049 default:
1050 goto ordchar;
1051 }
1052 case '[':
1053 moreset = read_predef_charset(input, error_p);
1054 if (!SCM_CHAR_SET_P(moreset)) goto err;
1055 addset:
1056 Scm_CharSetAdd(set, SCM_CHAR_SET(moreset));
1057 begin = FALSE;
1058 continue;
1059 ordchar:
1060 default:
1061 if (inrange) {
1062 if (lastchar < 0) {
1063 Scm_CharSetAddRange(set, '-', '-');
1064 Scm_CharSetAddRange(set, ch, ch);
1065 lastchar = ch;
1066 } else {
1067 Scm_CharSetAddRange(set, lastchar, ch);
1068 lastchar = -1;
1069 }
1070 inrange = FALSE;
1071 } else {
1072 Scm_CharSetAddRange(set, ch, ch);
1073 lastchar = ch;
1074 }
1075 begin = FALSE;
1076 continue;
1077 }
1078 break;
1079 }
1080
1081 if (inrange) {
1082 Scm_CharSetAddRange(set, '-', '-');
1083 if (lastchar >= 0) Scm_CharSetAddRange(set, lastchar, lastchar);
1084 }
1085 if (complement_p) {
1086 *complement_p = complement;
1087 return SCM_OBJ(set);
1088 } else {
1089 if (complement) Scm_CharSetComplement(set);
1090 return SCM_OBJ(set);
1091 }
1092 err:
1093 if (error_p) {
1094 /* TODO: We should deal with the case when input contains \0 */
1095 Scm_Error("Invalid charset syntax [%s%s...",
1096 complement? "^" : "",
1097 Scm_DStringGetz(&buf));
1098 }
1099 return SCM_FALSE;
1100 }
1101
1102 /* Predefined charset name table */
1103 struct predef_charset_posix_name_rec {
1104 const char *name;
1105 int cset; /* default cset */
1106 int cset_unicode; /* 'unicode' mode cset (:alnum: for unicode range)
1107 not used yet. */
1108 };
1109
1110 #define PREDEF_ENTRY(n, cs, csu) \
1111 { n, SCM_CPP_CAT(SCM_CHAR_SET_, cs), SCM_CPP_CAT(SCM_CHAR_SET_, csu) }
1112
1113 static struct predef_charset_posix_name_rec predef_charset_posix_names[] = {
1114 PREDEF_ENTRY("alpha:", ASCII_LETTER, LETTER),
1115 PREDEF_ENTRY("alnum:", ASCII_LETTER_DIGIT, LETTER_DIGIT),
1116 PREDEF_ENTRY("blank:", ASCII_BLANK, BLANK),
1117 PREDEF_ENTRY("cntrl:", ASCII_CONTROL, ISO_CONTROL),
1118 PREDEF_ENTRY("digit:", ASCII_DIGIT, DIGIT),
1119 PREDEF_ENTRY("graph:", ASCII_GRAPHIC, GRAPHIC),
1120 PREDEF_ENTRY("lower:", ASCII_LOWER, LOWER),
1121 PREDEF_ENTRY("print:", ASCII_PRINTING, PRINTING),
1122 PREDEF_ENTRY("punct:", ASCII_PUNCTUATION, PUNCTUATION),
1123 PREDEF_ENTRY("space:", ASCII_WHITESPACE, WHITESPACE),
1124 PREDEF_ENTRY("upper:", ASCII_UPPER, UPPER),
1125 PREDEF_ENTRY("word:", ASCII_WORD, WORD),
1126 PREDEF_ENTRY("xdigit:", HEX_DIGIT, HEX_DIGIT),
1127 PREDEF_ENTRY("ascii:", ASCII, ASCII), /* like Go */
1128
1129 /* Gauche extension - explicitly unicode range */
1130 PREDEF_ENTRY("ALPHA:", LETTER, LETTER),
1131 PREDEF_ENTRY("ALNUM:", LETTER_DIGIT, LETTER_DIGIT),
1132 PREDEF_ENTRY("BLANK:", BLANK, BLANK),
1133 PREDEF_ENTRY("CNTRL:", ISO_CONTROL, ISO_CONTROL),
1134 PREDEF_ENTRY("DIGIT:", DIGIT, DIGIT),
1135 PREDEF_ENTRY("GRAPH:", GRAPHIC, GRAPHIC),
1136 PREDEF_ENTRY("LOWER:", LOWER, LOWER),
1137 PREDEF_ENTRY("PRINT:", PRINTING, PRINTING),
1138 PREDEF_ENTRY("PUNCT:", PUNCTUATION, PUNCTUATION),
1139 PREDEF_ENTRY("SPACE:", WHITESPACE, WHITESPACE),
1140 PREDEF_ENTRY("UPPER:", UPPER, UPPER),
1141 PREDEF_ENTRY("TITLE:", TITLE, TITLE),
1142 PREDEF_ENTRY("WORD:", WORD, WORD),
1143 PREDEF_ENTRY("XDIGIT:", HEX_DIGIT, HEX_DIGIT),
1144
1145 { NULL, 0, 0 }
1146 };
1147
1148 /* Read posix [:alpha:] etc. The first '[' is already read.
1149 Return #f on error if errorp is FALSE. */
1150 static ScmObj read_predef_charset(ScmPort *input, int error_p)
1151 {
1152 #define MAX_CHARSET_NAME_LEN 11
1153 char name[MAX_CHARSET_NAME_LEN+1];
1154 int namecnt = 0;
1155 for (; namecnt < MAX_CHARSET_NAME_LEN; namecnt++) {
1156 int ch = Scm_Getc(input);
1157 if (ch == EOF && !SCM_CHAR_ASCII_P(ch)) {
1158 name[namecnt] = '\0';
1159 goto err;
1160 }
1161 if (ch == ']') break;
1162 name[namecnt] = (char)ch;
1163 }
1164 if (namecnt == MAX_CHARSET_NAME_LEN) goto err;
1165 name[namecnt] = '\0';
1166
1167 int complement = FALSE;
1168 const char *start = name+1;
1169
1170 if (*start == '^') {
1171 complement = TRUE;
1172 start++;
1173 }
1174
1175 struct predef_charset_posix_name_rec *e = predef_charset_posix_names;
1176 while (e->name != NULL) {
1177 if (strcmp(start, e->name) == 0) {
1178 if (!complement) {
1179 return Scm_GetStandardCharSet(e->cset);
1180 } else {
1181 return Scm_GetStandardCharSet(-e->cset);
1182 }
1183 }
1184 e++;
1185 }
1186 err:
1187 /* here we got invalid charset name */
1188 if (error_p) {
1189 Scm_Error("invalid or unsupported POSIX charset '[%s]'", name);
1190 }
1191 return SCM_FALSE;
1192 }
1193
1194 static struct predef_charset_category_name_rec {
1195 const char *cat;
1196 int cset;
1197 } predef_charset_category_name[] = {
1198 { "L", SCM_CHAR_SET_L },
1199 { "LC", SCM_CHAR_SET_LC },
1200 { "Lu", SCM_CHAR_SET_Lu },
1201 { "Ll", SCM_CHAR_SET_Ll },
1202 { "Lt", SCM_CHAR_SET_Lt },
1203 { "Lm", SCM_CHAR_SET_Lm },
1204 { "M", SCM_CHAR_SET_M },
1205 { "Mn", SCM_CHAR_SET_Mn },
1206 { "Mc", SCM_CHAR_SET_Mc },
1207 { "Me", SCM_CHAR_SET_Me },
1208 { "N", SCM_CHAR_SET_N },
1209 { "Nd", SCM_CHAR_SET_Nd },
1210 { "Nl", SCM_CHAR_SET_Nl },
1211 { "No", SCM_CHAR_SET_No },
1212 { "P", SCM_CHAR_SET_P },
1213 { "Pc", SCM_CHAR_SET_Pc },
1214 { "Pd", SCM_CHAR_SET_Pd },
1215 { "Ps", SCM_CHAR_SET_Ps },
1216 { "Pe", SCM_CHAR_SET_Pe },
1217 { "Pi", SCM_CHAR_SET_Pi },
1218 { "Pf", SCM_CHAR_SET_Pf },
1219 { "Po", SCM_CHAR_SET_Po },
1220 { "S", SCM_CHAR_SET_S },
1221 { "Sm", SCM_CHAR_SET_Sm },
1222 { "Sc", SCM_CHAR_SET_Sc },
1223 { "Sk", SCM_CHAR_SET_Sk },
1224 { "So", SCM_CHAR_SET_So },
1225 { "Z", SCM_CHAR_SET_Z },
1226 { "Zs", SCM_CHAR_SET_Zs },
1227 { "Zl", SCM_CHAR_SET_Zl },
1228 { "Zp", SCM_CHAR_SET_Zp },
1229 { "C", SCM_CHAR_SET_C },
1230 { "Cc", SCM_CHAR_SET_Cc },
1231 { "Cf", SCM_CHAR_SET_Cf },
1232 { "Cs", SCM_CHAR_SET_Cs },
1233 { "Co", SCM_CHAR_SET_Co },
1234 { "Cn", SCM_CHAR_SET_Cn },
1235 { NULL, 0 }
1236 };
1237
1238 /* Read \p{Category}, \P{Category}. INPUT must point right after 'p' or
1239 'P'. KEY is either 'p' or 'P'. On successful reading, Returns the
1240 charset number and update *cp to point right after the syntax.
1241 Otherwise, throws an error.
1242 */
1243 int Scm_CharSetParseCategory(ScmPort *input, char key)
1244 {
1245 int ch = Scm_Getc(input);
1246 if (ch != '{') {
1247 Scm_Error("\\%c must followed by '{'", key);
1248 }
1249 char name[3];
1250
1251 ch = Scm_Getc(input);
1252 if (ch == EOF || !SCM_CHAR_ASCII_P(ch)) {
1253 name[0] = '\0';
1254 goto bad;
1255 }
1256 name[0] = (char)ch;
1257
1258 ch = Scm_Getc(input);
1259 if (ch == EOF || !SCM_CHAR_ASCII_P(ch)) {
1260 name[1] = '\0';
1261 goto bad;
1262 }
1263 if (ch == '}') {
1264 name[1] = '\0';
1265 } else {
1266 name[1] = (char)ch;
1267 ch = Scm_Getc(input);
1268 name[2] = '\0';
1269 if (ch != '}') {
1270 goto bad;
1271 }
1272 }
1273
1274 for (int j=0; predef_charset_category_name[j].cat; j++) {
1275 if (strcmp(name, predef_charset_category_name[j].cat) == 0) {
1276 if (key == 'p') {
1277 return predef_charset_category_name[j].cset;
1278 } else {
1279 return -predef_charset_category_name[j].cset;
1280 }
1281 }
1282 }
1283 bad:
1284 Scm_Error("Bad charset category name near \\%c{%s...", key, name);
1285 return 0; /* dummy */
1286 }
1287
1288
1289 /*-----------------------------------------------------------------
1290 * Character attributes
1291 */
1292
1293 int Scm_CharGeneralCategory(ScmChar ch)
1294 {
1295 return (int)(Scm__LookupCharCategory(ch) & SCM_CHAR_CATEGORY_MASK);
1296 }
1297
1298 int Scm_CharAlphabeticP(ScmChar ch)
1299 {
1300 return (SCM_CHAR_ALPHA_MASK & Scm__LookupCharCategory(ch)) != 0;
1301 }
1302
1303 int Scm_CharUppercaseP(ScmChar ch)
1304 {
1305 return ((SCM_CHAR_ALPHA_MASK & Scm__LookupCharCategory(ch))
1306 == SCM_CHAR_UPPERCASE_BITS);
1307 }
1308
1309 int Scm_CharLowercaseP(ScmChar ch)
1310 {
1311 return ((SCM_CHAR_ALPHA_MASK & Scm__LookupCharCategory(ch))
1312 == SCM_CHAR_LOWERCASE_BITS);
1313 }
1314
1315 int Scm_CharTitlecaseP(ScmChar ch)
1316 {
1317 return (Scm_CharGeneralCategory(ch) == SCM_CHAR_CATEGORY_Lt);
1318 }
1319
1320 int Scm_CharNumericP(ScmChar ch)
1321 {
1322 return (Scm_CharGeneralCategory(ch) == SCM_CHAR_CATEGORY_Nd);
1323 }
1324
1325 /* An internal entry to extract case mapping info.
1326 * Internal table is compressed, so the caller must provide
1327 * the buffer for ScmCharCaseMap.
1328 * The function returns either the pointer to the given buffer
1329 * with information filled, or a pointer to a static read-only
1330 * data structure in the internal table.
1331 */
1332 static const ScmCharCaseMap casemap_identity = {
1333 0, 0, 0, {-1}, {-1}, {-1}
1334 };
1335
1336 const ScmCharCaseMap *Scm__CharCaseMap(ScmChar ch,
1337 ScmCharCaseMap *buf,
1338 int full)
1339 {
1340 if (ch < 0x10000) {
1341 int subtable = casemap_000[(ch >> 8) & 0xff];
1342 if (subtable == 255) return &casemap_identity;
1343
1344 unsigned short cmap =
1345 casemap_subtable[subtable][(unsigned char)(ch & 0xff)];
1346 if (cmap == SCM_CHAR_NO_CASE_MAPPING) return &casemap_identity;
1347 if (cmap & 0x8000) {
1348 /* mapping is extended. */
1349 return &(extended_casemaps[cmap & 0x7fff]);
1350 } else {
1351 /* mapping is simple */
1352 int off = (cmap & 0x2000)? (signed int)(cmap|~0x1fff) : cmap&0x1fff;
1353 if (cmap & 0x4000) {
1354 buf->to_upper_simple = off;
1355 buf->to_lower_simple = 0;
1356 buf->to_title_simple = off;
1357 } else {
1358 buf->to_upper_simple = 0;
1359 buf->to_lower_simple = off;
1360 buf->to_title_simple = 0;
1361 }
1362 if (full) {
1363 /* indicate no special mappings */
1364 buf->to_upper_full[0] = -1;
1365 buf->to_lower_full[0] = -1;
1366 buf->to_title_full[0] = -1;
1367 }
1368 return buf;
1369 }
1370 } else {
1371 /* TODO: 104xx*/
1372 return &casemap_identity;
1373 }
1374 }
1375
1376 /*
1377 * Case conversion API.
1378 */
1379 #define SIMPLE_CASE(code, buf, field) \
1380 (ScmChar)((code) + Scm__CharCaseMap((code), (buf), FALSE)->SCM_CPP_CAT3(to_, field, _simple))
1381
1382 #define SIMPLE_CASE_CV(code, buf, field) \
1383 ((code) = (ScmChar)Scm_CharToUcs((int)(code)), \
1384 (code) = SIMPLE_CASE(code, buf, field), \
1385 Scm_UcsToChar((int)(code)))
1386
1387 ScmChar Scm_CharUpcase(ScmChar ch)
1388 {
1389 ScmCharCaseMap cm;
1390 return SIMPLE_CASE(ch, &cm, upper);
1391 }
1392
1393 ScmChar Scm_CharDowncase(ScmChar ch)
1394 {
1395 ScmCharCaseMap cm;
1396 return SIMPLE_CASE(ch, &cm, lower);
1397 }
1398
1399 ScmChar Scm_CharTitlecase(ScmChar ch)
1400 {
1401 ScmCharCaseMap cm;
1402 return SIMPLE_CASE(ch, &cm, title);
1403 }
1404
1405 ScmChar Scm_CharFoldcase(ScmChar ch)
1406 {
1407 ScmCharCaseMap cm;
1408 if (ch == 0x130 || ch == 0x131) {
1409 /* char-foldcase is identity for
1410 U+0130 Turkish I (LATIN CAPITAL LETTER I WITH DOT ABOVE) and
1411 U+0131 Turkish i (LATIN SMALL LETTER DOTLESS I) */
1412 return ch;
1413 }
1414 const ScmCharCaseMap *pcm = Scm__CharCaseMap(ch, &cm, FALSE);
1415 if (pcm->to_lower_simple == 0 && pcm->to_upper_simple == 0) {
1416 /* we don't have case folding */
1417 return ch;
1418 }
1419 /* Otherwise, we do (char-downcase (char-upcase ch)) */
1420 if (pcm->to_upper_simple != 0) {
1421 ch += pcm->to_upper_simple;
1422 pcm = Scm__CharCaseMap(ch, &cm, FALSE);
1423 }
1424 return ch + pcm->to_lower_simple;
1425 }
1426
1427 /*-----------------------------------------------------------------
1428 * Pre-defined charset
1429 */
1430
1431 /* Most predefined charset are pre-generated as static immutable data.
1432 See gen-unicode.scm for the generation code. */
1433
1434 ScmObj Scm_GetStandardCharSet(int id)
1435 {
1436 if (id == 0
1437 || id >= SCM_CHAR_SET_NUM_PREDEFINED_SETS
1438 || id <= -SCM_CHAR_SET_NUM_PREDEFINED_SETS) {
1439 Scm_Error("bad id for predefined charset index: %d", id);
1440 }
1441
1442 if (id > 0) {
1443 return predef_sets[id];
1444 } else {
1445 if (!SCM_CHAR_SET_P(predef_sets_complement[-id])) {
1446 ScmObj cs = Scm_CharSetCopy(SCM_CHAR_SET(predef_sets[-id]));
1447 cs = Scm_CharSetComplement(SCM_CHAR_SET(cs));
1448 Scm_CharSetFreezeX(SCM_CHAR_SET(cs));
1449 predef_sets_complement[-id] = cs;
1450 }
1451 return predef_sets_complement[-id];
1452 }
1453 }
1454
1455 void Scm__InitChar(void)
1456 {
1457 ScmModule *mod = Scm_GaucheModule();
1458
1459 init_predefined_charsets();
1460 predef_sets[SCM_CHAR_SET_FULL] = Scm_CharSetComplement(make_charset());
1461
1462 #define DEFCS(name, id) \
1463 Scm_Define(mod, SCM_SYMBOL(SCM_INTERN("char-set:" name)), predef_sets[SCM_CPP_CAT(SCM_CHAR_SET_, id)])
1464
1465 DEFCS("L", L);
1466 DEFCS("LC", LC);
1467 DEFCS("Lu", Lu);
1468 DEFCS("Ll", Ll);
1469 DEFCS("Lt", Lt);
1470 DEFCS("Lm", Lm);
1471 DEFCS("Lo", Lo);
1472 DEFCS("M", M);
1473 DEFCS("Mn", Mn);
1474 DEFCS("Mc", Mc);
1475 DEFCS("Me", Me);
1476 DEFCS("N", N);
1477 DEFCS("Nd", Nd);
1478 DEFCS("Nl", Nl);
1479 DEFCS("No", No);
1480 DEFCS("P", P);
1481 DEFCS("Pc", Pc);
1482 DEFCS("Pd", Pd);
1483 DEFCS("Ps", Ps);
1484 DEFCS("Pe", Pe);
1485 DEFCS("Pi", Pi);
1486 DEFCS("Pf", Pf);
1487 DEFCS("Po", Po);
1488 DEFCS("S", S);
1489 DEFCS("Sm", Sm);
1490 DEFCS("Sc", Sc);
1491 DEFCS("Sk", Sk);
1492 DEFCS("So", So);
1493 DEFCS("Z", Z);
1494 DEFCS("Zs", Zs);
1495 DEFCS("Zl", Zl);
1496 DEFCS("Zp", Zp);
1497 DEFCS("C", C);
1498 DEFCS("Cc", Cc);
1499 DEFCS("Cf", Cf);
1500 DEFCS("Cs", Cs);
1501 DEFCS("Co", Co);
1502 DEFCS("Cn", Cn);
1503
1504 DEFCS("lower-case", LOWER);
1505 DEFCS("ascii-lower-case", ASCII_LOWER);
1506 DEFCS("upper-case", UPPER);
1507 DEFCS("ascii-upper-case", ASCII_UPPER);
1508 DEFCS("title-case", TITLE);
1509 DEFCS("letter", LETTER);
1510 DEFCS("ascii-letter", ASCII_LETTER);
1511 DEFCS("digit", DIGIT);
1512 DEFCS("ascii-digit", ASCII_DIGIT);
1513 DEFCS("letter+digit", LETTER_DIGIT);
1514 DEFCS("ascii-letter+digit", ASCII_LETTER_DIGIT);
1515 DEFCS("graphic", GRAPHIC);
1516 DEFCS("ascii-graphic", ASCII_GRAPHIC);
1517 DEFCS("printing", PRINTING);
1518 DEFCS("ascii-printing", ASCII_PRINTING);
1519 DEFCS("whitespace", WHITESPACE);
1520 DEFCS("ascii-whitespace", ASCII_WHITESPACE);
1521 DEFCS("iso-control", ISO_CONTROL);
1522 DEFCS("ascii-control", ASCII_CONTROL);
1523 DEFCS("punctuation", PUNCTUATION);
1524 DEFCS("ascii-punctuation", ASCII_PUNCTUATION);
1525 DEFCS("symbol", SYMBOL);
1526 DEFCS("ascii-symbol", ASCII_SYMBOL);
1527 DEFCS("hex-digit", HEX_DIGIT);
1528 DEFCS("blank", BLANK);
1529 DEFCS("ascii-blank", ASCII_BLANK);
1530 DEFCS("ascii", ASCII);
1531 DEFCS("word", WORD);
1532 DEFCS("ascii-word", ASCII_WORD);
1533 DEFCS("empty", EMPTY);
1534 DEFCS("full", FULL);
1535
1536 /* We initialize complement charset on demand, except EMPTY and FULL. */
1537 for (int i=0; i<SCM_CHAR_SET_NUM_PREDEFINED_SETS; i++) {
1538 predef_sets_complement[i] = SCM_FALSE;
1539 }
1540 predef_sets_complement[SCM_CHAR_SET_EMPTY]
1541 = predef_sets_complement[SCM_CHAR_SET_FULL];
1542 predef_sets_complement[SCM_CHAR_SET_FULL]
1543 = predef_sets_complement[SCM_CHAR_SET_EMPTY];
1544
1545 /* Expose internal charset */
1546 Scm_AddFeature("gauche.ces.utf8", NULL);
1547 }