Debian Codegraph @

src/backend/utils/mb/Unicode/UCS_to_EUC_JP.pl

raw
1 #! /usr/bin/perl
2 #
3 # Copyright (c) 2001-2025, PostgreSQL Global Development Group
4 #
5 # src/backend/utils/mb/Unicode/UCS_to_EUC_JP.pl
6 #
7 # Generate UTF-8 <--> EUC_JP code conversion tables from
8 # map files provided by Unicode organization.
9 # Unfortunately it is prohibited by the organization
10 # to distribute the map files. So if you try to use this script,
11 # you have to obtain CP932.TXT and JIS0212.TXT from the
12 # organization's ftp site.
13
14 use strict;
15 use warnings FATAL => 'all';
16
17 use convutils;
18
19 my $this_script = 'src/backend/utils/mb/Unicode/UCS_to_EUC_JP.pl';
20
21 # Load JIS0212.TXT
22 my $jis0212 = &read_source("JIS0212.TXT");
23
24 my @mapping;
25
26 foreach my $i (@$jis0212)
27 {
28
29 # We have a different mapping for this in the EUC_JP to UTF-8 direction.
30 if ($i->{code} == 0x2243)
31 {
32 $i->{direction} = FROM_UNICODE;
33 }
34
35 if ($i->{code} == 0x2271)
36 {
37 $i->{direction} = TO_UNICODE;
38 }
39
40 if ($i->{ucs} >= 0x080)
41 {
42 $i->{code} = $i->{code} | 0x8f8080;
43 }
44 else
45 {
46 next;
47 }
48
49 push @mapping, $i;
50 }
51
52 # Load CP932.TXT.
53 my $ct932 = &read_source("CP932.TXT");
54
55 foreach my $i (@$ct932)
56 {
57 my $sjis = $i->{code};
58
59 # We have a different mapping for this in the EUC_JP to UTF-8 direction.
60 if ( $sjis == 0xeefa
61 || $sjis == 0xeefb
62 || $sjis == 0xeefc)
63 {
64 next;
65 }
66
67 if ($sjis >= 0xa1)
68 {
69 my $jis = &sjis2jis($sjis);
70
71 $i->{code} = $jis | (
72 $jis < 0x100
73 ? 0x8e00
74 : ($sjis >= 0xeffd ? 0x8f8080 : 0x8080));
75
76 # Remember the SJIS code for later.
77 $i->{sjis} = $sjis;
78
79 push @mapping, $i;
80 }
81 }
82
83 # extract only SJIS characters
84 foreach my $i (grep defined $_->{sjis}, @mapping)
85 {
86 my $sjis = $i->{sjis};
87
88 # These SJIS characters are excluded completely.
89 if ( $sjis >= 0xed00 && $sjis <= 0xeef9
90 || $sjis >= 0xfa54 && $sjis <= 0xfa56
91 || $sjis >= 0xfa58 && $sjis <= 0xfc4b)
92 {
93 $i->{direction} = NONE;
94 next;
95 }
96
97 # These SJIS characters are only in the UTF-8 to EUC_JP table
98 if ($sjis == 0xeefa || $sjis == 0xeefb || $sjis == 0xeefc)
99 {
100 $i->{direction} = FROM_UNICODE;
101 next;
102 }
103
104 if ( $sjis == 0x8790
105 || $sjis == 0x8791
106 || $sjis == 0x8792
107 || $sjis == 0x8795
108 || $sjis == 0x8796
109 || $sjis == 0x8797
110 || $sjis == 0x879a
111 || $sjis == 0x879b
112 || $sjis == 0x879c
113 || ($sjis >= 0xfa4a && $sjis <= 0xfa53))
114 {
115 $i->{direction} = TO_UNICODE;
116 next;
117 }
118 }
119
120 push @mapping, (
121 {
122 direction => BOTH,
123 ucs => 0x4efc,
124 code => 0x8ff4af,
125 comment => '# CJK(4EFC)'
126 },
127 {
128 direction => BOTH,
129 ucs => 0x50f4,
130 code => 0x8ff4b0,
131 comment => '# CJK(50F4)'
132 },
133 {
134 direction => BOTH,
135 ucs => 0x51EC,
136 code => 0x8ff4b1,
137 comment => '# CJK(51EC)'
138 },
139 {
140 direction => BOTH,
141 ucs => 0x5307,
142 code => 0x8ff4b2,
143 comment => '# CJK(5307)'
144 },
145 {
146 direction => BOTH,
147 ucs => 0x5324,
148 code => 0x8ff4b3,
149 comment => '# CJK(5324)'
150 },
151 {
152 direction => BOTH,
153 ucs => 0x548A,
154 code => 0x8ff4b5,
155 comment => '# CJK(548A)'
156 },
157 {
158 direction => BOTH,
159 ucs => 0x5759,
160 code => 0x8ff4b6,
161 comment => '# CJK(5759)'
162 },
163 {
164 direction => BOTH,
165 ucs => 0x589E,
166 code => 0x8ff4b9,
167 comment => '# CJK(589E)'
168 },
169 {
170 direction => BOTH,
171 ucs => 0x5BEC,
172 code => 0x8ff4ba,
173 comment => '# CJK(5BEC)'
174 },
175 {
176 direction => BOTH,
177 ucs => 0x5CF5,
178 code => 0x8ff4bb,
179 comment => '# CJK(5CF5)'
180 },
181 {
182 direction => BOTH,
183 ucs => 0x5D53,
184 code => 0x8ff4bc,
185 comment => '# CJK(5D53)'
186 },
187 {
188 direction => BOTH,
189 ucs => 0x5FB7,
190 code => 0x8ff4be,
191 comment => '# CJK(5FB7)'
192 },
193 {
194 direction => BOTH,
195 ucs => 0x6085,
196 code => 0x8ff4bf,
197 comment => '# CJK(6085)'
198 },
199 {
200 direction => BOTH,
201 ucs => 0x6120,
202 code => 0x8ff4c0,
203 comment => '# CJK(6120)'
204 },
205 {
206 direction => BOTH,
207 ucs => 0x654E,
208 code => 0x8ff4c1,
209 comment => '# CJK(654E)'
210 },
211 {
212 direction => BOTH,
213 ucs => 0x663B,
214 code => 0x8ff4c2,
215 comment => '# CJK(663B)'
216 },
217 {
218 direction => BOTH,
219 ucs => 0x6665,
220 code => 0x8ff4c3,
221 comment => '# CJK(6665)'
222 },
223 {
224 direction => BOTH,
225 ucs => 0x6801,
226 code => 0x8ff4c6,
227 comment => '# CJK(6801)'
228 },
229 {
230 direction => BOTH,
231 ucs => 0x6A6B,
232 code => 0x8ff4c9,
233 comment => '# CJK(6A6B)'
234 },
235 {
236 direction => BOTH,
237 ucs => 0x6AE2,
238 code => 0x8ff4ca,
239 comment => '# CJK(6AE2)'
240 },
241 {
242 direction => BOTH,
243 ucs => 0x6DF2,
244 code => 0x8ff4cc,
245 comment => '# CJK(6DF2)'
246 },
247 {
248 direction => BOTH,
249 ucs => 0x6DF8,
250 code => 0x8ff4cb,
251 comment => '# CJK(6DF8)'
252 },
253 {
254 direction => BOTH,
255 ucs => 0x7028,
256 code => 0x8ff4cd,
257 comment => '# CJK(7028)'
258 },
259 {
260 direction => BOTH,
261 ucs => 0x70BB,
262 code => 0x8ff4ae,
263 comment => '# CJK(70BB)'
264 },
265 {
266 direction => BOTH,
267 ucs => 0x7501,
268 code => 0x8ff4d0,
269 comment => '# CJK(7501)'
270 },
271 {
272 direction => BOTH,
273 ucs => 0x7682,
274 code => 0x8ff4d1,
275 comment => '# CJK(7682)'
276 },
277 {
278 direction => BOTH,
279 ucs => 0x769E,
280 code => 0x8ff4d2,
281 comment => '# CJK(769E)'
282 },
283 {
284 direction => BOTH,
285 ucs => 0x7930,
286 code => 0x8ff4d4,
287 comment => '# CJK(7930)'
288 },
289 {
290 direction => BOTH,
291 ucs => 0x7AE7,
292 code => 0x8ff4d9,
293 comment => '# CJK(7AE7)'
294 },
295 {
296 direction => BOTH,
297 ucs => 0x7DA0,
298 code => 0x8ff4dc,
299 comment => '# CJK(7DA0)'
300 },
301 {
302 direction => BOTH,
303 ucs => 0x7DD6,
304 code => 0x8ff4dd,
305 comment => '# CJK(7DD6)'
306 },
307 {
308 direction => BOTH,
309 ucs => 0x8362,
310 code => 0x8ff4df,
311 comment => '# CJK(8362)'
312 },
313 {
314 direction => BOTH,
315 ucs => 0x85B0,
316 code => 0x8ff4e1,
317 comment => '# CJK(85B0)'
318 },
319 {
320 direction => BOTH,
321 ucs => 0x8807,
322 code => 0x8ff4e4,
323 comment => '# CJK(8807)'
324 },
325 {
326 direction => BOTH,
327 ucs => 0x8B7F,
328 code => 0x8ff4e6,
329 comment => '# CJK(8B7F)'
330 },
331 {
332 direction => BOTH,
333 ucs => 0x8CF4,
334 code => 0x8ff4e7,
335 comment => '# CJK(8CF4)'
336 },
337 {
338 direction => BOTH,
339 ucs => 0x8D76,
340 code => 0x8ff4e8,
341 comment => '# CJK(8D76)'
342 },
343 {
344 direction => BOTH,
345 ucs => 0x90DE,
346 code => 0x8ff4ec,
347 comment => '# CJK(90DE)'
348 },
349 {
350 direction => BOTH,
351 ucs => 0x9115,
352 code => 0x8ff4ee,
353 comment => '# CJK(9115)'
354 },
355 {
356 direction => BOTH,
357 ucs => 0x9592,
358 code => 0x8ff4f1,
359 comment => '# CJK(9592)'
360 },
361 {
362 direction => BOTH,
363 ucs => 0x973B,
364 code => 0x8ff4f4,
365 comment => '# CJK(973B)'
366 },
367 {
368 direction => BOTH,
369 ucs => 0x974D,
370 code => 0x8ff4f5,
371 comment => '# CJK(974D)'
372 },
373 {
374 direction => BOTH,
375 ucs => 0x9751,
376 code => 0x8ff4f6,
377 comment => '# CJK(9751)'
378 },
379 {
380 direction => BOTH,
381 ucs => 0x999E,
382 code => 0x8ff4fa,
383 comment => '# CJK(999E)'
384 },
385 {
386 direction => BOTH,
387 ucs => 0x9AD9,
388 code => 0x8ff4fb,
389 comment => '# CJK(9AD9)'
390 },
391 {
392 direction => BOTH,
393 ucs => 0x9B72,
394 code => 0x8ff4fc,
395 comment => '# CJK(9B72)'
396 },
397 {
398 direction => BOTH,
399 ucs => 0x9ED1,
400 code => 0x8ff4fe,
401 comment => '# CJK(9ED1)'
402 },
403 {
404 direction => BOTH,
405 ucs => 0xF929,
406 code => 0x8ff4c5,
407 comment => '# CJK COMPATIBILITY IDEOGRAPH-F929'
408 },
409 {
410 direction => BOTH,
411 ucs => 0xF9DC,
412 code => 0x8ff4f2,
413 comment => '# CJK COMPATIBILITY IDEOGRAPH-F9DC'
414 },
415 {
416 direction => BOTH,
417 ucs => 0xFA0E,
418 code => 0x8ff4b4,
419 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA0E'
420 },
421 {
422 direction => BOTH,
423 ucs => 0xFA0F,
424 code => 0x8ff4b7,
425 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA0F'
426 },
427 {
428 direction => BOTH,
429 ucs => 0xFA10,
430 code => 0x8ff4b8,
431 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA10'
432 },
433 {
434 direction => BOTH,
435 ucs => 0xFA11,
436 code => 0x8ff4bd,
437 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA11'
438 },
439 {
440 direction => BOTH,
441 ucs => 0xFA12,
442 code => 0x8ff4c4,
443 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA12'
444 },
445 {
446 direction => BOTH,
447 ucs => 0xFA13,
448 code => 0x8ff4c7,
449 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA13'
450 },
451 {
452 direction => BOTH,
453 ucs => 0xFA14,
454 code => 0x8ff4c8,
455 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA14'
456 },
457 {
458 direction => BOTH,
459 ucs => 0xFA15,
460 code => 0x8ff4ce,
461 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA15'
462 },
463 {
464 direction => BOTH,
465 ucs => 0xFA16,
466 code => 0x8ff4cf,
467 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA16'
468 },
469 {
470 direction => BOTH,
471 ucs => 0xFA17,
472 code => 0x8ff4d3,
473 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA17'
474 },
475 {
476 direction => BOTH,
477 ucs => 0xFA18,
478 code => 0x8ff4d5,
479 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA18'
480 },
481 {
482 direction => BOTH,
483 ucs => 0xFA19,
484 code => 0x8ff4d6,
485 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA19'
486 },
487 {
488 direction => BOTH,
489 ucs => 0xFA1A,
490 code => 0x8ff4d7,
491 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA1A'
492 },
493 {
494 direction => BOTH,
495 ucs => 0xFA1B,
496 code => 0x8ff4d8,
497 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA1B'
498 },
499 {
500 direction => BOTH,
501 ucs => 0xFA1C,
502 code => 0x8ff4da,
503 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA1C'
504 },
505 {
506 direction => BOTH,
507 ucs => 0xFA1D,
508 code => 0x8ff4db,
509 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA1D'
510 },
511 {
512 direction => BOTH,
513 ucs => 0xFA1E,
514 code => 0x8ff4de,
515 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA1E'
516 },
517 {
518 direction => BOTH,
519 ucs => 0xFA1F,
520 code => 0x8ff4e0,
521 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA1F'
522 },
523 {
524 direction => BOTH,
525 ucs => 0xFA20,
526 code => 0x8ff4e2,
527 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA20'
528 },
529 {
530 direction => BOTH,
531 ucs => 0xFA21,
532 code => 0x8ff4e3,
533 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA21'
534 },
535 {
536 direction => BOTH,
537 ucs => 0xFA22,
538 code => 0x8ff4e5,
539 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA22'
540 },
541 {
542 direction => BOTH,
543 ucs => 0xFA23,
544 code => 0x8ff4e9,
545 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA23'
546 },
547 {
548 direction => BOTH,
549 ucs => 0xFA24,
550 code => 0x8ff4ea,
551 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA24'
552 },
553 {
554 direction => BOTH,
555 ucs => 0xFA25,
556 code => 0x8ff4eb,
557 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA25'
558 },
559 {
560 direction => BOTH,
561 ucs => 0xFA26,
562 code => 0x8ff4ed,
563 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA26'
564 },
565 {
566 direction => BOTH,
567 ucs => 0xFA27,
568 code => 0x8ff4ef,
569 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA27'
570 },
571 {
572 direction => BOTH,
573 ucs => 0xFA28,
574 code => 0x8ff4f0,
575 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA28'
576 },
577 {
578 direction => BOTH,
579 ucs => 0xFA29,
580 code => 0x8ff4f3,
581 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA29'
582 },
583 {
584 direction => BOTH,
585 ucs => 0xFA2A,
586 code => 0x8ff4f7,
587 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA2A'
588 },
589 {
590 direction => BOTH,
591 ucs => 0xFA2B,
592 code => 0x8ff4f8,
593 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA2B'
594 },
595 {
596 direction => BOTH,
597 ucs => 0xFA2C,
598 code => 0x8ff4f9,
599 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA2C'
600 },
601 {
602 direction => BOTH,
603 ucs => 0xFA2D,
604 code => 0x8ff4fd,
605 comment => '# CJK COMPATIBILITY IDEOGRAPH-FA2D'
606 },
607 {
608 direction => BOTH,
609 ucs => 0xFF07,
610 code => 0x8ff4a9,
611 comment => '# FULLWIDTH APOSTROPHE'
612 },
613 {
614 direction => BOTH,
615 ucs => 0xFFE4,
616 code => 0x8fa2c3,
617 comment => '# FULLWIDTH BROKEN BAR'
618 },
619
620 # additional conversions for EUC_JP -> UTF-8 conversion
621 {
622 direction => TO_UNICODE,
623 ucs => 0x2116,
624 code => 0x8ff4ac,
625 comment => '# NUMERO SIGN'
626 },
627 {
628 direction => TO_UNICODE,
629 ucs => 0x2121,
630 code => 0x8ff4ad,
631 comment => '# TELEPHONE SIGN'
632 },
633 {
634 direction => TO_UNICODE,
635 ucs => 0x3231,
636 code => 0x8ff4ab,
637 comment => '# PARENTHESIZED IDEOGRAPH STOCK'
638 });
639
640 print_conversion_tables($this_script, "EUC_JP", \@mapping);
641
642
643 #######################################################################
644 # sjis2jis ; SJIS => JIS conversion
645 sub sjis2jis
646 {
647 my ($sjis) = @_;
648
649 return $sjis if ($sjis <= 0x100);
650
651 my $hi = $sjis >> 8;
652 my $lo = $sjis & 0xff;
653
654 if ($lo >= 0x80) { $lo--; }
655 $lo -= 0x40;
656 if ($hi >= 0xe0) { $hi -= 0x40; }
657 $hi -= 0x81;
658 my $pos = $lo + $hi * 0xbc;
659
660 if ($pos >= 114 * 0x5e && $pos <= 115 * 0x5e + 0x1b)
661 {
662
663 # This region (115-ku) is out of range of JIS code but for
664 # convenient to generate code in EUC CODESET 3, move this to
665 # seemingly duplicate region (83-84-ku).
666 $pos = $pos - ((31 * 0x5e) + 12);
667
668 # after 85-ku 82-ten needs to be moved 2 codepoints
669 $pos = $pos - 2 if ($pos >= 84 * 0x5c + 82);
670 }
671
672 my $hi2 = $pos / 0x5e;
673 my $lo2 = ($pos % 0x5e);
674
675 my $ret = $lo2 + 0x21 + (($hi2 + 0x21) << 8);
676
677 return $ret;
678 }