Debian Codegraph @

src/backend/utils/mb/Unicode/convutils.pm

raw
1 #
2 # Copyright (c) 2001-2025, PostgreSQL Global Development Group
3 #
4 # src/backend/utils/mb/Unicode/convutils.pm
5
6 package convutils;
7
8 use strict;
9 use warnings FATAL => 'all';
10
11 use Carp;
12 use Exporter 'import';
13
14 our @EXPORT =
15 qw( NONE TO_UNICODE FROM_UNICODE BOTH read_source print_conversion_tables);
16
17 # Constants used in the 'direction' field of the character maps
18 use constant {
19 NONE => 0,
20 TO_UNICODE => 1,
21 FROM_UNICODE => 2,
22 BOTH => 3
23 };
24
25 #######################################################################
26 # read_source - common routine to read source file
27 #
28 # fname ; input file name
29 #
30 sub read_source
31 {
32 my ($fname) = @_;
33 my @r;
34
35 open(my $in, '<', $fname) || die("cannot open $fname");
36
37 while (<$in>)
38 {
39 next if (/^#/);
40 chop;
41
42 next if (/^$/); # Ignore empty lines
43
44 next if (/^0x([0-9A-F]+)\s+(#.*)$/);
45
46 # The Unicode source files have three columns
47 # 1: The "foreign" code (in hex)
48 # 2: Unicode code point (in hex)
49 # 3: Unicode name
50 if (!/^0x([0-9A-Fa-f]+)\s+0x([0-9A-Fa-f]+)\s+(#.*)$/)
51 {
52 print STDERR "READ ERROR at line $. in $fname: $_\n";
53 exit;
54 }
55 my $out = {
56 code => hex($1),
57 ucs => hex($2),
58 comment => $4,
59 direction => BOTH,
60 f => $fname,
61 l => $.
62 };
63
64 # Ignore pure ASCII mappings. PostgreSQL character conversion code
65 # never even passes these to the conversion code.
66 next if ($out->{code} < 0x80 || $out->{ucs} < 0x80);
67
68 push(@r, $out);
69 }
70 close($in);
71
72 return \@r;
73 }
74
75 ##################################################################
76 # print_conversion_tables - output mapping tables
77 #
78 # print_conversion_tables($this_script, $csname, \%charset)
79 #
80 # this_script - the name of the *caller script* of this feature
81 # csname - character set name other than ucs
82 # charset - ref to character set array
83 #
84 # Input character set array format:
85 #
86 # Each element in the character set array is a hash. Each hash has the following fields:
87 # direction - BOTH, TO_UNICODE, or FROM_UNICODE (or NONE, to ignore the entry altogether)
88 # ucs - Unicode code point
89 # ucs_second - Second Unicode code point, if this is a "combined" character.
90 # code - Byte sequence in the "other" character set, as an integer
91 # comment - Text representation of the character
92 # f - Source filename
93 # l - Line number in source file
94 #
95 sub print_conversion_tables
96 {
97 my ($this_script, $csname, $charset) = @_;
98
99 print_conversion_tables_direction($this_script, $csname, FROM_UNICODE,
100 $charset);
101 print_conversion_tables_direction($this_script, $csname, TO_UNICODE,
102 $charset);
103 return;
104 }
105
106 #############################################################################
107 # INTERNAL ROUTINES
108
109 #######################################################################
110 # print_conversion_tables_direction - write the whole content of C source of radix tree
111 #
112 # print_conversion_tables_direction($this_script, $csname, $direction, \%charset, $tblwidth)
113 #
114 # this_script - the name of the *caller script* of this feature
115 # csname - character set name other than ucs
116 # direction - desired direction, TO_UNICODE or FROM_UNICODE
117 # charset - ref to character set array
118 #
119 sub print_conversion_tables_direction
120 {
121 my ($this_script, $csname, $direction, $charset) = @_;
122
123 my $fname;
124 my $tblname;
125 if ($direction == TO_UNICODE)
126 {
127 $fname = lc("${csname}_to_utf8.map");
128 $tblname = lc("${csname}_to_unicode_tree");
129
130 print "- Writing ${csname}=>UTF8 conversion table: $fname\n";
131 }
132 else
133 {
134 $fname = lc("utf8_to_${csname}.map");
135 $tblname = lc("${csname}_from_unicode_tree");
136
137 print "- Writing UTF8=>${csname} conversion table: $fname\n";
138 }
139
140 open(my $out, '>', $fname) || die("cannot open $fname");
141
142 print $out "/* src/backend/utils/mb/Unicode/$fname */\n";
143 print $out "/* This file is generated by $this_script */\n\n";
144
145 # Collect regular, non-combined, mappings, and create the radix tree from them.
146 my $charmap = &make_charmap($out, $charset, $direction, 0);
147 print_radix_table($out, $tblname, $charmap);
148
149 # Collect combined characters, and create combined character table (if any)
150 my $charmap_combined = &make_charmap_combined($charset, $direction);
151
152 if (scalar @{$charmap_combined} > 0)
153 {
154 if ($direction == TO_UNICODE)
155 {
156 print_to_utf8_combined_map($out, $csname, $charmap_combined, 1);
157 }
158 else
159 {
160 print_from_utf8_combined_map($out, $csname, $charmap_combined, 1);
161 }
162 }
163
164 close($out);
165 return;
166 }
167
168 sub print_from_utf8_combined_map
169 {
170 my ($out, $charset, $table, $verbose) = @_;
171
172 my $last_comment = "";
173
174 printf $out "\n/* Combined character map */\n";
175 printf $out
176 "static const pg_utf_to_local_combined ULmap${charset}_combined[%d] = {",
177 scalar(@$table);
178 my $first = 1;
179 foreach my $i (sort { $a->{utf8} <=> $b->{utf8} } @$table)
180 {
181 print($out ",") if (!$first);
182 $first = 0;
183 print $out "\t/* $last_comment */"
184 if ($verbose && $last_comment ne "");
185
186 printf $out "\n {0x%08x, 0x%08x, 0x%04x}",
187 $i->{utf8}, $i->{utf8_second}, $i->{code};
188 if ($verbose >= 2)
189 {
190 $last_comment =
191 sprintf("%s:%d %s", $i->{f}, $i->{l}, $i->{comment});
192 }
193 elsif ($verbose >= 1)
194 {
195 $last_comment = $i->{comment};
196 }
197 }
198 print $out "\t/* $last_comment */" if ($verbose && $last_comment ne "");
199 print $out "\n};\n";
200 return;
201 }
202
203 sub print_to_utf8_combined_map
204 {
205 my ($out, $charset, $table, $verbose) = @_;
206
207 my $last_comment = "";
208
209 printf $out "\n/* Combined character map */\n";
210 printf $out
211 "static const pg_local_to_utf_combined LUmap${charset}_combined[%d] = {",
212 scalar(@$table);
213
214 my $first = 1;
215 foreach my $i (sort { $a->{code} <=> $b->{code} } @$table)
216 {
217 print($out ",") if (!$first);
218 $first = 0;
219 print $out "\t/* $last_comment */"
220 if ($verbose && $last_comment ne "");
221
222 printf $out "\n {0x%04x, 0x%08x, 0x%08x}",
223 $i->{code}, $i->{utf8}, $i->{utf8_second};
224
225 if ($verbose >= 2)
226 {
227 $last_comment =
228 sprintf("%s:%d %s", $i->{f}, $i->{l}, $i->{comment});
229 }
230 elsif ($verbose >= 1)
231 {
232 $last_comment = $i->{comment};
233 }
234 }
235 print $out "\t/* $last_comment */" if ($verbose && $last_comment ne "");
236 print $out "\n};\n";
237 return;
238 }
239
240 #######################################################################
241 # print_radix_table(<output handle>, <table name>, <charmap hash ref>)
242 #
243 # Input: A hash, mapping an input character to an output character.
244 #
245 # Constructs a radix tree from the hash, and prints it out as a C-struct.
246 #
247 sub print_radix_table
248 {
249 my ($out, $tblname, $c) = @_;
250
251 ###
252 ### Build radix trees in memory, for 1-, 2-, 3- and 4-byte inputs. Each
253 ### radix tree is represented as a nested hash, each hash indexed by
254 ### input byte
255 ###
256 my %b1map;
257 my %b2map;
258 my %b3map;
259 my %b4map;
260 foreach my $in (keys %$c)
261 {
262 my $out = $c->{$in};
263
264 if ($in <= 0xff)
265 {
266 $b1map{$in} = $out;
267 }
268 elsif ($in <= 0xffff)
269 {
270 my $b1 = $in >> 8;
271 my $b2 = $in & 0xff;
272
273 $b2map{$b1}{$b2} = $out;
274 }
275 elsif ($in <= 0xffffff)
276 {
277 my $b1 = $in >> 16;
278 my $b2 = ($in >> 8) & 0xff;
279 my $b3 = $in & 0xff;
280
281 $b3map{$b1}{$b2}{$b3} = $out;
282 }
283 elsif ($in <= 0xffffffff)
284 {
285 my $b1 = $in >> 24;
286 my $b2 = ($in >> 16) & 0xff;
287 my $b3 = ($in >> 8) & 0xff;
288 my $b4 = $in & 0xff;
289
290 $b4map{$b1}{$b2}{$b3}{$b4} = $out;
291 }
292 else
293 {
294 die sprintf("up to 4 byte code is supported: %x", $in);
295 }
296 }
297
298 my @segments;
299
300 ###
301 ### Build a linear list of "segments", from the nested hashes.
302 ###
303 ### Each segment is a lookup table, keyed by the next byte in the input.
304 ### The segments are written out physically to one big array in the final
305 ### step, but logically, they form a radix tree. Or rather, four radix
306 ### trees: one for 1-byte inputs, another for 2-byte inputs, 3-byte
307 ### inputs, and 4-byte inputs.
308 ###
309 ### Each segment is represented by a hash with following fields:
310 ###
311 ### comment => <string to output as a comment>
312 ### label => <label that can be used to refer to this segment from elsewhere>
313 ### values => <a hash, keyed by byte, 0-0xff>
314 ###
315 ### Entries in 'values' can be integers (for leaf-level segments), or
316 ### string labels, pointing to a segment with that label. Any missing
317 ### values are treated as zeros. If 'values' hash is missing altogether,
318 ### it's treated as all-zeros.
319 ###
320 ### Subsequent steps will enrich the segments with more fields.
321 ###
322
323 # Add the segments for the radix trees themselves.
324 push @segments,
325 build_segments_from_tree("Single byte table", "1-byte", 1, \%b1map);
326 push @segments,
327 build_segments_from_tree("Two byte table", "2-byte", 2, \%b2map);
328 push @segments,
329 build_segments_from_tree("Three byte table", "3-byte", 3, \%b3map);
330 push @segments,
331 build_segments_from_tree("Four byte table", "4-byte", 4, \%b4map);
332
333 ###
334 ### Find min and max index used in each level of each tree.
335 ###
336 ### These are stored separately, and we can then leave out the unused
337 ### parts of every segment. (When using the resulting tree, you must
338 ### check each input byte against the min and max.)
339 ###
340 my %min_idx;
341 my %max_idx;
342 foreach my $seg (@segments)
343 {
344 my $this_min = $min_idx{ $seg->{depth} }->{ $seg->{level} };
345 my $this_max = $max_idx{ $seg->{depth} }->{ $seg->{level} };
346
347 foreach my $i (keys %{ $seg->{values} })
348 {
349 $this_min = $i if (!defined $this_min || $i < $this_min);
350 $this_max = $i if (!defined $this_max || $i > $this_max);
351 }
352
353 $min_idx{ $seg->{depth} }{ $seg->{level} } = $this_min;
354 $max_idx{ $seg->{depth} }{ $seg->{level} } = $this_max;
355 }
356
357 # Copy the mins and max's back to every segment, for convenience.
358 foreach my $seg (@segments)
359 {
360 $seg->{min_idx} = $min_idx{ $seg->{depth} }{ $seg->{level} };
361 $seg->{max_idx} = $max_idx{ $seg->{depth} }{ $seg->{level} };
362 }
363
364 ###
365 ### Prepend a dummy all-zeros map to the beginning.
366 ###
367 ### A 0 is an invalid value anywhere in the table, and this allows us to
368 ### point to 0 offset from any table, to get a 0 result.
369 ###
370
371 # Find the max range between min and max indexes in any of the segments.
372 my $widest_range = 0;
373 foreach my $seg (@segments)
374 {
375 my $this_range = $seg->{max_idx} - $seg->{min_idx};
376 $widest_range = $this_range if ($this_range > $widest_range);
377 }
378
379 unshift @segments,
380 {
381 header => "Dummy map, for invalid values",
382 min_idx => 0,
383 max_idx => $widest_range,
384 label => "dummy map"
385 };
386
387 ###
388 ### Eliminate overlapping zeros
389 ###
390 ### For each segment, if there are zero values at the end of, and there
391 ### are also zero values at the beginning of the next segment, we can
392 ### overlay the tail of this segment with the head of next segment, to
393 ### save space.
394 ###
395 ### To achieve that, we subtract the 'max_idx' of each segment with the
396 ### amount of zeros that can be overlaid.
397 ###
398 for (my $j = 0; $j < $#segments - 1; $j++)
399 {
400 my $seg = $segments[$j];
401 my $nextseg = $segments[ $j + 1 ];
402
403 # Count the number of zero values at the end of this segment.
404 my $this_trail_zeros = 0;
405 for (
406 my $i = $seg->{max_idx};
407 $i >= $seg->{min_idx} && !$seg->{values}->{$i};
408 $i--)
409 {
410 $this_trail_zeros++;
411 }
412
413 # Count the number of zeros at the beginning of next segment.
414 my $next_lead_zeros = 0;
415 for (
416 my $i = $nextseg->{min_idx};
417 $i <= $nextseg->{max_idx} && !$nextseg->{values}->{$i};
418 $i++)
419 {
420 $next_lead_zeros++;
421 }
422
423 # How many zeros in common?
424 my $overlaid_trail_zeros =
425 ($this_trail_zeros > $next_lead_zeros)
426 ? $next_lead_zeros
427 : $this_trail_zeros;
428
429 $seg->{overlaid_trail_zeros} = $overlaid_trail_zeros;
430 $seg->{max_idx} = $seg->{max_idx} - $overlaid_trail_zeros;
431 }
432
433 ###
434 ### Replace label references with real offsets.
435 ###
436 ### So far, the non-leaf segments have referred to other segments by
437 ### their labels. Replace them with numerical offsets from the beginning
438 ### of the final array. You cannot move, add, or remove segments after
439 ### this step, as that would invalidate the offsets calculated here!
440 ###
441 my $flatoff = 0;
442 my %segmap;
443
444 # First pass: assign offsets to each segment, and build hash
445 # of label => offset.
446 foreach my $seg (@segments)
447 {
448 $seg->{offset} = $flatoff;
449 $segmap{ $seg->{label} } = $flatoff;
450 $flatoff += $seg->{max_idx} - $seg->{min_idx} + 1;
451 }
452 my $tblsize = $flatoff;
453
454 # Second pass: look up the offset of each label reference in the hash.
455 foreach my $seg (@segments)
456 {
457 while (my ($i, $val) = each %{ $seg->{values} })
458 {
459 if (!($val =~ /^[0-9,.E]+$/))
460 {
461 my $segoff = $segmap{$val};
462 if ($segoff)
463 {
464 $seg->{values}->{$i} = $segoff;
465 }
466 else
467 {
468 die "no segment with label $val";
469 }
470 }
471 }
472 }
473
474 # Also look up the positions of the roots in the table.
475 # Missing map represents dummy mapping.
476 my $b1root = $segmap{"1-byte"} || 0;
477 my $b2root = $segmap{"2-byte"} || 0;
478 my $b3root = $segmap{"3-byte"} || 0;
479 my $b4root = $segmap{"4-byte"} || 0;
480
481 # And the lower-upper values of each level in each radix tree.
482 # Missing values represent zero.
483 my $b1_lower = $min_idx{1}{1} || 0;
484 my $b1_upper = $max_idx{1}{1} || 0;
485
486 my $b2_1_lower = $min_idx{2}{1} || 0;
487 my $b2_1_upper = $max_idx{2}{1} || 0;
488 my $b2_2_lower = $min_idx{2}{2} || 0;
489 my $b2_2_upper = $max_idx{2}{2} || 0;
490
491 my $b3_1_lower = $min_idx{3}{1} || 0;
492 my $b3_1_upper = $max_idx{3}{1} || 0;
493 my $b3_2_lower = $min_idx{3}{2} || 0;
494 my $b3_2_upper = $max_idx{3}{2} || 0;
495 my $b3_3_lower = $min_idx{3}{3} || 0;
496 my $b3_3_upper = $max_idx{3}{3} || 0;
497
498 my $b4_1_lower = $min_idx{4}{1} || 0;
499 my $b4_1_upper = $max_idx{4}{1} || 0;
500 my $b4_2_lower = $min_idx{4}{2} || 0;
501 my $b4_2_upper = $max_idx{4}{2} || 0;
502 my $b4_3_lower = $min_idx{4}{3} || 0;
503 my $b4_3_upper = $max_idx{4}{3} || 0;
504 my $b4_4_lower = $min_idx{4}{4} || 0;
505 my $b4_4_upper = $max_idx{4}{4} || 0;
506
507 ###
508 ### Find the maximum value in the whole table, to determine if we can
509 ### use uint16 or if we need to use uint32.
510 ###
511 my $max_val = 0;
512 foreach my $seg (@segments)
513 {
514 foreach my $val (values %{ $seg->{values} })
515 {
516 $max_val = $val if ($val > $max_val);
517 }
518 }
519
520 my $datatype = ($max_val <= 0xffff) ? "uint16" : "uint32";
521
522 # For formatting, determine how many values we can fit on a single
523 # line, and how wide each value needs to be to align nicely.
524 my $vals_per_line;
525 my $colwidth;
526
527 if ($max_val <= 0xffff)
528 {
529 $vals_per_line = 8;
530 $colwidth = 4;
531 }
532 elsif ($max_val <= 0xffffff)
533 {
534 $vals_per_line = 4;
535 $colwidth = 6;
536 }
537 else
538 {
539 $vals_per_line = 4;
540 $colwidth = 8;
541 }
542
543 ###
544 ### Print the struct and array.
545 ###
546 printf $out "static const $datatype ${tblname}_table[$tblsize];\n";
547 printf $out "\n";
548 printf $out "static const pg_mb_radix_tree $tblname =\n";
549 printf $out "{\n";
550 if ($datatype eq "uint16")
551 {
552 print $out " ${tblname}_table,\n";
553 print $out " NULL, /* 32-bit table not used */\n";
554 }
555 if ($datatype eq "uint32")
556 {
557 print $out " NULL, /* 16-bit table not used */\n";
558 print $out " ${tblname}_table,\n";
559 }
560 printf $out "\n";
561 printf $out " 0x%04x, /* offset of table for 1-byte inputs */\n",
562 $b1root;
563 printf $out " 0x%02x, /* b1_lower */\n", $b1_lower;
564 printf $out " 0x%02x, /* b1_upper */\n", $b1_upper;
565 printf $out "\n";
566 printf $out " 0x%04x, /* offset of table for 2-byte inputs */\n",
567 $b2root;
568 printf $out " 0x%02x, /* b2_1_lower */\n", $b2_1_lower;
569 printf $out " 0x%02x, /* b2_1_upper */\n", $b2_1_upper;
570 printf $out " 0x%02x, /* b2_2_lower */\n", $b2_2_lower;
571 printf $out " 0x%02x, /* b2_2_upper */\n", $b2_2_upper;
572 printf $out "\n";
573 printf $out " 0x%04x, /* offset of table for 3-byte inputs */\n",
574 $b3root;
575 printf $out " 0x%02x, /* b3_1_lower */\n", $b3_1_lower;
576 printf $out " 0x%02x, /* b3_1_upper */\n", $b3_1_upper;
577 printf $out " 0x%02x, /* b3_2_lower */\n", $b3_2_lower;
578 printf $out " 0x%02x, /* b3_2_upper */\n", $b3_2_upper;
579 printf $out " 0x%02x, /* b3_3_lower */\n", $b3_3_lower;
580 printf $out " 0x%02x, /* b3_3_upper */\n", $b3_3_upper;
581 printf $out "\n";
582 printf $out " 0x%04x, /* offset of table for 4-byte inputs */\n",
583 $b4root;
584 printf $out " 0x%02x, /* b4_1_lower */\n", $b4_1_lower;
585 printf $out " 0x%02x, /* b4_1_upper */\n", $b4_1_upper;
586 printf $out " 0x%02x, /* b4_2_lower */\n", $b4_2_lower;
587 printf $out " 0x%02x, /* b4_2_upper */\n", $b4_2_upper;
588 printf $out " 0x%02x, /* b4_3_lower */\n", $b4_3_lower;
589 printf $out " 0x%02x, /* b4_3_upper */\n", $b4_3_upper;
590 printf $out " 0x%02x, /* b4_4_lower */\n", $b4_4_lower;
591 printf $out " 0x%02x /* b4_4_upper */\n", $b4_4_upper;
592 print $out "};\n";
593 print $out "\n";
594 print $out "static const $datatype ${tblname}_table[$tblsize] =\n";
595 print $out "{";
596 my $off = 0;
597
598 foreach my $seg (@segments)
599 {
600 printf $out "\n";
601 printf $out " /*** %s - offset 0x%05x ***/\n", $seg->{header}, $off;
602 printf $out "\n";
603
604 for (my $i = $seg->{min_idx}; $i <= $seg->{max_idx};)
605 {
606
607 # Print the next line's worth of values.
608 # XXX pad to begin at a nice boundary
609 printf $out " /* %02x */ ", $i;
610 for (
611 my $j = 0;
612 $j < $vals_per_line && $i <= $seg->{max_idx};
613 $j++)
614 {
615 # missing values represent zero.
616 my $val = $seg->{values}->{$i} || 0;
617
618 printf $out " 0x%0*x", $colwidth, $val;
619 $off++;
620 if ($off != $tblsize)
621 {
622 print $out ",";
623 }
624 $i++;
625 }
626 print $out "\n";
627 }
628 if ($seg->{overlaid_trail_zeros})
629 {
630 printf $out
631 " /* $seg->{overlaid_trail_zeros} trailing zero values shared with next segment */\n";
632 }
633 }
634
635 # Sanity check.
636 if ($off != $tblsize) { die "table size didn't match!"; }
637
638 print $out "};\n";
639 return;
640 }
641
642 ###
643 sub build_segments_from_tree
644 {
645 my ($header, $rootlabel, $depth, $map) = @_;
646
647 my @segments;
648
649 if (%{$map})
650 {
651 @segments =
652 build_segments_recurse($header, $rootlabel, "", 1, $depth, $map);
653
654 # Sort the segments into "breadth-first" order. Not strictly required,
655 # but makes the maps nicer to read.
656 @segments =
657 sort { $a->{level} cmp $b->{level} or $a->{path} cmp $b->{path} }
658 @segments;
659 }
660
661 return @segments;
662 }
663
664 ###
665 sub build_segments_recurse
666 {
667 my ($header, $label, $path, $level, $depth, $map) = @_;
668
669 my @segments;
670
671 if ($level == $depth)
672 {
673 push @segments,
674 {
675 header => $header . ", leaf: ${path}xx",
676 label => $label,
677 level => $level,
678 depth => $depth,
679 path => $path,
680 values => $map
681 };
682 }
683 else
684 {
685 my %children;
686
687 while (my ($i, $val) = each %$map)
688 {
689 my $childpath = $path . sprintf("%02x", $i);
690 my $childlabel = "$depth-level-$level-$childpath";
691
692 push @segments,
693 build_segments_recurse($header, $childlabel, $childpath,
694 $level + 1, $depth, $val);
695 $children{$i} = $childlabel;
696 }
697
698 push @segments,
699 {
700 header => $header . ", byte #$level: ${path}xx",
701 label => $label,
702 level => $level,
703 depth => $depth,
704 path => $path,
705 values => \%children
706 };
707 }
708 return @segments;
709 }
710
711 #######################################################################
712 # make_charmap - convert charset table to charmap hash
713 #
714 # make_charmap(\@charset, $direction)
715 # charset - ref to charset table : see print_conversion_tables
716 # direction - conversion direction
717 #
718 sub make_charmap
719 {
720 my ($out, $charset, $direction, $verbose) = @_;
721
722 croak "unacceptable direction : $direction"
723 if ($direction != TO_UNICODE && $direction != FROM_UNICODE);
724
725 # In verbose mode, print a large comment with the source and comment of
726 # each character
727 if ($verbose)
728 {
729 print $out "/*\n";
730 print $out "<src> <dst> <file>:<lineno> <comment>\n";
731 }
732
733 my %charmap;
734 foreach my $c (@$charset)
735 {
736
737 # combined characters are handled elsewhere
738 next if (defined $c->{ucs_second});
739
740 next if ($c->{direction} != $direction && $c->{direction} != BOTH);
741
742 my ($src, $dst) =
743 $direction == TO_UNICODE
744 ? ($c->{code}, ucs2utf($c->{ucs}))
745 : (ucs2utf($c->{ucs}), $c->{code});
746
747 # check for duplicate source codes
748 if (defined $charmap{$src})
749 {
750 printf STDERR
751 "Error: duplicate source code on %s:%d: 0x%04x => 0x%04x, 0x%04x\n",
752 $c->{f}, $c->{l}, $src, $charmap{$src}, $dst;
753 exit;
754 }
755 $charmap{$src} = $dst;
756
757 if ($verbose)
758 {
759 printf $out "0x%04x 0x%04x %s:%d %s\n", $src, $dst, $c->{f},
760 $c->{l}, $c->{comment};
761 }
762 }
763 if ($verbose)
764 {
765 print $out "*/\n\n";
766 }
767
768 return \%charmap;
769 }
770
771 #######################################################################
772 # make_charmap_combined - convert charset table to charmap hash
773 # with checking duplicate source code
774 #
775 # make_charmap_combined(\@charset, $direction)
776 # charset - ref to charset table : see print_conversion_tables
777 # direction - conversion direction
778 #
779 sub make_charmap_combined
780 {
781 my ($charset, $direction) = @_;
782
783 croak "unacceptable direction : $direction"
784 if ($direction != TO_UNICODE && $direction != FROM_UNICODE);
785
786 my @combined;
787 foreach my $c (@$charset)
788 {
789 next if ($c->{direction} != $direction && $c->{direction} != BOTH);
790
791 if (defined $c->{ucs_second})
792 {
793 my $entry = {
794 utf8 => ucs2utf($c->{ucs}),
795 utf8_second => ucs2utf($c->{ucs_second}),
796 code => $c->{code},
797 comment => $c->{comment},
798 f => $c->{f},
799 l => $c->{l}
800 };
801 push @combined, $entry;
802 }
803 }
804
805 return \@combined;
806 }
807
808 #######################################################################
809 # convert UCS-4 to UTF-8
810 #
811 sub ucs2utf
812 {
813 my ($ucs) = @_;
814 my $utf;
815
816 if ($ucs <= 0x007f)
817 {
818 $utf = $ucs;
819 }
820 elsif ($ucs > 0x007f && $ucs <= 0x07ff)
821 {
822 $utf = (($ucs & 0x003f) | 0x80) | ((($ucs >> 6) | 0xc0) << 8);
823 }
824 elsif ($ucs > 0x07ff && $ucs <= 0xffff)
825 {
826 $utf =
827 ((($ucs >> 12) | 0xe0) << 16) |
828 (((($ucs & 0x0fc0) >> 6) | 0x80) << 8) | (($ucs & 0x003f) | 0x80);
829 }
830 else
831 {
832 $utf =
833 ((($ucs >> 18) | 0xf0) << 24) |
834 (((($ucs & 0x3ffff) >> 12) | 0x80) << 16) |
835 (((($ucs & 0x0fc0) >> 6) | 0x80) << 8) | (($ucs & 0x003f) | 0x80);
836 }
837 return $utf;
838 }
839
840 1;