Debian Codegraph @

lib/TEIHandlerxml_xml.pm

raw
1 # SAX handler module for tei2dictxml_xml converter
2
3 # V1.3 10/2006 Michael Bunk
4 # * removed trailing white space
5 # * added error handler, so XML:ESISParser doesn't die() on warnings
6 # V1.2 5/2004 Michael Bunk
7 # * added option to generate 00-database-allchars header
8 # * if reverse index is generated, 00-database-short will indicate it now
9 # V1.1 4/2004 Michael Bunk
10 # * when headwords contain entities, characters()
11 # is called multiple times for the same element:
12 # now only one call to add_headword()
13 # will be done
14 # V1.0 6/2003 Michael Bunk
15
16 # This program is free software; you can redistribute it and/or modify
17 # it under the terms of the GNU General Public License as published by
18 # the Free Software Foundation; either version 2 of the License, or
19 # (at your option) any later version.
20 #
21 # This program is distributed in the hope that it will be useful,
22 # but WITHOUT ANY WARRANTY; without even the implied warranty of
23 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
24 # GNU General Public License for more details.
25 #
26 # You should have received a copy of the GNU General Public License
27 # along with this program; if not, visit
28 # <http://www.gnu.org/licenses/gpl-2.0.txt>
29
30 # thinking about event & calling order of the Dict subs (there is no other doc?):
31 # 1. set_file_name() -> open_dict()
32 # 2. <orth> -> set_headword()
33 # 3. characters() -> $current_headword .= $data;
34 # 4. end_tag -> add_headword($current_headword)
35 # 5. </orth> -> end_headword()
36 # 6. several start_tags -> ( write_direct() | add_text() )
37 # 7. several end_tags -> write_text()
38 # 8. </entry> -> write_newline()
39 # 9. continue with 2. for all other entrys
40 # 10. closing the filehandles not needed
41
42 # for encoding tei header we need some extra:
43 # V1.1.:
44 # 1. same
45 # 2. <teiHeader> -> set_headword(); write_text("00-database-info"); end_headword();
46 # 3. header start_tags -> add_text(" Info: ");
47 # 4. tag content (if any) -> add_text($data); and occasionally save to some $var
48 # 5. header end_tags -> add_text("\n");
49 # 6. </teiHeader> -> write_text();
50 # set_headword(); write_text("00-database-short"); end_headword();
51 # write_direct($database-short);
52 # set_headword(); write_text("00-database-url"); end_headword();
53 # write_direct($database-url);
54 # 7. go on with entrys like in V1.0 (go on at 2. there)
55
56 package lib::TEIHandlerxml_xml;
57
58 use strict;
59 use warnings;
60 use lib::Dict;
61 use utf8;
62
63 # for filtering:
64 use FileHandle;
65 use IPC::Open2;
66
67 # for sablotron:
68 use XML::Sablotron;
69 use XML::Sablotron::DOM;
70
71 our ($header, $preLastStartTag, $lastStartTag,
72 $database_short, $database_url, @higherElements,
73 $file_name, $skip_header, $state, $quoted,
74 $filtercmd, $sit, $sab, $reverse_index, $sab_templ,
75 $current_headword);
76
77 # i copied _escape() from XML::Handler::CanonXMLWriter
78 # for escaping special chars in HTML/XML/SGML
79 my %char_entities = (
80 "\x09" => '&#9;',
81 "\x0a" => '&#10;',
82 "\x0d" => '&#13;',
83 '&' => '&amp;',
84 '<' => '&lt;',
85 '>' => '&gt;',
86 '"' => '&quot;',
87 );
88
89 sub _escape {
90 my $string = shift;
91
92 $string =~ s/([\x09\x0a\x0d&<>\"])/$char_entities{$1}/ge;
93
94 return $string;
95 }
96
97
98 sub new {
99 my ($type) = shift;
100 $header = 0;
101 $state="";
102 return bless {}, $type;
103 }
104
105 sub dumpTag {
106 my $element = shift;
107
108 #dump attributes
109 my $attr="";
110 if(!defined $element)
111 {
112 die "Please make sure the input file is valid!";
113 }
114 my %attrhash = %{$element->{Attributes}};
115 foreach(keys %attrhash) {
116 $attr .= ' '.$_.'="'._escape($attrhash{$_}).'"';
117 }
118
119 return "<".$element->{Name}.$attr.">";
120
121 }
122
123 sub apply_filter_cmd {
124 my ($filtercommand, $quoted) = @_;
125
126 if($filtercmd ne "") {
127 print STDERR ".";
128 #print "calling open2 with '$filtercmd'...";
129 #TODO: catch errors from executed filtercmd
130 # according to 'man perlipc'
131 my $pid = open2(*Reader, *Writer, $filtercmd );
132 #print "handing over...";flush STDOUT;
133 print Writer "$quoted\n";
134 close(Writer);
135 #print "reading...";flush STDOUT;
136 $quoted = <Reader>;
137 while(!eof(Reader)) { $quoted .= <Reader>; };
138 #print "\n";flush STDOUT;
139 waitpid $pid, 0;
140 }
141 elsif ($sab) {
142
143 # print STDERR "+";
144 $sab->addArg($sit, 'input', $quoted);
145
146 # print STDERR "\ncalling addArgTree...";
147 $sab->addArgTree($sit, 'template', $sab_templ);
148
149 # print STDERR "\ncalling process...";
150 $sab->process($sit, 'arg:/template', 'arg:/input', 'arg:/output');
151
152 # print STDERR "\ncalling getResultArg...";
153 $quoted = $sab->getResultArg('arg:/output');
154
155 # print STDERR "\ngot: $quoted\n\n\n\n";
156 }
157
158 return $quoted;
159 }
160
161 sub set_options {
162 my ($self, $filename, $askip_header, $aGenerate00DatabaseUtf8, $aLocale,
163 $aFilterCmd, $aStyleSheet, $aReverseindex, $aAllchars) = @_;
164 $file_name = $filename;
165 $file_name =~ s/^(\w*\/)+//g;
166 $file_name =~ s/(\S*)\.\w*/$1/;
167 $skip_header=$askip_header;
168 $filtercmd=$aFilterCmd;
169 $reverse_index=$aReverseindex;
170
171 if ($aStyleSheet) {
172 print STDERR "initializing Sablotron...\n";
173 $sab = new XML::Sablotron;
174 $sit = new XML::Sablotron::Situation;
175 $sab_templ = XML::Sablotron::DOM::parseStylesheet($sit, $aStyleSheet);
176 }
177
178 Dict->open_dict($file_name,$aLocale,$aGenerate00DatabaseUtf8);
179
180 if($aGenerate00DatabaseUtf8)
181 {
182 print STDERR "Generating 00-database-utf8\n";
183 Dict::set_headword(); Dict::add_text("00-database-utf8");
184 Dict::write_text(); Dict::end_headword();
185
186 # might not be needed at all, but we give a hint
187 Dict::write_direct("\n ");
188 Dict::write_direct("This dictionary is UTF-8 encoded. If you use dictd, ".
189 "make sure to start it with the appropriate --locale option.");
190 Dict::write_newline();
191 }
192
193 if($aAllchars)
194 {
195 print STDERR "Generating 00-database-allchars\n";
196 Dict::set_headword(); Dict::add_text("00-database-allchars");
197 Dict::write_text(); Dict::end_headword();
198 Dict::write_newline();
199 }
200 }
201
202 sub start_element {
203 my ($self, $element) = @_;
204 my $part = $element->{Name};
205
206 $preLastStartTag = $lastStartTag; $lastStartTag = $part;
207
208 # for multiple orths at beginning of <form>
209 if (!$reverse_index && ($Dict::head) && ($part ne "orth")) { Dict::end_headword() };
210
211
212 # see what tag we have (& remember XML is case sensitive!)
213 # most frequent tags on top
214 if (($part eq "entry") ||
215 ($part eq "teiHeader")) {
216 $quoted = "";
217 $state = "quoting";
218 }
219
220 elsif ($part eq "orth" && !$reverse_index) {
221 if ($preLastStartTag eq "form") {Dict::set_headword()}
222 $state = "orth";$current_headword='';
223 }
224
225 elsif ($part eq "tr" && $reverse_index) {
226 if (!$Dict::head) {Dict::set_headword()}
227 $state = "tr";$current_headword='';
228 }
229
230
231 if ($part eq "revisionDesc") {
232 # add myself as
233 # <change>
234 # <date>now</date>
235 # <respStmt><name>tei2dict_xml.pl</name></respStmt>
236 # <item>converted TEI database into .dict format</item>
237 # </change>
238 my $now_string = gmtime();
239 my $s .= "<change>\n <date>$now_string</date>\n";
240 $s .= " <respStmt><name>tei2dictxml_xml.pl</name></respStmt>\n";
241 $s .= " <item>converted TEI database into .dict format</item></change>\n";
242 $quoted .= dumpTag($element) . $s;
243 # printf STDERR "new revisionDesc: $quoted";
244 }
245 elsif (($state eq "quoting") ||
246 ($state eq "orth") ||
247 ($state eq "tr")) {
248 $quoted .= dumpTag($element);
249 }
250
251 elsif (($part eq "text") ||
252 ($part eq "body") ||
253 ($part eq "TEI.2")) {
254 # skip these tags without warnings
255 }
256 else {
257 print STDERR "warning: skipping start tag: $part\n" }
258 }
259
260 sub characters {
261 my ($self, $element) = @_;
262 my $data = $element->{Data};
263
264 # FIXME: maybe with other parser than ESISParser?
265 # my guess: sometimes our loved ESISParser calls this handler
266 # while it shouldn't: to notify us of a line break and white
267 # space. there are handlers end_record() and white_space() for
268 # this. anyway: since we don't have <pre>-element where newlines
269 # are important inside, we just replace all \n by ' '
270 # then all multiple white space by single ' '
271 # then remove all ' ' at beginning or end
272 # and skip everything if we get empty $data
273
274 # do some whitespace stripping:
275 return if ($data =~ /\A\s*\Z/);
276
277 $data =~ s/\n/ /;
278 $data =~ s/\s+/ /;
279
280 if ($data eq "") {
281 # can happen because of newlines
282 return;
283 }
284
285 if (($state eq "quoting") ||
286 ($state eq "orth") ||
287 ($state eq "tr")) {
288 $quoted .= _escape($data);
289 }
290
291 if (($preLastStartTag eq "titleStmt") &&
292 ($lastStartTag eq "title")) { $database_short = $data }
293 elsif ($lastStartTag eq "pubPlace") { $database_url=$data }
294
295 if ($reverse_index && ($lastStartTag eq "tr")) {
296 $current_headword .= $data;
297 }
298
299 elsif (!$reverse_index && ($lastStartTag eq "orth")) {
300 if (($preLastStartTag eq "form") ||
301 ($preLastStartTag eq "orth")) {
302 #warn "headwords add_text: '$data'\n";
303 $current_headword .= $data;
304 }
305 else {
306 print STDERR "multiple <orth>-s only at beginning of <form> supported. Skipping the others!\n" }
307 }
308 }
309
310
311 sub end_element {
312 my ($self, $element) = @_;
313 my $part = $element->{Name};
314
315 pop @higherElements;
316
317
318
319 if (($state eq "quoting") || ($state eq "orth") || ($state eq "tr")) {
320 $quoted .= "</$part>\n";
321 }
322
323 if ($part eq "orth") {
324 Dict::add_headword($current_headword);
325 # Dict::end_headword() has to be called later, because we don't know now
326 # if there is another orth coming (multiple headwords for one entry)
327 if($state eq "orth") {$state = "quoting"};
328 }
329
330 if ($part eq "tr") {
331 Dict::add_headword($current_headword);
332 # Dict::end_headword() has to be called later, because we don't know now
333 # if there is another tr coming
334 if($state eq "tr") {$state = "quoting"};
335 }
336
337 elsif ($part eq "entry") {
338 if ($reverse_index && $Dict::head) { Dict::end_headword() };
339
340 $quoted = apply_filter_cmd($filtercmd,$quoted);
341
342 Dict::write_direct($quoted);
343 Dict::write_newline();
344 $state = "";
345 }
346
347 elsif (($part eq "teiHeader")) {
348 return if $skip_header;
349
350 $state = "";
351
352 Dict::write_text();
353 Dict::write_newline();
354
355 # 00-database-short
356 Dict::set_headword(); Dict::add_text("00-database-short");
357 Dict::write_text(); Dict::end_headword();
358 Dict::write_direct("\n ");
359 Dict::write_direct($database_short);
360 if($reverse_index)
361 {
362 Dict::write_direct(' [reverse index]');
363 }
364 Dict::write_newline();
365
366 ## 00-database-info
367 Dict::set_headword();
368 $Dict::text="00-database-info";
369 Dict::write_text();
370 Dict::end_headword();
371
372 $quoted = apply_filter_cmd($filtercmd,$quoted);
373
374 Dict::add_text($quoted);# will do wrapping
375 Dict::write_text();
376 Dict::write_newline();
377
378 # FIXME who defined 00-database-url and where is it used? it seems not in kdict
379 Dict::set_headword(); Dict::add_text("00-database-url");
380 Dict::write_text(); Dict::end_headword();
381 Dict::write_direct(" ");
382 Dict::write_direct($database_url);
383 Dict::write_newline();
384
385 } # ^header
386
387 elsif ($part eq "body") {
388 Dict::set_headword();# to have the last headword appear in INDEX
389 };
390
391 }
392
393 sub error
394 {
395 my ($self, $args) = @_;
396 if ($args->{'Message'} =~ /^.+:.+:\d+:W:/)
397 {
398 # print warnings
399 print STDERR "Warning from XML::ESISParser: " . $args->{'Message'} . "\n";
400 return
401 }
402 use Data::Dumper;
403 print STDERR Dumper \@_;
404 print STDERR "Error from XML::ESISParser: " . $args->{'Message'} . "\n";
405 exit 3;
406 }
407
408 1;