| 1 |
# SAX handler module for tei2dictxml_xml converter |
| 2 |
|
| 3 |
# V1.3 10/2006 Michael Bunk |
| 4 |
# * removed trailing white space |
| 5 |
# * added error handler, so XML:ESISParser doesn't die() on warnings |
| 6 |
# V1.2 5/2004 Michael Bunk |
| 7 |
# * added option to generate 00-database-allchars header |
| 8 |
# * if reverse index is generated, 00-database-short will indicate it now |
| 9 |
# V1.1 4/2004 Michael Bunk |
| 10 |
# * when headwords contain entities, characters() |
| 11 |
# is called multiple times for the same element: |
| 12 |
# now only one call to add_headword() |
| 13 |
# will be done |
| 14 |
# V1.0 6/2003 Michael Bunk |
| 15 |
|
| 16 |
# This program is free software; you can redistribute it and/or modify |
| 17 |
# it under the terms of the GNU General Public License as published by |
| 18 |
# the Free Software Foundation; either version 2 of the License, or |
| 19 |
# (at your option) any later version. |
| 20 |
# |
| 21 |
# This program is distributed in the hope that it will be useful, |
| 22 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 23 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 24 |
# GNU General Public License for more details. |
| 25 |
# |
| 26 |
# You should have received a copy of the GNU General Public License |
| 27 |
# along with this program; if not, visit |
| 28 |
# <http://www.gnu.org/licenses/gpl-2.0.txt> |
| 29 |
|
| 30 |
# thinking about event & calling order of the Dict subs (there is no other doc?): |
| 31 |
# 1. set_file_name() -> open_dict() |
| 32 |
# 2. <orth> -> set_headword() |
| 33 |
# 3. characters() -> $current_headword .= $data; |
| 34 |
# 4. end_tag -> add_headword($current_headword) |
| 35 |
# 5. </orth> -> end_headword() |
| 36 |
# 6. several start_tags -> ( write_direct() | add_text() ) |
| 37 |
# 7. several end_tags -> write_text() |
| 38 |
# 8. </entry> -> write_newline() |
| 39 |
# 9. continue with 2. for all other entrys |
| 40 |
# 10. closing the filehandles not needed |
| 41 |
|
| 42 |
# for encoding tei header we need some extra: |
| 43 |
# V1.1.: |
| 44 |
# 1. same |
| 45 |
# 2. <teiHeader> -> set_headword(); write_text("00-database-info"); end_headword(); |
| 46 |
# 3. header start_tags -> add_text(" Info: "); |
| 47 |
# 4. tag content (if any) -> add_text($data); and occasionally save to some $var |
| 48 |
# 5. header end_tags -> add_text("\n"); |
| 49 |
# 6. </teiHeader> -> write_text(); |
| 50 |
# set_headword(); write_text("00-database-short"); end_headword(); |
| 51 |
# write_direct($database-short); |
| 52 |
# set_headword(); write_text("00-database-url"); end_headword(); |
| 53 |
# write_direct($database-url); |
| 54 |
# 7. go on with entrys like in V1.0 (go on at 2. there) |
| 55 |
|
| 56 |
package lib::TEIHandlerxml_xml; |
| 57 |
|
| 58 |
use strict; |
| 59 |
use warnings; |
| 60 |
use lib::Dict; |
| 61 |
use utf8; |
| 62 |
|
| 63 |
# for filtering: |
| 64 |
use FileHandle; |
| 65 |
use IPC::Open2; |
| 66 |
|
| 67 |
# for sablotron: |
| 68 |
use XML::Sablotron; |
| 69 |
use XML::Sablotron::DOM; |
| 70 |
|
| 71 |
our ($header, $preLastStartTag, $lastStartTag, |
| 72 |
$database_short, $database_url, @higherElements, |
| 73 |
$file_name, $skip_header, $state, $quoted, |
| 74 |
$filtercmd, $sit, $sab, $reverse_index, $sab_templ, |
| 75 |
$current_headword); |
| 76 |
|
| 77 |
# i copied _escape() from XML::Handler::CanonXMLWriter |
| 78 |
# for escaping special chars in HTML/XML/SGML |
| 79 |
my %char_entities = ( |
| 80 |
"\x09" => '	', |
| 81 |
"\x0a" => ' ', |
| 82 |
"\x0d" => ' ', |
| 83 |
'&' => '&', |
| 84 |
'<' => '<', |
| 85 |
'>' => '>', |
| 86 |
'"' => '"', |
| 87 |
); |
| 88 |
|
| 89 |
sub _escape { |
| 90 |
my $string = shift; |
| 91 |
|
| 92 |
$string =~ s/([\x09\x0a\x0d&<>\"])/$char_entities{$1}/ge; |
| 93 |
|
| 94 |
return $string; |
| 95 |
} |
| 96 |
|
| 97 |
|
| 98 |
sub new { |
| 99 |
my ($type) = shift; |
| 100 |
$header = 0; |
| 101 |
$state=""; |
| 102 |
return bless {}, $type; |
| 103 |
} |
| 104 |
|
| 105 |
sub dumpTag { |
| 106 |
my $element = shift; |
| 107 |
|
| 108 |
#dump attributes |
| 109 |
my $attr=""; |
| 110 |
if(!defined $element) |
| 111 |
{ |
| 112 |
die "Please make sure the input file is valid!"; |
| 113 |
} |
| 114 |
my %attrhash = %{$element->{Attributes}}; |
| 115 |
foreach(keys %attrhash) { |
| 116 |
$attr .= ' '.$_.'="'._escape($attrhash{$_}).'"'; |
| 117 |
} |
| 118 |
|
| 119 |
return "<".$element->{Name}.$attr.">"; |
| 120 |
|
| 121 |
} |
| 122 |
|
| 123 |
sub apply_filter_cmd { |
| 124 |
my ($filtercommand, $quoted) = @_; |
| 125 |
|
| 126 |
if($filtercmd ne "") { |
| 127 |
print STDERR "."; |
| 128 |
#print "calling open2 with '$filtercmd'..."; |
| 129 |
#TODO: catch errors from executed filtercmd |
| 130 |
# according to 'man perlipc' |
| 131 |
my $pid = open2(*Reader, *Writer, $filtercmd ); |
| 132 |
#print "handing over...";flush STDOUT; |
| 133 |
print Writer "$quoted\n"; |
| 134 |
close(Writer); |
| 135 |
#print "reading...";flush STDOUT; |
| 136 |
$quoted = <Reader>; |
| 137 |
while(!eof(Reader)) { $quoted .= <Reader>; }; |
| 138 |
#print "\n";flush STDOUT; |
| 139 |
waitpid $pid, 0; |
| 140 |
} |
| 141 |
elsif ($sab) { |
| 142 |
|
| 143 |
# print STDERR "+"; |
| 144 |
$sab->addArg($sit, 'input', $quoted); |
| 145 |
|
| 146 |
# print STDERR "\ncalling addArgTree..."; |
| 147 |
$sab->addArgTree($sit, 'template', $sab_templ); |
| 148 |
|
| 149 |
# print STDERR "\ncalling process..."; |
| 150 |
$sab->process($sit, 'arg:/template', 'arg:/input', 'arg:/output'); |
| 151 |
|
| 152 |
# print STDERR "\ncalling getResultArg..."; |
| 153 |
$quoted = $sab->getResultArg('arg:/output'); |
| 154 |
|
| 155 |
# print STDERR "\ngot: $quoted\n\n\n\n"; |
| 156 |
} |
| 157 |
|
| 158 |
return $quoted; |
| 159 |
} |
| 160 |
|
| 161 |
sub set_options { |
| 162 |
my ($self, $filename, $askip_header, $aGenerate00DatabaseUtf8, $aLocale, |
| 163 |
$aFilterCmd, $aStyleSheet, $aReverseindex, $aAllchars) = @_; |
| 164 |
$file_name = $filename; |
| 165 |
$file_name =~ s/^(\w*\/)+//g; |
| 166 |
$file_name =~ s/(\S*)\.\w*/$1/; |
| 167 |
$skip_header=$askip_header; |
| 168 |
$filtercmd=$aFilterCmd; |
| 169 |
$reverse_index=$aReverseindex; |
| 170 |
|
| 171 |
if ($aStyleSheet) { |
| 172 |
print STDERR "initializing Sablotron...\n"; |
| 173 |
$sab = new XML::Sablotron; |
| 174 |
$sit = new XML::Sablotron::Situation; |
| 175 |
$sab_templ = XML::Sablotron::DOM::parseStylesheet($sit, $aStyleSheet); |
| 176 |
} |
| 177 |
|
| 178 |
Dict->open_dict($file_name,$aLocale,$aGenerate00DatabaseUtf8); |
| 179 |
|
| 180 |
if($aGenerate00DatabaseUtf8) |
| 181 |
{ |
| 182 |
print STDERR "Generating 00-database-utf8\n"; |
| 183 |
Dict::set_headword(); Dict::add_text("00-database-utf8"); |
| 184 |
Dict::write_text(); Dict::end_headword(); |
| 185 |
|
| 186 |
# might not be needed at all, but we give a hint |
| 187 |
Dict::write_direct("\n "); |
| 188 |
Dict::write_direct("This dictionary is UTF-8 encoded. If you use dictd, ". |
| 189 |
"make sure to start it with the appropriate --locale option."); |
| 190 |
Dict::write_newline(); |
| 191 |
} |
| 192 |
|
| 193 |
if($aAllchars) |
| 194 |
{ |
| 195 |
print STDERR "Generating 00-database-allchars\n"; |
| 196 |
Dict::set_headword(); Dict::add_text("00-database-allchars"); |
| 197 |
Dict::write_text(); Dict::end_headword(); |
| 198 |
Dict::write_newline(); |
| 199 |
} |
| 200 |
} |
| 201 |
|
| 202 |
sub start_element { |
| 203 |
my ($self, $element) = @_; |
| 204 |
my $part = $element->{Name}; |
| 205 |
|
| 206 |
$preLastStartTag = $lastStartTag; $lastStartTag = $part; |
| 207 |
|
| 208 |
# for multiple orths at beginning of <form> |
| 209 |
if (!$reverse_index && ($Dict::head) && ($part ne "orth")) { Dict::end_headword() }; |
| 210 |
|
| 211 |
|
| 212 |
# see what tag we have (& remember XML is case sensitive!) |
| 213 |
# most frequent tags on top |
| 214 |
if (($part eq "entry") || |
| 215 |
($part eq "teiHeader")) { |
| 216 |
$quoted = ""; |
| 217 |
$state = "quoting"; |
| 218 |
} |
| 219 |
|
| 220 |
elsif ($part eq "orth" && !$reverse_index) { |
| 221 |
if ($preLastStartTag eq "form") {Dict::set_headword()} |
| 222 |
$state = "orth";$current_headword=''; |
| 223 |
} |
| 224 |
|
| 225 |
elsif ($part eq "tr" && $reverse_index) { |
| 226 |
if (!$Dict::head) {Dict::set_headword()} |
| 227 |
$state = "tr";$current_headword=''; |
| 228 |
} |
| 229 |
|
| 230 |
|
| 231 |
if ($part eq "revisionDesc") { |
| 232 |
# add myself as |
| 233 |
# <change> |
| 234 |
# <date>now</date> |
| 235 |
# <respStmt><name>tei2dict_xml.pl</name></respStmt> |
| 236 |
# <item>converted TEI database into .dict format</item> |
| 237 |
# </change> |
| 238 |
my $now_string = gmtime(); |
| 239 |
my $s .= "<change>\n <date>$now_string</date>\n"; |
| 240 |
$s .= " <respStmt><name>tei2dictxml_xml.pl</name></respStmt>\n"; |
| 241 |
$s .= " <item>converted TEI database into .dict format</item></change>\n"; |
| 242 |
$quoted .= dumpTag($element) . $s; |
| 243 |
# printf STDERR "new revisionDesc: $quoted"; |
| 244 |
} |
| 245 |
elsif (($state eq "quoting") || |
| 246 |
($state eq "orth") || |
| 247 |
($state eq "tr")) { |
| 248 |
$quoted .= dumpTag($element); |
| 249 |
} |
| 250 |
|
| 251 |
elsif (($part eq "text") || |
| 252 |
($part eq "body") || |
| 253 |
($part eq "TEI.2")) { |
| 254 |
# skip these tags without warnings |
| 255 |
} |
| 256 |
else { |
| 257 |
print STDERR "warning: skipping start tag: $part\n" } |
| 258 |
} |
| 259 |
|
| 260 |
sub characters { |
| 261 |
my ($self, $element) = @_; |
| 262 |
my $data = $element->{Data}; |
| 263 |
|
| 264 |
# FIXME: maybe with other parser than ESISParser? |
| 265 |
# my guess: sometimes our loved ESISParser calls this handler |
| 266 |
# while it shouldn't: to notify us of a line break and white |
| 267 |
# space. there are handlers end_record() and white_space() for |
| 268 |
# this. anyway: since we don't have <pre>-element where newlines |
| 269 |
# are important inside, we just replace all \n by ' ' |
| 270 |
# then all multiple white space by single ' ' |
| 271 |
# then remove all ' ' at beginning or end |
| 272 |
# and skip everything if we get empty $data |
| 273 |
|
| 274 |
# do some whitespace stripping: |
| 275 |
return if ($data =~ /\A\s*\Z/); |
| 276 |
|
| 277 |
$data =~ s/\n/ /; |
| 278 |
$data =~ s/\s+/ /; |
| 279 |
|
| 280 |
if ($data eq "") { |
| 281 |
# can happen because of newlines |
| 282 |
return; |
| 283 |
} |
| 284 |
|
| 285 |
if (($state eq "quoting") || |
| 286 |
($state eq "orth") || |
| 287 |
($state eq "tr")) { |
| 288 |
$quoted .= _escape($data); |
| 289 |
} |
| 290 |
|
| 291 |
if (($preLastStartTag eq "titleStmt") && |
| 292 |
($lastStartTag eq "title")) { $database_short = $data } |
| 293 |
elsif ($lastStartTag eq "pubPlace") { $database_url=$data } |
| 294 |
|
| 295 |
if ($reverse_index && ($lastStartTag eq "tr")) { |
| 296 |
$current_headword .= $data; |
| 297 |
} |
| 298 |
|
| 299 |
elsif (!$reverse_index && ($lastStartTag eq "orth")) { |
| 300 |
if (($preLastStartTag eq "form") || |
| 301 |
($preLastStartTag eq "orth")) { |
| 302 |
#warn "headwords add_text: '$data'\n"; |
| 303 |
$current_headword .= $data; |
| 304 |
} |
| 305 |
else { |
| 306 |
print STDERR "multiple <orth>-s only at beginning of <form> supported. Skipping the others!\n" } |
| 307 |
} |
| 308 |
} |
| 309 |
|
| 310 |
|
| 311 |
sub end_element { |
| 312 |
my ($self, $element) = @_; |
| 313 |
my $part = $element->{Name}; |
| 314 |
|
| 315 |
pop @higherElements; |
| 316 |
|
| 317 |
|
| 318 |
|
| 319 |
if (($state eq "quoting") || ($state eq "orth") || ($state eq "tr")) { |
| 320 |
$quoted .= "</$part>\n"; |
| 321 |
} |
| 322 |
|
| 323 |
if ($part eq "orth") { |
| 324 |
Dict::add_headword($current_headword); |
| 325 |
# Dict::end_headword() has to be called later, because we don't know now |
| 326 |
# if there is another orth coming (multiple headwords for one entry) |
| 327 |
if($state eq "orth") {$state = "quoting"}; |
| 328 |
} |
| 329 |
|
| 330 |
if ($part eq "tr") { |
| 331 |
Dict::add_headword($current_headword); |
| 332 |
# Dict::end_headword() has to be called later, because we don't know now |
| 333 |
# if there is another tr coming |
| 334 |
if($state eq "tr") {$state = "quoting"}; |
| 335 |
} |
| 336 |
|
| 337 |
elsif ($part eq "entry") { |
| 338 |
if ($reverse_index && $Dict::head) { Dict::end_headword() }; |
| 339 |
|
| 340 |
$quoted = apply_filter_cmd($filtercmd,$quoted); |
| 341 |
|
| 342 |
Dict::write_direct($quoted); |
| 343 |
Dict::write_newline(); |
| 344 |
$state = ""; |
| 345 |
} |
| 346 |
|
| 347 |
elsif (($part eq "teiHeader")) { |
| 348 |
return if $skip_header; |
| 349 |
|
| 350 |
$state = ""; |
| 351 |
|
| 352 |
Dict::write_text(); |
| 353 |
Dict::write_newline(); |
| 354 |
|
| 355 |
# 00-database-short |
| 356 |
Dict::set_headword(); Dict::add_text("00-database-short"); |
| 357 |
Dict::write_text(); Dict::end_headword(); |
| 358 |
Dict::write_direct("\n "); |
| 359 |
Dict::write_direct($database_short); |
| 360 |
if($reverse_index) |
| 361 |
{ |
| 362 |
Dict::write_direct(' [reverse index]'); |
| 363 |
} |
| 364 |
Dict::write_newline(); |
| 365 |
|
| 366 |
## 00-database-info |
| 367 |
Dict::set_headword(); |
| 368 |
$Dict::text="00-database-info"; |
| 369 |
Dict::write_text(); |
| 370 |
Dict::end_headword(); |
| 371 |
|
| 372 |
$quoted = apply_filter_cmd($filtercmd,$quoted); |
| 373 |
|
| 374 |
Dict::add_text($quoted);# will do wrapping |
| 375 |
Dict::write_text(); |
| 376 |
Dict::write_newline(); |
| 377 |
|
| 378 |
# FIXME who defined 00-database-url and where is it used? it seems not in kdict |
| 379 |
Dict::set_headword(); Dict::add_text("00-database-url"); |
| 380 |
Dict::write_text(); Dict::end_headword(); |
| 381 |
Dict::write_direct(" "); |
| 382 |
Dict::write_direct($database_url); |
| 383 |
Dict::write_newline(); |
| 384 |
|
| 385 |
} # ^header |
| 386 |
|
| 387 |
elsif ($part eq "body") { |
| 388 |
Dict::set_headword();# to have the last headword appear in INDEX |
| 389 |
}; |
| 390 |
|
| 391 |
} |
| 392 |
|
| 393 |
sub error |
| 394 |
{ |
| 395 |
my ($self, $args) = @_; |
| 396 |
if ($args->{'Message'} =~ /^.+:.+:\d+:W:/) |
| 397 |
{ |
| 398 |
# print warnings |
| 399 |
print STDERR "Warning from XML::ESISParser: " . $args->{'Message'} . "\n"; |
| 400 |
return |
| 401 |
} |
| 402 |
use Data::Dumper; |
| 403 |
print STDERR Dumper \@_; |
| 404 |
print STDERR "Error from XML::ESISParser: " . $args->{'Message'} . "\n"; |
| 405 |
exit 3; |
| 406 |
} |
| 407 |
|
| 408 |
1; |