Debian Codegraph @

teidiff

raw
1 #!/usr/bin/perl
2 # $Revision$
3
4 use warnings;
5 use strict;
6 use Getopt::Long;
7 use Pod::Usage;
8 use XML::DOM;
9 use Text::Diff;
10
11 my ($verbose, $help);
12 my $count = 1;
13 my $base;
14 my $output;
15 GetOptions (
16 "verbose" => \$verbose,
17 "help|?" => \$help,
18 "count" => \$count,
19 "base=s" => \$base,
20 "output=s" => \$output) or exit 1;
21
22 pod2usage(-exitval => 1, -verbose => 2) if $help;
23
24 if (scalar @ARGV != 2)
25 {
26 print STDERR
27 "I need exactly 2 filenames to operate. For help use --help.\n";
28 exit 1
29 }
30
31 if (defined $output)
32 {
33 $output = uc $output;
34 unless ($output =~
35 /^FILES|UNMODIFIED|ADDED|DELETED|MODIFIED_(LA1|LA2|MULTIMATCH)$/)
36 {
37 print STDERR "Invalid category: $output\n";
38 exit 1
39 }
40 }
41
42 my $parser = new XML::DOM::Parser;
43 print STDERR "Parsing $ARGV[0]... " if $verbose;
44 my $doc1 = $parser->parsefile ($ARGV[0]);
45 my @entries1 = $doc1->getElementsByTagName ("entry");
46 my $entries1 = scalar @entries1;
47 print STDERR $entries1, " entries\nParsing $ARGV[1]... " if $verbose;
48 my $doc2 = $parser->parsefile ($ARGV[1]);
49 undef $parser;
50 my @entries2 = $doc2->getElementsByTagName ("entry");
51 my $entries2 = scalar @entries2;
52 print STDERR $entries2, " entries\n" if $verbose;
53
54 sub pretty_print
55 {
56 my ($node, $indent) = @_;
57 my $type = $node->getNodeType;
58 return $node->getNodeValue if $type == TEXT_NODE;
59 if ($type == ELEMENT_NODE)
60 {
61 $indent = 0 unless defined $indent;
62 my $contents = undef;
63 for my $c ($node->getChildNodes)
64 {
65 my $cpp = pretty_print ($c, $indent+2);
66 if (defined $contents) { $contents .= $cpp; next }
67 $contents = $cpp
68 }
69 my $attrs = '';
70 for my $a ($node->getAttributes->getValues)
71 {
72 $attrs .= ' ' . $a->getNodeName . '=' . $a->getNodeValue
73 }
74
75 return ' ' x $indent
76 . '<' . $node->getNodeName . $attrs . '/>' unless defined $contents;
77 return ' ' x $indent
78 . '<' . $node->getNodeName . $attrs . '>'
79 . ($node->getFirstChild->getNodeType == ELEMENT_NODE ? "\n" : '')
80 . $contents
81 . ($node->getLastChild->getNodeType == ELEMENT_NODE ? ' ' x $indent : '')
82 . '</' . $node->getNodeName . ">\n"
83 }
84 die "Unhandled node type: ", $node->getNodeType
85 }
86
87 my (@unmodified, @added, @deleted, @modified_form, @modified_senses,
88 @modified_multimatch, %entry2form, %entry2senses, %entry2complete,
89 %complete2entry1, %form2entry1, %senses2entry1,
90 %complete2entry2, %form2entry2, %senses2entry2);
91
92 # adds an entry to an arrayref
93 sub put
94 {
95 my ($hashref, $index, $entry) = @_;
96 if (exists $hashref->{$index})
97 { push @{ $hashref->{$index} }, $entry }
98 else
99 { $hashref->{$index} = [ $entry ] }
100 }
101
102 # deletes an entry from an arrayref
103 # the entry may occur only once
104 sub remove
105 {
106 my ($hashref, $index, $entry) = @_;
107 die join ' ', caller unless defined $hashref && defined $index;
108 return unless exists $hashref->{$index};
109 my @entries = @{ $hashref->{$index} };
110 my $count = scalar @entries;
111 if ($count == 1)
112 {
113 return unless $entries[$#entries] eq $entry;
114 delete $hashref->{$index};
115 return
116 }
117 for (my $i=0; $i <= $count ; $i++)
118 {
119 next unless exists $entries[$i];
120 next unless $entries[$i] == $entry;
121 delete $entries[$i];
122 return
123 }
124 warn "No entry removed";
125 }
126
127 # Removes the entries in $entry2arrayref from the $x2entryhashrefs given.
128 # Each $x2entryhashref argument must be followed by an $entry2xhashref.
129 sub removeall
130 {
131 my $entryarrayref = shift;
132 while(my $x2entryhashref = shift)
133 {
134 my $entry2x = shift;
135 die unless defined $entry2x;
136 for my $entry (@$entryarrayref)
137 { remove $x2entryhashref, $entry2x->{$entry}, $entry }
138 }
139 }
140
141 sub find_unmodified_or_modified
142 {
143 my $found = 0;
144
145 for (keys %complete2entry1)
146 {
147 next if 1 < scalar @{ $complete2entry1{$_} };
148 next unless exists $complete2entry2{$_};
149 next if 1 != scalar @{ $complete2entry2{$_} };
150 my $entry1 = $complete2entry1{$_}->[0];
151 my $entry2 = $complete2entry2{$_}->[0];
152
153 push @unmodified, $entry1;
154 delete $complete2entry1{$_};
155 delete $complete2entry2{$_};
156 remove \%form2entry1, $entry2form{$entry1}, $entry1;
157 remove \%form2entry2, $entry2form{$entry1}, $entry2;
158 remove \%senses2entry1, $entry2senses{$entry1}, $entry1;
159 remove \%senses2entry2, $entry2senses{$entry1}, $entry2;
160 $found++
161 }
162
163 for (keys %form2entry1)
164 {
165 next if 1 < scalar @{ $form2entry1{$_} };
166 next unless exists $form2entry2{$_};
167 next if 1 != scalar @{ $form2entry2{$_} };
168 my $entry1 = $form2entry1{$_}->[0];
169 my $entry2 = $form2entry2{$_}->[0];
170
171 push @modified_senses, [ $entry1, $entry2 ] ;
172
173 remove \%complete2entry1, $entry2complete{$entry1}, $entry1;
174 remove \%complete2entry2, $entry2complete{$entry2}, $entry2;
175 delete $form2entry1{$_};
176 delete $form2entry2{$_};
177 remove \%senses2entry1, $entry2senses{$entry1}, $entry1;
178 remove \%senses2entry2, $entry2senses{$entry2}, $entry2;
179 $found++
180 }
181
182 for (keys %senses2entry1)
183 {
184 next if 1 < scalar @{ $senses2entry1{$_} };
185 next unless exists $senses2entry2{$_};
186 next if 1 != scalar @{ $senses2entry2{$_} };
187 my $entry1 = $senses2entry1{$_}->[0];
188 my $entry2 = $senses2entry2{$_}->[0];
189
190 push @modified_form, [ $entry1, $entry2 ] ;
191
192 remove \%complete2entry1, $entry2complete{$entry1}, $entry1;
193 remove \%complete2entry2, $entry2complete{$entry2}, $entry2;
194 remove \%form2entry1, $entry2form{$entry1}, $entry1;
195 remove \%form2entry2, $entry2form{$entry2}, $entry2;
196 delete $senses2entry1{$_};
197 delete $senses2entry2{$_};
198 $found++
199 }
200
201 print STDERR "$found entries matched\n" if $verbose;
202 return $found
203 }
204
205 sub find_added_deleted
206 {
207 print STDERR "Finding added and deleted entries... " if $verbose;
208 # an entry was deleted/added when no match
209 # for it exists in the other file
210 # since single matches should be sorted out by now,
211 # only multimatches remain to be skipped
212 for (keys %complete2entry1)
213 {
214 # the next line should never apply
215 next if exists $complete2entry2{$_};
216 my $skip = 0;
217 for my $entry1 (@{ $complete2entry1{$_} })
218 {
219 next unless exists $form2entry2{ $entry2form{$entry1} }
220 || exists $senses2entry2{ $entry2senses{$entry1} };
221 $skip = 1;
222 last
223 }
224 next if $skip;
225
226 push @deleted, @{ $complete2entry1{$_} };
227 removeall $complete2entry1{$_},
228 \%form2entry1, \%entry2form,
229 \%senses2entry1, \%entry2senses;
230 delete $complete2entry1{$_}
231 }
232 for (keys %complete2entry2)
233 {
234 next if exists $complete2entry1{$_};
235 my $skip = 0;
236 for my $entry2 (@{ $complete2entry2{$_} })
237 {
238 next unless exists $form2entry1{ $entry2form{$entry2} }
239 || exists $senses2entry1{ $entry2senses{$entry2} };
240 $skip = 1;
241 last
242 }
243 next if $skip;
244
245 push @added, @{ $complete2entry2{$_} };
246 removeall $complete2entry2{$_},
247 \%form2entry2, \%entry2form,
248 \%senses2entry2, \%entry2senses;
249 delete $complete2entry2{$_}
250 }
251 print STDERR "\n" if $verbose;
252 }
253
254 # Actually there is nothing more to find. We just put together the remains
255 # of the lists.
256 sub find_modified_multimatch
257 {
258 print STDERR "Collecting the multimatches... " if $verbose;
259 my (@from1, @from2);
260 for (values %complete2entry1)
261 { push @from1, @$_ }
262 for (values %complete2entry2)
263 { push @from2, @$_ }
264 push @modified_multimatch, [ @from1 ], [ @from2 ];
265 print STDERR "\n" if $verbose;
266 }
267
268 sub cache_toStrings
269 {
270 my ($entries, $complete2entry, $form2entry, $senses2entry) = @_;
271 for (@$entries)
272 {
273 # a complete comparison key is easy (if canonicalised XML is used)
274 my $scomplete = $_->toString ();
275 put $complete2entry, $scomplete, $_;
276 $entry2complete{$_} = $scomplete;
277 # comparison key for form: concat <form> and <gramGrp> child nodes
278 # comparison key for senses: concatenate all other child nodes
279 my @children = $_->getChildNodes ();
280 my $ssenses = my $sforms = '';
281 for my $child (@children)
282 {
283 if ($child->getTagName () =~ /^form|gramGrp$/)
284 {
285 $sforms .= $child->toString ();
286 next
287 }
288 $ssenses .= $child->toString ()
289 }
290 put $form2entry, $sforms, $_;
291 $entry2form{$_} = $sforms;
292 put $senses2entry, $ssenses, $_;
293 $entry2senses{$_} = $ssenses
294 }
295 }
296
297 print STDERR "Caching results of toString()..." if $verbose;
298 cache_toStrings \@entries1,
299 \%complete2entry1, \%form2entry1, \%senses2entry1;
300 undef @entries1;
301 cache_toStrings \@entries2,
302 \%complete2entry2, \%form2entry2, \%senses2entry2;
303 undef @entries2;
304 print STDERR "\n" if $verbose;
305
306 my $i = 1;
307 do
308 {
309 print STDERR "Comparison iteration ", $i, "... " if $verbose;
310 $i++
311 }
312 while find_unmodified_or_modified;
313
314 find_added_deleted;
315 find_modified_multimatch;
316
317 print STDERR "Finished comparison.\n" if $verbose;
318
319 undef %complete2entry1;
320 undef %form2entry1;
321 undef %senses2entry1;
322 undef %complete2entry2;
323 undef %form2entry2;
324 undef %senses2entry2;
325
326 if ($count)
327 {
328 my $unmodified = scalar @unmodified;
329 my $added = scalar @added;
330 my $deleted = scalar @deleted;
331 my $modified_form = scalar @modified_form;
332 my $modified_senses = scalar @modified_senses;
333 my $mm1 = scalar @{ $modified_multimatch[0] };
334 my $mm2 = scalar @{ $modified_multimatch[1] };
335 print "Unmodified:\t\t$unmodified\n";
336 print "Added:\t\t\t$added\n";
337 print "Deleted:\t\t$deleted\n";
338 print "Modified <form>:\t$modified_form\n";
339 print "Modified <sense>s:\t$modified_senses\n";
340 print "Modified multimatch:\t$mm1 + $mm2\n";
341 my $sum1 =
342 $unmodified + $deleted + $modified_form + $modified_senses + $mm1;
343 my $sum2 =
344 $unmodified + $added + $modified_form + $modified_senses + $mm2;
345 print STDERR "file1: unmod + del + mod_form + mod_sense + mm1 = ", $sum1,
346 ($sum1 == $entries1 ? ' OK' : ' MISMATCH') . " \n" if $verbose;
347 print STDERR "file2: unmod + add + mod_form + mod_sense + mm2 = ", $sum2,
348 ($sum2 == $entries2 ? ' OK' : ' MISMATCH') . " \n" if $verbose;
349 }
350
351 sub print_category_fh
352 {
353 my ($fh, $category) = @_;
354 print STDERR "Outputting category $category... " if defined $verbose and $verbose>1;
355 if ($category eq 'UNMODIFIED')
356 {
357 print $fh pretty_print $_ for (@unmodified)
358 }
359 elsif ($category eq 'ADDED')
360 {
361 print $fh pretty_print $_ for (@added)
362 }
363 elsif ($category eq 'DELETED')
364 {
365 print $fh pretty_print $_ for (@deleted)
366 }
367 elsif ($category eq 'MODIFIED_LA1')
368 {
369 for (@modified_form)
370 {
371 my $pretty1 = pretty_print $_->[0];
372 my $pretty2 = pretty_print $_->[1];
373 my %options = ();
374 my $diff = diff \$pretty1, \$pretty2, \%options;
375 print $fh $diff
376 }
377 }
378 elsif ($category eq 'MODIFIED_LA2')
379 {
380 for (@modified_senses)
381 {
382 my $pretty1 = pretty_print $_->[0];
383 my $pretty2 = pretty_print $_->[1];
384 my %options = ();
385 my $diff = diff \$pretty1, \$pretty2, \%options;
386 print $fh $diff
387 }
388 }
389 elsif ($category eq 'MODIFIED_MULTIMATCH')
390 {
391 # output entry sets
392 print $fh "<file1>\n";
393 print STDERR "file1... " if $verbose;
394 print $fh pretty_print $_ for (@{ $modified_multimatch[0] });
395 print $fh "</file1>\n<file2>\n";
396 print STDERR "file2... " if $verbose;
397 print $fh pretty_print $_ for (@{ $modified_multimatch[1] });
398 print $fh "</file2>\n"
399 }
400 else { warn "Invalid category: $category" }
401 print STDERR "done.\n" if defined $verbose and $verbose>1
402 }
403
404 sub print_category_file
405 {
406 my ($filename, $category) = @_;
407 my $fh;
408 unless (open $fh, '>:utf8', $filename)
409 {
410 print STDERR "Could not open $filename for writing: $!\n";
411 return
412 }
413 print STDERR "Writing $filename... " if $verbose;
414 print_category_fh $fh, $category;
415 close $fh;
416 print STDERR "OK\n" if $verbose
417 }
418
419 exit unless defined $output;
420 $base = $ARGV[0] . '-' . $ARGV[1] unless defined $base;
421 if ($output eq 'FILES')
422 {
423 print_category_file $base . '-unmodified.teipart', 'UNMODIFIED';
424 undef @unmodified;
425 print_category_file $base . '-added.teipart', 'ADDED';
426 undef @added;
427 print_category_file $base . '-deleted.teipart', 'DELETED';
428 undef @deleted;
429 print_category_file $base . '-modified_la1.udiff', 'MODIFIED_LA1';
430 undef @modified_form;
431 print_category_file $base . '-modified_la2.udiff', 'MODIFIED_LA2';
432 undef @modified_senses;
433 print_category_file $base . '-modified_multimatch.xml',
434 'MODIFIED_MULTIMATCH';
435 exit
436 }
437
438 # print only category in $output to STDOUT
439 open my $fh, '>:utf8',' -' or die;
440 print_category_fh $fh, $output;
441 close $fh;
442
443 # __DATA__, keep a file1 and a file2 and the outfiles
444 # XXX testcases, call with a commandline switch
445
446 __END__
447
448 =head1 NAME
449
450 teidiff - Compare the entries in two TEI XML dictionary files
451
452 =head1 SYNOPSIS
453
454 teidiff [options] file1 file2
455
456 Options:
457 --base Basename for output files
458 --[no]count Print number of added/modified/deleted entries to STDOUT
459 --help This manual page
460 --output Entry category to print to STDOUT
461 --verbose Tell what I'm doing
462
463 =head1 OPTIONS
464
465 =over 8
466
467 =item B<--base BASENAME>
468
469 Basename of output files. Default: F<file1-file2>
470
471 =item B<--[no]count>
472
473 Print number of added/modified/deleted entries to STDOUT. Default: on.
474
475 =item B<--help>
476
477 Print this manual page and exit.
478
479 =item B<--verbose>
480
481 Print information about the current work performed to STDERR.
482
483 =item B<--output CATEGORY>
484
485 Can be set to the string "FILES" or a category of entries to output to
486 STDOUT. By default nothing is output. If "FILES" is used, the
487 filenames will follow the scheme F<BASENAME-CATEGORY.(diff|teipart)>
488
489 The following categories are allowed:
490
491 =over 4
492
493 =item UNMODIFIED
494
495 Unmodified entries. They appear exactly the same in both versions of
496 the dictionary.
497
498 =item ADDED
499
500 Entries not present in file1, but in file2. They were added in
501 F<file2>.
502
503 =item DELETED
504
505 Entries present in file1, but not in file2. They were deleted in
506 F<file2>.
507
508 =item MODIFIED_LA1
509
510 Entries where the contents of the <form> element were modified, but
511 the contents of the <sense> elements are unmodified.
512
513 =item MODIFIED_LA2
514
515 Entries where the contents of the <sense> element(s) were modified,
516 but the contents of the <form> elements are unmodified.
517
518 =item MODIFIED_MULTIMATCH
519
520 Entries where the contents of the <form> or <sense> elements were modified (but
521 not both at the same time), but a corresponding entry was not found, because
522 multiple matches exist ("multimatch").
523
524 This category is output as two sets - the entries from file1 and the entries
525 from file2. One could build finer sets, consisting of only the entries
526 between which the multimatch relationship exists. But this information is
527 not useful enough for implementation currently.
528
529 =back
530
531 =back
532
533 =head1 DESCRIPTION
534
535 This program parses the given input files and compares the entries
536 in them, putting them in B<categories>. One or all categories can be
537 output.
538
539 =head1 NOTES
540
541 Normally ignorable blank space should not account for differences.
542 C<xmllint> can be used as a filter to drop ignorable blank spaces:
543
544 xmllint --noblanks infile.xml >outfile.xml
545
546 C<xmllint> also comes with a C<--c14n> option, but using it is unadvisable,
547 because it adds default attributes, bloating the file submerging
548 relevant differences in a sea of explicitness.
549
550 =head1 TODO
551
552 Integrating the functionality of this command line tool into
553 FreeDict-Editor would facilitate comfortable browsing of the
554 differences.
555
556 =head1 AUTHOR AND LICENCE
557
558 Author: Michael Bunk, 2006
559
560 This is free software, licensed under GPL.
561
562 =cut
563