Debian Codegraph @

mkxmltype

raw
1 #!/usr/bin/perl -w -s
2 use XML::DT;
3 use Data::Dumper;
4 use File::Temp;
5 use Term::ReadLine;
6 #use locale;
7
8 use strict;
9
10 our ($latin1,$html,$show_att,$expand_att_id,$class);
11 our ($lines,$t,$shell);
12
13 my (@files)=@ARGV;
14 @ARGV=();
15
16
17
18 mkxmltypes (@files);
19
20
21 sub mkxmltypes {
22 my %type=();
23 my @files = @_;
24 my %root = ();
25 my %att=();
26 my %dom=();
27
28 my %ele=();
29 my %elel=();
30 my %atl=();
31 my %handler=(
32 # '-outputenc' => 'ISO-8859-1',
33 '-default' => sub{
34 $c =~ s/,$//;
35 if(not $class){
36 push(@{$type{$q}}, (eval("[$c]") || "?$c"));
37 $elel{$q}++;
38 if(ctxt(1)){ $ele{ctxt(1)}{$q} ++;}
39 else { $root{$q}++}
40 for(keys(%v)){
41 $atl{$_}++;
42 $att{$q}{$_}{tipo($v{$_})||"_str"} ++ ;
43 $dom{$q}{$_}{$v{$_}} ++ }
44 "'$q',";
45 }
46 else{
47 my $qcl=$q;
48 if($v{class}){$qcl .="+$v{class}"}
49 if($v{id} ){$qcl .="+$v{id}"}
50
51 push(@{$type{$qcl}}, (eval("[$c]") || "?$c"));
52 $elel{$qcl}++;
53 if(ctxt(1)){
54 my $fcl=ctxt(1);
55 if(father->{class}){$fcl .="+".father->{class}}
56 if(father->{id} ){$fcl .="+".father->{id}}
57 $ele{$fcl}{$qcl} ++;}
58 else { $root{$qcl}++}
59
60 for(keys(%v)){
61 next if ($_ eq "class");
62 next if ($_ eq "id");
63 $atl{$_}++;
64 $att{$qcl}{$_}{tipo($v{$_})||"_str"} ++ ;
65 $dom{$qcl}{$_}{$v{$_}} ++ }
66 "'$qcl',";
67 }
68 },
69 '-pcdata' => sub{ if ($c =~ /[^ \t\n]/){ $ele{ctxt(1)}{"#PCDATA"}=1;
70 "'#PCDATA'," } else {""}},
71 );
72
73 if ($html) { $handler{'-html'} = 1;}
74 if($latin1) { $handler{'-inputenc'}='ISO-8859-1';}
75
76 for my $fname (@files){
77 if($lines){
78 my $tmpfile = File::Temp->new( UNLINK => 0 )->filename;
79 system("head -$lines $fname | xmllint --recover - > $tmpfile");
80 $fname = $tmpfile;
81 }
82 dt($fname,%handler);
83 unlink($fname) if $lines;
84 }
85
86 ### print "DEBUG",Dumper(\%att,\%ele,\%dom,\%atl);
87 print "# ", join(" ",keys %root)," ...", scalar(localtime(time)) ,"\n";
88
89 my %resumofinal=();
90 for (keys %type){
91 my @tipo=();
92 for my $lista (@{$type{$_}}){ push (@tipo, processa($lista)) }
93 $resumofinal{$_}=resumele(processa2([@tipo])).resumeatts($att{$_});
94 }
95
96 if($shell){ shell($t,\%root,\%ele,\%att,\%dom,\%resumofinal,\%atl); }
97 else{ pprint(\%resumofinal,ordem(\%ele,(($t) ||(keys %root) ))); }
98 }
99
100 sub shell{
101 my ($t,$root,$ele,$att,$dom,$resumofinal,$atl) = @_;
102 my $last=(keys %$root)[0];
103 my $elepat = q{[\w:]+};
104 my $max = 10;
105 my $term = new Term::ReadLine 'sample';
106 my $tas = $term->Attribs;
107 $tas->{completion_entry_function}= $tas->{list_completion_function};
108 $tas->{completion_word} = [ keys(%$ele), keys(%$atl) ];
109
110 pprint($resumofinal,ordem($ele,(($t) ||(keys %$root) )));
111 while ( defined ($_ = $term->readline("\npfs> ")) ) {
112 chomp();
113 $term->addhistory($_) if /\S/;
114 s/^\s*(.*?)\s*$/$1/;
115 if(/($elepat)\[\@?($elepat)\]/){
116 print resumeatt($att->{$1}{$2},$dom->{$1}{$2},$max);
117 $last = $1}
118 elsif(/\!max\s*=?\s*(\d+)/){$max=$1;}
119 elsif(/\.($elepat)/){
120 print resumeatt($att->{$last}{$1},$dom->{$last}{$1},$max);}
121 elsif(!$_ or defined $ele->{$_}) {
122 $last=$_;
123 pprint($resumofinal,ordem($ele,(($_) ||(keys %$root) ))); }
124 else{
125 for my $e (keys %$att){
126 for my $a (keys %{$att->{$e}}){
127 print "$e($a):",
128 resumeatt($att->{$e}{$a},$dom->{$e}{$a},$max) if($a eq $_)
129 }
130 }
131 }
132 }
133 }
134
135 sub ordem{
136 my ($rel,@st)=@_;
137 my @r=();
138 my %visited = ('#PCDATA' => 1);
139
140 while(@st){
141 my $next = shift(@st);
142 next if $visited{$next};
143 push(@r,$next);
144 $visited{$next} = 1;
145 push(@st, (grep {! $visited{$_}} (keys %{$rel->{$next}})));
146 }
147 \@r;
148 }
149
150 sub pprint{
151 my $r = shift;
152 my $order = shift;
153 for (@$order){ print "$_ \t=> $r->{$_}\n";}
154 }
155
156 sub resumeatts{
157 my $a=shift;
158 my $r="";
159 for (keys(%{$a})) {
160 if($expand_att_id){ $r .= "\n\t\t * $_:(".join(",",keys %{$a->{$_}}) . ")" }
161 else { $r .= " * $_" }
162 }
163 $r
164 }
165
166 sub resumeatt{
167 my $a=shift;
168 my $d=shift;
169 my $max = shift(@_) || 10;
170 my $r= join("|",keys %{$a}) ;
171 my @domact = (grep {defined $_} ((keys %{$d}))[0..$max]);
172 $domact[$max] = '...' if $domact[$max];
173 $r . " = {". join(",",@domact) . "}\n";
174 }
175
176 sub processa{
177 my $a=shift;
178 if( @$a == 0 ) { +{ _isa =>"empty"} }
179 elsif( @$a == 1 && $a->[0] eq '#PCDATA') {
180 +{ _isa =>"text" ,$a->[0] =>[1,1]} }
181 elsif( @$a == 1 ) { +{ _isa =>"singleton",$a->[0] =>[1,1]} }
182 else{ my %f = ();
183 for (@$a){$f{$_}[0]++,$f{$_}[1]++}
184 my $dif = scalar keys %f;
185 if($dif == 1) { +{ _isa =>"seq", %f} ; }
186 elsif($dif == @$a) { +{ _isa =>"tup", %f}; }
187 elsif($f{'#PCDATA'}){ +{ _isa =>"mixed", %f }; }
188 else { +{ _isa =>"mtup", %f } }
189 }
190 }
191
192 sub processa2{
193 my $a=shift;
194 if ( @$a == 0 ) { die("no sons????") }
195 elsif( @$a == 1 ) { $a->[0] }
196 else{
197 my %f = ();
198 my %maybe = ();
199 for (@$a){$f{sons2str($_)}++;
200 $maybe{$_->{_isa}}++ }
201 my $dif = scalar keys %f;
202 if ($dif == 1) { $a->[0]; }
203 elsif($maybe{mixed} || $maybe{text}){ +{%{join_sons($a)}, _isa=> "mixed"} }
204 else { my %s= %{join_sons($a)};
205 if(keys %s == 1) { +{%s, _isa => "seq"}}
206 else { +{%s, _isa => "mtup"} }
207 }
208 }
209 }
210
211 sub resumele{
212 my $a=shift;
213 ## print Dumper($a);
214 my $i = $a->{_isa};
215 delete $a->{_isa};
216 if ($i eq "text") {"text"}
217 elsif ($i eq "empty") {"empty"}
218 elsif ($i eq "singleton") {join(", ", keys %{$a}) }
219 elsif ($i eq "mixed") {delete $a->{'#PCDATA'};
220 if(keys %{$a}){ "mixed(".join(", ", keys %{$a}).")"}
221 else {"text"}
222 }
223 elsif ($i eq "tup") {"tup(".join(", ", keys %{$a}).")"}
224 elsif ($i eq "seq") {"seq(".join(", ", keys %{$a}).")"}
225 else { my $r= "mtup(";
226 for(sort keys %$a){
227 $r .= "$_, " if ( $a->{$_}[0] == 1 && $a->{$_}[1] == 1 );
228 $r .= "$_?, " if ( $a->{$_}[0] == 0 && $a->{$_}[1] == 1 );
229 $r .= "$_*, " if ( $a->{$_}[0] == 0 && $a->{$_}[1] > 1 );
230 $r .= "$_+, " if ( $a->{$_}[0] > 0 && $a->{$_}[1] > 1 );
231 $r .= Dumper($a) if($r =~ /\($/ );
232 }
233 $r =~ s/, $//;
234 $r.=")";
235 }
236 }
237
238 sub join_sons{
239 my $a = shift;
240 my %final = ( map { ($_ => [$a->[0]{$_}[0], $a->[0]{$_}[1]])}
241 grep {$_ ne "_isa"} keys %{$a->[0]});
242 my %todas=();
243 for (@$a){
244 my @novas=keys %{$_};
245 @todas{@novas}= @novas;
246 for my $k (keys %todas){
247 next if $k =~ /_isa/;
248 $final{$k}[0]=0 unless $final{$k}[0];
249 unless (exists $_->{$k}){ $final{$k}[0]=0;
250 next}
251 $final{$k}[1]=$_->{$k}[1] if $_->{$k}[1] > ($final{$k}[1] || 0);
252 $final{$k}[0]=$_->{$k}[0] if $_->{$k}[0] < $final{$k}[0];
253 }
254 }
255 \%final
256 }
257
258 sub sons2str{
259 my $a = shift;
260 join(' ',($a->{_isa},map { $_ . ($a->{$_}[0]==1 ? "" : "+") }
261 grep {$_ ne "_isa"} sort keys %$a));
262 }
263
264 sub tipo{
265 my $a=shift;
266 for ($a){
267 if(/^\s*\d+\s*$/) {return "_int" }
268 elsif(m{^\s*(https?|ftp|file)://\w[~&=?\w:/.-]+\s*$}i){return "_url" }
269 elsif(/^\s*\d+\.\d+\s*$/) {return "_real" }
270 elsif(/^\w+$/) {return "_id" }
271 elsif(m{^\s*[\w.-]+\@\w[\w_:/.-]+\s*$}) {return "_email" }
272 else {return undef }
273 }
274 }
275
276 __END__
277
278 =head1 NAME
279
280 mkxmltype - Make XML analysis using XML::DT
281
282 =head1 SYNOPSIS
283
284 mkxmltype <xmlfile>
285
286 =head1 DESCRIPTION
287
288 This command tries to infer DTD and Camlila-like types for a specific XML file;
289
290 =head1 Options
291
292 -latin1 input file encoding is forced to be latin1
293 -html uses html (libxml2) parser
294 -show_att Show attribute values
295 -expand_att_id
296 -lines=20000 just reads the first 20000 lines of the XML file
297 -t
298 -class '<div class="a" id="b">'... is treated as 'div+a+b'
299 -shell Enter interactive shell mode
300
301 =head1 SEE ALSO
302
303 XML::DT(1), mkdtskel(1), mkdtdskel and perl(1)
304
305 =cut