Debian Codegraph @

samples/mhtml-tool

raw
1 #!/usr/bin/env perl
2
3 use strict;
4 use warnings;
5
6 $| = 1;
7
8 =pod
9
10 =head1 NAME
11
12 B<mhtml-tool> - Unpack a MIME HTML archive
13
14
15 =head1 SYNOPSIS
16
17 B<mhtml-tool> unpacks MIME HTML archives that some browsers (such as Opera)
18 save by default. The file extensions of such archives are ".mht" or ".mhtml".
19
20 The first HTML file in the archive is taken to be the primary web page, the
21 other contained files for "page requisites" such as images or frames. The
22 primary web page is written to the output directory (the current directory by
23 default), the requisites to a subdirectory named after the primary HTML file
24 name without extension, with "_files" appended. Link URLs in all HTML
25 files referring to requisites are rewritten to point to the saved files.
26
27
28 =head1 OPTIONS
29
30 =over
31
32 =item B<-e> program
33
34 =item B<--exec> program
35
36 Unpack the archive into a temporary directory and execute the given program
37 in that directory, passing the primary HTML (first text/html entry) of the
38 archive as a parameter.
39
40 If B<-o> is given, use that directory location with the program.
41
42 If the program is "lynx" or "(e)links(2)", this script uses the "-force-html"
43 option to help when the entrypoint lacks a ".html" file-suffix.
44
45 =item B<-h>
46
47 =item B<-?>
48
49 =item B<--help>
50
51 Print a brief usage summary.
52
53 =item B<-l>
54
55 =item B<--list>
56
57 List archive contents instead of unpacking. Four columns are output: file
58 name, MIME type, size and URL. Unavailable entries are replaced by "(?)".
59
60 =item B<-o> I<directory/ or name>
61
62 =item B<--output> I<directory/ or name>
63
64 If the argument ends in a slash or is an existing directory, unpack to that
65 directory instead of current directory. Otherwise the argument is taken as a
66 path to the file name to write the primary HTML file to. If the output
67 directory does not exist, it is created.
68
69 =back
70
71
72 =head1 SEE ALSO
73
74 https://github.com/ThomasDickey/mhtml-tool
75
76 https://github.com/clapautius/mhtml-tool
77
78 http://www.volkerschatz.com/unix/uware/unmht.html
79
80 =head1 COPYLEFT
81
82 B<unmht> is Copyright (c) 2012 Volker Schatz. It may be copied and/or
83 modified under the same terms as Perl.
84
85 B<mhtml-tool> is Copyright (c) 2018 Tudor M. Pristavu. It may be copied and/or
86 modified under the same terms as Perl.
87
88 B<mhtml-tool> is Copyright (c) 2024 Thomas E. Dickey. It may be copied and/or
89 modified under the same terms as Perl.
90
91 =cut
92
93 use Cwd;
94 use File::Path;
95 use File::Copy;
96 use File::Glob;
97 use File::Temp qw/ tempdir /;
98 use URI;
99 use MIME::Base64;
100 use MIME::QuotedPrint;
101 use HTML::PullParser;
102 use HTML::Tagset;
103 use Getopt::Long;
104
105 our $path_limit = 255;
106 our %shorten_path;
107
108 # RFC 7230 (obsoletes RFC 2616) indicates that URI length is limited by the
109 # server line-length, which it suggests should be at least 8000 octets. The
110 # problem is that components of pathnames might be limited to 255 bytes, while
111 # the URI components can be much longer. As an additional complication, the
112 # too-long components can be both directory- and file-names, and truncating
113 # the components can produce collisions.
114 sub shorten_path {
115 my $actual = $_[0];
116 if ( not defined $shorten_path{$actual} ) {
117 my @parts = split /\//, $actual;
118 for my $n ( 0 .. $#parts ) {
119 $parts[$n] = substr( $parts[$n], 0, $path_limit );
120 }
121 $shorten_path{$actual} = join "/", @parts;
122 }
123 return $shorten_path{$actual};
124 }
125
126 # URLs can include "&" and other characters which require quoting to use in
127 # a shell command. We use system() rather than exec() to allow the temporary
128 # directory to be cleaned up after running lynx.
129 sub quoted {
130 my $param = $_[0];
131 $param =~ s/'/'\"'\"'/g;
132 return sprintf( "'%s'", $param );
133 }
134
135 # Add appropriate ordinal suffix to a number.
136 # -> Number
137 # <- String of number with ordinal suffix
138 sub ordinal {
139 return $_[0] . "th" if $_[0] > 3 && $_[0] < 20;
140 my $unitdig = $_[0] % 10;
141 return $_[0] . "st" if $unitdig == 1;
142 return $_[0] . "nd" if $unitdig == 2;
143 return $_[0] . "rd" if $unitdig == 3;
144 return $_[0] . "th";
145 }
146
147 {
148 my %taken;
149
150 # Find unique file name.
151 # -> Preferred file name, or undef
152 # MHT archive name (as a fallback if no name given)
153 # <- File name not conflicting with names returned by previous calls, but which
154 # may exist!
155 sub unique_name {
156 my ( $fname, $mhtname ) = @_;
157 my ( $trunc, $ext );
158
159 if ( defined $fname ) {
160 $fname =~ s/^\s+//;
161 $fname =~ s/\s+$//;
162 $taken{$fname} = 1, return $fname unless $taken{$fname};
163 ( $trunc, $ext ) = $fname =~ /^(.*?)(\.\w+)?$/;
164 $ext //= "";
165 }
166 else {
167 $trunc = $mhtname || "unpack";
168 $trunc =~ s/\.mht(?:ml?)?$//i;
169 $ext = "";
170 }
171 for my $suff ( 1 .. 9999 ) {
172 $fname = "${trunc}_$suff$ext";
173 $taken{$fname} = 1, return $fname unless $taken{$fname};
174 ++$suff;
175 }
176 return undef;
177 }
178
179 }
180
181 # Output error message and exit with return value 1.
182 sub abort {
183 print STDERR "$_[0]\n";
184 exit 1;
185 }
186
187 # Generate output file directories and primary HTML file name depending on
188 # --output option and primary HTML file name from archive. In case the primary
189 # HTML file is not the first file in the archive, the secondary files directory
190 # is renamed or the files moved on the second call, when the primary HTML file
191 # name is known.
192 # -> Value of --output option (or undef)
193 # Primary HTML file name from MHT archive
194 # Flag indicating if .._files subdirectory should be created
195 # Hash reference to store resulting paths to
196 sub mkfiledir {
197 my ( $outputopt, $firsthtmlname, $needfilesdir, $out ) = @_;
198
199 my $prevfilespath = $$out{filespath};
200 $firsthtmlname = "unpackmht-$$" unless defined $firsthtmlname;
201 if ( !defined $outputopt ) {
202 $$out{toppath} = ".";
203 }
204 elsif ( -d $outputopt || $outputopt =~ m!/$! ) {
205 $$out{toppath} = $outputopt;
206 }
207 else {
208 ( $$out{toppath}, $firsthtmlname ) = $outputopt =~ m!^(.*/)?([^/]+)$!;
209 abort "Empty output file name." unless defined $firsthtmlname;
210 $firsthtmlname .= ".html" unless $firsthtmlname =~ /\./;
211 $$out{toppath} = "." unless defined $$out{toppath};
212 }
213 $$out{toppath} =~ s!/$!!;
214 $$out{firstout} = "$$out{toppath}/$firsthtmlname";
215 $$out{filesdir} = $firsthtmlname;
216 $$out{filesdir} =~ s/\.[^.]+$//;
217 $$out{filesdir} = substr( $$out{filesdir}, 0, $path_limit - 6 );
218 $$out{filesdir} .= "_files";
219 $$out{filespath} = "$$out{toppath}/$$out{filesdir}";
220
221 if ( defined $prevfilespath ) {
222 return unless $prevfilespath ne $$out{filespath};
223 return unless -d $prevfilespath;
224 if ( !-d $$out{filespath} ) {
225 File::Copy::move( $prevfilespath, $$out{filespath} )
226 or abort "Could not rename secondary files directory.";
227 }
228 else {
229 for ( File::Glob::bsd_glob("$prevfilespath/*") ) {
230 File::Copy::move( $_, $$out{filespath} )
231 or abort "Could not move secondary files.";
232 }
233 }
234 }
235 else {
236 my $createall = $needfilesdir ? $$out{filespath} : $$out{toppath};
237 if ( !-d $createall ) {
238 File::Path::make_path($createall)
239 or abort "Could not create output directory $createall.";
240 }
241 }
242 }
243
244 my %opt;
245 my @optdescr = ( 'exec|e=s', 'output|o=s', 'list|l!', 'help|h|?!', 'debug|d!' );
246 my %config;
247
248 my $status = GetOptions( \%opt, @optdescr );
249
250 if ( !$status || $opt{help} ) {
251 print <<EOF;
252 Usage: mhtml-tool [options] <MHT file>
253
254 By default, mhtml-tool unpacks an MHT archive (an archive type saved by some
255 browsers) to the current directory. The first HTML file in the archive is
256 taken for the primary web page, and all other contained files are written to a
257 directory named after that HTML file.
258
259 Options:
260 -e, --exec Execute the given program on a temporarily-unpacked archive
261 -l, --list List archive contents (file name, MIME type, size and URL)
262 -o, --output Unpack to directory <dir/> or to file <name>.html
263
264 Use the command "pod2man mhtml-tool > mhtml-tool.1" or
265 "pod2html mhtml-tool > mhtml-tool.html" to extract the manual.
266 EOF
267 exit !$status;
268 }
269
270 my $origin = getcwd;
271 my $tempdir = "";
272 if ( $opt{exec} and not defined $opt{output} ) {
273 $tempdir = tempdir( CLEANUP => 1 );
274 $opt{output} = $tempdir . "/";
275 }
276
277 my $orig_sep = $/;
278 my $crlf_file = 0;
279
280 # Print a message to stdout (if debug is enabled).
281 sub debug_msg {
282 if ( $opt{debug} ) {
283 print ":debug: $_[0]\n";
284 }
285 }
286
287 # Read next line. Remove line endings (both unix & windows style).
288 sub read_next_line {
289 my $cur_line = <>;
290 chomp $cur_line;
291 if ( $cur_line =~ /\r$/ ) {
292 debug_msg("It's a CRLF file") if ( $crlf_file == 0 );
293 $crlf_file++;
294 }
295 $cur_line =~ s/\r//;
296 return $cur_line;
297 }
298
299 # Parse headers.
300 # Handle multi-line headers (lines starting with spaces are part of a
301 # multi-line header).
302 # Read data from <> until the first empty line.
303 # Set $crlf_file to 1 if it's a CRLF (windoze) file.
304 sub parse_headers {
305 my %headers;
306 my $sep = $/;
307 $/ = $orig_sep;
308 my $cur_line = read_next_line;
309 my $full_line = $cur_line;
310
311 while ( not( $cur_line =~ /^$/ ) ) {
312 if ( $cur_line =~ /;$/ ) { # if a continued line...
313 while ( ( $cur_line = read_next_line ) =~ /^\h+/ ) {
314 $cur_line =~ s/^\h+//;
315 $full_line = $full_line . " " . $cur_line;
316 }
317 if ( $full_line =~ s/^([-\w]+): (.*)$// ) {
318 $headers{$1} = $2;
319 debug_msg("(1) New header $1 with value $2");
320 }
321 }
322 else {
323 if ( $full_line =~ s/^([-\w]+): (.*)$// ) {
324 $headers{$1} = $2;
325 debug_msg("(2) New header $1 with value $2");
326 }
327 }
328 $cur_line = read_next_line;
329 $full_line = $cur_line;
330 }
331 $/ = $sep;
332 return %headers;
333 }
334
335 my %global_headers = parse_headers();
336
337 abort "Can't find Content-Type header - not a MIME HTML file?"
338 if ( !defined( $global_headers{'Content-Type'} ) );
339 debug_msg("Global Content-Type: $global_headers{'Content-Type'}");
340
341 my $boundary = '';
342 my $endcode = '';
343
344 # FIXME: - add other possible mime types
345 # Types implemented so far are:
346 # Content-Type: multipart/related; boundary="----=_NextPart_01D8BCC9.47C2B260"
347 # Content-Type: text/html; charset="utf-8"
348 if ( $global_headers{'Content-Type'} =~
349 m!multipart/related;.*\h+boundary="?([^"]*)"?$! )
350 {
351 $endcode = $boundary = $1;
352 $endcode =~ s/\s+$//;
353 if ($crlf_file) {
354 $/ = "\r\n--$boundary\r\n";
355 }
356 else {
357 $/ = "\n--$boundary\n";
358 }
359 debug_msg("Boundary: $boundary");
360 }
361 elsif ( $global_headers{'Content-Type'} =~ m!text/html! ) {
362 $/ = '';
363 }
364 else {
365 die "Error: Unknown Content-Type: $global_headers{'Content-Type'}\n";
366 }
367
368 my %by_url;
369 my @htmlfiles;
370 my $fh;
371
372 {
373 my $fileind = 1;
374 my $leading = "--$boundary\n";
375 while ( defined( my $data = <> ) ) {
376 chomp $data;
377 $data =~ s/\R/\n/g; # handle various other line-endings (windows, mac)
378 while ( index( $data, $leading ) == 0 ) {
379 $data = substr( $data, length($leading) );
380 }
381 my %headers;
382 while ( $data =~ s/^([-\w]+): (.*)\n// ) {
383 $headers{$1} = $2;
384 debug_msg("New header $1 with value $2");
385
386 # read (and ignore atm) lines starting with space (multi-line headers)
387 while ( $data =~ s/^\h+.*\n// ) {
388 debug_msg("empty line");
389 }
390 }
391 if ( scalar(%headers) == 0 ) {
392 %headers = %global_headers
393 ; # fallback to the global headers as needed, usually for "text"
394 }
395 $data =~ s/^\n//;
396 $data =~ s/\n--$endcode--\r?\n$/\n/s;
397 my ( $type, $origname );
398 if ( defined( $headers{"Content-Type"} ) ) {
399 ($type) = $headers{"Content-Type"} =~ /^(\w+\/\w+)\b/;
400 debug_msg("type=$type");
401 }
402 else {
403 print "Error: No Content-Type found, skipping chunk\n";
404
405 # we could print the bad chunk here...
406 # but it's probably just a "warning" about
407 # "if you see this your software isn't recognizing a web archive file"
408 next;
409 }
410 if ( defined( $headers{"Content-Type"} )
411 && $headers{"Content-Type"} =~ /\bname=([^;]*)/ )
412 {
413 $origname = $1;
414 ($type) = $headers{"Content-Type"} =~ /^(\w+\/\w+)\b/;
415 $type //= "";
416 }
417 elsif ( defined( $headers{"Content-Disposition"} )
418 && $headers{"Content-Disposition"} =~ /\bfilename=([^;]*)/ )
419 {
420 $origname = $1;
421 if ( !defined($type) ) {
422 $type = $origname =~ /\.html?$/i ? "text/html" : "";
423 }
424 }
425 elsif ( defined( $headers{"Content-Location"} ) ) {
426 $origname = $headers{"Content-Location"};
427
428 # for unknown reasons, files generated by IE11 may contain some
429 # local paths with '\' instead of '/'
430 if ( $origname =~ m!^file://.*\\! ) {
431 debug_msg("Windows-style path detected: $origname");
432 $origname =~ s!^.*\\!!;
433 }
434 $origname =~ s!^.*/!!;
435 if ( !defined($type) ) {
436 $type = "";
437 }
438 }
439
440 $origname = "noname" unless defined $origname;
441 $origname = "noname" if ( $origname eq "" );
442 my $fname = unique_name( $origname, $ARGV[0] );
443 if ( !defined( $headers{"Content-Transfer-Encoding"} ) ) {
444 print STDERR "Info: Encoding of ", ordinal($fileind),
445 " file not found - leaving as-is.\n";
446 }
447 elsif ( $headers{"Content-Transfer-Encoding"} =~ /\bbase64\b/i ) {
448 $data = MIME::Base64::decode($data);
449 }
450 elsif (
451 $headers{"Content-Transfer-Encoding"} =~ /\bquoted-printable\b/i )
452 {
453 $data = MIME::QuotedPrint::decode($data);
454 }
455 debug_msg("origname=$origname; fname=$fname; type=$type");
456 debug_msg( "Content-Type: " . $headers{"Content-Type"} );
457 debug_msg( "Data size: " . length($data) );
458 if ( $opt{list} ) {
459 $origname =~ s/\s+$// if defined $origname;
460 my $size = length($data);
461 print $fname // "(?)", "\t", $type || "(?)", "\t$size\t",
462 $headers{"Content-Location"} // "(?)", "\n";
463 next;
464 }
465 $headers{fname} = $fname;
466 if ( $headers{"Content-Location"} ) {
467 $headers{url} = $headers{"Content-Location"};
468 $headers{url} =~ s/\s+$//;
469 $by_url{ $headers{url} } = \%headers;
470 debug_msg("Content-Location: $headers{url}");
471 }
472 else {
473 debug_msg("?? no location");
474 }
475 if ( $type eq "text/html" ) {
476 $headers{data} = $data;
477 if ( scalar @htmlfiles == 0 ) { # first html file must have a name
478 if ( !$headers{fname} ) {
479 $headers{fname} = "index.html";
480 }
481 }
482 push @htmlfiles, \%headers;
483 debug_msg("New html file in list: $headers{fname}");
484 }
485 else {
486 mkfiledir( $opt{output}, $htmlfiles[0]->{fname}, 1, \%config );
487 $fname = shorten_path "$config{filespath}/$fname";
488 open $fh, ">$fname" or abort "Could not create file $fname.";
489 print $fh $data;
490 close $fh;
491 }
492 }
493 continue {
494 debug_msg("");
495 ++$fileind;
496 }
497 }
498
499 if ( $opt{list} ) {
500 exit(0);
501 }
502
503 mkfiledir( $opt{output}, $htmlfiles[0]->{fname}, 0, \%config );
504
505 my $entrypoint = ".";
506 my $filesprefix = $config{filesdir} . "/";
507 my $outname = $config{firstout};
508 print "primary html output name: $outname\n";
509
510 for my $html (@htmlfiles) {
511 my $linksubst = "";
512 my $p = HTML::PullParser->new(
513 doc => \$html->{data},
514 "start" => 'text, attr, tagname',
515 "text" => 'text',
516 "end" => 'text'
517 );
518 while ( defined( my $tok = $p->get_token() ) ) {
519 my $linkary;
520 my @linkattrs;
521 if ( ref( $tok->[1] )
522 && ( $linkary = $HTML::Tagset::linkElements{ $tok->[2] } )
523 && ( @linkattrs = grep $tok->[1]->{$_}, @$linkary ) )
524 {
525 for my $attr (@linkattrs) {
526 my $uri = URI->new( $tok->[1]->{$attr} );
527 next unless defined $uri;
528 next unless defined $html->{url};
529 $uri = $uri->abs( $html->{url} );
530 $tok->[1]->{$attr} =
531 "$filesprefix" . $by_url{ $uri->as_string() }->{fname}
532 if $by_url{ $uri->as_string() };
533 }
534 delete $tok->[1]->{"/"};
535 $linksubst .=
536 "<$tok->[2] "
537 . join( " ",
538 map( "$_=\"$tok->[1]->{$_}\"", keys %{ $tok->[1] } ) )
539 . ">";
540 }
541 else {
542 $linksubst .= $tok->[0];
543 }
544 }
545 $outname = "$config{filespath}/$html->{fname}" unless defined $outname;
546 $outname = shorten_path $outname;
547 open $fh, ">$outname" or abort "Could not create file $outname.";
548 print $fh $linksubst;
549 close $fh;
550 $entrypoint = $outname if ( $entrypoint eq "." );
551
552 # for all except the first HTML file:
553 $filesprefix = "";
554 $outname = undef;
555 }
556
557 if ( $tempdir ne "" ) {
558 chdir $tempdir;
559 my $enforce = "";
560 $enforce = "-force-html"
561 if ( $entrypoint ne "."
562 and $entrypoint !~ /\.htm(l)?$/
563 and $opt{exec} =~ /(lynx|((e)?links(2)?))/ );
564 system( sprintf( "%s %s %s", $opt{exec}, $enforce, quoted $entrypoint ) );
565 chdir $origin;
566 }
567
568 1;