| 1 |
.\" -*- mode: troff; coding: utf-8 -*- |
| 2 |
.\" Automatically generated by Pod::Man 5.0102 (Pod::Simple 3.45) |
| 3 |
.\" |
| 4 |
.\" Standard preamble: |
| 5 |
.\" ======================================================================== |
| 6 |
.de Sp \" Vertical space (when we can't use .PP) |
| 7 |
.if t .sp .5v |
| 8 |
.if n .sp |
| 9 |
.. |
| 10 |
.de Vb \" Begin verbatim text |
| 11 |
.ft CW |
| 12 |
.nf |
| 13 |
.ne \\$1 |
| 14 |
.. |
| 15 |
.de Ve \" End verbatim text |
| 16 |
.ft R |
| 17 |
.fi |
| 18 |
.. |
| 19 |
.\" \*(C` and \*(C' are quotes in nroff, nothing in troff, for use with C<>. |
| 20 |
.ie n \{\ |
| 21 |
. ds C` "" |
| 22 |
. ds C' "" |
| 23 |
'br\} |
| 24 |
.el\{\ |
| 25 |
. ds C` |
| 26 |
. ds C' |
| 27 |
'br\} |
| 28 |
.\" |
| 29 |
.\" Escape single quotes in literal strings from groff's Unicode transform. |
| 30 |
.ie \n(.g .ds Aq \(aq |
| 31 |
.el .ds Aq ' |
| 32 |
.\" |
| 33 |
.\" If the F register is >0, we'll generate index entries on stderr for |
| 34 |
.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index |
| 35 |
.\" entries marked with X<> in POD. Of course, you'll have to process the |
| 36 |
.\" output yourself in some meaningful fashion. |
| 37 |
.\" |
| 38 |
.\" Avoid warning from groff about undefined register 'F'. |
| 39 |
.de IX |
| 40 |
.. |
| 41 |
.nr rF 0 |
| 42 |
.if \n(.g .if rF .nr rF 1 |
| 43 |
.if (\n(rF:(\n(.g==0)) \{\ |
| 44 |
. if \nF \{\ |
| 45 |
. de IX |
| 46 |
. tm Index:\\$1\t\\n%\t"\\$2" |
| 47 |
.. |
| 48 |
. if !\nF==2 \{\ |
| 49 |
. nr % 0 |
| 50 |
. nr F 2 |
| 51 |
. \} |
| 52 |
. \} |
| 53 |
.\} |
| 54 |
.rr rF |
| 55 |
.\" ======================================================================== |
| 56 |
.\" |
| 57 |
.IX Title "IO::Compress::FAQ 3perl" |
| 58 |
.TH IO::Compress::FAQ 3perl 2025-11-16 "perl v5.40.1" "Perl Programmers Reference Guide" |
| 59 |
.\" For nroff, turn off justification. Always turn off hyphenation; it makes |
| 60 |
.\" way too many mistakes in technical documents. |
| 61 |
.if n .ad l |
| 62 |
.nh |
| 63 |
.SH NAME |
| 64 |
IO::Compress::FAQ \-\- Frequently Asked Questions about IO::Compress |
| 65 |
.SH DESCRIPTION |
| 66 |
.IX Header "DESCRIPTION" |
| 67 |
Common questions answered. |
| 68 |
.SH GENERAL |
| 69 |
.IX Header "GENERAL" |
| 70 |
.SS "Compatibility with Unix compress/uncompress." |
| 71 |
.IX Subsection "Compatibility with Unix compress/uncompress." |
| 72 |
Although \f(CW\*(C`Compress::Zlib\*(C'\fR has a pair of functions called \f(CW\*(C`compress\*(C'\fR and |
| 73 |
\&\f(CW\*(C`uncompress\*(C'\fR, they are \fInot\fR related to the Unix programs of the same |
| 74 |
name. The \f(CW\*(C`Compress::Zlib\*(C'\fR module is not compatible with Unix |
| 75 |
\&\f(CW\*(C`compress\*(C'\fR. |
| 76 |
.PP |
| 77 |
If you have the \f(CW\*(C`uncompress\*(C'\fR program available, you can use this to read |
| 78 |
compressed files |
| 79 |
.PP |
| 80 |
.Vb 4 |
| 81 |
\& open F, "uncompress \-c $filename |"; |
| 82 |
\& while (<F>) |
| 83 |
\& { |
| 84 |
\& ... |
| 85 |
.Ve |
| 86 |
.PP |
| 87 |
Alternatively, if you have the \f(CW\*(C`gunzip\*(C'\fR program available, you can use |
| 88 |
this to read compressed files |
| 89 |
.PP |
| 90 |
.Vb 4 |
| 91 |
\& open F, "gunzip \-c $filename |"; |
| 92 |
\& while (<F>) |
| 93 |
\& { |
| 94 |
\& ... |
| 95 |
.Ve |
| 96 |
.PP |
| 97 |
and this to write compress files, if you have the \f(CW\*(C`compress\*(C'\fR program |
| 98 |
available |
| 99 |
.PP |
| 100 |
.Vb 4 |
| 101 |
\& open F, "| compress \-c $filename "; |
| 102 |
\& print F "data"; |
| 103 |
\& ... |
| 104 |
\& close F ; |
| 105 |
.Ve |
| 106 |
.SS "Accessing .tar.Z files" |
| 107 |
.IX Subsection "Accessing .tar.Z files" |
| 108 |
The \f(CW\*(C`Archive::Tar\*(C'\fR module can optionally use \f(CW\*(C`Compress::Zlib\*(C'\fR (via the |
| 109 |
\&\f(CW\*(C`IO::Zlib\*(C'\fR module) to access tar files that have been compressed with |
| 110 |
\&\f(CW\*(C`gzip\*(C'\fR. Unfortunately tar files compressed with the Unix \f(CW\*(C`compress\*(C'\fR |
| 111 |
utility cannot be read by \f(CW\*(C`Compress::Zlib\*(C'\fR and so cannot be directly |
| 112 |
accessed by \f(CW\*(C`Archive::Tar\*(C'\fR. |
| 113 |
.PP |
| 114 |
If the \f(CW\*(C`uncompress\*(C'\fR or \f(CW\*(C`gunzip\*(C'\fR programs are available, you can use one |
| 115 |
of these workarounds to read \f(CW\*(C`.tar.Z\*(C'\fR files from \f(CW\*(C`Archive::Tar\*(C'\fR |
| 116 |
.PP |
| 117 |
Firstly with \f(CW\*(C`uncompress\*(C'\fR |
| 118 |
.PP |
| 119 |
.Vb 3 |
| 120 |
\& use strict; |
| 121 |
\& use warnings; |
| 122 |
\& use Archive::Tar; |
| 123 |
\& |
| 124 |
\& open F, "uncompress \-c $filename |"; |
| 125 |
\& my $tar = Archive::Tar\->new(*F); |
| 126 |
\& ... |
| 127 |
.Ve |
| 128 |
.PP |
| 129 |
and this with \f(CW\*(C`gunzip\*(C'\fR |
| 130 |
.PP |
| 131 |
.Vb 3 |
| 132 |
\& use strict; |
| 133 |
\& use warnings; |
| 134 |
\& use Archive::Tar; |
| 135 |
\& |
| 136 |
\& open F, "gunzip \-c $filename |"; |
| 137 |
\& my $tar = Archive::Tar\->new(*F); |
| 138 |
\& ... |
| 139 |
.Ve |
| 140 |
.PP |
| 141 |
Similarly, if the \f(CW\*(C`compress\*(C'\fR program is available, you can use this to |
| 142 |
write a \f(CW\*(C`.tar.Z\*(C'\fR file |
| 143 |
.PP |
| 144 |
.Vb 4 |
| 145 |
\& use strict; |
| 146 |
\& use warnings; |
| 147 |
\& use Archive::Tar; |
| 148 |
\& use IO::File; |
| 149 |
\& |
| 150 |
\& my $fh = IO::File\->new( "| compress \-c >$filename" ); |
| 151 |
\& my $tar = Archive::Tar\->new(); |
| 152 |
\& ... |
| 153 |
\& $tar\->write($fh); |
| 154 |
\& $fh\->close ; |
| 155 |
.Ve |
| 156 |
.SS "How do I recompress using a different compression?" |
| 157 |
.IX Subsection "How do I recompress using a different compression?" |
| 158 |
This is easier that you might expect if you realise that all the |
| 159 |
\&\f(CW\*(C`IO::Compress::*\*(C'\fR objects are derived from \f(CW\*(C`IO::File\*(C'\fR and that all the |
| 160 |
\&\f(CW\*(C`IO::Uncompress::*\*(C'\fR modules can read from an \f(CW\*(C`IO::File\*(C'\fR filehandle. |
| 161 |
.PP |
| 162 |
So, for example, say you have a file compressed with gzip that you want to |
| 163 |
recompress with bzip2. Here is all that is needed to carry out the |
| 164 |
recompression. |
| 165 |
.PP |
| 166 |
.Vb 2 |
| 167 |
\& use IO::Uncompress::Gunzip \*(Aq:all\*(Aq; |
| 168 |
\& use IO::Compress::Bzip2 \*(Aq:all\*(Aq; |
| 169 |
\& |
| 170 |
\& my $gzipFile = "somefile.gz"; |
| 171 |
\& my $bzipFile = "somefile.bz2"; |
| 172 |
\& |
| 173 |
\& my $gunzip = IO::Uncompress::Gunzip\->new( $gzipFile ) |
| 174 |
\& or die "Cannot gunzip $gzipFile: $GunzipError\en" ; |
| 175 |
\& |
| 176 |
\& bzip2 $gunzip => $bzipFile |
| 177 |
\& or die "Cannot bzip2 to $bzipFile: $Bzip2Error\en" ; |
| 178 |
.Ve |
| 179 |
.PP |
| 180 |
Note, there is a limitation of this technique. Some compression file |
| 181 |
formats store extra information along with the compressed data payload. For |
| 182 |
example, gzip can optionally store the original filename and Zip stores a |
| 183 |
lot of information about the original file. If the original compressed file |
| 184 |
contains any of this extra information, it will not be transferred to the |
| 185 |
new compressed file using the technique above. |
| 186 |
.SH ZIP |
| 187 |
.IX Header "ZIP" |
| 188 |
.SS "What Compression Types do IO::Compress::Zip & IO::Uncompress::Unzip support?" |
| 189 |
.IX Subsection "What Compression Types do IO::Compress::Zip & IO::Uncompress::Unzip support?" |
| 190 |
The following compression formats are supported by \f(CW\*(C`IO::Compress::Zip\*(C'\fR and |
| 191 |
\&\f(CW\*(C`IO::Uncompress::Unzip\*(C'\fR |
| 192 |
.IP \(bu 5 |
| 193 |
Store (method 0) |
| 194 |
.Sp |
| 195 |
No compression at all. |
| 196 |
.IP \(bu 5 |
| 197 |
Deflate (method 8) |
| 198 |
.Sp |
| 199 |
This is the default compression used when creating a zip file with |
| 200 |
\&\f(CW\*(C`IO::Compress::Zip\*(C'\fR. |
| 201 |
.IP \(bu 5 |
| 202 |
Bzip2 (method 12) |
| 203 |
.Sp |
| 204 |
Only supported if the \f(CW\*(C`IO\-Compress\-Bzip2\*(C'\fR module is installed. |
| 205 |
.IP \(bu 5 |
| 206 |
Lzma (method 14) |
| 207 |
.Sp |
| 208 |
Only supported if the \f(CW\*(C`IO\-Compress\-Lzma\*(C'\fR module is installed. |
| 209 |
.SS "Can I Read/Write Zip files larger the 4 Gig?" |
| 210 |
.IX Subsection "Can I Read/Write Zip files larger the 4 Gig?" |
| 211 |
Yes, both the \f(CW\*(C`IO\-Compress\-Zip\*(C'\fR and \f(CW\*(C`IO\-Uncompress\-Unzip\*(C'\fR modules |
| 212 |
support the zip feature called \fIZip64\fR. That allows them to read/write |
| 213 |
files/buffers larger than 4Gig. |
| 214 |
.PP |
| 215 |
If you are creating a Zip file using the one-shot interface, and any of the |
| 216 |
input files is greater than 4Gig, a zip64 complaint zip file will be |
| 217 |
created. |
| 218 |
.PP |
| 219 |
.Vb 1 |
| 220 |
\& zip "really\-large\-file" => "my.zip"; |
| 221 |
.Ve |
| 222 |
.PP |
| 223 |
Similarly with the one-shot interface, if the input is a buffer larger than |
| 224 |
4 Gig, a zip64 complaint zip file will be created. |
| 225 |
.PP |
| 226 |
.Vb 1 |
| 227 |
\& zip \e$really_large_buffer => "my.zip"; |
| 228 |
.Ve |
| 229 |
.PP |
| 230 |
The one-shot interface allows you to force the creation of a zip64 zip file |
| 231 |
by including the \f(CW\*(C`Zip64\*(C'\fR option. |
| 232 |
.PP |
| 233 |
.Vb 1 |
| 234 |
\& zip $filehandle => "my.zip", Zip64 => 1; |
| 235 |
.Ve |
| 236 |
.PP |
| 237 |
If you want to create a zip64 zip file with the OO interface you must |
| 238 |
specify the \f(CW\*(C`Zip64\*(C'\fR option. |
| 239 |
.PP |
| 240 |
.Vb 1 |
| 241 |
\& my $zip = IO::Compress::Zip\->new( "whatever", Zip64 => 1 ); |
| 242 |
.Ve |
| 243 |
.PP |
| 244 |
When uncompressing with \f(CW\*(C`IO\-Uncompress\-Unzip\*(C'\fR, it will automatically |
| 245 |
detect if the zip file is zip64. |
| 246 |
.PP |
| 247 |
If you intend to manipulate the Zip64 zip files created with |
| 248 |
\&\f(CW\*(C`IO\-Compress\-Zip\*(C'\fR using an external zip/unzip, make sure that it supports |
| 249 |
Zip64. |
| 250 |
.PP |
| 251 |
In particular, if you are using Info-Zip you need to have zip version 3.x |
| 252 |
or better to update a Zip64 archive and unzip version 6.x to read a zip64 |
| 253 |
archive. |
| 254 |
.SS "Can I write more that 64K entries is a Zip files?" |
| 255 |
.IX Subsection "Can I write more that 64K entries is a Zip files?" |
| 256 |
Yes. Zip64 allows this. See previous question. |
| 257 |
.SS "Zip Resources" |
| 258 |
.IX Subsection "Zip Resources" |
| 259 |
The primary reference for zip files is the "appnote" document available at |
| 260 |
<http://www.pkware.com/documents/casestudies/APPNOTE.TXT> |
| 261 |
.PP |
| 262 |
An alternatively is the Info-Zip appnote. This is available from |
| 263 |
<ftp://ftp.info\-zip.org/pub/infozip/doc/> |
| 264 |
.SH GZIP |
| 265 |
.IX Header "GZIP" |
| 266 |
.SS "Gzip Resources" |
| 267 |
.IX Subsection "Gzip Resources" |
| 268 |
The primary reference for gzip files is RFC 1952 |
| 269 |
<https://datatracker.ietf.org/doc/html/rfc1952> |
| 270 |
.PP |
| 271 |
The primary site for gzip is <http://www.gzip.org>. |
| 272 |
.SS "Dealing with concatenated gzip files" |
| 273 |
.IX Subsection "Dealing with concatenated gzip files" |
| 274 |
If the gunzip program encounters a file containing multiple gzip files |
| 275 |
concatenated together it will automatically uncompress them all. |
| 276 |
The example below illustrates this behaviour |
| 277 |
.PP |
| 278 |
.Vb 5 |
| 279 |
\& $ echo abc | gzip \-c >x.gz |
| 280 |
\& $ echo def | gzip \-c >>x.gz |
| 281 |
\& $ gunzip \-c x.gz |
| 282 |
\& abc |
| 283 |
\& def |
| 284 |
.Ve |
| 285 |
.PP |
| 286 |
By default \f(CW\*(C`IO::Uncompress::Gunzip\*(C'\fR will \fInot\fR behave like the gunzip |
| 287 |
program. It will only uncompress the first gzip data stream in the file, as |
| 288 |
shown below |
| 289 |
.PP |
| 290 |
.Vb 2 |
| 291 |
\& $ perl \-MIO::Uncompress::Gunzip=:all \-e \*(Aqgunzip "x.gz" => \e*STDOUT\*(Aq |
| 292 |
\& abc |
| 293 |
.Ve |
| 294 |
.PP |
| 295 |
To force \f(CW\*(C`IO::Uncompress::Gunzip\*(C'\fR to uncompress all the gzip data streams, |
| 296 |
include the \f(CW\*(C`MultiStream\*(C'\fR option, as shown below |
| 297 |
.PP |
| 298 |
.Vb 3 |
| 299 |
\& $ perl \-MIO::Uncompress::Gunzip=:all \-e \*(Aqgunzip "x.gz" => \e*STDOUT, MultiStream => 1\*(Aq |
| 300 |
\& abc |
| 301 |
\& def |
| 302 |
.Ve |
| 303 |
.SS "Reading bgzip files with IO::Uncompress::Gunzip" |
| 304 |
.IX Subsection "Reading bgzip files with IO::Uncompress::Gunzip" |
| 305 |
A \f(CW\*(C`bgzip\*(C'\fR file consists of a series of valid gzip-compliant data streams |
| 306 |
concatenated together. To read a file created by \f(CW\*(C`bgzip\*(C'\fR with |
| 307 |
\&\f(CW\*(C`IO::Uncompress::Gunzip\*(C'\fR use the \f(CW\*(C`MultiStream\*(C'\fR option as shown in the |
| 308 |
previous section. |
| 309 |
.PP |
| 310 |
See the section titled "The BGZF compression format" in |
| 311 |
<http://samtools.github.io/hts\-specs/SAMv1.pdf> for a definition of |
| 312 |
\&\f(CW\*(C`bgzip\*(C'\fR. |
| 313 |
.SH ZLIB |
| 314 |
.IX Header "ZLIB" |
| 315 |
.SS "Zlib Resources" |
| 316 |
.IX Subsection "Zlib Resources" |
| 317 |
The primary site for the \fIzlib\fR compression library is |
| 318 |
<http://www.zlib.org>. |
| 319 |
.SH Bzip2 |
| 320 |
.IX Header "Bzip2" |
| 321 |
.SS "Bzip2 Resources" |
| 322 |
.IX Subsection "Bzip2 Resources" |
| 323 |
The primary site for bzip2 is <http://www.bzip.org>. |
| 324 |
.SS "Dealing with Concatenated bzip2 files" |
| 325 |
.IX Subsection "Dealing with Concatenated bzip2 files" |
| 326 |
If the bunzip2 program encounters a file containing multiple bzip2 files |
| 327 |
concatenated together it will automatically uncompress them all. |
| 328 |
The example below illustrates this behaviour |
| 329 |
.PP |
| 330 |
.Vb 5 |
| 331 |
\& $ echo abc | bzip2 \-c >x.bz2 |
| 332 |
\& $ echo def | bzip2 \-c >>x.bz2 |
| 333 |
\& $ bunzip2 \-c x.bz2 |
| 334 |
\& abc |
| 335 |
\& def |
| 336 |
.Ve |
| 337 |
.PP |
| 338 |
By default \f(CW\*(C`IO::Uncompress::Bunzip2\*(C'\fR will \fInot\fR behave like the bunzip2 |
| 339 |
program. It will only uncompress the first bunzip2 data stream in the file, as |
| 340 |
shown below |
| 341 |
.PP |
| 342 |
.Vb 2 |
| 343 |
\& $ perl \-MIO::Uncompress::Bunzip2=:all \-e \*(Aqbunzip2 "x.bz2" => \e*STDOUT\*(Aq |
| 344 |
\& abc |
| 345 |
.Ve |
| 346 |
.PP |
| 347 |
To force \f(CW\*(C`IO::Uncompress::Bunzip2\*(C'\fR to uncompress all the bzip2 data streams, |
| 348 |
include the \f(CW\*(C`MultiStream\*(C'\fR option, as shown below |
| 349 |
.PP |
| 350 |
.Vb 3 |
| 351 |
\& $ perl \-MIO::Uncompress::Bunzip2=:all \-e \*(Aqbunzip2 "x.bz2" => \e*STDOUT, MultiStream => 1\*(Aq |
| 352 |
\& abc |
| 353 |
\& def |
| 354 |
.Ve |
| 355 |
.SS "Interoperating with Pbzip2" |
| 356 |
.IX Subsection "Interoperating with Pbzip2" |
| 357 |
Pbzip2 (<http://compression.ca/pbzip2/>) is a parallel implementation of |
| 358 |
bzip2. The output from pbzip2 consists of a series of concatenated bzip2 |
| 359 |
data streams. |
| 360 |
.PP |
| 361 |
By default \f(CW\*(C`IO::Uncompress::Bzip2\*(C'\fR will only uncompress the first bzip2 |
| 362 |
data stream in a pbzip2 file. To uncompress the complete pbzip2 file you |
| 363 |
must include the \f(CW\*(C`MultiStream\*(C'\fR option, like this. |
| 364 |
.PP |
| 365 |
.Vb 2 |
| 366 |
\& bunzip2 $input => \e$output, MultiStream => 1 |
| 367 |
\& or die "bunzip2 failed: $Bunzip2Error\en"; |
| 368 |
.Ve |
| 369 |
.SH "HTTP & NETWORK" |
| 370 |
.IX Header "HTTP & NETWORK" |
| 371 |
.SS "Apache::GZip Revisited" |
| 372 |
.IX Subsection "Apache::GZip Revisited" |
| 373 |
Below is a mod_perl Apache compression module, called \f(CW\*(C`Apache::GZip\*(C'\fR, |
| 374 |
taken from |
| 375 |
<http://perl.apache.org/docs/tutorials/tips/mod_perl_tricks/mod_perl_tricks.html#On_the_Fly_Compression> |
| 376 |
.PP |
| 377 |
.Vb 2 |
| 378 |
\& package Apache::GZip; |
| 379 |
\& #File: Apache::GZip.pm |
| 380 |
\& |
| 381 |
\& use strict vars; |
| 382 |
\& use Apache::Constants \*(Aq:common\*(Aq; |
| 383 |
\& use Compress::Zlib; |
| 384 |
\& use IO::File; |
| 385 |
\& use constant GZIP_MAGIC => 0x1f8b; |
| 386 |
\& use constant OS_MAGIC => 0x03; |
| 387 |
\& |
| 388 |
\& sub handler { |
| 389 |
\& my $r = shift; |
| 390 |
\& my ($fh,$gz); |
| 391 |
\& my $file = $r\->filename; |
| 392 |
\& return DECLINED unless $fh=IO::File\->new($file); |
| 393 |
\& $r\->header_out(\*(AqContent\-Encoding\*(Aq=>\*(Aqgzip\*(Aq); |
| 394 |
\& $r\->send_http_header; |
| 395 |
\& return OK if $r\->header_only; |
| 396 |
\& |
| 397 |
\& tie *STDOUT,\*(AqApache::GZip\*(Aq,$r; |
| 398 |
\& print($_) while <$fh>; |
| 399 |
\& untie *STDOUT; |
| 400 |
\& return OK; |
| 401 |
\& } |
| 402 |
\& |
| 403 |
\& sub TIEHANDLE { |
| 404 |
\& my($class,$r) = @_; |
| 405 |
\& # initialize a deflation stream |
| 406 |
\& my $d = deflateInit(\-WindowBits=>\-MAX_WBITS()) || return undef; |
| 407 |
\& |
| 408 |
\& # gzip header \-\- don\*(Aqt ask how I found out |
| 409 |
\& $r\->print(pack("nccVcc",GZIP_MAGIC,Z_DEFLATED,0,time(),0,OS_MAGIC)); |
| 410 |
\& |
| 411 |
\& return bless { r => $r, |
| 412 |
\& crc => crc32(undef), |
| 413 |
\& d => $d, |
| 414 |
\& l => 0 |
| 415 |
\& },$class; |
| 416 |
\& } |
| 417 |
\& |
| 418 |
\& sub PRINT { |
| 419 |
\& my $self = shift; |
| 420 |
\& foreach (@_) { |
| 421 |
\& # deflate the data |
| 422 |
\& my $data = $self\->{d}\->deflate($_); |
| 423 |
\& $self\->{r}\->print($data); |
| 424 |
\& # keep track of its length and crc |
| 425 |
\& $self\->{l} += length($_); |
| 426 |
\& $self\->{crc} = crc32($_,$self\->{crc}); |
| 427 |
\& } |
| 428 |
\& } |
| 429 |
\& |
| 430 |
\& sub DESTROY { |
| 431 |
\& my $self = shift; |
| 432 |
\& |
| 433 |
\& # flush the output buffers |
| 434 |
\& my $data = $self\->{d}\->flush; |
| 435 |
\& $self\->{r}\->print($data); |
| 436 |
\& |
| 437 |
\& # print the CRC and the total length (uncompressed) |
| 438 |
\& $self\->{r}\->print(pack("LL",@{$self}{qw/crc l/})); |
| 439 |
\& } |
| 440 |
\& |
| 441 |
\& 1; |
| 442 |
.Ve |
| 443 |
.PP |
| 444 |
Here's the Apache configuration entry you'll need to make use of it. Once |
| 445 |
set it will result in everything in the /compressed directory will be |
| 446 |
compressed automagically. |
| 447 |
.PP |
| 448 |
.Vb 4 |
| 449 |
\& <Location /compressed> |
| 450 |
\& SetHandler perl\-script |
| 451 |
\& PerlHandler Apache::GZip |
| 452 |
\& </Location> |
| 453 |
.Ve |
| 454 |
.PP |
| 455 |
Although at first sight there seems to be quite a lot going on in |
| 456 |
\&\f(CW\*(C`Apache::GZip\*(C'\fR, you could sum up what the code was doing as follows \-\- |
| 457 |
read the contents of the file in \f(CW\*(C`$r\->filename\*(C'\fR, compress it and write |
| 458 |
the compressed data to standard output. That's all. |
| 459 |
.PP |
| 460 |
This code has to jump through a few hoops to achieve this because |
| 461 |
.IP 1. 4 |
| 462 |
The gzip support in \f(CW\*(C`Compress::Zlib\*(C'\fR version 1.x can only work with a real |
| 463 |
filesystem filehandle. The filehandles used by Apache modules are not |
| 464 |
associated with the filesystem. |
| 465 |
.IP 2. 4 |
| 466 |
That means all the gzip support has to be done by hand \- in this case by |
| 467 |
creating a tied filehandle to deal with creating the gzip header and |
| 468 |
trailer. |
| 469 |
.PP |
| 470 |
\&\f(CW\*(C`IO::Compress::Gzip\*(C'\fR doesn't have that filehandle limitation (this was one |
| 471 |
of the reasons for writing it in the first place). So if |
| 472 |
\&\f(CW\*(C`IO::Compress::Gzip\*(C'\fR is used instead of \f(CW\*(C`Compress::Zlib\*(C'\fR the whole tied |
| 473 |
filehandle code can be removed. Here is the rewritten code. |
| 474 |
.PP |
| 475 |
.Vb 1 |
| 476 |
\& package Apache::GZip; |
| 477 |
\& |
| 478 |
\& use strict vars; |
| 479 |
\& use Apache::Constants \*(Aq:common\*(Aq; |
| 480 |
\& use IO::Compress::Gzip; |
| 481 |
\& use IO::File; |
| 482 |
\& |
| 483 |
\& sub handler { |
| 484 |
\& my $r = shift; |
| 485 |
\& my ($fh,$gz); |
| 486 |
\& my $file = $r\->filename; |
| 487 |
\& return DECLINED unless $fh=IO::File\->new($file); |
| 488 |
\& $r\->header_out(\*(AqContent\-Encoding\*(Aq=>\*(Aqgzip\*(Aq); |
| 489 |
\& $r\->send_http_header; |
| 490 |
\& return OK if $r\->header_only; |
| 491 |
\& |
| 492 |
\& my $gz = IO::Compress::Gzip\->new( \*(Aq\-\*(Aq, Minimal => 1 ) |
| 493 |
\& or return DECLINED ; |
| 494 |
\& |
| 495 |
\& print $gz $_ while <$fh>; |
| 496 |
\& |
| 497 |
\& return OK; |
| 498 |
\& } |
| 499 |
.Ve |
| 500 |
.PP |
| 501 |
or even more succinctly, like this, using a one-shot gzip |
| 502 |
.PP |
| 503 |
.Vb 1 |
| 504 |
\& package Apache::GZip; |
| 505 |
\& |
| 506 |
\& use strict vars; |
| 507 |
\& use Apache::Constants \*(Aq:common\*(Aq; |
| 508 |
\& use IO::Compress::Gzip qw(gzip); |
| 509 |
\& |
| 510 |
\& sub handler { |
| 511 |
\& my $r = shift; |
| 512 |
\& $r\->header_out(\*(AqContent\-Encoding\*(Aq=>\*(Aqgzip\*(Aq); |
| 513 |
\& $r\->send_http_header; |
| 514 |
\& return OK if $r\->header_only; |
| 515 |
\& |
| 516 |
\& gzip $r\->filename => \*(Aq\-\*(Aq, Minimal => 1 |
| 517 |
\& or return DECLINED ; |
| 518 |
\& |
| 519 |
\& return OK; |
| 520 |
\& } |
| 521 |
\& |
| 522 |
\& 1; |
| 523 |
.Ve |
| 524 |
.PP |
| 525 |
The use of one-shot \f(CW\*(C`gzip\*(C'\fR above just reads from \f(CW\*(C`$r\->filename\*(C'\fR and |
| 526 |
writes the compressed data to standard output. |
| 527 |
.PP |
| 528 |
Note the use of the \f(CW\*(C`Minimal\*(C'\fR option in the code above. When using gzip |
| 529 |
for Content-Encoding you should \fIalways\fR use this option. In the example |
| 530 |
above it will prevent the filename being included in the gzip header and |
| 531 |
make the size of the gzip data stream a slight bit smaller. |
| 532 |
.SS "Compressed files and Net::FTP" |
| 533 |
.IX Subsection "Compressed files and Net::FTP" |
| 534 |
The \f(CW\*(C`Net::FTP\*(C'\fR module provides two low-level methods called \f(CW\*(C`stor\*(C'\fR and |
| 535 |
\&\f(CW\*(C`retr\*(C'\fR that both return filehandles. These filehandles can used with the |
| 536 |
\&\f(CW\*(C`IO::Compress/Uncompress\*(C'\fR modules to compress or uncompress files read |
| 537 |
from or written to an FTP Server on the fly, without having to create a |
| 538 |
temporary file. |
| 539 |
.PP |
| 540 |
Firstly, here is code that uses \f(CW\*(C`retr\*(C'\fR to uncompressed a file as it is |
| 541 |
read from the FTP Server. |
| 542 |
.PP |
| 543 |
.Vb 2 |
| 544 |
\& use Net::FTP; |
| 545 |
\& use IO::Uncompress::Gunzip qw(:all); |
| 546 |
\& |
| 547 |
\& my $ftp = Net::FTP\->new( ... ) |
| 548 |
\& |
| 549 |
\& my $retr_fh = $ftp\->retr($compressed_filename); |
| 550 |
\& gunzip $retr_fh => $outFilename, AutoClose => 1 |
| 551 |
\& or die "Cannot uncompress \*(Aq$compressed_file\*(Aq: $GunzipError\en"; |
| 552 |
.Ve |
| 553 |
.PP |
| 554 |
and this to compress a file as it is written to the FTP Server |
| 555 |
.PP |
| 556 |
.Vb 2 |
| 557 |
\& use Net::FTP; |
| 558 |
\& use IO::Compress::Gzip qw(:all); |
| 559 |
\& |
| 560 |
\& my $stor_fh = $ftp\->stor($filename); |
| 561 |
\& gzip "filename" => $stor_fh, AutoClose => 1 |
| 562 |
\& or die "Cannot compress \*(Aq$filename\*(Aq: $GzipError\en"; |
| 563 |
.Ve |
| 564 |
.SH MISC |
| 565 |
.IX Header "MISC" |
| 566 |
.ie n .SS "Using ""InputLength"" to uncompress data embedded in a larger file/buffer." |
| 567 |
.el .SS "Using \f(CWInputLength\fP to uncompress data embedded in a larger file/buffer." |
| 568 |
.IX Subsection "Using InputLength to uncompress data embedded in a larger file/buffer." |
| 569 |
A fairly common use-case is where compressed data is embedded in a larger |
| 570 |
file/buffer and you want to read both. |
| 571 |
.PP |
| 572 |
As an example consider the structure of a zip file. This is a well-defined |
| 573 |
file format that mixes both compressed and uncompressed sections of data in |
| 574 |
a single file. |
| 575 |
.PP |
| 576 |
For the purposes of this discussion you can think of a zip file as sequence |
| 577 |
of compressed data streams, each of which is prefixed by an uncompressed |
| 578 |
local header. The local header contains information about the compressed |
| 579 |
data stream, including the name of the compressed file and, in particular, |
| 580 |
the length of the compressed data stream. |
| 581 |
.PP |
| 582 |
To illustrate how to use \f(CW\*(C`InputLength\*(C'\fR here is a script that walks a zip |
| 583 |
file and prints out how many lines are in each compressed file (if you |
| 584 |
intend write code to walking through a zip file for real see |
| 585 |
"Walking through a zip file" in IO::Uncompress::Unzip ). Also, although |
| 586 |
this example uses the zlib-based compression, the technique can be used by |
| 587 |
the other \f(CW\*(C`IO::Uncompress::*\*(C'\fR modules. |
| 588 |
.PP |
| 589 |
.Vb 2 |
| 590 |
\& use strict; |
| 591 |
\& use warnings; |
| 592 |
\& |
| 593 |
\& use IO::File; |
| 594 |
\& use IO::Uncompress::RawInflate qw(:all); |
| 595 |
\& |
| 596 |
\& use constant ZIP_LOCAL_HDR_SIG => 0x04034b50; |
| 597 |
\& use constant ZIP_LOCAL_HDR_LENGTH => 30; |
| 598 |
\& |
| 599 |
\& my $file = $ARGV[0] ; |
| 600 |
\& |
| 601 |
\& my $fh = IO::File\->new( "<$file" ) |
| 602 |
\& or die "Cannot open \*(Aq$file\*(Aq: $!\en"; |
| 603 |
\& |
| 604 |
\& while (1) |
| 605 |
\& { |
| 606 |
\& my $sig; |
| 607 |
\& my $buffer; |
| 608 |
\& |
| 609 |
\& my $x ; |
| 610 |
\& ($x = $fh\->read($buffer, ZIP_LOCAL_HDR_LENGTH)) == ZIP_LOCAL_HDR_LENGTH |
| 611 |
\& or die "Truncated file: $!\en"; |
| 612 |
\& |
| 613 |
\& my $signature = unpack ("V", substr($buffer, 0, 4)); |
| 614 |
\& |
| 615 |
\& last unless $signature == ZIP_LOCAL_HDR_SIG; |
| 616 |
\& |
| 617 |
\& # Read Local Header |
| 618 |
\& my $gpFlag = unpack ("v", substr($buffer, 6, 2)); |
| 619 |
\& my $compressedMethod = unpack ("v", substr($buffer, 8, 2)); |
| 620 |
\& my $compressedLength = unpack ("V", substr($buffer, 18, 4)); |
| 621 |
\& my $uncompressedLength = unpack ("V", substr($buffer, 22, 4)); |
| 622 |
\& my $filename_length = unpack ("v", substr($buffer, 26, 2)); |
| 623 |
\& my $extra_length = unpack ("v", substr($buffer, 28, 2)); |
| 624 |
\& |
| 625 |
\& my $filename ; |
| 626 |
\& $fh\->read($filename, $filename_length) == $filename_length |
| 627 |
\& or die "Truncated file\en"; |
| 628 |
\& |
| 629 |
\& $fh\->read($buffer, $extra_length) == $extra_length |
| 630 |
\& or die "Truncated file\en"; |
| 631 |
\& |
| 632 |
\& if ($compressedMethod != 8 && $compressedMethod != 0) |
| 633 |
\& { |
| 634 |
\& warn "Skipping file \*(Aq$filename\*(Aq \- not deflated $compressedMethod\en"; |
| 635 |
\& $fh\->read($buffer, $compressedLength) == $compressedLength |
| 636 |
\& or die "Truncated file\en"; |
| 637 |
\& next; |
| 638 |
\& } |
| 639 |
\& |
| 640 |
\& if ($compressedMethod == 0 && $gpFlag & 8 == 8) |
| 641 |
\& { |
| 642 |
\& die "Streamed Stored not supported for \*(Aq$filename\*(Aq\en"; |
| 643 |
\& } |
| 644 |
\& |
| 645 |
\& next if $compressedLength == 0; |
| 646 |
\& |
| 647 |
\& # Done reading the Local Header |
| 648 |
\& |
| 649 |
\& my $inf = IO::Uncompress::RawInflate\->new( $fh, |
| 650 |
\& Transparent => 1, |
| 651 |
\& InputLength => $compressedLength ) |
| 652 |
\& or die "Cannot uncompress $file [$filename]: $RawInflateError\en" ; |
| 653 |
\& |
| 654 |
\& my $line_count = 0; |
| 655 |
\& |
| 656 |
\& while (<$inf>) |
| 657 |
\& { |
| 658 |
\& ++ $line_count; |
| 659 |
\& } |
| 660 |
\& |
| 661 |
\& print "$filename: $line_count\en"; |
| 662 |
\& } |
| 663 |
.Ve |
| 664 |
.PP |
| 665 |
The majority of the code above is concerned with reading the zip local |
| 666 |
header data. The code that I want to focus on is at the bottom. |
| 667 |
.PP |
| 668 |
.Vb 1 |
| 669 |
\& while (1) { |
| 670 |
\& |
| 671 |
\& # read local zip header data |
| 672 |
\& # get $filename |
| 673 |
\& # get $compressedLength |
| 674 |
\& |
| 675 |
\& my $inf = IO::Uncompress::RawInflate\->new( $fh, |
| 676 |
\& Transparent => 1, |
| 677 |
\& InputLength => $compressedLength ) |
| 678 |
\& or die "Cannot uncompress $file [$filename]: $RawInflateError\en" ; |
| 679 |
\& |
| 680 |
\& my $line_count = 0; |
| 681 |
\& |
| 682 |
\& while (<$inf>) |
| 683 |
\& { |
| 684 |
\& ++ $line_count; |
| 685 |
\& } |
| 686 |
\& |
| 687 |
\& print "$filename: $line_count\en"; |
| 688 |
\& } |
| 689 |
.Ve |
| 690 |
.PP |
| 691 |
The call to \f(CW\*(C`IO::Uncompress::RawInflate\*(C'\fR creates a new filehandle \f(CW$inf\fR |
| 692 |
that can be used to read from the parent filehandle \f(CW$fh\fR, uncompressing |
| 693 |
it as it goes. The use of the \f(CW\*(C`InputLength\*(C'\fR option will guarantee that |
| 694 |
\&\fIat most\fR \f(CW$compressedLength\fR bytes of compressed data will be read from |
| 695 |
the \f(CW$fh\fR filehandle (The only exception is for an error case like a |
| 696 |
truncated file or a corrupt data stream). |
| 697 |
.PP |
| 698 |
This means that once RawInflate is finished \f(CW$fh\fR will be left at the |
| 699 |
byte directly after the compressed data stream. |
| 700 |
.PP |
| 701 |
Now consider what the code looks like without \f(CW\*(C`InputLength\*(C'\fR |
| 702 |
.PP |
| 703 |
.Vb 1 |
| 704 |
\& while (1) { |
| 705 |
\& |
| 706 |
\& # read local zip header data |
| 707 |
\& # get $filename |
| 708 |
\& # get $compressedLength |
| 709 |
\& |
| 710 |
\& # read all the compressed data into $data |
| 711 |
\& read($fh, $data, $compressedLength); |
| 712 |
\& |
| 713 |
\& my $inf = IO::Uncompress::RawInflate\->new( \e$data, |
| 714 |
\& Transparent => 1 ) |
| 715 |
\& or die "Cannot uncompress $file [$filename]: $RawInflateError\en" ; |
| 716 |
\& |
| 717 |
\& my $line_count = 0; |
| 718 |
\& |
| 719 |
\& while (<$inf>) |
| 720 |
\& { |
| 721 |
\& ++ $line_count; |
| 722 |
\& } |
| 723 |
\& |
| 724 |
\& print "$filename: $line_count\en"; |
| 725 |
\& } |
| 726 |
.Ve |
| 727 |
.PP |
| 728 |
The difference here is the addition of the temporary variable \f(CW$data\fR. |
| 729 |
This is used to store a copy of the compressed data while it is being |
| 730 |
uncompressed. |
| 731 |
.PP |
| 732 |
If you know that \f(CW$compressedLength\fR isn't that big then using temporary |
| 733 |
storage won't be a problem. But if \f(CW$compressedLength\fR is very large or |
| 734 |
you are writing an application that other people will use, and so have no |
| 735 |
idea how big \f(CW$compressedLength\fR will be, it could be an issue. |
| 736 |
.PP |
| 737 |
Using \f(CW\*(C`InputLength\*(C'\fR avoids the use of temporary storage and means the |
| 738 |
application can cope with large compressed data streams. |
| 739 |
.PP |
| 740 |
One final point \-\- obviously \f(CW\*(C`InputLength\*(C'\fR can only be used whenever you |
| 741 |
know the length of the compressed data beforehand, like here with a zip |
| 742 |
file. |
| 743 |
.SH SUPPORT |
| 744 |
.IX Header "SUPPORT" |
| 745 |
General feedback/questions/bug reports should be sent to |
| 746 |
<https://github.com/pmqs//issues> (preferred) or |
| 747 |
<https://rt.cpan.org/Public/Dist/Display.html?Name=>. |
| 748 |
.SH "SEE ALSO" |
| 749 |
.IX Header "SEE ALSO" |
| 750 |
Compress::Zlib, IO::Compress::Gzip, IO::Uncompress::Gunzip, IO::Compress::Deflate, IO::Uncompress::Inflate, IO::Compress::RawDeflate, IO::Uncompress::RawInflate, IO::Compress::Bzip2, IO::Uncompress::Bunzip2, IO::Compress::Lzma, IO::Uncompress::UnLzma, IO::Compress::Xz, IO::Uncompress::UnXz, IO::Compress::Lzip, IO::Uncompress::UnLzip, IO::Compress::Lzop, IO::Uncompress::UnLzop, IO::Compress::Lzf, IO::Uncompress::UnLzf, IO::Compress::Zstd, IO::Uncompress::UnZstd, IO::Uncompress::AnyInflate, IO::Uncompress::AnyUncompress |
| 751 |
.PP |
| 752 |
IO::Compress::FAQ |
| 753 |
.PP |
| 754 |
File::GlobMapper, Archive::Zip, |
| 755 |
Archive::Tar, |
| 756 |
IO::Zlib |
| 757 |
.SH AUTHOR |
| 758 |
.IX Header "AUTHOR" |
| 759 |
This module was written by Paul Marquess, \f(CW\*(C`[email protected]\*(C'\fR. |
| 760 |
.SH "MODIFICATION HISTORY" |
| 761 |
.IX Header "MODIFICATION HISTORY" |
| 762 |
See the Changes file. |
| 763 |
.SH "COPYRIGHT AND LICENSE" |
| 764 |
.IX Header "COPYRIGHT AND LICENSE" |
| 765 |
Copyright (c) 2005\-2024 Paul Marquess. All rights reserved. |
| 766 |
.PP |
| 767 |
This program is free software; you can redistribute it and/or |
| 768 |
modify it under the same terms as Perl itself. |