Debian Codegraph @

mkjigsnap

raw
1 #!/usr/bin/perl -w
2 #
3 # mkjigsnap
4 #
5 # (c) 2004-2019 Steve McIntyre <[email protected]>
6 #
7 # Server-side wrapper; run this on a machine with a mirror to set up
8 # the snapshots for jigit / jigdo downloading
9 #
10 # GPL v2 - see COPYING
11 #
12 # This script can be run in two modes:
13 #
14 # 1. To build a jigit .conf file for a single jigdo file:
15 # add the "-n" option with a CD name on the command line
16 # and only specify a single jigdo to work with using "-j".
17 #
18 # 2. To build a snapshot tree for (potentially multiple) jigdo files:
19 # do *not* specify the "-n" option, and list as many jigdo files as
20 # desired, either on the command line using multiple "-j <jigdo>" options
21 # or (better) via a file listing them with the "-J" option.
22 #
23 # Some things needed:
24 # (single-jigdo mode only) the CD name of the jigit
25 # (single-jigdo mode only) the output location; where the jigdo, template
26 # file and snapshot will be written
27 # (single-jigdo mode only) the locations of the input jigdo and template
28 # files
29 # the location of the mirror
30 # the keyword(s) to look for (e.g. Debian)
31 # the snapshot dirname (e.g. today's date)
32 #
33 # Example #1: (single-jigdo mode, used for Ubuntu jigit generation)
34 #
35 # mkjigsnap -o /tmp/mjs-test -n mjs-test -m /tmp/mirror \
36 # -j ~/jigdo/update/debian-update-3.0r2.01-i386.jigdo \
37 # -t ~/jigdo/update/debian-update-3.0r2.01-i386.template \
38 # -k Debian -k Non-US
39 # -d 20041017
40 #
41 # (This creates a single jigit conf file using the supplied jigdo/template
42 # file pair, looking for jigdo references to files in the "Debian" and
43 # "Non-US" areas. Output the files into /tmp/mjs-test and call them
44 # "mjs-test.<ext>", creating a snapshot of the needed files in
45 # /tmp/mjs-test/20041017 by linking files from /tmp/mirror as needed.)
46 #
47 # Example #2: (multi-jigdo mode, as run to keep
48 # http://us.cdimage.debian.org/cdimage/snapshot/ up to date)
49 #
50 # mkjigsnap -m /org/ftp/debian -J ~/jigdo.list \
51 # -k Debian \
52 # -d /org/jigdo-area/snapshot/Debian \
53 # -f ~/mkjigsnap-failed.log \
54 # -i ~/mkjigsnap-ignore.list
55 #
56 # (This reads in all the jigdo files listed in ~/jigdo.list, building a
57 # list of all the files referenced in the "Debian" area. It will then
58 # attempt to build a snapshot tree of all those files under
59 # /org/jigdo-area/snapshot/Debian by linking from /org/ftp/debian. Any
60 # files that are missing will be listed into the output "missing" file
61 # ~/mkjigsnap-failed.log for later checking, UNLESS they are already listed
62 # in the "ignore" file ~/mkjigsnap-ignore.list.)
63 #
64
65 use strict;
66 use Getopt::Long;
67 use File::Basename;
68 use File::Find;
69 use File::Copy;
70 use Compress::Zlib;
71 Getopt::Long::Configure ('no_ignore_case');
72 Getopt::Long::Configure ('no_auto_abbrev');
73
74 my $mode = "multi";
75 my $dryrun = 0;
76 my $verbose = 0;
77 my $startdate = `date -u`;
78 my ($jlistdonedate, $parsedonedate, $snapdonedate);
79 my @jigdos;
80 my $single_jigdo;
81 my @keywords;
82 my @mirrors;
83 my ($dirname, $failedfile, $ignorefile, $jigdolist, $mirror, $cdname,
84 $outdir, $tempdir, $template, $check_checksums, $checksum_out, $backref_file);
85 my $result;
86 my $num_jigdos = 0;
87 my $num_unsorted = 0;
88 my $num_unique = 0;
89 my @failed_files;
90 my @ck_failed_files;
91 my $old_deleted = 0;
92 my %ignored_fails;
93 my %file_list;
94 my %ref;
95 my %jigdo_backref;
96
97 GetOptions("b=s" => \$backref_file,
98 "c" => \$check_checksums,
99 "C=s" => \$checksum_out,
100 "d=s" => \$dirname,
101 "f=s" => \$failedfile,
102 "i=s" => \$ignorefile,
103 "J=s" => \$jigdolist,
104 "j=s" => \@jigdos,
105 "k=s" => \@keywords,
106 "m=s" => \@mirrors,
107 "N" => \$dryrun,
108 "n=s" => \$cdname,
109 "o=s" => \$outdir,
110 "T=s" => \$tempdir,
111 "t=s" => \$template,
112 "v" => \$verbose)
113 or die "Error in command line arguments, bailing out\n";
114
115 # Sanity-check arguments
116 if (!defined ($dirname)) {
117 die "You must specify the snapshot directory name!\n";
118 }
119 if (!@keywords) {
120 die "You must specify the keywords to match!\n";
121 }
122 if (!@mirrors) {
123 die "You must specify the location(s) of the mirror(s)!\n";
124 }
125 if (@jigdos) {
126 $num_jigdos += scalar(@jigdos);
127 }
128 if (defined($jigdolist)) {
129 $num_jigdos += `wc -w < $jigdolist`;
130 }
131 if ($num_jigdos == 0) {
132 die "No jigdo file(s) specified!\n";
133 }
134 if (defined($cdname)) {
135 $mode = "single";
136 }
137
138 if ($mode eq "single") {
139 if (!defined($cdname)) {
140 die "You must specify the output name for the jigit conf!\n";
141 }
142 if (!defined($outdir)) {
143 die "You must specify where to set up the snapshot!\n";
144 }
145 if (!defined($template)) {
146 die "You must specify the template file!\n";
147 }
148 if ($num_jigdos != 1) {
149 die "More than one jigdo file specified ($num_jigdos) in single-jigdo mode!\n";
150 }
151 # In single-jigdo mode, the snapshot directory is relative to the
152 # output dir
153 $dirname="$outdir/$dirname";
154 # And store the path to the jigdo file for later use
155 $single_jigdo = $jigdos[0];
156 } else {
157 if (defined($cdname)) {
158 die "Output name is meaningless for multi-jigdo mode!\n";
159 }
160 if (defined($outdir)) {
161 die "Output dir is meaningless for multi-jigdo mode!\n";
162 }
163 if (defined($template)) {
164 die "Template file name is meaningless for multi-jigdo mode!\n";
165 }
166 }
167
168 # Make a dir tree
169 sub mkdirs {
170 my $input = shift;
171 my $dir;
172 my @components;
173 my $need_slash = 0;
174
175 if (! -d $input) {
176 if ($verbose) {
177 print "mkdirs($input)\n";
178 }
179 if (!$dryrun) {
180 @components = split /\//,$input;
181 foreach my $component (@components) {
182 if ($need_slash) {
183 $dir = join ("/", $dir, $component);
184 } else {
185 $dir = $component;
186 $need_slash = 1;
187 }
188 mkdir $dir;
189 }
190 } else {
191 print "DRYRUN: not making directory tree $input\n";
192 }
193 }
194 }
195
196 sub delete_redundant {
197 my $link;
198
199 if (-f) {
200 $link = $file_list{$File::Find::name};
201 if (!defined($link)) {
202 if ($verbose) {
203 print "delete_redundant($File::Find::name)\n";
204 }
205 if (!$dryrun) {
206 unlink($File::Find::name);
207 } else {
208 print "DRYRUN: not deleting $File::Find::name\n";
209 }
210 $old_deleted++;
211 if ( !($old_deleted % 1000) ) {
212 print "$old_deleted\n";
213 }
214 }
215 }
216 }
217
218 sub parse_ignore_file {
219 my $inputfile = shift;
220 my $num_ignored_loaded = 0;
221 open(INLIST, "$inputfile") or return;
222 while (defined (my $pkg = <INLIST>)) {
223 chomp $pkg;
224 $ignored_fails{$pkg}++;
225 $num_ignored_loaded++;
226 }
227 print "parse_ignore_file: loaded $num_ignored_loaded entries from file $inputfile\n";
228 }
229
230 # Iff we have a checksum of the right type, calculate the checksum of
231 # the file on disk and validate
232 sub validate_checksum($$$) {
233 my $file = shift;
234 my $full_path = shift;
235 my $type = shift;
236 my $jigsum;
237 my $checksum = "";
238
239 if (! exists $ref{$file}{$type}) {
240 return 0; # Nothing to compare, so we're good!
241 }
242
243 # else
244 if ($type eq "md5") {
245 $jigsum= `jigsum $full_path 2>/dev/null`;
246 if ($jigsum =~ m/^(.{22}) /) {
247 $checksum = $1;
248 }
249 } elsif ($type eq "sha256") {
250 $jigsum= `jigsum-sha256 $full_path 2>/dev/null`;
251 if ($jigsum =~ m/^(.{43}) /) {
252 $checksum = $1;
253 }
254 }
255
256 if (length($checksum) < 2) {
257 # Didn't find a checksum in the jigsum output, so failed
258 return -2;
259 }
260
261 # else
262 if (!($ref{$file}{$type} =~ m/\Q$checksum\E/)) {
263 return -1;
264 }
265
266 # else
267 return 0;
268 }
269
270 sub generate_snapshot_tree () {
271 my $done = 0;
272 my $failed = 0;
273 my $ignored = 0;
274 my $ck_failed = 0;
275
276 $| = 1;
277
278 # Sorting is important here for performance, to help with
279 # directory lookups
280 foreach $_ (sort (keys %ref)) {
281 my $outfile = $dirname . "/" . $_;
282
283 $file_list{$outfile}++;
284 if ($verbose) {
285 print "file_list hash updated for $outfile\n";
286 }
287 if (! -e $outfile) {
288 my $dir = dirname($_);
289 my $filename = basename($_);
290 my $link;
291 my $link_ok = 0;
292 my $infile;
293
294 mkdirs($dirname . "/" . $dir);
295
296 foreach my $mirror (@mirrors) {
297 $infile = $mirror . "/" . $_;
298 if (-l $infile) {
299 $link = readlink($infile);
300 if ($link =~ m#^/#) {
301 $infile = $link;
302 } else {
303 $infile = dirname($infile) . "/" . $link;
304 }
305 }
306 if ($verbose) {
307 print "look for $_:\n";
308 }
309 $outfile = $dirname . "/" . $_;
310 if (!$dryrun) {
311 if ($verbose) {
312 print " try $infile\n";
313 }
314 if (link ($infile, $outfile)) {
315 $link_ok = 1;
316 last;
317 }
318 } else {
319 print "DRYRUN: not linking $infile to $outfile\n";
320 $link_ok = 1;
321 last;
322 }
323 $infile = $mirror . "/" . $filename;
324 if ($verbose) {
325 print " fallback: try $infile\n";
326 }
327 if (!$dryrun) {
328 if (link ($infile, $outfile)) {
329 $link_ok = 1;
330 last;
331 }
332 } else {
333 print "DRYRUN: not linking $infile to $outfile\n";
334 $link_ok = 1;
335 last;
336 }
337 }
338 if ($link_ok == 0) {
339 if ($ignored_fails{$_}) {
340 $ignored++;
341 } else {
342 if (!defined($failedfile)) {
343 # No logfile, print to stdout then
344 print "\nFailed to create link $outfile\n";
345 }
346 $failed++;
347 push (@failed_files, $_);
348 }
349 } else {
350 if ($ignored_fails{$_}) {
351 print "\n$_ marked as failed, but we found it anyway!\n";
352 }
353 }
354 }
355
356 if (-e $outfile && $check_checksums) {
357 my $csum_result;
358 $csum_result = validate_checksum($_, $outfile, "md5");
359 if (0 == $csum_result) {
360 # no problems
361 $csum_result = validate_checksum($_, $outfile, "sha256");
362 }
363 if ($csum_result == -1) {
364 print "\nChecksum failure: $_\n";
365 $ck_failed++;
366 push (@ck_failed_files, $_);
367 } elsif ($csum_result == -2) {
368 print "\nFailed to jigsum $_\n";
369 }
370 }
371
372 $done++;
373 if ( !($done % 10000) or ($check_checksums && !($done % 100))) {
374 print "$done done, ignored $ignored, failed $failed ck_failed $ck_failed out of $num_unique\n";
375 }
376 }
377 print " Finished: $done/$num_unique, $failed failed, $ck_failed ck_failed, ignored $ignored\n\n";
378
379 if (defined($failedfile) && ($failed > 0)) {
380 print "Writing list of failed files to $failedfile\n";
381 open(FAIL_LOG, "> $failedfile") or die "Failed to open $failedfile: $!\n";
382 if ($backref_file) {
383 open (BACKREF, "> $backref_file") or die "Failed to open $backref_file: $!\n";
384 }
385 foreach my $missing (@failed_files) {
386 print FAIL_LOG "$missing\n";
387 if ($backref_file) {
388 print BACKREF "$missing:\n";
389 print BACKREF $jigdo_backref{$missing};
390 }
391 }
392 close FAIL_LOG;
393 if ($backref_file) {
394 close BACKREF;
395 }
396 }
397
398 # Now walk the tree and delete files that we no longer need
399 print "Scanning for now-redundant files\n";
400 find(\&delete_redundant, $dirname);
401 print " Finished: $old_deleted old files removed\n";
402 }
403
404 # Parse jigdo_list file if we have one
405 if (defined($jigdolist)) {
406 if ($verbose) {
407 print "Checking for jigdos in $jigdolist\n";
408 }
409 open (INLIST, "$jigdolist") or die "Can't open file $jigdolist: $!\n";
410 while ($_ = <INLIST>) {
411 chomp;
412 if (length($_) > 1) {
413 push (@jigdos, $_);
414 }
415 }
416 close INLIST;
417 }
418 $jlistdonedate = `date -u`;
419
420 print "Working on $num_jigdos jigdo file(s)\n";
421 # Walk through the list of jigdos, parsing as we go
422 my $num_parsed = 0;
423 print "Reading / parsing jigdo file(s)\n";
424
425 foreach my $injig (sort @jigdos) {
426 open (INJIG, "zcat -f $injig |");
427 $num_parsed++;
428 while (<INJIG>) {
429 my ($file, $jigsum, $type);
430 chomp;
431 foreach my $keyword (@keywords) {
432 # Look for a jigdo format v1 match first, with
433 # base64(ish)-encoded md5 checksums (22 chars before the
434 # "=")
435 if (m/^(.{22})=$keyword:(.*)$/) {
436 $jigsum = $1;
437 $file = $2;
438 $file =~ s?^/??;
439 $type = "md5";
440 }
441 # Otherwise, look for a jigdo format v2 match, with
442 # base64(ish)-encoded sha256 checksums (43 chars before
443 # the "=")
444 if (m/^(.{43})=$keyword:(.*)$/) {
445 $jigsum = $1;
446 $file = $2;
447 $file =~ s?^/??;
448 $type = "sha256";
449 }
450 }
451 if (defined($file)) {
452 $num_unsorted++;
453 # Only count a ref of any kind as unique
454 if (!exists $ref{$file}) {
455 $num_unique++;
456 }
457 # Even though we have to treat different checksums
458 # differently
459 if (!exists $ref{$file}{$type}) {
460 $ref{$file}{$type} = $jigsum;
461 } else {
462 if (!($ref{$file}{$type} =~ /\Q$jigsum\E/ )) {
463 print " ERROR: $file referenced again with different checksum!\n";
464 print " (old " . $ref{$file}{$type} . " new $jigsum\n";
465 }
466 }
467 if ($backref_file) {
468 if (!defined $jigdo_backref{$file}) {
469 $jigdo_backref{$file} = " $injig\n";
470 } else {
471 $jigdo_backref{$file} .= " $injig\n";
472 }
473 }
474 if (!($num_unsorted % 100000) ) {
475 print " found $num_unsorted total, $num_unique unique file refs, $num_parsed / $num_jigdos jigdo files ($injig)\n";
476 }
477 }
478 }
479 close(INJIG);
480 }
481 $parsedonedate = `date -u`;
482 print " found $num_unsorted total, $num_unique unique file refs in $num_jigdos jigdo files\n";
483
484 if ($checksum_out) {
485 open(CK_OUT, "> $checksum_out") or die "Can't open $checksum_out for writing: $!\n";
486 foreach $_ (sort (keys %ref)) {
487 if (exists $ref{$_}{"md5"}) {
488 print CK_OUT $ref{$_}{"md5"} . " $_\n";
489 }
490 if (exists $ref{$_}{"sha256"}) {
491 print CK_OUT $ref{$_}{"sha256"} . " $_\n";
492 }
493 }
494 close(CK_OUT);
495 }
496
497 if ($num_unique < 5) {
498 die "Only $num_unique for the snapshot? Something is wrong; abort!\n"
499 }
500
501 # Now look at the snapshot dir
502 if (! -d $dirname) {
503 print "$dirname does not exist\n";
504 if (!$dryrun) {
505 mkdirs($dirname);
506 } else {
507 die "DRYRUN: not making it, so aborting\n";
508 }
509 }
510 if (defined($ignorefile)) {
511 parse_ignore_file($ignorefile);
512 }
513
514 print "Trying to snapshot-link $num_unique files into $dirname\n";
515 if ($check_checksums) {
516 print " (and checksumming every file, so this may take a while)\n";
517 }
518 generate_snapshot_tree();
519 $snapdonedate = `date -u`;
520
521 chomp ($startdate, $jlistdonedate, $parsedonedate, $snapdonedate);
522
523 print "$startdate: startup\n";
524 print "$jlistdonedate: found $num_jigdos jigdo files\n";
525 print "$parsedonedate: found $num_unsorted files referenced in those jigdo files, $num_unique unique\n";
526 print "$snapdonedate: snapshot done\n";
527
528 if ($mode eq "single") {
529 if ($dryrun) {
530 print "DRYRUN: Not creating files in $outdir\n";
531 } else {
532 my ($gzin, $gzout, $line);
533 $gzin = gzopen($single_jigdo, "rb") or
534 die "Unable to open jigdo file $single_jigdo for reading: $!\n";
535 $gzout = gzopen("$outdir/$cdname.jigdo", "wb9") or
536 die "Unable to open new jigdo file $outdir/$cdname.jigdo for writing: $!\n";
537 while ($gzin->gzreadline($line) > 0) {
538 $line =~ s:^Template=.*$:Template=$cdname.template:;
539 $gzout->gzwrite($line);
540 }
541 $gzin->close();
542 $gzout->close();
543 copy("$template", "$outdir/$cdname.template") or
544 die "Failed to copy template file $template: $!\n";
545 open (CONF, "> $outdir/$cdname.conf") or
546 die "Failed to open conf file $outdir/$cdname.conf for writing: $!\n";
547 print CONF "JIGDO=$cdname.jigdo\n";
548 print CONF "TEMPLATE=$cdname.template\n";
549 print CONF "SNAPSHOT=snapshot/$dirname\n";
550 close(CONF);
551 print "Jigdo files, config and snapshot made in $outdir\n";
552 }
553 }