Debian Codegraph @

contrib/mkjigsnap

raw
1 #!/bin/sh
2 #
3 # mkjigsnap
4 #
5 # (c) 2004 Steve McIntyre
6 #
7 # Server-side wrapper; run this on a machine with a mirror to set up
8 # the snapshots for jigit
9 #
10 # GPL v2 - see COPYING
11 #
12 # Some things needed:
13 # location of the mirror
14 # the keyword to look for (e.g. Debian)
15 # the snapshot dirname (e.g. today's date)
16 # the jigdo files
17 # Example:
18 # ./mkjigsnap -m /tmp/mirror \
19 # -k Debian -k Non-US \
20 # -d 20041017 \
21 # jigdo1 jigdo2 jigdo3 ...
22
23 STARTDATE=`date`
24
25 while [ $# -gt 0 ]
26 do
27 case "$1"x in
28 "-m"x)
29 shift
30 MIRROR=$1
31 shift
32 ;;
33 "-d"x)
34 shift
35 DIRNAME=$1
36 shift
37 ;;
38 "-j"x)
39 shift
40 JIGDOLIST=$1
41 shift
42 ;;
43 "-t"x)
44 shift
45 TF_TMPDIR="--directory $1"
46 SORT_TMPDIR="-T $1"
47 shift
48 ;;
49 *)
50 JIGDOS="$JIGDOS $1"
51 shift
52 ;;
53 esac
54 done
55
56 if [ "$MIRROR"x = ""x ] ; then
57 echo "You must specify the location of the mirror!"
58 exit 1
59 fi
60
61 if [ "$DIRNAME"x = ""x ] ; then
62 echo "You must specify the snapshot directory name!"
63 exit 1
64 fi
65
66 if [ "$JIGDOS"x = ""x ] ; then
67 if [ "$JIGDOLIST"x = ""x ] ; then
68 echo "You must specify some jigdo files!"
69 exit 1
70 fi
71 fi
72
73 TMPFILE=`tempfile $TF_TMPDIR`
74 if [ "$JIGDOLIST"x != ""x ] ; then
75 for JIGDO in `cat $JIGDOLIST`
76 do
77 zcat -f $JIGDO | sed -n "s/^.*Debian://gp" >> $TMPFILE
78 done
79 fi
80
81 if [ "$JIGDOS"x != ""x ] ; then
82 zcat -f $JIGDOS | sed -n "s/^.*Debian://gp" >> $TMPFILE
83 fi
84
85 LISTDONEDATE=`date`
86
87 TMPFILE1=`tempfile $TF_TMPDIR`
88 TOTAL_FILES=`wc -l < $TMPFILE`
89
90 cat $TMPFILE | sort -u $SORT_TMPDIR > $TMPFILE1
91
92 SORTDONEDATE=`date`
93 NUM_FILES=`wc -l < $TMPFILE1`
94
95 mv -f $TMPFILE1 $TMPFILE
96
97 NUM=`cat $TMPFILE | wc -l`
98
99 LINKS_DONE=0
100 (echo $NUM; echo $MIRROR; echo $DIRNAME; cat $TMPFILE ) | perl -we '
101
102 use File::Basename;
103 use File::Find;
104 my $num;
105 my $mirrorpath;
106 my $outdir;
107 my $dirname;
108 my $filename;
109 my $done = 0;
110 my $failed = 0;
111 my @file_list;
112 my $old_deleted = 0;
113 my $link;
114 $| = 1;
115
116 # Make a dir tree
117 sub mkdirs {
118 my $input = shift;
119 my $dir;
120 my @components;
121 my $need_slash = 0;
122
123 if (! -d $input) {
124 @components = split /\//,$input;
125 foreach $component (@components) {
126 if ($need_slash) {
127 $dir = join ("/", $dir, $component);
128 } else {
129 $dir = $component;
130 $need_slash = 1;
131 }
132 if (! -d $dir) {
133 mkdir $dir;
134 }
135 }
136 }
137 }
138
139 sub delete_redundant {
140 my $ref;
141
142 if (-f) {
143 $ref = $file_list{$File::Find::name};
144 if (!defined($ref)) {
145 unlink($File::Find::name);
146 $old_deleted++;
147 if ( !($old_deleted % 1000) ) {
148 print "$old_deleted\n";
149 }
150 }
151 }
152 }
153
154 while (<>) {
155 chomp;
156
157 if (!defined($num)) {
158 $num = $_;
159 next;
160 }
161 if (!defined($mirrorpath)) {
162 $mirrorpath = $_;
163 next;
164 }
165 if (!defined($outdir)) {
166 $outdir = $_;
167 print "Linking $num files from $mirrorpath to $outdir\n";
168 next;
169 }
170
171 $outfile = $outdir . "/" . $_;
172 $file_list{$outfile}++;
173 if (! -e $outfile) {
174 $dirname = dirname($_);
175 $filename = basename($_);
176
177 mkdirs($outdir . "/" . $dirname);
178 my $link_ok = 0;
179 foreach $mirror (split /:/,$mirrorpath) {
180 $infile = $mirror . "/" . $_;
181 if (-l $infile) {
182 $link = readlink($infile);
183 if ($link =~ m#^/#) {
184 $infile = $link;
185 } else {
186 $infile = dirname($infile) . "/" . $link;
187 }
188 }
189 $outfile = $outdir . "/" . $_;
190 if (link ($infile, $outfile)) {
191 $link_ok = 1;
192 last;
193 }
194 $infile = $mirror . "/" . $filename;
195 # print "Falling back to $infile\n";
196 if (link ($infile, $outfile)) {
197 $link_ok = 1;
198 last;
199 }
200 }
201 if ($link_ok == 0) {
202 print "\nFailed to create link $outfile\n";
203 $failed++;
204 }
205 }
206 $done++;
207 if ( !($done % 1000) ) {
208 print "$done/$num\n";
209 }
210 }
211 print " Finished: $done/$num, $failed failed\n\n";
212
213 # Now walk the tree and delete files that we no longer need
214 print "Scanning for now-redundant files\n";
215 find(\&delete_redundant, $outdir);
216 print " Finished: $old_deleted old files removed\n";
217 '
218
219 rm -f $TMPFILE
220 echo
221
222 echo "$STARTDATE: startup"
223 echo "$LISTDONEDATE: $TOTAL_FILES found"
224 echo "$SORTDONEDATE: $NUM_FILES after sorting"
225 echo `date`": mkjigsnap finished successfully"
226