| 1 |
#!/bin/sh |
| 2 |
# |
| 3 |
# mkjigsnap |
| 4 |
# |
| 5 |
# (c) 2004 Steve McIntyre |
| 6 |
# |
| 7 |
# Server-side wrapper; run this on a machine with a mirror to set up |
| 8 |
# the snapshots for jigit |
| 9 |
# |
| 10 |
# GPL v2 - see COPYING |
| 11 |
# |
| 12 |
# Some things needed: |
| 13 |
# location of the mirror |
| 14 |
# the keyword to look for (e.g. Debian) |
| 15 |
# the snapshot dirname (e.g. today's date) |
| 16 |
# the jigdo files |
| 17 |
# Example: |
| 18 |
# ./mkjigsnap -m /tmp/mirror \ |
| 19 |
# -k Debian -k Non-US \ |
| 20 |
# -d 20041017 \ |
| 21 |
# jigdo1 jigdo2 jigdo3 ... |
| 22 |
|
| 23 |
STARTDATE=`date` |
| 24 |
|
| 25 |
while [ $# -gt 0 ] |
| 26 |
do |
| 27 |
case "$1"x in |
| 28 |
"-m"x) |
| 29 |
shift |
| 30 |
MIRROR=$1 |
| 31 |
shift |
| 32 |
;; |
| 33 |
"-d"x) |
| 34 |
shift |
| 35 |
DIRNAME=$1 |
| 36 |
shift |
| 37 |
;; |
| 38 |
"-j"x) |
| 39 |
shift |
| 40 |
JIGDOLIST=$1 |
| 41 |
shift |
| 42 |
;; |
| 43 |
"-t"x) |
| 44 |
shift |
| 45 |
TF_TMPDIR="--directory $1" |
| 46 |
SORT_TMPDIR="-T $1" |
| 47 |
shift |
| 48 |
;; |
| 49 |
*) |
| 50 |
JIGDOS="$JIGDOS $1" |
| 51 |
shift |
| 52 |
;; |
| 53 |
esac |
| 54 |
done |
| 55 |
|
| 56 |
if [ "$MIRROR"x = ""x ] ; then |
| 57 |
echo "You must specify the location of the mirror!" |
| 58 |
exit 1 |
| 59 |
fi |
| 60 |
|
| 61 |
if [ "$DIRNAME"x = ""x ] ; then |
| 62 |
echo "You must specify the snapshot directory name!" |
| 63 |
exit 1 |
| 64 |
fi |
| 65 |
|
| 66 |
if [ "$JIGDOS"x = ""x ] ; then |
| 67 |
if [ "$JIGDOLIST"x = ""x ] ; then |
| 68 |
echo "You must specify some jigdo files!" |
| 69 |
exit 1 |
| 70 |
fi |
| 71 |
fi |
| 72 |
|
| 73 |
TMPFILE=`tempfile $TF_TMPDIR` |
| 74 |
if [ "$JIGDOLIST"x != ""x ] ; then |
| 75 |
for JIGDO in `cat $JIGDOLIST` |
| 76 |
do |
| 77 |
zcat -f $JIGDO | sed -n "s/^.*Debian://gp" >> $TMPFILE |
| 78 |
done |
| 79 |
fi |
| 80 |
|
| 81 |
if [ "$JIGDOS"x != ""x ] ; then |
| 82 |
zcat -f $JIGDOS | sed -n "s/^.*Debian://gp" >> $TMPFILE |
| 83 |
fi |
| 84 |
|
| 85 |
LISTDONEDATE=`date` |
| 86 |
|
| 87 |
TMPFILE1=`tempfile $TF_TMPDIR` |
| 88 |
TOTAL_FILES=`wc -l < $TMPFILE` |
| 89 |
|
| 90 |
cat $TMPFILE | sort -u $SORT_TMPDIR > $TMPFILE1 |
| 91 |
|
| 92 |
SORTDONEDATE=`date` |
| 93 |
NUM_FILES=`wc -l < $TMPFILE1` |
| 94 |
|
| 95 |
mv -f $TMPFILE1 $TMPFILE |
| 96 |
|
| 97 |
NUM=`cat $TMPFILE | wc -l` |
| 98 |
|
| 99 |
LINKS_DONE=0 |
| 100 |
(echo $NUM; echo $MIRROR; echo $DIRNAME; cat $TMPFILE ) | perl -we ' |
| 101 |
|
| 102 |
use File::Basename; |
| 103 |
use File::Find; |
| 104 |
my $num; |
| 105 |
my $mirrorpath; |
| 106 |
my $outdir; |
| 107 |
my $dirname; |
| 108 |
my $filename; |
| 109 |
my $done = 0; |
| 110 |
my $failed = 0; |
| 111 |
my @file_list; |
| 112 |
my $old_deleted = 0; |
| 113 |
my $link; |
| 114 |
$| = 1; |
| 115 |
|
| 116 |
# Make a dir tree |
| 117 |
sub mkdirs { |
| 118 |
my $input = shift; |
| 119 |
my $dir; |
| 120 |
my @components; |
| 121 |
my $need_slash = 0; |
| 122 |
|
| 123 |
if (! -d $input) { |
| 124 |
@components = split /\//,$input; |
| 125 |
foreach $component (@components) { |
| 126 |
if ($need_slash) { |
| 127 |
$dir = join ("/", $dir, $component); |
| 128 |
} else { |
| 129 |
$dir = $component; |
| 130 |
$need_slash = 1; |
| 131 |
} |
| 132 |
if (! -d $dir) { |
| 133 |
mkdir $dir; |
| 134 |
} |
| 135 |
} |
| 136 |
} |
| 137 |
} |
| 138 |
|
| 139 |
sub delete_redundant { |
| 140 |
my $ref; |
| 141 |
|
| 142 |
if (-f) { |
| 143 |
$ref = $file_list{$File::Find::name}; |
| 144 |
if (!defined($ref)) { |
| 145 |
unlink($File::Find::name); |
| 146 |
$old_deleted++; |
| 147 |
if ( !($old_deleted % 1000) ) { |
| 148 |
print "$old_deleted\n"; |
| 149 |
} |
| 150 |
} |
| 151 |
} |
| 152 |
} |
| 153 |
|
| 154 |
while (<>) { |
| 155 |
chomp; |
| 156 |
|
| 157 |
if (!defined($num)) { |
| 158 |
$num = $_; |
| 159 |
next; |
| 160 |
} |
| 161 |
if (!defined($mirrorpath)) { |
| 162 |
$mirrorpath = $_; |
| 163 |
next; |
| 164 |
} |
| 165 |
if (!defined($outdir)) { |
| 166 |
$outdir = $_; |
| 167 |
print "Linking $num files from $mirrorpath to $outdir\n"; |
| 168 |
next; |
| 169 |
} |
| 170 |
|
| 171 |
$outfile = $outdir . "/" . $_; |
| 172 |
$file_list{$outfile}++; |
| 173 |
if (! -e $outfile) { |
| 174 |
$dirname = dirname($_); |
| 175 |
$filename = basename($_); |
| 176 |
|
| 177 |
mkdirs($outdir . "/" . $dirname); |
| 178 |
my $link_ok = 0; |
| 179 |
foreach $mirror (split /:/,$mirrorpath) { |
| 180 |
$infile = $mirror . "/" . $_; |
| 181 |
if (-l $infile) { |
| 182 |
$link = readlink($infile); |
| 183 |
if ($link =~ m#^/#) { |
| 184 |
$infile = $link; |
| 185 |
} else { |
| 186 |
$infile = dirname($infile) . "/" . $link; |
| 187 |
} |
| 188 |
} |
| 189 |
$outfile = $outdir . "/" . $_; |
| 190 |
if (link ($infile, $outfile)) { |
| 191 |
$link_ok = 1; |
| 192 |
last; |
| 193 |
} |
| 194 |
$infile = $mirror . "/" . $filename; |
| 195 |
# print "Falling back to $infile\n"; |
| 196 |
if (link ($infile, $outfile)) { |
| 197 |
$link_ok = 1; |
| 198 |
last; |
| 199 |
} |
| 200 |
} |
| 201 |
if ($link_ok == 0) { |
| 202 |
print "\nFailed to create link $outfile\n"; |
| 203 |
$failed++; |
| 204 |
} |
| 205 |
} |
| 206 |
$done++; |
| 207 |
if ( !($done % 1000) ) { |
| 208 |
print "$done/$num\n"; |
| 209 |
} |
| 210 |
} |
| 211 |
print " Finished: $done/$num, $failed failed\n\n"; |
| 212 |
|
| 213 |
# Now walk the tree and delete files that we no longer need |
| 214 |
print "Scanning for now-redundant files\n"; |
| 215 |
find(\&delete_redundant, $outdir); |
| 216 |
print " Finished: $old_deleted old files removed\n"; |
| 217 |
' |
| 218 |
|
| 219 |
rm -f $TMPFILE |
| 220 |
echo |
| 221 |
|
| 222 |
echo "$STARTDATE: startup" |
| 223 |
echo "$LISTDONEDATE: $TOTAL_FILES found" |
| 224 |
echo "$SORTDONEDATE: $NUM_FILES after sorting" |
| 225 |
echo `date`": mkjigsnap finished successfully" |
| 226 |
|