| 1 |
#!/bin/sh |
| 2 |
# |
| 3 |
# merge_package_lists |
| 4 |
# |
| 5 |
# Quick, hacky script to merge the package lists for multiple arches |
| 6 |
# together, with 2 priorities: |
| 7 |
# |
| 8 |
# 1. keep the ordering of the original files wherever possible |
| 9 |
# 2. collect together instances of the same package on multiple arches |
| 10 |
# |
| 11 |
# expects multiple files containing lines of the form, one arch per file |
| 12 |
# |
| 13 |
# <arch>:<component>:<pkgname> |
| 14 |
# |
| 15 |
# and will output similar |
| 16 |
# |
| 17 |
# The files MUST be passed by filename rather than simply piped in, |
| 18 |
# otherwise the ordering constraint will be lost |
| 19 |
# |
| 20 |
# Things also get quite complicated here due to considering |
| 21 |
# source. Depending on how things are being run: |
| 22 |
# |
| 23 |
# 1. If we're not doing source, life is easy - simply interleave the |
| 24 |
# binary packages |
| 25 |
# 2. If we're *only* doing source, then we won't be called at all - |
| 26 |
# the Makefile will just dump the sources directly into "packages" |
| 27 |
# without needing our help |
| 28 |
# 3. If we're doing mixed source/binary discs, then we have to work |
| 29 |
# quite hard here. We generate the list of sources from the |
| 30 |
# binaries given to us. For each binary, we will attempt to place |
| 31 |
# its source *first* onto the disc so we can make nice GPL-compliant |
| 32 |
# media. However, we also need to check whether or not that source |
| 33 |
# has already been placed due to an *earlier* binary package (a |
| 34 |
# different binary built from the same source, or the same for |
| 35 |
# another arch). To allow us to so that, we need to parse the Sources |
| 36 |
# files first to build up the src<->bin relationship. Ick... :-/ |
| 37 |
# |
| 38 |
# An added bit of grot: to make sure that sources are placed first, we |
| 39 |
# *temporarily* call the source architecture "aaaaaaaaaaaaaaaaaa" to force |
| 40 |
# it to sort first, then replace it woth "source" again later. *grin* |
| 41 |
# |
| 42 |
# Steve McIntyre <[email protected]> 2006, GPL v2 |
| 43 |
|
| 44 |
BDIR=$1 |
| 45 |
ADIR=$2 |
| 46 |
ARCHES="$3" |
| 47 |
OUT="$4" |
| 48 |
|
| 49 |
case "$ARCHES" in |
| 50 |
*source*) source=yes;; |
| 51 |
*) source=no;; |
| 52 |
esac |
| 53 |
|
| 54 |
for ARCH in $ARCHES_NOSRC |
| 55 |
do |
| 56 |
list="$list $BDIR/packages.$ARCH" |
| 57 |
done |
| 58 |
|
| 59 |
echo "# Starting merge at "`date` >> $BDIR/list.mid |
| 60 |
if [ $source = "yes" ] ; then |
| 61 |
echo "DONEDONE" > $BDIR/DONE |
| 62 |
awk -v max_size=$MAX_PKG_SIZE ' |
| 63 |
|
| 64 |
# First parse the Sources files |
| 65 |
/^Package:/ { |
| 66 |
srcname=$2 |
| 67 |
srcs_done++ |
| 68 |
} |
| 69 |
|
| 70 |
/^Version:/ { |
| 71 |
version=$2 |
| 72 |
# Cannot cope with : in output |
| 73 |
gsub(":", "%", version) |
| 74 |
if (srcname in versions) { |
| 75 |
versions[srcname]=sprintf("%s:%s", versions[srcname], version) |
| 76 |
} else { |
| 77 |
versions[srcname] = version |
| 78 |
} |
| 79 |
next |
| 80 |
} |
| 81 |
|
| 82 |
/^Binary:/ { |
| 83 |
bins=$0 |
| 84 |
gsub("^Binary:","",bins) |
| 85 |
gsub(",","",bins) |
| 86 |
nb=split(bins, binarray) |
| 87 |
for (i=1; i <= nb; i++) { |
| 88 |
bin2src[binarray[i]] = srcname |
| 89 |
bins_done++ |
| 90 |
} |
| 91 |
in_binary = 1 |
| 92 |
in_files = 0 |
| 93 |
next |
| 94 |
} |
| 95 |
|
| 96 |
/^Files:/ { |
| 97 |
in_files = 1 |
| 98 |
in_binary = 0 |
| 99 |
next |
| 100 |
} |
| 101 |
|
| 102 |
/^ / { |
| 103 |
if (in_files) { # Cope with multiple lines of files for source packages |
| 104 |
size[srcname]+=$2 |
| 105 |
sizes[srcname, version]+=$2 |
| 106 |
next |
| 107 |
} |
| 108 |
if (in_binary) { # Cope with multi-line Binary: fields |
| 109 |
bins=$0 |
| 110 |
gsub(",","",bins) |
| 111 |
nb=split(bins, binarray) |
| 112 |
for (i=1; i <= nb; i++) { |
| 113 |
bin2src[binarray[i]] = srcname |
| 114 |
bins_done++ |
| 115 |
} |
| 116 |
next |
| 117 |
} |
| 118 |
} |
| 119 |
|
| 120 |
# Done reading the Sources |
| 121 |
/^DONEDONE/ { |
| 122 |
parsed=1 |
| 123 |
printf("# Parsed Sources files: %d sources producing %d binaries\n", srcs_done, bins_done) |
| 124 |
next |
| 125 |
} |
| 126 |
|
| 127 |
# Now start placing source packages, depending on the order of the binary packages |
| 128 |
/^./ { |
| 129 |
in_files = 0 |
| 130 |
in_binary = 0 |
| 131 |
if (parsed) { |
| 132 |
num_sources = 0 |
| 133 |
split($0,fields,":") |
| 134 |
arch=fields[1] |
| 135 |
component=fields[2] |
| 136 |
pkg=fields[3] |
| 137 |
pkgsize=fields[4] |
| 138 |
srcpkg=bin2src[pkg] |
| 139 |
built_using=fields[5] |
| 140 |
if ("" != built_using) { |
| 141 |
num_sources = split(built_using, sources, ",") |
| 142 |
} |
| 143 |
if ("" == srcpkg) { |
| 144 |
if ("main-installer" == component) { |
| 145 |
printf("# Ignoring source for udeb %s\n", pkg) |
| 146 |
} else { |
| 147 |
printf("# Source for pkg %s is UNKNOWN!\n", pkg) |
| 148 |
} |
| 149 |
} else { |
| 150 |
sources[++num_sources] = srcpkg |
| 151 |
} |
| 152 |
if (num_sources) { |
| 153 |
for(i = 1; i <= num_sources; i++) { |
| 154 |
source = sources[i] |
| 155 |
if (!included[source]) { |
| 156 |
printf("# Adding source %s at %d because of %s:%s (%s)\n", |
| 157 |
source, FNR, arch, pkg, (i == num_sources ? "dep" : "b-u")) |
| 158 |
included[source] = pkg |
| 159 |
incarch[source] = arch |
| 160 |
indexnr[source] = FNR |
| 161 |
sourcecomp[source] = component |
| 162 |
} else { |
| 163 |
if (FNR < indexnr[source]) { |
| 164 |
printf("# Updating source %s: was due to %d:%s:%s, now moved earlier because of %d:%s:%s (%s)\n", |
| 165 |
source, indexnr[source], |
| 166 |
incarch[source], included[source], |
| 167 |
FNR, arch, pkg, (i == num_sources ? "dep" : "b-u")) |
| 168 |
included[source] = pkg |
| 169 |
incarch[source] = arch |
| 170 |
indexnr[source] = FNR |
| 171 |
} |
| 172 |
} |
| 173 |
} |
| 174 |
} |
| 175 |
if (size[source] <= max_size) { |
| 176 |
# List the binary |
| 177 |
printf("%d:%s:%s:%s:%s:<DEFAULT>\n", FNR, arch, component, pkg, pkgsize) |
| 178 |
} else { |
| 179 |
printf("%d:%s-DEPTOOBIG\n", FNR, $0) |
| 180 |
} |
| 181 |
} |
| 182 |
} |
| 183 |
|
| 184 |
END { |
| 185 |
for (source in included) { |
| 186 |
if (size[source] > max_size) { |
| 187 |
printf("%d:aaaaaaaaaaaaaaaaaa:%s:%s-SRCTOOBIG:%d\n", indexnr[source], sourcecomp[source], source, size[source]) |
| 188 |
} else { |
| 189 |
num_versions = split(versions[source], vers_this, ":") |
| 190 |
for(i = 1; i <= num_versions; i++) { |
| 191 |
printf("%d:aaaaaaaaaaaaaaaaaa:%s:%s:%d:%s\n", indexnr[source], sourcecomp[source], source, sizes[source, vers_this[i]], vers_this[i]) |
| 192 |
} |
| 193 |
} |
| 194 |
} |
| 195 |
}' $ADIR/$CODENAME-source/apt-state/lists/*Sources $BDIR/DONE $list | sort -nk 1 -t : >> $BDIR/list.mid |
| 196 |
else |
| 197 |
# Use "<DEFAULT>" as a version for binaries - it will match up |
| 198 |
# later in make_disc_trees.pl |
| 199 |
awk -F : '{printf("%d:%s:%s:%s:%s:<DEFAULT>\n", FNR, $1, $2, $3, $4)}' $list | sort -nk 1 -t : >> $BDIR/list.mid |
| 200 |
fi |
| 201 |
echo "# Ending merge pass 1 at "`date` >> $BDIR/list.mid |
| 202 |
|
| 203 |
# We now have all the files stuck together, with line numbers |
| 204 |
# prepended to allow us to shuffle them properly. Remove any comments |
| 205 |
# logged from the awk code above, then sort and remove the line numbers |
| 206 |
# again |
| 207 |
grep -v -e ^# -e TOOBIG $BDIR/list.mid | awk -F : ' |
| 208 |
{ |
| 209 |
if (!($4 in add)) { |
| 210 |
pkgname[number_packages++] = $4 |
| 211 |
} |
| 212 |
gsub("aaaaaaaaaaaaaaaaaa", "source", $2) # Undo the source sorting hack |
| 213 |
gsub("main-installer", "main", $3) |
| 214 |
add[$4] = add[$4] $2 ":" $3 ":" $4 ":" $5 ":" $6 " " |
| 215 |
} |
| 216 |
END { |
| 217 |
for (i = 0; i < number_packages; i++) { |
| 218 |
gsub(" $", "", add[pkgname[i]]) |
| 219 |
gsub(" ", "\n", add[pkgname[i]]) |
| 220 |
print add[pkgname[i]] |
| 221 |
} |
| 222 |
}' > $OUT |
| 223 |
echo "# Ending merge pass 2 at "`date` >> $BDIR/list.mid |
| 224 |
|
| 225 |
awk -F : ' |
| 226 |
/SRCTOOBIG/ { print $3 } |
| 227 |
' $BDIR/list.mid > $BDIR/sourcetoobig |
| 228 |
|
| 229 |
awk -F : ' |
| 230 |
/DEPTOOBIG/ { print $3 } |
| 231 |
' $BDIR/list.mid > $BDIR/sourcetoobig-deps |
| 232 |
|
| 233 |
exit 0 |