Debian Codegraph @

tools/merge_package_lists

raw
1 #!/bin/sh
2 #
3 # merge_package_lists
4 #
5 # Quick, hacky script to merge the package lists for multiple arches
6 # together, with 2 priorities:
7 #
8 # 1. keep the ordering of the original files wherever possible
9 # 2. collect together instances of the same package on multiple arches
10 #
11 # expects multiple files containing lines of the form, one arch per file
12 #
13 # <arch>:<component>:<pkgname>
14 #
15 # and will output similar
16 #
17 # The files MUST be passed by filename rather than simply piped in,
18 # otherwise the ordering constraint will be lost
19 #
20 # Things also get quite complicated here due to considering
21 # source. Depending on how things are being run:
22 #
23 # 1. If we're not doing source, life is easy - simply interleave the
24 # binary packages
25 # 2. If we're *only* doing source, then we won't be called at all -
26 # the Makefile will just dump the sources directly into "packages"
27 # without needing our help
28 # 3. If we're doing mixed source/binary discs, then we have to work
29 # quite hard here. We generate the list of sources from the
30 # binaries given to us. For each binary, we will attempt to place
31 # its source *first* onto the disc so we can make nice GPL-compliant
32 # media. However, we also need to check whether or not that source
33 # has already been placed due to an *earlier* binary package (a
34 # different binary built from the same source, or the same for
35 # another arch). To allow us to so that, we need to parse the Sources
36 # files first to build up the src<->bin relationship. Ick... :-/
37 #
38 # An added bit of grot: to make sure that sources are placed first, we
39 # *temporarily* call the source architecture "aaaaaaaaaaaaaaaaaa" to force
40 # it to sort first, then replace it woth "source" again later. *grin*
41 #
42 # Steve McIntyre <[email protected]> 2006, GPL v2
43
44 BDIR=$1
45 ADIR=$2
46 ARCHES="$3"
47 OUT="$4"
48
49 case "$ARCHES" in
50 *source*) source=yes;;
51 *) source=no;;
52 esac
53
54 for ARCH in $ARCHES_NOSRC
55 do
56 list="$list $BDIR/packages.$ARCH"
57 done
58
59 echo "# Starting merge at "`date` >> $BDIR/list.mid
60 if [ $source = "yes" ] ; then
61 echo "DONEDONE" > $BDIR/DONE
62 awk -v max_size=$MAX_PKG_SIZE '
63
64 # First parse the Sources files
65 /^Package:/ {
66 srcname=$2
67 srcs_done++
68 }
69
70 /^Version:/ {
71 version=$2
72 # Cannot cope with : in output
73 gsub(":", "%", version)
74 if (srcname in versions) {
75 versions[srcname]=sprintf("%s:%s", versions[srcname], version)
76 } else {
77 versions[srcname] = version
78 }
79 next
80 }
81
82 /^Binary:/ {
83 bins=$0
84 gsub("^Binary:","",bins)
85 gsub(",","",bins)
86 nb=split(bins, binarray)
87 for (i=1; i <= nb; i++) {
88 bin2src[binarray[i]] = srcname
89 bins_done++
90 }
91 in_binary = 1
92 in_files = 0
93 next
94 }
95
96 /^Files:/ {
97 in_files = 1
98 in_binary = 0
99 next
100 }
101
102 /^ / {
103 if (in_files) { # Cope with multiple lines of files for source packages
104 size[srcname]+=$2
105 sizes[srcname, version]+=$2
106 next
107 }
108 if (in_binary) { # Cope with multi-line Binary: fields
109 bins=$0
110 gsub(",","",bins)
111 nb=split(bins, binarray)
112 for (i=1; i <= nb; i++) {
113 bin2src[binarray[i]] = srcname
114 bins_done++
115 }
116 next
117 }
118 }
119
120 # Done reading the Sources
121 /^DONEDONE/ {
122 parsed=1
123 printf("# Parsed Sources files: %d sources producing %d binaries\n", srcs_done, bins_done)
124 next
125 }
126
127 # Now start placing source packages, depending on the order of the binary packages
128 /^./ {
129 in_files = 0
130 in_binary = 0
131 if (parsed) {
132 num_sources = 0
133 split($0,fields,":")
134 arch=fields[1]
135 component=fields[2]
136 pkg=fields[3]
137 pkgsize=fields[4]
138 srcpkg=bin2src[pkg]
139 built_using=fields[5]
140 if ("" != built_using) {
141 num_sources = split(built_using, sources, ",")
142 }
143 if ("" == srcpkg) {
144 if ("main-installer" == component) {
145 printf("# Ignoring source for udeb %s\n", pkg)
146 } else {
147 printf("# Source for pkg %s is UNKNOWN!\n", pkg)
148 }
149 } else {
150 sources[++num_sources] = srcpkg
151 }
152 if (num_sources) {
153 for(i = 1; i <= num_sources; i++) {
154 source = sources[i]
155 if (!included[source]) {
156 printf("# Adding source %s at %d because of %s:%s (%s)\n",
157 source, FNR, arch, pkg, (i == num_sources ? "dep" : "b-u"))
158 included[source] = pkg
159 incarch[source] = arch
160 indexnr[source] = FNR
161 sourcecomp[source] = component
162 } else {
163 if (FNR < indexnr[source]) {
164 printf("# Updating source %s: was due to %d:%s:%s, now moved earlier because of %d:%s:%s (%s)\n",
165 source, indexnr[source],
166 incarch[source], included[source],
167 FNR, arch, pkg, (i == num_sources ? "dep" : "b-u"))
168 included[source] = pkg
169 incarch[source] = arch
170 indexnr[source] = FNR
171 }
172 }
173 }
174 }
175 if (size[source] <= max_size) {
176 # List the binary
177 printf("%d:%s:%s:%s:%s:<DEFAULT>\n", FNR, arch, component, pkg, pkgsize)
178 } else {
179 printf("%d:%s-DEPTOOBIG\n", FNR, $0)
180 }
181 }
182 }
183
184 END {
185 for (source in included) {
186 if (size[source] > max_size) {
187 printf("%d:aaaaaaaaaaaaaaaaaa:%s:%s-SRCTOOBIG:%d\n", indexnr[source], sourcecomp[source], source, size[source])
188 } else {
189 num_versions = split(versions[source], vers_this, ":")
190 for(i = 1; i <= num_versions; i++) {
191 printf("%d:aaaaaaaaaaaaaaaaaa:%s:%s:%d:%s\n", indexnr[source], sourcecomp[source], source, sizes[source, vers_this[i]], vers_this[i])
192 }
193 }
194 }
195 }' $ADIR/$CODENAME-source/apt-state/lists/*Sources $BDIR/DONE $list | sort -nk 1 -t : >> $BDIR/list.mid
196 else
197 # Use "<DEFAULT>" as a version for binaries - it will match up
198 # later in make_disc_trees.pl
199 awk -F : '{printf("%d:%s:%s:%s:%s:<DEFAULT>\n", FNR, $1, $2, $3, $4)}' $list | sort -nk 1 -t : >> $BDIR/list.mid
200 fi
201 echo "# Ending merge pass 1 at "`date` >> $BDIR/list.mid
202
203 # We now have all the files stuck together, with line numbers
204 # prepended to allow us to shuffle them properly. Remove any comments
205 # logged from the awk code above, then sort and remove the line numbers
206 # again
207 grep -v -e ^# -e TOOBIG $BDIR/list.mid | awk -F : '
208 {
209 if (!($4 in add)) {
210 pkgname[number_packages++] = $4
211 }
212 gsub("aaaaaaaaaaaaaaaaaa", "source", $2) # Undo the source sorting hack
213 gsub("main-installer", "main", $3)
214 add[$4] = add[$4] $2 ":" $3 ":" $4 ":" $5 ":" $6 " "
215 }
216 END {
217 for (i = 0; i < number_packages; i++) {
218 gsub(" $", "", add[pkgname[i]])
219 gsub(" ", "\n", add[pkgname[i]])
220 print add[pkgname[i]]
221 }
222 }' > $OUT
223 echo "# Ending merge pass 2 at "`date` >> $BDIR/list.mid
224
225 awk -F : '
226 /SRCTOOBIG/ { print $3 }
227 ' $BDIR/list.mid > $BDIR/sourcetoobig
228
229 awk -F : '
230 /DEPTOOBIG/ { print $3 }
231 ' $BDIR/list.mid > $BDIR/sourcetoobig-deps
232
233 exit 0