Debian Codegraph @

update-cd

raw
1 #!/bin/bash
2 #
3 # (c) Steve McIntyre <[email protected]> 2001-2017
4 # GNU GPL v2
5 #
6 # Quick and dirty script to create update CDs for people to upgrade
7 # from an initial stable release (r0) to the latest point release level
8 #
9
10 set -e
11
12 # Configuration goes here.
13
14 # Where is your mirror?
15 MIRROR_NORM=/srv/cdbuilder.debian.org/src/ftp/debian
16
17 # Do you want non-free? 1 for yes, 0 for no
18 NONFREE=0
19 NONFREE_FIRMWARE=1
20
21 # What release version is this?
22 VER=12.1.0
23
24 # Is this an official CD?
25 OFFICIAL=Official
26 # OFFICIAL="Unofficial snapshot"
27
28 # Path to use with mkisofs/mkhybrid
29 MKISOFS="xorriso -as mkisofs -jigdo-template-compress bzip2 -r -checksum_algorithm_iso md5,sha256,sha512"
30
31 # Use JTE extensions to mkisofs to make jigdo files?
32 JTE=1
33
34 # The working directory to use. MUST be on the same partition as the mirror.
35 TDIR=/srv/cdbuilder.debian.org/src/deb-cd/tmp/bookworm-update/cd-work
36
37 # Building bookworm cd set ...
38 CODENAME=bookworm
39
40 # Path where the images will be written
41 OUT=/srv/cdbuilder.debian.org/src/deb-cd/tmp/bookworm-update/cd-out
42
43 # Location(s) for the snapshot(s)
44 SNAPSHOT='Debian=https://us.cdimage.debian.org/cdimage/snapshot/Debian/ --try-last'
45
46 JIGOO_CHECKSUM=md5
47
48 # Type of disc image, cd or dvd (lower case)
49 if [ "$1"x = ""x ] ; then
50 TYPE=cd
51 else
52 TYPE=$1
53 fi
54 case $TYPE in
55 cd|CD)
56 TYPE=cd
57 TYPEUP=CD;;
58 dvd|DVD)
59 TYPE=dvd
60 TYPEUP=DVD;;
61 *)
62 echo "Specify a disk type..."
63 exit 1;;
64 esac
65
66 # Location of the diff file to use to determine the changes. If you leave
67 # this blank, we'll try to determine the changes from the ChangeLog files,
68 # which is probably less accurate.
69 DIFF=/home/debian-cd/lists/bookworm/r0-r1.diff
70
71 if [ "$TYPE" = "cd" ] ; then
72 CDSIZE=644 # megabytes, leaving space for metadata
73 else
74 CDSIZE=4300 # megabytes, leaving space for metadata
75 fi
76
77 CLOG=dists/$CODENAME/ChangeLog
78 UPD=$TDIR/$CODENAME-update
79 DATE=`date +%Y%m%d`
80 BASEDIR=`pwd`
81 SECTS="main non-free non-free-firmware contrib"
82 VERBOSE=2
83 GRAB_CHECKSUMS=$BASEDIR/tools/grab_checksums
84 export VERBOSE BASEDIR SECTS
85
86 FIRSTVER=`echo $VER | cut -c 1-4`0
87 CAPCODE=`perl -e "print ucfirst("$CODENAME")"`
88
89 export FIRSTVER CAPCODE
90
91 if [ "$ARCHLIST"x = ""x ] ; then
92 ARCHLIST="arm64 armel armhf amd64 i386 loong64 mips mipsel mips64el ppc64el s390x source" # amd64 # -all dealt with specially
93 fi
94
95 export TDIR NONFREE NONFREE_FIRMWARE VER MIRROR CODENAME OUT BASEDIR
96
97 CDSIZE=$(($CDSIZE * 1024))
98
99 create_control=$BASEDIR/tools/create_control
100 set_mkisofs_opts=$BASEDIR/tools/set_mkisofs_opts
101 addfiles=$BASEDIR/tools/add_files
102
103 copy_file () {
104 ROOTDIR=$1
105 shift
106 FILE=$1
107 DIR=`dirname $FILE`
108 if [ ! -d $ROOTDIR/$DIR ] ; then
109 mkdir -p $ROOTDIR/$DIR
110 fi
111 if [ -e $MIRROR/$FILE ] ; then
112 cp -l $MIRROR/$FILE $ROOTDIR/$FILE
113 else
114 echo
115 echo "File $FILE not found!"
116 exit 1
117 fi
118 }
119
120 make_cd () {
121 CDNUM=$1
122 shift
123 THISNUM=$1
124 shift
125 THISARCH=$1
126 shift
127 SRCFILES=$1
128 shift
129 ARCHFILES=$1
130 shift
131 ALLFILES=$1
132 echo
133 echo " Creating Packages and Sources files for CD$CDNUM"
134 cd $UPD
135 $create_control CD$CDNUM $THISARCH
136 echo " Creating image for CD$CDNUM ($SRCFILES source files, $ARCHFILES $ARCH debs, $ALLFILES all debs)"
137 echo -n " "
138
139 BASENAME="debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM"
140 ISODIR=$OUT/$THISARCH/iso-$TYPE
141 JIGDODIR=$OUT/$THISARCH/jigdo-$TYPE
142 LISTDIR=$OUT/$THISARCH/list-$TYPE
143
144 if [ ! -d ${ISODIR} ] ; then
145 mkdir -p ${ISODIR}
146 fi
147 if [ $JTE = 1 ] && [ ! -d ${JIGDODIR} ] ; then
148 mkdir -p ${JIGDODIR}
149 fi
150 if [ ! -d ${LISTDIR} ] ; then
151 mkdir -p ${LISTDIR}
152 fi
153
154 if [ $JTE = 1 ] ; then
155 set +e
156 ${MKISOFS} -J -joliet-long -r -V "Debian $VER update $TYPEUP" -o \
157 ${ISODIR}/${BASENAME}.iso \
158 -jigdo-jigdo ${JIGDODIR}/${BASENAME}.jigdo \
159 -jigdo-template ${JIGDODIR}/${BASENAME}.template \
160 -jigdo-map Debian=$MIRROR/ \
161 -jigdo-force-md5 /pool/ \
162 -md5-list $UPD/checksum-check \
163 $UPD/CD$CDNUM
164 # $UPD/CD$CDNUM 2>&1 | grep "extents written"
165 if [ $? -ne 0 ] ; then
166 echo "${MKISOFS} failed, error $?"
167 exit 1
168 fi
169 set -e
170 $BASEDIR/tools/jigdo_cleanup \
171 ${JIGDODIR}/${BASENAME}.jigdo \
172 ${BASENAME}.iso \
173 $OUT ${BASENAME}.template \
174 "Debian ${DEBIAN_KERNEL} $VER - $OFFICIAL $THISARCH $FIRSTVER->$VER update $TYPEUP #$THISNUM"
175 echo $SNAPSHOT >> ${JIGDODIR}/${BASENAME}.jigdo
176
177 # Make sure we compres the jigdo file *before* we checksum it
178 pigz -9nm ${JIGDODIR}/${BASENAME}.jigdo
179 mv ${JIGDODIR}/${BASENAME}.jigdo.gz ${JIGDODIR}/${BASENAME}.jigdo
180
181 for SHA_SIZE in 256 512; do
182 SHA=`zcat ${JIGDODIR}/${BASENAME}.jigdo | \
183 awk "/Image Hex SHA${SHA_SIZE}Sum/ {print \\$5}"`
184 if [ "$SHA"x = ""x ] ; then
185 echo "SHA fail"
186 exit 1
187 fi
188 echo "$SHA ${BASENAME}.iso" >> ${JIGDODIR}/SHA${SHA_SIZE}SUMS.update
189 echo "$SHA ${BASENAME}.iso" >> ${ISODIR}/SHA${SHA_SIZE}SUMS.update
190 (cd ${JIGDODIR} && sha${SHA_SIZE}sum ${BASENAME}.jigdo ${BASENAME}.template >> SHA${SHA_SIZE}SUMS.update)
191 done
192
193 # Make sure that the ISO is as new/newer than the jigdo file; #587774
194 touch ${ISODIR}/${BASENAME}.iso
195 else
196 set +e
197 ${MKISOFS} -J -joliet-long -r -V "Debian $VER update $TYPEUP" -o \
198 $OUT/iso-$TYPE/${BASENAME}.iso \
199 $UPD/CD$CDNUM 2>&1 | grep "extents written"
200 if [ $? -ne 0 ] ; then
201 echo "${MKISOFS} failed, error $?"
202 exit 1
203 fi
204 set -e
205 fi
206 find $UPD/CD$CDNUM/pool -type f | sed 's?^.*/??g' | pigz -9nm > ${LISTDIR}/${BASENAME}.list.gz
207 }
208
209 add_file () {
210 file=$1
211 case "$THISARCH" in
212 source)
213 case "$file" in
214 *.xz|*.lzma|*.gz|*.bz2|*.dsc)
215 copy_file $UPD/CD$CDNUM $file
216 SRCFILES=$(($SRCFILES + 1))
217 ;;
218 esac
219 echo -en "\r$SRCFILES copied"
220 ;;
221 *)
222 case "$file" in
223 *_$THISARCH.deb)
224 copy_file $UPD/CD$CDNUM $file
225 ARCHFILES=$(($ARCHFILES + 1))
226 ;;
227 *_all.deb)
228 copy_file $UPD/CD$CDNUM $file
229 ALLFILES=$(($ALLFILES + 1))
230 ;;
231 esac
232 echo -en "\r$THISARCH:$ARCHFILES all:$ALLFILES copied"
233 ;;
234 esac
235 }
236
237 start_new_disc () {
238 echo
239 cd $MIRROR
240 THISNUM=$((THISNUM + 1))
241 if [ $THISARCH = "source" ] ; then
242 echo "Creating $TYPEUP$CDNUM for source (part $THISNUM)"
243 else
244 echo "Creating $TYPEUP$CDNUM for binary-$THISARCH (part $THISNUM)"
245 fi
246 mkdir $UPD/CD$CDNUM $UPD/CD$CDNUM/.disk
247 INFO="Debian ${DEBIAN_KERNEL} $VER Update $TYPEUP $DATE: $THISARCH $TYPEUP $THISNUM"
248 echo $INFO > $UPD/CD$CDNUM/.disk/info
249 SIZE_USED=0
250 SRCFILES=0
251 ARCHFILES=0
252 ALLFILES=0
253 }
254
255 echo Cleaning up
256 rm -rf $UPD
257 mkdir -p $UPD
258 CDNUM=1
259
260 echo Creating file list
261
262 if [ ! -e $DIFF ] ; then
263 echo "NO DIFF FOUND. ABORT!"
264 exit 1
265 fi
266
267 cp $DIFF $UPD/list
268 if [ "$NONFREE"x != "1"x ] ; then
269 echo "Removing non-free files from the list"
270 grep -v non-free/ $UPD/list > $UPD/list1
271 mv -f $UPD/list1 $UPD/list
272 fi
273 if [ "$NONFREE_FIRMWARE"x != "1"x ] ; then
274 echo "Removing non-free-firmware files from the list"
275 grep -v non-free-firmware/ $UPD/list > $UPD/list1
276 mv -f $UPD/list1 $UPD/list
277 fi
278
279 $GRAB_CHECKSUMS $JIGOO_CHECKSUM $MIRROR_NORM "$ARCHLIST all" $CODENAME $CODENAME $UPD/checksum-check
280
281 for THISARCH in $ARCHLIST
282 do
283 THISNUM=0
284
285 MIRROR=$MIRROR_NORM
286
287 case "$THISARCH" in
288 kfreebsd) DEBIAN_KERNEL="GNU/kFreeBSD";;
289 hurd) DEBIAN_KERNEL="GNU/Hurd";;
290 source) DEBIAN_KERNEL="";;
291 *) DEBIAN_KERNEL="GNU/Linux";;
292 esac
293 export DEBIAN_KERNEL
294
295 rm -rf $OUT/$THISARCH/*-$TYPE
296
297 for file in `cat $UPD/list`
298 do
299 if [ ! -d $UPD/CD$CDNUM ] ; then
300 start_new_disc
301 fi
302
303 if [ "$BACKOUT_FILE"x != ""x ] ; then
304 echo "Starting with backed-out file $BACKOUT_FILE"
305 add_file $BACKOUT_FILE
306 BACKOUT_FILE=""
307 fi
308
309 size=$(stat -c %s $MIRROR/$file)
310 size_kib=$(($size / 1024))
311 if [ $size_kib -gt $CDSIZE ] ; then
312 echo "Ignoring file $file - it won't fit!"
313 else
314 add_file $file
315
316 SIZE_USED=`du -sk $UPD/CD$CDNUM | awk '{print $1}'`
317 if [ $SIZE_USED -gt $CDSIZE ] ; then
318 # Back out the last file added
319 size=`stat -c %s $UPD/CD$CDNUM/$file`
320 rm -f $UPD/CD$CDNUM/$file
321 echo
322 echo " Overfull: back out $file ($size bytes)"
323 DIR=`dirname $file`
324 rmdir $DIR > /dev/null 2>&1 || true
325 BACKOUT_FILE=$file
326 make_cd $CDNUM $THISNUM $THISARCH $SRCFILES $ARCHFILES $ALLFILES
327 CDNUM=$(($CDNUM + 1))
328 continue
329 fi
330 fi
331 done
332
333 echo
334 echo "Making last disc in set for $THISARCH: size $SIZE_USED, BACKOUT_FILE \"$BACKOUT_FILE\""
335 # Catch the case where we still have a backout file at the very
336 # end of the set
337 if [ "$BACKOUT_FILE"x != ""x ] ; then
338 if [ ! -d $UPD/CD$CDNUM ] ; then
339 start_new_disc
340 fi
341 echo "Starting last disc with backed-out file $BACKOUT_FILE"
342 add_file $BACKOUT_FILE
343 BACKOUT_FILE=""
344 fi
345
346 make_cd $CDNUM $THISNUM $THISARCH $SRCFILES $ARCHFILES $ALLFILES
347 CDNUM=$(($CDNUM + 1))
348 done
349