Debian Codegraph @

debdeltas

raw
1 #!/usr/bin/python3
2
3 # Copyright (C) 2006-2020 Andrea Mennucci.
4 # License: GNU Library General Public License, version 2 or later
5
6 EMAIL="[email protected]"
7
8 #### messages and translations
9 # Early errors (such as errors in args parsing) are printed on stderr.
10 # Subsequent errors,warnings,messages are printed on stdout,using the 'logging' facility
11 # but the progress bars are on stderr
12
13 # Messages printed at verbosity 0 or 1 are translated, higher verbosities are not
14 # Errors are sometimes translated, sometimes not...
15 # obscure error messages that would be printed only in very rare cases are not translated
16 # e.g.: a malformed http header, a gnupg unexptected error, damaged input files...
17 # more common error messages are translated,
18 # e.g.: out of disk space while using debdelta-upgrade , file does not exist...
19
20 import sys
21
22 try:
23 import gettext
24 gettext.bindtextdomain('debdelta','/usr/share/locale')
25 gettext.textdomain('debdelta')
26 _ = gettext.gettext
27 except Exception:
28 a=sys.exc_info[1]
29 sys.stderr.write('Could not initialize "gettext", translations will be unavailable\n'+str(a))
30 def _(x): return x
31
32 doc={}
33 doc['delta']=_("""\
34 Usage: debdelta [ option... ] fromfile tofile delta
35 Computes the difference of two deb files, from fromfile to tofile, and writes it to delta
36
37 Options:
38 --signing-key KEY
39 gnupg key used to sign the delta
40 --no-md5 do not include MD5 info in delta
41 --needsold create a delta that can only be used if the old deb is available
42 -M Mb maximum memory to use (for 'bsdiff' or 'xdelta')
43 --delta-algo ALGO
44 use a specific backend for computing binary diffs
45 """)
46
47
48 doc['deltas']=_("""\
49 Usage: debdeltas [ option... ] [deb files and dirs, or 'Packages' files]
50 Computes all missing deltas for deb files.
51 It orders by version number and produce deltas to the newest version
52
53 Options:
54 --signing-key KEY
55 key used to sign the deltas (using GnuPG)
56 --dir DIR force saving of deltas in this DIR
57 (otherwise they go in the dir of the newer deb_file)
58 --old ARGS 'Packages' files containing list of old versions of debs
59 --alt ARGS for any cmdline argument, search for debs also in this place
60 -n N how many deltas to produce for each deb (default unlimited)
61 --no-md5 do not include MD5 info in delta
62 --needsold create a delta that can only be used if the old .deb is available
63 --delta-algo ALGO
64 use a specific backend for computing binary diffs;
65 possible values are: xdelta xdelta-bzip xdelta3 bsdiff
66 -M Mb maximum memory to use (for 'bsdiff' or 'xdelta')
67 --clean-deltas delete deltas if newer deb is not in archive
68 --cache cache parsed version of Packages.bz2 as Packages.debdelta_cache
69 """)
70
71 ## implement : --search search in the directory of the above debs for older versions
72
73 doc['patch']=_("""\
74 Usage: debpatch [ option... ] delta fromfile tofile
75 Applies delta to fromfile and produces a reconstructed version of tofile.
76
77 (When using 'debpatch' and the old .deb is not available,
78 use the unpack directory, usually '/', for the fromfile.)
79
80 Usage: debpatch --info delta
81 Write info on delta.
82
83 Options:
84 --no-md5 do not verify MD5 (if found in info in delta)
85 -A accept unsigned deltas
86 --format FORMAT
87 format of created deb
88 """)
89
90 doc['delta-upgrade']=_("""\
91 Usage: debdelta-upgrade [package names]
92 Downloads all deltas and apply them to create the debs
93 that are needed by 'apt-get upgrade'.
94
95 Options:
96 --dir DIR directory where to save results
97 --deb-policy POLICY
98 policy to decide which debs to download,
99 -A accept unsigned deltas
100 --format FORMAT
101 format of created debs
102 --timeout SECONDS
103 adjust timeout for connections, default is
104 15 seconds
105 """)
106
107 doc['patch-url']=_("""\
108 Usage: debpatch-url [package names]
109 Show URL wherefrom to downloads all deltas that may be used to upgrade the given package names
110 """)
111
112 doc_common=_("""\
113 -v verbose (can be added multiple times)
114 --no-act do not do that (whatever it is!)
115 -d add extra debugging checks
116 -k keep temporary files (use for debugging)
117 --gpg-home HOME
118 specify a different home for GPG
119
120 See man page for more options and details.
121 """)
122
123 minigzip='/usr/lib/debdelta/minigzip'
124 minibzip2='/usr/lib/debdelta/minibzip2'
125
126
127 ####################################################################
128
129 import os , tempfile , getopt , tarfile , shutil , time, traceback, subprocess, time, tarfile, stat, hashlib, random, gzip
130
131
132
133 try:
134 import debian.deb822
135 debian_deb822 = debian.deb822
136 except ImportError:
137 debian_deb822 = None
138
139 try:
140 import cPickle as pickle
141 except ImportError:
142 import pickle
143
144 from stat import ST_SIZE, ST_MTIME, ST_MODE, S_IMODE, S_IRUSR, S_IWUSR, S_IXUSR
145 from os.path import abspath, expanduser
146 from copy import copy
147
148 from types import FunctionType, LambdaType
149
150 import logging, logging.handlers
151 if __name__ == '__main__':
152 logging.basicConfig(format='%(message)s', stream=sys.stdout, level=logging.INFO)
153 logger=logging.getLogger(__name__)
154
155 if sys.version_info.major == 2:
156 string_types = (str, unicode) # python2
157 # flags for subprocess.Popen
158 Popen_args = {}
159 #
160 from ConfigParser import SafeConfigParser as ConfigParser
161 from tarfile import filemode as stat_filemode
162 #
163 def str2bytes(a):
164 return a
165 def bytes2str(b):
166 return b
167 def is_bytes(b):
168 return False
169 # see https://docs.python.org/2/library/tarfile.html#unicode-issues
170 if sys.getfilesystemencoding() != 'UTF-8':
171 logger.warning('The filesystem encoding is %r, but this program assumes `UTF-8`.'%\
172 (sys.getfilesystemencoding(),))
173 else:
174 string_types = (str, bytes) # python3
175 intern = sys.intern
176 # flags for subprocess.Popen to open stdin,stdout as text streams
177 Popen_args = {'text':True}
178 #
179 from configparser import ConfigParser
180 from stat import filemode as stat_filemode
181 #
182 long = int
183 def str2bytes(a):
184 return a.encode('UTF-8',errors='surrogateescape')
185 def bytes2str(b):
186 return b.decode('UTF-8',errors='surrogateescape')
187 def is_bytes(b):
188 return isinstance(b,bytes)
189 # see https://docs.python.org/3/library/tarfile.html#unicode-issues
190 if sys.getfilesystemencoding() != 'utf-8':
191 logger.warning('The filesystem encoding is %r, but this program assumes `utf-8`.'%\
192 (sys.getfilesystemencoding(),))
193 elif hasattr(sys,'getfilesystemencodeerrors') and \
194 sys.getfilesystemencodeerrors() != 'surrogateescape':
195 logger.warning('The filesystem encoding is `utf-8` with %r, but this program uses `surrogateescape`.'%\
196 (sys.getfilesystemencodeerrors(),))
197
198 def get_termsize():
199 import termios, fcntl, struct
200 s = struct.pack("HHHH", 0, 0, 0, 0)
201 fd_stdout = sys.stdout.fileno()
202 x = fcntl.ioctl(fd_stdout, termios.TIOCGWINSZ, s)
203 return struct.unpack("HHHH", x)[:2]
204
205 try:
206 (terminalrows , terminalcolumns) = get_termsize()
207 except:
208 (terminalrows , terminalcolumns) = (None, None) #(24, 80)
209
210 ################################################# main program, read options
211
212 #target of: maximum memory that bsdiff will use
213 MAXMEMORY = 1024 * 1024 * 50
214
215 #this is +-10% , depending on the package size
216 MAX_DELTA_PERCENT = 70
217
218 #min size of .deb that debdelta will consider
219 #very small packages cannot be effectively delta-ed
220 MIN_DEB_SIZE = 10 * 1024
221
222
223 N_DELTAS=None
224
225 USE_DELTA_ALGO = 'bsdiff'
226
227 TIMEOUT = 15
228 DEBUG = 0
229 VERBOSE = 0
230 KEEP = False
231 INFO = False
232 NEEDSOLD= False
233 DIR = None
234 ALT = []
235 OLD = []
236 ACT = True
237 DO_MD5 = True
238 DEB_POLICY = ['b','s','e','t','f','q']
239 DO_PROGRESS = terminalcolumns != None and __name__ == '__main__'
240
241 #where/how debpatch/debdelta-upgrade will send forensic data, when patching fails
242 #possible values:
243 # False : do not send them
244 # do : compute forensic but not send them, just list them
245 # mail : automatically send by email to default address (uses `mutt`)
246 # mail:user@domain : automatically send by email to address (uses `mutt`)
247 # mutt : as above
248 # mutt:user@domain : as above
249 # http://domain/cgi : send them automatically thru a CGI script
250 #Warning: the above is mostly TODO
251 FORENSIC='do'
252
253 FORENSIC_EMAIL='[email protected]'
254
255 #directory tree where forensic info are stored by 'debdeltas'
256 FORENSICDIR=None
257
258 DEB_FORMAT='deb'
259 DEB_FORMAT_LIST=('deb','unzipped','preunpacked') #not yet implemented on patching side : (,'piped')
260
261 #for debdeltas: test patches internally
262 DO_TEST = False
263
264 DO_GPG = True #this is changed a few lines below
265 GPG_SIGNING_KEY = None
266 if os.getuid() == 0:
267 GPG_HOME="/etc/debdelta/gnupg"
268 else:
269 GPG_HOME=None
270 GPG_MASTER_PUB_KEYRING="/usr/share/keyrings/debian-debdelta-archive-keyring.gpg"
271
272 GPG_CMD='gpg'
273 if os.path.exists('/usr/bin/gpg2'):
274 GPG_CMD='/usr/bin/gpg2'
275
276 CLEAN_DELTAS = False
277 CLEAN_DELTAS_MTIME = 2 # two days grace period
278 CLEAN_ALT = False
279
280 DO_PREDICTOR = False
281
282 DO_CACHE = False #cache parsed version of Packages.bz2 as Packages.debdelta_cache
283
284 #see README.features
285 DISABLEABLE_FEATURES=['xz', 'lzma', 'xdelta3-fifo']
286 DISABLED_FEATURES=[]
287
288 HTTP_USER_AGENT={'User-Agent': ('Debian debdelta-upgrade' ) }
289
290
291 DPKG_MULTIARCH=( 0 == os.system('dpkg --assert-multi-arch 2> /dev/null') )
292
293
294 if __name__ != "__main__":
295 action = None
296 elif os.path.dirname(sys.argv[0]) == '/usr/lib/apt/methods' :
297 action = None
298 else:
299 action=(os.path.basename(sys.argv[0]))[3:]
300 actions = ('delta','patch','deltas','delta-upgrade', 'patch-url')
301 #
302 if action not in actions:
303 sys.stderr.write('wrong filename: should be "deb" + '+repr(actions))
304 raise SystemExit(4)
305
306 __doc__ = doc[action] + doc_common
307
308
309
310 #GPG signatures are required for debdelta-upgrade and debpatch
311 DO_GPG = action in ( "delta-upgrade", "patch")
312
313 try:
314 ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:n:A' ,
315 ('help','info','needsold','dir=','no-act','alt=','old=','delta-algo=',
316 'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug','forensicdir=','forensic=',
317 'signing-key=', "accept-unsigned", "gpg-home=", "disable-feature=", "test", "format=", "cache", "timeout=") )
318 except getopt.GetoptError:
319 a=sys.exc_info()[1]
320 sys.stderr.write(sys.argv[0] +': '+ str(a)+'\n')
321 raise SystemExit(3)
322
323 for o , v in opts :
324 if o == '-v' :
325 VERBOSE += 1
326 if VERBOSE > 1 :
327 logger.setLevel(logging.DEBUG)
328 elif o == '-d' or o == '--debug' :
329 DEBUG += 1
330 logger.setLevel(logging.DEBUG)
331 elif o == '-k' : KEEP = True
332 elif o == '--no-act': ACT=False
333 elif o == '--no-md5': DO_MD5=False
334 elif o == '--clean-deltas' : CLEAN_DELTAS = True
335 elif o == '--clean-alt' :
336 CLEAN_ALT = True
337 sys.stderr.write(_('Warning, currently --clean-alt does nothing.')+'\n')
338 elif o == '--needsold' : NEEDSOLD = True
339 elif o == '--delta-algo': USE_DELTA_ALGO=v
340 elif o == '--max-percent': MAX_DELTA_PERCENT=int(v)
341 elif o == '--deb-policy' : DEB_POLICY = [j[0] for j in v.split(',') if j]
342 elif o == '--timeout' :
343 if int(v) < 0:
344 sys.stderr.write(_('Error: "--timeout %s" is too small.') % v + '\n')
345 raise SystemExit(3)
346 TIMEOUT = int(v)
347 elif o == '-M' :
348 if int(v) <= 1:
349 sys.stderr.write(_('Error: "-M %s" is too small.') % v + '\n')
350 raise SystemExit(3)
351 if int(v) <= 12:
352 sys.stderr.write(_('Warning: "-M %s" is quite small.') % v + '\n')
353 MAXMEMORY = 1024 * 1024 * int(v)
354 elif o == '-n' :
355 N_DELTAS = int(v)
356 if N_DELTAS < 0:
357 sys.stderr.write(_('Error: -n value is negative.')+'\n')
358 raise SystemExit(3)
359 elif o == '--test' and action == 'deltas' : DO_TEST = True
360 elif o == '--info' and action == 'patch' : INFO = True
361 elif o == '--dir' :
362 DIR = abspath(expanduser(v))
363 if v[-2:] == '//':
364 DIR += '//'
365 if not os.path.isdir(DIR):
366 sys.stderr.write( _("Error: argument of --dir is not a directory:") +' '+ DIR +'\n')
367 raise SystemExit(3)
368
369 elif o == '--forensicdir' :
370 FORENSICDIR = abspath(expanduser(v))
371 if v[-2:] == '//':
372 FORENSICDIR += '//'
373 if not os.path.isdir(FORENSICDIR):
374 sys.stderr.write( _("Error: argument of --forensicdir is not a directory:") +' '+ FORENSICDIR +'\n')
375 raise SystemExit(3)
376
377 elif o == '--forensic' :
378 FORENSIC = v
379 if FORENSIC.startswith('http'):
380 try:
381 import poster
382 except:
383 sys.stderr.write(_('To use the http forensic, you must install the package "python-poster".')+'\n')
384 raise SystemExit(3)
385 if FORENSIC[:4] in ('mutt','mail') and not os.path.exists('/usr/bin/mutt'):
386 sys.stderr.write(_('To use this forensic, you must install the package "mutt".')+'\n')
387 raise SystemExit(3)
388 elif o == '--alt' :
389 if not (os.path.isfile(v) or os.path.isdir(v)) :
390 sys.stderr.write(_('Error: argument of --alt is not a directory or a regular file:')+' '+v +'\n')
391 raise SystemExit(3)
392 ALT.append(v)
393 elif o == '--old' and action == 'deltas' :
394 if not (os.path.isfile(v) or os.path.isdir(v)) :
395 sys.stderr.write(_("Error: argument of --old is not a directory or a regular file:")+' '+v +'\n')
396 raise SystemExit(3)
397 OLD.append(v)
398 elif o == '--help' or o == '-h':
399 sys.stderr.write(__doc__)
400 raise SystemExit(0)
401 elif (o == '--disable-feature') and action in ("delta", "deltas"):
402 DISABLED_FEATURES += v.split(',')
403 elif (o == '--signing-key') and action in ("delta", "deltas"):
404 GPG_SIGNING_KEY=v
405 DO_GPG=True
406 elif (o == '--accept-unsigned' or o == '-A') and action in ("delta-upgrade", "patch"):
407 DO_GPG=False
408 elif (o == '--gpg-home'):
409 GPG_HOME=abspath(expanduser(v))
410 if not os.path.isdir(GPG_HOME):
411 sys.stderr.write(_("Error: --gpg-home `%s' does not exist.") % GPG_HOME)
412 raise SystemExit(3)
413 elif o == '--format' : # maybe, and action in ("delta-upgrade", "patch"):
414 if v not in DEB_FORMAT_LIST:
415 sys.stderr.write(_("Error: output format `%s' is unknown.") % v + '\n')
416 raise SystemExit(3)
417 DEB_FORMAT=v
418 elif o == '--cache': DO_CACHE=True
419 else:
420 sys.stderr.write(_("Error: option `%s' is unknown, try --help") % o + '\n')
421 raise SystemExit(3)
422
423 for i in DISABLED_FEATURES:
424 if i not in DISABLEABLE_FEATURES:
425 sys.stderr.write(_("Error: feature `%s' cannot be disabled.") % i + '\n')
426 raise SystemExit(3)
427
428 try:
429 a = subprocess.Popen(b'grep bogomips /proc/cpuinfo',shell=True, stdout=subprocess.PIPE).stdout.read()
430 a = bytes2str(a)
431 BOGOMIPS=float(a.split(':')[-1])
432 except:
433 if VERBOSE:
434 logger.exception(' Warning, /proc not mounted, using bogus BOGOMIPS\n')
435 BOGOMIPS=3000.0
436
437 SP=subprocess.Popen(['hostname','-f'], shell=False, stdout=subprocess.PIPE)
438 HOSTID=hashlib.md5( SP.stdout.read() ).hexdigest()
439 SP.wait()
440 del SP
441
442 TMPDIR = ( os.getenv('TMPDIR') or '/tmp' ).rstrip('/')
443
444 if KEEP:
445 def unlink(a):
446 if VERBOSE > 2: logger.debug(' -k: would unlink '+repr(a))
447 def rmdir(a):
448 if VERBOSE > 2: logger.debug(' -k: would rmdir '+repr(a))
449 def rmtree(a):
450 if VERBOSE > 2: logger.debug(' -k: would rm -r '+repr(a))
451 else:
452 def __wrap__(a,cmd):
453 c=cmd.__name__+"("+a+")"
454 if a[ : len(TMPDIR)+9 ] != TMPDIR+'/debdelta' :
455 raise DebDeltaError('Internal error! refuse to '+c)
456 try:
457 cmd(a)
458 except OSError:
459 logger.exception(' Warning! when trying to '+repr(c)+' got OSError ')
460 raise
461
462 def unlink(a):
463 return __wrap__(a,os.unlink)
464 def rmdir(a):
465 return __wrap__(a,os.rmdir)
466 def rmtree(a):
467 return __wrap__(a,shutil.rmtree)
468
469 ################################################################## check debian.deb822
470
471 _fake_Packages="""Package: coreutils
472 Version: 8.30-3
473 Essential: yes
474 Installed-Size: 15719
475 Maintainer: Michael Stone <[email protected]>
476 Architecture: amd64
477 Pre-Depends: libacl1 (>= 2.2.23), libattr1 (>= 1:2.4.44), libc6 (>= 2.28), libselinux1 (>= 2.1.13)
478 Description: GNU core utilities
479 Multi-Arch: foreign
480 Homepage: http://gnu.org/software/coreutils
481 Description-md5: d0d975dec3625409d24be1238cede238
482 Tag: admin::configuring, implemented-in::c, interface::commandline,
483 role::program, scope::utility, suite::gnu, works-with::file
484 Section: utils
485 Priority: required
486 Filename: pool/main/c/coreutils/coreutils_8.30-3_amd64.deb
487 Size: 2707932
488 MD5sum: 4a658a39c2810530df6642d22f65ddd1
489 SHA256: ae6e5cd6e9aaf74d66edded3931a7a6c916625b8b890379189c75574f6856bf4
490
491 Package: bash
492 Version: 5.0-4
493 Essential: yes
494 Installed-Size: 6439
495 Maintainer: Matthias Klose <[email protected]>
496 Architecture: amd64
497 Replaces: bash-completion (<< 20060301-0), bash-doc (<= 2.05-1)
498 Depends: base-files (>= 2.1.12), debianutils (>= 2.15)
499 Pre-Depends: libc6 (>= 2.15), libtinfo6 (>= 6)
500 Recommends: bash-completion (>= 20060301-0)
501 Suggests: bash-doc
502 Conflicts: bash-completion (<< 20060301-0)
503 Description: GNU Bourne Again SHell
504 Multi-Arch: foreign
505 Homepage: http://tiswww.case.edu/php/chet/bash/bashtop.html
506 Description-md5: 3522aa7b4374048d6450e348a5bb45d9
507 Tag: admin::TODO, devel::TODO, devel::interpreter, implemented-in::c,
508 interface::shell, interface::text-mode, role::program,
509 scope::application, suite::gnu, uitoolkit::ncurses
510 Section: shells
511 Priority: required
512 Filename: pool/main/b/bash/bash_5.0-4_amd64.deb
513 Size: 1377592
514 MD5sum: 6f118d96e29a5208da9033bf0b2c772c
515 SHA256: d07f3528c615545bb182269362926bc171adab565dade3e8c35cb3bae8ad9e5e
516
517 """
518
519 def test_deb822(debian_deb822, shared_storage):
520 F = tempfile.NamedTemporaryFile()
521 open(F.name,'w').write(_fake_Packages)
522 I=debian_deb822.Packages.iter_paragraphs(sequence=open(F.name,'rb'),shared_storage=shared_storage)
523 c=next(I)
524 assert c['Package'] == 'coreutils'
525 assert c['Size'] == '2707932'
526 b=next(I)
527 assert b['Package'] == 'bash'
528 assert c['Size'] == '2707932'
529 b['foo']='bar'
530 c['foo']='tock'
531 assert b['foo']=='bar'
532 # Deb822 only accepts strings
533 #b['foo']=(1,None)
534
535
536 if debian_deb822 is not None:
537 try:
538 test_deb822(debian_deb822,False)
539 test_deb822(debian_deb822,True)
540 except:
541 logger.exception("While testing debian.822")
542
543 #################################################### various routines
544
545 def my_popen_read(cmd, ** args):
546 " opens a pipe (for text mode, pass `**Popen_args` )"
547 return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=open(os.devnull),
548 close_fds=True, **args).stdout
549
550 def freespace(w):
551 assert(os.path.exists(w))
552 try:
553 a=os.statvfs(w)
554 freespace= long(a[0]) * long(a[4])
555 except Exception:
556 logger.exception('Statvfs error on %r', w)
557 freespace=None
558 return freespace
559
560 dpkg_keeps_controls = (
561 'conffiles','config','list','md5sums','postinst',
562 'postrm','preinst','prerm','shlibs','templates')
563
564 def parse_dist(f,d):
565 a=f.readline()
566 p={}
567 while a:
568 if a[:4] in ('Pack','Vers','Arch','Stat','Inst','File','Size','MD5s'):
569 a=de_n(a)
570 i=a.index(':')
571 assert(a[i:i+2] == ': ')
572 p[a[:i]] = a[i+2:]
573 elif a == '\n':
574 d[p['Package']] = p
575 p={}
576 a=f.readline()
577
578
579 def scan_control(p,params=None,prefix=None,info=None):
580 if prefix == None:
581 prefix = ''
582 else:
583 prefix += '/'
584 a=p.readline()
585 while a:
586 a=de_n(a)
587 if a[:4] in ('Pack','Vers','Arch','Stat','Inst','File'):
588 if info != None :
589 info.append(prefix+a)
590 if params != None:
591 i=a.index(':')
592 assert(a[i:i+2] == ': ')
593 params[prefix+a[:i]] = a[i+2:]
594 a=p.readline()
595
596 def append_info(delta,info):
597 "insert into the delta (that is an AR archive) the info file, as a first element, possibly removing a previous occurrence"
598 #new style : special info file
599 TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR))
600 infofile=open(TD+'/info','w')
601 for i in info:
602 infofile.write(i+'\n')
603 infofile.close()
604 if DO_GPG:
605 r=_compute_hashes_(TD+"/info")
606 else:
607 r=None
608 system(['ar','rSi','0',delta, 'info'], TD)
609 rmtree(TD)
610 return r
611
612 def de_n(a):
613 assert isinstance(a, str)
614 if a and a[-1:] == '\n' :
615 a = a[:-1]
616 return a
617
618 def de_n_b(a):
619 assert sys.version_info.major == 2 or isinstance(a, bytes)
620 if a and a[-1:] == b'\n' :
621 a = a[:-1]
622 return a
623
624 def de_bar(a):
625 assert isinstance(a, str)
626 if a and a[:2] == './' :
627 a=a[2:]
628 elif a == '/.' :
629 a=''
630 elif a and a[0] == '/' :
631 a=a[1:]
632 return a
633
634 def de_bar_b(a):
635 assert sys.version_info.major == 2 or isinstance(a, bytes)
636 if a and a[:2] == b'./' :
637 a=a[2:]
638 elif a == b'/.' :
639 a=''
640 elif a and a[0] == b'/' :
641 a=a[1:]
642 return a
643
644
645 def list_ar(f):
646 assert(os.path.exists(f))
647 ar_list = []
648 p=my_popen_read(['ar','t', f])
649 while 1:
650 a=p.readline()
651 if not a : break
652 a=de_n_b(a)
653 ar_list.append(a)
654 p.close()
655 return ar_list
656
657 def list_tar(f):
658 " returns content of the tar file `f`, as a list of `bytes`"
659 assert(os.path.exists(f))
660 ar_list = []
661 p=my_popen_read(['tar','t',f])
662 while 1:
663 a=p.readline()
664 if not a : break
665 a=de_n_b(a)
666 ar_list.append(a)
667 p.close()
668 return ar_list
669
670 def untar_control_in_deb(ar_ls):
671 " ar_ls is usually list_ar(n) ; it returns (c , b, z) \
672 where c 'name of control' , b 'option for tar' , z 'unzip command' "
673 c = None
674 for a in ar_ls :
675 if a[:12]==b'control.tar.':
676 c=a
677 assert c
678 # command for tar
679 bdb={b'.gz':'-z', b'.bz2':'-j', b'.lzma':'--lzma', b'.xz':'-J'}
680 zdb={b'.gz' : ['gzip','-cd'], b'.bz2': ['bzip2','-cd'],
681 b'.lzma':['unlzma','-c'], b'.xz': ['unxz','-c'] }
682 return c, bdb.get(c[11:]), zdb.get(c[11:])
683
684 class cache_sequence(object):
685 cache_filename=None
686 cache_file=None
687 cache=None
688 exists=None
689 broken=None
690 suffix='.debdelta_cache'
691 def __init__(self, filename, cache_filename=None):
692 """manages a cache file that store a sequence of python object;
693 'filename' is a reference file that is related to the data being cached,
694 and is used to create the name of the cache file, unless 'cache_filename'
695 is provided
696 """
697 self.cache_filename=os.path.splitext(filename)[0]+self.suffix \
698 if cache_filename==None else cache_filename
699 self.cache_file=None
700 self.cache=None
701 self.broken=None
702 self.exists=os.path.isfile(self.cache_filename) and \
703 os.path.getmtime(filename) <= os.path.getmtime(self.cache_filename)
704
705 def __iter__(self):
706 assert self.exists and not self.cache
707 self.cache=gzip.GzipFile(self.cache_filename)
708 return self
709
710 def __next__(self):
711 assert self.cache
712 try:
713 return pickle.load(self.cache)
714 except EOFError:
715 self.cache=None
716 raise StopIteration
717 except Exception:
718 logger.exception('Cache file is broken, deleting %r' % self.cache_filename)
719 if ACT: os.unlink(self.cache_filename)
720 self.cache=None
721 self.broken=True
722 # do not kill program
723 raise StopIteration
724
725 def next(self):
726 return self.__next__()
727
728 def __prepare_for_write__(self):
729 if not self.cache:
730 if DEBUG: logger.debug('Creating cache file : %r', self.cache_filename)
731 self.cache_file=open(self.cache_filename+'~tmp~', 'wb')
732 self.cache=gzip.GzipFile(filename='', fileobj=self.cache_file, mode='wb') # 'mtime=0' needs python2.7 or higher
733
734 def close(self):
735 try:
736 if self.cache:
737 self.cache.close()
738 self.cache=None
739 if self.cache_file:
740 self.cache_file.close()
741 self.cache_file=None
742 os.rename(self.cache_filename+'~tmp~', self.cache_filename)
743 except Exception:
744 logger.exception('Cannot close the cache file %r' % self.cache_filename)
745 self.broken=True
746
747 __del__=close
748
749 def write(self,s):
750 " write one object"
751 assert not self.exists
752 if self.broken:
753 return
754 self.__prepare_for_write__()
755 try:
756 self.cache.write(pickle.dumps(s))
757 except Exception:
758 logger.exception('Cannot write to cache file, deleting %r', self.cache_filename)
759 self.close()
760 if ACT: os.unlink(self.cache_filename)
761 self.broken=True
762
763 class cache_same_dict(cache_sequence):
764 "cache occurrences of a dict that uses always the same keys; omit the keys to optimize"
765 def __init__(self, filename, keys):
766 super(cache_same_dict, self).__init__(filename)
767 self.keys=keys
768
769 def write(self, s):
770 n=[s[k] for k in self.keys]
771 super(cache_same_dict, self).write(n)
772
773 def __next__(self):
774 n=(super(cache_same_dict, self)).__next__()
775 return dict(map(lambda x,y: (x,y) , self.keys, n)) # dict comprehension may be used instead
776
777 #####################################################################
778
779 ALLOWED = '<>()[]{}.,;:!_-+/ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
780 ALLOWED_SET = set(ord(j) for j in ALLOWED)
781 NUMBERS_SET = set(ord(j) for j in '0123456789')
782
783 ECHO_TEST = r"""c='\0151\0141'
784 E='echo -ne'
785 if test c`$E 'i'"$c" `o = ciiao ; then
786 :
787 else
788 E='echo -n'
789 if test c`$E 'i'"$c" `o = ciiao ; then
790 :
791 else
792 #echo WARNING : BUILTIN echo DOES NOT WORK OK
793 E='/bin/echo -ne'
794 test c`$E 'i'"$c" `o = ciiao
795 fi
796 fi
797 """
798
799 def prepare_for_echo_py2_(s):
800 assert ( type (s) in string_types )
801 r=''
802 shortquoted=False
803 for a in s:
804 if a in ALLOWED :
805 r += a
806 shortquoted = False
807 elif a in '0123456789' :
808 if shortquoted :
809 a = "\\" + ('000' +oct(ord(a)))[-4:]
810 shortquoted = False
811 r += a
812 elif a == 0:
813 r += '\\0'
814 shortquoted = True
815 else:
816 a = "\\" + oct(ord(a))
817 r += a
818 shortquoted = len(a) < 5
819 return r
820
821 def prepare_for_echo_py3_(s):
822 " `s` may be a `str` or `bytes` ; returns `str`"
823 #assert isinstance(s,bytes)
824 if isinstance(s,str):
825 s = str2bytes(s)
826 r=''
827 O = lambda x : oct(x)[2:]
828 shortquoted=False
829 for a in s:
830 #j = ord(a)
831 if a in ALLOWED_SET :
832 r += chr(a)
833 shortquoted = False
834 elif a in NUMBERS_SET :
835 if shortquoted :
836 a = "\\" + ('000' + O(a) )[-4:]
837 r += a
838 shortquoted = False
839 else:
840 r += chr(a)
841 elif a == 0:
842 r += '\\0'
843 shortquoted = True
844 else:
845 a = "\\0" + O(a)
846 r += a
847 shortquoted = len(a) < 5
848 return r
849
850 if sys.version_info.major == 2:
851 prepare_for_echo__ = prepare_for_echo_py2_
852 else:
853 prepare_for_echo__ = prepare_for_echo_py3_
854
855
856 def apply_prepare_for_echo(shell,repres):
857 a=ECHO_TEST + " $E '" + repres + "' \n exit "
858 p = subprocess.Popen([shell], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
859 close_fds=True) #, ** Popen_args)
860 (i, o) = (p.stdout, p.stdin)
861 o.write(str2bytes(a))
862 o.close()
863 a=i.read()
864 i.close()
865 return a
866
867 #ack! I wanted to use 'dash' as preferred shell, but bug 379227 stopped me
868 SHELL = '/bin/bash'
869 #check my code
870 s=b'\x00'+b'1ciao88\n77\r566'+b'\x00'+b'99\n'
871 r=prepare_for_echo__(s)
872 a=apply_prepare_for_echo(SHELL,r)
873 if a != s :
874 sys.stderr.write( 'string='+repr(s)+'\n')
875 sys.stderr.write( 'repres='+repr(r)+'\n')
876 sys.stderr.write( 'shell='+repr(SHELL)+'\n')
877 sys.stderr.write( 'output='+repr(a)+'\n')
878 sys.stderr.write( 'Errror in prepare_for_echo.\n')
879 raise SystemExit(4)
880 del r,s,a
881
882 ###
883
884 def prepare_for_echo(s):
885 r=prepare_for_echo__(s)
886 if DEBUG > 2 :
887 a=apply_prepare_for_echo(SHELL,r)
888 if a != s:
889 z = 'Error in prepare_for_echo()\n'
890 z += 'string='+repr(s)+'\n'
891 z += 'repres='+repr(r)+'\n'
892 z += 'shell='+repr(SHELL)+'\n'
893 z += 'output='+repr(a)+'\n'
894 raise DebDeltaError(z, exitcode=4)
895 return r
896
897 #####################################################################
898
899
900 def version_mangle(v):
901 if ':' in v :
902 return str.join('%3a',v.split(':'))
903 else:
904 return v
905
906 def version_demangle(v):
907 if '%' in v :
908 return str.join(':',v.split('%3a'))
909 else:
910 return v
911
912 def delta_base_name(pkg, old_ver, new_ver, arch, ext='.debdelta'):
913 assert(':' not in pkg)
914 return pkg + \
915 '_' + version_mangle(old_ver) + \
916 '_' + version_mangle(new_ver) + \
917 '_' + arch + ext
918
919 def tempo():
920 TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR))
921 for i in 'OLD','NEW','PATCH' :
922 os.mkdir(TD+'/'+i)
923 if VERBOSE > 2 or KEEP : logger.debug('Temporary in '+TD)
924 return TD
925
926 ##########
927
928
929 class DebDeltaError(Exception): #should derive from (Exception):http://docs.python.org/dev/whatsnew/pep-352.html
930 # Subclasses that define an __init__ must call Exception.__init__
931 # or define self.args. Otherwise, str() will fail.
932 def __init__(self,s,retriable=False,exitcode=None,logs=None):
933 assert(type(s) in string_types)
934 self.retriable = retriable
935 if retriable:
936 self.args=(s + ' (retriable) ',)
937 else:
938 self.args=(s + ' (non retriable) ',)
939 if exitcode == None:
940 if retriable:
941 exitcode = 1
942 else:
943 exitcode = 2
944 self.exitcode=exitcode
945 self.logs=logs
946
947 def die(s):
948 #if s : sys.stderr.write(s+'\n')
949 assert type(s) in string_types
950 raise DebDeltaError(s)
951
952 class PopenPipe(object):
953 def __init__(self, a, **dictargs):
954 #special code for pipes http://docs.python.org/library/subprocess.html#replacing-shell-pipeline
955 old_stdin=dictargs.pop('stdin',None)
956 old_stdout=dictargs.pop('stdout',None)
957 self.pros=[]
958 while '|' in a:
959 l=a.index('|') ; a1=a[:l] ; a=a[l+1:]
960 p=subprocess.Popen(args=a1, stdin=old_stdin, stdout=subprocess.PIPE, **dictargs)
961 self.pros.append( p )
962 old_stdin=p.stdout
963 final_pro= subprocess.Popen(args=a, stdin=old_stdin, stdout=old_stdout, **dictargs)
964 for p in self.pros:
965 p.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits.
966 self.pros.append(final_pro)
967 #
968 self.returncode=None
969 self.stdout=self.pros[-1].stdout
970 self.stdin=self.pros[0].stdin
971 self.read=getattr(self.pros[-1].stdout,'read',None)
972 self.write=getattr(self.pros[0].stdin,'write',None)
973 def __set_returncode(self):
974 if any([ (p.returncode<0) for p in self.pros]):
975 self.returncode = min([ p.returncode for p in self.pros])
976 else:
977 self.returncode = max([ p.returncode for p in self.pros])
978 return self.returncode
979 def poll(self):
980 for p in self.pros:
981 p.poll()
982 return self.__set_returncode()
983 def wait(self):
984 for p in self.pros:
985 p.wait()
986 return self.__set_returncode()
987 def terminate(self):
988 self.pros[0].poll()
989 if self.pros[0].returncode is None:
990 self.pros[0].terminate()
991 def close(self):
992 self.terminate()
993 self.wait()
994
995
996 def system(a,TD,saveargs=None,ignore_output=False,return_output=False):
997 "a must be a tuple, TD the temporary directory ; if return_output , it will return (stdout,stderr,exitcode) regardless"
998 assert type(a) in (list, tuple)
999 # mvo: compat with python2.5 where tuple does not have index
1000 a = list(a)
1001 if VERBOSE and TD[: (len(TMPDIR)+9) ] != TMPDIR+'/debdelta' :
1002 logger.debug(' Warning "system()" in '+TD+' for '+repr(a))
1003 (temp_fd, temp_name) = tempfile.mkstemp(prefix="debdelta_out_system")
1004 (temp_err_fd, temp_err_name) = tempfile.mkstemp(prefix="debdelta_err_system")
1005
1006 MPP=PopenPipe(a,stdin=open(os.devnull), stdout=temp_fd, stderr=temp_err_fd, cwd=TD, close_fds=True)
1007 ret=MPP.wait()
1008 os.close(temp_fd)
1009 os.close(temp_err_fd)
1010 if VERBOSE > 3 : logger.debug(' system(%r)=%d', a, ret)
1011 if ignore_output==False and (os.stat(temp_name)[ST_SIZE] > 0 or os.stat(temp_err_name)[ST_SIZE] > 0 ):
1012 logger.debug(' command "%s" returned %d and produced output as follows' % (a,ret))
1013 for i in open(temp_name):
1014 logger.debug('stdout: '+repr(i))
1015 for i in open(temp_err_name):
1016 logger.debug('stderr: '+repr(i))
1017 if return_output:
1018 return temp_name, temp_err_name, ret
1019 os.unlink(temp_err_name)
1020 os.unlink(temp_name)
1021 if ret == 0:
1022 return
1023 elif ret != 1 or a[0] != 'xdelta' :
1024 s='Error , non zero return status '+str(ret)+' for command "'+repr(a)+'"'
1025 try:
1026 if DEBUG and saveargs:
1027 T=abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR))
1028 open(T+'/command','w').write(repr(a))
1029 for l in saveargs:
1030 if l[0] != '/':
1031 l = TD+'/'+l
1032 if os.path.exists(l):
1033 shutil.copy2(l,T)
1034 s=s+'\n saved argument '+l+' in '+T
1035 else:
1036 s=s+'\n did not find argument '+l
1037 except OSError:
1038 o=sys.exc_info()[1]
1039 s=s+'\n (there was an additional OSError "'+str(o)+'" when trying to save arguments)'
1040 die(s)
1041
1042 def check_deb(f, die_ = True):
1043 if not os.path.exists(f) :
1044 die(_("Error: the file `%s' does not exist.") % f)
1045 if not os.path.isfile(f) :
1046 die(_("Error: `%s' is not a regular file.") % f)
1047 p=open(f,'rb')
1048 if p.read(21) != b"!<arch>\ndebian-binary" :
1049 if die_:
1050 die(_("Error: `%s' does not seem to be a Debian package.") % f)
1051 else:
1052 return False
1053 p.close()
1054 return True
1055
1056 def check_is_delta(f):
1057 if not os.path.exists(f) :
1058 die(_("Error: the file `%s' does not exist.") % f)
1059 if not os.path.isfile(f) :
1060 die(_("Error: `%s' is not a regular file.") % f)
1061 p=open(f,'rb')
1062 if p.read(8) != b"!<arch>\n" :
1063 die(_("Error: `%s' does not seem to be a Debian delta.") % f)
1064 p.close()
1065
1066 def puke(s,e=None):
1067 """ write informations on stderr; if DEBUG also prints traceback;
1068 if DEBUG>1 and `e` is given, rereaise the exception"""
1069 (typ, value, trace)=sys.exc_info()
1070 if e == None or len(str(e)) < 2:
1071 logger.error(str(s)+' : '+str(e)+' '+str(typ)+' '+str(value))
1072 else:
1073 logger.error(str(s)+' : '+str(e))
1074 if DEBUG > 1 and e is not None and trace:
1075 raise
1076 elif DEBUG and trace and traceback.print_tb(trace):
1077 logger.error( traceback.print_tb(trace))
1078
1079
1080 ################################################################### GPG
1081
1082 def gpg_base_commandline():
1083 if GPG_HOME:
1084 GPG_BASE_CMD_LINE=[GPG_CMD,"--homedir",GPG_HOME]
1085 else:
1086 GPG_BASE_CMD_LINE=[GPG_CMD,"--keyring",GPG_MASTER_PUB_KEYRING]
1087
1088 if VERBOSE < 1 :
1089 GPG_BASE_CMD_LINE+=['--quiet']
1090
1091 return GPG_BASE_CMD_LINE
1092
1093 def gpg_sign_command():
1094 return gpg_base_commandline()+["--batch","--armor","--clearsign","--default-key",GPG_SIGNING_KEY,"--sign"]
1095
1096 def compute_md5_up_to_len(o, length):
1097 "hash initial part of a file using MD5. 'o' may be a string (in which case the file is opened) or a file type; returns MD5 and bytes effectively read"
1098 assert type(length) in (int,long) and length >= 0
1099 if type(o) in string_types:
1100 o = open(o,'rb')
1101 m=hashlib.md5()
1102 a=True
1103 l=length
1104 while l>0 and a:
1105 a=o.read(min(1024,l))
1106 l -= len(a)
1107 m.update(a)
1108 return m.hexdigest(), length-l
1109
1110
1111 def compute_md5_len(o):
1112 "hash the file using MD5. 'o' may be a string (in which case the file is opened) or a file type; returns MD5 and length"
1113 if type(o) in string_types:
1114 o = open(o,'rb')
1115 m=hashlib.md5()
1116 a=o.read(1024)
1117 l=0
1118 while a:
1119 l+=len(a)
1120 m.update(a)
1121 a=o.read(1024)
1122 return m.hexdigest(), l
1123
1124 def compute_md5(o):
1125 "hash the file using MD5. 'o' may be a string (in which case the file is opened) or a file type; returns MD5 (as a string of hexes)"
1126 return compute_md5_len(o)[0]
1127
1128 def _compute_hashes_(na):
1129 "hash the file"
1130 o = open(na,'rb')
1131 m=hashlib.md5()
1132 s=hashlib.sha1()
1133 a=o.read(1024)
1134 while a:
1135 m.update(a)
1136 s.update(a)
1137 a=o.read(1024)
1138 r = ( m.hexdigest(), s.hexdigest(), os.stat(na)[ST_SIZE])
1139 return r
1140
1141 def _compute_hashes_db_(li,DIR):
1142 db={}
1143 for na in li:
1144 db[na] = _compute_hashes_(DIR+'/'+na)
1145 return db
1146
1147 def verify_signature(signature, DIR):
1148 a=b"-----BEGIN PGP SIGNED MESSAGE-----\n"
1149 if open(signature,'rb').read(len(a)) != a:
1150 return ('BAD_FORMAT',signature)
1151
1152 role=os.path.basename(signature)
1153 assert role[:4] == "_gpg"
1154 role=role[4:]
1155
1156 (temp_fd, temp_name) = tempfile.mkstemp(prefix="debdelta_gpg_verified")
1157 #(read_end, write_end) = os.pipe()
1158 p=subprocess.Popen(gpg_base_commandline() + ['--batch','--status-fd',"2",'--output',"-",signature],
1159 stdout=subprocess.PIPE, stderr=temp_fd, stdin=open(os.devnull), close_fds=True,
1160 **Popen_args)
1161 r=_verify_signature_no_gpg(p.stdout, DIR, role)
1162 p.wait()
1163
1164 os.close(temp_fd)
1165
1166 if VERBOSE > 2 or p.returncode:
1167 for j in open(temp_name):
1168 logger.debug(' GPG> '+j.rstrip())
1169
1170 os.unlink(temp_name)
1171
1172 if p.returncode:
1173 return ('GPG_VERIFY_FAILED',signature)
1174
1175 return r
1176
1177 def _verify_signature_no_gpg(signature, DIR, role):
1178 #list stuff, skipping signatures
1179 dir_list = [a for a in os.listdir(DIR) if a[:4] != '_gpg']
1180 #compute signatures
1181 hashes = _compute_hashes_db_(dir_list, DIR)
1182 #scan hashes file (GPG already verified)
1183 if type(signature) in string_types:
1184 f=open(signature)
1185 elif hasattr(signature,'readline'):
1186 f=signature
1187 else: raise AssertionError
1188 a=f.readline()
1189 if a != "Version: 4\n":
1190 return ("UNSUPPORTED_VERSION",a)
1191 a=f.readline()
1192 while a:
1193 if a[:5] == "Role:":
1194 if a[5:].strip() != role :
1195 return ("ROLE_MISMATCH",a)
1196 a=f.readline()
1197 elif a[:6] == "Files:" :
1198 #parse files
1199 a=f.readline()
1200 while a and a[0] in ( '\t' , ' ') :
1201 a=a.rstrip('\n')
1202 a=a.lstrip()
1203 a=a.split(' ')
1204 if VERBOSE > 3 : logger.debug(' checking hashes '+repr(a))
1205 (md5,sha1,le,na)=a
1206 if na not in dir_list:
1207 return ('ABSENT',na)
1208 (cmd5,csha1,cle)=hashes[na]
1209 if int(le) != cle:
1210 return ('SIZE',na)
1211 # check hashes
1212 if md5 != cmd5 :
1213 return ('MD5',na)
1214 if sha1 != csha1 :
1215 return ('SHA1',na)
1216 dir_list.remove(na)
1217 a=f.readline()
1218 elif VERBOSE > 2 :
1219 logger.debug(' signature header ignored: '+ a)
1220 a=f.readline()
1221 else:
1222 a=f.readline()
1223 #end parsing
1224 if dir_list:
1225 return ("UNCHECKED",dir_list)
1226 return True
1227
1228 def _write_signature(db,filename,role):
1229 "starting from a database of hashes, see _compute_hashes_, it writes a signature file"
1230 f=open(filename,mode='w')
1231 ##this is the format of dpkg-sig, but is redundant, since the "date" and "signer"
1232 ##are already available thru the gpg signature
1233 #f.write("Version: 4\nSigner: \nDate: %s\nRole: %s\nFiles: \n" % (time.ctime(),role))
1234 ##and actually dpkg-sig will validate also a simpler file, so, lets save a few bytes
1235 f.write("Version: 4\nRole: %s\nFiles:\n" % (role,))
1236 for a in db:
1237 (m,s,l) = db[a]
1238 f.write('\t'+m+" "+s+" "+str(l)+" "+a+"\n")
1239 f.close()
1240
1241 def sign_delta(delta, db, role="maker"):
1242 TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR))
1243 try:
1244 _write_signature(db,TD+'/_temp',role)
1245 p=subprocess.Popen(gpg_sign_command() +['--output',TD+'/_gpg'+role,TD+'/_temp'], stdin=open(os.devnull), close_fds=True)
1246 p.wait()
1247 if p.returncode==0:
1248 r=system(("ar","qS",delta,TD+"/_gpg"+role),TD)
1249 except:
1250 rmtree(TD)
1251 raise
1252 rmtree(TD)
1253 if p.returncode:
1254 raise DebDeltaError('GnuPG fails to sign')
1255 if r:
1256 raise DebDeltaError('ar fails to add the signature')
1257
1258
1259
1260
1261 #################################################################### apply patch
1262
1263 ########### info auxiliary routines
1264
1265 def _info_patch_unzip_(TD):
1266 "unzip info and patch.sh"
1267 if os.path.exists(TD+'PATCH/info.gz'):
1268 system(('gunzip','PATCH/info.gz'),TD)
1269 if os.path.exists(TD+'PATCH/patch.sh.gz'):
1270 system(('gunzip','PATCH/patch.sh.gz'),TD)
1271 elif os.path.exists(TD+'PATCH/patch.sh.bz2'):
1272 system(('bunzip2','PATCH/patch.sh.bz2'),TD)
1273 elif os.path.exists(TD+'PATCH/patch.sh.lzma'):
1274 if not os.path.exists('/usr/bin/unlzma'):
1275 raise DebDeltaError('This patch needs lzma. Please install the Debian package "lzma".',retriable=True)
1276 system(('unlzma','PATCH/patch.sh.lzma'),TD)
1277 elif os.path.exists(TD+'PATCH/patch.sh.xz'):
1278 if not os.path.exists('/usr/bin/unxz'):
1279 raise DebDeltaError('This patch needs xz. Please install the Debian package "xz-utils".',retriable=True)
1280 system(('unxz','PATCH/patch.sh.xz'),TD)
1281
1282 def get_info_slow(delta,T=None):
1283 if T:
1284 TD=T
1285 else:
1286 TD=tempo()
1287 if TD[-1] != '/':
1288 TD = TD + '/'
1289 delta=abspath(expanduser(delta))
1290 system(('ar','x',delta,'info','info.gz','patch.sh','patch.sh.gz','patch.sh.bz2','patch.sh.lzma','patch.sh.xz'),
1291 TD+'/PATCH', ignore_output=True)
1292 _info_patch_unzip_(TD)
1293 info = _scan_delta_info_(TD)
1294 if T == None:
1295 rmtree(TD)
1296 return info
1297
1298 def get_info_fast(delta):
1299 f=open(delta,'rb')
1300 s=f.readline()
1301 if b"!<arch>\n" != s :
1302 raise DebDeltaError('This is not a delta file: '+delta)
1303 s = f.read(60)
1304 if len(s) != 60 :
1305 logger.warning('(Warning, cannot get info from truncated: '+delta+' )')
1306 return None
1307 if s[:4] != b'info':
1308 #old style debdelta, with info in patch.sh
1309 if VERBOSE > 1 : logger.warning(' (Warning, cannot get info from old style: '+delta+' )')
1310 return None
1311 ##parse ar segment
1312 ## see /usr/include/ar.h
1313 if s[-2:] != b'`\n' :
1314 logger.warning('(Warning, cannot get info from '+delta+' , format not known)')
1315 return None
1316 l=int(s[ -12:-2 ])
1317 s=bytes2str(f.read(l))
1318 if len(s) != l :
1319 logger.warning('(Warning, cannot get info from truncated: '+delta+' )')
1320 return None
1321 info= s.split('\n')
1322 f.close()
1323 return info
1324
1325 def get_info(delta,TD=None):
1326 info=get_info_fast(delta)
1327 if info == None:
1328 info=get_info_slow(delta,TD)
1329 return info
1330
1331 def _scan_delta_info_(TD):
1332 info=[]
1333 if os.path.isfile(TD+'PATCH/info'):
1334 #new style debdelta, with info file
1335 p=open(TD+'PATCH/info')
1336 info=p.read().split('\n')
1337 p.close()
1338 if info[-1] == '': info.pop()
1339 else:
1340 #old style debdelta, with info in patch.sh
1341 p=open(TD+'PATCH/patch.sh')
1342 s=p.readline()
1343 s=p.readline()
1344 while s:
1345 if s[0] == '#' :
1346 s=de_n(s)
1347 info.append(s[1:])
1348 s=p.readline()
1349 p.close()
1350 return info
1351
1352 def info_2_db(info):
1353 params={}
1354 for s in info:
1355 if ':' in s:
1356 i=s.index(':')
1357 params[s[:i]] = s[i+2:]
1358 elif s:
1359 params[s] = True
1360 return params
1361
1362 ########### other auxiliary routines
1363
1364 def patch_check_tmp_space(params,olddeb):
1365 if type(params) != dict:
1366 params=info_2_db(params)
1367 if 'NEW/Installed-Size' not in params or 'OLD/Installed-Size' not in params:
1368 logger.warning('(Warning... Installed size unknown...)')
1369 return True
1370 free=freespace(TMPDIR)
1371 if free == None : return True
1372 free = free / 1024
1373 if os.path.isdir(olddeb):
1374 instsize=int(params['NEW/Installed-Size'])
1375 #the last action of the script is to gzip the data.tar, so
1376 if 'NEW/Size' in params :
1377 instsize += int(params['NEW/Size']) / 1024
1378 else:
1379 instsize = instsize * 1.8
1380 else:
1381 instsize=int(params['NEW/Installed-Size'])+int(params['OLD/Installed-Size'])
1382 instsize += 2**13
1383 if free < instsize :
1384 return _('not enough disk space (%(free)dkB) in directory %(dir)s for applying delta (needs %(size)dkB)') % \
1385 {'free' : int(free), 'dir' : TMPDIR, 'size' : instsize}
1386 else:
1387 return True
1388
1389
1390 def scan_diversions():
1391 f=open('/var/lib/dpkg/diversions')
1392 d={}
1393
1394 a=1
1395 while 1:
1396 a=f.readline()
1397 if not a: break
1398 a=de_n(a)
1399 b=de_n(f.readline())
1400 p=de_n(f.readline())
1401 d[a]=(b,p)
1402 f.close()
1403 return d
1404
1405 ###################################################### debforensic extract
1406
1407 #in base-passwd 3.5.11
1408 #/usr/share/base-passwd/passwd.master
1409 base_passwd="""root::0:0:root:/root:/bin/bash
1410 daemon:*:1:1:daemon:/usr/sbin:/bin/sh
1411 bin:*:2:2:bin:/bin:/bin/sh
1412 sys:*:3:3:sys:/dev:/bin/sh
1413 sync:*:4:65534:sync:/bin:/bin/sync
1414 games:*:5:60:games:/usr/games:/bin/sh
1415 man:*:6:12:man:/var/cache/man:/bin/sh
1416 lp:*:7:7:lp:/var/spool/lpd:/bin/sh
1417 mail:*:8:8:mail:/var/mail:/bin/sh
1418 news:*:9:9:news:/var/spool/news:/bin/sh
1419 uucp:*:10:10:uucp:/var/spool/uucp:/bin/sh
1420 proxy:*:13:13:proxy:/bin:/bin/sh
1421 www-data:*:33:33:www-data:/var/www:/bin/sh
1422 backup:*:34:34:backup:/var/backups:/bin/sh
1423 list:*:38:38:Mailing List Manager:/var/list:/bin/sh
1424 irc:*:39:39:ircd:/var/run/ircd:/bin/sh
1425 gnats:*:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/bin/sh
1426 nobody:*:65534:65534:nobody:/nonexistent:/bin/sh"""
1427 base_passwd_db={}
1428 base_passwd_anti_db={}
1429 for a in base_passwd.split('\n'):
1430 a=a.split(':')
1431 base_passwd_db[a[0]]=int(a[2])
1432 base_passwd_anti_db[int(a[2])]=a[0]
1433
1434 base_group="""root:*:0:
1435 daemon:*:1:
1436 bin:*:2:
1437 sys:*:3:
1438 adm:*:4:
1439 tty:*:5:
1440 disk:*:6:
1441 lp:*:7:
1442 mail:*:8:
1443 news:*:9:
1444 uucp:*:10:
1445 man:*:12:
1446 proxy:*:13:
1447 kmem:*:15:
1448 dialout:*:20:
1449 fax:*:21:
1450 voice:*:22:
1451 cdrom:*:24:
1452 floppy:*:25:
1453 tape:*:26:
1454 sudo:*:27:
1455 audio:*:29:
1456 dip:*:30:
1457 www-data:*:33:
1458 backup:*:34:
1459 operator:*:37:
1460 list:*:38:
1461 irc:*:39:
1462 src:*:40:
1463 gnats:*:41:
1464 shadow:*:42:
1465 utmp:*:43:
1466 video:*:44:
1467 sasl:*:45:
1468 plugdev:*:46:
1469 staff:*:50:
1470 games:*:60:
1471 users:*:100:
1472 nogroup:*:65534:"""
1473
1474 base_group_db={}
1475 base_group_anti_db={}
1476 for a in base_group.split('\n'):
1477 a=a.split(':')
1478 base_group_db[a[0]]=int(a[2])
1479 base_group_anti_db[int(a[2])]=a[0]
1480
1481 # all code following return name,mode,tartype,uid,gid,uname,gname
1482
1483 #adapted from tarfile.py, a Python module
1484 def stat_to_tar(name):
1485 "returns name,mode,tartype,uid,gid,uname,gname,data"
1486 statres = os.lstat(name)
1487 stmd = statres.st_mode
1488 data = None
1489 if stat.S_ISREG(stmd):
1490 tartype = tarfile.REGTYPE
1491 # here ideally we should SHA1 the file ;
1492 # but this is done elsewhere for performance,
1493 # and to have multi_hash in the future
1494 elif stat.S_ISDIR(stmd):
1495 tartype = tarfile.DIRTYPE
1496 elif stat.S_ISFIFO(stmd):
1497 tartype = tarfile.FIFOTYPE
1498 elif stat.S_ISLNK(stmd):
1499 tartype = tarfile.SYMTYPE
1500 data = os.readlink(name)
1501 elif stat.S_ISCHR(stmd):
1502 tartype = tarfile.CHRTYPE
1503 elif stat.S_ISBLK(stmd):
1504 tartype = tarfile.BLKTYPE
1505 elif stat.S_ISSOCK(stmd):
1506 tartype = 'SOCKET' #SOCKETs are not supported in tar files
1507 else: raise TypeError
1508
1509 if tartype in (tarfile.CHRTYPE, tarfile.BLKTYPE):
1510 data = str(os.major(statres.st_rdev))+' '+str( os.minor(statres.st_rdev))
1511
1512 uid,gid = statres.st_uid, statres.st_gid
1513
1514 if uid in base_passwd_anti_db :
1515 uname = base_passwd_anti_db[uid]
1516 else:
1517 import pwd
1518 try:
1519 uname = pwd.getpwuid(uid)[0]
1520 except KeyError:
1521 uname = None
1522
1523 if gid in base_group_anti_db :
1524 gname = base_group_anti_db[gid]
1525 else:
1526 import grp
1527 try:
1528 gname = grp.getgrgid(gid)[0]
1529 except KeyError:
1530 gname = None
1531
1532 #07777 is used in tarfile.TarInfo.tobuf
1533 return name.lstrip('/'), stmd & 0o7777, tartype, uid, gid, uname, gname, data
1534
1535
1536 def tarinfo_to_ls(tartype,tarmode):
1537 "returns a string -rwxrwxrwx such as what ls -l prints "
1538 if ord(tartype) == 0 :
1539 a='_'
1540 else:
1541 if tartype >= b'0' and tartype <= b'6' :
1542 a="-hlcbdp"[ord(tartype) - ord('0')]
1543 else:
1544 a='?'
1545 return a + stat_filemode(tarmode)[1:]
1546
1547
1548 def sha1_hash_file(f):
1549 "returns hex digest of file"
1550 s=hashlib.sha1()
1551 if type(f) in string_types:
1552 f=open(f,'rb')
1553 a=f.read(1024)
1554 while a:
1555 s.update(a)
1556 a=f.read(1024)
1557 f.close()
1558 return s.hexdigest()
1559
1560 def forensics_rfc(o,db,bytar,controlfiles,files,conffiles,diverted=[],diversions={},localepurged=[],prelink_u_failed=[]):
1561 " this is invoked by do_patch_() as well as do_delta_() ; in the former case, by_tar=False"
1562 assert type(diversions) == dict
1563 if type(db) == dict:
1564 for a in sorted(db.keys()):
1565 assert isinstance(a,str)
1566 if a[:3] == 'OLD':
1567 o.write(a[4:]+': '+db[a]+'\n')
1568 else:
1569 for a in sorted(db):
1570 assert isinstance(a,str)
1571 if a[:3] == 'OLD':
1572 o.write(a[4:]+'\n')
1573 if diverted:
1574 o.write("Diversions:\n")
1575 for a in sorted(diverted):
1576 b,p = diversions[a]
1577 o.write(" From: "+a+'\n')
1578 o.write(" To: "+b+'\n')
1579 o.write(" By: "+p+'\n')
1580 if conffiles:
1581 o.write("Conffiles:\n")
1582 for a in sorted(conffiles):
1583 o.write(' '+a+'\n')
1584 for L,N in ((controlfiles,"Control"),(files,"Files")):
1585 o.write(N+":\n")
1586 for l in sorted(L):
1587 if bytar:
1588 name,mode,tartype,uid,gid,uname,gname,data=l
1589 tmpcopy=None
1590 divert=None
1591 else:
1592 name,divert,tmpcopy=l
1593 if os.path.exists(divert):
1594 fullname,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(divert)
1595 else:
1596 fullname,mode,tartype,uid,gid,uname,gname,data='',0,'?',0,0,'?','?','?'
1597 if tartype == tarfile.REGTYPE:
1598 if tmpcopy and os.path.exists(tmpcopy):
1599 data=sha1_hash_file(tmpcopy)
1600 elif os.path.exists(divert):
1601 data=sha1_hash_file(divert)
1602 if name in ('.', '/', './', '/.') and tartype == tarfile.DIRTYPE: #skip root
1603 continue
1604 if uname == None: uname=str(uid)
1605 if gname == None: gname=str(gid)
1606 name=de_bar(name)
1607 o.write(' '+tarinfo_to_ls(tartype,mode)+" "+uname+' '+gname)
1608 if N == "Files" and tartype == tarfile.REGTYPE and name in conffiles:
1609 o.write(" [conffile]")
1610 if N == "Files" and tartype == tarfile.REGTYPE and name in localepurged:
1611 o.write(" [localpurged]")
1612 if N == "Files" and tartype == tarfile.REGTYPE and name in prelink_u_failed:
1613 o.write(" [prelink-u failed]")
1614 if divert and not os.path.exists(divert):
1615 o.write(" [missing file %r]" % divert)
1616 if tmpcopy:
1617 o.write(" [prelink-u]")
1618 o.write("\n "+name+"\n")
1619 if data!=None:
1620 o.write(" "+data+"\n")
1621 else:
1622 o.write(" \n")
1623
1624 def tar_those(f):
1625 " tar multiple files in one tar (all in the same base directory!). Note that f may be a list of lists or strings or mixed."
1626 temptar=tempfile.mktemp(suffix='.tgz')
1627 tar=tarfile.open(name=temptar,mode='w:gz')
1628 for z in f:
1629 if type(z) in (list,tuple):
1630 for j in z:
1631 tar.add(j,arcname=os.path.basename(j))
1632 elif type(z) in string_types:
1633 tar.add(z,arcname=os.path.basename(z))
1634 else: DebDeltaError(' internal error m92ksy')
1635 tar.close()
1636 return temptar
1637
1638 def forensic_send(f,forensic=FORENSIC):
1639 " note that f must be a list of lists (or None)"
1640 assert type(f) == list
1641 if forensic is False or forensic == 'False' or forensic == '':
1642 if f:
1643 logger.error(_('(Faulty delta. Please consider retrying with the option "--forensic=do" ).')+'\n')
1644 return
1645 if not f:
1646 return
1647 if all([(z == None) for z in f]):
1648 logger.warning('Sorry, no forensic logs were generated')
1649 return
1650 if forensic[:4] in ('mutt','mail') or forensic[:7] == 'icedove' or forensic[:10]=='thunderbird':
1651 email=FORENSIC_EMAIL
1652 if ':' in forensic:
1653 a=forensic.find(':')
1654 email == forensic[a:]
1655 forensic=forensic[:a]
1656 logger.warning(_("There were faulty deltas.")+' '+_("Now invoking the mail sender to send the logs."))
1657 if forensic in ('mutt','mail'):
1658 if sys.version_info.major == 2:
1659 raw_input( _('(hit any key)') )
1660 else:
1661 input( _('(hit any key)') )
1662 args=[]
1663 for z in f:
1664 if z:
1665 for j in z:
1666 args+=['-a',j]
1667 subprocess.call(['mutt',email,'-s','delta_failures']+args)
1668 else:
1669 temptar=tempfile.mktemp(suffix='.tgz')
1670 tar=tarfile.open(name=temptar,mode='w:gz')
1671 for z in f:
1672 if z:
1673 for j in z:
1674 tar.add(j,arcname=os.path.basename(j))
1675 tar.close()
1676 args="to=%s,subject=delta_failures,attachment='file:///%s'" % (email,temptar)
1677 subprocess.call([forensic,'-compose',args])
1678 return
1679 elif forensic.startswith('http'):
1680 logger.warning(_("There were faulty deltas.")+' '+_('Sending logs to server.'))
1681 temptar=tempfile.mktemp(suffix='.tgz')
1682 tar=tarfile.open(name=temptar,mode='w:gz')
1683 for z in f:
1684 if z:
1685 for j in z:
1686 tar.add(j,arcname=os.path.basename(j))
1687 tar.close()
1688 #http://atlee.ca/software/poster
1689 ## FIXME this will not work with python3
1690 import urllib, urllib2, httplib, poster
1691 poster.streaminghttp.register_openers()
1692 datagen, headers = poster.encode.multipart_encode({'auth_userid':'debdelta','auth_password':'slartibartfast',"thefile": open(temptar, "rb")})
1693 # Create the Request object
1694 request = urllib2.Request("http://debdelta.debian.net:7890/receive", datagen, headers)
1695 # Actually do the request, and get the response
1696 logger.info(' '+_('Server answers:'),repr(urllib2.urlopen(request).read()))
1697 return
1698 else:
1699 logger.warning(_('Faulty delta. Please send by email to %s the following files:\n') % EMAIL)
1700 for z in f:
1701 if z:
1702 logger.warning(' '+str.join(' ',z)+'\n')
1703 return
1704 logger.warning(_('(Faulty delta. Please consider retrying with the option "--forensic=do" ).')+'\n')
1705
1706 def elf_info(f):
1707 "returns (is_elf, ei_class, ei_data, ei_osabi, e_type)"
1708 import struct
1709 elfheader=open(f,'rb').read(32)
1710 if len(elfheader) == 32:
1711 #parse as specified in /usr/include/elf.h from libelf-dev
1712 EI_CLASS={1:'ELFCLASS32',
1713 2:'ELFCLASS64'}
1714 EI_DATA={1:'ELFDATA2LSB', # 2's complement, little endian
1715 2:'ELFDATA2MSB'} # 2's complement, big endian
1716 EI_OSABI={0:'ELFOSABI_SYSV',# UNIX System V ABI
1717 1:'ELFOSABI_HPUX',
1718 2:'ELFOSABI_NETBSD',
1719 3:'ELFOSABI_LINUX',
1720 #fixme insert other values
1721 9:'ELFOSABI_FREEBSD',
1722 12:'ELFOSABI_OPENBSD',
1723 97:'ELFOSABI_ARM'}
1724 #fixme what is ET_LOOS , ET_HIOS , ET_LOPROC, ET_HIPROC ??
1725 ET_TYPE={1:'ET_REL', #Relocatable file
1726 2:'ET_EXEC', #Executable file
1727 3:'ET_DYN', #Shared object file
1728 4:'ET_CORE'} #Core file
1729
1730 ei_magic, ei_class, ei_data, ei_version, ei_osabi, ei_abiversion = \
1731 struct.unpack_from('4sBBBBB',elfheader)
1732 e_type, e_machine, e_version = struct.unpack_from('HHI',elfheader,16)
1733 #FIXME I think I am getting ei_osabi wrong.. it is always 0
1734 is_elf = b'\x7fELF' == ei_magic
1735 #and ei_class in (1,2) and \
1736 # ei_version == 1 and \
1737 # ei_data in (1,2) and e_type>0 and e_machine>0 and e_version>0
1738 return is_elf, EI_CLASS.get(ei_class), \
1739 EI_DATA.get(ei_data), EI_OSABI.get(ei_osabi), ET_TYPE.get(e_type)
1740 #, e_machine, e_version
1741 else:
1742 return False, 0, 0, 0, 0
1743
1744 def parse_prelink_conf():
1745 " fixme , currently unused and incomplete "
1746 prelinked_dirs=[]
1747 prelinked_blacklist=[]
1748 prelinked_blacklist_glob=[]
1749 for a in open('/etc/prelink.conf'):
1750 if a[0] == '#':
1751 continue
1752 a=a.strip()
1753 b=a.split()
1754 if len(b) != 2:
1755 logger.warning(' (sorry this line of /etc/prelink.conf cannot be parsed currently: "'+a+'")')
1756 continue
1757 if '-b' == b[0]:
1758 if '/' in b[1]:
1759 prelinked_blacklist.append(b[1])
1760 else:
1761 prelinked_blacklist_glob.append(b[1])
1762 elif '-l' == b[0]:
1763 prelinked_dirs.append(b[1])
1764
1765
1766 ############ do_patch
1767
1768 def do_patch(delta,olddeb,newdeb, info=None, diversions=None, do_gpg=DO_GPG):
1769 runtime={}
1770 T=None
1771 try:
1772 T=tempo()
1773 r=do_patch_(delta,olddeb,newdeb, T, runtime,
1774 info=info, diversions=diversions, do_gpg=do_gpg)
1775 except:
1776 if T : rmtree(T)
1777 if newdeb and os.path.exists(newdeb):
1778 os.unlink(newdeb)
1779 raise
1780 rmtree(T)
1781 return r
1782
1783 def do_patch_(delta, olddeb, newdeb, TD, runtime,
1784 info=None, diversions=None, do_gpg=DO_GPG, do_progress=DO_PROGRESS):
1785
1786 import threading
1787
1788 if TD[-1] != '/':
1789 TD = TD + '/'
1790
1791 HAVE_PRELINK=os.path.exists('/usr/sbin/prelink')
1792 # some people purge locales w/o using 'localepurge' , see e.g. http://bugs.debian.org/619086
1793 #HAVE_LOCALEPURGE=os.path.exists('/etc/locale.nopurge') or os.path.exists('/usr/sbin/localepurge')
1794
1795 delta=abspath(delta)
1796 newdebshortname='-'
1797 if newdeb:
1798 newdebshortname=newdeb
1799 newdeb=abspath(newdeb)
1800 if not os.path.isdir(olddeb):
1801 olddeb=abspath(olddeb)
1802 elif diversions == None:
1803 diversions=scan_diversions()
1804
1805 start_sec = time.time()
1806 runtime['patchprogress']=0
1807
1808 check_is_delta(delta)
1809
1810 if not os.path.isdir(olddeb):
1811 check_deb(olddeb)
1812
1813 temp_name, temp_err_name, ret=system(('ar','xvo',delta), TD+'/PATCH', return_output=True, ignore_output=True)
1814 if ret :
1815 raise DebDeltaError('Cannot extract from '+delta)
1816 ar_list_delta=[a[4:] for a in open(temp_name).read().split('\n') if a]
1817 os.unlink(temp_name)
1818 os.unlink(temp_err_name)
1819
1820 runtime['patchprogress']=1
1821
1822 is_signed=False
1823 for a in ar_list_delta:
1824 if a[:4] == '_gpg':
1825 r = verify_signature(TD+'/PATCH/'+a,TD+'/PATCH')
1826 if True != r:
1827 die(delta+": the signature file "+a+" fails as follows: "+repr(r))
1828 is_signed=True
1829 if VERBOSE > 1 : logger.debug(' The signature by "'+a[4:]+'" is correctly verified for '+delta)
1830 if not is_signed:
1831 if do_gpg:
1832 die(_("Delta is not signed:")+' '+delta)
1833 elif do_gpg != None:
1834 logger.warning(_("WARNING, delta is not signed:")+' '+delta)
1835
1836 runtime['patchprogress']=2
1837
1838 _info_patch_unzip_(TD)
1839
1840 if not os.path.isfile(TD+'PATCH/patch.sh'):
1841 die("Error. File `%s' is not a delta file." % delta)
1842
1843 os.symlink(minigzip,TD+'minigzip')
1844 os.symlink(minibzip2,TD+'minibzip2')
1845
1846 #lets scan parameters, to see what it does and what it requires
1847 if info == None :
1848 info=_scan_delta_info_(TD)
1849 params=info_2_db(info)
1850
1851 runtime['patchprogress']=3
1852
1853 #this is not needed in preparing the patch, but may help in forensic
1854 conf_files=[]
1855 z='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles'
1856 if FORENSIC and os.path.isfile(z):
1857 #note that filenames do not have leading /
1858 conf_files=[de_bar(p) for p in open(z).read().split('\n') if p]
1859 del z
1860
1861 ###
1862 s=patch_check_tmp_space(params,olddeb)
1863 if s != True:
1864 raise DebDeltaError('Sorry, '+s, True )
1865
1866 if not os.path.isdir(olddeb):
1867 os.symlink(olddeb,TD+'/OLD.file')
1868 ar_list_old= list_ar(TD+'OLD.file')
1869 #unpack the old control structure, if available
1870 os.mkdir(TD+'/OLD/CONTROL')
1871 #unpack control.tar.gz
1872 f,b,z = untar_control_in_deb(ar_list_old)
1873 system(('ar','p',TD+'OLD.file',f,'|','tar',b,'-xp','-f','-','-C',TD+'OLD/CONTROL'),TD)
1874 #then we check for the conformance
1875 if not os.path.isdir(olddeb) and 'OLD/Size' in params:
1876 olddebsize = os.stat(olddeb)[ST_SIZE]
1877 if olddebsize != int(params['OLD/Size']):
1878 raise DebDeltaError('Old deb size is '+str(olddebsize)+' instead of '+params['OLD/Size'])
1879
1880 runtime['patchprogress']=4
1881
1882 if DEBUG > 1 :
1883 #this is currently disabled, since 'dpkg -s' is vey slow (~ 1.6 sec)
1884 dpkg_params={}
1885 b=params['OLD/Package']
1886 if os.path.isdir(olddeb):
1887 p=my_popen_read(['env', '-i', 'dpkg', '-s',b], **Popen_args)
1888 else:
1889 p=open(TD+'OLD/CONTROL/control')
1890 scan_control(p,params=dpkg_params,prefix='OLD')
1891 p.close()
1892 if os.path.isdir(olddeb):
1893 if 'OLD/Status' not in dpkg_params:
1894 die('Error: package %s is not known to dpkg.' % b)
1895 if dpkg_params['OLD/Status'] != 'install ok installed' :
1896 die('Error: package %s is not installed, status is %s.'
1897 % ( b , dpkg_params['OLD/Status'] ) )
1898 for a in params:
1899 if a[:3] == 'OLD' and a != 'OLD/Installed-Size' and a != 'OLD/Size':
1900 if a not in dpkg_params:
1901 die('Error parsing old control file , parameter %s not found' % a)
1902 elif params[a] != dpkg_params[a] :
1903 die( 'Error : in delta , '+a+' = ' +params[a] +\
1904 '\nin old/installed deb, '+a+' = ' +dpkg_params[a])
1905 del b,p #cannot delete 'a', python raise a SyntaxError
1906
1907 runtime['patchprogress']=5
1908
1909 ### some auxiliary routines, separated to make code more readable
1910
1911 def dpkg_L_faster(pa,ar,diversions):
1912 "Scan dpkg -L . 'diversions' must be prepared by scan_diversions() . Returns list of pairs of files ,and list of diverted files. "
1913 s=[]
1914 diverted=[]
1915 n='/var/lib/dpkg/info/'+pa+':'+ar+'.list'
1916 if not DPKG_MULTIARCH or not os.path.exists(n):
1917 n='/var/lib/dpkg/info/'+pa+'.list'
1918 f=open(n)
1919 while 1:
1920 a=f.readline()
1921 if not a: break
1922 a=de_n(a)
1923 if a in diversions:
1924 b,p= diversions[a]
1925 if p != pa:
1926 s.append((a,b))
1927 diverted.append(a)
1928 else:
1929 s.append((a,a))
1930 else: s.append((a,a))
1931 f.close()
1932 return s,diverted
1933
1934 def dpkg_L(pa,ar):
1935 "Scan dpkg -L . Currently unused, see previous function."
1936 #sys.stderr.write('INTERNAL WARNING: USING OBSOLETE dpkg_L\n')
1937 s=[]
1938 diverted=[]
1939 if DPKG_MULTIARCH:
1940 p=my_popen_read(['env', '-i', 'dpkg-query', '-L', pa+':'+ar])
1941 else:
1942 p=my_popen_read(['env', '-i', 'dpkg-query', '-L', pa])
1943 a=p.readline()
1944 while a:
1945 a=de_n(a)
1946 #support diversions
1947 if a[:26] == 'package diverts others to:':
1948 continue
1949 if s and a[:11] == 'diverted by' or a[:20] == 'locally diverted to:':
1950 orig,divert=s.pop()
1951 i = a.index(':')
1952 divert = a[i+2:]
1953 s.append( (orig,divert) )
1954 diverted.append(orig)
1955 else:
1956 s.append( (a,a) )
1957 a=p.readline()
1958 p.close()
1959 return s, diverted
1960
1961 def _symlink_data_tree(pa,ar,TD,diversions,runtime):
1962 localepurged=[]
1963 prelink_u_failed=[]
1964 file_triples=[]
1965 prelink_time = 0
1966 prelink_datasize = 0
1967 if diversions:
1968 s,diverted=dpkg_L_faster(pa,ar,diversions)
1969 else:
1970 s,diverted=dpkg_L(pa,ar)
1971 progressline=0
1972 progresslen=float(len(s))
1973 for orig,divert in s:
1974 progressline+=1
1975 progress=6.0 + 6.0 * float(progressline) / progresslen
1976 runtime['patchprogress']=progress
1977 if do_progress:
1978 sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname))
1979 if os.path.isfile(divert) and not os.path.islink(divert) :
1980 tmpcopy=TD+'OLD/DATA'+orig
1981 d=os.path.dirname(tmpcopy)
1982 if not os.path.exists(d):
1983 os.makedirs(d)
1984 #the following code idea was provided by [email protected]
1985 unprelink=False
1986 if HAVE_PRELINK :
1987 prelink_time -= time.time()
1988 is_elf, ei_class, ei_data, ei_osabi, e_type = elf_info(divert)
1989 #according to prelink-0.0.20090925/src/main.c
1990 unprelink = is_elf and e_type in ('ET_DYN','ET_EXEC')
1991 prelink_time += time.time()
1992 if unprelink:
1993 prelink_time -= time.time()
1994 prelink_datasize += os.path.getsize(divert)
1995 if VERBOSE > 3 :
1996 logger.debug(' copying/unprelinking '+divert+' to '+ tmpcopy)
1997 #unfortunately 'prelink -o' sometimes alters files, see http://bugs.debian.org/627932
1998 shutil.copy2(divert, tmpcopy)
1999 proc=subprocess.Popen(["/usr/sbin/prelink","-u",tmpcopy],stdin=open(os.devnull),\
2000 stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
2001 close_fds=True, **Popen_args)
2002 out=proc.stdout.read().strip()
2003 proc.wait()
2004 if proc.returncode:
2005 if not os.path.exists(tmpcopy):
2006 if VERBOSE > 4 : logger.debug(' (prelink failed, symlinking '+divert+' to '+tmpcopy+')')
2007 os.symlink(divert, tmpcopy)
2008 prelink_u_failed.append(de_bar(orig))
2009 unprelink=False
2010 elif VERBOSE > 4 : logger.debug(' (prelink failed, but file was copied)')
2011 thestat = os.statvfs(tmpcopy)
2012 if out[-39:] == 'does not have .gnu.prelink_undo section':
2013 if DEBUG > 1:
2014 logger.debug(' '+repr(out))
2015 elif (thestat.f_bsize * thestat.f_bavail / 1024) < 50000 :
2016 logger.warning('!!Prelink -u failed, it needs at least 50000KB of free disk space\n')
2017 prelink_u_failed.append(de_bar(orig))
2018 unprelink=False
2019 else:
2020 logger.warning('!!Prelink -u failed on %r : %r\n', tmpcopy, out)
2021 prelink_u_failed.append(de_bar(orig))
2022 unprelink=False
2023 prelink_time += time.time()
2024 else:
2025 if VERBOSE > 3 : logger.debug(' symlinking '+divert+' to '+a)
2026 os.symlink(divert, tmpcopy)
2027 if unprelink and FORENSIC:
2028 #unfortunately the script will delete the 'tmpcopy', so we hardlink it
2029 z=tempfile.mktemp(prefix=TD)
2030 os.link(tmpcopy,z)
2031 file_triples.append((orig,divert,z))
2032 else:
2033 file_triples.append((orig,divert,None))
2034 elif not os.path.exists(divert) and os.path.islink(divert):
2035 file_triples.append((orig,divert,None))
2036 if VERBOSE > 1 : logger.debug(' Broken symlink? %r',divert)
2037 elif not os.path.exists(divert):
2038 file_triples.append((orig,divert,None))
2039 if VERBOSE : logger.debug(' Disappeared file? %r',divert)
2040 for z in ('locale','man','gnome/help','omf','doc/kde/HTML'):
2041 w='/usr/share/'+z
2042 if orig[:len(w)] == w:
2043 localepurged.append(de_bar(orig))
2044 else:
2045 file_triples.append((orig,divert,None))
2046 if VERBOSE > 3 : logger.debug(' not symlinking %r to %r',divert,orig)
2047 return file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize
2048
2049 def chmod_add(n,m):
2050 "same as 'chmod ...+... n '"
2051 om=S_IMODE(os.stat(n)[ST_MODE])
2052 nm=om | m
2053 if nm != om :
2054 if VERBOSE > 2 : logger.debug(' Performing chmod %s %s %s',n,oct(om),oct(nm))
2055 os.chmod(n,nm)
2056
2057 def _fix_data_tree_(TD):
2058 for (dirpath, dirnames, filenames) in os.walk(TD+'OLD/DATA'):
2059 chmod_add(dirpath, S_IRUSR | S_IWUSR| S_IXUSR )
2060 for i in filenames:
2061 i=os.path.join(dirpath,i)
2062 if os.path.isfile(i):
2063 chmod_add(i, S_IRUSR | S_IWUSR )
2064 for i in dirnames:
2065 i=os.path.join(dirpath,i)
2066 chmod_add(i, S_IRUSR | S_IWUSR| S_IXUSR )
2067
2068 #initialize, just in case
2069 control_file_triples=[]
2070 file_triples=[]
2071 localepurged=[]
2072 prelink_u_failed=[]
2073 diverted=[]
2074 prelink_time=0
2075 prelink_datasize=0
2076
2077 ###see into parameters: the patch may need extra info and data
2078
2079 runtime['patchprogress']=6
2080
2081 prelink_time=None
2082
2083 for a in params:
2084 if 'needs-old' == a:
2085 if os.path.isdir(olddeb):
2086 die('This patch needs the old version Debian package')
2087 elif 'old-data-tree' == a :
2088 os.mkdir(TD+'/OLD/DATA')
2089 if os.path.isdir(olddeb):
2090 file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize=\
2091 _symlink_data_tree(params['OLD/Package'],params['OLD/Architecture'],TD,diversions,runtime)
2092 else:
2093 ar_list_old= list_ar(TD+'OLD.file')
2094 if b'data.tar.bz2' in ar_list_old:
2095 system(('ar','p',TD+'OLD.file','data.tar.bz2','|','tar','-xp','--bzip2','-f','-','-C',TD+'OLD/DATA'), TD)
2096 elif b'data.tar.gz' in ar_list_old:
2097 system(('ar','p',TD+'OLD.file','data.tar.gz','|','tar','-xp','-z','-f','-','-C',TD+'OLD/DATA'), TD)
2098 elif b'data.tar.lzma' in ar_list_old:
2099 if not os.path.exists('/usr/bin/lzma'):
2100 raise DebDeltaError('This patch needs lzma. Please install the Debian package "lzma".',retriable=True)
2101 system(('ar','p',TD+'OLD.file','data.tar.lzma','|','unlzma','-c','|','tar','-xpf','-','-C',TD+'OLD/DATA'), TD)
2102 elif b'data.tar.xz' in ar_list_old:
2103 if not os.path.exists('/usr/bin/xz'):
2104 raise DebDeltaError('This patch needs xz. Please install the Debian package "xz-utils".',retriable=True)
2105 system(('ar','p',TD+'OLD.file','data.tar.xz','|','unxz','-c','|','tar','-xpf','-','-C',TD+'OLD/DATA'), TD)
2106 else: assert(0)
2107 _fix_data_tree_(TD)
2108 elif 'old-control-tree' == a:
2109 if os.path.isdir(olddeb):
2110 if not os.path.isdir(TD+'OLD/CONTROL'):
2111 os.mkdir(TD+'OLD/CONTROL')
2112 p=params['OLD/Package']
2113 a=params['OLD/Architecture']
2114 for b in dpkg_keeps_controls :
2115 z='/var/lib/dpkg/info/' + p + ':' + a + '.'+b
2116 if not DPKG_MULTIARCH or not os.path.exists(z):
2117 z='/var/lib/dpkg/info/' + p +'.'+b
2118 if os.path.exists(z):
2119 os.symlink(z,TD+'OLD/CONTROL/'+b)
2120 control_file_triples.append((b,z,None))
2121 del z,p #cannot delete 'a', python raise a SyntaxError
2122 #else... we always unpack the control of a .deb
2123 elif 'needs-xdelta3' == a:
2124 if not os.path.exists('/usr/bin/xdelta3'):
2125 raise DebDeltaError('This patch needs xdelta3. Please install the Debian package "xdelta3".',retriable=True)
2126 elif 'needs-xdelta3-fifo' == a:
2127 # not doing a specific check, I am using debian/control Conflicts
2128 if not os.path.exists('/usr/bin/xdelta3'):
2129 raise DebDeltaError('This patch needs xdelta3, at least version 3.0y. Please install the Debian package "xdelta3".',retriable=True)
2130 elif 'needs-xdelta' == a:
2131 if not os.path.exists('/usr/bin/xdelta'):
2132 raise DebDeltaError('This patch needs xdelta. Please install the Debian package "xdelta".',retriable=True)
2133 elif 'needs-bsdiff' == a:
2134 if not os.path.exists('/usr/bin/bsdiff'):
2135 raise DebDeltaError('This patch needs bsdiff. Please install the Debian package "bsdiff".',retriable=True)
2136 elif 'needs-lzma' == a:
2137 if not os.path.exists('/usr/bin/lzma'):
2138 raise DebDeltaError('This patch needs lzma. Please install the Debian package "lzma".',retriable=True)
2139 elif 'needs-xz' == a:
2140 if not os.path.exists('/usr/bin/xz'):
2141 raise DebDeltaError('This patch needs xz. Please install the Debian package "xz-utils".',retriable=True)
2142 elif 'needs-minibzip2' == a:
2143 pass #its your lucky day
2144 elif a[:6] == 'needs-':
2145 raise DebDeltaError('patch says "'+a+"' and this is unsupported. Get a newer debdelta.",retriable=True)
2146 elif params[a] == True:
2147 logger.warning('WARNING patch says %r and this is unsupported. Get a newer debdelta.',a)
2148
2149 if localepurged and not DEBUG:
2150 #actually we cannot be 100% sure that the delta really needs those files, but it is quite plausible
2151 raise DebDeltaError('Error, '+str(len(localepurged))+' locale files are absent.')
2152
2153 runtime['patchprogress']=12
2154
2155 script_time = - time.time()
2156
2157 this_deb_format=DEB_FORMAT
2158 if this_deb_format != 'deb' and 'NEW/data.tar' not in params:
2159 #this patch does not support streaming
2160 logger.warning('Warning, created as standard deb: '+newdeb)
2161 this_deb_format='deb'
2162
2163 if this_deb_format == 'preunpacked' : #need a fifo
2164 os.mkfifo(TD+'data.pipe')
2165
2166 temp_err_name_fd, temp_err_name = tempfile.mkstemp(prefix='debdeltaE')
2167 temp_name_fd, temp_name = tempfile.mkstemp(prefix='debdeltaO')
2168 cmd=[SHELL,'-e','PATCH/patch.sh']
2169 if DEBUG > 2 :
2170 cmd=[SHELL,'-evx','PATCH/patch.sh']
2171 if this_deb_format == 'unzipped':
2172 cmd+=['unzipped']
2173 elif this_deb_format == 'preunpacked':
2174 cmd+=['piped']
2175
2176 env={'PATH':os.getenv('PATH')}
2177 F=subprocess.Popen(cmd, cwd=TD,
2178 bufsize=4096,close_fds=True,
2179 stdin=open(os.devnull),env=env,
2180 stderr=temp_err_name_fd, stdout=temp_name_fd)
2181
2182 ### data used by the preunpacked method
2183 data_md5=None # md5 of uncompressed data.tar
2184 tar_status=[] # should be [True] if 'preunpacked' went fine
2185 md5_status=[] # idem
2186 #this list contains tuples of (unpacked_temporary_filename, real_filename, owner, group, tartype, mode, mtime, linkname)
2187 preunpacked_filelist=[]
2188 def do_cleanup():
2189 for a in preunpacked_filelist:
2190 pass
2191 ##CHEAT we are not (yet) writing anything to disk
2192 #if a[0]: os.unlink(a[0])
2193
2194 if this_deb_format == 'preunpacked' : #do progress reporting and unpacking in filesystem
2195 def do_extension():
2196 return '_'+str(random.randint(1,9999)).rjust(4,'0')+'_debdelta_preunpacked'
2197
2198 def do_pipe_md5(i,o,rm,ms,ts):
2199 try:
2200 a=i.read(1024)
2201 while a and (ts == [] or ts == [True]) :
2202 rm.update(a)
2203 o.write(a)
2204 a=i.read(1024)
2205 #TODO implement progress reporting here as well
2206 o.close()
2207 ms.append(True)
2208 except:
2209 import sys
2210 ms.append(sys.exc_info())
2211 if DEBUG: logger.debug(' do_pipe_md5 crashed: '+repr(ms))
2212
2213 def do_tar(i,fl,s):
2214 try:
2215 dt=tarfile.open(mode='r|',fileobj=i)
2216 for tarmember in dt:
2217 if tarmember.isreg():
2218 n='/'+tarmember.name+do_extension()
2219 while os.path.exists(n) : #wont overwrite existing stuff, never ever
2220 n='/'+tarmember.name+do_extension()
2221 else:
2222 n=''
2223 a=(n, tarmember.name, tarmember.uname, tarmember.gname,\
2224 tarmember.type, tarmember.mode, tarmember.mtime, tarmember.linkname)
2225 fl.append(a)
2226 if n:
2227 pass
2228 ##CHEAT we are not writing anything today!
2229 #datatar.extract(tarmember,path=n)
2230 #successfully untarred!
2231 s.append(True)
2232 except:#catch problems such as "out of disk space" or corrupted data
2233 import sys
2234 s.append(sys.exc_info())
2235 if DEBUG: logger.debug(' do_tar crashed: '+repr(s))
2236 #flush input (note that do_pipe_md5 will soon stop writing) no it seems useless #i.read()
2237
2238 try:
2239 datapipe=open(TD+'data.pipe')
2240 rolling_md5=hashlib.md5()
2241 (piper,pipew)=os.pipe()
2242 md5_thread=threading.Thread(target=do_pipe_md5,
2243 args=(open(TD+'data.pipe'),os.fdopen(pipew,'w'),rolling_md5,\
2244 md5_status,tar_status))
2245 tar_thread=threading.Thread(target=do_tar,
2246 args=(os.fdopen(piper),preunpacked_filelist,tar_status))
2247 #yeah maybe using two threads is more complex than strictly needed
2248 md5_thread.daemon=True
2249 md5_thread.start()
2250 tar_thread.daemon=True
2251 tar_thread.start()
2252 #join back the md5
2253 md5_thread.join()
2254 data_md5=rolling_md5.hexdigest()
2255 if md5_status == [True] and params['NEW/data.tar'][:32] == data_md5:
2256 tar_thread.join()
2257 if tar_status == [True]:
2258 F.wait()
2259 #write data_list
2260 a=open(TD+'data_list','w')
2261 a.write('Files:\n')
2262 for (unpacked_temporary_filename, real_filename, owner,\
2263 group, tartype, mode, mtime, linkname) in preunpacked_filelist:
2264 ## to convert 'tar' type into 'ls' type
2265 if tartype >= '0' and tartype <= '6' :
2266 tartype="-hlcbdp"[ord(tartype) - ord('0')]
2267 else:
2268 tartype='?'
2269 logger.debug('WARNING unsupported tar type '+repr(tartype)+' for: '+repr(real_filename))
2270 #if tartype < '0' or tartype > '6' :
2271 # tartype='?'
2272 # print 'WARNING unsupported tar type '+repr(tartype)+' for: '+repr(real_filename)
2273 mode=oct(mode).rjust(4,'0')
2274 a.write(' %s %s %s %s %s\n %s\n %s\n %s\n' % (tartype,mode,owner,group,mtime,\
2275 unpacked_temporary_filename,real_filename,linkname))
2276 a.close()
2277 #append it into deb
2278 system(['ar','q','NEW.file', 'data_list'], TD)
2279 except:
2280 do_cleanup()
2281 raise
2282 else: #progress reporting for deb_format != 'preunpacked'
2283 runtime['patchprogress']=12
2284 if 'NEW/Size' in params and do_progress:
2285 NEW_size=int(params['NEW/Size'])
2286 while None == F.poll():
2287 if os.path.exists(TD+'NEW.file'):
2288 a=os.path.getsize(TD+'NEW.file')
2289 progress=(int(12.0 + 84.0 * a / NEW_size))
2290 else:
2291 progress=12
2292 runtime['patchprogress']=progress
2293 time.sleep(0.1)
2294 if do_progress:
2295 sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname))
2296 F.wait()
2297 if do_progress and terminalcolumns: #clean up
2298 sys.stderr.write(' ' * terminalcolumns + '\r')
2299 ret=F.returncode
2300 os.close(temp_err_name_fd)
2301 os.close(temp_name_fd)
2302
2303 script_time += time.time() # for --format='preunpacked' this time also include data MD5
2304 runtime['patchprogress']=97
2305
2306 #helper for debugging
2307 def tempos(f):
2308 if os.path.getsize(temp_name):
2309 f.append(temp_name)
2310 if os.path.getsize(temp_err_name):
2311 f.append(temp_err_name)
2312
2313 if not FORENSIC:
2314 def fore():
2315 return None
2316 elif not os.path.isdir(olddeb):
2317 def fore():
2318 f=[delta,olddeb]
2319 tempos(f)
2320 return f
2321 else:
2322 def fore():
2323 temp_fore_name=''
2324 f=[]
2325 tempos(f)
2326 try:
2327 (temp_fd,temp_fore_name) = tempfile.mkstemp(prefix="debforensic_"+params['NEW/Package']+"_")
2328 temp_file=os.fdopen(temp_fd,'w')
2329 temp_file.write('Delta: '+delta+'\n')
2330 temp_file.write('DeltaSHA1: ' + sha1_hash_file(delta) + '\n')
2331 temp_file.write('LocalePurgedFilesN: '+str(len(localepurged))+'\n')
2332 temp_file.write('PrelinkUFailedN: '+str(len(prelink_u_failed))+'\n')
2333 if ret:
2334 temp_file.write('PatchExitCode: '+str(ret)+'\n')
2335 forensics_rfc(temp_file,params,False,control_file_triples,file_triples,conf_files,
2336 diverted,diversions,localepurged,prelink_u_failed)
2337 #copy short content here and remove from list
2338 for i in copy(f):
2339 if os.path.getsize(i) < 2000:
2340 f.remove(i)
2341 temp_file.write('PatchLogFile_'+str(i)+'_content:\n')
2342 for ll in open(i):
2343 temp_file.write(' '+repr(ll)+'\n')
2344 else: temp_file.write('PatchLogFileIs: '+str(i)+'\n')
2345 temp_file.close()
2346 except OSError: #Exception,s:
2347 die('!!While creating forensic '+temp_fore_name+' error:'+str(s)+'\n')
2348 f.append(temp_fore_name)
2349 return f
2350
2351 if ret:
2352 if localepurged:
2353 raise DebDeltaError('"debdelta" is incompatible with "localepurge".')
2354 else:
2355 f=fore()
2356 raise DebDeltaError('error in patch.sh.',logs=f)
2357
2358 #then we check for the conformance
2359 if this_deb_format == 'deb':
2360 if 'NEW/Size' in params:
2361 newdebsize = os.stat(TD+'NEW.file')[ST_SIZE]
2362 if newdebsize != int(params['NEW/Size']):
2363 f=fore()
2364 raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size'],logs=f)
2365
2366 if DO_MD5:
2367 if 'NEW/MD5sum' in params:
2368 if VERBOSE > 1 : logger.debug(' verifying MD5 for %r',os.path.basename(newdeb or delta))
2369 m= compute_md5(open(TD+'NEW.file','rb'))
2370 if params['NEW/MD5sum'] != m :
2371 f=fore()
2372 raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) , logs=f)
2373 else: logger.warning(' Warning! no MD5 was verified for %r',os.path.basename(newdeb or delta))
2374 elif this_deb_format == 'unzipped' :
2375 if DO_MD5:
2376 c,b,z=untar_control_in_deb(list_ar(TD+'NEW.file'))
2377 p=PopenPipe(['ar','p',(TD+'NEW.file'),c,'|']+z, stdout=subprocess.PIPE)
2378 m=compute_md5(p.stdout)
2379 p.wait()
2380 if params['NEW/control.tar'][:32] != m:
2381 f=fore()
2382 raise DebDeltaError('MD5 mismatch for control.tar' , logs=f)
2383 p=subprocess.Popen(['ar','p',(TD+'NEW.file'),'data.tar'], stdout=subprocess.PIPE)
2384 m=compute_md5(p.stdout)
2385 p.wait()
2386 if params['NEW/data.tar'][:32] != m:
2387 f=fore()
2388 raise DebDeltaError('MD5 mismatch for data.tar', logs=f)
2389 elif this_deb_format == 'preunpacked' :
2390 if tar_status != [True]:
2391 f=fore()
2392 do_cleanup()
2393 raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1]), logs=f) #todo format me better
2394 if md5_status != [True]:
2395 f=fore()
2396 do_cleanup()
2397 raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1]), logs=f) #todo format me better
2398 #if DO_MD5: #actually we always do MD5
2399 m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'),
2400 stdout=subprocess.PIPE,shell=True).stdout)
2401 if params['NEW/control.tar'][:32] != m:
2402 f=fore()
2403 do_cleanup()
2404 raise DebDeltaError('MD5 mismatch for control.tar', logs=f)
2405 if params['NEW/data.tar'][:32] != data_md5:
2406 f=fore()
2407 do_cleanup()
2408 raise DebDeltaError('MD5 mismatch for data.tar', logs=f)
2409 else: assert('unimplemented'=='')
2410
2411 os.unlink(temp_name)
2412 os.unlink(temp_err_name)
2413
2414 runtime['patchprogress']=99
2415
2416 if newdeb:
2417 shutil.move(TD+'NEW.file',newdeb)
2418
2419 end_sec = time.time()
2420 elaps=(end_sec - start_sec)
2421
2422 if VERBOSE :
2423 if newdeb:
2424 debsize = os.stat(newdeb)[ST_SIZE]
2425 else:
2426 debsize = os.stat(olddeb)[ST_SIZE]
2427 #this printout uses kibibytes, and not SizeToStr, to ease statistics
2428 out = ' ' + _('Patching done, time %(time).2fsec, speed %(speed)dk/sec') % \
2429 {'time' : elaps, 'speed' : (debsize / 1024 / (elaps+.001))}
2430 out += ' ' + _('(script %(time).2fsec %(speed)dk/sec)') % \
2431 {'time' : script_time, 'speed' : (debsize / 1024 / (script_time+.001))}
2432 if prelink_time != None and prelink_time > 0 :
2433 prelink_datasize=prelink_datasize/1024
2434 out += _('(prelink %(time).2fsec, %(size)dk, %(speed)dk/s)') % \
2435 {'time' : prelink_time, 'size' : prelink_datasize, 'speed' : prelink_datasize / (prelink_time + .001)}
2436 out += _('(unaccounted %.2fsec)') % (elaps-prelink_time-script_time)
2437 else:
2438 out += _('(unaccounted %.2fsec)') % (elaps-script_time)
2439 logger.info(out)
2440 ## this is useless, if 'debpatch' it is in the command line, if 'debdelta-upgrade'
2441 ## it is printed elsewhere
2442 #if newdeb != None:
2443 # print 'result: '+os.path.basename(newdeb),
2444 return (newdeb,elaps)
2445
2446 ##################################################### compute delta
2447 def do_delta(olddeb,newdeb,delta):
2448 T=None
2449 try:
2450 T=tempo()
2451 if os.path.exists(delta+'_tmp_') :
2452 os.unlink(delta+'_tmp_')
2453 r=do_delta_(olddeb,newdeb,delta+'_tmp_',TD=T)
2454 (deltatmp, percent, elaps, info, gpg_hashes) = r
2455 info_hashes=append_info(deltatmp,info)
2456 if DO_GPG:
2457 gpg_hashes['info']=info_hashes
2458 sign_delta(deltatmp,gpg_hashes)
2459 if os.path.exists(delta) :
2460 os.rename(delta,delta+'~')
2461 os.rename(deltatmp,delta)
2462 except:
2463 if delta and os.path.exists(delta):
2464 os.unlink(delta)
2465 if delta and os.path.exists(delta+'_tmp_'):
2466 os.unlink(delta+'_tmp_')
2467 if T : rmtree(T)
2468 raise
2469 else:
2470 if T : rmtree(T)
2471 return r
2472
2473 def do_delta_(olddeb, newdeb, delta, TD, forensic_file=None, info=[]):
2474 """This function creates a delta. The delta is 'ar' archive (see 'man ar').
2475 The delta contains data, a script, and optional gpg signatures.
2476 The script recreates the new deb. Note that the deb is (again) an 'ar' archive,
2477 and has multiple members.
2478 Simple deb members are managed directly.
2479 The complex members are 'data.tar.xxx' and 'control.tar.gz' ;
2480 these are studied in the delta_tar() function, that in turn uses
2481 the Script() class.
2482 (Note that there is just one object instanced from Script(), in variable 'script').
2483 To recreate one of the complex members, there are two main steps in the script:
2484 (1) rebuild the member, uncompressed
2485 (2) recompress the member
2486 The part (2) is managed by script.zip_piped().
2487 The part (1) is more complex:
2488 (1a) there is a first subshell where data from the old version of the deb
2489 are piped to stdout (moreover some gzipped files may be transparently
2490 unzipped, delta-ed and re-gzipped, see delta_gzipped_files)
2491 (1b) the former (1a) is "piped" into the delta backend, to transform the old data into new.
2492 If the delta-backend is 'bsdiff' then the above process (1a,b) is repeated
2493 in chunks (indeed bsdiff cannot manage large files w/o exausting all your memory!)
2494 All the (chunks of) steps (1a,b) are in a subshell, and its result stdout is piped in (2).
2495 """
2496 if TD[-1] != '/':
2497 TD = TD + '/'
2498
2499 import fnmatch
2500
2501 start_sec = time.time()
2502
2503 #I do not like global variables but I do not know of another solution
2504 global bsdiff_time, bsdiff_datasize
2505 bsdiff_time = 0
2506 bsdiff_datasize = 0
2507
2508 olddeb=abspath(olddeb)
2509 check_deb(olddeb)
2510 os.symlink(olddeb,TD+'/OLD.file')
2511 olddebsize = os.stat(olddeb)[ST_SIZE]
2512
2513 newdeb=abspath(newdeb)
2514 check_deb(newdeb)
2515 os.symlink(newdeb,TD+'/NEW.file')
2516 newdebsize = os.stat(newdeb)[ST_SIZE]
2517
2518 #process all contents of old vs new .deb
2519 ar_list_old= list_ar(TD+'OLD.file')
2520 ar_list_new= list_ar(TD+'NEW.file')
2521
2522 free=freespace(TD)
2523 if free and free < newdebsize :
2524 raise DebDeltaError('Error: not enough disk space in '+TD, True)
2525
2526 delta=abspath(delta)
2527
2528 #generater for numbered files
2529 def a_numb_file_gen():
2530 deltacount = 0
2531 while 1:
2532 yield str(deltacount)
2533 deltacount+=1
2534 a_numb_file=a_numb_file_gen()
2535 a_numb_patch=a_numb_file_gen()
2536
2537 ####################### class Script ########################
2538 class Script:
2539 """This class helps create the script 'patch.sh' that is the core of the delta.
2540 The script recreates the new deb. See documentation of do_delta_() for details.
2541 """
2542 def __init__(self, delta_uses_infifo):
2543 #start writing script
2544 self.fd=open(TD+'PATCH/patch.sh','w')
2545 self.fd.write('#!/bin/bash -e\n')
2546 self.member=None
2547 self.current_chunk_name=None
2548 self.delta_uses_infifo=delta_uses_infifo
2549 if delta_uses_infifo : #create the fifo as input for xdelta3
2550 self.the_fifo=next(a_numb_file)
2551 self.fd.write("mkfifo %s\n" % self.the_fifo)
2552 else:
2553 self.the_fifo=None
2554 #this is used when recompressing data.tar.zx
2555 self.xz_parameters=None
2556 #this is used when recompressing data.tar.gz
2557 self.gz_command=None
2558
2559 def write(self,s):
2560 "verbatim write in the script"
2561 self.fd.write(s)
2562 def close(self):
2563 if self.the_fifo:
2564 self.fd.write("rm %s\n" % self.the_fifo)
2565 self.fd.close()
2566
2567 def zip(self, n, cn, newhead=None):
2568 """inverts the unzip() function ; optionally, forces .gz header (to fight changes in libz)
2569 This is obsolete, not efficient, left as a compatibility layer."""
2570 self.fd.write('cat "'+n+'" | ')
2571 self.zip_piped(cn, newhead)
2572 self.fd.write(" > '"+n+cn+"' && rm '"+n+"'\n")
2573
2574 def zip_piped(self, cn, newhead=None):
2575 "inverts the unzip() function, with piped behaviour"
2576 if cn == '.gz' :
2577 cmd=str.join(' ',self.gz_command)
2578 if newhead:
2579 s=prepare_for_echo(newhead)
2580 self.fd.write("($E '"+ s +"' && "+cmd+" | tail -c +"+str(len(newhead)+1)+')')
2581 else:
2582 self.fd.write(cmd)
2583 elif cn == '.bz2' :
2584 info_append('needs-minibzip2')
2585 self.fd.write('./minibzip2 -9')
2586 elif cn == '.lzma' :
2587 info_append('needs-lzma')
2588 self.fd.write('lzma -9')
2589 elif cn == '.xz' :
2590 info_append('needs-xz')
2591 if self.xz_parameters==None:
2592 self.fd.write('xz -c')
2593 else:
2594 self.fd.write('xz -c '+self.xz_parameters)
2595 else: assert(0)
2596
2597 def start_member(self, ar_line, newname, extrachar):
2598 "start a new 'ar' member"
2599 self.member=newname
2600 self.ar_line=ar_line
2601 self.extrachar=extrachar
2602 assert(self.current_chunk_name==None)
2603 self.fd.write('{\n')
2604
2605 def end_member(self):
2606 assert(self.member)
2607 self.member=None
2608 self.fd.write('}\n')
2609
2610 def start_rebuilding(self):
2611 "starts the first part of the delta/recompressing pipe (for a 'ar' member)"
2612 self.fd.write('(')
2613 def end_rebuilding(self):
2614 "ends the first part of the delta/recompressing pipe"
2615 self.fd.write(')|') #pipe the delta/recompressing pipe
2616
2617 def recompressing(self,new_filename,new_filename_ext,new_file_zip_head):
2618 self.fd.write('(')
2619 append_NEW_file(self.ar_line)
2620 script.zip_piped(new_filename_ext,new_file_zip_head)
2621 self.fd.write(' >> NEW.file\n') #end delta tar
2622 if self.extrachar :
2623 append_NEW_file(self.extrachar)
2624 self.fd.write(')\n')
2625
2626 def recompressing_by_arg(self,new_filename,new_filename_ext,new_file_zip_head,new_file_size):
2627 "flexible recompressing for data.tar , depending on first argument passed to the script"
2628 self.fd.write('( if test "$1" = "" \n then\n')
2629 self.recompressing(new_filename,new_filename_ext,new_file_zip_head)
2630 self.fd.write('elif test "$1" = "unzipped" \n then\n')
2631 #http://en.wikipedia.org/wiki/Ar_(Unix)
2632 assert(new_filename == 'NEW/data.tar')
2633 ar_line_unzipped=b'data.tar'.ljust(16)+self.ar_line[16:48]+str2bytes(str(new_file_size)).ljust(10)+b'`\n'
2634 append_NEW_file(ar_line_unzipped) #there is no extra char, tar is 512b blocks
2635 self.fd.write('cat >> NEW.file \n elif test "$1" = "piped" \n then cat >> data.pipe \n fi )\n')
2636
2637 def start_chunk(self, current_chunk_name):
2638 "start the pipe to create the chunk. The chunk is always piped"
2639 self.fd.write('(')
2640 self.current_chunk_name=current_chunk_name
2641 def end_chunk(self, current_chunk_name):
2642 """this ends the 'data part' of a chunk, and writes the old data somewhere;
2643 a successive script code (generated by delta_files() )
2644 will then delta it to transform old data into new data."""
2645 assert(self.current_chunk_name==current_chunk_name)
2646 if self.the_fifo:
2647 self.fd.write(') > '+self.the_fifo+'&\n') #write to fifo, background
2648 else:
2649 self.fd.write(') > '+current_chunk_name+'\n') #write chunk
2650 self.current_chunk_name=None
2651
2652 def md5_check_file(self, n, md5=None):
2653 "add a md5 check in the script (this is done only if a lot -d are passed on cmdline)"
2654 if md5==None:
2655 assert(os.path.isfile(TD+n))
2656 md5=compute_md5(TD+n)
2657 logger.debug(" adding extra MD5 for %r",n)
2658 self.fd.write('if ! echo "%s %s" | md5sum -c --quiet ; then echo "%s is currupt!"; exit 1; fi\n' % (md5,n,n))
2659
2660 ####################### end of class Script ########################
2661
2662 ##### unpack control.tar.gz, scan control, write parameters
2663 #
2664 def info_append(s):
2665 "smart appending that avoids duplicate entries"
2666 if s not in info:
2667 info.append(s)
2668
2669 for o,l in ('OLD',ar_list_old) , ('NEW',ar_list_new):
2670 os.mkdir(TD+o+'/CONTROL')
2671 #unpack control.tar.gz
2672 f=None
2673 for a in l :
2674 if a[:12] == b'control.tar.':
2675 f=a
2676 b={b'.gz':'-z', b'.xz':'-J', b'.bz2':'-j'}.get(f[11:])
2677 assert b != None
2678 system(('ar','p',TD+o+'.file',f,'|','tar',b,'-xf','-','-C',TD+o+'/CONTROL'),TD)
2679 ## scan control
2680 p=open(TD+'/'+o+'/CONTROL/control')
2681 s=[]
2682 scan_control(p,params=None,prefix=o,info=s)
2683 p.close()
2684 if VERBOSE : logger.debug(' '+o+': '+str.join(' ',[o[4:] for o in s]))
2685 info = info + s
2686 info.append('OLD/Size: '+str(olddebsize))
2687 info.append('NEW/Size: '+str(newdebsize))
2688 params=info_2_db(info)
2689
2690 #scan debdelta.conf to find any special requirement
2691 debdelta_conf=ConfigParser()
2692 debdelta_conf.read(['/etc/debdelta/debdelta.conf', expanduser('~/.debdelta/debdelta.conf') ])
2693
2694 debdelta_conf_skip=[]
2695 for s in debdelta_conf.sections():
2696 if fnmatch.fnmatch(params['OLD/Package'],s):
2697 opt=debdelta_conf.options(s)
2698 if 'skip' in opt:
2699 debdelta_conf_skip += debdelta_conf.get(s,'skip').split(';')
2700 break
2701
2702 if VERBOSE > 1 : logger.debug(' debdelta.conf says we will skip: %r', debdelta_conf_skip)
2703
2704 gpg_hashes = {}
2705
2706 if DO_MD5 :
2707 # compute a MD5 of NEW deb
2708 newdeb_md5sum=compute_md5(TD+'NEW.file')
2709 info.append('NEW/MD5sum: '+ newdeb_md5sum)
2710 else:
2711 newdeb_md5sum=None
2712
2713 if NEEDSOLD :
2714 #this delta needs the old deb
2715 info.append('needs-old')
2716 else:
2717 info.append('old-data-tree')
2718 info.append('old-control-tree')
2719
2720 #do we use a fifo as input for xdelta3
2721 delta_uses_infifo = ('xdelta3-fifo' not in DISABLED_FEATURES) and (USE_DELTA_ALGO == 'xdelta3')
2722
2723 #Note that there is just one object instanced from class Script()
2724 script=Script(delta_uses_infifo)
2725
2726 a=USE_DELTA_ALGO
2727 if a == 'xdelta-bzip':
2728 a='xdelta'
2729 if not os.path.exists('/usr/bin/'+a):
2730 raise DebDeltaError('please install the package "'+a+'".', retriable=True)
2731 if delta_uses_infifo:
2732 info.append('needs-xdelta3-fifo')
2733 else:
2734 info.append('needs-'+a)
2735 del a
2736
2737 #### check for disk space
2738 if 'NEW/Installed-Size' in params and 'OLD/Installed-Size' in params:
2739 free=freespace(TD)
2740 instsize=int(params['NEW/Installed-Size']) + int(params['OLD/Installed-Size'])
2741 if free and free < ( instsize * 1024 + + 2**23 + MAXMEMORY / 6 ) :
2742 raise DebDeltaError(' Not enough disk space (%dkB) for creating delta (needs %dkB).' % \
2743 ( int(free/1024) , instsize ) , True )
2744
2745
2746 ############# check for conffiles
2747 a=TD+'/OLD/CONTROL/conffiles'
2748 if os.path.exists(a):
2749 p=open(a)
2750 #files do not have leading /
2751 old_conffiles=[ de_bar(a) for a in p.read().split('\n') if a]
2752 p.close()
2753 else:
2754 old_conffiles=[]
2755
2756 ## a=TD+'/OLD/CONTROL/list'
2757 ## if os.path.exists(a):
2758 ## p=open(a)
2759 ## for a in p:
2760 ## a=de_bar(de_n(a))
2761 ## for j in debdelta_conf_skip:
2762 ## if fnmatch(a,j):
2763 ## old_conffiles.append(a) #OK, this abuses the name of the var a bit
2764 ## print ' REPR skip ',repr(a)
2765 ## else:
2766 ## print ' The old debian package ',olddeb,' does not contain a file list?!?'
2767
2768 def shell_not_allowed(name):
2769 "Strings that I do not trust to inject into the shell script; maybe I am a tad too paranoid..."
2770 #FIXME should use it , by properly quoting for the shell script
2771 return '"' in name or "'" in name or '\\' in name or '`' in name
2772
2773 # uses MD5 to detect identical files (even when renamed)
2774 def scan_md5(n):
2775 md5={}
2776 f=open(n)
2777 a=de_n(f.readline())
2778 while a:
2779 m , n = a[:32] , de_bar( a[34:] )
2780 md5[n]=m
2781 a=de_n(f.readline())
2782 f.close()
2783 return md5
2784
2785
2786 new_md5=None
2787 if os.path.exists(TD+'/NEW/CONTROL/md5sums'):
2788 new_md5=scan_md5(TD+'/NEW/CONTROL/md5sums')
2789
2790 old_md5=None
2791 if os.path.exists(TD+'/OLD/CONTROL/md5sums') :
2792 old_md5=scan_md5(TD+'/OLD/CONTROL/md5sums')
2793
2794 ############### some routines to prepare delta of two files
2795
2796 def patch_append(f):
2797 #possibly GPG
2798 if DO_GPG:
2799 gpg_hashes[f] = _compute_hashes_(TD+'PATCH/'+f)
2800 if VERBOSE > 2 :
2801 a=os.stat(TD+'PATCH/'+f)[ST_SIZE]
2802 logger.debug(' appending %r of size %d to delta, %3.2f%% of new .deb',f,a,( a * 100. / newdebsize ))
2803 system(['ar','qSc', delta,f], TD+'/PATCH')
2804 unlink(TD+'PATCH/'+f)
2805
2806 def verbatim(f):
2807 pp=next(a_numb_patch)
2808 p = 'PATCH/'+pp
2809 if VERBOSE > 1 : logger.debug(' including %r verbatim in patch', name)
2810 os.rename(TD+f,TD+p)
2811 patch_append(pp)
2812 return p
2813
2814 def unzip(f):
2815 c=''
2816 if f[-3:] == '.gz' :
2817 system(('gunzip',f),TD)
2818 f=f[:-3]
2819 c='.gz'
2820 elif f[-4:] == '.bz2' :
2821 system(('bunzip2',f),TD)
2822 f=f[:-4]
2823 c='.bz2'
2824 elif f[-5:] == '.lzma' :
2825 info_append('needs-lzma')
2826 system(('unlzma',f),TD)
2827 f=f[:-5]
2828 c='.lzma'
2829 elif f[-3:] == '.xz' :
2830 info_append('needs-xz')
2831 system(('unxz',f),TD)
2832 f=f[:-3]
2833 c='.xz'
2834 else: raise NotImplementedError(' dont know how to decompress '+repr(f))
2835 return (f,c)
2836
2837
2838
2839 def delta_files__(o, n, p, algo, outpiped, infifo):
2840 "delta of file 'o' to 'n' using/producing patch 'p' ; xdelta3 can also pipe"
2841 this_delta_outpiped=False
2842 #bdiff
2843 #http://www.webalice.it/g_pochini/bdiff/
2844 if algo == 'bdiff':
2845 system(('~/debdelta/bdiff-1.0.5/bdiff','-q','-nooldmd5','-nonewmd5','-d',o,n,p),TD)
2846 script.write('~/debdelta/bdiff-1.0.5/bdiff -p '+o+' '+p+' '+n+'\n')
2847 #zdelta
2848 #http://cis.poly.edu/zdelta/
2849 elif algo == 'zdelta':
2850 system(('~/debdelta/zdelta-2.1/zdc',o,n,p),TD)
2851 script.write('~/debdelta/zdelta-2.1/zdu '+o+' '+p+' '+n+'\n')
2852 #bdelta
2853 #http://deltup.sf.net
2854 elif algo == 'bdelta':
2855 system(('~/debdelta/bdelta-0.1.0/bdelta',o,n,p),TD)
2856 script.write('~/debdelta/bdelta-0.1.0/bpatch '+o+' '+n+' '+p+'\n')
2857 #diffball
2858 #http://developer.berlios.de/projects/diffball/
2859 elif algo == 'diffball':
2860 system(('~/debdelta/diffball-0.7.2/differ',o,n,p),TD)
2861 script.write('~/debdelta/diffball-0.7.2/patcher '+o+' '+p+' '+n+'\n')
2862 #rdiff
2863 elif algo == 'rdiff':
2864 system(('rdiff','signature',o,'sign_file.tmp'),TD)
2865 system(('rdiff','delta','sign_file.tmp',n,p),TD)
2866 script.write('rdiff patch '+o+' '+p+' '+n+'\n')
2867 #xdelta3
2868 elif algo == 'xdelta3' :
2869 system(('xdelta3','-9','-R','-D','-n','-S','djw','-s',o,n,p),TD)
2870 if infifo : o = infifo #use fifo as input
2871 if outpiped:
2872 this_delta_outpiped=True
2873 script.write('xdelta3 -d -R -D -c -s '+o+' '+p+'\n')
2874 else:
2875 script.write('xdelta3 -d -R -D -s '+o+' '+p+' '+n+'\n')
2876 ## according to the man page,
2877 ## bsdiff uses memory equal to 17 times the size of oldfile
2878 ## but , in my experiments, this number is more like 12.
2879 ##But bsdiff is sooooo slow!
2880 elif algo == 'bsdiff' : # not ALLOW_XDELTA or ( osize < (MAXMEMORY / 12)):
2881 system(('bsdiff',o,n,p),TD)
2882 script.write('bspatch '+o+' '+n+' '+p+'\n')
2883 #seems that 'xdelta' is buggy on 64bit and different-endian machines
2884 #xdelta does not deal with different endianness!
2885 elif algo == 'xdelta-bzip' :
2886 system(('xdelta','delta','--pristine','--noverify','-0','-m'+str(int(MAXMEMORY/1024))+'k',o,n,p),TD)
2887 system('bzip2 -9 '+p,TD,(p,))
2888 script.write('bunzip2 '+p+'.bz2 ; xdelta patch '+p+' '+o+' '+n+'\n')
2889 p += '.bz2'
2890 elif algo == 'xdelta' :
2891 system(('xdelta','delta','--pristine','--noverify','-9','-m'+str(int(MAXMEMORY/1024))+'k',o,n,p),TD)
2892 script.write('xdelta patch '+p+' '+o+' '+n+'\n')
2893 elif algo == 'jojodiff' :
2894 system(('~/debdelta/jdiff06/src/jdiff','-b',o,n,p),TD)
2895 script.write('~/debdelta/jdiff06/src/jpatch '+o+' '+p+' '+n+'\n')
2896 else: raise AssertionError(' unsupported delta algo ')
2897 return p, this_delta_outpiped
2898
2899 def delta_files(o, n, outpiped=None, infifo=None):
2900 " compute delta of two files , and prepare the script consequently"
2901 nsize = os.path.getsize(TD+n)
2902 osize = os.path.getsize(TD+o)
2903 if VERBOSE > 1 : logger.debug(' compute delta for %s (%dkB) and %s (%dkB)',
2904 o,osize/1024,n,nsize/1024)
2905 #
2906 p = 'PATCH/'+next(a_numb_patch)
2907 tim = -time.time()
2908 #
2909 if DEBUG > 3 : script.md5_check_file(o)
2910 #
2911 if USE_DELTA_ALGO == 'bsdiff' and osize > ( 1.1 * (MAXMEMORY / 12)) and VERBOSE :
2912 logger.debug(' Warning, memory usage by bsdiff on the order of %dMb', (12 * osize / 2**20))
2913 #
2914 p, this_delta_outpiped = delta_files__(o, n, p, USE_DELTA_ALGO, outpiped, infifo)
2915 #script.write(s)
2916 #
2917 if DEBUG > 2 and not this_delta_outpiped : script.md5_check_file(n)
2918 #
2919 tim += time.time()
2920 #
2921 global bsdiff_time, bsdiff_datasize
2922 bsdiff_time += tim
2923 bsdiff_datasize += nsize
2924 #
2925 if infifo:
2926 script.write('rm '+p+'\n')
2927 else:
2928 script.write('rm '+o+' '+p+'\n')
2929 ## how did we fare ?
2930 deltasize = os.path.getsize(TD+p)
2931 if VERBOSE > 1 :
2932 logger.debug(' delta is %3.2f%% of %s, speed: %dkB /sec',
2933 ( deltasize * 100. / nsize ) , n, (nsize / 1024. / ( tim + 0.001 )))
2934 #save it
2935 patch_append(p[6:])
2936 #clean up
2937 unlink(TD+o)
2938 return this_delta_outpiped
2939
2940 def cmp_gz(o,n):
2941 "compare gzip files, ignoring header; returns first different byte (+-10), or True if equal"
2942 of=open(o,'rb')
2943 nf=open(n,'rb')
2944 oa=of.read(10)
2945 na=nf.read(10)
2946 myord = ord if sys.version_info.major == 2 else (lambda x:x)
2947 if na[:3] != b'\037\213\010' :
2948 logger.warning(' Warning: was not created with gzip: %r',n)
2949 nf.close() ; of.close()
2950 return 0
2951 if oa[:3] != b'\037\213\010' :
2952 logger.warning(' Warning: was not created with gzip: %r',o)
2953 nf.close() ; of.close()
2954 return 0
2955 oflag=myord(oa[3])
2956 if oflag & 0xf7:
2957 logger.warning(' Warning: unsupported .gz flags: %r %r',oct(oflag),o)
2958 if oflag & 8 : #skip orig name
2959 oa=of.read(1)
2960 while oa and myord(oa) != 0:
2961 oa=of.read(1)
2962 if not oa:
2963 logger.warning(' Warning: premature EOF: %r',o)
2964 nf.close() ; of.close()
2965 return 0
2966 l=10
2967 nflag=myord(na[3])
2968 if nflag & 0xf7:
2969 logger.warning(' Warning: unsupported .gz flags: %r %r',oct(nflag),n)
2970 if nflag & 8 : #skip orig name
2971 na=nf.read(1)
2972 #s=na
2973 while na and myord(na) != 0:
2974 na=nf.read(1)
2975 #s+=na
2976 if not na:
2977 logger.warning(' Warning: premature EOF: %r',n)
2978 nf.close() ; of.close()
2979 return 0
2980 l+=len(s)
2981 #print repr(s)
2982 while oa and na:
2983 oa=of.read(2)
2984 na=nf.read(2)
2985 if oa != na:
2986 return l
2987 l+=2
2988 if oa or na: return l
2989 return True
2990
2991 def parse_gzip_header(n):
2992 " n may be a file name, a function returning a stream, or a stream"
2993 myord = ord if sys.version_info.major == 2 else (lambda x:x)
2994 if type(n) in string_types:
2995 f=open(TD+n,'rb')
2996 a=f.read(10)
2997 f.close()
2998 elif type(n) in (FunctionType,LambdaType):
2999 f=n()
3000 a=f.read(10)
3001 if isinstance(f,PopenPipe): f.close()
3002 else:
3003 a=n.read(10)
3004 if a[:3] != b'\037\213\010' :
3005 logger.debug(' Warning: was not created with gzip: %r',n)
3006 return
3007 flag=myord(a[3]) # mostly ignored :->
3008 orig_name='-n'
3009 if flag & 8:
3010 orig_name='-N'
3011 if flag & 0xf7:
3012 logger.debug(' Warning: unsupported .gz flags: %r %r',oct(flag),n)
3013 #a[4:8] #mtime ! ignored ! FIXME will be changed...
3014 #from deflate.c in gzip source code
3015 format=myord(a[8])
3016 FAST=4
3017 SLOW=2 #unfortunately intermediate steps are lost....
3018 pack_level=6
3019 if format == 0 :
3020 pass
3021 elif format == FAST :
3022 pack_level = 1
3023 elif format == SLOW :
3024 pack_level = 9
3025 else:
3026 logger.debug(' Warning: unsupported compression .gz format: %r %r',oct(format),n)
3027 return
3028 if myord(a[9]) != 3 :
3029 if VERBOSE : logger.debug(' Warning: unknown OS in .gz format: %r %r',oct(myord(a[9])),n)
3030 pack_list=[1,2,3,4,5,6,7,8,9]
3031 del pack_list[pack_level-1]
3032 pack_list.append(pack_level)
3033 pack_list.reverse()
3034 #print 'format ', repr(format), type(format), format==FAST, pack_list
3035 return pack_list,orig_name
3036
3037 def delta_gzipped_files(o,n):
3038 "delta o and n, replace o with n"
3039 assert(o[-3:] == '.gz' and n[-3:] == '.gz')
3040 before=cmp_gz(TD+o,TD+n)
3041 if before == True:
3042 if VERBOSE > 3: logger.debug(' equal but for header: %r',n)
3043 return
3044 #compare the cost of leaving as is , VS the minimum cost of delta
3045 newsize=os.path.getsize(TD+n)
3046 if ( newsize - before + 10 ) < 200 :
3047 if VERBOSE > 3: logger.debug(' not worthwhile gunzipping: %r',n)
3048 return
3049 z=parse_gzip_header(n)
3050 if z==None:
3051 return
3052 pack_list,orig_name=z
3053 pack_level=pack_list[0]
3054 #OK, it seems we can play our trick
3055 p='_tmp_'
3056 #unzip new file
3057 pro=subprocess.Popen(('zcat',TD+n),stdout=open(TD+p+'.new','wb'), cwd=TD)
3058 pro.wait()
3059 if pro.returncode: raise DebDeltaError('argghh, zcat failed on us')
3060 #test our ability of recompressing
3061 best_r=0
3062 best_flag=None
3063 for i in pack_list:
3064 #force -n ... no problem with timestamps
3065 gzip_flags="-"+str(i)
3066 pro=subprocess.Popen(("gzip",'-c','-n',gzip_flags,TD+p+'.new'),stdout=open(TD+p+'.faked.gz','wb'))
3067 pro.wait()
3068 if pro.returncode: DebDeltaError('Argh, gzip failed on us')
3069 r=cmp_gz(TD+n,TD+p+'.faked.gz')
3070 if r > best_r :
3071 best_r = r
3072 best_flag = gzip_flags
3073 if r == True:
3074 break
3075 if i == pack_level and VERBOSE > 3:
3076 logger.debug(' warning: wrong guess to re-gzip to equal file: %r %r %r',gzip_flags,r,n)
3077 if r != True:
3078 if VERBOSE > 1 : logger.debug(' warning: cannot re-gzip to equal file, best was %d / %d , "%s" : %s ',best_r,newsize,best_flag,n)
3079 os.unlink(TD+p+".new") ; os.unlink(TD+p+'.faked.gz')
3080 return
3081 #actual delta of decompressed files
3082 pro=subprocess.Popen(('zcat',TD+o),stdout=open(TD+p+'.old','wb'), cwd=TD)
3083 pro.wait()
3084 if pro.returncode: raise DebDeltaError('argh, zcat failed on us')
3085 script.write("zcat '"+o+"' > "+p+".old ; rm '"+o+"' \n")
3086 if VERBOSE > 2 : logger.debug(' '+n[9:]+(' (= to %d%%): ' % (100*before/newsize)))
3087 delta_files(p+'.old',p+'.new')
3088 script.write("gzip -c -n "+gzip_flags+" < "+p+".new > '"+o+"' ; rm "+p+".new\n")
3089 #replace the old file with the best that we can do re-gzipping the new file
3090 #this is important in the rest of the delta-ing process
3091 os.rename(TD+p+'.faked.gz',TD+o)
3092 if DEBUG > 1 :
3093 script.md5_check_file(o, compute_md5(TD+o))
3094
3095 def guess_gz_command(o, check=True):
3096 """tries to guess the parameters used to compress, returns a command vector
3097 if it fails, returns False
3098 o may be a file name, a function returning a stream, or a stream
3099 """
3100 h=parse_gzip_header(o)
3101 if h==None:
3102 return False
3103 if type(o) in (FunctionType,LambdaType):
3104 o=o()
3105 elif type(o) in string_types:
3106 o=open(o,'rb')
3107 z=tempfile.NamedTemporaryFile(suffix='.gz',delete=False)
3108 shutil.copyfileobj(o,z)
3109 z.flush()
3110 if isinstance(o,PopenPipe): o.close()
3111 pack_list,orig_name=h
3112 pack_list.reverse()
3113 cmd_list=[(['gzip','-nc'],l) for l in pack_list]
3114 cmd_list.append(([minigzip],9)) ##old method up to dpkg-deb in 2014-01-15
3115 wcmd=False #fixme, there is no way to distinguish output of minigzip or gzip from header
3116 if check:
3117 redo=True
3118 while redo and cmd_list:
3119 cmd,par=cmd_list.pop()
3120 wcmd=copy(cmd)
3121 wcmd.append('-'+str(par))
3122 if VERBOSE > 2 : logger.debug(' Testing command %r',wcmd)
3123 w=PopenPipe(['zcat',z.name,'|']+wcmd+['|','cmp','-',z.name],\
3124 stdout=open(os.devnull,'w'),stderr=open(os.devnull,'wb'),close_fds=True)
3125 redo=False
3126 w.wait()
3127 if w.returncode :
3128 redo=True
3129 if VERBOSE or DEBUG : logger.debug(' Tried gzip options but failed: %r',wcmd)
3130 elif VERBOSE > 2 : logger.debug(' Success with command %r',wcmd)
3131 if redo and not cmd_list:
3132 logger.debug(' NO MORE OPTIONS !')
3133 os.unlink(z.name)
3134 return False
3135 os.unlink(z.name)
3136 return wcmd
3137
3138 def guess_xz_parameters(o, check=True):
3139 """tries to guess the parameters used to compress, returns a a string of options
3140 if it fails, returns False
3141 o may be a file name, a function returning a stream, or a stream
3142 """
3143 par=''
3144 crc=''
3145 thread=''
3146 if type(o) in (FunctionType,LambdaType):
3147 o=o()
3148 elif type(o) in string_types:
3149 o=open(o,'rb')
3150 z=tempfile.NamedTemporaryFile(suffix='.xz',delete=False)
3151 #unfortunately 'xz --list' does not work on pipes!
3152 shutil.copyfileobj(o,z)
3153 z.flush()
3154 if isinstance(o,PopenPipe): o.close()
3155 b=subprocess.Popen(['xz','-vv','--robot','--list',z.name],stdout=subprocess.PIPE,
3156 **Popen_args)
3157 for a in b.stdout:
3158 a=a.rstrip('\n')
3159 a=str.split(a,'\t')
3160 if a[0]=='block':
3161 if crc and crc != a[9]:
3162 logger.warning(" warning : this xz -- compressed file was compressed with variable blocks crc ?! '%s' != '%s'",crc,a[9])
3163 crc=a[9]
3164 if par and par != a[15]:
3165 logger.warning(" warning : this xz -- compressed file was compressed with variable blocks options ?! '%s' != '%s'",par,a[15])
3166 par=a[15]
3167 if not thread and a[12]=='cu':
3168 thread='-T2'
3169 #print ' guessed par crc ',par,crc
3170 if crc:
3171 crc=crc.lower()
3172 if crc=='sha-256': crc='sha256'
3173 if crc not in ('crc32','crc64','sha256'):
3174 logger.warning(' Unknown XZ crc %r',crc)
3175 crc=''
3176 PARS=['-6e','-9','-9e']
3177 if par:
3178 PARS.append(par)
3179 if par == '--lzma2=dict=1MiB':
3180 # dbgsym deb files are compressed with -1e
3181 PARS.append('-1')
3182 PARS.append('-1e')
3183 if check:
3184 redo=True
3185 while redo and PARS:
3186 par=PARS.pop()
3187 w=['xz','-c']
3188 if par:
3189 w.append(par)
3190 if crc:
3191 w+=['-C',crc]
3192 if thread:
3193 w.append(thread)
3194 w.append('-')
3195 if VERBOSE > 2 : logger.debug(' Testing XZ options %r',w)
3196 redo=False
3197 c=PopenPipe(['unxz','-c',z.name,'|']+w+['|','cmp','-',z.name],\
3198 stdout=open(os.devnull,'wb'),stderr=open(os.devnull,'wb'),close_fds=True)
3199 c.wait()
3200 if c.returncode :
3201 redo=True
3202 if VERBOSE or DEBUG : logger.debug(' Tried XZ options but failed: %r %r %r',par,crc,thread)
3203 if redo and not PARS:
3204 #print ' HO FINITO LE OPZIONI !'
3205 os.unlink(z.name)
3206 return False
3207 if crc:
3208 crc=' -C '+crc
3209 if thread:
3210 thread=' '+thread
3211 os.unlink(z.name)
3212 return par+crc+thread
3213
3214 ########### helper sh functions for script, for delta_tar()
3215
3216 import difflib, re
3217 re_numbers=re.compile(r'^[0-9][0-9]*$')
3218
3219 def file_similarity_premangle(fp):
3220 fps=fp.split('/')
3221 bns=fps[-1].split('.')
3222 j=len(bns)-1 #search first "non numeric" extension, and put it last
3223 while j >= 0 and re_numbers.match(bns[j]):
3224 j-=1
3225 if j >= 0:
3226 a=bns.pop(j)
3227 r = fps[:-1] + bns + [a,]
3228 else:
3229 r = fps[:-1] + bns
3230 return r
3231
3232 def files_similarity_score__noext__(oo,nn):
3233 ln=len(nn)
3234 lo=len(oo)
3235 l=0
3236 while oo and nn:
3237 while oo and nn and oo[-1] == nn[-1]:
3238 oo=oo[:-1]
3239 nn=nn[:-1]
3240 if not oo or not nn: break
3241 while oo and nn and oo[0] == nn[0]:
3242 oo=oo[1:]
3243 nn=nn[1:]
3244 if not oo or not nn: break
3245 if len(nn) > 1 and oo[0] == nn[1]:
3246 l+=1
3247 nn=nn[1:]
3248 if len(oo) > 1 and oo[1] == nn[0]:
3249 l+=1
3250 oo=oo[1:]
3251 if not oo or not nn: break
3252 if oo[-1] != nn[-1]:
3253 oo=oo[:-1]
3254 nn=nn[:-1]
3255 l+=2
3256 if not oo or not nn: break
3257 if oo[0] != nn[0]:
3258 oo=oo[1:]
3259 nn=nn[1:]
3260 l+=2
3261 return (l +len(oo) + len(nn)) * 2.0 / float(ln+lo)
3262
3263 def files_similarity_score__(oo,nn):
3264 oo=copy(oo)
3265 nn=copy(nn)
3266 if oo.pop() != nn.pop() :
3267 return 0.2 + files_similarity_score__noext__(oo,nn)
3268 else:
3269 return files_similarity_score__noext__(oo,nn)
3270
3271 def files_similarity_score__difflib__(oo,nn):
3272 "compute similarity by difflib. Too slow."
3273 if oo == nn :
3274 return 0
3275 d=difflib.context_diff(oo,nn,'','','','',0,'')
3276 d=[a for a in tuple(d) if a and a[:3] != '---' and a[:3] != '***' ]
3277 if oo[-1] != nn[-1] : #penalty for wrong extension
3278 return 0.2+float(len(d)) * 2.0 / float(len(oo)+len(nn))
3279 else:
3280 return float(len(d)) * 2.0 / float(len(oo)+len(nn))
3281
3282 def files_similarity_score(oo,nn):
3283 if oo == nn :
3284 return 0
3285 if type(oo) in string_types:
3286 oo=file_similarity_premangle(oo)
3287 if type(nn) in string_types:
3288 nn=file_similarity_premangle(nn)
3289 return files_similarity_score__(oo,nn)
3290
3291 def fake_tar_header_2nd():
3292 " returns the second part of a tar header , for regular files and dirs"
3293 # The following code was contributed by Detlef Lannert.
3294 # into /usr/lib/python2.3/tarfile.py
3295 MAGIC = b"ustar" # magic tar string
3296 VERSION = b"00" # version number
3297 NUL = b"\0" # the null character
3298 parts = []
3299 for value, fieldsize in (
3300 (b"", 100),
3301 # unfortunately this is not what DPKG does
3302 #(MAGIC, 6),
3303 #(VERSION, 2),
3304 # this is what DPKG does
3305 (b'ustar \x00',8),
3306 (b"root", 32),
3307 (b"root", 32),
3308 (b"%07o" % 0, 8),
3309 (b"%07o" % 0, 8),
3310 (b"", 155)
3311 ):
3312 l = len(value)
3313 parts.append(value + (fieldsize - l) * NUL)
3314 buf = b"".join(parts)
3315 return buf
3316
3317 fake_tar_2nd=fake_tar_header_2nd()
3318 fake_tar_2nd_echo=prepare_for_echo(fake_tar_2nd)
3319 script.write("FTH='"+fake_tar_2nd_echo+"'\n")
3320 script.write("E='echo -ne'\n")
3321
3322 global time_corr
3323 time_corr=0
3324
3325 #################### vvv delta_tar vvv ###########################
3326 def delta_tar(old_filename, new_filename, CWD, old_forensic,\
3327 skip=[], old_md5={}, new_md5={},\
3328 chunked_p=(not delta_uses_infifo) ,debdelta_conf_skip=()):
3329 " compute delta of two tar files, and prepare the script consequently"
3330 assert( type(old_filename) in string_types or type(old_filename) == FunctionType )
3331
3332 script.write('ECR () { $E "$1" ; $E "${FTH}" ; cat OLD/'+CWD+'/"$1" ; rm OLD/'+CWD+'/"$1" ;}\n')
3333 script.write('EC () { $E "$1" ; $E "${FTH}" ; cat OLD/'+CWD+'/"$1" ;}\n')
3334
3335 ###### uncompress and scan the old tar file, extract regular files
3336 if type(old_filename) in string_types :
3337 (old_filename,old_filename_ext) = unzip(old_filename)
3338 oldtar = tarfile.open(TD+old_filename, "r")
3339 elif type(old_filename) in (FunctionType,LambdaType) :
3340 old_filename_ext=None
3341 oldfileobj = old_filename()
3342 oldtar = tarfile.open(mode="r|", fileobj=oldfileobj)
3343 else:
3344 old_filename_ext=None
3345 oldtar = tarfile.open(mode="r|", fileobj=old_filename)
3346 oldnames = []
3347 oldtarinfos = {}
3348 for oldtarinfo in oldtar:
3349 oldname = de_bar(oldtarinfo.name)
3350 if old_forensic != None:
3351 #fixme : devices are not supported (but debian policy does not allow them)
3352 old_forensic.append([oldtarinfo.name,oldtarinfo.mode,oldtarinfo.type,\
3353 oldtarinfo.uid,oldtarinfo.gid,oldtarinfo.uname,oldtarinfo.gname,oldtarinfo.linkname])
3354 #this always happens
3355 #if VERBOSE > 3 and oldname != de_bar(oldname):
3356 # print ' filename in old tar has weird ./ in front: ' , oldname
3357
3358 if not oldtarinfo.isreg():
3359 if VERBOSE > 2 : logger.debug(' skipping old non-regular %r',oldname)
3360 continue
3361
3362 if oldtarinfo.size == 0:
3363 if VERBOSE > 2 : logger.debug(' skipping old empty %r',oldname)
3364 continue
3365
3366 if shell_not_allowed(oldname):
3367 if VERBOSE > 2 : logger.debug(' skipping non-allowed-name %r',oldname)
3368 continue
3369
3370 for j in debdelta_conf_skip:
3371 if fnmatch.fnmatch(oldname,j):
3372 if VERBOSE > 2 : logger.debug(' skipping following as per rule %r',j)
3373 skip.append(oldname)
3374 break
3375
3376 if oldname in skip:
3377 if VERBOSE > 2 : logger.debug(' skipping %r',oldname)
3378 if old_forensic != None:
3379 oldtar.extract(oldtarinfo,TD+"OLD/"+CWD )
3380 old_forensic.append(old_forensic.pop()[:-1] + \
3381 [sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname))])
3382 continue
3383
3384 oldnames.append(oldname)
3385 oldtarinfos[oldname] = oldtarinfo
3386 oldtar.extract(oldtarinfo,TD+"OLD/"+CWD )
3387 if old_forensic != None:
3388 old_forensic.append(old_forensic.pop()[:-1] + \
3389 [sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname))])
3390
3391 oldtar.close()
3392 if type(old_filename) in string_types :
3393 unlink(TD+old_filename)
3394 else:
3395 while oldfileobj.read(512):
3396 pass
3397 ###### scan the new tarfile, save info regarding regular files therein
3398 #save header part of new_filename, since it changes in newer versions
3399 f=open(TD+new_filename,'rb')
3400 new_file_zip_head=f.read(20)
3401 f.close()
3402 (new_filename,new_filename_ext) = unzip(new_filename)
3403 new_file_md5=compute_md5(TD+new_filename)
3404 new_file_size=os.path.getsize(TD+new_filename)
3405 info_append(new_filename+': '+new_file_md5+' '+str(new_file_size))
3406 #### scan the new tarfile, compare to the old tar contents
3407 assert(0 == (new_file_size % 512))
3408 newtar = tarfile.open(TD+new_filename, "r")
3409 # dpkg lately uses PAX format, dunno the difference, debdelta seems to work fine
3410 logger.log(logging.DEBUG if newtar.format in (tarfile.GNU_FORMAT , tarfile.PAX_FORMAT) else logging.WARNING,
3411 ' tar file format %r for %r'%(newtar.format,new_filename))
3412 newnames = []
3413 newtarinfos = {}
3414 for newtarinfo in newtar:
3415 newname = newtarinfo.name
3416 #just curious to know
3417 t=newtarinfo.type
3418 a=newtarinfo.mode
3419 if VERBOSE and (( t == '2' and a != 0o777 ) or \
3420 ( t == '0' and ( (a & 0o400 ) == 0 )) or \
3421 ( t == '5' and ( (a & 0o500 ) == 0 ))):
3422 logger.debug(' weird permission %r %r %r', newname, oct(a), repr(newtarinfo.type))
3423 ###
3424 if not newtarinfo.isreg():
3425 continue
3426 if VERBOSE > 3 and newname != de_bar(newname):
3427 logger.debug(' filename in new tar has weird ./ in front: %r' , newname)
3428 newname = de_bar(newname)
3429 newnames.append(newname)
3430 newtarinfos[newname] = newtarinfo
3431
3432 old_used={}
3433 correspondence={}
3434
3435 ###### find correspondences between old tar and new tar contents
3436 global time_corr
3437 time_corr=-time.time()
3438
3439 if VERBOSE > 2 : logger.debug(' finding correspondences for %r', new_filename)
3440
3441 reverse_old_md5={}
3442 if old_md5:
3443 for o in old_md5:
3444 if o in oldnames:
3445 reverse_old_md5[old_md5[o]] = o
3446 else:
3447 #would you believe? many packages contain MD5 for files they do not ship...
3448 if VERBOSE > 1 and o not in skip: logger.debug(' hmmm... there is a md5 but not a file: %r',o)
3449
3450 #database of databases of premangled old names , by "extension" and name
3451 oldnames_premangle={}
3452 for o in oldnames:
3453 om=file_similarity_premangle(o)
3454 a=om[-1] # "extension"
3455 if a not in oldnames_premangle:
3456 oldnames_premangle[a]={}
3457 oldnames_premangle[a][o]=om
3458
3459 for newname in newnames:
3460 newtarinfo=newtarinfos[newname]
3461 oldname=None
3462 #ignore empty files
3463 if newtarinfo.size == 0:
3464 continue
3465 #try correspondence by MD5
3466 if new_md5 and newname in new_md5:
3467 md5=new_md5[newname]
3468 if md5 in reverse_old_md5:
3469 oldname=reverse_old_md5[md5]
3470 if VERBOSE > 2 :
3471 if oldname == newname :
3472 logger.debug(' use identical old file: %r',newname)
3473 else:
3474 logger.debug(' use identical old file: %r %r',oldname, newname)
3475 #try correspondence by file name
3476 if oldname == None and newname in oldnames:
3477 oldname=newname
3478 if VERBOSE > 2 : logger.debug(' use same name old file: %r',newname)
3479 #try correspondence by file name and len similarity
3480 np=file_similarity_premangle(newname)
3481 ne=np[-1] # "extension"
3482 if oldname == None and ne in oldnames_premangle:
3483 basescore=1.6
3484 nl=newtarinfo.size
3485 for o in oldnames_premangle[ne]:
3486 op=oldnames_premangle[ne][o]
3487 l=oldtarinfos[o].size
3488 sfile=files_similarity_score__noext__(op,np)
3489 slen = abs(float(l - nl))/float(l+nl)
3490 s=slen+sfile
3491 if VERBOSE > 3 : logger.debug(' name/len diff %.2f+%.2f=%.2f %r',slen,sfile,s, o)
3492 if s < basescore:
3493 oldname=o
3494 basescore=s
3495 if oldname and VERBOSE > 2 : logger.debug(' best similar %.3f %r %r',basescore,newname,oldname)
3496 if not oldname:
3497 if VERBOSE > 2 : logger.debug(' no correspondence for: %r',newname)
3498 continue
3499 #we have correspondence, lets store
3500 if oldname not in old_used:
3501 old_used[oldname]=[]
3502 old_used[oldname].append(newname)
3503 correspondence[newname]=oldname
3504
3505 time_corr+=time.time()
3506 if VERBOSE > 1 : logger.debug(' time lost so far in finding correspondence %.2f',time_corr)
3507
3508 ######### final pass : scan new tar, extract regular files, prepare deltas
3509 if VERBOSE > 2 : logger.debug(' scanning %r',new_filename)
3510
3511 script.start_rebuilding()
3512
3513 current_chunk_name=next(a_numb_file)
3514 script.start_chunk(current_chunk_name)
3515 mega_cat=open(TD+'/'+current_chunk_name,'wb')
3516
3517 #helper function
3518 def _append_(p, w, rm):
3519 mega_cat.write(str2bytes(w)+fake_tar_2nd)
3520 f=open(TD+p+'/'+w,'rb')
3521 a=f.read(1024)
3522 while a:
3523 try:
3524 mega_cat.write(a)
3525 except OSError:
3526 s=sys.exc_info()[1]
3527 raise DebDeltaError(' OSError (at _a_) while writing: '+str(s), True)
3528 a=f.read(1024)
3529 f.close()
3530 if rm:
3531 script.write("ECR '"+w+"'\n")
3532 unlink(TD+p+'/'+w)
3533 else:
3534 script.write("EC '"+w+"'\n")
3535
3536 global something_backgrounded #FIXME I hate using globals for this :-(
3537 something_backgrounded=False
3538 #helper function
3539 def mega_cat_chunk(oldoffset,newoffset, background=True):
3540 global something_backgrounded
3541 p = next(a_numb_file)
3542 f=open(TD+new_filename,'rb')
3543 f.seek(oldoffset)
3544 of=open(TD+p,'wb')
3545 l=oldoffset
3546 while l<newoffset:
3547 s=f.read(512)
3548 l+=len(s)
3549 assert(len(s))
3550 try:
3551 of.write(s)
3552 except OSError:
3553 s=sys.exc_info()[1]
3554 raise DebDeltaError(' OSError (at MCK) while writing: '+str(s), True)
3555 f.close()
3556 of.close()
3557 #do delta, in background there
3558 if something_backgrounded:
3559 script.write('wait\n')
3560 if background:
3561 script.write('(')
3562 this_delta_piped=delta_files(current_chunk_name, p, True, script.the_fifo)
3563 if not this_delta_piped:
3564 script.write('cat '+p+'\n')
3565 if not this_delta_piped:
3566 script.write('rm '+p+'\n')
3567 if background:
3568 script.write(')&\n') #the delta+cat is backgrounded
3569 something_backgrounded=True
3570 os.unlink(TD+p)
3571
3572 #there may be files that have been renamed and edited...
3573 def some_old_file_gen():
3574 for oldname in oldnames :
3575 if (oldname in skip) or (oldname in old_used ) :
3576 continue
3577 if VERBOSE > 2 : logger.debug(' provide also old file %r', oldname)
3578 yield oldname
3579 while 1:
3580 yield None
3581
3582 some_old_file=some_old_file_gen()
3583 one_old_file=next(some_old_file)
3584
3585 max_chunk_size = MAXMEMORY / 12
3586 chunk_discount = 0.3
3587
3588 progressive_new_offset=0
3589
3590 for newtarinfo in newtar:
3591 #progressive mega_cat
3592 a=mega_cat.tell()
3593 if chunked_p and ((a >= max_chunk_size * chunk_discount) or \
3594 (a >= max_chunk_size * chunk_discount * 0.9 and one_old_file ) or \
3595 (a>0 and (a+newtarinfo.size) >= max_chunk_size * chunk_discount )):
3596 #provide some old unused files, if any
3597 while one_old_file:
3598 _append_("OLD/"+CWD, one_old_file, False)
3599 if mega_cat.tell() >= max_chunk_size * chunk_discount :
3600 break
3601 one_old_file=next(some_old_file)
3602 #write the chunk into a temporary
3603 mega_cat.close()
3604 script.end_chunk(current_chunk_name)
3605 #delta the chunk
3606 mega_cat_chunk(progressive_new_offset, newtarinfo.offset )
3607 #start a new chunk
3608 current_chunk_name=next(a_numb_file)
3609 script.start_chunk(current_chunk_name)
3610 mega_cat=open(TD+'/'+current_chunk_name,'wb')
3611 #
3612 progressive_new_offset=newtarinfo.offset
3613 chunk_discount = min( 1. , chunk_discount * 1.2 )
3614 #
3615 name = de_bar( newtarinfo.name )
3616
3617 if newtarinfo.isdir():
3618 #recreate also parts of the tar headers
3619 mega_cat.write(str2bytes(newtarinfo.name)+fake_tar_2nd)
3620 script.write("$E '"+prepare_for_echo(str2bytes(newtarinfo.name))+"'\"${FTH}\"\n")
3621 if VERBOSE > 2 : logger.debug(' directory in new : %r', name)
3622 continue
3623
3624 if not newtarinfo.isreg():
3625 #recreate also parts of the tar headers
3626 mega_cat.write(str2bytes(newtarinfo.name)+fake_tar_2nd)
3627 script.write("$E '"+prepare_for_echo(newtarinfo.name)+"'\"${FTH}\"\n")
3628 if VERBOSE > 2 : logger.debug(' not regular in new : %r', name)
3629 continue
3630
3631 if newtarinfo.size == 0:
3632 #recreate also parts of the tar headers
3633 mega_cat.write(str2bytes(newtarinfo.name)+fake_tar_2nd)
3634 script.write("$E '"+prepare_for_echo(newtarinfo.name)+"'\"${FTH}\"\n")
3635 if VERBOSE > 2 : logger.debug(' empty new file : %r', name)
3636 continue
3637
3638 if name not in correspondence:
3639 #recreate also parts of the tar headers
3640 mega_cat.write(str2bytes(newtarinfo.name)+fake_tar_2nd)
3641 script.write("$E '"+prepare_for_echo(newtarinfo.name)+"'\"${FTH}\"\n")
3642 if VERBOSE > 2: logger.debug(' no corresponding fil: %r', name)
3643 continue
3644
3645 oldname = correspondence[name]
3646
3647 mul=len( old_used[oldname]) > 1 #multiple usage
3648
3649 if not mul and oldname == name and oldname[-3:] == '.gz' and \
3650 newtarinfo.size > 120 and \
3651 not ( new_md5 and name in new_md5 and old_md5 and name in old_md5 and \
3652 new_md5[name] == old_md5[name]):
3653 newtar.extract(newtarinfo,TD+"NEW/"+CWD )
3654 delta_gzipped_files("OLD/"+CWD+'/'+name,"NEW/"+CWD+'/'+name)
3655
3656 if VERBOSE > 2 : logger.debug(' adding reg file: %r %r', oldname, mul and '(multiple)' or '')
3657 _append_("OLD/"+CWD, oldname , not mul )
3658 old_used[oldname].pop()
3659 #end of for loop
3660
3661 #write the chunk into a temporary
3662 mega_cat.close()
3663 script.end_chunk(current_chunk_name)
3664 if os.path.exists(TD+'/OLD/'+CWD):
3665 rmtree(TD+'/OLD/'+CWD)
3666 if os.path.getsize(TD+'/'+current_chunk_name) > 0 :
3667 mega_cat_chunk(progressive_new_offset, os.path.getsize(TD+new_filename), background=False)
3668 else:
3669 #the (tail of the) new tar did not match anything in the old tar, nothing to delta
3670 p=verbatim(new_filename)
3671 script.write("cat '"+p+"'\n")
3672 script.end_rebuilding() #pipes the rebuilding part into the recompressing part
3673 if new_filename == 'NEW/data.tar':
3674 script.recompressing_by_arg(new_filename,new_filename_ext,new_file_zip_head,new_file_size)
3675 else:
3676 script.recompressing(new_filename,new_filename_ext,new_file_zip_head)
3677
3678 #################### ^^^^ delta_tar ^^^^ ###########################
3679
3680 ############ start computing deltas
3681 def append_NEW_file(s):
3682 'appends some data to NEW.file'
3683 s=prepare_for_echo(s)
3684 script.write("$E '"+ s +"' >> NEW.file\n")
3685
3686 #this following is actually
3687 #def delta_debs_using_old(old,new):
3688
3689 ### start scanning the new deb
3690 newdeb_file=open(newdeb,'rb')
3691 # pop the "!<arch>\n"
3692 s = newdeb_file.readline()
3693 assert( b"!<arch>\n" == s)
3694 append_NEW_file(s)
3695
3696
3697 if forensic_file==None:
3698 control_forensic=None
3699 data_forensic=None
3700 else:
3701 control_forensic=[]
3702 data_forensic=[]
3703
3704 def decompress_pipe_from_ar(oldnew, name, p=None):
3705 " use p=False to not decompress"
3706 assert oldnew in ('OLD','NEW')
3707 if p==None:
3708 p=os.path.splitext(name)[1]
3709 db={b'.gz' : ['gzip','-cd'], b'.bz2': ['bzip2','-cd'],
3710 b'.lzma':['unlzma','-c'], b'.xz': ['unxz','-c'] }
3711 assert p==False or p == b'.tar' or p in db
3712 if p==b'.tar' or p==False:
3713 def x():
3714 return PopenPipe(['ar','p', oldnew+'.file', name] , stdout=subprocess.PIPE, cwd=TD)
3715 else:
3716 def x():
3717 return PopenPipe(['ar','p', oldnew+'.file', name,'|']+db[p] , stdout=subprocess.PIPE, cwd=TD)
3718 return x
3719
3720 for name in ar_list_new :
3721 newname = 'NEW/'+bytes2str(name)
3722 system(('ar','x',TD+'NEW.file',name), TD+'/NEW/')
3723 script.xz_parameters=None
3724 newsize = os.stat(TD+newname)[ST_SIZE]
3725 if VERBOSE > 1: logger.debug(' studying %r of len %dkB' , name, (newsize/1024))
3726 #add 'ar' structure
3727 ar_line = newdeb_file.read(60)
3728 if VERBOSE > 3: logger.debug(' ar line: %r',ar_line)
3729 assert( ar_line[:len(name)] == name and ar_line[-2:] == b'`\n' )
3730 #sometimes there is an extra \n, depending if the previous was odd length
3731 newdeb_file.seek(newsize ,1)
3732 if newsize & 1 :
3733 extrachar = newdeb_file.read(1)
3734 else:
3735 extrachar = ''
3736 #add file to delta
3737 if newsize < 128: #file is too short to compute a delta,
3738 append_NEW_file(ar_line)
3739 p=open(TD+newname,'rb')
3740 append_NEW_file( p.read(newsize))
3741 p.close()
3742 unlink(TD+newname)
3743 #pad new deb
3744 if extrachar :
3745 append_NEW_file(extrachar)
3746 elif not NEEDSOLD and ( name[:11] == b'control.tar' or name[:8] == b'data.tar'):
3747 script.start_member(ar_line, newname, extrachar)
3748 basename,ext=os.path.splitext(name)
3749 xd=decompress_pipe_from_ar('NEW',name,False)
3750 #delta it
3751 if b'.lzma' == ext :
3752 info_append('needs-lzma')
3753 elif b'.xz' == ext :
3754 info_append('needs-xz')
3755 script.xz_parameters=guess_xz_parameters(xd)
3756 if script.xz_parameters==False:
3757 raise DebDeltaError('Cannot guess XZ parameters for new %r' % name)
3758 elif b'.gz' == ext :
3759 script.gz_command=guess_gz_command(xd)
3760 if script.gz_command==False:
3761 raise DebDeltaError('Cannot guess GZ parameters for new %r' % name)
3762 del xd
3763 x=None
3764 for oldext in (b'', b'.gz', b'.bz2', b'.lzma', b'.xz'):
3765 if basename+oldext in ar_list_old:
3766 x=decompress_pipe_from_ar('OLD',basename+oldext)
3767 break
3768 assert x
3769 if name[:11] == b'control.tar':
3770 skip=[]
3771 ##avoid using strange files that dpkg may not install in /var...info/
3772 for a in os.listdir(TD+'OLD/CONTROL') :
3773 if a not in dpkg_keeps_controls:
3774 skip.append(a)
3775 delta_tar(x,newname,'CONTROL',control_forensic,skip)
3776 else:
3777 delta_tar(x,newname,'DATA',data_forensic,old_conffiles,old_md5,new_md5,\
3778 debdelta_conf_skip=debdelta_conf_skip)
3779 del x, basename, oldext
3780 script.end_member()
3781 if DEBUG>3 and name[:11] == b'control.tar':
3782 length=newdeb_file.tell()
3783 md5_len=compute_md5_up_to_len(newdeb, length)
3784 assert md5_len[1] == length
3785 script.md5_check_file('NEW.file', md5_len[0])
3786 elif not NEEDSOLD or name not in ar_list_old : #or it is not in old deb
3787 append_NEW_file(ar_line)
3788 patchname=verbatim(newname)
3789 script.write('cat '+patchname+' >> NEW.file ; rm '+patchname+'\n')
3790 #pad new deb
3791 if extrachar :
3792 append_NEW_file(extrachar)
3793 elif NEEDSOLD :
3794 append_NEW_file(ar_line)
3795 #file is long, and has old version ; lets compute a delta
3796 oldname = 'OLD/'+name
3797 system(('ar','x',TD+'OLD.file',name), TD+'OLD')
3798 script.write('ar p OLD.file '+name+' >> '+oldname+'\n')
3799 (oldname,co) = unzip(oldname)
3800 (newname,cn) = unzip(newname)
3801 delta_files(oldname,newname)
3802 script.zip(newname,cn)
3803 script.write('cat '+newname+cn+' >> NEW.file ; rm '+newname+cn+'\n')
3804 unlink(TD+newname)
3805 #pad new deb
3806 if extrachar :
3807 append_NEW_file(extrachar)
3808 del co,cn
3809 else:
3810 die('internal error j98')
3811 # put in script any leftover
3812 s = newdeb_file.read()
3813 if s:
3814 if VERBOSE > 2: logger.debug(' ar leftover character: %r',s)
3815 append_NEW_file(s)
3816
3817 #this is done already from the receiving end
3818 if DEBUG > 2 and newdeb_md5sum :
3819 script.md5_check_file("NEW.file",md5=newdeb_md5sum)
3820
3821 #script is done
3822 script.close()
3823
3824 if forensic_file:
3825 forensics_rfc(forensic_file,info,True,control_forensic,data_forensic,old_conffiles)
3826
3827 patchsize = os.stat(TD+'PATCH/patch.sh')[ST_SIZE]
3828 patch_files = []
3829 if 'lzma' not in DISABLED_FEATURES and os.path.exists('/usr/bin/lzma'):
3830 system(('lzma','-q','-9','-k','PATCH/patch.sh'), TD)
3831 patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.lzma'), 'lzma', 'patch.sh.lzma'))
3832 if 'xz' not in DISABLED_FEATURES and os.path.exists('/usr/bin/xz'):
3833 system(('xz','-q','-9','-k','PATCH/patch.sh'), TD)
3834 patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.xz'), 'xz', 'patch.sh.xz'))
3835 system(('bzip2','-q','--keep','-9','PATCH/patch.sh'), TD)
3836 patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.bz2'), 'bzip2', 'patch.sh.bz2'))
3837 system(('gzip','-q','-9','-n','PATCH/patch.sh'), TD)
3838 patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.gz'), 'gzip', 'patch.sh.gz'))
3839
3840 # Use the smallest compressed patch.sh
3841 patch_files.sort()
3842 if VERBOSE > 1 : logger.debug(' '+patch_files[0][1]+' wins on patch.sh')
3843 if patch_files[0][1] == 'lzma':
3844 info_append('needs-lzma')
3845 if patch_files[0][1] == 'xz':
3846 info_append('needs-xz')
3847 patch_append(patch_files[0][2])
3848 del patch_files
3849
3850 #OK, OK... this is not yet correct, since I will add the info file later on
3851 elaps = time.time() - start_sec
3852 info.append('DeltaTime: %.2f' % elaps)
3853 deltasize = os.stat(delta)[ST_SIZE] + 60 + sum(map(len,info))
3854 percent = deltasize * 100. / newdebsize
3855 info.append('Ratio: %.4f' % (float(deltasize) / float(newdebsize)) )
3856
3857 if VERBOSE:
3858 #note that sizes are written as kB but are actually kibybytes, that is 1024 bytes
3859 logger.info(' ' + _('delta is %(perc)3.1f%% of deb; that is, %(save)dkB are saved, on a total of %(tot)dkB.'),
3860 {'perc' : percent , 'save' : (( newdebsize -deltasize ) / 1024), 'tot' : ( newdebsize/ 1024)})
3861 logger.info(' ' + _('delta time %(time).2f sec, speed %(speed)dkB /sec, (%(algo)s time %(algotime).2fsec speed %(algospeed)dkB /sec) (corr %(corrtime).2f sec)') % \
3862 {'time' : elaps, 'speed' : newdebsize / 1024. / (elaps+0.001),
3863 'algo' : USE_DELTA_ALGO, 'algotime' : bsdiff_time,
3864 'algospeed' : bsdiff_datasize / 1024. / (bsdiff_time + 0.001) , 'corrtime' : time_corr })
3865 return (delta, percent, elaps, info, gpg_hashes)
3866
3867
3868 ##################################################### compute many deltas
3869
3870 def info_by_pack_arch_add(f, info_by_pack_arch):
3871 pack = f['Package']
3872 arch = f['Architecture']
3873 if (pack,arch) not in info_by_pack_arch :
3874 info_by_pack_arch[ (pack,arch) ]=[]
3875 info_by_pack_arch[ (pack,arch) ].append( f )
3876
3877
3878 def iterate_Packages(packages, use_debian_822=True):
3879 fields=('Package','Architecture','Version','Filename')
3880 for f in fields: intern(f)
3881
3882 packages=abspath(packages)
3883 assert os.path.isfile(packages)
3884 assert os.path.basename(packages) in ('Packages', 'Packages.gz','Packages.bz2','Packages.xz')
3885 dir=os.path.dirname(packages)
3886 dir=dir.split('/')
3887 try:
3888 a=dir.index('dists')
3889 except ValueError:
3890 logger.error('Error: pathname "%s" does not contain "dists"\n' % packages)
3891 return
3892 base = str.join('/',dir[:a])
3893 #
3894 cache=cache_same_dict(packages, fields)
3895 if DO_CACHE and cache.exists:
3896 for i in cache:
3897 i['Basepath'] = base
3898 yield i
3899 if not cache.broken:
3900 return
3901 #
3902 if packages[-3:] == '.gz':
3903 import gzip
3904 F=gzip.GzipFile(packages)
3905 SP=None
3906 elif packages[-4:] == '.bz2':
3907 import bz2
3908 F=bz2.BZ2File(packages)
3909 SP=None
3910 elif packages[-3:] == '.xz':
3911 SP=subprocess.Popen(["xzcat",packages], stdout=subprocess.PIPE, stdin=open(os.devnull), close_fds=True)
3912 F=SP.stdout
3913 else:
3914 F=open(packages,'rb')
3915 SP=None
3916 #
3917 if debian_deb822 and use_debian_822: #use faster implementation
3918 #P=debian_deb822.Packages(F,fields=fields)
3919 for a in debian_deb822.Packages.iter_paragraphs(sequence=F,shared_storage=False,fields=fields):
3920 if DO_CACHE and not cache.exists: cache.write(a)
3921 a['Basepath'] = base
3922 yield a
3923 if SP:
3924 F.read()
3925 SP.wait()
3926 return
3927 #
3928 of,pack,vers,arch=None,None,None,None
3929 for l in F:
3930 l = bytes2str(l)
3931 l=l.rstrip('\n')
3932 if l[:9] == 'Package: ':
3933 pack=l[9:]
3934 elif l[:14] == 'Architecture: ':
3935 arch = l[14:]
3936 elif l[:9] == 'Version: ':
3937 vers = l[9:]
3938 elif l[:10] == 'Filename: ':
3939 of=l[10:]
3940 elif l == '':
3941 if of == None or pack == None or vers == None or arch == None:
3942 logger.warning('Warning, skipping incomplete record in index: %r %r %r %r',of,pack,vers,arch)
3943 continue
3944 if of[-4:] == '.udeb':
3945 if VERBOSE > 2 : logger.debug(' skip udeb')
3946 continue
3947 a={}
3948 a['Filename'] = of
3949 a['Package'] = pack
3950 a['Architecture'] = arch
3951 a['Version'] = vers
3952 if DO_CACHE and not cache.exists: cache.write(a)
3953 a['Basepath'] = base
3954 yield a
3955 of,pack,vers,arch=None,None,None,None
3956 if SP:
3957 F.read()
3958 SP.wait()
3959
3960 def scan_deb_dir(d, debname, label, lazy, info_by_pack_arch, info_by_file):
3961 assert (debname == None or type(debname) in string_types) and type(label) in string_types
3962 if not os.path.isdir(d):
3963 logger.debug('Error, skip non dir: %r',d)
3964 return
3965 if lazy:
3966 scan=scan_deb_byfile_lazy
3967 else:
3968 scan=scan_deb_byfile
3969 for n in os.listdir(d):
3970 if n[-4:] != '.deb':
3971 continue
3972 if debname != None and debname != n.split('_')[0]:
3973 continue
3974 if not check_deb(os.path.join(d,n), die_ = False):
3975 logger.warning('Is not a deb file: %r',n)
3976 continue
3977 a=scan(os.path.join(d,n), info_by_file)
3978 a['Label'] = label
3979 info_by_pack_arch_add(a, info_by_pack_arch)
3980
3981 def scan_deb_byfile_lazy(f, info_by_file):
3982 n=os.path.basename(f).split('_')
3983 a={}
3984 a['File']=f
3985 a['Package']=n[0]
3986 #version cannot be trusted, due to epochs (deleted in filenames)
3987 #it is read using scan_deb_bydict, later on
3988 a['Architecture']=n[2][:-4]
3989 return a
3990
3991 def scan_deb_byfile(f, info_by_file):
3992 #DEBUG: assert( os.path.isfile(f) )
3993 if f in info_by_file and 'Version' in info_by_file[f]:
3994 #already scanned in non-lazy mode
3995 return info_by_file[f]
3996 a={}
3997 a['File']=f
3998 c,b,z = untar_control_in_deb( list_ar(f) )
3999 p=PopenPipe(['ar','p',f,c,'|','tar',b,'-xp','-f','-','-O','./control'],
4000 stdout = subprocess.PIPE, ** Popen_args)
4001 scan_control(p.stdout, params=a)
4002 p.close()
4003 return a
4004
4005 def scan_deb_bydict(a):
4006 c,b,z = untar_control_in_deb( list_ar(a['File']) )
4007 p=PopenPipe(['ar','p',a['File'],c,'|','tar',b,'-xp','-f','-','-O','./control'],
4008 stdout = subprocess.PIPE)
4009 scan_control(p.stdout, params=a)
4010 p.close()
4011 return a
4012
4013 def scan_delta_dir(d, debname, scanned_delta_dirs, old_deltas_by_pack_arch):
4014 if (d,debname) in scanned_delta_dirs or (d,None) in scanned_delta_dirs :
4015 return
4016 if not os.path.isdir(d) :
4017 if VERBOSE > 2 and DEBUG : logger.debug(' No such delta dir: %r',d)
4018 scanned_delta_dirs.add((d,None)) #trick, if aint there no need to retry
4019 return
4020 assert debname == None or type(debname) in string_types
4021 scanned_delta_dirs.add((d,debname))
4022 for n in os.listdir(d):
4023 if debname != None and debname != n.split('_')[0]:
4024 continue
4025 scan_delta( os.path.join(d,n) , old_deltas_by_pack_arch)
4026
4027 def scan_delta(f, old_deltas_by_pack_arch):
4028 assert( os.path.isfile(f) )
4029 if f[-9:] == '.debdelta' :
4030 a=f[:-9]
4031 elif f[-17:] == '.debdelta-too-big' :
4032 a=f[:-17]
4033 elif f[-15:] == '.debdelta-fails' :
4034 a=f[:-15]
4035 else: return
4036 a=os.path.basename(a)
4037 a=a.split('_')
4038 pa=a[0]
4039 ar=a[3]
4040 if (pa,ar) not in old_deltas_by_pack_arch:
4041 old_deltas_by_pack_arch[ (pa,ar) ]=[]
4042 ov=version_demangle(a[1])
4043 nv=version_demangle(a[2])
4044 if (f,ov,nv) not in old_deltas_by_pack_arch[ (pa,ar) ]:
4045 old_deltas_by_pack_arch[ (pa,ar) ].append( (f, ov, nv ) )
4046
4047 def delta_dirname(f,altdir):
4048 "f=directory, altdir=ALT or DIR with // convention. Returns augmented dirname"
4049 if altdir != None:
4050 if altdir[-2:] == '//' :
4051 if '../' in f:
4052 logger.warning('Warning: cannot combine .. and // ! Saving in %r', altdir)
4053 return altdir
4054 #os.path.join has a weird behaviour with absolute paths!
4055 f=f.lstrip('/')
4056 return os.path.join(altdir[:-2], f)
4057 else:
4058 return altdir
4059 else:
4060 #this happens when DIR is not set, and the delta goes in the same directory as the deb
4061 return f
4062
4063 def go_fishing(deb,others, loglevel=logging.ERROR):
4064 "Find a package filepath if missing, save it in deb['File']. deb = dict representing package ; others: list of such dicts."
4065 if 'File' in deb:
4066 assert os.path.isfile(deb['File'])
4067 return deb['File']
4068 of=deb['Filename']
4069 ob=os.path.basename(of)
4070 #try to build it from its own info
4071 if 'Basepath' in deb:
4072 f=os.path.join(deb['Basepath'],of)
4073 if os.path.exists(f):
4074 deb['File']=f
4075 if DEBUG > 1 : logger.debug('Fish! %r',deb)
4076 return f
4077 #try to build it from others info
4078 others=[a for a in others if id(a) != id(deb) ]
4079 for new in others:
4080 if 'File' in new:
4081 f=new['File']
4082 if os.path.basename(f) == ob:
4083 deb['File']=f
4084 if DEBUG > 1 : logger.debug('Fish! %r %r',deb,new)
4085 return f
4086 if 'Basepath' in new:
4087 for a in of,ob:
4088 f=os.path.join(new['Basepath'],a)
4089 if os.path.exists(f):
4090 deb['File']=f
4091 if DEBUG > 1 : logger.debug('Fish! %r %r',deb,new)
4092 return f
4093 logger.log(loglevel, 'bad fishing for %r %r', deb, others)
4094 return False
4095
4096 def order_by_version(a, b, VersionCompare):
4097 # lazy packages do not have versions; but are always ALT, so we sort them at bottom
4098 if 'Version' not in a:
4099 return -1
4100 if 'Version' not in b:
4101 return 1
4102 return VersionCompare( a['Version'] , b['Version'] )
4103
4104
4105 def iter_deltas_one_pack_arch(pa,ar,info_pack,thedir,theforensicdir,VersionCompare,
4106 loglevel=logging.ERROR):
4107 " iterate deltas to be created for package pa, architecture ar, by analyzing info_pack"
4108 if all([('CMDLINE' != a['Label']) for a in info_pack ]):
4109 #this happens a lot, e.g. when we are scanning non-free/Packages.gz,
4110 # all free packages in the trash are to be ignored
4111 if DEBUG > 2 or VERBOSE > 3 : logger.debug(' No cmdline for: %r %r',pa,ar)
4112 return
4113
4114 #do we need the versions of --alt packages ? seems not
4115 # for a in info_pack:
4116 # #scan all lazy packages
4117 # if 'Version' not in a:
4118 # assert a['Label'] == 'ALT'
4119 # scan_deb_bydict(a)
4120
4121 import functools
4122 mykey = functools.cmp_to_key(lambda x,y : order_by_version(x, y, VersionCompare))
4123 info_pack.sort(key = mykey)
4124
4125 how_many= len(info_pack)
4126
4127 if how_many <= 1 :
4128 if VERBOSE > 3 : logger.debug(' Only one version: %r %r', pa,ar)
4129 return
4130
4131 newest = how_many -1
4132 while newest >= 0 :
4133 new=info_pack[newest]
4134 if new['Label'] != 'CMDLINE' :
4135 if VERBOSE > 3 : logger.debug(' Newest version deb was not in cmdline, skip down one: %r', new)
4136 else:
4137 break
4138 newest -= 1
4139
4140 if newest <= 0 :
4141 if VERBOSE > 3 : logger.debug(' No older versions: %r', new)
4142 return
4143
4144 if not go_fishing(new,[], loglevel):
4145 logger.log(loglevel, 'Cannot locate new file: %r',new)
4146 return
4147
4148 if VERBOSE > 2 :
4149 logger.debug(' Package: %r %r Versions: %r', pa, ar, [ (o.get('Version'),o['Label'],o.get('Packages'),o.get('File')) for o in info_pack])
4150
4151 newdebsize=os.path.getsize(new['File'])
4152 #very small packages cannot be effectively delta-ed
4153 if newdebsize <= MIN_DEB_SIZE :
4154 if VERBOSE > 1 : logger.debug(' Skip , too small: %r', new['File'])
4155 return
4156
4157 oldn = newest
4158 generated=0
4159 seen_versions=[]
4160 while (oldn>0) :
4161 oldn -= 1
4162
4163 old=info_pack[oldn]
4164
4165 if old['Label'] != 'OLD':
4166 if VERBOSE > 2 : logger.debug(' Not old, skip: %r',old)
4167 continue
4168
4169 if old['Version'] == new['Version'] :
4170 if VERBOSE > 3 and old != new : logger.debug(' Identical versions: %r %r', old, new)
4171 continue
4172
4173 assert( old['Package'] == pa and pa == new['Package'] )
4174 deltabasename = delta_base_name(pa, old['Version'], new['Version'], ar)
4175
4176 if 'Filename' in new:
4177 deltadirname=delta_dirname(os.path.dirname(new['Filename']),thedir)
4178 elif 'File' in new:
4179 deltadirname=delta_dirname(os.path.dirname(new['File']),thedir)
4180 else:
4181 assert(0)
4182 if deltadirname == '' :
4183 deltadirname = '.'
4184
4185 delta=os.path.join(deltadirname,deltabasename)
4186
4187 generated+=1 #count also those already generated
4188 if N_DELTAS != None and (generated > N_DELTAS):
4189 continue
4190
4191 if os.path.exists(delta):
4192 if VERBOSE > 1 : logger.debug(' Skip , already exists: %r',delta)
4193 continue
4194
4195 if old['Package'] in seen_versions:
4196 if VERBOSE > 3 : logger.debug(' Skip , already considered: %r',delta)
4197 continue
4198
4199 if os.path.exists(delta+'-too-big'):
4200 if VERBOSE > 1 : logger.debug(' Skip , tried and too big: %r',delta)
4201 continue
4202
4203 if os.path.exists(delta+'-fails'):
4204 if VERBOSE > 1 : logger.debug(' Skip , tried and fails: %r',delta)
4205 continue
4206
4207 if not go_fishing(old,info_pack, loglevel):
4208 logger.log(loglevel, 'Cannot locate old file: %r',old)
4209 continue
4210
4211 forensicfile=None
4212 if theforensicdir:
4213 if 'Filename' in new:
4214 forensicdirname=delta_dirname(os.path.dirname(new['Filename']),theforensicdir)
4215 elif 'File' in new:
4216 forensicdirname=delta_dirname(os.path.dirname(new['File']),theforensicdir)
4217 else:
4218 assert(0)
4219 forensicbasename = pa +'_'+ version_mangle(old['Version']) +'_'+ar+'.forensic'
4220 forensicfile=os.path.join(forensicdirname,forensicbasename)
4221
4222 seen_versions.append(old['Package'])
4223
4224 yield (old,new, delta, forensicfile)
4225
4226
4227 def do_delta_and_test(old_File, new_File, delta, forensicfile):
4228 " returns (error_code, delta_size, delta_elaps, patch_elaps, info_delta, error_string) : for error_code, see EXIT STATUS in 'man debdelta' "
4229 exitstatus=0
4230 if VERBOSE : logger.info('Creating: %r',delta)
4231
4232 deltatmp=delta+'_tmp_'
4233 ret= None
4234 tdir=tempo()
4235 ret = None # data returned from calls
4236 elaps = None # delta time
4237 delta_size = None
4238 p_elaps = None # patching time
4239 info_delta = [] # informations about created delta
4240 error_string = '' # error when delta fails
4241
4242 deltadirname=os.path.dirname(delta)
4243 if not os.path.exists(deltadirname): #FIXME this does not respect --no-act
4244 os.makedirs(deltadirname)
4245
4246 free=freespace(deltadirname)
4247 newdebsize=os.path.getsize(new_File)
4248 if free and (free < (newdebsize /2 + 2**15)) :
4249 if VERBOSE : logger.warning(' '+(_("Not enough disk space for storing `%s'.")), delta)
4250 return (1, None, None, None, [], 'not_enough_disk_space')
4251
4252 if forensicfile and not os.path.exists(forensicfile):
4253 forensicdirname=os.path.dirname(forensicfile)
4254 if not os.path.exists(forensicdirname): #FIXME this does not respect --no-act
4255 os.makedirs(forensicdirname)
4256 forensic_file=open(forensicfile,'w')
4257 else:
4258 forensic_file=None
4259
4260 try:
4261 ret=do_delta_(old_File,new_File, deltatmp, TD=tdir, forensic_file=forensic_file, info=info_delta)
4262 (deltatmp_, percent, elaps, info_delta, gpg_hashes) = ret
4263 if os.path.exists(deltatmp):
4264 delta_size = os.path.getsize(deltatmp)
4265 except (KeyboardInterrupt, SystemExit):
4266 if os.path.exists(deltatmp):
4267 os.unlink(deltatmp)
4268 rmtree(tdir)
4269 raise
4270 except DebDeltaError:
4271 s=sys.exc_info()[1]
4272 error_string=str(s)
4273 if not VERBOSE : logger.info(_('Creating:')+' '+repr(delta))
4274 logger.info(' Creation of delta failed, reason: %r', error_string)
4275 if os.path.exists(deltatmp):
4276 if KEEP:
4277 logger.info(' '+_('You may want to examine:')+' '+str(deltatmp))
4278 else:
4279 os.unlink(deltatmp)
4280 if not s.retriable :
4281 open(delta+'-fails','w').close()
4282 exitstatus=max(exitstatus, s.exitcode)
4283 ret = None
4284 except:
4285 exitstatus=4
4286 if os.path.exists(deltatmp):
4287 os.unlink(deltatmp)
4288 puke( " *** Error while creating delta "+delta)
4289 open(delta+'-fails','w').close()
4290 ret = None
4291
4292 rmtree(tdir)
4293
4294 if ret == None:
4295 return (exitstatus, delta_size, elaps, p_elaps, info_delta, error_string)
4296
4297 info_delta.append('ServerID: '+HOSTID)
4298 info_delta.append('ServerBogomips: '+str(BOGOMIPS))
4299
4300 if MAX_DELTA_PERCENT and percent > MAX_DELTA_PERCENT:
4301 os.unlink(deltatmp)
4302 if VERBOSE : logger.info(' Warning, too big!')
4303 open(delta+'-too-big','w').close()
4304 return (exitstatus, delta_size, elaps, p_elaps, info_delta, 'too_big')
4305 #
4306 if DO_TEST :
4307 ##patch test
4308 pret=None
4309 try:
4310 #test, ignoring gpg, that is added later on
4311 pret=do_patch(deltatmp,old_File,None , info=info_delta, do_gpg=None)
4312 except DebDeltaError:
4313 s=sys.exc_info()[1]
4314 logger.warning(' '+_('Error: testing of delta failed:')+' '+str(s))
4315 if os.path.exists(deltatmp):
4316 if KEEP:
4317 logger.warning(' '+_('You may want to examine:')+' '+str(deltatmp))
4318 else:
4319 os.unlink(deltatmp)
4320 if not s.retriable :
4321 open(delta+'-fails','w').close()
4322 except (KeyboardInterrupt, SystemExit):
4323 if os.path.exists(deltatmp):
4324 os.unlink(deltatmp)
4325 raise
4326 except Exception:
4327 s=sys.exc_info()[1]
4328 exitstatus=max(exitstatus,4)
4329 puke(" *** Error while testing delta "+delta,s)
4330 if os.path.exists(deltatmp):
4331 if KEEP:
4332 logger.warning(' '+_('You may want to examine:')+' '+str(deltatmp))
4333 else:
4334 os.unlink(deltatmp)
4335 open(delta+'-fails','w').close()
4336 if pret == None:
4337 return (max(exitstatus, 4), delta_size, elaps, p_elaps, info_delta, 'patch_fails')
4338
4339 (newdeb_,p_elaps)=pret
4340 info_delta.append('PatchTime: %.2f' % p_elaps)
4341 ##end patch test
4342 #complete the delta file prepending to it the info
4343 try:
4344 hashes_info=append_info(deltatmp,info_delta)
4345 # sign the delta
4346 if DO_GPG:
4347 gpg_hashes['info']=hashes_info
4348 sign_delta(deltatmp,gpg_hashes)
4349 except:
4350 puke('debdeltas signing')
4351 if os.path.exists(deltatmp):
4352 os.unlink(deltatmp)
4353
4354 #eventually, put in its place
4355 if os.path.isfile(deltatmp):
4356 # update delta size , since info and signatures were added
4357 delta_size = os.path.getsize(deltatmp)
4358 os.rename(deltatmp,delta)
4359 return (exitstatus, delta_size, elaps, p_elaps, info_delta, '')
4360 else:
4361 return (max(exitstatus, 4), delta_size, elaps, p_elaps, info_delta, 'no_delta')
4362
4363
4364 def init_apt_return_VersionCompare():
4365 try:
4366 import apt_pkg
4367 if 'init_system' in dir(apt_pkg):
4368 apt_pkg.init_system()
4369 elif 'InitSystem' in dir(apt_pkg):
4370 apt_pkg.InitSystem()
4371 else: assert 'fegrwq' == 0
4372 if 'VersionCompare' in dir(apt_pkg):
4373 from apt_pkg import VersionCompare
4374 elif 'version_compare' in dir(apt_pkg):
4375 from apt_pkg import version_compare as VersionCompare
4376 else: assert 'lopadf' == 0
4377 except ImportError:
4378 raise DebDeltaError('python module "apt_pkg" is missing. Please install python-apt', retriable=True)
4379 return VersionCompare
4380
4381
4382 ############
4383
4384 def do_deltas(debs):
4385 exitstatus=0
4386
4387
4388 VersionCompare = init_apt_return_VersionCompare()
4389
4390 if not debs:
4391 logger.warning(_('Warning, no non-option arguments, debdeltas does nothing.'))
4392 if not DEBUG:
4393 return
4394 elif not OLD and not CLEAN_DELTAS:
4395 logger.warning(_('Warning, no --old arguments, debdeltas will not generate any deltas.'))
4396 if not DEBUG:
4397 return
4398
4399 original_cwd = os.getcwd()
4400 start_time = time.time()
4401
4402 info_by_pack_arch={}
4403
4404 info_by_file = {} #cache to speed up scanning file packages
4405
4406 scanned_delta_dirs=set()
4407
4408 # contains list of triples (filename,oldversion,newversion)
4409 old_deltas_by_pack_arch={}
4410
4411 ALT_DOUBLEDIR=[] # directories ending in //
4412 for arg in ALT:
4413 if os.path.isfile(arg):
4414 if os.path.basename(arg) in ('Packages', 'Packages.gz','Packages.bz2'):
4415 for a in iterate_Packages(arg):
4416 a['Label'] = 'ALT'
4417 #DEBUG a['Packages'] = arg
4418 info_by_pack_arch_add(a, info_by_pack_arch)
4419 elif arg[-4: ] == '.deb' :
4420 a=scan_deb_byfile_lazy(arg, info_by_file)
4421 if a:
4422 a['Label'] = 'ALT'
4423 info_by_pack_arch_add(a, info_by_pack_arch)
4424 else: logger.error('Error! skipping file --alt %r',arg)
4425 elif os.path.isdir(arg):
4426 if arg[-2:] != '//' :
4427 scan_deb_dir(arg, None, 'ALT', True, info_by_pack_arch)
4428 else:
4429 ALT_DOUBLEDIR.append(arg)
4430 else: logger.error('Error, ignored --alt %r',arg)
4431
4432 for arg in OLD:
4433 if os.path.isfile(arg):
4434 if os.path.basename(arg) in ('Packages', 'Packages.gz','Packages.bz2'):
4435 for a in iterate_Packages(arg):
4436 a['Label'] = 'OLD'
4437 #DEBUG a['Packages'] = arg
4438 info_by_pack_arch_add(a, info_by_pack_arch)
4439 elif arg[-4: ] == '.deb' :
4440 a=scan_deb_byfile(arg, info_by_file)
4441 if a:
4442 a['Label'] = 'OLD'
4443 info_by_pack_arch_add(a, info_by_pack_arch)
4444 else: logger.error('Error! skipping file --old %r',arg)
4445 elif os.path.isdir(arg):
4446 # no // support yet
4447 scan_deb_dir(arg, None, 'OLD', False, info_by_pack_arch,info_by_file)
4448 else: logger.error('Error, ignored --old %r',arg)
4449
4450 #scan cmdline arguments and prepare list of debs and deltas
4451 for arg in debs:
4452 if os.path.isfile(arg):
4453 if os.path.basename(arg) in ('Packages', 'Packages.gz','Packages.bz2'):
4454 for a in iterate_Packages(arg):
4455 a['Label'] = 'CMDLINE' #note that, if a file is scanned more than once, the last label is CMDLINE
4456 #DEBUG a['Packages'] = arg
4457 ofd = os.path.dirname(a['Filename'])
4458 pa = a['Package']
4459 info_by_pack_arch_add(a, info_by_pack_arch)
4460 for alt in ALT_DOUBLEDIR:
4461 scan_deb_dir(delta_dirname(ofd,alt), pa , 'ALT', True, info_by_pack_arch)
4462 if CLEAN_DELTAS:
4463 scan_delta_dir(delta_dirname(ofd,DIR), pa, scanned_delta_dirs, old_deltas_by_pack_arch)
4464 elif arg[-4: ] == '.deb':
4465 a=scan_deb_byfile(arg, info_by_file)
4466 if a :
4467 a['Label'] = 'CMDLINE'
4468 info_by_pack_arch_add(a, info_by_pack_arch)
4469 ofd = os.path.dirname(arg)
4470 for alt in ALT_DOUBLEDIR:
4471 scan_deb_dir(delta_dirname(ofd,alt), a['Package'], 'ALT', True, info_by_pack_arch, info_by_file)
4472 if CLEAN_DELTAS:
4473 scan_delta_dir(delta_dirname(ofd,DIR), a['Package'], scanned_delta_dirs, old_deltas_by_pack_arch)
4474 else: logger.error('Error! skipping cmdline argument: %r',arg)
4475 elif os.path.isdir(arg):
4476 scan_deb_dir(arg, None, 'CMDLINE', False, info_by_pack_arch, info_by_file)
4477 for alt in ALT_DOUBLEDIR:
4478 scan_deb_dir(delta_dirname(arg,alt), None, 'ALT', True, info_by_pack_arch, info_by_file)
4479 if CLEAN_DELTAS:
4480 scan_delta_dir(delta_dirname(arg,DIR), None, scanned_delta_dirs, old_deltas_by_pack_arch)
4481 else:
4482 logger.error('Error, skipping cmd line argument: %r',arg)
4483
4484 if VERBOSE > 1 : logger.info(' total parsing time: %.1f ', ( -start_time + time.time()))
4485
4486 for pa,ar in info_by_pack_arch :
4487 info_pack=info_by_pack_arch[ (pa,ar) ]
4488 assert(info_pack)
4489 # delete old deltas
4490 if CLEAN_DELTAS :
4491 if (pa,ar) in old_deltas_by_pack_arch :
4492 versions=[(o['Version']) for o in info_pack if (o['Label'] == 'CMDLINE')]
4493 timegrace=time.time() - CLEAN_DELTAS_MTIME * 24 * 3600
4494 #DEBUG print pa,ar,versions
4495 for f_d,o_d,n_d in old_deltas_by_pack_arch[ (pa,ar) ] :
4496 if n_d not in versions :
4497 if os.path.exists(f_d):
4498 if os.stat(f_d)[ST_MTIME] < timegrace:
4499 if VERBOSE: logger.debug(' Removing: %r', f_d)
4500 if ACT: os.unlink(f_d)
4501 elif VERBOSE > 1 : logger.debug(' Obsolete but still young, graced: %r',f_d)
4502 else:
4503 logger.debug('bellandata %r',f_d)
4504 del versions
4505 elif DEBUG > 1:
4506 logger.debug('No deltas where found for: %r %r',pa,ar)
4507 if N_DELTAS != None and 0 == N_DELTAS:
4508 #debdeltas was invoked just to clean the archive
4509 pass
4510 else:
4511 for old, new, delta, forensicfile in \
4512 iter_deltas_one_pack_arch(pa,ar,info_pack, DIR, FORENSICDIR, VersionCompare):
4513 if not ACT:
4514 logger.info('Would create: %r',delta)
4515 else:
4516 e=do_delta_and_test(old['File'], new['File'], delta, forensicfile)
4517 exitstatus=max(exitstatus,e[0])
4518
4519 if VERBOSE : logger.info(' '+ ( _('Total running time: %.1f')),( -start_time + time.time()))
4520 return exitstatus
4521
4522
4523 ##################################################### delta-upgrade
4524
4525 class Predictor:
4526 package_stats = None
4527 upgrade_stats = None
4528 def __init__(self):
4529 import shelve
4530 #self.shelve=shelve
4531 if os.getuid() == 0:
4532 basedir='/var/lib/debdelta'
4533 else:
4534 if not os.path.exists(os.path.expanduser('~/')):
4535 logger.warning('(home directory does not exists, Predictor disabled)')
4536 return
4537 basedir=os.path.expanduser('~/.debdelta')
4538
4539 s=os.path.join(basedir,'upgrade.db')
4540 if not os.path.exists(basedir):
4541 logger.info('Creating: %r',basedir)
4542 os.makedirs(basedir)
4543 self.upgrade_stats=shelve.open(s,flag='c')
4544
4545 s=os.path.join(basedir,'packages_stats.db')
4546
4547 if os.path.exists(s) or DEBUG > 1 :
4548 self.package_stats=shelve.open(s,flag='c')
4549
4550 self.patch_time_predictor=self.patch_time_predictor_math
4551
4552 ##### predictor for patching time
4553 def patch_time_predictor_simple(self,p):
4554 if 'ServerBogomips' in p and 'PatchTime' in p:
4555 return (float(p[ 'PatchTime']) / BOGOMIPS * float(p['ServerBogomips']) )
4556 else:
4557 return None
4558
4559 def update(self,p,t):
4560 #save delta info
4561 if self.package_stats != None :
4562 n=p['NEW/Package']
4563 d=copy(p)
4564 d['LocalDeltaTime']=t
4565 try:
4566 self.package_stats[n]=d
4567 except Exception:
4568 logger.exception('ERROR:Cannot update package_stats')
4569
4570 if self.upgrade_stats == None :
4571 return
4572
4573 s='ServerID'
4574 if s not in p :
4575 return
4576 s=s+':'+p[s]
4577 if s not in self.upgrade_stats:
4578 r=1
4579 if 'ServerBogomips' in p :
4580 r= float(p['ServerBogomips']) / BOGOMIPS
4581 try:
4582 self.upgrade_stats[s]={ 'PatchSpeedRatio' : r }
4583 except Exception:
4584 logger.exception('ERROR:Cannot update upgrade_stats')
4585
4586 if 'PatchTime' not in p:
4587 return
4588 ut=float(p[ 'PatchTime'])
4589
4590 r=self.upgrade_stats[s]['PatchSpeedRatio']
4591
4592 nr = 0.95 * r + 0.05 * ( t / ut )
4593 a=self.upgrade_stats[s]
4594 a['PatchSpeedRatio'] = nr
4595 try:
4596 self.upgrade_stats[s]=a
4597 except Exception:
4598 logger.exception('ERROR:Cannot update upgrade_stats')
4599 if VERBOSE > 1 :
4600 logger.debug(' Upstream %r PatchSpeedRatio from %r to %r , %r', ut, r, nr, self.upgrade_stats[s]['PatchSpeedRatio'])
4601
4602 def patch_time_predictor_math(self,p):
4603 "Predicts time to patch."
4604 if 'PatchTime' not in p:
4605 return None
4606 ut=float(p[ 'PatchTime'])
4607 #
4608 s='ServerID'
4609 if s not in p :
4610 return self.patch_time_predictor_simple(p)
4611 s=s+':'+p[s]
4612 if s not in self.upgrade_stats:
4613 return self.patch_time_predictor_simple(p)
4614
4615 r=self.upgrade_stats[s]['PatchSpeedRatio']
4616 return r * ut
4617
4618 def delta_uri_from_config(config, **dictio):
4619 secs=config.sections()
4620 for s in secs:
4621 opt=config.options(s)
4622 if 'delta_uri' not in opt:
4623 raise DebDeltaError('sources.conf section '+repr(s)+'does not contain delta_uri',exitcode=3)
4624 match=True
4625 for a in dictio:
4626 #damn it, ConfigParser changes everything to lowercase !
4627 if ( a.lower() in opt ) and ( dictio[a] != config.get( s, a) ) :
4628 #print '!!',a, repr(dictio[a]) , ' != ',repr(config.get( s, a))
4629 match=False
4630 break
4631 if match:
4632 return config.get( s, 'delta_uri' )
4633 if VERBOSE : logger.warning(' '+_("(sources.conf does not provide a server for `%s')"), repr(dictio['PackageName']))
4634
4635
4636
4637 def delta_upgrade_(args):
4638 # a list of all error exitcodes that derive from downloading and applying
4639 mainexitcodes = [0]
4640
4641 original_cwd = os.getcwd()
4642
4643 import threading, fcntl, atexit, signal, socket
4644 if sys.version_info.major == 2:
4645 import Queue, urllib2, thread
4646 from httplib import HTTPConnection, HTTPException
4647 def http_responses_get(j):
4648 import httplib
4649 return httplib.responses.get(j,str(j))
4650 from urlparse import urlparse, urlunparse
4651 from urllib2 import quote as urlquote
4652 proxies=urllib2.getproxies()
4653 else:
4654 import queue as Queue
4655 import _thread as thread
4656 from http.client import HTTPConnection, HTTPException
4657 #FIXME
4658 http_responses_get = lambda j : str(j)
4659 from urllib.parse import urlparse, urlunparse
4660 from urllib.parse import quote as urlquote
4661 #FIXME
4662 proxies = []
4663
4664 try:
4665 import requests
4666 requests_session = requests.Session()
4667 except ImportError:
4668 if (sys.version_info.major == 2):
4669 logger.info('You may want to install `requests`\n# sudo apt install python-requests\n')
4670 else:
4671 logger.warning('You should install `requests`\n# sudo apt install python3-requests\n')
4672 requests=None
4673 requests_session=None
4674
4675 if VERBOSE and proxies:
4676 #note that this message is indented, I dont know what's best in translations
4677 logger.warning(_(' Proxy settings detected in the environment; using "urllib2" for downloading; but\n this disables some features and is in general slower and buggier. See man page.'))
4678 #for example, urllib2 transforms http response "401" into "404" , and "302" into "200"
4679
4680 config=ConfigParser()
4681 a=config.read(['/etc/debdelta/sources.conf', expanduser('~/.debdelta/sources.conf') ])
4682 # FIXME this does not work as documented in Python
4683 #if VERBOSE > 1 : print 'Read config files: ',repr(a)
4684
4685 import warnings
4686 warnings.simplefilter("ignore",FutureWarning)
4687
4688 if DO_PROGRESS:
4689 sys.stderr.write((_('Initializing APT cache...')).ljust(terminalcolumns)+'\r')
4690 elif VERBOSE:
4691 logger.debug(_('Initializing APT cache...'))
4692
4693 try:
4694 import apt_pkg
4695 except ImportError:
4696 raise DebDeltaError('python module "apt_pkg" is missing. Please install python-apt',True)
4697
4698 try:
4699 import apt
4700 except ImportError:
4701 raise DebDeltaError('python module "apt" is missing. Please install a newer version of python-apt (newer than 0.6.12).',True)
4702
4703 apt_pkg.init()
4704
4705 #from apt import SizeToStr
4706 ## Return a string describing the size in a human-readable manner using
4707 ## SI prefix and base-10 units, e.g. '1k' for 1000, '1M' for 1000000, etc.
4708
4709 def SizeToKibiStr(a):
4710 "this uses kibibytes (altough the program prints them as kilobytes)"
4711 if a < 8096 :
4712 return str(int(a))+'B'
4713 elif a < 8096*1024:
4714 return str(int(a/1024))+'kB'
4715 else:
4716 return str(int(a/1024/1024))+'MB'
4717
4718 if DO_PROGRESS:
4719 sys.stderr.write((_('Upgrading APT cache...')).ljust(terminalcolumns)+'\r')
4720 elif VERBOSE:
4721 logger.debug(_('Upgrading APT cache...'))
4722
4723 cache=apt.Cache()
4724 try:
4725 cache.upgrade(True)
4726 except:
4727 if DO_PROGRESS:
4728 sys.stderr.write((_('Failed! Safe upgrading APT cache...')).ljust(terminalcolumns)+'\r')
4729 elif VERBOSE:
4730 logger.debug(_('Failed! Safe upgrading APT cache...'))
4731 cache.upgrade(False)
4732
4733 if DO_PROGRESS:
4734 sys.stderr.write((_('Upgraded APT cache.')).ljust(terminalcolumns)+'\r')
4735 elif VERBOSE:
4736 logger.debug(_('Upgraded APT cache.'))
4737
4738 diversions=scan_diversions()
4739
4740 if DIR == None:
4741 if os.getuid() == 0:
4742 DEB_DIR='/var/cache/apt/archives'
4743 else:
4744 DEB_DIR='/tmp/archives'
4745 else:
4746 DEB_DIR=DIR
4747 if not os.path.exists(DEB_DIR):
4748 os.mkdir(DEB_DIR)
4749 if not os.path.exists(DEB_DIR+'/partial'):
4750 os.mkdir(DEB_DIR+'/partial')
4751
4752 try:
4753 ##APT does (according to strace)
4754 #open("/var/cache/apt/archives/lock", O_RDWR|O_CREAT|O_TRUNC, 0640) = 17
4755 #fcntl64(17, F_SETFD, FD_CLOEXEC) = 0
4756 #fcntl64(17, F_SETLK, {type=F_WRLCK, whence=SEEK_SET, start=0, len=0}) = 0
4757 ##so
4758 a=os.open(DEB_DIR+'/lock', os.O_RDWR | os.O_TRUNC | os.O_CREAT, 0o640)
4759 fcntl.fcntl(a, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
4760 # synopsis lockf( fd, operation, [length, [start, [whence]]])
4761 fcntl.lockf(a, fcntl.LOCK_EX | fcntl.LOCK_NB, 0,0,0)
4762 except IOError:
4763 s=sys.exc_info()[1]
4764 from errno import EAGAIN
4765 if s.errno == EAGAIN :
4766 a=' already locked!'
4767 else:
4768 a=str(s)
4769 if DEB_DIR == '/var/cache/apt/archives' :
4770 a=a+' (is APT running?)'
4771 raise DebDeltaError('could not lock dir: '+DEB_DIR+' '+a, retriable=True)
4772
4773 if VERBOSE or DEB_DIR != '/var/cache/apt/archives' :
4774 logger.info(_('Recreated debs are saved in the directory %s'), DEB_DIR)
4775
4776 #these are the packages that do not have a delta
4777 no_delta = []
4778
4779 total_time = -time.time()
4780
4781 ##### predictor for patching time
4782 if DO_PREDICTOR:
4783 predictor = Predictor()
4784
4785 #this is a dictonary (key is package name) of parameters of deltas
4786 #(to add some math in the future)
4787 params_of_delta={}
4788
4789 patching_queue=Queue.Queue()
4790 thread_returns={}
4791 ######################## thread_do_patch
4792 def thread_do_patch(que, no_delta, returns, exitcodes, forensics):
4793 if VERBOSE > 1 : logger.info(' Patching thread started. ')
4794 debs_size=0
4795 debs_time=0
4796 # in case of early exception
4797 returns['debs_size']=debs_size
4798 returns['debs_time']=debs_time
4799 while 1:
4800 a = que.get()
4801 if a == None:
4802 break
4803 (name, delta , newdeb, deb_uri) = a
4804 debs_time -= time.time()
4805 TD=tempo()
4806 if not ACT:
4807 logger.info('Would create: %r ',newdeb)
4808 else:
4809 if VERBOSE >= 2 : logger.debug(' Now patching for: %r',name)
4810 try:
4811 #start_time=time.time()
4812 returns['patchname']=os.path.basename(newdeb)
4813 ret=do_patch_(delta, '/', newdeb , TD, returns,
4814 diversions=diversions, do_progress=False)
4815 del returns['patchname']
4816 l = os.path.getsize(newdeb)
4817 #a=time.time() - start_time
4818 a=ret[1]
4819 #dear translator, please align this line with the line saying 'Downloaded, ...'
4820 msg=_("Created, time %(time)5.2fsec, speed %(speed)4s/sec, %(name)s")
4821 msgdata={'time' : a, 'speed' : SizeToKibiStr(l / (a+0.001)), 'name' : os.path.basename(newdeb)}
4822 if DO_PROGRESS:
4823 sys.stderr.write((msg % msgdata).ljust(terminalcolumns)+'\n')
4824 else:
4825 logger.info(msg % msgdata)
4826 except KeyboardInterrupt:
4827 thread.interrupt_main()
4828 rmtree(TD)
4829 return
4830 except DebDeltaError:
4831 s=sys.exc_info()[1]
4832 puke(' Error: applying of delta for '+name+' failed: ',s)
4833 if 'e' in DEB_POLICY:
4834 no_delta.append( (deb_uri, newdeb) )
4835 elif VERBOSE > 1 : logger.debug(' No deb-policy "e", no download of %r',deb_uri)
4836 forensics.append(s.logs)
4837 exitcodes.append(s.exitcode)
4838 except:
4839 if puke == None: return
4840 puke( " *** Error while applying delta for "+name+": ")
4841 if 'e' in DEB_POLICY:
4842 no_delta.append( (deb_uri, newdeb) )
4843 elif VERBOSE > 1 : logger.debug(' No deb-policy "e", no download of %r',deb_uri)
4844 exitcodes.append(4)
4845 else:
4846 if name in params_of_delta :
4847 p= params_of_delta[name]
4848 name,elaps=ret
4849 if DO_PREDICTOR:
4850 predictor.update(p,elaps)
4851 if VERBOSE > 1 :
4852 t=predictor.patch_time_predictor(p)
4853 if t: logger.debug(' (Predicted %.3f sec )', t)
4854 debs_size += os.path.getsize(newdeb)
4855 if os.path.exists(delta):
4856 os.unlink(delta)
4857 rmtree(TD)
4858 debs_time += time.time()
4859 returns['debs_size']=debs_size
4860 returns['debs_time']=debs_time
4861 if VERBOSE > 1 : logger.debug(' Patching thread ended , bye bye. ')
4862
4863 #####################################
4864
4865 def progress_string(statusdb):
4866 download=''
4867 if 'downloaduri' in statusdb:
4868 download="D %2d%% (%4s/s) %s " % \
4869 (statusdb.get('downloadprogress',-1),
4870 statusdb.get('downloadspeed','-'),
4871 statusdb['downloaduri'])
4872 patch=''
4873 if 'patchname' in statusdb:
4874 patch='P %2d%% %s'% (statusdb.get('patchprogress',-1), statusdb['patchname'])
4875 if terminalcolumns == None:
4876 return download+' ; '+patch
4877 if not patch:
4878 return download[:terminalcolumns]
4879 if not download:
4880 return patch[:terminalcolumns]
4881 ld=len(download)
4882 lp=len(patch)
4883 b=ld + lp
4884 if b < terminalcolumns - 3 :
4885 return download+' ; '+patch
4886 a=float(terminalcolumns-5)/float(b)
4887 ld=int(ld*a)
4888 lp=int(lp*a)
4889 return download[:ld] + ' ; ' + patch[:lp] + '..'
4890
4891
4892 #################### manage connections
4893 #keeps a cache of all connections, by URL
4894 http_conns={}
4895
4896 def conn_by_url(url):
4897 url=urlparse(url)[1]
4898 if url not in http_conns:
4899 if VERBOSE > 1 : logger.debug(' Opening connection to: %r',url)
4900 http_conns[url] = HTTPConnection(url, timeout=TIMEOUT)
4901 return http_conns[url]
4902
4903 def conn_close(url,fatal=False):
4904 url=urlparse(url)[1]
4905 conn=http_conns.get(url)
4906 if fatal:
4907 http_conns[url] = None
4908 else:
4909 del http_conns[url]
4910 if conn != None :
4911 if VERBOSE > 1 : logger.debug(' Closing connection to: %r',url)
4912 conn.close()
4913
4914 ####
4915
4916 def _connect(uri, headers):
4917 """connects for a GET ; returns (filetype, statuscode, servermessage, getheaders)
4918 in case of errors returns (exception, statuscode, None, None)
4919 """
4920 uri_p=urlparse(uri)
4921 ###### python3 code
4922 if requests and uri_p.scheme in ('http','https') :
4923 try:
4924 req = requests_session.get(uri, headers=headers, stream = True)
4925 req.read = req.raw.read
4926 return req, req.raw.status, getattr(req.raw,'msg','(no message)'), req.raw.info()
4927 except requests.exceptions.ConnectionError:
4928 e=sys.exc_info()[1]
4929 puke( 'Connection error (fatal): %r'%(uri,),e)
4930 return e, None, None, None
4931 except requests.exceptions.RequestException:
4932 e=sys.exc_info()[1]
4933 puke( 'Error in http request (fatal): %r'%(uri,),e)
4934 return e, None, None, None
4935 elif sys.version_info.major == 3:
4936 import urllib.request
4937 try:
4938 req = urllib.request.urlopen(uri,)
4939 return req, getattr(req,'code',None), getattr(req,'msg','(no message)'), req.info()
4940 except urllib.request.HTTPError:
4941 e=sys.exc_info()[1]
4942 return e, e.code, None, None
4943 except Exception:
4944 e=sys.exc_info()[1]
4945 #puke( 'Exception while accessing',uri)
4946 return DebDeltaError('for %s : %s'%(uri,e)), None, None, None
4947 ###### python2 code
4948 if uri_p.scheme in ('http','https') and not proxies:
4949 #use persistent http connections
4950 conn=conn_by_url(uri)
4951 if conn == None :
4952 return None, None, None, None
4953 try:
4954 conn.request("GET", urlquote(uri_p[2]),headers=headers)
4955 r = conn.getresponse()
4956 return r, r.status, r.reason, r.msg
4957 except (HTTPException, socket.error):
4958 e=sys.exc_info()[1]
4959 if VERBOSE : puke( ' Connection error (retrying): %r'%(uri_p[1],),e)
4960 conn_close(uri)
4961 try:
4962 conn=conn_by_url(uri)
4963 conn.request("GET", urlquote(uri_p[2]),headers=headers)
4964 r = conn.getresponse()
4965 return r, r.status, r.reason, r.msg
4966 except (HTTPException, socket.error):
4967 e=sys.exc_info()[1]
4968 puke( 'Connection error (fatal): ',uri_p[1])
4969 mainexitcodes.append(1)
4970 try:
4971 conn_close(uri,fatal=True)
4972 except: pass
4973 mainexitcodes.append(1)
4974 return e, None, None, None
4975 else: #use urllib2
4976 if uri_p.scheme in ('http','https'):
4977 a=[copy(z) for z in uri_p]
4978 a[2]=urlquote(uri_p[2])
4979 uri=urlunparse(a)
4980 import urllib2
4981 try:
4982 req = urllib2.Request(uri, headers=headers)
4983 r = urllib2.urlopen(req)
4984 #print r.info(),dir(r),r.code
4985 return r, getattr(r,'code',None), getattr(r,'msg','(no message)'), r.info()
4986 except urllib2.HTTPError:
4987 e=sys.exc_info()[1]
4988 return e.code, None, None, None
4989 except (HTTPException,socket.error,urllib2.URLError):
4990 e=sys.exc_info()[1]
4991 puke( 'Connection error (fatal): %r'%(uri,), e)
4992 mainexitcodes.append(1)
4993 return e, None, None, None
4994
4995 ################################################# various HTTP facilities
4996
4997 def _parse_ContentRange(s):
4998 #bytes 0-1023/25328
4999 if not s or s[:6] != "bytes " :
5000 logger.warning("Malformed Content-Range %r", s)
5001 return
5002 a=s[6:].split('/')
5003 if len(a) != 2 :
5004 logger.warning("Malformed Content-Range %r", s)
5005 return
5006 b=a[0].split('-')
5007 if len(b) != 2 :
5008 logger.warning("Malformed Content-Range %r", s)
5009 return
5010 return int(b[0]),int(b[1]),int(a[1])
5011 ###################################### test_uri
5012 def test_uri(uri):
5013 conn=conn_by_url(uri)
5014 if conn == None: return None
5015 uri_p=urlparse(uri)
5016 assert(uri_p[0] in ('http','https'))
5017 conn.request("HEAD", urlquote(uri_p[2]),headers=HTTP_USER_AGENT)
5018 r = conn.getresponse()
5019 r.read()
5020 r.close()
5021 return r.status
5022
5023 ###################################### download_10k_uri
5024 def download_10k_uri(uri,outname):
5025 "in case of connection error, returns the (error, None, None, None) ; otherwise returns (status,len,outname,complete)"
5026 #download
5027 uri_p=urlparse(uri)
5028 assert(uri_p[0] in ('http', 'https'))
5029 re=copy(HTTP_USER_AGENT)
5030 re["Range"] = "bytes=0-10239"
5031 complete=False
5032 r, status, msg, responseheaders=_connect(uri, re)
5033 if not hasattr(r,'read') and responseheaders==None:
5034 return r, None, None, None
5035 if status == 301 and 'location' in responseheaders:
5036 r.read()
5037 r.close()
5038 return download_10k_uri(responseheaders['location'], outname)
5039 elif status == 206:
5040 outnametemp=os.path.join(os.path.dirname(outname),'partial',os.path.basename(outname))
5041 try:
5042 l = _parse_ContentRange(responseheaders['Content-Range'])[2]
5043 except (KeyError, ValueError):
5044 l = None
5045 elif status == 200:
5046 outnametemp=outname
5047 complete=True
5048 try:
5049 l=long(responseheaders.get('Content-Length'))
5050 except:
5051 l=None
5052 else: #FIXME how do we deal with a FTP mirror of deltas ?
5053 r.read()
5054 r.close()
5055 return status, None, None, None
5056 s=r.read()
5057 r.close()
5058 #(maybe I did not understand the status 206 ?)
5059 if not complete and l != None and len(s) >= l:
5060 outnametemp=outname
5061 complete=True
5062 if os.path.exists(outnametemp) and os.path.getsize(outnametemp) >= len(s) :
5063 #do not truncate preexisting file
5064 complete=(os.path.getsize(outnametemp) >= l)
5065 return status, outnametemp, l, complete
5066 out=open(outnametemp,'wb')
5067 out.write(s)
5068 out.close()
5069 return status, outnametemp, l, complete
5070
5071 ###################################### download_uri
5072 def download_uri(uri, outname, conn_time, len_downloaded, statusdb):
5073 "returns `(time_elapsed, bytes_downloaded)`; or `None` in case of error, "
5074 outnametemp=os.path.join(os.path.dirname(outname),'partial',os.path.basename(outname))
5075 re=copy(HTTP_USER_AGENT)
5076 #content range
5077 l=None
5078 if os.path.exists(outnametemp):
5079 #shamelessly adapted from APT, methods/http.cc
5080 s=os.stat(outnametemp)
5081 l=s[ST_SIZE]
5082 #t=s[ST_MTIME]
5083 ### unfortunately these do not yet work
5084 #thank god for http://docs.python.org/lib/module-time.html
5085 #actually APT does
5086 #t=time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(t))
5087 ##re["If-Range"] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(t))
5088 ####re["If-Range"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(t))
5089 re["Range"] = "bytes=%li-" % ( (long(l)-1) )
5090 #start downloading
5091 start_time=time.time()
5092 r, status, message, responseheaders = _connect(uri, re)
5093 if not hasattr(r,'read') and responseheaders==None:
5094 return
5095 if status in (301,302) and 'location' in responseheaders:
5096 r.read()
5097 r.close()
5098 if VERBOSE > 1 : logger.debug(_(' Redirect to:')+' '+repr(responseheaders['location']))
5099 return download_uri(responseheaders['location'], outname, conn_time, len_downloaded, statusdb)
5100 if not ( status == None or status == 200 or ( status == 206 and l != None ) ):
5101 if VERBOSE :
5102 logger.warning('Connection problem, status %r msg: %r uri: %r', status, message, uri)
5103 r.read()
5104 r.close()
5105 return
5106 #print 'ooK Content-Range', r.getheader('Content-Range') #HACK
5107 if l and status == 200 :
5108 logger.warning(' Hmmm... our HTTP range request failed, %r %r %r',re,status,message)
5109 if status == 200 :
5110 out=open(outnametemp,'wb')
5111 try:
5112 total_len = long(responseheaders['Content-Length'])
5113 except (KeyError,ValueError):
5114 total_len=None
5115 elif status == 206 :
5116 #APT does scanf of "bytes %lu-%*u/%lu",&StartPos,&Size
5117 #first-byte-pos "-" last-byte-pos "/" instance-length
5118 out=open(outnametemp,'ab')
5119 try:
5120 a,b,total_len =_parse_ContentRange(responseheaders['Content-Range'])
5121 except (KeyError,ValueError):
5122 e=sys.exc_info()[1]
5123 logger.warning('! problem, http response [206], Content Range %s , error %s , uri %s\n' %\
5124 (responseheaders.get('Content-Range'),e,uri))
5125 return
5126 out.seek(a)
5127 out.truncate()
5128 else:
5129 out=open(outnametemp,'wb')
5130 try:
5131 total_len=long(responseheaders.get('Content-length'))
5132 except ValueError:
5133 total_len=None
5134
5135 free=freespace(os.path.dirname(outname))
5136 if total_len != None and free and (free + 2**14 ) < total_len :
5137 logger.warning(_('Not enough disk space to download:')+' '+os.path.basename(uri))
5138 r.read()
5139 r.close()
5140 mainexitcodes.append(1)
5141 return
5142 j=0
5143
5144 s=r.read(1024)
5145 while s and (total_len == None or out.tell() < total_len ):
5146 j+=len(s)
5147 out.write(s)
5148 if total_len:
5149 statusdb['downloadprogress']=99.9 * out.tell() / total_len
5150 a=time.time() + conn_time - start_time
5151 if a > 0.5 :
5152 statusdb['downloadspeed']=SizeToKibiStr(float(j+len_downloaded)/a)
5153 s=r.read(1024)
5154 out.close()
5155 r.close()
5156 #end of download
5157 a=time.time() - start_time
5158 #if total_len == None:
5159 # total_len = os.path.getsize(outnametemp)
5160
5161 #dear translator, please align this line with the line saying 'Created, ...'
5162 msg=_("Downloaded, time %(time)5.2fsec, speed %(speed)4s/sec, %(name)s")
5163 msgdata={'time' : a , 'speed' : SizeToKibiStr(j / (a+0.001)) , 'name' : os.path.basename(uri)}
5164 if DO_PROGRESS:
5165 sys.stderr.write((msg % msgdata).ljust(terminalcolumns)+'\n')
5166 else:
5167 logger.info(msg % msgdata)
5168
5169 os.rename(outnametemp,outname)
5170 #FIXME this is incorrect by 1024 bytes
5171 return conn_time + a, (j+len_downloaded)
5172
5173 ###################################### end of HTTP stuff
5174
5175 ################### start patching thread
5176 forensics=[]
5177
5178 patching_thread=threading.Thread(
5179 target=thread_do_patch ,
5180 args=(patching_queue, no_delta, thread_returns, mainexitcodes, forensics) )
5181 patching_thread.daemon=True
5182 patching_thread.start()
5183
5184 ################### first merry-go-round
5185
5186 deltas_down_size=0
5187 deltas_down_time=0
5188
5189 #this is a list of tuples of .....
5190 available_deltas=[]
5191
5192 not_available_deltas=[]
5193
5194 if hasattr(apt.package.Package,'is_installed'):
5195 is_installed=lambda p : p.is_installed
5196 elif hasattr(apt.package.Package,'isInstalled'):
5197 is_installed=lambda p : p.isInstalled
5198 else: assert 0
5199
5200 if hasattr(apt.package.Package,'marked_upgrade'):
5201 marked_upgrade=lambda p : p.marked_upgrade
5202 elif hasattr(apt.package.Package,'markedUpgrade'):
5203 marked_upgrade=lambda p : p.markedUpgrade
5204 else: assert 0
5205
5206 progress_count=0
5207
5208 ## first merry-go-round, use package cache to fill available_deltas, download 10kB of each delta
5209 for p in cache :
5210 #print progress
5211 if DO_PROGRESS :
5212 progress_count+=1
5213 if 0 == (progress_count & 2047):
5214 sys.stderr.write(('%2.1f%%' % (float(progress_count) * 100.0 / len(cache))).ljust(terminalcolumns) + '\r')
5215
5216 if is_installed(p) and marked_upgrade(p) :
5217 if args and p.name not in args:
5218 continue
5219 #thanks a lot to Julian Andres Klode
5220 candidate=p.candidate
5221 origin = p.candidate.origins[0]
5222 arch=candidate.architecture
5223 deb_uri=candidate.uri
5224 installed_version=p.installed.version
5225 candidate_version=p.candidate.version
5226 newsize=p.candidate.size
5227 deb_path=str.split(deb_uri,'/')
5228 try:
5229 thepoolindex=deb_path.index('pool')
5230 except ValueError:
5231 logger.warning('! Package "%s" (version %s) does not have "pool" in the uri %s \n' % (p.name, candidate_version, deb_uri))
5232 continue
5233 deb_path=str.join('/',deb_path[(thepoolindex):])
5234
5235 #try all possible variants of the filename
5236 newdebs=[p.shortname+'_'+candidate_version+'_'+arch+'.deb',
5237 os.path.basename(deb_uri)]
5238 if ':' in candidate_version:
5239 a=candidate_version.split(':')
5240 newdebs.append(p.shortname+'_'+a[1]+'_'+arch+'.deb')
5241 newdebs.append(p.shortname+'_'+a[0]+'%3A'+a[1]+'_'+arch+'.deb')
5242 newdebs.append(p.shortname+'_'+a[0]+'%3a'+a[1]+'_'+arch+'.deb')
5243
5244 for newdeb in newdebs:
5245 if os.path.exists(DEB_DIR+'/'+newdeb) or \
5246 os.path.exists('/var/cache/apt/archives/'+newdeb):
5247 if VERBOSE > 1 : logger.debug(' Already downloaded %r %r as %r',p.name,candidate_version,newdeb)
5248 newdeb=None
5249 break
5250 if newdeb == None:
5251 continue
5252 newdeb = DEB_DIR+'/'+newdebs[0]
5253
5254 if VERBOSE > 1 : logger.debug(' Looking for a delta for %s from %s to %s ', p.name, installed_version, candidate_version)
5255 delta_uri_base=delta_uri_from_config(config,
5256 Origin=origin.origin,
5257 Label=origin.label,
5258 Site=origin.site,
5259 Archive=origin.archive,
5260 PackageName=p.name)
5261 if delta_uri_base == None:
5262 if 's' in DEB_POLICY:
5263 no_delta.append( (deb_uri, newdeb) )
5264 continue
5265
5266 a=urlparse(delta_uri_base)
5267 assert(a[0] in ('http', 'https'))
5268
5269 #delta name
5270 delta_name = delta_base_name(p.shortname, installed_version,
5271 candidate_version, arch)
5272
5273 uri=delta_uri_base+'/'+os.path.dirname(deb_path)+'/'+delta_name
5274
5275 #download first part of delta
5276 abs_delta_name= DEB_DIR+'/'+delta_name
5277
5278 #maybe it is already here
5279 if os.path.exists(abs_delta_name):
5280 a = abs_delta_name
5281 else:
5282 a = DEB_DIR+'/partial/'+delta_name
5283 if not os.path.exists(a):
5284 a = None
5285 if a:
5286 l=os.path.getsize(a)
5287 if VERBOSE > 1 : logger.debug(' Already here: %r',abs_delta_name)
5288 s=get_info_fast(a)
5289 if s:
5290 params_of_delta[p.name]=info_2_db(s)
5291 available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, a, True) )
5292 continue
5293 #if not, download its first part
5294
5295 if DO_PROGRESS:
5296 sys.stderr.write(('%2.1f%% ' % (float(progress_count) * 100.0 / len(cache)) + \
5297 _('Downloading head of %s...') % p.name).ljust(terminalcolumns) + '\r')
5298 deltas_down_time-=time.time()
5299 status, tempname, l, complete = download_10k_uri(uri,abs_delta_name)
5300 deltas_down_time+=time.time()
5301
5302 #some strange error in remote server?
5303 #FIXME this does not support ftp delta repositories
5304 if status != 200 and status != 206 and status != 404:
5305 logger.warning('Delta is not downloadable (%s):%s', http_responses_get(status), uri)
5306 continue
5307
5308 if status == 404:
5309 not_available_deltas.append(p.name)
5310 # check if delta is too big
5311 if uri.startswith('http') and not proxies and 200 == test_uri(uri+'-too-big'): # FIXME support ftp or proxies
5312 logger.info(_('Delta is too big:')+' '+delta_name)
5313 if 'b' in DEB_POLICY:
5314 no_delta.append( (deb_uri, newdeb) )
5315 elif VERBOSE > 1 : logger.debug(' No deb-policy "b", no download of %r',deb_uri)
5316 continue
5317 # check if delta is queued in the server but not yet done
5318 if uri.startswith('http') and not proxies and 200 == test_uri(uri+'-queued'): # FIXME support ftp or proxies
5319 logger.info(_('Delta is not yet ready in the server:')+' '+delta_name)
5320 if 'q' in DEB_POLICY:
5321 no_delta.append( (deb_uri, newdeb) )
5322 elif VERBOSE > 1 : logger.debug(' No deb-policy "q", no download of %r',deb_uri)
5323 continue
5324 #check if delta failed upstream
5325 if uri.startswith('http') and not proxies and 200 == test_uri(uri+'-fails'): # FIXME support ftp or proxies
5326 logger.info(_('Delta missing, server failed to create it:')+' '+delta_name)
5327 if 'f' in DEB_POLICY:
5328 no_delta.append( (deb_uri, newdeb) )
5329 elif VERBOSE > 1 : logger.debug(' No deb-policy "f", no download of %r',deb_uri)
5330 continue
5331 ### FIXME the server is not generating these stamps !
5332 #if uri.startswith('http') and not proxies and newsize <= 2 * MIN_DEB_SIZE : # check only on small packages
5333 # smallstatus = test_uri(uri+'-smalldeb')
5334 # if smallstatus == 200: ETC ETC
5335 ###
5336 # packages smaller than MIN_DEB_SIZE are ignored by the server.
5337 if newsize <= MIN_DEB_SIZE : # check only on small packages
5338 logger.info(_('Delta was not created since new package is too small:')+' '+delta_name)
5339 if 't' in DEB_POLICY:
5340 no_delta.append( (deb_uri, newdeb) )
5341 elif VERBOSE > 1 : logger.debug(' No deb-policy "t", no download of %r',deb_uri)
5342 continue
5343 if DEBUG and VERBOSE:
5344 logger.info(_('Delta is not present:')+' '+uri)
5345 else:
5346 logger.info(_('Delta is not present:')+' '+delta_name)
5347 if 'u' in DEB_POLICY:
5348 no_delta.append( (deb_uri, newdeb) )
5349 elif VERBOSE > 1 : logger.debug(' No deb-policy "u", no download of %r',deb_uri)
5350 continue
5351
5352 if VERBOSE > 1:
5353 logger.debug('Delta is present: %r %r', delta_name, tempname)
5354 elif DO_PROGRESS:
5355 sys.stderr.write(('%2.1f%%' % (float(progress_count) * 100.0 / len(cache)) + \
5356 _('Downloaded head of %s.') % p.name).ljust(terminalcolumns) + '\r')
5357
5358 if os.path.isfile(tempname):
5359 deltas_down_size+=os.path.getsize(tempname)
5360
5361 #parse file and save info
5362 try:
5363 s=get_info_fast(tempname)
5364 except DebDeltaError:
5365 e=sys.exc_info()[1]
5366 logger.error("!!"+str(e)+'\n')
5367 logger.error("!! (renamed to "+tempname+'~~NOT~A~DELTA~~ )\n')
5368 os.rename(tempname,tempname+'~~NOT~A~DELTA~~')
5369 if proxies:
5370 logger.error("!!maybe a proxy is returning an error page??\n")
5371 else:
5372 logger.error("!!damaged delta??\n")
5373 continue
5374 if s:
5375 params_of_delta[p.name]=info_2_db(s)
5376 s=patch_check_tmp_space(params_of_delta[p.name], '/')
5377 if s != True:
5378 logger.warning('%r : sorry %r', p.name, s)
5379 #neither download deb nor delta..
5380 #the user may wish to free space and retry
5381 continue
5382 #FIXME may check that parameters are conformant to what we expect
5383
5384 if complete:
5385 patching_queue.put( (p.name, abs_delta_name ,newdeb, deb_uri ) )
5386 else:
5387 available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, tempname, complete) )
5388 ## end of first merry-go-round
5389
5390 available_deltas.sort()
5391
5392 if DEBUG or VERBOSE:
5393 if DO_PROGRESS:
5394 sys.stderr.write(' ' * (terminalcolumns) +'\r')
5395 logger.info(' '+ _('Deltas: %(present)d present and %(absent)d not,') % \
5396 {'present' : len(available_deltas), 'absent' : len(not_available_deltas)})
5397 logger.info(' '+ _('downloaded so far: time %(time).2fsec, size %(size)s, speed %(speed)4s/sec.') % \
5398 {'time' : deltas_down_time, 'size' : SizeToKibiStr(deltas_down_size), 'speed' : SizeToKibiStr(deltas_down_size/float(deltas_down_time+0.001))})
5399 if available_deltas:
5400 logger.info(' '+_('Need to get %s of deltas.') , SizeToKibiStr(sum([a[0] for a in available_deltas])))
5401
5402 ###################### check available space
5403 a=freespace('/var/cache/apt/archives') / 1024
5404 b=sum([int(s.get('NEW/Installed-Size','0')) for s in params_of_delta.values()])
5405 c=sum([int(s.get('NEW/Size','0')) for s in params_of_delta.values()]) / 1024
5406 if DEB_FORMAT == 'deb' and a < c:
5407 logger.warning('**'+_('Very low disk space, need %(need)d kB have %(have)d kB'),
5408 {'need':c,'have':a})
5409 if DEB_FORMAT == 'unzipped' and a < b:
5410 logger.warning('**'+_('Very low disk space, need %(need)d kB have %(have)d kB'),
5411 {'need':b,'have':a})
5412
5413 ###################### start progress thread
5414
5415 def print_progress(common_db):
5416 while sys and 'STOP' not in common_db:
5417 sys.stderr.write(progress_string(common_db)+'\r')
5418 time.sleep(0.2)
5419
5420 if DO_PROGRESS and terminalcolumns > 4:
5421 progress_thread=threading.Thread(target=print_progress, args=( thread_returns, ) )
5422 progress_thread.daemon=True
5423 progress_thread.start()
5424 else:
5425 progress_thread=None
5426
5427 ################ second merry-go-round, download rest of available deltas , queue them
5428 for delta_len, name, uri, abs_delta_name , newdeb, deb_uri, tempname, complete in available_deltas :
5429 ## this seems to create problems....
5430 #if not os.path.exists(abs_delta_name) and os.path.exists(tempname) and os.path.getsize(tempname) == delta_len:
5431 # print 'Just rename:',name #this actually should never happen, but , who knows...
5432 # os.rename(tempname,abs_delta_name)
5433 # tempname=abs_delta_name
5434
5435 if name in params_of_delta:
5436 s=patch_check_tmp_space(params_of_delta[name], '/')
5437 if s != True:
5438 logger.warning('%r : sorry, %r', name, s)
5439 #argh, we ran out of space in meantime
5440 continue
5441
5442 if not os.path.exists(abs_delta_name):
5443 thread_returns['downloaduri']=os.path.basename(uri)
5444 r=download_uri(uri, abs_delta_name, deltas_down_time, deltas_down_size, thread_returns)
5445 del thread_returns['downloaduri']
5446 if r == None or isinstance(r, HTTPException) :
5447 if VERBOSE : logger.info(' '+_('You may wish to rerun, to get also:')+' '+uri)
5448 continue
5449 else:
5450 deltas_down_time = r[0]
5451 deltas_down_size = r[1]
5452
5453 #queue to apply delta
5454 if os.path.exists(abs_delta_name):
5455 #append to queue
5456 patching_queue.put( (name, abs_delta_name ,newdeb, deb_uri ) )
5457 else:
5458 if 'u' in DEB_POLICY:
5459 no_delta.append( (deb_uri, newdeb) )
5460 elif VERBOSE > 1 : logger.debug(' No deb-policy "u", no download of %r',deb_uri)
5461 ## end of second merry-go-round
5462
5463 #terminate queue
5464 patching_queue.put(None)
5465
5466 #do something useful in the meantime
5467 debs_down_size=0
5468 debs_down_time=0
5469 if patching_thread.is_alive() and no_delta and VERBOSE > 1 :
5470 logger.info(' Downloading deltas done, downloading debs while waiting for patching thread.')
5471 while patching_thread.is_alive() or ('a' in DEB_POLICY and no_delta):
5472 if no_delta:
5473 uri, newdeb = no_delta.pop()
5474 thread_returns['downloaduri']=os.path.basename(uri)
5475 r=download_uri(uri, newdeb, debs_down_time, debs_down_size, thread_returns)
5476 del thread_returns['downloaduri']
5477 if r == None or isinstance(r, HTTPException) :
5478 if VERBOSE : logger.debug(' You may wish to rerun, to get also: %r',uri)
5479 continue
5480 if r:
5481 debs_down_time = r[0]
5482 debs_down_size = r[1]
5483 if not no_delta :
5484 time.sleep(0.1)
5485
5486 for i in http_conns:
5487 if http_conns[i] != None :
5488 http_conns[i].close()
5489
5490 while patching_thread.is_alive():
5491 time.sleep(0.1)
5492
5493 #terminate progress report
5494 thread_returns['STOP']=True
5495 while progress_thread != None and progress_thread.is_alive():
5496 time.sleep(0.1)
5497
5498 if DO_PROGRESS:
5499 sys.stderr.write(' ' * terminalcolumns +'\r')
5500
5501 total_time += time.time()
5502 logger.info(_('Delta-upgrade statistics:'))
5503 msg=_("size %(size)s time %(time)dsec speed %(speed)s/sec")
5504 if VERBOSE:
5505 if deltas_down_time :
5506 a=float(deltas_down_size)
5507 t=deltas_down_time
5508 logger.info(' ' + _('downloaded deltas, ') + msg % \
5509 {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )})
5510 if thread_returns['debs_time'] :
5511 a=float(thread_returns['debs_size'])
5512 t=thread_returns['debs_time']
5513 logger.info(' ' + _('patching to debs, ') + msg % \
5514 {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )})
5515 if debs_down_time :
5516 a=float(debs_down_size)
5517 t=debs_down_time
5518 logger.info(' ' + _('downloaded debs, ') + msg % \
5519 {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )})
5520 if total_time:
5521 a=float(debs_down_size + thread_returns['debs_size'])
5522 t=total_time
5523 logger.info(' ' + _('total resulting debs, size %(size)s time %(time)dsec virtual speed %(speed)s/sec') % \
5524 {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )})
5525
5526 if forensics:
5527 forensic_send(forensics)
5528 return max(mainexitcodes)
5529
5530 ################################################# main program, do stuff
5531
5532 def act():
5533 "fake function that marks where the action starts"
5534 pass
5535
5536 if action == 'patch':
5537 if INFO :
5538 if len(argv) > 1 and VERBOSE :
5539 sys.stderr.write(' (printing info - extra arguments are ignored)\n')
5540 elif len(argv) == 0 :
5541 sys.stderr.write(_('Need a filename; try --help.')+'\n')
5542 raise SystemExit(3)
5543 try:
5544 delta=abspath(argv[0])
5545 check_is_delta(delta)
5546 info=get_info(delta)
5547 for s in info:
5548 if s:
5549 logger.debug(' info: %r',s)
5550 except KeyboardInterrupt:
5551 puke('debpatch exited by keyboard interrupt')
5552 raise SystemExit(5)
5553 except DebDeltaError:
5554 s=sys.exc_info()[1]
5555 puke('debpatch',s)
5556 raise SystemExit(s.exitcode)
5557 except Exception:
5558 s=sys.exc_info()[1]
5559 puke("debpatch",s)
5560 raise SystemExit(4)
5561 raise SystemExit(0)
5562 #really patch
5563 if len(argv) != 3 :
5564 sys.stderr.write(_('Need 3 filenames; try --help.')+'\n')
5565 raise SystemExit(3)
5566
5567 newdeb=abspath(argv[2])
5568 if newdeb == '/dev/null':
5569 newdeb = None
5570
5571 try:
5572 do_patch(abspath(argv[0]), abspath(argv[1]), newdeb)
5573 except KeyboardInterrupt:
5574 puke('debpatch exited by keyboard interrupt')
5575 raise SystemExit(5)
5576 except DebDeltaError:
5577 s=sys.exc_info()[1]
5578 puke('debpatch',s)
5579 if s.logs:
5580 forensic_send([s.logs])
5581 raise SystemExit(s.exitcode)
5582 except Exception:
5583 s=sys.exc_info()[1]
5584 puke('debpatch',s)
5585 raise SystemExit(4)
5586 raise SystemExit(0)
5587
5588 elif action == 'delta' :
5589 if len(argv) != 3 :
5590 sys.stderr.write(_('Need 3 filenames; try --help.')+'\n')
5591 raise SystemExit(3)
5592
5593 delta=abspath(argv[2])
5594 try:
5595 r = do_delta(abspath(argv[0]), abspath(argv[1]), delta)
5596 except KeyboardInterrupt:
5597 puke('debdelta exited by keyboard interrupt')
5598 raise SystemExit(5)
5599 except DebDeltaError:
5600 s=sys.exc_info()[1]
5601 puke('debdelta',s)
5602 raise SystemExit(s.exitcode)
5603 except Exception:
5604 s=sys.exc_info()[1]
5605 puke('debdelta',s)
5606 raise SystemExit(4)
5607 raise SystemExit(0)
5608
5609 elif action == 'deltas' :
5610 for v in argv:
5611 if not (os.path.isfile(v) or os.path.isdir(v)) :
5612 sys.stderr.write(_("Error: argument is not a directory or a regular file:")+' '+v)
5613 raise SystemExit(3)
5614 try:
5615 exitcode=do_deltas(argv)
5616 except KeyboardInterrupt:
5617 puke('debdeltas exited by keyboard interrupt')
5618 raise SystemExit(5)
5619 except DebDeltaError:
5620 s=sys.exc_info()[1]
5621 puke('debdeltas',s)
5622 raise SystemExit(s.exitcode)
5623 except Exception:
5624 s=sys.exc_info()[1]
5625 puke('debdeltas',s)
5626 raise SystemExit(4)
5627 raise SystemExit(exitcode)
5628
5629 elif action == 'delta-upgrade':
5630 import warnings
5631 warnings.simplefilter("ignore",FutureWarning)
5632 try:
5633 exitcode=delta_upgrade_(argv)
5634 except KeyboardInterrupt:
5635 puke('debdelta-upgrade exited due to keyboard interrupt')
5636 raise SystemExit(5)
5637 except DebDeltaError:
5638 s=sys.exc_info()[1]
5639 puke('debdelta-upgrade',s)
5640 raise SystemExit(s.exitcode)
5641 except Exception:
5642 s=sys.exc_info()[1]
5643 puke('debdelta-upgrade',s)
5644 raise SystemExit(4)
5645 raise SystemExit(exitcode)
5646
5647 elif action == 'patch-url':
5648 config=ConfigParser()
5649 config.read(['/etc/debdelta/sources.conf', expanduser('~/.debdelta/sources.conf') ])
5650
5651 try:
5652 import apt_pkg
5653 except ImportError:
5654 logger.error('ERROR!!! python module "apt_pkg" is missing. Please install python-apt')
5655 raise SystemExit(1)
5656
5657 try:
5658 import apt
5659 except ImportError:
5660 logger.error('ERROR!!! python module "apt" is missing. Please install a newer version of python-apt (newer than 0.6.12)')
5661 raise SystemExit(1)
5662
5663 apt_pkg.init()
5664
5665 cache=apt.Cache()
5666 cache.upgrade(True)
5667
5668 for a in argv:
5669 logger.info(_('Lookup %s') % a)
5670 p = cache[a]
5671 candidate=p.candidate
5672 origin = p.candidate.origins[0]
5673 arch=candidate.architecture
5674 if not candidate.uris :
5675 logger.warning(_("Sorry, cannot find an URI to download the debian package of `%s'.") % a)
5676 continue
5677 deb_uri = candidate.uri
5678 installed_version=p.installed.version
5679 candidate_version=p.candidate.version
5680 deb_path=str.split(deb_uri,'/')
5681 deb_path=str.join('/',deb_path[(deb_path.index('pool')):])
5682
5683 delta_uri_base=delta_uri_from_config(config,
5684 Origin=origin.origin,
5685 Label=origin.label,
5686 Site=origin.site,
5687 Archive=origin.archive,
5688 PackageName=p.name)
5689
5690 if delta_uri_base == None:
5691 logger.warning(_("Sorry, no source is available to upgrade `%s'.") % a)
5692 continue
5693
5694 if installed_version == candidate_version:
5695 logger.warning(_("Sorry, the package `%s' is already at its newest version.") % a)
5696 continue
5697
5698 #delta name
5699 delta_name=p.shortname+'_'+version_mangle(installed_version)+\
5700 '_'+ version_mangle(candidate_version)+'_'+\
5701 arch+'.debdelta'
5702
5703 delta_name = delta_base_name(p.shortname, installed_version,
5704 candidate_version, arch)
5705
5706 uri=delta_uri_base+'/'+os.path.dirname(deb_path)+'/'+delta_name
5707
5708 logger.info(a+' : '+uri)
5709 raise SystemExit(0)
5710
5711 ##################################################### apt method
5712
5713 ### still work in progress
5714 if os.path.dirname(sys.argv[0]) == '/usr/lib/apt/methods' :
5715 import os,sys, select, fcntl, apt, thread, threading, time
5716
5717 apt_cache=apt.Cache()
5718
5719 log=open('/tmp/log','a')
5720 log.write(' --- here we go\n')
5721
5722 ( hi, ho , he) = os.popen3('/usr/lib/apt/methods/http.distrib','b',2)
5723
5724 nthreads=3
5725
5726 class cheat_apt_gen:
5727 def __init__(self):
5728 self.uri=None
5729 self.filename=None
5730 self.acquire=False
5731 def process(self,cmd):
5732 if self.uri:
5733 self.filename=cmd[10:-1]
5734 log.write(' download %s for %s\n' % (repr(self.uri),repr(self.filename)))
5735 self.uri=None
5736 self.filename=None
5737 self.acquire=False
5738 return cmd
5739 elif self.acquire:
5740 self.uri=cmd[5:-1]
5741 return cmd
5742 elif cmd[:3] == '600' :
5743 self.acquire=True
5744 else:
5745 return cmd
5746
5747 def copyin():
5748 bufin=''
5749 while 1:
5750 #print ' o'
5751 s=os.read(ho.fileno(),1)
5752 bufin += s
5753 if log and bufin and (s == '' or s == '\n') :
5754 log.write( ' meth ' +repr(bufin)+'\n' )
5755 bufin=''
5756 if s == '':
5757 thread.interrupt_main( )
5758 global nthreads
5759 if nthreads:
5760 nthreads-=1
5761 #log.write( ' in closed \n' )
5762 #return
5763 os.write(1,s)
5764
5765
5766 def copyerr():
5767 buferr=''
5768 while 1:
5769 s=os.read(he.fileno(),1)
5770 buferr += s
5771 if log and buferr and (s == '' or s == '\n') :
5772 log.write( ' err ' +repr(buferr)+'\n' )
5773 buferr=''
5774 if s == '':
5775 thread.interrupt_main( )
5776 global nthreads
5777 if nthreads:
5778 nthreads-=1
5779 log.write( ' err closed \n' )
5780 #return
5781 os.write(2,s)
5782
5783 def copyout():
5784 gen=cheat_apt_gen()
5785 bufout=''
5786 while 1:
5787 s=os.read(0,1)
5788 bufout += s
5789 if log and bufout and (s == '' or s == '\n') :
5790 log.write( ' apt ' +repr(bufout)+'\n' )
5791
5792 bufout=gen.process(bufout)
5793
5794 bufout=''
5795 if s == '':
5796 thread.interrupt_main()
5797 global nthreads
5798 if nthreads:
5799 nthreads-=1
5800 #log.write( ' out closed \n' )
5801 #return
5802 os.write(hi.fileno(),(s))
5803
5804
5805 tin=thread.start_new_thread(copyin,())
5806 tout=thread.start_new_thread(copyout,())
5807 terr=thread.start_new_thread(copyerr,())
5808 while nthreads>0 :
5809 log.write( ' nthreads %d \n' % nthreads )
5810 try:
5811 while nthreads>0 :
5812 time.sleep(1)
5813 except KeyboardInterrupt:
5814 pass
5815 raise SystemExit(0)
5816