#!/usr/bin/python3 # Copyright (C) 2006-2020 Andrea Mennucci. # License: GNU Library General Public License, version 2 or later EMAIL="mennucc1@debian.org" #### messages and translations # Early errors (such as errors in args parsing) are printed on stderr. # Subsequent errors,warnings,messages are printed on stdout,using the 'logging' facility # but the progress bars are on stderr # Messages printed at verbosity 0 or 1 are translated, higher verbosities are not # Errors are sometimes translated, sometimes not... # obscure error messages that would be printed only in very rare cases are not translated # e.g.: a malformed http header, a gnupg unexptected error, damaged input files... # more common error messages are translated, # e.g.: out of disk space while using debdelta-upgrade , file does not exist... import sys try: import gettext gettext.bindtextdomain('debdelta','/usr/share/locale') gettext.textdomain('debdelta') _ = gettext.gettext except Exception: a=sys.exc_info[1] sys.stderr.write('Could not initialize "gettext", translations will be unavailable\n'+str(a)) def _(x): return x doc={} doc['delta']=_("""\ Usage: debdelta [ option... ] fromfile tofile delta Computes the difference of two deb files, from fromfile to tofile, and writes it to delta Options: --signing-key KEY gnupg key used to sign the delta --no-md5 do not include MD5 info in delta --needsold create a delta that can only be used if the old deb is available -M Mb maximum memory to use (for 'bsdiff' or 'xdelta') --delta-algo ALGO use a specific backend for computing binary diffs """) doc['deltas']=_("""\ Usage: debdeltas [ option... ] [deb files and dirs, or 'Packages' files] Computes all missing deltas for deb files. It orders by version number and produce deltas to the newest version Options: --signing-key KEY key used to sign the deltas (using GnuPG) --dir DIR force saving of deltas in this DIR (otherwise they go in the dir of the newer deb_file) --old ARGS 'Packages' files containing list of old versions of debs --alt ARGS for any cmdline argument, search for debs also in this place -n N how many deltas to produce for each deb (default unlimited) --no-md5 do not include MD5 info in delta --needsold create a delta that can only be used if the old .deb is available --delta-algo ALGO use a specific backend for computing binary diffs; possible values are: xdelta xdelta-bzip xdelta3 bsdiff -M Mb maximum memory to use (for 'bsdiff' or 'xdelta') --clean-deltas delete deltas if newer deb is not in archive --cache cache parsed version of Packages.bz2 as Packages.debdelta_cache """) ## implement : --search search in the directory of the above debs for older versions doc['patch']=_("""\ Usage: debpatch [ option... ] delta fromfile tofile Applies delta to fromfile and produces a reconstructed version of tofile. (When using 'debpatch' and the old .deb is not available, use the unpack directory, usually '/', for the fromfile.) Usage: debpatch --info delta Write info on delta. Options: --no-md5 do not verify MD5 (if found in info in delta) -A accept unsigned deltas --format FORMAT format of created deb """) doc['delta-upgrade']=_("""\ Usage: debdelta-upgrade [package names] Downloads all deltas and apply them to create the debs that are needed by 'apt-get upgrade'. Options: --dir DIR directory where to save results --deb-policy POLICY policy to decide which debs to download, -A accept unsigned deltas --format FORMAT format of created debs --timeout SECONDS adjust timeout for connections, default is 15 seconds """) doc['patch-url']=_("""\ Usage: debpatch-url [package names] Show URL wherefrom to downloads all deltas that may be used to upgrade the given package names """) doc_common=_("""\ -v verbose (can be added multiple times) --no-act do not do that (whatever it is!) -d add extra debugging checks -k keep temporary files (use for debugging) --gpg-home HOME specify a different home for GPG See man page for more options and details. """) minigzip='/usr/lib/debdelta/minigzip' minibzip2='/usr/lib/debdelta/minibzip2' #################################################################### import os , tempfile , getopt , tarfile , shutil , time, traceback, subprocess, time, tarfile, stat, hashlib, random, gzip try: import debian.deb822 debian_deb822 = debian.deb822 except ImportError: debian_deb822 = None try: import cPickle as pickle except ImportError: import pickle from stat import ST_SIZE, ST_MTIME, ST_MODE, S_IMODE, S_IRUSR, S_IWUSR, S_IXUSR from os.path import abspath, expanduser from copy import copy from types import FunctionType, LambdaType import logging, logging.handlers if __name__ == '__main__': logging.basicConfig(format='%(message)s', stream=sys.stdout, level=logging.INFO) logger=logging.getLogger(__name__) if sys.version_info.major == 2: string_types = (str, unicode) # python2 # flags for subprocess.Popen Popen_args = {} # from ConfigParser import SafeConfigParser as ConfigParser from tarfile import filemode as stat_filemode # def str2bytes(a): return a def bytes2str(b): return b def is_bytes(b): return False # see https://docs.python.org/2/library/tarfile.html#unicode-issues if sys.getfilesystemencoding() != 'UTF-8': logger.warning('The filesystem encoding is %r, but this program assumes `UTF-8`.'%\ (sys.getfilesystemencoding(),)) else: string_types = (str, bytes) # python3 intern = sys.intern # flags for subprocess.Popen to open stdin,stdout as text streams Popen_args = {'text':True} # from configparser import ConfigParser from stat import filemode as stat_filemode # long = int def str2bytes(a): return a.encode('UTF-8',errors='surrogateescape') def bytes2str(b): return b.decode('UTF-8',errors='surrogateescape') def is_bytes(b): return isinstance(b,bytes) # see https://docs.python.org/3/library/tarfile.html#unicode-issues if sys.getfilesystemencoding() != 'utf-8': logger.warning('The filesystem encoding is %r, but this program assumes `utf-8`.'%\ (sys.getfilesystemencoding(),)) elif hasattr(sys,'getfilesystemencodeerrors') and \ sys.getfilesystemencodeerrors() != 'surrogateescape': logger.warning('The filesystem encoding is `utf-8` with %r, but this program uses `surrogateescape`.'%\ (sys.getfilesystemencodeerrors(),)) def get_termsize(): import termios, fcntl, struct s = struct.pack("HHHH", 0, 0, 0, 0) fd_stdout = sys.stdout.fileno() x = fcntl.ioctl(fd_stdout, termios.TIOCGWINSZ, s) return struct.unpack("HHHH", x)[:2] try: (terminalrows , terminalcolumns) = get_termsize() except: (terminalrows , terminalcolumns) = (None, None) #(24, 80) ################################################# main program, read options #target of: maximum memory that bsdiff will use MAXMEMORY = 1024 * 1024 * 50 #this is +-10% , depending on the package size MAX_DELTA_PERCENT = 70 #min size of .deb that debdelta will consider #very small packages cannot be effectively delta-ed MIN_DEB_SIZE = 10 * 1024 N_DELTAS=None USE_DELTA_ALGO = 'bsdiff' TIMEOUT = 15 DEBUG = 0 VERBOSE = 0 KEEP = False INFO = False NEEDSOLD= False DIR = None ALT = [] OLD = [] ACT = True DO_MD5 = True DEB_POLICY = ['b','s','e','t','f','q'] DO_PROGRESS = terminalcolumns != None and __name__ == '__main__' #where/how debpatch/debdelta-upgrade will send forensic data, when patching fails #possible values: # False : do not send them # do : compute forensic but not send them, just list them # mail : automatically send by email to default address (uses `mutt`) # mail:user@domain : automatically send by email to address (uses `mutt`) # mutt : as above # mutt:user@domain : as above # http://domain/cgi : send them automatically thru a CGI script #Warning: the above is mostly TODO FORENSIC='do' FORENSIC_EMAIL='debdelta_fail@debdelta.debian.net' #directory tree where forensic info are stored by 'debdeltas' FORENSICDIR=None DEB_FORMAT='deb' DEB_FORMAT_LIST=('deb','unzipped','preunpacked') #not yet implemented on patching side : (,'piped') #for debdeltas: test patches internally DO_TEST = False DO_GPG = True #this is changed a few lines below GPG_SIGNING_KEY = None if os.getuid() == 0: GPG_HOME="/etc/debdelta/gnupg" else: GPG_HOME=None GPG_MASTER_PUB_KEYRING="/usr/share/keyrings/debian-debdelta-archive-keyring.gpg" GPG_CMD='gpg' if os.path.exists('/usr/bin/gpg2'): GPG_CMD='/usr/bin/gpg2' CLEAN_DELTAS = False CLEAN_DELTAS_MTIME = 2 # two days grace period CLEAN_ALT = False DO_PREDICTOR = False DO_CACHE = False #cache parsed version of Packages.bz2 as Packages.debdelta_cache #see README.features DISABLEABLE_FEATURES=['xz', 'lzma', 'xdelta3-fifo'] DISABLED_FEATURES=[] HTTP_USER_AGENT={'User-Agent': ('Debian debdelta-upgrade' ) } DPKG_MULTIARCH=( 0 == os.system('dpkg --assert-multi-arch 2> /dev/null') ) if __name__ != "__main__": action = None elif os.path.dirname(sys.argv[0]) == '/usr/lib/apt/methods' : action = None else: action=(os.path.basename(sys.argv[0]))[3:] actions = ('delta','patch','deltas','delta-upgrade', 'patch-url') # if action not in actions: sys.stderr.write('wrong filename: should be "deb" + '+repr(actions)) raise SystemExit(4) __doc__ = doc[action] + doc_common #GPG signatures are required for debdelta-upgrade and debpatch DO_GPG = action in ( "delta-upgrade", "patch") try: ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:n:A' , ('help','info','needsold','dir=','no-act','alt=','old=','delta-algo=', 'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug','forensicdir=','forensic=', 'signing-key=', "accept-unsigned", "gpg-home=", "disable-feature=", "test", "format=", "cache", "timeout=") ) except getopt.GetoptError: a=sys.exc_info()[1] sys.stderr.write(sys.argv[0] +': '+ str(a)+'\n') raise SystemExit(3) for o , v in opts : if o == '-v' : VERBOSE += 1 if VERBOSE > 1 : logger.setLevel(logging.DEBUG) elif o == '-d' or o == '--debug' : DEBUG += 1 logger.setLevel(logging.DEBUG) elif o == '-k' : KEEP = True elif o == '--no-act': ACT=False elif o == '--no-md5': DO_MD5=False elif o == '--clean-deltas' : CLEAN_DELTAS = True elif o == '--clean-alt' : CLEAN_ALT = True sys.stderr.write(_('Warning, currently --clean-alt does nothing.')+'\n') elif o == '--needsold' : NEEDSOLD = True elif o == '--delta-algo': USE_DELTA_ALGO=v elif o == '--max-percent': MAX_DELTA_PERCENT=int(v) elif o == '--deb-policy' : DEB_POLICY = [j[0] for j in v.split(',') if j] elif o == '--timeout' : if int(v) < 0: sys.stderr.write(_('Error: "--timeout %s" is too small.') % v + '\n') raise SystemExit(3) TIMEOUT = int(v) elif o == '-M' : if int(v) <= 1: sys.stderr.write(_('Error: "-M %s" is too small.') % v + '\n') raise SystemExit(3) if int(v) <= 12: sys.stderr.write(_('Warning: "-M %s" is quite small.') % v + '\n') MAXMEMORY = 1024 * 1024 * int(v) elif o == '-n' : N_DELTAS = int(v) if N_DELTAS < 0: sys.stderr.write(_('Error: -n value is negative.')+'\n') raise SystemExit(3) elif o == '--test' and action == 'deltas' : DO_TEST = True elif o == '--info' and action == 'patch' : INFO = True elif o == '--dir' : DIR = abspath(expanduser(v)) if v[-2:] == '//': DIR += '//' if not os.path.isdir(DIR): sys.stderr.write( _("Error: argument of --dir is not a directory:") +' '+ DIR +'\n') raise SystemExit(3) elif o == '--forensicdir' : FORENSICDIR = abspath(expanduser(v)) if v[-2:] == '//': FORENSICDIR += '//' if not os.path.isdir(FORENSICDIR): sys.stderr.write( _("Error: argument of --forensicdir is not a directory:") +' '+ FORENSICDIR +'\n') raise SystemExit(3) elif o == '--forensic' : FORENSIC = v if FORENSIC.startswith('http'): try: import poster except: sys.stderr.write(_('To use the http forensic, you must install the package "python-poster".')+'\n') raise SystemExit(3) if FORENSIC[:4] in ('mutt','mail') and not os.path.exists('/usr/bin/mutt'): sys.stderr.write(_('To use this forensic, you must install the package "mutt".')+'\n') raise SystemExit(3) elif o == '--alt' : if not (os.path.isfile(v) or os.path.isdir(v)) : sys.stderr.write(_('Error: argument of --alt is not a directory or a regular file:')+' '+v +'\n') raise SystemExit(3) ALT.append(v) elif o == '--old' and action == 'deltas' : if not (os.path.isfile(v) or os.path.isdir(v)) : sys.stderr.write(_("Error: argument of --old is not a directory or a regular file:")+' '+v +'\n') raise SystemExit(3) OLD.append(v) elif o == '--help' or o == '-h': sys.stderr.write(__doc__) raise SystemExit(0) elif (o == '--disable-feature') and action in ("delta", "deltas"): DISABLED_FEATURES += v.split(',') elif (o == '--signing-key') and action in ("delta", "deltas"): GPG_SIGNING_KEY=v DO_GPG=True elif (o == '--accept-unsigned' or o == '-A') and action in ("delta-upgrade", "patch"): DO_GPG=False elif (o == '--gpg-home'): GPG_HOME=abspath(expanduser(v)) if not os.path.isdir(GPG_HOME): sys.stderr.write(_("Error: --gpg-home `%s' does not exist.") % GPG_HOME) raise SystemExit(3) elif o == '--format' : # maybe, and action in ("delta-upgrade", "patch"): if v not in DEB_FORMAT_LIST: sys.stderr.write(_("Error: output format `%s' is unknown.") % v + '\n') raise SystemExit(3) DEB_FORMAT=v elif o == '--cache': DO_CACHE=True else: sys.stderr.write(_("Error: option `%s' is unknown, try --help") % o + '\n') raise SystemExit(3) for i in DISABLED_FEATURES: if i not in DISABLEABLE_FEATURES: sys.stderr.write(_("Error: feature `%s' cannot be disabled.") % i + '\n') raise SystemExit(3) try: a = subprocess.Popen(b'grep bogomips /proc/cpuinfo',shell=True, stdout=subprocess.PIPE).stdout.read() a = bytes2str(a) BOGOMIPS=float(a.split(':')[-1]) except: if VERBOSE: logger.exception(' Warning, /proc not mounted, using bogus BOGOMIPS\n') BOGOMIPS=3000.0 SP=subprocess.Popen(['hostname','-f'], shell=False, stdout=subprocess.PIPE) HOSTID=hashlib.md5( SP.stdout.read() ).hexdigest() SP.wait() del SP TMPDIR = ( os.getenv('TMPDIR') or '/tmp' ).rstrip('/') if KEEP: def unlink(a): if VERBOSE > 2: logger.debug(' -k: would unlink '+repr(a)) def rmdir(a): if VERBOSE > 2: logger.debug(' -k: would rmdir '+repr(a)) def rmtree(a): if VERBOSE > 2: logger.debug(' -k: would rm -r '+repr(a)) else: def __wrap__(a,cmd): c=cmd.__name__+"("+a+")" if a[ : len(TMPDIR)+9 ] != TMPDIR+'/debdelta' : raise DebDeltaError('Internal error! refuse to '+c) try: cmd(a) except OSError: logger.exception(' Warning! when trying to '+repr(c)+' got OSError ') raise def unlink(a): return __wrap__(a,os.unlink) def rmdir(a): return __wrap__(a,os.rmdir) def rmtree(a): return __wrap__(a,shutil.rmtree) ################################################################## check debian.deb822 _fake_Packages="""Package: coreutils Version: 8.30-3 Essential: yes Installed-Size: 15719 Maintainer: Michael Stone Architecture: amd64 Pre-Depends: libacl1 (>= 2.2.23), libattr1 (>= 1:2.4.44), libc6 (>= 2.28), libselinux1 (>= 2.1.13) Description: GNU core utilities Multi-Arch: foreign Homepage: http://gnu.org/software/coreutils Description-md5: d0d975dec3625409d24be1238cede238 Tag: admin::configuring, implemented-in::c, interface::commandline, role::program, scope::utility, suite::gnu, works-with::file Section: utils Priority: required Filename: pool/main/c/coreutils/coreutils_8.30-3_amd64.deb Size: 2707932 MD5sum: 4a658a39c2810530df6642d22f65ddd1 SHA256: ae6e5cd6e9aaf74d66edded3931a7a6c916625b8b890379189c75574f6856bf4 Package: bash Version: 5.0-4 Essential: yes Installed-Size: 6439 Maintainer: Matthias Klose Architecture: amd64 Replaces: bash-completion (<< 20060301-0), bash-doc (<= 2.05-1) Depends: base-files (>= 2.1.12), debianutils (>= 2.15) Pre-Depends: libc6 (>= 2.15), libtinfo6 (>= 6) Recommends: bash-completion (>= 20060301-0) Suggests: bash-doc Conflicts: bash-completion (<< 20060301-0) Description: GNU Bourne Again SHell Multi-Arch: foreign Homepage: http://tiswww.case.edu/php/chet/bash/bashtop.html Description-md5: 3522aa7b4374048d6450e348a5bb45d9 Tag: admin::TODO, devel::TODO, devel::interpreter, implemented-in::c, interface::shell, interface::text-mode, role::program, scope::application, suite::gnu, uitoolkit::ncurses Section: shells Priority: required Filename: pool/main/b/bash/bash_5.0-4_amd64.deb Size: 1377592 MD5sum: 6f118d96e29a5208da9033bf0b2c772c SHA256: d07f3528c615545bb182269362926bc171adab565dade3e8c35cb3bae8ad9e5e """ def test_deb822(debian_deb822, shared_storage): F = tempfile.NamedTemporaryFile() open(F.name,'w').write(_fake_Packages) I=debian_deb822.Packages.iter_paragraphs(sequence=open(F.name,'rb'),shared_storage=shared_storage) c=next(I) assert c['Package'] == 'coreutils' assert c['Size'] == '2707932' b=next(I) assert b['Package'] == 'bash' assert c['Size'] == '2707932' b['foo']='bar' c['foo']='tock' assert b['foo']=='bar' # Deb822 only accepts strings #b['foo']=(1,None) if debian_deb822 is not None: try: test_deb822(debian_deb822,False) test_deb822(debian_deb822,True) except: logger.exception("While testing debian.822") #################################################### various routines def my_popen_read(cmd, ** args): " opens a pipe (for text mode, pass `**Popen_args` )" return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=open(os.devnull), close_fds=True, **args).stdout def freespace(w): assert(os.path.exists(w)) try: a=os.statvfs(w) freespace= long(a[0]) * long(a[4]) except Exception: logger.exception('Statvfs error on %r', w) freespace=None return freespace dpkg_keeps_controls = ( 'conffiles','config','list','md5sums','postinst', 'postrm','preinst','prerm','shlibs','templates') def parse_dist(f,d): a=f.readline() p={} while a: if a[:4] in ('Pack','Vers','Arch','Stat','Inst','File','Size','MD5s'): a=de_n(a) i=a.index(':') assert(a[i:i+2] == ': ') p[a[:i]] = a[i+2:] elif a == '\n': d[p['Package']] = p p={} a=f.readline() def scan_control(p,params=None,prefix=None,info=None): if prefix == None: prefix = '' else: prefix += '/' a=p.readline() while a: a=de_n(a) if a[:4] in ('Pack','Vers','Arch','Stat','Inst','File'): if info != None : info.append(prefix+a) if params != None: i=a.index(':') assert(a[i:i+2] == ': ') params[prefix+a[:i]] = a[i+2:] a=p.readline() def append_info(delta,info): "insert into the delta (that is an AR archive) the info file, as a first element, possibly removing a previous occurrence" #new style : special info file TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR)) infofile=open(TD+'/info','w') for i in info: infofile.write(i+'\n') infofile.close() if DO_GPG: r=_compute_hashes_(TD+"/info") else: r=None system(['ar','rSi','0',delta, 'info'], TD) rmtree(TD) return r def de_n(a): assert isinstance(a, str) if a and a[-1:] == '\n' : a = a[:-1] return a def de_n_b(a): assert sys.version_info.major == 2 or isinstance(a, bytes) if a and a[-1:] == b'\n' : a = a[:-1] return a def de_bar(a): assert isinstance(a, str) if a and a[:2] == './' : a=a[2:] elif a == '/.' : a='' elif a and a[0] == '/' : a=a[1:] return a def de_bar_b(a): assert sys.version_info.major == 2 or isinstance(a, bytes) if a and a[:2] == b'./' : a=a[2:] elif a == b'/.' : a='' elif a and a[0] == b'/' : a=a[1:] return a def list_ar(f): assert(os.path.exists(f)) ar_list = [] p=my_popen_read(['ar','t', f]) while 1: a=p.readline() if not a : break a=de_n_b(a) ar_list.append(a) p.close() return ar_list def list_tar(f): " returns content of the tar file `f`, as a list of `bytes`" assert(os.path.exists(f)) ar_list = [] p=my_popen_read(['tar','t',f]) while 1: a=p.readline() if not a : break a=de_n_b(a) ar_list.append(a) p.close() return ar_list def untar_control_in_deb(ar_ls): " ar_ls is usually list_ar(n) ; it returns (c , b, z) \ where c 'name of control' , b 'option for tar' , z 'unzip command' " c = None for a in ar_ls : if a[:12]==b'control.tar.': c=a assert c # command for tar bdb={b'.gz':'-z', b'.bz2':'-j', b'.lzma':'--lzma', b'.xz':'-J'} zdb={b'.gz' : ['gzip','-cd'], b'.bz2': ['bzip2','-cd'], b'.lzma':['unlzma','-c'], b'.xz': ['unxz','-c'] } return c, bdb.get(c[11:]), zdb.get(c[11:]) class cache_sequence(object): cache_filename=None cache_file=None cache=None exists=None broken=None suffix='.debdelta_cache' def __init__(self, filename, cache_filename=None): """manages a cache file that store a sequence of python object; 'filename' is a reference file that is related to the data being cached, and is used to create the name of the cache file, unless 'cache_filename' is provided """ self.cache_filename=os.path.splitext(filename)[0]+self.suffix \ if cache_filename==None else cache_filename self.cache_file=None self.cache=None self.broken=None self.exists=os.path.isfile(self.cache_filename) and \ os.path.getmtime(filename) <= os.path.getmtime(self.cache_filename) def __iter__(self): assert self.exists and not self.cache self.cache=gzip.GzipFile(self.cache_filename) return self def __next__(self): assert self.cache try: return pickle.load(self.cache) except EOFError: self.cache=None raise StopIteration except Exception: logger.exception('Cache file is broken, deleting %r' % self.cache_filename) if ACT: os.unlink(self.cache_filename) self.cache=None self.broken=True # do not kill program raise StopIteration def next(self): return self.__next__() def __prepare_for_write__(self): if not self.cache: if DEBUG: logger.debug('Creating cache file : %r', self.cache_filename) self.cache_file=open(self.cache_filename+'~tmp~', 'wb') self.cache=gzip.GzipFile(filename='', fileobj=self.cache_file, mode='wb') # 'mtime=0' needs python2.7 or higher def close(self): try: if self.cache: self.cache.close() self.cache=None if self.cache_file: self.cache_file.close() self.cache_file=None os.rename(self.cache_filename+'~tmp~', self.cache_filename) except Exception: logger.exception('Cannot close the cache file %r' % self.cache_filename) self.broken=True __del__=close def write(self,s): " write one object" assert not self.exists if self.broken: return self.__prepare_for_write__() try: self.cache.write(pickle.dumps(s)) except Exception: logger.exception('Cannot write to cache file, deleting %r', self.cache_filename) self.close() if ACT: os.unlink(self.cache_filename) self.broken=True class cache_same_dict(cache_sequence): "cache occurrences of a dict that uses always the same keys; omit the keys to optimize" def __init__(self, filename, keys): super(cache_same_dict, self).__init__(filename) self.keys=keys def write(self, s): n=[s[k] for k in self.keys] super(cache_same_dict, self).write(n) def __next__(self): n=(super(cache_same_dict, self)).__next__() return dict(map(lambda x,y: (x,y) , self.keys, n)) # dict comprehension may be used instead ##################################################################### ALLOWED = '<>()[]{}.,;:!_-+/ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' ALLOWED_SET = set(ord(j) for j in ALLOWED) NUMBERS_SET = set(ord(j) for j in '0123456789') ECHO_TEST = r"""c='\0151\0141' E='echo -ne' if test c`$E 'i'"$c" `o = ciiao ; then : else E='echo -n' if test c`$E 'i'"$c" `o = ciiao ; then : else #echo WARNING : BUILTIN echo DOES NOT WORK OK E='/bin/echo -ne' test c`$E 'i'"$c" `o = ciiao fi fi """ def prepare_for_echo_py2_(s): assert ( type (s) in string_types ) r='' shortquoted=False for a in s: if a in ALLOWED : r += a shortquoted = False elif a in '0123456789' : if shortquoted : a = "\\" + ('000' +oct(ord(a)))[-4:] shortquoted = False r += a elif a == 0: r += '\\0' shortquoted = True else: a = "\\" + oct(ord(a)) r += a shortquoted = len(a) < 5 return r def prepare_for_echo_py3_(s): " `s` may be a `str` or `bytes` ; returns `str`" #assert isinstance(s,bytes) if isinstance(s,str): s = str2bytes(s) r='' O = lambda x : oct(x)[2:] shortquoted=False for a in s: #j = ord(a) if a in ALLOWED_SET : r += chr(a) shortquoted = False elif a in NUMBERS_SET : if shortquoted : a = "\\" + ('000' + O(a) )[-4:] r += a shortquoted = False else: r += chr(a) elif a == 0: r += '\\0' shortquoted = True else: a = "\\0" + O(a) r += a shortquoted = len(a) < 5 return r if sys.version_info.major == 2: prepare_for_echo__ = prepare_for_echo_py2_ else: prepare_for_echo__ = prepare_for_echo_py3_ def apply_prepare_for_echo(shell,repres): a=ECHO_TEST + " $E '" + repres + "' \n exit " p = subprocess.Popen([shell], stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True) #, ** Popen_args) (i, o) = (p.stdout, p.stdin) o.write(str2bytes(a)) o.close() a=i.read() i.close() return a #ack! I wanted to use 'dash' as preferred shell, but bug 379227 stopped me SHELL = '/bin/bash' #check my code s=b'\x00'+b'1ciao88\n77\r566'+b'\x00'+b'99\n' r=prepare_for_echo__(s) a=apply_prepare_for_echo(SHELL,r) if a != s : sys.stderr.write( 'string='+repr(s)+'\n') sys.stderr.write( 'repres='+repr(r)+'\n') sys.stderr.write( 'shell='+repr(SHELL)+'\n') sys.stderr.write( 'output='+repr(a)+'\n') sys.stderr.write( 'Errror in prepare_for_echo.\n') raise SystemExit(4) del r,s,a ### def prepare_for_echo(s): r=prepare_for_echo__(s) if DEBUG > 2 : a=apply_prepare_for_echo(SHELL,r) if a != s: z = 'Error in prepare_for_echo()\n' z += 'string='+repr(s)+'\n' z += 'repres='+repr(r)+'\n' z += 'shell='+repr(SHELL)+'\n' z += 'output='+repr(a)+'\n' raise DebDeltaError(z, exitcode=4) return r ##################################################################### def version_mangle(v): if ':' in v : return str.join('%3a',v.split(':')) else: return v def version_demangle(v): if '%' in v : return str.join(':',v.split('%3a')) else: return v def delta_base_name(pkg, old_ver, new_ver, arch, ext='.debdelta'): assert(':' not in pkg) return pkg + \ '_' + version_mangle(old_ver) + \ '_' + version_mangle(new_ver) + \ '_' + arch + ext def tempo(): TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR)) for i in 'OLD','NEW','PATCH' : os.mkdir(TD+'/'+i) if VERBOSE > 2 or KEEP : logger.debug('Temporary in '+TD) return TD ########## class DebDeltaError(Exception): #should derive from (Exception):http://docs.python.org/dev/whatsnew/pep-352.html # Subclasses that define an __init__ must call Exception.__init__ # or define self.args. Otherwise, str() will fail. def __init__(self,s,retriable=False,exitcode=None,logs=None): assert(type(s) in string_types) self.retriable = retriable if retriable: self.args=(s + ' (retriable) ',) else: self.args=(s + ' (non retriable) ',) if exitcode == None: if retriable: exitcode = 1 else: exitcode = 2 self.exitcode=exitcode self.logs=logs def die(s): #if s : sys.stderr.write(s+'\n') assert type(s) in string_types raise DebDeltaError(s) class PopenPipe(object): def __init__(self, a, **dictargs): #special code for pipes http://docs.python.org/library/subprocess.html#replacing-shell-pipeline old_stdin=dictargs.pop('stdin',None) old_stdout=dictargs.pop('stdout',None) self.pros=[] while '|' in a: l=a.index('|') ; a1=a[:l] ; a=a[l+1:] p=subprocess.Popen(args=a1, stdin=old_stdin, stdout=subprocess.PIPE, **dictargs) self.pros.append( p ) old_stdin=p.stdout final_pro= subprocess.Popen(args=a, stdin=old_stdin, stdout=old_stdout, **dictargs) for p in self.pros: p.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits. self.pros.append(final_pro) # self.returncode=None self.stdout=self.pros[-1].stdout self.stdin=self.pros[0].stdin self.read=getattr(self.pros[-1].stdout,'read',None) self.write=getattr(self.pros[0].stdin,'write',None) def __set_returncode(self): if any([ (p.returncode<0) for p in self.pros]): self.returncode = min([ p.returncode for p in self.pros]) else: self.returncode = max([ p.returncode for p in self.pros]) return self.returncode def poll(self): for p in self.pros: p.poll() return self.__set_returncode() def wait(self): for p in self.pros: p.wait() return self.__set_returncode() def terminate(self): self.pros[0].poll() if self.pros[0].returncode is None: self.pros[0].terminate() def close(self): self.terminate() self.wait() def system(a,TD,saveargs=None,ignore_output=False,return_output=False): "a must be a tuple, TD the temporary directory ; if return_output , it will return (stdout,stderr,exitcode) regardless" assert type(a) in (list, tuple) # mvo: compat with python2.5 where tuple does not have index a = list(a) if VERBOSE and TD[: (len(TMPDIR)+9) ] != TMPDIR+'/debdelta' : logger.debug(' Warning "system()" in '+TD+' for '+repr(a)) (temp_fd, temp_name) = tempfile.mkstemp(prefix="debdelta_out_system") (temp_err_fd, temp_err_name) = tempfile.mkstemp(prefix="debdelta_err_system") MPP=PopenPipe(a,stdin=open(os.devnull), stdout=temp_fd, stderr=temp_err_fd, cwd=TD, close_fds=True) ret=MPP.wait() os.close(temp_fd) os.close(temp_err_fd) if VERBOSE > 3 : logger.debug(' system(%r)=%d', a, ret) if ignore_output==False and (os.stat(temp_name)[ST_SIZE] > 0 or os.stat(temp_err_name)[ST_SIZE] > 0 ): logger.debug(' command "%s" returned %d and produced output as follows' % (a,ret)) for i in open(temp_name): logger.debug('stdout: '+repr(i)) for i in open(temp_err_name): logger.debug('stderr: '+repr(i)) if return_output: return temp_name, temp_err_name, ret os.unlink(temp_err_name) os.unlink(temp_name) if ret == 0: return elif ret != 1 or a[0] != 'xdelta' : s='Error , non zero return status '+str(ret)+' for command "'+repr(a)+'"' try: if DEBUG and saveargs: T=abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR)) open(T+'/command','w').write(repr(a)) for l in saveargs: if l[0] != '/': l = TD+'/'+l if os.path.exists(l): shutil.copy2(l,T) s=s+'\n saved argument '+l+' in '+T else: s=s+'\n did not find argument '+l except OSError: o=sys.exc_info()[1] s=s+'\n (there was an additional OSError "'+str(o)+'" when trying to save arguments)' die(s) def check_deb(f, die_ = True): if not os.path.exists(f) : die(_("Error: the file `%s' does not exist.") % f) if not os.path.isfile(f) : die(_("Error: `%s' is not a regular file.") % f) p=open(f,'rb') if p.read(21) != b"!\ndebian-binary" : if die_: die(_("Error: `%s' does not seem to be a Debian package.") % f) else: return False p.close() return True def check_is_delta(f): if not os.path.exists(f) : die(_("Error: the file `%s' does not exist.") % f) if not os.path.isfile(f) : die(_("Error: `%s' is not a regular file.") % f) p=open(f,'rb') if p.read(8) != b"!\n" : die(_("Error: `%s' does not seem to be a Debian delta.") % f) p.close() def puke(s,e=None): """ write informations on stderr; if DEBUG also prints traceback; if DEBUG>1 and `e` is given, rereaise the exception""" (typ, value, trace)=sys.exc_info() if e == None or len(str(e)) < 2: logger.error(str(s)+' : '+str(e)+' '+str(typ)+' '+str(value)) else: logger.error(str(s)+' : '+str(e)) if DEBUG > 1 and e is not None and trace: raise elif DEBUG and trace and traceback.print_tb(trace): logger.error( traceback.print_tb(trace)) ################################################################### GPG def gpg_base_commandline(): if GPG_HOME: GPG_BASE_CMD_LINE=[GPG_CMD,"--homedir",GPG_HOME] else: GPG_BASE_CMD_LINE=[GPG_CMD,"--keyring",GPG_MASTER_PUB_KEYRING] if VERBOSE < 1 : GPG_BASE_CMD_LINE+=['--quiet'] return GPG_BASE_CMD_LINE def gpg_sign_command(): return gpg_base_commandline()+["--batch","--armor","--clearsign","--default-key",GPG_SIGNING_KEY,"--sign"] def compute_md5_up_to_len(o, length): "hash initial part of a file using MD5. 'o' may be a string (in which case the file is opened) or a file type; returns MD5 and bytes effectively read" assert type(length) in (int,long) and length >= 0 if type(o) in string_types: o = open(o,'rb') m=hashlib.md5() a=True l=length while l>0 and a: a=o.read(min(1024,l)) l -= len(a) m.update(a) return m.hexdigest(), length-l def compute_md5_len(o): "hash the file using MD5. 'o' may be a string (in which case the file is opened) or a file type; returns MD5 and length" if type(o) in string_types: o = open(o,'rb') m=hashlib.md5() a=o.read(1024) l=0 while a: l+=len(a) m.update(a) a=o.read(1024) return m.hexdigest(), l def compute_md5(o): "hash the file using MD5. 'o' may be a string (in which case the file is opened) or a file type; returns MD5 (as a string of hexes)" return compute_md5_len(o)[0] def _compute_hashes_(na): "hash the file" o = open(na,'rb') m=hashlib.md5() s=hashlib.sha1() a=o.read(1024) while a: m.update(a) s.update(a) a=o.read(1024) r = ( m.hexdigest(), s.hexdigest(), os.stat(na)[ST_SIZE]) return r def _compute_hashes_db_(li,DIR): db={} for na in li: db[na] = _compute_hashes_(DIR+'/'+na) return db def verify_signature(signature, DIR): a=b"-----BEGIN PGP SIGNED MESSAGE-----\n" if open(signature,'rb').read(len(a)) != a: return ('BAD_FORMAT',signature) role=os.path.basename(signature) assert role[:4] == "_gpg" role=role[4:] (temp_fd, temp_name) = tempfile.mkstemp(prefix="debdelta_gpg_verified") #(read_end, write_end) = os.pipe() p=subprocess.Popen(gpg_base_commandline() + ['--batch','--status-fd',"2",'--output',"-",signature], stdout=subprocess.PIPE, stderr=temp_fd, stdin=open(os.devnull), close_fds=True, **Popen_args) r=_verify_signature_no_gpg(p.stdout, DIR, role) p.wait() os.close(temp_fd) if VERBOSE > 2 or p.returncode: for j in open(temp_name): logger.debug(' GPG> '+j.rstrip()) os.unlink(temp_name) if p.returncode: return ('GPG_VERIFY_FAILED',signature) return r def _verify_signature_no_gpg(signature, DIR, role): #list stuff, skipping signatures dir_list = [a for a in os.listdir(DIR) if a[:4] != '_gpg'] #compute signatures hashes = _compute_hashes_db_(dir_list, DIR) #scan hashes file (GPG already verified) if type(signature) in string_types: f=open(signature) elif hasattr(signature,'readline'): f=signature else: raise AssertionError a=f.readline() if a != "Version: 4\n": return ("UNSUPPORTED_VERSION",a) a=f.readline() while a: if a[:5] == "Role:": if a[5:].strip() != role : return ("ROLE_MISMATCH",a) a=f.readline() elif a[:6] == "Files:" : #parse files a=f.readline() while a and a[0] in ( '\t' , ' ') : a=a.rstrip('\n') a=a.lstrip() a=a.split(' ') if VERBOSE > 3 : logger.debug(' checking hashes '+repr(a)) (md5,sha1,le,na)=a if na not in dir_list: return ('ABSENT',na) (cmd5,csha1,cle)=hashes[na] if int(le) != cle: return ('SIZE',na) # check hashes if md5 != cmd5 : return ('MD5',na) if sha1 != csha1 : return ('SHA1',na) dir_list.remove(na) a=f.readline() elif VERBOSE > 2 : logger.debug(' signature header ignored: '+ a) a=f.readline() else: a=f.readline() #end parsing if dir_list: return ("UNCHECKED",dir_list) return True def _write_signature(db,filename,role): "starting from a database of hashes, see _compute_hashes_, it writes a signature file" f=open(filename,mode='w') ##this is the format of dpkg-sig, but is redundant, since the "date" and "signer" ##are already available thru the gpg signature #f.write("Version: 4\nSigner: \nDate: %s\nRole: %s\nFiles: \n" % (time.ctime(),role)) ##and actually dpkg-sig will validate also a simpler file, so, lets save a few bytes f.write("Version: 4\nRole: %s\nFiles:\n" % (role,)) for a in db: (m,s,l) = db[a] f.write('\t'+m+" "+s+" "+str(l)+" "+a+"\n") f.close() def sign_delta(delta, db, role="maker"): TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR)) try: _write_signature(db,TD+'/_temp',role) p=subprocess.Popen(gpg_sign_command() +['--output',TD+'/_gpg'+role,TD+'/_temp'], stdin=open(os.devnull), close_fds=True) p.wait() if p.returncode==0: r=system(("ar","qS",delta,TD+"/_gpg"+role),TD) except: rmtree(TD) raise rmtree(TD) if p.returncode: raise DebDeltaError('GnuPG fails to sign') if r: raise DebDeltaError('ar fails to add the signature') #################################################################### apply patch ########### info auxiliary routines def _info_patch_unzip_(TD): "unzip info and patch.sh" if os.path.exists(TD+'PATCH/info.gz'): system(('gunzip','PATCH/info.gz'),TD) if os.path.exists(TD+'PATCH/patch.sh.gz'): system(('gunzip','PATCH/patch.sh.gz'),TD) elif os.path.exists(TD+'PATCH/patch.sh.bz2'): system(('bunzip2','PATCH/patch.sh.bz2'),TD) elif os.path.exists(TD+'PATCH/patch.sh.lzma'): if not os.path.exists('/usr/bin/unlzma'): raise DebDeltaError('This patch needs lzma. Please install the Debian package "lzma".',retriable=True) system(('unlzma','PATCH/patch.sh.lzma'),TD) elif os.path.exists(TD+'PATCH/patch.sh.xz'): if not os.path.exists('/usr/bin/unxz'): raise DebDeltaError('This patch needs xz. Please install the Debian package "xz-utils".',retriable=True) system(('unxz','PATCH/patch.sh.xz'),TD) def get_info_slow(delta,T=None): if T: TD=T else: TD=tempo() if TD[-1] != '/': TD = TD + '/' delta=abspath(expanduser(delta)) system(('ar','x',delta,'info','info.gz','patch.sh','patch.sh.gz','patch.sh.bz2','patch.sh.lzma','patch.sh.xz'), TD+'/PATCH', ignore_output=True) _info_patch_unzip_(TD) info = _scan_delta_info_(TD) if T == None: rmtree(TD) return info def get_info_fast(delta): f=open(delta,'rb') s=f.readline() if b"!\n" != s : raise DebDeltaError('This is not a delta file: '+delta) s = f.read(60) if len(s) != 60 : logger.warning('(Warning, cannot get info from truncated: '+delta+' )') return None if s[:4] != b'info': #old style debdelta, with info in patch.sh if VERBOSE > 1 : logger.warning(' (Warning, cannot get info from old style: '+delta+' )') return None ##parse ar segment ## see /usr/include/ar.h if s[-2:] != b'`\n' : logger.warning('(Warning, cannot get info from '+delta+' , format not known)') return None l=int(s[ -12:-2 ]) s=bytes2str(f.read(l)) if len(s) != l : logger.warning('(Warning, cannot get info from truncated: '+delta+' )') return None info= s.split('\n') f.close() return info def get_info(delta,TD=None): info=get_info_fast(delta) if info == None: info=get_info_slow(delta,TD) return info def _scan_delta_info_(TD): info=[] if os.path.isfile(TD+'PATCH/info'): #new style debdelta, with info file p=open(TD+'PATCH/info') info=p.read().split('\n') p.close() if info[-1] == '': info.pop() else: #old style debdelta, with info in patch.sh p=open(TD+'PATCH/patch.sh') s=p.readline() s=p.readline() while s: if s[0] == '#' : s=de_n(s) info.append(s[1:]) s=p.readline() p.close() return info def info_2_db(info): params={} for s in info: if ':' in s: i=s.index(':') params[s[:i]] = s[i+2:] elif s: params[s] = True return params ########### other auxiliary routines def patch_check_tmp_space(params,olddeb): if type(params) != dict: params=info_2_db(params) if 'NEW/Installed-Size' not in params or 'OLD/Installed-Size' not in params: logger.warning('(Warning... Installed size unknown...)') return True free=freespace(TMPDIR) if free == None : return True free = free / 1024 if os.path.isdir(olddeb): instsize=int(params['NEW/Installed-Size']) #the last action of the script is to gzip the data.tar, so if 'NEW/Size' in params : instsize += int(params['NEW/Size']) / 1024 else: instsize = instsize * 1.8 else: instsize=int(params['NEW/Installed-Size'])+int(params['OLD/Installed-Size']) instsize += 2**13 if free < instsize : return _('not enough disk space (%(free)dkB) in directory %(dir)s for applying delta (needs %(size)dkB)') % \ {'free' : int(free), 'dir' : TMPDIR, 'size' : instsize} else: return True def scan_diversions(): f=open('/var/lib/dpkg/diversions') d={} a=1 while 1: a=f.readline() if not a: break a=de_n(a) b=de_n(f.readline()) p=de_n(f.readline()) d[a]=(b,p) f.close() return d ###################################################### debforensic extract #in base-passwd 3.5.11 #/usr/share/base-passwd/passwd.master base_passwd="""root::0:0:root:/root:/bin/bash daemon:*:1:1:daemon:/usr/sbin:/bin/sh bin:*:2:2:bin:/bin:/bin/sh sys:*:3:3:sys:/dev:/bin/sh sync:*:4:65534:sync:/bin:/bin/sync games:*:5:60:games:/usr/games:/bin/sh man:*:6:12:man:/var/cache/man:/bin/sh lp:*:7:7:lp:/var/spool/lpd:/bin/sh mail:*:8:8:mail:/var/mail:/bin/sh news:*:9:9:news:/var/spool/news:/bin/sh uucp:*:10:10:uucp:/var/spool/uucp:/bin/sh proxy:*:13:13:proxy:/bin:/bin/sh www-data:*:33:33:www-data:/var/www:/bin/sh backup:*:34:34:backup:/var/backups:/bin/sh list:*:38:38:Mailing List Manager:/var/list:/bin/sh irc:*:39:39:ircd:/var/run/ircd:/bin/sh gnats:*:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/bin/sh nobody:*:65534:65534:nobody:/nonexistent:/bin/sh""" base_passwd_db={} base_passwd_anti_db={} for a in base_passwd.split('\n'): a=a.split(':') base_passwd_db[a[0]]=int(a[2]) base_passwd_anti_db[int(a[2])]=a[0] base_group="""root:*:0: daemon:*:1: bin:*:2: sys:*:3: adm:*:4: tty:*:5: disk:*:6: lp:*:7: mail:*:8: news:*:9: uucp:*:10: man:*:12: proxy:*:13: kmem:*:15: dialout:*:20: fax:*:21: voice:*:22: cdrom:*:24: floppy:*:25: tape:*:26: sudo:*:27: audio:*:29: dip:*:30: www-data:*:33: backup:*:34: operator:*:37: list:*:38: irc:*:39: src:*:40: gnats:*:41: shadow:*:42: utmp:*:43: video:*:44: sasl:*:45: plugdev:*:46: staff:*:50: games:*:60: users:*:100: nogroup:*:65534:""" base_group_db={} base_group_anti_db={} for a in base_group.split('\n'): a=a.split(':') base_group_db[a[0]]=int(a[2]) base_group_anti_db[int(a[2])]=a[0] # all code following return name,mode,tartype,uid,gid,uname,gname #adapted from tarfile.py, a Python module def stat_to_tar(name): "returns name,mode,tartype,uid,gid,uname,gname,data" statres = os.lstat(name) stmd = statres.st_mode data = None if stat.S_ISREG(stmd): tartype = tarfile.REGTYPE # here ideally we should SHA1 the file ; # but this is done elsewhere for performance, # and to have multi_hash in the future elif stat.S_ISDIR(stmd): tartype = tarfile.DIRTYPE elif stat.S_ISFIFO(stmd): tartype = tarfile.FIFOTYPE elif stat.S_ISLNK(stmd): tartype = tarfile.SYMTYPE data = os.readlink(name) elif stat.S_ISCHR(stmd): tartype = tarfile.CHRTYPE elif stat.S_ISBLK(stmd): tartype = tarfile.BLKTYPE elif stat.S_ISSOCK(stmd): tartype = 'SOCKET' #SOCKETs are not supported in tar files else: raise TypeError if tartype in (tarfile.CHRTYPE, tarfile.BLKTYPE): data = str(os.major(statres.st_rdev))+' '+str( os.minor(statres.st_rdev)) uid,gid = statres.st_uid, statres.st_gid if uid in base_passwd_anti_db : uname = base_passwd_anti_db[uid] else: import pwd try: uname = pwd.getpwuid(uid)[0] except KeyError: uname = None if gid in base_group_anti_db : gname = base_group_anti_db[gid] else: import grp try: gname = grp.getgrgid(gid)[0] except KeyError: gname = None #07777 is used in tarfile.TarInfo.tobuf return name.lstrip('/'), stmd & 0o7777, tartype, uid, gid, uname, gname, data def tarinfo_to_ls(tartype,tarmode): "returns a string -rwxrwxrwx such as what ls -l prints " if ord(tartype) == 0 : a='_' else: if tartype >= b'0' and tartype <= b'6' : a="-hlcbdp"[ord(tartype) - ord('0')] else: a='?' return a + stat_filemode(tarmode)[1:] def sha1_hash_file(f): "returns hex digest of file" s=hashlib.sha1() if type(f) in string_types: f=open(f,'rb') a=f.read(1024) while a: s.update(a) a=f.read(1024) f.close() return s.hexdigest() def forensics_rfc(o,db,bytar,controlfiles,files,conffiles,diverted=[],diversions={},localepurged=[],prelink_u_failed=[]): " this is invoked by do_patch_() as well as do_delta_() ; in the former case, by_tar=False" assert type(diversions) == dict if type(db) == dict: for a in sorted(db.keys()): assert isinstance(a,str) if a[:3] == 'OLD': o.write(a[4:]+': '+db[a]+'\n') else: for a in sorted(db): assert isinstance(a,str) if a[:3] == 'OLD': o.write(a[4:]+'\n') if diverted: o.write("Diversions:\n") for a in sorted(diverted): b,p = diversions[a] o.write(" From: "+a+'\n') o.write(" To: "+b+'\n') o.write(" By: "+p+'\n') if conffiles: o.write("Conffiles:\n") for a in sorted(conffiles): o.write(' '+a+'\n') for L,N in ((controlfiles,"Control"),(files,"Files")): o.write(N+":\n") for l in sorted(L): if bytar: name,mode,tartype,uid,gid,uname,gname,data=l tmpcopy=None divert=None else: name,divert,tmpcopy=l if os.path.exists(divert): fullname,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(divert) else: fullname,mode,tartype,uid,gid,uname,gname,data='',0,'?',0,0,'?','?','?' if tartype == tarfile.REGTYPE: if tmpcopy and os.path.exists(tmpcopy): data=sha1_hash_file(tmpcopy) elif os.path.exists(divert): data=sha1_hash_file(divert) if name in ('.', '/', './', '/.') and tartype == tarfile.DIRTYPE: #skip root continue if uname == None: uname=str(uid) if gname == None: gname=str(gid) name=de_bar(name) o.write(' '+tarinfo_to_ls(tartype,mode)+" "+uname+' '+gname) if N == "Files" and tartype == tarfile.REGTYPE and name in conffiles: o.write(" [conffile]") if N == "Files" and tartype == tarfile.REGTYPE and name in localepurged: o.write(" [localpurged]") if N == "Files" and tartype == tarfile.REGTYPE and name in prelink_u_failed: o.write(" [prelink-u failed]") if divert and not os.path.exists(divert): o.write(" [missing file %r]" % divert) if tmpcopy: o.write(" [prelink-u]") o.write("\n "+name+"\n") if data!=None: o.write(" "+data+"\n") else: o.write(" \n") def tar_those(f): " tar multiple files in one tar (all in the same base directory!). Note that f may be a list of lists or strings or mixed." temptar=tempfile.mktemp(suffix='.tgz') tar=tarfile.open(name=temptar,mode='w:gz') for z in f: if type(z) in (list,tuple): for j in z: tar.add(j,arcname=os.path.basename(j)) elif type(z) in string_types: tar.add(z,arcname=os.path.basename(z)) else: DebDeltaError(' internal error m92ksy') tar.close() return temptar def forensic_send(f,forensic=FORENSIC): " note that f must be a list of lists (or None)" assert type(f) == list if forensic is False or forensic == 'False' or forensic == '': if f: logger.error(_('(Faulty delta. Please consider retrying with the option "--forensic=do" ).')+'\n') return if not f: return if all([(z == None) for z in f]): logger.warning('Sorry, no forensic logs were generated') return if forensic[:4] in ('mutt','mail') or forensic[:7] == 'icedove' or forensic[:10]=='thunderbird': email=FORENSIC_EMAIL if ':' in forensic: a=forensic.find(':') email == forensic[a:] forensic=forensic[:a] logger.warning(_("There were faulty deltas.")+' '+_("Now invoking the mail sender to send the logs.")) if forensic in ('mutt','mail'): if sys.version_info.major == 2: raw_input( _('(hit any key)') ) else: input( _('(hit any key)') ) args=[] for z in f: if z: for j in z: args+=['-a',j] subprocess.call(['mutt',email,'-s','delta_failures']+args) else: temptar=tempfile.mktemp(suffix='.tgz') tar=tarfile.open(name=temptar,mode='w:gz') for z in f: if z: for j in z: tar.add(j,arcname=os.path.basename(j)) tar.close() args="to=%s,subject=delta_failures,attachment='file:///%s'" % (email,temptar) subprocess.call([forensic,'-compose',args]) return elif forensic.startswith('http'): logger.warning(_("There were faulty deltas.")+' '+_('Sending logs to server.')) temptar=tempfile.mktemp(suffix='.tgz') tar=tarfile.open(name=temptar,mode='w:gz') for z in f: if z: for j in z: tar.add(j,arcname=os.path.basename(j)) tar.close() #http://atlee.ca/software/poster ## FIXME this will not work with python3 import urllib, urllib2, httplib, poster poster.streaminghttp.register_openers() datagen, headers = poster.encode.multipart_encode({'auth_userid':'debdelta','auth_password':'slartibartfast',"thefile": open(temptar, "rb")}) # Create the Request object request = urllib2.Request("http://debdelta.debian.net:7890/receive", datagen, headers) # Actually do the request, and get the response logger.info(' '+_('Server answers:'),repr(urllib2.urlopen(request).read())) return else: logger.warning(_('Faulty delta. Please send by email to %s the following files:\n') % EMAIL) for z in f: if z: logger.warning(' '+str.join(' ',z)+'\n') return logger.warning(_('(Faulty delta. Please consider retrying with the option "--forensic=do" ).')+'\n') def elf_info(f): "returns (is_elf, ei_class, ei_data, ei_osabi, e_type)" import struct elfheader=open(f,'rb').read(32) if len(elfheader) == 32: #parse as specified in /usr/include/elf.h from libelf-dev EI_CLASS={1:'ELFCLASS32', 2:'ELFCLASS64'} EI_DATA={1:'ELFDATA2LSB', # 2's complement, little endian 2:'ELFDATA2MSB'} # 2's complement, big endian EI_OSABI={0:'ELFOSABI_SYSV',# UNIX System V ABI 1:'ELFOSABI_HPUX', 2:'ELFOSABI_NETBSD', 3:'ELFOSABI_LINUX', #fixme insert other values 9:'ELFOSABI_FREEBSD', 12:'ELFOSABI_OPENBSD', 97:'ELFOSABI_ARM'} #fixme what is ET_LOOS , ET_HIOS , ET_LOPROC, ET_HIPROC ?? ET_TYPE={1:'ET_REL', #Relocatable file 2:'ET_EXEC', #Executable file 3:'ET_DYN', #Shared object file 4:'ET_CORE'} #Core file ei_magic, ei_class, ei_data, ei_version, ei_osabi, ei_abiversion = \ struct.unpack_from('4sBBBBB',elfheader) e_type, e_machine, e_version = struct.unpack_from('HHI',elfheader,16) #FIXME I think I am getting ei_osabi wrong.. it is always 0 is_elf = b'\x7fELF' == ei_magic #and ei_class in (1,2) and \ # ei_version == 1 and \ # ei_data in (1,2) and e_type>0 and e_machine>0 and e_version>0 return is_elf, EI_CLASS.get(ei_class), \ EI_DATA.get(ei_data), EI_OSABI.get(ei_osabi), ET_TYPE.get(e_type) #, e_machine, e_version else: return False, 0, 0, 0, 0 def parse_prelink_conf(): " fixme , currently unused and incomplete " prelinked_dirs=[] prelinked_blacklist=[] prelinked_blacklist_glob=[] for a in open('/etc/prelink.conf'): if a[0] == '#': continue a=a.strip() b=a.split() if len(b) != 2: logger.warning(' (sorry this line of /etc/prelink.conf cannot be parsed currently: "'+a+'")') continue if '-b' == b[0]: if '/' in b[1]: prelinked_blacklist.append(b[1]) else: prelinked_blacklist_glob.append(b[1]) elif '-l' == b[0]: prelinked_dirs.append(b[1]) ############ do_patch def do_patch(delta,olddeb,newdeb, info=None, diversions=None, do_gpg=DO_GPG): runtime={} T=None try: T=tempo() r=do_patch_(delta,olddeb,newdeb, T, runtime, info=info, diversions=diversions, do_gpg=do_gpg) except: if T : rmtree(T) if newdeb and os.path.exists(newdeb): os.unlink(newdeb) raise rmtree(T) return r def do_patch_(delta, olddeb, newdeb, TD, runtime, info=None, diversions=None, do_gpg=DO_GPG, do_progress=DO_PROGRESS): import threading if TD[-1] != '/': TD = TD + '/' HAVE_PRELINK=os.path.exists('/usr/sbin/prelink') # some people purge locales w/o using 'localepurge' , see e.g. http://bugs.debian.org/619086 #HAVE_LOCALEPURGE=os.path.exists('/etc/locale.nopurge') or os.path.exists('/usr/sbin/localepurge') delta=abspath(delta) newdebshortname='-' if newdeb: newdebshortname=newdeb newdeb=abspath(newdeb) if not os.path.isdir(olddeb): olddeb=abspath(olddeb) elif diversions == None: diversions=scan_diversions() start_sec = time.time() runtime['patchprogress']=0 check_is_delta(delta) if not os.path.isdir(olddeb): check_deb(olddeb) temp_name, temp_err_name, ret=system(('ar','xvo',delta), TD+'/PATCH', return_output=True, ignore_output=True) if ret : raise DebDeltaError('Cannot extract from '+delta) ar_list_delta=[a[4:] for a in open(temp_name).read().split('\n') if a] os.unlink(temp_name) os.unlink(temp_err_name) runtime['patchprogress']=1 is_signed=False for a in ar_list_delta: if a[:4] == '_gpg': r = verify_signature(TD+'/PATCH/'+a,TD+'/PATCH') if True != r: die(delta+": the signature file "+a+" fails as follows: "+repr(r)) is_signed=True if VERBOSE > 1 : logger.debug(' The signature by "'+a[4:]+'" is correctly verified for '+delta) if not is_signed: if do_gpg: die(_("Delta is not signed:")+' '+delta) elif do_gpg != None: logger.warning(_("WARNING, delta is not signed:")+' '+delta) runtime['patchprogress']=2 _info_patch_unzip_(TD) if not os.path.isfile(TD+'PATCH/patch.sh'): die("Error. File `%s' is not a delta file." % delta) os.symlink(minigzip,TD+'minigzip') os.symlink(minibzip2,TD+'minibzip2') #lets scan parameters, to see what it does and what it requires if info == None : info=_scan_delta_info_(TD) params=info_2_db(info) runtime['patchprogress']=3 #this is not needed in preparing the patch, but may help in forensic conf_files=[] z='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles' if FORENSIC and os.path.isfile(z): #note that filenames do not have leading / conf_files=[de_bar(p) for p in open(z).read().split('\n') if p] del z ### s=patch_check_tmp_space(params,olddeb) if s != True: raise DebDeltaError('Sorry, '+s, True ) if not os.path.isdir(olddeb): os.symlink(olddeb,TD+'/OLD.file') ar_list_old= list_ar(TD+'OLD.file') #unpack the old control structure, if available os.mkdir(TD+'/OLD/CONTROL') #unpack control.tar.gz f,b,z = untar_control_in_deb(ar_list_old) system(('ar','p',TD+'OLD.file',f,'|','tar',b,'-xp','-f','-','-C',TD+'OLD/CONTROL'),TD) #then we check for the conformance if not os.path.isdir(olddeb) and 'OLD/Size' in params: olddebsize = os.stat(olddeb)[ST_SIZE] if olddebsize != int(params['OLD/Size']): raise DebDeltaError('Old deb size is '+str(olddebsize)+' instead of '+params['OLD/Size']) runtime['patchprogress']=4 if DEBUG > 1 : #this is currently disabled, since 'dpkg -s' is vey slow (~ 1.6 sec) dpkg_params={} b=params['OLD/Package'] if os.path.isdir(olddeb): p=my_popen_read(['env', '-i', 'dpkg', '-s',b], **Popen_args) else: p=open(TD+'OLD/CONTROL/control') scan_control(p,params=dpkg_params,prefix='OLD') p.close() if os.path.isdir(olddeb): if 'OLD/Status' not in dpkg_params: die('Error: package %s is not known to dpkg.' % b) if dpkg_params['OLD/Status'] != 'install ok installed' : die('Error: package %s is not installed, status is %s.' % ( b , dpkg_params['OLD/Status'] ) ) for a in params: if a[:3] == 'OLD' and a != 'OLD/Installed-Size' and a != 'OLD/Size': if a not in dpkg_params: die('Error parsing old control file , parameter %s not found' % a) elif params[a] != dpkg_params[a] : die( 'Error : in delta , '+a+' = ' +params[a] +\ '\nin old/installed deb, '+a+' = ' +dpkg_params[a]) del b,p #cannot delete 'a', python raise a SyntaxError runtime['patchprogress']=5 ### some auxiliary routines, separated to make code more readable def dpkg_L_faster(pa,ar,diversions): "Scan dpkg -L . 'diversions' must be prepared by scan_diversions() . Returns list of pairs of files ,and list of diverted files. " s=[] diverted=[] n='/var/lib/dpkg/info/'+pa+':'+ar+'.list' if not DPKG_MULTIARCH or not os.path.exists(n): n='/var/lib/dpkg/info/'+pa+'.list' f=open(n) while 1: a=f.readline() if not a: break a=de_n(a) if a in diversions: b,p= diversions[a] if p != pa: s.append((a,b)) diverted.append(a) else: s.append((a,a)) else: s.append((a,a)) f.close() return s,diverted def dpkg_L(pa,ar): "Scan dpkg -L . Currently unused, see previous function." #sys.stderr.write('INTERNAL WARNING: USING OBSOLETE dpkg_L\n') s=[] diverted=[] if DPKG_MULTIARCH: p=my_popen_read(['env', '-i', 'dpkg-query', '-L', pa+':'+ar]) else: p=my_popen_read(['env', '-i', 'dpkg-query', '-L', pa]) a=p.readline() while a: a=de_n(a) #support diversions if a[:26] == 'package diverts others to:': continue if s and a[:11] == 'diverted by' or a[:20] == 'locally diverted to:': orig,divert=s.pop() i = a.index(':') divert = a[i+2:] s.append( (orig,divert) ) diverted.append(orig) else: s.append( (a,a) ) a=p.readline() p.close() return s, diverted def _symlink_data_tree(pa,ar,TD,diversions,runtime): localepurged=[] prelink_u_failed=[] file_triples=[] prelink_time = 0 prelink_datasize = 0 if diversions: s,diverted=dpkg_L_faster(pa,ar,diversions) else: s,diverted=dpkg_L(pa,ar) progressline=0 progresslen=float(len(s)) for orig,divert in s: progressline+=1 progress=6.0 + 6.0 * float(progressline) / progresslen runtime['patchprogress']=progress if do_progress: sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) if os.path.isfile(divert) and not os.path.islink(divert) : tmpcopy=TD+'OLD/DATA'+orig d=os.path.dirname(tmpcopy) if not os.path.exists(d): os.makedirs(d) #the following code idea was provided by roman@khimov.ru unprelink=False if HAVE_PRELINK : prelink_time -= time.time() is_elf, ei_class, ei_data, ei_osabi, e_type = elf_info(divert) #according to prelink-0.0.20090925/src/main.c unprelink = is_elf and e_type in ('ET_DYN','ET_EXEC') prelink_time += time.time() if unprelink: prelink_time -= time.time() prelink_datasize += os.path.getsize(divert) if VERBOSE > 3 : logger.debug(' copying/unprelinking '+divert+' to '+ tmpcopy) #unfortunately 'prelink -o' sometimes alters files, see http://bugs.debian.org/627932 shutil.copy2(divert, tmpcopy) proc=subprocess.Popen(["/usr/sbin/prelink","-u",tmpcopy],stdin=open(os.devnull),\ stdout=subprocess.PIPE,stderr=subprocess.STDOUT, close_fds=True, **Popen_args) out=proc.stdout.read().strip() proc.wait() if proc.returncode: if not os.path.exists(tmpcopy): if VERBOSE > 4 : logger.debug(' (prelink failed, symlinking '+divert+' to '+tmpcopy+')') os.symlink(divert, tmpcopy) prelink_u_failed.append(de_bar(orig)) unprelink=False elif VERBOSE > 4 : logger.debug(' (prelink failed, but file was copied)') thestat = os.statvfs(tmpcopy) if out[-39:] == 'does not have .gnu.prelink_undo section': if DEBUG > 1: logger.debug(' '+repr(out)) elif (thestat.f_bsize * thestat.f_bavail / 1024) < 50000 : logger.warning('!!Prelink -u failed, it needs at least 50000KB of free disk space\n') prelink_u_failed.append(de_bar(orig)) unprelink=False else: logger.warning('!!Prelink -u failed on %r : %r\n', tmpcopy, out) prelink_u_failed.append(de_bar(orig)) unprelink=False prelink_time += time.time() else: if VERBOSE > 3 : logger.debug(' symlinking '+divert+' to '+a) os.symlink(divert, tmpcopy) if unprelink and FORENSIC: #unfortunately the script will delete the 'tmpcopy', so we hardlink it z=tempfile.mktemp(prefix=TD) os.link(tmpcopy,z) file_triples.append((orig,divert,z)) else: file_triples.append((orig,divert,None)) elif not os.path.exists(divert) and os.path.islink(divert): file_triples.append((orig,divert,None)) if VERBOSE > 1 : logger.debug(' Broken symlink? %r',divert) elif not os.path.exists(divert): file_triples.append((orig,divert,None)) if VERBOSE : logger.debug(' Disappeared file? %r',divert) for z in ('locale','man','gnome/help','omf','doc/kde/HTML'): w='/usr/share/'+z if orig[:len(w)] == w: localepurged.append(de_bar(orig)) else: file_triples.append((orig,divert,None)) if VERBOSE > 3 : logger.debug(' not symlinking %r to %r',divert,orig) return file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize def chmod_add(n,m): "same as 'chmod ...+... n '" om=S_IMODE(os.stat(n)[ST_MODE]) nm=om | m if nm != om : if VERBOSE > 2 : logger.debug(' Performing chmod %s %s %s',n,oct(om),oct(nm)) os.chmod(n,nm) def _fix_data_tree_(TD): for (dirpath, dirnames, filenames) in os.walk(TD+'OLD/DATA'): chmod_add(dirpath, S_IRUSR | S_IWUSR| S_IXUSR ) for i in filenames: i=os.path.join(dirpath,i) if os.path.isfile(i): chmod_add(i, S_IRUSR | S_IWUSR ) for i in dirnames: i=os.path.join(dirpath,i) chmod_add(i, S_IRUSR | S_IWUSR| S_IXUSR ) #initialize, just in case control_file_triples=[] file_triples=[] localepurged=[] prelink_u_failed=[] diverted=[] prelink_time=0 prelink_datasize=0 ###see into parameters: the patch may need extra info and data runtime['patchprogress']=6 prelink_time=None for a in params: if 'needs-old' == a: if os.path.isdir(olddeb): die('This patch needs the old version Debian package') elif 'old-data-tree' == a : os.mkdir(TD+'/OLD/DATA') if os.path.isdir(olddeb): file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize=\ _symlink_data_tree(params['OLD/Package'],params['OLD/Architecture'],TD,diversions,runtime) else: ar_list_old= list_ar(TD+'OLD.file') if b'data.tar.bz2' in ar_list_old: system(('ar','p',TD+'OLD.file','data.tar.bz2','|','tar','-xp','--bzip2','-f','-','-C',TD+'OLD/DATA'), TD) elif b'data.tar.gz' in ar_list_old: system(('ar','p',TD+'OLD.file','data.tar.gz','|','tar','-xp','-z','-f','-','-C',TD+'OLD/DATA'), TD) elif b'data.tar.lzma' in ar_list_old: if not os.path.exists('/usr/bin/lzma'): raise DebDeltaError('This patch needs lzma. Please install the Debian package "lzma".',retriable=True) system(('ar','p',TD+'OLD.file','data.tar.lzma','|','unlzma','-c','|','tar','-xpf','-','-C',TD+'OLD/DATA'), TD) elif b'data.tar.xz' in ar_list_old: if not os.path.exists('/usr/bin/xz'): raise DebDeltaError('This patch needs xz. Please install the Debian package "xz-utils".',retriable=True) system(('ar','p',TD+'OLD.file','data.tar.xz','|','unxz','-c','|','tar','-xpf','-','-C',TD+'OLD/DATA'), TD) else: assert(0) _fix_data_tree_(TD) elif 'old-control-tree' == a: if os.path.isdir(olddeb): if not os.path.isdir(TD+'OLD/CONTROL'): os.mkdir(TD+'OLD/CONTROL') p=params['OLD/Package'] a=params['OLD/Architecture'] for b in dpkg_keeps_controls : z='/var/lib/dpkg/info/' + p + ':' + a + '.'+b if not DPKG_MULTIARCH or not os.path.exists(z): z='/var/lib/dpkg/info/' + p +'.'+b if os.path.exists(z): os.symlink(z,TD+'OLD/CONTROL/'+b) control_file_triples.append((b,z,None)) del z,p #cannot delete 'a', python raise a SyntaxError #else... we always unpack the control of a .deb elif 'needs-xdelta3' == a: if not os.path.exists('/usr/bin/xdelta3'): raise DebDeltaError('This patch needs xdelta3. Please install the Debian package "xdelta3".',retriable=True) elif 'needs-xdelta3-fifo' == a: # not doing a specific check, I am using debian/control Conflicts if not os.path.exists('/usr/bin/xdelta3'): raise DebDeltaError('This patch needs xdelta3, at least version 3.0y. Please install the Debian package "xdelta3".',retriable=True) elif 'needs-xdelta' == a: if not os.path.exists('/usr/bin/xdelta'): raise DebDeltaError('This patch needs xdelta. Please install the Debian package "xdelta".',retriable=True) elif 'needs-bsdiff' == a: if not os.path.exists('/usr/bin/bsdiff'): raise DebDeltaError('This patch needs bsdiff. Please install the Debian package "bsdiff".',retriable=True) elif 'needs-lzma' == a: if not os.path.exists('/usr/bin/lzma'): raise DebDeltaError('This patch needs lzma. Please install the Debian package "lzma".',retriable=True) elif 'needs-xz' == a: if not os.path.exists('/usr/bin/xz'): raise DebDeltaError('This patch needs xz. Please install the Debian package "xz-utils".',retriable=True) elif 'needs-minibzip2' == a: pass #its your lucky day elif a[:6] == 'needs-': raise DebDeltaError('patch says "'+a+"' and this is unsupported. Get a newer debdelta.",retriable=True) elif params[a] == True: logger.warning('WARNING patch says %r and this is unsupported. Get a newer debdelta.',a) if localepurged and not DEBUG: #actually we cannot be 100% sure that the delta really needs those files, but it is quite plausible raise DebDeltaError('Error, '+str(len(localepurged))+' locale files are absent.') runtime['patchprogress']=12 script_time = - time.time() this_deb_format=DEB_FORMAT if this_deb_format != 'deb' and 'NEW/data.tar' not in params: #this patch does not support streaming logger.warning('Warning, created as standard deb: '+newdeb) this_deb_format='deb' if this_deb_format == 'preunpacked' : #need a fifo os.mkfifo(TD+'data.pipe') temp_err_name_fd, temp_err_name = tempfile.mkstemp(prefix='debdeltaE') temp_name_fd, temp_name = tempfile.mkstemp(prefix='debdeltaO') cmd=[SHELL,'-e','PATCH/patch.sh'] if DEBUG > 2 : cmd=[SHELL,'-evx','PATCH/patch.sh'] if this_deb_format == 'unzipped': cmd+=['unzipped'] elif this_deb_format == 'preunpacked': cmd+=['piped'] env={'PATH':os.getenv('PATH')} F=subprocess.Popen(cmd, cwd=TD, bufsize=4096,close_fds=True, stdin=open(os.devnull),env=env, stderr=temp_err_name_fd, stdout=temp_name_fd) ### data used by the preunpacked method data_md5=None # md5 of uncompressed data.tar tar_status=[] # should be [True] if 'preunpacked' went fine md5_status=[] # idem #this list contains tuples of (unpacked_temporary_filename, real_filename, owner, group, tartype, mode, mtime, linkname) preunpacked_filelist=[] def do_cleanup(): for a in preunpacked_filelist: pass ##CHEAT we are not (yet) writing anything to disk #if a[0]: os.unlink(a[0]) if this_deb_format == 'preunpacked' : #do progress reporting and unpacking in filesystem def do_extension(): return '_'+str(random.randint(1,9999)).rjust(4,'0')+'_debdelta_preunpacked' def do_pipe_md5(i,o,rm,ms,ts): try: a=i.read(1024) while a and (ts == [] or ts == [True]) : rm.update(a) o.write(a) a=i.read(1024) #TODO implement progress reporting here as well o.close() ms.append(True) except: import sys ms.append(sys.exc_info()) if DEBUG: logger.debug(' do_pipe_md5 crashed: '+repr(ms)) def do_tar(i,fl,s): try: dt=tarfile.open(mode='r|',fileobj=i) for tarmember in dt: if tarmember.isreg(): n='/'+tarmember.name+do_extension() while os.path.exists(n) : #wont overwrite existing stuff, never ever n='/'+tarmember.name+do_extension() else: n='' a=(n, tarmember.name, tarmember.uname, tarmember.gname,\ tarmember.type, tarmember.mode, tarmember.mtime, tarmember.linkname) fl.append(a) if n: pass ##CHEAT we are not writing anything today! #datatar.extract(tarmember,path=n) #successfully untarred! s.append(True) except:#catch problems such as "out of disk space" or corrupted data import sys s.append(sys.exc_info()) if DEBUG: logger.debug(' do_tar crashed: '+repr(s)) #flush input (note that do_pipe_md5 will soon stop writing) no it seems useless #i.read() try: datapipe=open(TD+'data.pipe') rolling_md5=hashlib.md5() (piper,pipew)=os.pipe() md5_thread=threading.Thread(target=do_pipe_md5, args=(open(TD+'data.pipe'),os.fdopen(pipew,'w'),rolling_md5,\ md5_status,tar_status)) tar_thread=threading.Thread(target=do_tar, args=(os.fdopen(piper),preunpacked_filelist,tar_status)) #yeah maybe using two threads is more complex than strictly needed md5_thread.daemon=True md5_thread.start() tar_thread.daemon=True tar_thread.start() #join back the md5 md5_thread.join() data_md5=rolling_md5.hexdigest() if md5_status == [True] and params['NEW/data.tar'][:32] == data_md5: tar_thread.join() if tar_status == [True]: F.wait() #write data_list a=open(TD+'data_list','w') a.write('Files:\n') for (unpacked_temporary_filename, real_filename, owner,\ group, tartype, mode, mtime, linkname) in preunpacked_filelist: ## to convert 'tar' type into 'ls' type if tartype >= '0' and tartype <= '6' : tartype="-hlcbdp"[ord(tartype) - ord('0')] else: tartype='?' logger.debug('WARNING unsupported tar type '+repr(tartype)+' for: '+repr(real_filename)) #if tartype < '0' or tartype > '6' : # tartype='?' # print 'WARNING unsupported tar type '+repr(tartype)+' for: '+repr(real_filename) mode=oct(mode).rjust(4,'0') a.write(' %s %s %s %s %s\n %s\n %s\n %s\n' % (tartype,mode,owner,group,mtime,\ unpacked_temporary_filename,real_filename,linkname)) a.close() #append it into deb system(['ar','q','NEW.file', 'data_list'], TD) except: do_cleanup() raise else: #progress reporting for deb_format != 'preunpacked' runtime['patchprogress']=12 if 'NEW/Size' in params and do_progress: NEW_size=int(params['NEW/Size']) while None == F.poll(): if os.path.exists(TD+'NEW.file'): a=os.path.getsize(TD+'NEW.file') progress=(int(12.0 + 84.0 * a / NEW_size)) else: progress=12 runtime['patchprogress']=progress time.sleep(0.1) if do_progress: sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) F.wait() if do_progress and terminalcolumns: #clean up sys.stderr.write(' ' * terminalcolumns + '\r') ret=F.returncode os.close(temp_err_name_fd) os.close(temp_name_fd) script_time += time.time() # for --format='preunpacked' this time also include data MD5 runtime['patchprogress']=97 #helper for debugging def tempos(f): if os.path.getsize(temp_name): f.append(temp_name) if os.path.getsize(temp_err_name): f.append(temp_err_name) if not FORENSIC: def fore(): return None elif not os.path.isdir(olddeb): def fore(): f=[delta,olddeb] tempos(f) return f else: def fore(): temp_fore_name='' f=[] tempos(f) try: (temp_fd,temp_fore_name) = tempfile.mkstemp(prefix="debforensic_"+params['NEW/Package']+"_") temp_file=os.fdopen(temp_fd,'w') temp_file.write('Delta: '+delta+'\n') temp_file.write('DeltaSHA1: ' + sha1_hash_file(delta) + '\n') temp_file.write('LocalePurgedFilesN: '+str(len(localepurged))+'\n') temp_file.write('PrelinkUFailedN: '+str(len(prelink_u_failed))+'\n') if ret: temp_file.write('PatchExitCode: '+str(ret)+'\n') forensics_rfc(temp_file,params,False,control_file_triples,file_triples,conf_files, diverted,diversions,localepurged,prelink_u_failed) #copy short content here and remove from list for i in copy(f): if os.path.getsize(i) < 2000: f.remove(i) temp_file.write('PatchLogFile_'+str(i)+'_content:\n') for ll in open(i): temp_file.write(' '+repr(ll)+'\n') else: temp_file.write('PatchLogFileIs: '+str(i)+'\n') temp_file.close() except OSError: #Exception,s: die('!!While creating forensic '+temp_fore_name+' error:'+str(s)+'\n') f.append(temp_fore_name) return f if ret: if localepurged: raise DebDeltaError('"debdelta" is incompatible with "localepurge".') else: f=fore() raise DebDeltaError('error in patch.sh.',logs=f) #then we check for the conformance if this_deb_format == 'deb': if 'NEW/Size' in params: newdebsize = os.stat(TD+'NEW.file')[ST_SIZE] if newdebsize != int(params['NEW/Size']): f=fore() raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size'],logs=f) if DO_MD5: if 'NEW/MD5sum' in params: if VERBOSE > 1 : logger.debug(' verifying MD5 for %r',os.path.basename(newdeb or delta)) m= compute_md5(open(TD+'NEW.file','rb')) if params['NEW/MD5sum'] != m : f=fore() raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) , logs=f) else: logger.warning(' Warning! no MD5 was verified for %r',os.path.basename(newdeb or delta)) elif this_deb_format == 'unzipped' : if DO_MD5: c,b,z=untar_control_in_deb(list_ar(TD+'NEW.file')) p=PopenPipe(['ar','p',(TD+'NEW.file'),c,'|']+z, stdout=subprocess.PIPE) m=compute_md5(p.stdout) p.wait() if params['NEW/control.tar'][:32] != m: f=fore() raise DebDeltaError('MD5 mismatch for control.tar' , logs=f) p=subprocess.Popen(['ar','p',(TD+'NEW.file'),'data.tar'], stdout=subprocess.PIPE) m=compute_md5(p.stdout) p.wait() if params['NEW/data.tar'][:32] != m: f=fore() raise DebDeltaError('MD5 mismatch for data.tar', logs=f) elif this_deb_format == 'preunpacked' : if tar_status != [True]: f=fore() do_cleanup() raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1]), logs=f) #todo format me better if md5_status != [True]: f=fore() do_cleanup() raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1]), logs=f) #todo format me better #if DO_MD5: #actually we always do MD5 m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/control.tar'][:32] != m: f=fore() do_cleanup() raise DebDeltaError('MD5 mismatch for control.tar', logs=f) if params['NEW/data.tar'][:32] != data_md5: f=fore() do_cleanup() raise DebDeltaError('MD5 mismatch for data.tar', logs=f) else: assert('unimplemented'=='') os.unlink(temp_name) os.unlink(temp_err_name) runtime['patchprogress']=99 if newdeb: shutil.move(TD+'NEW.file',newdeb) end_sec = time.time() elaps=(end_sec - start_sec) if VERBOSE : if newdeb: debsize = os.stat(newdeb)[ST_SIZE] else: debsize = os.stat(olddeb)[ST_SIZE] #this printout uses kibibytes, and not SizeToStr, to ease statistics out = ' ' + _('Patching done, time %(time).2fsec, speed %(speed)dk/sec') % \ {'time' : elaps, 'speed' : (debsize / 1024 / (elaps+.001))} out += ' ' + _('(script %(time).2fsec %(speed)dk/sec)') % \ {'time' : script_time, 'speed' : (debsize / 1024 / (script_time+.001))} if prelink_time != None and prelink_time > 0 : prelink_datasize=prelink_datasize/1024 out += _('(prelink %(time).2fsec, %(size)dk, %(speed)dk/s)') % \ {'time' : prelink_time, 'size' : prelink_datasize, 'speed' : prelink_datasize / (prelink_time + .001)} out += _('(unaccounted %.2fsec)') % (elaps-prelink_time-script_time) else: out += _('(unaccounted %.2fsec)') % (elaps-script_time) logger.info(out) ## this is useless, if 'debpatch' it is in the command line, if 'debdelta-upgrade' ## it is printed elsewhere #if newdeb != None: # print 'result: '+os.path.basename(newdeb), return (newdeb,elaps) ##################################################### compute delta def do_delta(olddeb,newdeb,delta): T=None try: T=tempo() if os.path.exists(delta+'_tmp_') : os.unlink(delta+'_tmp_') r=do_delta_(olddeb,newdeb,delta+'_tmp_',TD=T) (deltatmp, percent, elaps, info, gpg_hashes) = r info_hashes=append_info(deltatmp,info) if DO_GPG: gpg_hashes['info']=info_hashes sign_delta(deltatmp,gpg_hashes) if os.path.exists(delta) : os.rename(delta,delta+'~') os.rename(deltatmp,delta) except: if delta and os.path.exists(delta): os.unlink(delta) if delta and os.path.exists(delta+'_tmp_'): os.unlink(delta+'_tmp_') if T : rmtree(T) raise else: if T : rmtree(T) return r def do_delta_(olddeb, newdeb, delta, TD, forensic_file=None, info=[]): """This function creates a delta. The delta is 'ar' archive (see 'man ar'). The delta contains data, a script, and optional gpg signatures. The script recreates the new deb. Note that the deb is (again) an 'ar' archive, and has multiple members. Simple deb members are managed directly. The complex members are 'data.tar.xxx' and 'control.tar.gz' ; these are studied in the delta_tar() function, that in turn uses the Script() class. (Note that there is just one object instanced from Script(), in variable 'script'). To recreate one of the complex members, there are two main steps in the script: (1) rebuild the member, uncompressed (2) recompress the member The part (2) is managed by script.zip_piped(). The part (1) is more complex: (1a) there is a first subshell where data from the old version of the deb are piped to stdout (moreover some gzipped files may be transparently unzipped, delta-ed and re-gzipped, see delta_gzipped_files) (1b) the former (1a) is "piped" into the delta backend, to transform the old data into new. If the delta-backend is 'bsdiff' then the above process (1a,b) is repeated in chunks (indeed bsdiff cannot manage large files w/o exausting all your memory!) All the (chunks of) steps (1a,b) are in a subshell, and its result stdout is piped in (2). """ if TD[-1] != '/': TD = TD + '/' import fnmatch start_sec = time.time() #I do not like global variables but I do not know of another solution global bsdiff_time, bsdiff_datasize bsdiff_time = 0 bsdiff_datasize = 0 olddeb=abspath(olddeb) check_deb(olddeb) os.symlink(olddeb,TD+'/OLD.file') olddebsize = os.stat(olddeb)[ST_SIZE] newdeb=abspath(newdeb) check_deb(newdeb) os.symlink(newdeb,TD+'/NEW.file') newdebsize = os.stat(newdeb)[ST_SIZE] #process all contents of old vs new .deb ar_list_old= list_ar(TD+'OLD.file') ar_list_new= list_ar(TD+'NEW.file') free=freespace(TD) if free and free < newdebsize : raise DebDeltaError('Error: not enough disk space in '+TD, True) delta=abspath(delta) #generater for numbered files def a_numb_file_gen(): deltacount = 0 while 1: yield str(deltacount) deltacount+=1 a_numb_file=a_numb_file_gen() a_numb_patch=a_numb_file_gen() ####################### class Script ######################## class Script: """This class helps create the script 'patch.sh' that is the core of the delta. The script recreates the new deb. See documentation of do_delta_() for details. """ def __init__(self, delta_uses_infifo): #start writing script self.fd=open(TD+'PATCH/patch.sh','w') self.fd.write('#!/bin/bash -e\n') self.member=None self.current_chunk_name=None self.delta_uses_infifo=delta_uses_infifo if delta_uses_infifo : #create the fifo as input for xdelta3 self.the_fifo=next(a_numb_file) self.fd.write("mkfifo %s\n" % self.the_fifo) else: self.the_fifo=None #this is used when recompressing data.tar.zx self.xz_parameters=None #this is used when recompressing data.tar.gz self.gz_command=None def write(self,s): "verbatim write in the script" self.fd.write(s) def close(self): if self.the_fifo: self.fd.write("rm %s\n" % self.the_fifo) self.fd.close() def zip(self, n, cn, newhead=None): """inverts the unzip() function ; optionally, forces .gz header (to fight changes in libz) This is obsolete, not efficient, left as a compatibility layer.""" self.fd.write('cat "'+n+'" | ') self.zip_piped(cn, newhead) self.fd.write(" > '"+n+cn+"' && rm '"+n+"'\n") def zip_piped(self, cn, newhead=None): "inverts the unzip() function, with piped behaviour" if cn == '.gz' : cmd=str.join(' ',self.gz_command) if newhead: s=prepare_for_echo(newhead) self.fd.write("($E '"+ s +"' && "+cmd+" | tail -c +"+str(len(newhead)+1)+')') else: self.fd.write(cmd) elif cn == '.bz2' : info_append('needs-minibzip2') self.fd.write('./minibzip2 -9') elif cn == '.lzma' : info_append('needs-lzma') self.fd.write('lzma -9') elif cn == '.xz' : info_append('needs-xz') if self.xz_parameters==None: self.fd.write('xz -c') else: self.fd.write('xz -c '+self.xz_parameters) else: assert(0) def start_member(self, ar_line, newname, extrachar): "start a new 'ar' member" self.member=newname self.ar_line=ar_line self.extrachar=extrachar assert(self.current_chunk_name==None) self.fd.write('{\n') def end_member(self): assert(self.member) self.member=None self.fd.write('}\n') def start_rebuilding(self): "starts the first part of the delta/recompressing pipe (for a 'ar' member)" self.fd.write('(') def end_rebuilding(self): "ends the first part of the delta/recompressing pipe" self.fd.write(')|') #pipe the delta/recompressing pipe def recompressing(self,new_filename,new_filename_ext,new_file_zip_head): self.fd.write('(') append_NEW_file(self.ar_line) script.zip_piped(new_filename_ext,new_file_zip_head) self.fd.write(' >> NEW.file\n') #end delta tar if self.extrachar : append_NEW_file(self.extrachar) self.fd.write(')\n') def recompressing_by_arg(self,new_filename,new_filename_ext,new_file_zip_head,new_file_size): "flexible recompressing for data.tar , depending on first argument passed to the script" self.fd.write('( if test "$1" = "" \n then\n') self.recompressing(new_filename,new_filename_ext,new_file_zip_head) self.fd.write('elif test "$1" = "unzipped" \n then\n') #http://en.wikipedia.org/wiki/Ar_(Unix) assert(new_filename == 'NEW/data.tar') ar_line_unzipped=b'data.tar'.ljust(16)+self.ar_line[16:48]+str2bytes(str(new_file_size)).ljust(10)+b'`\n' append_NEW_file(ar_line_unzipped) #there is no extra char, tar is 512b blocks self.fd.write('cat >> NEW.file \n elif test "$1" = "piped" \n then cat >> data.pipe \n fi )\n') def start_chunk(self, current_chunk_name): "start the pipe to create the chunk. The chunk is always piped" self.fd.write('(') self.current_chunk_name=current_chunk_name def end_chunk(self, current_chunk_name): """this ends the 'data part' of a chunk, and writes the old data somewhere; a successive script code (generated by delta_files() ) will then delta it to transform old data into new data.""" assert(self.current_chunk_name==current_chunk_name) if self.the_fifo: self.fd.write(') > '+self.the_fifo+'&\n') #write to fifo, background else: self.fd.write(') > '+current_chunk_name+'\n') #write chunk self.current_chunk_name=None def md5_check_file(self, n, md5=None): "add a md5 check in the script (this is done only if a lot -d are passed on cmdline)" if md5==None: assert(os.path.isfile(TD+n)) md5=compute_md5(TD+n) logger.debug(" adding extra MD5 for %r",n) self.fd.write('if ! echo "%s %s" | md5sum -c --quiet ; then echo "%s is currupt!"; exit 1; fi\n' % (md5,n,n)) ####################### end of class Script ######################## ##### unpack control.tar.gz, scan control, write parameters # def info_append(s): "smart appending that avoids duplicate entries" if s not in info: info.append(s) for o,l in ('OLD',ar_list_old) , ('NEW',ar_list_new): os.mkdir(TD+o+'/CONTROL') #unpack control.tar.gz f=None for a in l : if a[:12] == b'control.tar.': f=a b={b'.gz':'-z', b'.xz':'-J', b'.bz2':'-j'}.get(f[11:]) assert b != None system(('ar','p',TD+o+'.file',f,'|','tar',b,'-xf','-','-C',TD+o+'/CONTROL'),TD) ## scan control p=open(TD+'/'+o+'/CONTROL/control') s=[] scan_control(p,params=None,prefix=o,info=s) p.close() if VERBOSE : logger.debug(' '+o+': '+str.join(' ',[o[4:] for o in s])) info = info + s info.append('OLD/Size: '+str(olddebsize)) info.append('NEW/Size: '+str(newdebsize)) params=info_2_db(info) #scan debdelta.conf to find any special requirement debdelta_conf=ConfigParser() debdelta_conf.read(['/etc/debdelta/debdelta.conf', expanduser('~/.debdelta/debdelta.conf') ]) debdelta_conf_skip=[] for s in debdelta_conf.sections(): if fnmatch.fnmatch(params['OLD/Package'],s): opt=debdelta_conf.options(s) if 'skip' in opt: debdelta_conf_skip += debdelta_conf.get(s,'skip').split(';') break if VERBOSE > 1 : logger.debug(' debdelta.conf says we will skip: %r', debdelta_conf_skip) gpg_hashes = {} if DO_MD5 : # compute a MD5 of NEW deb newdeb_md5sum=compute_md5(TD+'NEW.file') info.append('NEW/MD5sum: '+ newdeb_md5sum) else: newdeb_md5sum=None if NEEDSOLD : #this delta needs the old deb info.append('needs-old') else: info.append('old-data-tree') info.append('old-control-tree') #do we use a fifo as input for xdelta3 delta_uses_infifo = ('xdelta3-fifo' not in DISABLED_FEATURES) and (USE_DELTA_ALGO == 'xdelta3') #Note that there is just one object instanced from class Script() script=Script(delta_uses_infifo) a=USE_DELTA_ALGO if a == 'xdelta-bzip': a='xdelta' if not os.path.exists('/usr/bin/'+a): raise DebDeltaError('please install the package "'+a+'".', retriable=True) if delta_uses_infifo: info.append('needs-xdelta3-fifo') else: info.append('needs-'+a) del a #### check for disk space if 'NEW/Installed-Size' in params and 'OLD/Installed-Size' in params: free=freespace(TD) instsize=int(params['NEW/Installed-Size']) + int(params['OLD/Installed-Size']) if free and free < ( instsize * 1024 + + 2**23 + MAXMEMORY / 6 ) : raise DebDeltaError(' Not enough disk space (%dkB) for creating delta (needs %dkB).' % \ ( int(free/1024) , instsize ) , True ) ############# check for conffiles a=TD+'/OLD/CONTROL/conffiles' if os.path.exists(a): p=open(a) #files do not have leading / old_conffiles=[ de_bar(a) for a in p.read().split('\n') if a] p.close() else: old_conffiles=[] ## a=TD+'/OLD/CONTROL/list' ## if os.path.exists(a): ## p=open(a) ## for a in p: ## a=de_bar(de_n(a)) ## for j in debdelta_conf_skip: ## if fnmatch(a,j): ## old_conffiles.append(a) #OK, this abuses the name of the var a bit ## print ' REPR skip ',repr(a) ## else: ## print ' The old debian package ',olddeb,' does not contain a file list?!?' def shell_not_allowed(name): "Strings that I do not trust to inject into the shell script; maybe I am a tad too paranoid..." #FIXME should use it , by properly quoting for the shell script return '"' in name or "'" in name or '\\' in name or '`' in name # uses MD5 to detect identical files (even when renamed) def scan_md5(n): md5={} f=open(n) a=de_n(f.readline()) while a: m , n = a[:32] , de_bar( a[34:] ) md5[n]=m a=de_n(f.readline()) f.close() return md5 new_md5=None if os.path.exists(TD+'/NEW/CONTROL/md5sums'): new_md5=scan_md5(TD+'/NEW/CONTROL/md5sums') old_md5=None if os.path.exists(TD+'/OLD/CONTROL/md5sums') : old_md5=scan_md5(TD+'/OLD/CONTROL/md5sums') ############### some routines to prepare delta of two files def patch_append(f): #possibly GPG if DO_GPG: gpg_hashes[f] = _compute_hashes_(TD+'PATCH/'+f) if VERBOSE > 2 : a=os.stat(TD+'PATCH/'+f)[ST_SIZE] logger.debug(' appending %r of size %d to delta, %3.2f%% of new .deb',f,a,( a * 100. / newdebsize )) system(['ar','qSc', delta,f], TD+'/PATCH') unlink(TD+'PATCH/'+f) def verbatim(f): pp=next(a_numb_patch) p = 'PATCH/'+pp if VERBOSE > 1 : logger.debug(' including %r verbatim in patch', name) os.rename(TD+f,TD+p) patch_append(pp) return p def unzip(f): c='' if f[-3:] == '.gz' : system(('gunzip',f),TD) f=f[:-3] c='.gz' elif f[-4:] == '.bz2' : system(('bunzip2',f),TD) f=f[:-4] c='.bz2' elif f[-5:] == '.lzma' : info_append('needs-lzma') system(('unlzma',f),TD) f=f[:-5] c='.lzma' elif f[-3:] == '.xz' : info_append('needs-xz') system(('unxz',f),TD) f=f[:-3] c='.xz' else: raise NotImplementedError(' dont know how to decompress '+repr(f)) return (f,c) def delta_files__(o, n, p, algo, outpiped, infifo): "delta of file 'o' to 'n' using/producing patch 'p' ; xdelta3 can also pipe" this_delta_outpiped=False #bdiff #http://www.webalice.it/g_pochini/bdiff/ if algo == 'bdiff': system(('~/debdelta/bdiff-1.0.5/bdiff','-q','-nooldmd5','-nonewmd5','-d',o,n,p),TD) script.write('~/debdelta/bdiff-1.0.5/bdiff -p '+o+' '+p+' '+n+'\n') #zdelta #http://cis.poly.edu/zdelta/ elif algo == 'zdelta': system(('~/debdelta/zdelta-2.1/zdc',o,n,p),TD) script.write('~/debdelta/zdelta-2.1/zdu '+o+' '+p+' '+n+'\n') #bdelta #http://deltup.sf.net elif algo == 'bdelta': system(('~/debdelta/bdelta-0.1.0/bdelta',o,n,p),TD) script.write('~/debdelta/bdelta-0.1.0/bpatch '+o+' '+n+' '+p+'\n') #diffball #http://developer.berlios.de/projects/diffball/ elif algo == 'diffball': system(('~/debdelta/diffball-0.7.2/differ',o,n,p),TD) script.write('~/debdelta/diffball-0.7.2/patcher '+o+' '+p+' '+n+'\n') #rdiff elif algo == 'rdiff': system(('rdiff','signature',o,'sign_file.tmp'),TD) system(('rdiff','delta','sign_file.tmp',n,p),TD) script.write('rdiff patch '+o+' '+p+' '+n+'\n') #xdelta3 elif algo == 'xdelta3' : system(('xdelta3','-9','-R','-D','-n','-S','djw','-s',o,n,p),TD) if infifo : o = infifo #use fifo as input if outpiped: this_delta_outpiped=True script.write('xdelta3 -d -R -D -c -s '+o+' '+p+'\n') else: script.write('xdelta3 -d -R -D -s '+o+' '+p+' '+n+'\n') ## according to the man page, ## bsdiff uses memory equal to 17 times the size of oldfile ## but , in my experiments, this number is more like 12. ##But bsdiff is sooooo slow! elif algo == 'bsdiff' : # not ALLOW_XDELTA or ( osize < (MAXMEMORY / 12)): system(('bsdiff',o,n,p),TD) script.write('bspatch '+o+' '+n+' '+p+'\n') #seems that 'xdelta' is buggy on 64bit and different-endian machines #xdelta does not deal with different endianness! elif algo == 'xdelta-bzip' : system(('xdelta','delta','--pristine','--noverify','-0','-m'+str(int(MAXMEMORY/1024))+'k',o,n,p),TD) system('bzip2 -9 '+p,TD,(p,)) script.write('bunzip2 '+p+'.bz2 ; xdelta patch '+p+' '+o+' '+n+'\n') p += '.bz2' elif algo == 'xdelta' : system(('xdelta','delta','--pristine','--noverify','-9','-m'+str(int(MAXMEMORY/1024))+'k',o,n,p),TD) script.write('xdelta patch '+p+' '+o+' '+n+'\n') elif algo == 'jojodiff' : system(('~/debdelta/jdiff06/src/jdiff','-b',o,n,p),TD) script.write('~/debdelta/jdiff06/src/jpatch '+o+' '+p+' '+n+'\n') else: raise AssertionError(' unsupported delta algo ') return p, this_delta_outpiped def delta_files(o, n, outpiped=None, infifo=None): " compute delta of two files , and prepare the script consequently" nsize = os.path.getsize(TD+n) osize = os.path.getsize(TD+o) if VERBOSE > 1 : logger.debug(' compute delta for %s (%dkB) and %s (%dkB)', o,osize/1024,n,nsize/1024) # p = 'PATCH/'+next(a_numb_patch) tim = -time.time() # if DEBUG > 3 : script.md5_check_file(o) # if USE_DELTA_ALGO == 'bsdiff' and osize > ( 1.1 * (MAXMEMORY / 12)) and VERBOSE : logger.debug(' Warning, memory usage by bsdiff on the order of %dMb', (12 * osize / 2**20)) # p, this_delta_outpiped = delta_files__(o, n, p, USE_DELTA_ALGO, outpiped, infifo) #script.write(s) # if DEBUG > 2 and not this_delta_outpiped : script.md5_check_file(n) # tim += time.time() # global bsdiff_time, bsdiff_datasize bsdiff_time += tim bsdiff_datasize += nsize # if infifo: script.write('rm '+p+'\n') else: script.write('rm '+o+' '+p+'\n') ## how did we fare ? deltasize = os.path.getsize(TD+p) if VERBOSE > 1 : logger.debug(' delta is %3.2f%% of %s, speed: %dkB /sec', ( deltasize * 100. / nsize ) , n, (nsize / 1024. / ( tim + 0.001 ))) #save it patch_append(p[6:]) #clean up unlink(TD+o) return this_delta_outpiped def cmp_gz(o,n): "compare gzip files, ignoring header; returns first different byte (+-10), or True if equal" of=open(o,'rb') nf=open(n,'rb') oa=of.read(10) na=nf.read(10) myord = ord if sys.version_info.major == 2 else (lambda x:x) if na[:3] != b'\037\213\010' : logger.warning(' Warning: was not created with gzip: %r',n) nf.close() ; of.close() return 0 if oa[:3] != b'\037\213\010' : logger.warning(' Warning: was not created with gzip: %r',o) nf.close() ; of.close() return 0 oflag=myord(oa[3]) if oflag & 0xf7: logger.warning(' Warning: unsupported .gz flags: %r %r',oct(oflag),o) if oflag & 8 : #skip orig name oa=of.read(1) while oa and myord(oa) != 0: oa=of.read(1) if not oa: logger.warning(' Warning: premature EOF: %r',o) nf.close() ; of.close() return 0 l=10 nflag=myord(na[3]) if nflag & 0xf7: logger.warning(' Warning: unsupported .gz flags: %r %r',oct(nflag),n) if nflag & 8 : #skip orig name na=nf.read(1) #s=na while na and myord(na) != 0: na=nf.read(1) #s+=na if not na: logger.warning(' Warning: premature EOF: %r',n) nf.close() ; of.close() return 0 l+=len(s) #print repr(s) while oa and na: oa=of.read(2) na=nf.read(2) if oa != na: return l l+=2 if oa or na: return l return True def parse_gzip_header(n): " n may be a file name, a function returning a stream, or a stream" myord = ord if sys.version_info.major == 2 else (lambda x:x) if type(n) in string_types: f=open(TD+n,'rb') a=f.read(10) f.close() elif type(n) in (FunctionType,LambdaType): f=n() a=f.read(10) if isinstance(f,PopenPipe): f.close() else: a=n.read(10) if a[:3] != b'\037\213\010' : logger.debug(' Warning: was not created with gzip: %r',n) return flag=myord(a[3]) # mostly ignored :-> orig_name='-n' if flag & 8: orig_name='-N' if flag & 0xf7: logger.debug(' Warning: unsupported .gz flags: %r %r',oct(flag),n) #a[4:8] #mtime ! ignored ! FIXME will be changed... #from deflate.c in gzip source code format=myord(a[8]) FAST=4 SLOW=2 #unfortunately intermediate steps are lost.... pack_level=6 if format == 0 : pass elif format == FAST : pack_level = 1 elif format == SLOW : pack_level = 9 else: logger.debug(' Warning: unsupported compression .gz format: %r %r',oct(format),n) return if myord(a[9]) != 3 : if VERBOSE : logger.debug(' Warning: unknown OS in .gz format: %r %r',oct(myord(a[9])),n) pack_list=[1,2,3,4,5,6,7,8,9] del pack_list[pack_level-1] pack_list.append(pack_level) pack_list.reverse() #print 'format ', repr(format), type(format), format==FAST, pack_list return pack_list,orig_name def delta_gzipped_files(o,n): "delta o and n, replace o with n" assert(o[-3:] == '.gz' and n[-3:] == '.gz') before=cmp_gz(TD+o,TD+n) if before == True: if VERBOSE > 3: logger.debug(' equal but for header: %r',n) return #compare the cost of leaving as is , VS the minimum cost of delta newsize=os.path.getsize(TD+n) if ( newsize - before + 10 ) < 200 : if VERBOSE > 3: logger.debug(' not worthwhile gunzipping: %r',n) return z=parse_gzip_header(n) if z==None: return pack_list,orig_name=z pack_level=pack_list[0] #OK, it seems we can play our trick p='_tmp_' #unzip new file pro=subprocess.Popen(('zcat',TD+n),stdout=open(TD+p+'.new','wb'), cwd=TD) pro.wait() if pro.returncode: raise DebDeltaError('argghh, zcat failed on us') #test our ability of recompressing best_r=0 best_flag=None for i in pack_list: #force -n ... no problem with timestamps gzip_flags="-"+str(i) pro=subprocess.Popen(("gzip",'-c','-n',gzip_flags,TD+p+'.new'),stdout=open(TD+p+'.faked.gz','wb')) pro.wait() if pro.returncode: DebDeltaError('Argh, gzip failed on us') r=cmp_gz(TD+n,TD+p+'.faked.gz') if r > best_r : best_r = r best_flag = gzip_flags if r == True: break if i == pack_level and VERBOSE > 3: logger.debug(' warning: wrong guess to re-gzip to equal file: %r %r %r',gzip_flags,r,n) if r != True: if VERBOSE > 1 : logger.debug(' warning: cannot re-gzip to equal file, best was %d / %d , "%s" : %s ',best_r,newsize,best_flag,n) os.unlink(TD+p+".new") ; os.unlink(TD+p+'.faked.gz') return #actual delta of decompressed files pro=subprocess.Popen(('zcat',TD+o),stdout=open(TD+p+'.old','wb'), cwd=TD) pro.wait() if pro.returncode: raise DebDeltaError('argh, zcat failed on us') script.write("zcat '"+o+"' > "+p+".old ; rm '"+o+"' \n") if VERBOSE > 2 : logger.debug(' '+n[9:]+(' (= to %d%%): ' % (100*before/newsize))) delta_files(p+'.old',p+'.new') script.write("gzip -c -n "+gzip_flags+" < "+p+".new > '"+o+"' ; rm "+p+".new\n") #replace the old file with the best that we can do re-gzipping the new file #this is important in the rest of the delta-ing process os.rename(TD+p+'.faked.gz',TD+o) if DEBUG > 1 : script.md5_check_file(o, compute_md5(TD+o)) def guess_gz_command(o, check=True): """tries to guess the parameters used to compress, returns a command vector if it fails, returns False o may be a file name, a function returning a stream, or a stream """ h=parse_gzip_header(o) if h==None: return False if type(o) in (FunctionType,LambdaType): o=o() elif type(o) in string_types: o=open(o,'rb') z=tempfile.NamedTemporaryFile(suffix='.gz',delete=False) shutil.copyfileobj(o,z) z.flush() if isinstance(o,PopenPipe): o.close() pack_list,orig_name=h pack_list.reverse() cmd_list=[(['gzip','-nc'],l) for l in pack_list] cmd_list.append(([minigzip],9)) ##old method up to dpkg-deb in 2014-01-15 wcmd=False #fixme, there is no way to distinguish output of minigzip or gzip from header if check: redo=True while redo and cmd_list: cmd,par=cmd_list.pop() wcmd=copy(cmd) wcmd.append('-'+str(par)) if VERBOSE > 2 : logger.debug(' Testing command %r',wcmd) w=PopenPipe(['zcat',z.name,'|']+wcmd+['|','cmp','-',z.name],\ stdout=open(os.devnull,'w'),stderr=open(os.devnull,'wb'),close_fds=True) redo=False w.wait() if w.returncode : redo=True if VERBOSE or DEBUG : logger.debug(' Tried gzip options but failed: %r',wcmd) elif VERBOSE > 2 : logger.debug(' Success with command %r',wcmd) if redo and not cmd_list: logger.debug(' NO MORE OPTIONS !') os.unlink(z.name) return False os.unlink(z.name) return wcmd def guess_xz_parameters(o, check=True): """tries to guess the parameters used to compress, returns a a string of options if it fails, returns False o may be a file name, a function returning a stream, or a stream """ par='' crc='' thread='' if type(o) in (FunctionType,LambdaType): o=o() elif type(o) in string_types: o=open(o,'rb') z=tempfile.NamedTemporaryFile(suffix='.xz',delete=False) #unfortunately 'xz --list' does not work on pipes! shutil.copyfileobj(o,z) z.flush() if isinstance(o,PopenPipe): o.close() b=subprocess.Popen(['xz','-vv','--robot','--list',z.name],stdout=subprocess.PIPE, **Popen_args) for a in b.stdout: a=a.rstrip('\n') a=str.split(a,'\t') if a[0]=='block': if crc and crc != a[9]: logger.warning(" warning : this xz -- compressed file was compressed with variable blocks crc ?! '%s' != '%s'",crc,a[9]) crc=a[9] if par and par != a[15]: logger.warning(" warning : this xz -- compressed file was compressed with variable blocks options ?! '%s' != '%s'",par,a[15]) par=a[15] if not thread and a[12]=='cu': thread='-T2' #print ' guessed par crc ',par,crc if crc: crc=crc.lower() if crc=='sha-256': crc='sha256' if crc not in ('crc32','crc64','sha256'): logger.warning(' Unknown XZ crc %r',crc) crc='' PARS=['-6e','-9','-9e'] if par: PARS.append(par) if par == '--lzma2=dict=1MiB': # dbgsym deb files are compressed with -1e PARS.append('-1') PARS.append('-1e') if check: redo=True while redo and PARS: par=PARS.pop() w=['xz','-c'] if par: w.append(par) if crc: w+=['-C',crc] if thread: w.append(thread) w.append('-') if VERBOSE > 2 : logger.debug(' Testing XZ options %r',w) redo=False c=PopenPipe(['unxz','-c',z.name,'|']+w+['|','cmp','-',z.name],\ stdout=open(os.devnull,'wb'),stderr=open(os.devnull,'wb'),close_fds=True) c.wait() if c.returncode : redo=True if VERBOSE or DEBUG : logger.debug(' Tried XZ options but failed: %r %r %r',par,crc,thread) if redo and not PARS: #print ' HO FINITO LE OPZIONI !' os.unlink(z.name) return False if crc: crc=' -C '+crc if thread: thread=' '+thread os.unlink(z.name) return par+crc+thread ########### helper sh functions for script, for delta_tar() import difflib, re re_numbers=re.compile(r'^[0-9][0-9]*$') def file_similarity_premangle(fp): fps=fp.split('/') bns=fps[-1].split('.') j=len(bns)-1 #search first "non numeric" extension, and put it last while j >= 0 and re_numbers.match(bns[j]): j-=1 if j >= 0: a=bns.pop(j) r = fps[:-1] + bns + [a,] else: r = fps[:-1] + bns return r def files_similarity_score__noext__(oo,nn): ln=len(nn) lo=len(oo) l=0 while oo and nn: while oo and nn and oo[-1] == nn[-1]: oo=oo[:-1] nn=nn[:-1] if not oo or not nn: break while oo and nn and oo[0] == nn[0]: oo=oo[1:] nn=nn[1:] if not oo or not nn: break if len(nn) > 1 and oo[0] == nn[1]: l+=1 nn=nn[1:] if len(oo) > 1 and oo[1] == nn[0]: l+=1 oo=oo[1:] if not oo or not nn: break if oo[-1] != nn[-1]: oo=oo[:-1] nn=nn[:-1] l+=2 if not oo or not nn: break if oo[0] != nn[0]: oo=oo[1:] nn=nn[1:] l+=2 return (l +len(oo) + len(nn)) * 2.0 / float(ln+lo) def files_similarity_score__(oo,nn): oo=copy(oo) nn=copy(nn) if oo.pop() != nn.pop() : return 0.2 + files_similarity_score__noext__(oo,nn) else: return files_similarity_score__noext__(oo,nn) def files_similarity_score__difflib__(oo,nn): "compute similarity by difflib. Too slow." if oo == nn : return 0 d=difflib.context_diff(oo,nn,'','','','',0,'') d=[a for a in tuple(d) if a and a[:3] != '---' and a[:3] != '***' ] if oo[-1] != nn[-1] : #penalty for wrong extension return 0.2+float(len(d)) * 2.0 / float(len(oo)+len(nn)) else: return float(len(d)) * 2.0 / float(len(oo)+len(nn)) def files_similarity_score(oo,nn): if oo == nn : return 0 if type(oo) in string_types: oo=file_similarity_premangle(oo) if type(nn) in string_types: nn=file_similarity_premangle(nn) return files_similarity_score__(oo,nn) def fake_tar_header_2nd(): " returns the second part of a tar header , for regular files and dirs" # The following code was contributed by Detlef Lannert. # into /usr/lib/python2.3/tarfile.py MAGIC = b"ustar" # magic tar string VERSION = b"00" # version number NUL = b"\0" # the null character parts = [] for value, fieldsize in ( (b"", 100), # unfortunately this is not what DPKG does #(MAGIC, 6), #(VERSION, 2), # this is what DPKG does (b'ustar \x00',8), (b"root", 32), (b"root", 32), (b"%07o" % 0, 8), (b"%07o" % 0, 8), (b"", 155) ): l = len(value) parts.append(value + (fieldsize - l) * NUL) buf = b"".join(parts) return buf fake_tar_2nd=fake_tar_header_2nd() fake_tar_2nd_echo=prepare_for_echo(fake_tar_2nd) script.write("FTH='"+fake_tar_2nd_echo+"'\n") script.write("E='echo -ne'\n") global time_corr time_corr=0 #################### vvv delta_tar vvv ########################### def delta_tar(old_filename, new_filename, CWD, old_forensic,\ skip=[], old_md5={}, new_md5={},\ chunked_p=(not delta_uses_infifo) ,debdelta_conf_skip=()): " compute delta of two tar files, and prepare the script consequently" assert( type(old_filename) in string_types or type(old_filename) == FunctionType ) script.write('ECR () { $E "$1" ; $E "${FTH}" ; cat OLD/'+CWD+'/"$1" ; rm OLD/'+CWD+'/"$1" ;}\n') script.write('EC () { $E "$1" ; $E "${FTH}" ; cat OLD/'+CWD+'/"$1" ;}\n') ###### uncompress and scan the old tar file, extract regular files if type(old_filename) in string_types : (old_filename,old_filename_ext) = unzip(old_filename) oldtar = tarfile.open(TD+old_filename, "r") elif type(old_filename) in (FunctionType,LambdaType) : old_filename_ext=None oldfileobj = old_filename() oldtar = tarfile.open(mode="r|", fileobj=oldfileobj) else: old_filename_ext=None oldtar = tarfile.open(mode="r|", fileobj=old_filename) oldnames = [] oldtarinfos = {} for oldtarinfo in oldtar: oldname = de_bar(oldtarinfo.name) if old_forensic != None: #fixme : devices are not supported (but debian policy does not allow them) old_forensic.append([oldtarinfo.name,oldtarinfo.mode,oldtarinfo.type,\ oldtarinfo.uid,oldtarinfo.gid,oldtarinfo.uname,oldtarinfo.gname,oldtarinfo.linkname]) #this always happens #if VERBOSE > 3 and oldname != de_bar(oldname): # print ' filename in old tar has weird ./ in front: ' , oldname if not oldtarinfo.isreg(): if VERBOSE > 2 : logger.debug(' skipping old non-regular %r',oldname) continue if oldtarinfo.size == 0: if VERBOSE > 2 : logger.debug(' skipping old empty %r',oldname) continue if shell_not_allowed(oldname): if VERBOSE > 2 : logger.debug(' skipping non-allowed-name %r',oldname) continue for j in debdelta_conf_skip: if fnmatch.fnmatch(oldname,j): if VERBOSE > 2 : logger.debug(' skipping following as per rule %r',j) skip.append(oldname) break if oldname in skip: if VERBOSE > 2 : logger.debug(' skipping %r',oldname) if old_forensic != None: oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) old_forensic.append(old_forensic.pop()[:-1] + \ [sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname))]) continue oldnames.append(oldname) oldtarinfos[oldname] = oldtarinfo oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) if old_forensic != None: old_forensic.append(old_forensic.pop()[:-1] + \ [sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname))]) oldtar.close() if type(old_filename) in string_types : unlink(TD+old_filename) else: while oldfileobj.read(512): pass ###### scan the new tarfile, save info regarding regular files therein #save header part of new_filename, since it changes in newer versions f=open(TD+new_filename,'rb') new_file_zip_head=f.read(20) f.close() (new_filename,new_filename_ext) = unzip(new_filename) new_file_md5=compute_md5(TD+new_filename) new_file_size=os.path.getsize(TD+new_filename) info_append(new_filename+': '+new_file_md5+' '+str(new_file_size)) #### scan the new tarfile, compare to the old tar contents assert(0 == (new_file_size % 512)) newtar = tarfile.open(TD+new_filename, "r") # dpkg lately uses PAX format, dunno the difference, debdelta seems to work fine logger.log(logging.DEBUG if newtar.format in (tarfile.GNU_FORMAT , tarfile.PAX_FORMAT) else logging.WARNING, ' tar file format %r for %r'%(newtar.format,new_filename)) newnames = [] newtarinfos = {} for newtarinfo in newtar: newname = newtarinfo.name #just curious to know t=newtarinfo.type a=newtarinfo.mode if VERBOSE and (( t == '2' and a != 0o777 ) or \ ( t == '0' and ( (a & 0o400 ) == 0 )) or \ ( t == '5' and ( (a & 0o500 ) == 0 ))): logger.debug(' weird permission %r %r %r', newname, oct(a), repr(newtarinfo.type)) ### if not newtarinfo.isreg(): continue if VERBOSE > 3 and newname != de_bar(newname): logger.debug(' filename in new tar has weird ./ in front: %r' , newname) newname = de_bar(newname) newnames.append(newname) newtarinfos[newname] = newtarinfo old_used={} correspondence={} ###### find correspondences between old tar and new tar contents global time_corr time_corr=-time.time() if VERBOSE > 2 : logger.debug(' finding correspondences for %r', new_filename) reverse_old_md5={} if old_md5: for o in old_md5: if o in oldnames: reverse_old_md5[old_md5[o]] = o else: #would you believe? many packages contain MD5 for files they do not ship... if VERBOSE > 1 and o not in skip: logger.debug(' hmmm... there is a md5 but not a file: %r',o) #database of databases of premangled old names , by "extension" and name oldnames_premangle={} for o in oldnames: om=file_similarity_premangle(o) a=om[-1] # "extension" if a not in oldnames_premangle: oldnames_premangle[a]={} oldnames_premangle[a][o]=om for newname in newnames: newtarinfo=newtarinfos[newname] oldname=None #ignore empty files if newtarinfo.size == 0: continue #try correspondence by MD5 if new_md5 and newname in new_md5: md5=new_md5[newname] if md5 in reverse_old_md5: oldname=reverse_old_md5[md5] if VERBOSE > 2 : if oldname == newname : logger.debug(' use identical old file: %r',newname) else: logger.debug(' use identical old file: %r %r',oldname, newname) #try correspondence by file name if oldname == None and newname in oldnames: oldname=newname if VERBOSE > 2 : logger.debug(' use same name old file: %r',newname) #try correspondence by file name and len similarity np=file_similarity_premangle(newname) ne=np[-1] # "extension" if oldname == None and ne in oldnames_premangle: basescore=1.6 nl=newtarinfo.size for o in oldnames_premangle[ne]: op=oldnames_premangle[ne][o] l=oldtarinfos[o].size sfile=files_similarity_score__noext__(op,np) slen = abs(float(l - nl))/float(l+nl) s=slen+sfile if VERBOSE > 3 : logger.debug(' name/len diff %.2f+%.2f=%.2f %r',slen,sfile,s, o) if s < basescore: oldname=o basescore=s if oldname and VERBOSE > 2 : logger.debug(' best similar %.3f %r %r',basescore,newname,oldname) if not oldname: if VERBOSE > 2 : logger.debug(' no correspondence for: %r',newname) continue #we have correspondence, lets store if oldname not in old_used: old_used[oldname]=[] old_used[oldname].append(newname) correspondence[newname]=oldname time_corr+=time.time() if VERBOSE > 1 : logger.debug(' time lost so far in finding correspondence %.2f',time_corr) ######### final pass : scan new tar, extract regular files, prepare deltas if VERBOSE > 2 : logger.debug(' scanning %r',new_filename) script.start_rebuilding() current_chunk_name=next(a_numb_file) script.start_chunk(current_chunk_name) mega_cat=open(TD+'/'+current_chunk_name,'wb') #helper function def _append_(p, w, rm): mega_cat.write(str2bytes(w)+fake_tar_2nd) f=open(TD+p+'/'+w,'rb') a=f.read(1024) while a: try: mega_cat.write(a) except OSError: s=sys.exc_info()[1] raise DebDeltaError(' OSError (at _a_) while writing: '+str(s), True) a=f.read(1024) f.close() if rm: script.write("ECR '"+w+"'\n") unlink(TD+p+'/'+w) else: script.write("EC '"+w+"'\n") global something_backgrounded #FIXME I hate using globals for this :-( something_backgrounded=False #helper function def mega_cat_chunk(oldoffset,newoffset, background=True): global something_backgrounded p = next(a_numb_file) f=open(TD+new_filename,'rb') f.seek(oldoffset) of=open(TD+p,'wb') l=oldoffset while l 2 : logger.debug(' provide also old file %r', oldname) yield oldname while 1: yield None some_old_file=some_old_file_gen() one_old_file=next(some_old_file) max_chunk_size = MAXMEMORY / 12 chunk_discount = 0.3 progressive_new_offset=0 for newtarinfo in newtar: #progressive mega_cat a=mega_cat.tell() if chunked_p and ((a >= max_chunk_size * chunk_discount) or \ (a >= max_chunk_size * chunk_discount * 0.9 and one_old_file ) or \ (a>0 and (a+newtarinfo.size) >= max_chunk_size * chunk_discount )): #provide some old unused files, if any while one_old_file: _append_("OLD/"+CWD, one_old_file, False) if mega_cat.tell() >= max_chunk_size * chunk_discount : break one_old_file=next(some_old_file) #write the chunk into a temporary mega_cat.close() script.end_chunk(current_chunk_name) #delta the chunk mega_cat_chunk(progressive_new_offset, newtarinfo.offset ) #start a new chunk current_chunk_name=next(a_numb_file) script.start_chunk(current_chunk_name) mega_cat=open(TD+'/'+current_chunk_name,'wb') # progressive_new_offset=newtarinfo.offset chunk_discount = min( 1. , chunk_discount * 1.2 ) # name = de_bar( newtarinfo.name ) if newtarinfo.isdir(): #recreate also parts of the tar headers mega_cat.write(str2bytes(newtarinfo.name)+fake_tar_2nd) script.write("$E '"+prepare_for_echo(str2bytes(newtarinfo.name))+"'\"${FTH}\"\n") if VERBOSE > 2 : logger.debug(' directory in new : %r', name) continue if not newtarinfo.isreg(): #recreate also parts of the tar headers mega_cat.write(str2bytes(newtarinfo.name)+fake_tar_2nd) script.write("$E '"+prepare_for_echo(newtarinfo.name)+"'\"${FTH}\"\n") if VERBOSE > 2 : logger.debug(' not regular in new : %r', name) continue if newtarinfo.size == 0: #recreate also parts of the tar headers mega_cat.write(str2bytes(newtarinfo.name)+fake_tar_2nd) script.write("$E '"+prepare_for_echo(newtarinfo.name)+"'\"${FTH}\"\n") if VERBOSE > 2 : logger.debug(' empty new file : %r', name) continue if name not in correspondence: #recreate also parts of the tar headers mega_cat.write(str2bytes(newtarinfo.name)+fake_tar_2nd) script.write("$E '"+prepare_for_echo(newtarinfo.name)+"'\"${FTH}\"\n") if VERBOSE > 2: logger.debug(' no corresponding fil: %r', name) continue oldname = correspondence[name] mul=len( old_used[oldname]) > 1 #multiple usage if not mul and oldname == name and oldname[-3:] == '.gz' and \ newtarinfo.size > 120 and \ not ( new_md5 and name in new_md5 and old_md5 and name in old_md5 and \ new_md5[name] == old_md5[name]): newtar.extract(newtarinfo,TD+"NEW/"+CWD ) delta_gzipped_files("OLD/"+CWD+'/'+name,"NEW/"+CWD+'/'+name) if VERBOSE > 2 : logger.debug(' adding reg file: %r %r', oldname, mul and '(multiple)' or '') _append_("OLD/"+CWD, oldname , not mul ) old_used[oldname].pop() #end of for loop #write the chunk into a temporary mega_cat.close() script.end_chunk(current_chunk_name) if os.path.exists(TD+'/OLD/'+CWD): rmtree(TD+'/OLD/'+CWD) if os.path.getsize(TD+'/'+current_chunk_name) > 0 : mega_cat_chunk(progressive_new_offset, os.path.getsize(TD+new_filename), background=False) else: #the (tail of the) new tar did not match anything in the old tar, nothing to delta p=verbatim(new_filename) script.write("cat '"+p+"'\n") script.end_rebuilding() #pipes the rebuilding part into the recompressing part if new_filename == 'NEW/data.tar': script.recompressing_by_arg(new_filename,new_filename_ext,new_file_zip_head,new_file_size) else: script.recompressing(new_filename,new_filename_ext,new_file_zip_head) #################### ^^^^ delta_tar ^^^^ ########################### ############ start computing deltas def append_NEW_file(s): 'appends some data to NEW.file' s=prepare_for_echo(s) script.write("$E '"+ s +"' >> NEW.file\n") #this following is actually #def delta_debs_using_old(old,new): ### start scanning the new deb newdeb_file=open(newdeb,'rb') # pop the "!\n" s = newdeb_file.readline() assert( b"!\n" == s) append_NEW_file(s) if forensic_file==None: control_forensic=None data_forensic=None else: control_forensic=[] data_forensic=[] def decompress_pipe_from_ar(oldnew, name, p=None): " use p=False to not decompress" assert oldnew in ('OLD','NEW') if p==None: p=os.path.splitext(name)[1] db={b'.gz' : ['gzip','-cd'], b'.bz2': ['bzip2','-cd'], b'.lzma':['unlzma','-c'], b'.xz': ['unxz','-c'] } assert p==False or p == b'.tar' or p in db if p==b'.tar' or p==False: def x(): return PopenPipe(['ar','p', oldnew+'.file', name] , stdout=subprocess.PIPE, cwd=TD) else: def x(): return PopenPipe(['ar','p', oldnew+'.file', name,'|']+db[p] , stdout=subprocess.PIPE, cwd=TD) return x for name in ar_list_new : newname = 'NEW/'+bytes2str(name) system(('ar','x',TD+'NEW.file',name), TD+'/NEW/') script.xz_parameters=None newsize = os.stat(TD+newname)[ST_SIZE] if VERBOSE > 1: logger.debug(' studying %r of len %dkB' , name, (newsize/1024)) #add 'ar' structure ar_line = newdeb_file.read(60) if VERBOSE > 3: logger.debug(' ar line: %r',ar_line) assert( ar_line[:len(name)] == name and ar_line[-2:] == b'`\n' ) #sometimes there is an extra \n, depending if the previous was odd length newdeb_file.seek(newsize ,1) if newsize & 1 : extrachar = newdeb_file.read(1) else: extrachar = '' #add file to delta if newsize < 128: #file is too short to compute a delta, append_NEW_file(ar_line) p=open(TD+newname,'rb') append_NEW_file( p.read(newsize)) p.close() unlink(TD+newname) #pad new deb if extrachar : append_NEW_file(extrachar) elif not NEEDSOLD and ( name[:11] == b'control.tar' or name[:8] == b'data.tar'): script.start_member(ar_line, newname, extrachar) basename,ext=os.path.splitext(name) xd=decompress_pipe_from_ar('NEW',name,False) #delta it if b'.lzma' == ext : info_append('needs-lzma') elif b'.xz' == ext : info_append('needs-xz') script.xz_parameters=guess_xz_parameters(xd) if script.xz_parameters==False: raise DebDeltaError('Cannot guess XZ parameters for new %r' % name) elif b'.gz' == ext : script.gz_command=guess_gz_command(xd) if script.gz_command==False: raise DebDeltaError('Cannot guess GZ parameters for new %r' % name) del xd x=None for oldext in (b'', b'.gz', b'.bz2', b'.lzma', b'.xz'): if basename+oldext in ar_list_old: x=decompress_pipe_from_ar('OLD',basename+oldext) break assert x if name[:11] == b'control.tar': skip=[] ##avoid using strange files that dpkg may not install in /var...info/ for a in os.listdir(TD+'OLD/CONTROL') : if a not in dpkg_keeps_controls: skip.append(a) delta_tar(x,newname,'CONTROL',control_forensic,skip) else: delta_tar(x,newname,'DATA',data_forensic,old_conffiles,old_md5,new_md5,\ debdelta_conf_skip=debdelta_conf_skip) del x, basename, oldext script.end_member() if DEBUG>3 and name[:11] == b'control.tar': length=newdeb_file.tell() md5_len=compute_md5_up_to_len(newdeb, length) assert md5_len[1] == length script.md5_check_file('NEW.file', md5_len[0]) elif not NEEDSOLD or name not in ar_list_old : #or it is not in old deb append_NEW_file(ar_line) patchname=verbatim(newname) script.write('cat '+patchname+' >> NEW.file ; rm '+patchname+'\n') #pad new deb if extrachar : append_NEW_file(extrachar) elif NEEDSOLD : append_NEW_file(ar_line) #file is long, and has old version ; lets compute a delta oldname = 'OLD/'+name system(('ar','x',TD+'OLD.file',name), TD+'OLD') script.write('ar p OLD.file '+name+' >> '+oldname+'\n') (oldname,co) = unzip(oldname) (newname,cn) = unzip(newname) delta_files(oldname,newname) script.zip(newname,cn) script.write('cat '+newname+cn+' >> NEW.file ; rm '+newname+cn+'\n') unlink(TD+newname) #pad new deb if extrachar : append_NEW_file(extrachar) del co,cn else: die('internal error j98') # put in script any leftover s = newdeb_file.read() if s: if VERBOSE > 2: logger.debug(' ar leftover character: %r',s) append_NEW_file(s) #this is done already from the receiving end if DEBUG > 2 and newdeb_md5sum : script.md5_check_file("NEW.file",md5=newdeb_md5sum) #script is done script.close() if forensic_file: forensics_rfc(forensic_file,info,True,control_forensic,data_forensic,old_conffiles) patchsize = os.stat(TD+'PATCH/patch.sh')[ST_SIZE] patch_files = [] if 'lzma' not in DISABLED_FEATURES and os.path.exists('/usr/bin/lzma'): system(('lzma','-q','-9','-k','PATCH/patch.sh'), TD) patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.lzma'), 'lzma', 'patch.sh.lzma')) if 'xz' not in DISABLED_FEATURES and os.path.exists('/usr/bin/xz'): system(('xz','-q','-9','-k','PATCH/patch.sh'), TD) patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.xz'), 'xz', 'patch.sh.xz')) system(('bzip2','-q','--keep','-9','PATCH/patch.sh'), TD) patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.bz2'), 'bzip2', 'patch.sh.bz2')) system(('gzip','-q','-9','-n','PATCH/patch.sh'), TD) patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.gz'), 'gzip', 'patch.sh.gz')) # Use the smallest compressed patch.sh patch_files.sort() if VERBOSE > 1 : logger.debug(' '+patch_files[0][1]+' wins on patch.sh') if patch_files[0][1] == 'lzma': info_append('needs-lzma') if patch_files[0][1] == 'xz': info_append('needs-xz') patch_append(patch_files[0][2]) del patch_files #OK, OK... this is not yet correct, since I will add the info file later on elaps = time.time() - start_sec info.append('DeltaTime: %.2f' % elaps) deltasize = os.stat(delta)[ST_SIZE] + 60 + sum(map(len,info)) percent = deltasize * 100. / newdebsize info.append('Ratio: %.4f' % (float(deltasize) / float(newdebsize)) ) if VERBOSE: #note that sizes are written as kB but are actually kibybytes, that is 1024 bytes logger.info(' ' + _('delta is %(perc)3.1f%% of deb; that is, %(save)dkB are saved, on a total of %(tot)dkB.'), {'perc' : percent , 'save' : (( newdebsize -deltasize ) / 1024), 'tot' : ( newdebsize/ 1024)}) logger.info(' ' + _('delta time %(time).2f sec, speed %(speed)dkB /sec, (%(algo)s time %(algotime).2fsec speed %(algospeed)dkB /sec) (corr %(corrtime).2f sec)') % \ {'time' : elaps, 'speed' : newdebsize / 1024. / (elaps+0.001), 'algo' : USE_DELTA_ALGO, 'algotime' : bsdiff_time, 'algospeed' : bsdiff_datasize / 1024. / (bsdiff_time + 0.001) , 'corrtime' : time_corr }) return (delta, percent, elaps, info, gpg_hashes) ##################################################### compute many deltas def info_by_pack_arch_add(f, info_by_pack_arch): pack = f['Package'] arch = f['Architecture'] if (pack,arch) not in info_by_pack_arch : info_by_pack_arch[ (pack,arch) ]=[] info_by_pack_arch[ (pack,arch) ].append( f ) def iterate_Packages(packages, use_debian_822=True): fields=('Package','Architecture','Version','Filename') for f in fields: intern(f) packages=abspath(packages) assert os.path.isfile(packages) assert os.path.basename(packages) in ('Packages', 'Packages.gz','Packages.bz2','Packages.xz') dir=os.path.dirname(packages) dir=dir.split('/') try: a=dir.index('dists') except ValueError: logger.error('Error: pathname "%s" does not contain "dists"\n' % packages) return base = str.join('/',dir[:a]) # cache=cache_same_dict(packages, fields) if DO_CACHE and cache.exists: for i in cache: i['Basepath'] = base yield i if not cache.broken: return # if packages[-3:] == '.gz': import gzip F=gzip.GzipFile(packages) SP=None elif packages[-4:] == '.bz2': import bz2 F=bz2.BZ2File(packages) SP=None elif packages[-3:] == '.xz': SP=subprocess.Popen(["xzcat",packages], stdout=subprocess.PIPE, stdin=open(os.devnull), close_fds=True) F=SP.stdout else: F=open(packages,'rb') SP=None # if debian_deb822 and use_debian_822: #use faster implementation #P=debian_deb822.Packages(F,fields=fields) for a in debian_deb822.Packages.iter_paragraphs(sequence=F,shared_storage=False,fields=fields): if DO_CACHE and not cache.exists: cache.write(a) a['Basepath'] = base yield a if SP: F.read() SP.wait() return # of,pack,vers,arch=None,None,None,None for l in F: l = bytes2str(l) l=l.rstrip('\n') if l[:9] == 'Package: ': pack=l[9:] elif l[:14] == 'Architecture: ': arch = l[14:] elif l[:9] == 'Version: ': vers = l[9:] elif l[:10] == 'Filename: ': of=l[10:] elif l == '': if of == None or pack == None or vers == None or arch == None: logger.warning('Warning, skipping incomplete record in index: %r %r %r %r',of,pack,vers,arch) continue if of[-4:] == '.udeb': if VERBOSE > 2 : logger.debug(' skip udeb') continue a={} a['Filename'] = of a['Package'] = pack a['Architecture'] = arch a['Version'] = vers if DO_CACHE and not cache.exists: cache.write(a) a['Basepath'] = base yield a of,pack,vers,arch=None,None,None,None if SP: F.read() SP.wait() def scan_deb_dir(d, debname, label, lazy, info_by_pack_arch, info_by_file): assert (debname == None or type(debname) in string_types) and type(label) in string_types if not os.path.isdir(d): logger.debug('Error, skip non dir: %r',d) return if lazy: scan=scan_deb_byfile_lazy else: scan=scan_deb_byfile for n in os.listdir(d): if n[-4:] != '.deb': continue if debname != None and debname != n.split('_')[0]: continue if not check_deb(os.path.join(d,n), die_ = False): logger.warning('Is not a deb file: %r',n) continue a=scan(os.path.join(d,n), info_by_file) a['Label'] = label info_by_pack_arch_add(a, info_by_pack_arch) def scan_deb_byfile_lazy(f, info_by_file): n=os.path.basename(f).split('_') a={} a['File']=f a['Package']=n[0] #version cannot be trusted, due to epochs (deleted in filenames) #it is read using scan_deb_bydict, later on a['Architecture']=n[2][:-4] return a def scan_deb_byfile(f, info_by_file): #DEBUG: assert( os.path.isfile(f) ) if f in info_by_file and 'Version' in info_by_file[f]: #already scanned in non-lazy mode return info_by_file[f] a={} a['File']=f c,b,z = untar_control_in_deb( list_ar(f) ) p=PopenPipe(['ar','p',f,c,'|','tar',b,'-xp','-f','-','-O','./control'], stdout = subprocess.PIPE, ** Popen_args) scan_control(p.stdout, params=a) p.close() return a def scan_deb_bydict(a): c,b,z = untar_control_in_deb( list_ar(a['File']) ) p=PopenPipe(['ar','p',a['File'],c,'|','tar',b,'-xp','-f','-','-O','./control'], stdout = subprocess.PIPE) scan_control(p.stdout, params=a) p.close() return a def scan_delta_dir(d, debname, scanned_delta_dirs, old_deltas_by_pack_arch): if (d,debname) in scanned_delta_dirs or (d,None) in scanned_delta_dirs : return if not os.path.isdir(d) : if VERBOSE > 2 and DEBUG : logger.debug(' No such delta dir: %r',d) scanned_delta_dirs.add((d,None)) #trick, if aint there no need to retry return assert debname == None or type(debname) in string_types scanned_delta_dirs.add((d,debname)) for n in os.listdir(d): if debname != None and debname != n.split('_')[0]: continue scan_delta( os.path.join(d,n) , old_deltas_by_pack_arch) def scan_delta(f, old_deltas_by_pack_arch): assert( os.path.isfile(f) ) if f[-9:] == '.debdelta' : a=f[:-9] elif f[-17:] == '.debdelta-too-big' : a=f[:-17] elif f[-15:] == '.debdelta-fails' : a=f[:-15] else: return a=os.path.basename(a) a=a.split('_') pa=a[0] ar=a[3] if (pa,ar) not in old_deltas_by_pack_arch: old_deltas_by_pack_arch[ (pa,ar) ]=[] ov=version_demangle(a[1]) nv=version_demangle(a[2]) if (f,ov,nv) not in old_deltas_by_pack_arch[ (pa,ar) ]: old_deltas_by_pack_arch[ (pa,ar) ].append( (f, ov, nv ) ) def delta_dirname(f,altdir): "f=directory, altdir=ALT or DIR with // convention. Returns augmented dirname" if altdir != None: if altdir[-2:] == '//' : if '../' in f: logger.warning('Warning: cannot combine .. and // ! Saving in %r', altdir) return altdir #os.path.join has a weird behaviour with absolute paths! f=f.lstrip('/') return os.path.join(altdir[:-2], f) else: return altdir else: #this happens when DIR is not set, and the delta goes in the same directory as the deb return f def go_fishing(deb,others, loglevel=logging.ERROR): "Find a package filepath if missing, save it in deb['File']. deb = dict representing package ; others: list of such dicts." if 'File' in deb: assert os.path.isfile(deb['File']) return deb['File'] of=deb['Filename'] ob=os.path.basename(of) #try to build it from its own info if 'Basepath' in deb: f=os.path.join(deb['Basepath'],of) if os.path.exists(f): deb['File']=f if DEBUG > 1 : logger.debug('Fish! %r',deb) return f #try to build it from others info others=[a for a in others if id(a) != id(deb) ] for new in others: if 'File' in new: f=new['File'] if os.path.basename(f) == ob: deb['File']=f if DEBUG > 1 : logger.debug('Fish! %r %r',deb,new) return f if 'Basepath' in new: for a in of,ob: f=os.path.join(new['Basepath'],a) if os.path.exists(f): deb['File']=f if DEBUG > 1 : logger.debug('Fish! %r %r',deb,new) return f logger.log(loglevel, 'bad fishing for %r %r', deb, others) return False def order_by_version(a, b, VersionCompare): # lazy packages do not have versions; but are always ALT, so we sort them at bottom if 'Version' not in a: return -1 if 'Version' not in b: return 1 return VersionCompare( a['Version'] , b['Version'] ) def iter_deltas_one_pack_arch(pa,ar,info_pack,thedir,theforensicdir,VersionCompare, loglevel=logging.ERROR): " iterate deltas to be created for package pa, architecture ar, by analyzing info_pack" if all([('CMDLINE' != a['Label']) for a in info_pack ]): #this happens a lot, e.g. when we are scanning non-free/Packages.gz, # all free packages in the trash are to be ignored if DEBUG > 2 or VERBOSE > 3 : logger.debug(' No cmdline for: %r %r',pa,ar) return #do we need the versions of --alt packages ? seems not # for a in info_pack: # #scan all lazy packages # if 'Version' not in a: # assert a['Label'] == 'ALT' # scan_deb_bydict(a) import functools mykey = functools.cmp_to_key(lambda x,y : order_by_version(x, y, VersionCompare)) info_pack.sort(key = mykey) how_many= len(info_pack) if how_many <= 1 : if VERBOSE > 3 : logger.debug(' Only one version: %r %r', pa,ar) return newest = how_many -1 while newest >= 0 : new=info_pack[newest] if new['Label'] != 'CMDLINE' : if VERBOSE > 3 : logger.debug(' Newest version deb was not in cmdline, skip down one: %r', new) else: break newest -= 1 if newest <= 0 : if VERBOSE > 3 : logger.debug(' No older versions: %r', new) return if not go_fishing(new,[], loglevel): logger.log(loglevel, 'Cannot locate new file: %r',new) return if VERBOSE > 2 : logger.debug(' Package: %r %r Versions: %r', pa, ar, [ (o.get('Version'),o['Label'],o.get('Packages'),o.get('File')) for o in info_pack]) newdebsize=os.path.getsize(new['File']) #very small packages cannot be effectively delta-ed if newdebsize <= MIN_DEB_SIZE : if VERBOSE > 1 : logger.debug(' Skip , too small: %r', new['File']) return oldn = newest generated=0 seen_versions=[] while (oldn>0) : oldn -= 1 old=info_pack[oldn] if old['Label'] != 'OLD': if VERBOSE > 2 : logger.debug(' Not old, skip: %r',old) continue if old['Version'] == new['Version'] : if VERBOSE > 3 and old != new : logger.debug(' Identical versions: %r %r', old, new) continue assert( old['Package'] == pa and pa == new['Package'] ) deltabasename = delta_base_name(pa, old['Version'], new['Version'], ar) if 'Filename' in new: deltadirname=delta_dirname(os.path.dirname(new['Filename']),thedir) elif 'File' in new: deltadirname=delta_dirname(os.path.dirname(new['File']),thedir) else: assert(0) if deltadirname == '' : deltadirname = '.' delta=os.path.join(deltadirname,deltabasename) generated+=1 #count also those already generated if N_DELTAS != None and (generated > N_DELTAS): continue if os.path.exists(delta): if VERBOSE > 1 : logger.debug(' Skip , already exists: %r',delta) continue if old['Package'] in seen_versions: if VERBOSE > 3 : logger.debug(' Skip , already considered: %r',delta) continue if os.path.exists(delta+'-too-big'): if VERBOSE > 1 : logger.debug(' Skip , tried and too big: %r',delta) continue if os.path.exists(delta+'-fails'): if VERBOSE > 1 : logger.debug(' Skip , tried and fails: %r',delta) continue if not go_fishing(old,info_pack, loglevel): logger.log(loglevel, 'Cannot locate old file: %r',old) continue forensicfile=None if theforensicdir: if 'Filename' in new: forensicdirname=delta_dirname(os.path.dirname(new['Filename']),theforensicdir) elif 'File' in new: forensicdirname=delta_dirname(os.path.dirname(new['File']),theforensicdir) else: assert(0) forensicbasename = pa +'_'+ version_mangle(old['Version']) +'_'+ar+'.forensic' forensicfile=os.path.join(forensicdirname,forensicbasename) seen_versions.append(old['Package']) yield (old,new, delta, forensicfile) def do_delta_and_test(old_File, new_File, delta, forensicfile): " returns (error_code, delta_size, delta_elaps, patch_elaps, info_delta, error_string) : for error_code, see EXIT STATUS in 'man debdelta' " exitstatus=0 if VERBOSE : logger.info('Creating: %r',delta) deltatmp=delta+'_tmp_' ret= None tdir=tempo() ret = None # data returned from calls elaps = None # delta time delta_size = None p_elaps = None # patching time info_delta = [] # informations about created delta error_string = '' # error when delta fails deltadirname=os.path.dirname(delta) if not os.path.exists(deltadirname): #FIXME this does not respect --no-act os.makedirs(deltadirname) free=freespace(deltadirname) newdebsize=os.path.getsize(new_File) if free and (free < (newdebsize /2 + 2**15)) : if VERBOSE : logger.warning(' '+(_("Not enough disk space for storing `%s'.")), delta) return (1, None, None, None, [], 'not_enough_disk_space') if forensicfile and not os.path.exists(forensicfile): forensicdirname=os.path.dirname(forensicfile) if not os.path.exists(forensicdirname): #FIXME this does not respect --no-act os.makedirs(forensicdirname) forensic_file=open(forensicfile,'w') else: forensic_file=None try: ret=do_delta_(old_File,new_File, deltatmp, TD=tdir, forensic_file=forensic_file, info=info_delta) (deltatmp_, percent, elaps, info_delta, gpg_hashes) = ret if os.path.exists(deltatmp): delta_size = os.path.getsize(deltatmp) except (KeyboardInterrupt, SystemExit): if os.path.exists(deltatmp): os.unlink(deltatmp) rmtree(tdir) raise except DebDeltaError: s=sys.exc_info()[1] error_string=str(s) if not VERBOSE : logger.info(_('Creating:')+' '+repr(delta)) logger.info(' Creation of delta failed, reason: %r', error_string) if os.path.exists(deltatmp): if KEEP: logger.info(' '+_('You may want to examine:')+' '+str(deltatmp)) else: os.unlink(deltatmp) if not s.retriable : open(delta+'-fails','w').close() exitstatus=max(exitstatus, s.exitcode) ret = None except: exitstatus=4 if os.path.exists(deltatmp): os.unlink(deltatmp) puke( " *** Error while creating delta "+delta) open(delta+'-fails','w').close() ret = None rmtree(tdir) if ret == None: return (exitstatus, delta_size, elaps, p_elaps, info_delta, error_string) info_delta.append('ServerID: '+HOSTID) info_delta.append('ServerBogomips: '+str(BOGOMIPS)) if MAX_DELTA_PERCENT and percent > MAX_DELTA_PERCENT: os.unlink(deltatmp) if VERBOSE : logger.info(' Warning, too big!') open(delta+'-too-big','w').close() return (exitstatus, delta_size, elaps, p_elaps, info_delta, 'too_big') # if DO_TEST : ##patch test pret=None try: #test, ignoring gpg, that is added later on pret=do_patch(deltatmp,old_File,None , info=info_delta, do_gpg=None) except DebDeltaError: s=sys.exc_info()[1] logger.warning(' '+_('Error: testing of delta failed:')+' '+str(s)) if os.path.exists(deltatmp): if KEEP: logger.warning(' '+_('You may want to examine:')+' '+str(deltatmp)) else: os.unlink(deltatmp) if not s.retriable : open(delta+'-fails','w').close() except (KeyboardInterrupt, SystemExit): if os.path.exists(deltatmp): os.unlink(deltatmp) raise except Exception: s=sys.exc_info()[1] exitstatus=max(exitstatus,4) puke(" *** Error while testing delta "+delta,s) if os.path.exists(deltatmp): if KEEP: logger.warning(' '+_('You may want to examine:')+' '+str(deltatmp)) else: os.unlink(deltatmp) open(delta+'-fails','w').close() if pret == None: return (max(exitstatus, 4), delta_size, elaps, p_elaps, info_delta, 'patch_fails') (newdeb_,p_elaps)=pret info_delta.append('PatchTime: %.2f' % p_elaps) ##end patch test #complete the delta file prepending to it the info try: hashes_info=append_info(deltatmp,info_delta) # sign the delta if DO_GPG: gpg_hashes['info']=hashes_info sign_delta(deltatmp,gpg_hashes) except: puke('debdeltas signing') if os.path.exists(deltatmp): os.unlink(deltatmp) #eventually, put in its place if os.path.isfile(deltatmp): # update delta size , since info and signatures were added delta_size = os.path.getsize(deltatmp) os.rename(deltatmp,delta) return (exitstatus, delta_size, elaps, p_elaps, info_delta, '') else: return (max(exitstatus, 4), delta_size, elaps, p_elaps, info_delta, 'no_delta') def init_apt_return_VersionCompare(): try: import apt_pkg if 'init_system' in dir(apt_pkg): apt_pkg.init_system() elif 'InitSystem' in dir(apt_pkg): apt_pkg.InitSystem() else: assert 'fegrwq' == 0 if 'VersionCompare' in dir(apt_pkg): from apt_pkg import VersionCompare elif 'version_compare' in dir(apt_pkg): from apt_pkg import version_compare as VersionCompare else: assert 'lopadf' == 0 except ImportError: raise DebDeltaError('python module "apt_pkg" is missing. Please install python-apt', retriable=True) return VersionCompare ############ def do_deltas(debs): exitstatus=0 VersionCompare = init_apt_return_VersionCompare() if not debs: logger.warning(_('Warning, no non-option arguments, debdeltas does nothing.')) if not DEBUG: return elif not OLD and not CLEAN_DELTAS: logger.warning(_('Warning, no --old arguments, debdeltas will not generate any deltas.')) if not DEBUG: return original_cwd = os.getcwd() start_time = time.time() info_by_pack_arch={} info_by_file = {} #cache to speed up scanning file packages scanned_delta_dirs=set() # contains list of triples (filename,oldversion,newversion) old_deltas_by_pack_arch={} ALT_DOUBLEDIR=[] # directories ending in // for arg in ALT: if os.path.isfile(arg): if os.path.basename(arg) in ('Packages', 'Packages.gz','Packages.bz2'): for a in iterate_Packages(arg): a['Label'] = 'ALT' #DEBUG a['Packages'] = arg info_by_pack_arch_add(a, info_by_pack_arch) elif arg[-4: ] == '.deb' : a=scan_deb_byfile_lazy(arg, info_by_file) if a: a['Label'] = 'ALT' info_by_pack_arch_add(a, info_by_pack_arch) else: logger.error('Error! skipping file --alt %r',arg) elif os.path.isdir(arg): if arg[-2:] != '//' : scan_deb_dir(arg, None, 'ALT', True, info_by_pack_arch) else: ALT_DOUBLEDIR.append(arg) else: logger.error('Error, ignored --alt %r',arg) for arg in OLD: if os.path.isfile(arg): if os.path.basename(arg) in ('Packages', 'Packages.gz','Packages.bz2'): for a in iterate_Packages(arg): a['Label'] = 'OLD' #DEBUG a['Packages'] = arg info_by_pack_arch_add(a, info_by_pack_arch) elif arg[-4: ] == '.deb' : a=scan_deb_byfile(arg, info_by_file) if a: a['Label'] = 'OLD' info_by_pack_arch_add(a, info_by_pack_arch) else: logger.error('Error! skipping file --old %r',arg) elif os.path.isdir(arg): # no // support yet scan_deb_dir(arg, None, 'OLD', False, info_by_pack_arch,info_by_file) else: logger.error('Error, ignored --old %r',arg) #scan cmdline arguments and prepare list of debs and deltas for arg in debs: if os.path.isfile(arg): if os.path.basename(arg) in ('Packages', 'Packages.gz','Packages.bz2'): for a in iterate_Packages(arg): a['Label'] = 'CMDLINE' #note that, if a file is scanned more than once, the last label is CMDLINE #DEBUG a['Packages'] = arg ofd = os.path.dirname(a['Filename']) pa = a['Package'] info_by_pack_arch_add(a, info_by_pack_arch) for alt in ALT_DOUBLEDIR: scan_deb_dir(delta_dirname(ofd,alt), pa , 'ALT', True, info_by_pack_arch) if CLEAN_DELTAS: scan_delta_dir(delta_dirname(ofd,DIR), pa, scanned_delta_dirs, old_deltas_by_pack_arch) elif arg[-4: ] == '.deb': a=scan_deb_byfile(arg, info_by_file) if a : a['Label'] = 'CMDLINE' info_by_pack_arch_add(a, info_by_pack_arch) ofd = os.path.dirname(arg) for alt in ALT_DOUBLEDIR: scan_deb_dir(delta_dirname(ofd,alt), a['Package'], 'ALT', True, info_by_pack_arch, info_by_file) if CLEAN_DELTAS: scan_delta_dir(delta_dirname(ofd,DIR), a['Package'], scanned_delta_dirs, old_deltas_by_pack_arch) else: logger.error('Error! skipping cmdline argument: %r',arg) elif os.path.isdir(arg): scan_deb_dir(arg, None, 'CMDLINE', False, info_by_pack_arch, info_by_file) for alt in ALT_DOUBLEDIR: scan_deb_dir(delta_dirname(arg,alt), None, 'ALT', True, info_by_pack_arch, info_by_file) if CLEAN_DELTAS: scan_delta_dir(delta_dirname(arg,DIR), None, scanned_delta_dirs, old_deltas_by_pack_arch) else: logger.error('Error, skipping cmd line argument: %r',arg) if VERBOSE > 1 : logger.info(' total parsing time: %.1f ', ( -start_time + time.time())) for pa,ar in info_by_pack_arch : info_pack=info_by_pack_arch[ (pa,ar) ] assert(info_pack) # delete old deltas if CLEAN_DELTAS : if (pa,ar) in old_deltas_by_pack_arch : versions=[(o['Version']) for o in info_pack if (o['Label'] == 'CMDLINE')] timegrace=time.time() - CLEAN_DELTAS_MTIME * 24 * 3600 #DEBUG print pa,ar,versions for f_d,o_d,n_d in old_deltas_by_pack_arch[ (pa,ar) ] : if n_d not in versions : if os.path.exists(f_d): if os.stat(f_d)[ST_MTIME] < timegrace: if VERBOSE: logger.debug(' Removing: %r', f_d) if ACT: os.unlink(f_d) elif VERBOSE > 1 : logger.debug(' Obsolete but still young, graced: %r',f_d) else: logger.debug('bellandata %r',f_d) del versions elif DEBUG > 1: logger.debug('No deltas where found for: %r %r',pa,ar) if N_DELTAS != None and 0 == N_DELTAS: #debdeltas was invoked just to clean the archive pass else: for old, new, delta, forensicfile in \ iter_deltas_one_pack_arch(pa,ar,info_pack, DIR, FORENSICDIR, VersionCompare): if not ACT: logger.info('Would create: %r',delta) else: e=do_delta_and_test(old['File'], new['File'], delta, forensicfile) exitstatus=max(exitstatus,e[0]) if VERBOSE : logger.info(' '+ ( _('Total running time: %.1f')),( -start_time + time.time())) return exitstatus ##################################################### delta-upgrade class Predictor: package_stats = None upgrade_stats = None def __init__(self): import shelve #self.shelve=shelve if os.getuid() == 0: basedir='/var/lib/debdelta' else: if not os.path.exists(os.path.expanduser('~/')): logger.warning('(home directory does not exists, Predictor disabled)') return basedir=os.path.expanduser('~/.debdelta') s=os.path.join(basedir,'upgrade.db') if not os.path.exists(basedir): logger.info('Creating: %r',basedir) os.makedirs(basedir) self.upgrade_stats=shelve.open(s,flag='c') s=os.path.join(basedir,'packages_stats.db') if os.path.exists(s) or DEBUG > 1 : self.package_stats=shelve.open(s,flag='c') self.patch_time_predictor=self.patch_time_predictor_math ##### predictor for patching time def patch_time_predictor_simple(self,p): if 'ServerBogomips' in p and 'PatchTime' in p: return (float(p[ 'PatchTime']) / BOGOMIPS * float(p['ServerBogomips']) ) else: return None def update(self,p,t): #save delta info if self.package_stats != None : n=p['NEW/Package'] d=copy(p) d['LocalDeltaTime']=t try: self.package_stats[n]=d except Exception: logger.exception('ERROR:Cannot update package_stats') if self.upgrade_stats == None : return s='ServerID' if s not in p : return s=s+':'+p[s] if s not in self.upgrade_stats: r=1 if 'ServerBogomips' in p : r= float(p['ServerBogomips']) / BOGOMIPS try: self.upgrade_stats[s]={ 'PatchSpeedRatio' : r } except Exception: logger.exception('ERROR:Cannot update upgrade_stats') if 'PatchTime' not in p: return ut=float(p[ 'PatchTime']) r=self.upgrade_stats[s]['PatchSpeedRatio'] nr = 0.95 * r + 0.05 * ( t / ut ) a=self.upgrade_stats[s] a['PatchSpeedRatio'] = nr try: self.upgrade_stats[s]=a except Exception: logger.exception('ERROR:Cannot update upgrade_stats') if VERBOSE > 1 : logger.debug(' Upstream %r PatchSpeedRatio from %r to %r , %r', ut, r, nr, self.upgrade_stats[s]['PatchSpeedRatio']) def patch_time_predictor_math(self,p): "Predicts time to patch." if 'PatchTime' not in p: return None ut=float(p[ 'PatchTime']) # s='ServerID' if s not in p : return self.patch_time_predictor_simple(p) s=s+':'+p[s] if s not in self.upgrade_stats: return self.patch_time_predictor_simple(p) r=self.upgrade_stats[s]['PatchSpeedRatio'] return r * ut def delta_uri_from_config(config, **dictio): secs=config.sections() for s in secs: opt=config.options(s) if 'delta_uri' not in opt: raise DebDeltaError('sources.conf section '+repr(s)+'does not contain delta_uri',exitcode=3) match=True for a in dictio: #damn it, ConfigParser changes everything to lowercase ! if ( a.lower() in opt ) and ( dictio[a] != config.get( s, a) ) : #print '!!',a, repr(dictio[a]) , ' != ',repr(config.get( s, a)) match=False break if match: return config.get( s, 'delta_uri' ) if VERBOSE : logger.warning(' '+_("(sources.conf does not provide a server for `%s')"), repr(dictio['PackageName'])) def delta_upgrade_(args): # a list of all error exitcodes that derive from downloading and applying mainexitcodes = [0] original_cwd = os.getcwd() import threading, fcntl, atexit, signal, socket if sys.version_info.major == 2: import Queue, urllib2, thread from httplib import HTTPConnection, HTTPException def http_responses_get(j): import httplib return httplib.responses.get(j,str(j)) from urlparse import urlparse, urlunparse from urllib2 import quote as urlquote proxies=urllib2.getproxies() else: import queue as Queue import _thread as thread from http.client import HTTPConnection, HTTPException #FIXME http_responses_get = lambda j : str(j) from urllib.parse import urlparse, urlunparse from urllib.parse import quote as urlquote #FIXME proxies = [] try: import requests requests_session = requests.Session() except ImportError: if (sys.version_info.major == 2): logger.info('You may want to install `requests`\n# sudo apt install python-requests\n') else: logger.warning('You should install `requests`\n# sudo apt install python3-requests\n') requests=None requests_session=None if VERBOSE and proxies: #note that this message is indented, I dont know what's best in translations logger.warning(_(' Proxy settings detected in the environment; using "urllib2" for downloading; but\n this disables some features and is in general slower and buggier. See man page.')) #for example, urllib2 transforms http response "401" into "404" , and "302" into "200" config=ConfigParser() a=config.read(['/etc/debdelta/sources.conf', expanduser('~/.debdelta/sources.conf') ]) # FIXME this does not work as documented in Python #if VERBOSE > 1 : print 'Read config files: ',repr(a) import warnings warnings.simplefilter("ignore",FutureWarning) if DO_PROGRESS: sys.stderr.write((_('Initializing APT cache...')).ljust(terminalcolumns)+'\r') elif VERBOSE: logger.debug(_('Initializing APT cache...')) try: import apt_pkg except ImportError: raise DebDeltaError('python module "apt_pkg" is missing. Please install python-apt',True) try: import apt except ImportError: raise DebDeltaError('python module "apt" is missing. Please install a newer version of python-apt (newer than 0.6.12).',True) apt_pkg.init() #from apt import SizeToStr ## Return a string describing the size in a human-readable manner using ## SI prefix and base-10 units, e.g. '1k' for 1000, '1M' for 1000000, etc. def SizeToKibiStr(a): "this uses kibibytes (altough the program prints them as kilobytes)" if a < 8096 : return str(int(a))+'B' elif a < 8096*1024: return str(int(a/1024))+'kB' else: return str(int(a/1024/1024))+'MB' if DO_PROGRESS: sys.stderr.write((_('Upgrading APT cache...')).ljust(terminalcolumns)+'\r') elif VERBOSE: logger.debug(_('Upgrading APT cache...')) cache=apt.Cache() try: cache.upgrade(True) except: if DO_PROGRESS: sys.stderr.write((_('Failed! Safe upgrading APT cache...')).ljust(terminalcolumns)+'\r') elif VERBOSE: logger.debug(_('Failed! Safe upgrading APT cache...')) cache.upgrade(False) if DO_PROGRESS: sys.stderr.write((_('Upgraded APT cache.')).ljust(terminalcolumns)+'\r') elif VERBOSE: logger.debug(_('Upgraded APT cache.')) diversions=scan_diversions() if DIR == None: if os.getuid() == 0: DEB_DIR='/var/cache/apt/archives' else: DEB_DIR='/tmp/archives' else: DEB_DIR=DIR if not os.path.exists(DEB_DIR): os.mkdir(DEB_DIR) if not os.path.exists(DEB_DIR+'/partial'): os.mkdir(DEB_DIR+'/partial') try: ##APT does (according to strace) #open("/var/cache/apt/archives/lock", O_RDWR|O_CREAT|O_TRUNC, 0640) = 17 #fcntl64(17, F_SETFD, FD_CLOEXEC) = 0 #fcntl64(17, F_SETLK, {type=F_WRLCK, whence=SEEK_SET, start=0, len=0}) = 0 ##so a=os.open(DEB_DIR+'/lock', os.O_RDWR | os.O_TRUNC | os.O_CREAT, 0o640) fcntl.fcntl(a, fcntl.F_SETFD, fcntl.FD_CLOEXEC) # synopsis lockf( fd, operation, [length, [start, [whence]]]) fcntl.lockf(a, fcntl.LOCK_EX | fcntl.LOCK_NB, 0,0,0) except IOError: s=sys.exc_info()[1] from errno import EAGAIN if s.errno == EAGAIN : a=' already locked!' else: a=str(s) if DEB_DIR == '/var/cache/apt/archives' : a=a+' (is APT running?)' raise DebDeltaError('could not lock dir: '+DEB_DIR+' '+a, retriable=True) if VERBOSE or DEB_DIR != '/var/cache/apt/archives' : logger.info(_('Recreated debs are saved in the directory %s'), DEB_DIR) #these are the packages that do not have a delta no_delta = [] total_time = -time.time() ##### predictor for patching time if DO_PREDICTOR: predictor = Predictor() #this is a dictonary (key is package name) of parameters of deltas #(to add some math in the future) params_of_delta={} patching_queue=Queue.Queue() thread_returns={} ######################## thread_do_patch def thread_do_patch(que, no_delta, returns, exitcodes, forensics): if VERBOSE > 1 : logger.info(' Patching thread started. ') debs_size=0 debs_time=0 # in case of early exception returns['debs_size']=debs_size returns['debs_time']=debs_time while 1: a = que.get() if a == None: break (name, delta , newdeb, deb_uri) = a debs_time -= time.time() TD=tempo() if not ACT: logger.info('Would create: %r ',newdeb) else: if VERBOSE >= 2 : logger.debug(' Now patching for: %r',name) try: #start_time=time.time() returns['patchname']=os.path.basename(newdeb) ret=do_patch_(delta, '/', newdeb , TD, returns, diversions=diversions, do_progress=False) del returns['patchname'] l = os.path.getsize(newdeb) #a=time.time() - start_time a=ret[1] #dear translator, please align this line with the line saying 'Downloaded, ...' msg=_("Created, time %(time)5.2fsec, speed %(speed)4s/sec, %(name)s") msgdata={'time' : a, 'speed' : SizeToKibiStr(l / (a+0.001)), 'name' : os.path.basename(newdeb)} if DO_PROGRESS: sys.stderr.write((msg % msgdata).ljust(terminalcolumns)+'\n') else: logger.info(msg % msgdata) except KeyboardInterrupt: thread.interrupt_main() rmtree(TD) return except DebDeltaError: s=sys.exc_info()[1] puke(' Error: applying of delta for '+name+' failed: ',s) if 'e' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : logger.debug(' No deb-policy "e", no download of %r',deb_uri) forensics.append(s.logs) exitcodes.append(s.exitcode) except: if puke == None: return puke( " *** Error while applying delta for "+name+": ") if 'e' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : logger.debug(' No deb-policy "e", no download of %r',deb_uri) exitcodes.append(4) else: if name in params_of_delta : p= params_of_delta[name] name,elaps=ret if DO_PREDICTOR: predictor.update(p,elaps) if VERBOSE > 1 : t=predictor.patch_time_predictor(p) if t: logger.debug(' (Predicted %.3f sec )', t) debs_size += os.path.getsize(newdeb) if os.path.exists(delta): os.unlink(delta) rmtree(TD) debs_time += time.time() returns['debs_size']=debs_size returns['debs_time']=debs_time if VERBOSE > 1 : logger.debug(' Patching thread ended , bye bye. ') ##################################### def progress_string(statusdb): download='' if 'downloaduri' in statusdb: download="D %2d%% (%4s/s) %s " % \ (statusdb.get('downloadprogress',-1), statusdb.get('downloadspeed','-'), statusdb['downloaduri']) patch='' if 'patchname' in statusdb: patch='P %2d%% %s'% (statusdb.get('patchprogress',-1), statusdb['patchname']) if terminalcolumns == None: return download+' ; '+patch if not patch: return download[:terminalcolumns] if not download: return patch[:terminalcolumns] ld=len(download) lp=len(patch) b=ld + lp if b < terminalcolumns - 3 : return download+' ; '+patch a=float(terminalcolumns-5)/float(b) ld=int(ld*a) lp=int(lp*a) return download[:ld] + ' ; ' + patch[:lp] + '..' #################### manage connections #keeps a cache of all connections, by URL http_conns={} def conn_by_url(url): url=urlparse(url)[1] if url not in http_conns: if VERBOSE > 1 : logger.debug(' Opening connection to: %r',url) http_conns[url] = HTTPConnection(url, timeout=TIMEOUT) return http_conns[url] def conn_close(url,fatal=False): url=urlparse(url)[1] conn=http_conns.get(url) if fatal: http_conns[url] = None else: del http_conns[url] if conn != None : if VERBOSE > 1 : logger.debug(' Closing connection to: %r',url) conn.close() #### def _connect(uri, headers): """connects for a GET ; returns (filetype, statuscode, servermessage, getheaders) in case of errors returns (exception, statuscode, None, None) """ uri_p=urlparse(uri) ###### python3 code if requests and uri_p.scheme in ('http','https') : try: req = requests_session.get(uri, headers=headers, stream = True) req.read = req.raw.read return req, req.raw.status, getattr(req.raw,'msg','(no message)'), req.raw.info() except requests.exceptions.ConnectionError: e=sys.exc_info()[1] puke( 'Connection error (fatal): %r'%(uri,),e) return e, None, None, None except requests.exceptions.RequestException: e=sys.exc_info()[1] puke( 'Error in http request (fatal): %r'%(uri,),e) return e, None, None, None elif sys.version_info.major == 3: import urllib.request try: req = urllib.request.urlopen(uri,) return req, getattr(req,'code',None), getattr(req,'msg','(no message)'), req.info() except urllib.request.HTTPError: e=sys.exc_info()[1] return e, e.code, None, None except Exception: e=sys.exc_info()[1] #puke( 'Exception while accessing',uri) return DebDeltaError('for %s : %s'%(uri,e)), None, None, None ###### python2 code if uri_p.scheme in ('http','https') and not proxies: #use persistent http connections conn=conn_by_url(uri) if conn == None : return None, None, None, None try: conn.request("GET", urlquote(uri_p[2]),headers=headers) r = conn.getresponse() return r, r.status, r.reason, r.msg except (HTTPException, socket.error): e=sys.exc_info()[1] if VERBOSE : puke( ' Connection error (retrying): %r'%(uri_p[1],),e) conn_close(uri) try: conn=conn_by_url(uri) conn.request("GET", urlquote(uri_p[2]),headers=headers) r = conn.getresponse() return r, r.status, r.reason, r.msg except (HTTPException, socket.error): e=sys.exc_info()[1] puke( 'Connection error (fatal): ',uri_p[1]) mainexitcodes.append(1) try: conn_close(uri,fatal=True) except: pass mainexitcodes.append(1) return e, None, None, None else: #use urllib2 if uri_p.scheme in ('http','https'): a=[copy(z) for z in uri_p] a[2]=urlquote(uri_p[2]) uri=urlunparse(a) import urllib2 try: req = urllib2.Request(uri, headers=headers) r = urllib2.urlopen(req) #print r.info(),dir(r),r.code return r, getattr(r,'code',None), getattr(r,'msg','(no message)'), r.info() except urllib2.HTTPError: e=sys.exc_info()[1] return e.code, None, None, None except (HTTPException,socket.error,urllib2.URLError): e=sys.exc_info()[1] puke( 'Connection error (fatal): %r'%(uri,), e) mainexitcodes.append(1) return e, None, None, None ################################################# various HTTP facilities def _parse_ContentRange(s): #bytes 0-1023/25328 if not s or s[:6] != "bytes " : logger.warning("Malformed Content-Range %r", s) return a=s[6:].split('/') if len(a) != 2 : logger.warning("Malformed Content-Range %r", s) return b=a[0].split('-') if len(b) != 2 : logger.warning("Malformed Content-Range %r", s) return return int(b[0]),int(b[1]),int(a[1]) ###################################### test_uri def test_uri(uri): conn=conn_by_url(uri) if conn == None: return None uri_p=urlparse(uri) assert(uri_p[0] in ('http','https')) conn.request("HEAD", urlquote(uri_p[2]),headers=HTTP_USER_AGENT) r = conn.getresponse() r.read() r.close() return r.status ###################################### download_10k_uri def download_10k_uri(uri,outname): "in case of connection error, returns the (error, None, None, None) ; otherwise returns (status,len,outname,complete)" #download uri_p=urlparse(uri) assert(uri_p[0] in ('http', 'https')) re=copy(HTTP_USER_AGENT) re["Range"] = "bytes=0-10239" complete=False r, status, msg, responseheaders=_connect(uri, re) if not hasattr(r,'read') and responseheaders==None: return r, None, None, None if status == 301 and 'location' in responseheaders: r.read() r.close() return download_10k_uri(responseheaders['location'], outname) elif status == 206: outnametemp=os.path.join(os.path.dirname(outname),'partial',os.path.basename(outname)) try: l = _parse_ContentRange(responseheaders['Content-Range'])[2] except (KeyError, ValueError): l = None elif status == 200: outnametemp=outname complete=True try: l=long(responseheaders.get('Content-Length')) except: l=None else: #FIXME how do we deal with a FTP mirror of deltas ? r.read() r.close() return status, None, None, None s=r.read() r.close() #(maybe I did not understand the status 206 ?) if not complete and l != None and len(s) >= l: outnametemp=outname complete=True if os.path.exists(outnametemp) and os.path.getsize(outnametemp) >= len(s) : #do not truncate preexisting file complete=(os.path.getsize(outnametemp) >= l) return status, outnametemp, l, complete out=open(outnametemp,'wb') out.write(s) out.close() return status, outnametemp, l, complete ###################################### download_uri def download_uri(uri, outname, conn_time, len_downloaded, statusdb): "returns `(time_elapsed, bytes_downloaded)`; or `None` in case of error, " outnametemp=os.path.join(os.path.dirname(outname),'partial',os.path.basename(outname)) re=copy(HTTP_USER_AGENT) #content range l=None if os.path.exists(outnametemp): #shamelessly adapted from APT, methods/http.cc s=os.stat(outnametemp) l=s[ST_SIZE] #t=s[ST_MTIME] ### unfortunately these do not yet work #thank god for http://docs.python.org/lib/module-time.html #actually APT does #t=time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(t)) ##re["If-Range"] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(t)) ####re["If-Range"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(t)) re["Range"] = "bytes=%li-" % ( (long(l)-1) ) #start downloading start_time=time.time() r, status, message, responseheaders = _connect(uri, re) if not hasattr(r,'read') and responseheaders==None: return if status in (301,302) and 'location' in responseheaders: r.read() r.close() if VERBOSE > 1 : logger.debug(_(' Redirect to:')+' '+repr(responseheaders['location'])) return download_uri(responseheaders['location'], outname, conn_time, len_downloaded, statusdb) if not ( status == None or status == 200 or ( status == 206 and l != None ) ): if VERBOSE : logger.warning('Connection problem, status %r msg: %r uri: %r', status, message, uri) r.read() r.close() return #print 'ooK Content-Range', r.getheader('Content-Range') #HACK if l and status == 200 : logger.warning(' Hmmm... our HTTP range request failed, %r %r %r',re,status,message) if status == 200 : out=open(outnametemp,'wb') try: total_len = long(responseheaders['Content-Length']) except (KeyError,ValueError): total_len=None elif status == 206 : #APT does scanf of "bytes %lu-%*u/%lu",&StartPos,&Size #first-byte-pos "-" last-byte-pos "/" instance-length out=open(outnametemp,'ab') try: a,b,total_len =_parse_ContentRange(responseheaders['Content-Range']) except (KeyError,ValueError): e=sys.exc_info()[1] logger.warning('! problem, http response [206], Content Range %s , error %s , uri %s\n' %\ (responseheaders.get('Content-Range'),e,uri)) return out.seek(a) out.truncate() else: out=open(outnametemp,'wb') try: total_len=long(responseheaders.get('Content-length')) except ValueError: total_len=None free=freespace(os.path.dirname(outname)) if total_len != None and free and (free + 2**14 ) < total_len : logger.warning(_('Not enough disk space to download:')+' '+os.path.basename(uri)) r.read() r.close() mainexitcodes.append(1) return j=0 s=r.read(1024) while s and (total_len == None or out.tell() < total_len ): j+=len(s) out.write(s) if total_len: statusdb['downloadprogress']=99.9 * out.tell() / total_len a=time.time() + conn_time - start_time if a > 0.5 : statusdb['downloadspeed']=SizeToKibiStr(float(j+len_downloaded)/a) s=r.read(1024) out.close() r.close() #end of download a=time.time() - start_time #if total_len == None: # total_len = os.path.getsize(outnametemp) #dear translator, please align this line with the line saying 'Created, ...' msg=_("Downloaded, time %(time)5.2fsec, speed %(speed)4s/sec, %(name)s") msgdata={'time' : a , 'speed' : SizeToKibiStr(j / (a+0.001)) , 'name' : os.path.basename(uri)} if DO_PROGRESS: sys.stderr.write((msg % msgdata).ljust(terminalcolumns)+'\n') else: logger.info(msg % msgdata) os.rename(outnametemp,outname) #FIXME this is incorrect by 1024 bytes return conn_time + a, (j+len_downloaded) ###################################### end of HTTP stuff ################### start patching thread forensics=[] patching_thread=threading.Thread( target=thread_do_patch , args=(patching_queue, no_delta, thread_returns, mainexitcodes, forensics) ) patching_thread.daemon=True patching_thread.start() ################### first merry-go-round deltas_down_size=0 deltas_down_time=0 #this is a list of tuples of ..... available_deltas=[] not_available_deltas=[] if hasattr(apt.package.Package,'is_installed'): is_installed=lambda p : p.is_installed elif hasattr(apt.package.Package,'isInstalled'): is_installed=lambda p : p.isInstalled else: assert 0 if hasattr(apt.package.Package,'marked_upgrade'): marked_upgrade=lambda p : p.marked_upgrade elif hasattr(apt.package.Package,'markedUpgrade'): marked_upgrade=lambda p : p.markedUpgrade else: assert 0 progress_count=0 ## first merry-go-round, use package cache to fill available_deltas, download 10kB of each delta for p in cache : #print progress if DO_PROGRESS : progress_count+=1 if 0 == (progress_count & 2047): sys.stderr.write(('%2.1f%%' % (float(progress_count) * 100.0 / len(cache))).ljust(terminalcolumns) + '\r') if is_installed(p) and marked_upgrade(p) : if args and p.name not in args: continue #thanks a lot to Julian Andres Klode candidate=p.candidate origin = p.candidate.origins[0] arch=candidate.architecture deb_uri=candidate.uri installed_version=p.installed.version candidate_version=p.candidate.version newsize=p.candidate.size deb_path=str.split(deb_uri,'/') try: thepoolindex=deb_path.index('pool') except ValueError: logger.warning('! Package "%s" (version %s) does not have "pool" in the uri %s \n' % (p.name, candidate_version, deb_uri)) continue deb_path=str.join('/',deb_path[(thepoolindex):]) #try all possible variants of the filename newdebs=[p.shortname+'_'+candidate_version+'_'+arch+'.deb', os.path.basename(deb_uri)] if ':' in candidate_version: a=candidate_version.split(':') newdebs.append(p.shortname+'_'+a[1]+'_'+arch+'.deb') newdebs.append(p.shortname+'_'+a[0]+'%3A'+a[1]+'_'+arch+'.deb') newdebs.append(p.shortname+'_'+a[0]+'%3a'+a[1]+'_'+arch+'.deb') for newdeb in newdebs: if os.path.exists(DEB_DIR+'/'+newdeb) or \ os.path.exists('/var/cache/apt/archives/'+newdeb): if VERBOSE > 1 : logger.debug(' Already downloaded %r %r as %r',p.name,candidate_version,newdeb) newdeb=None break if newdeb == None: continue newdeb = DEB_DIR+'/'+newdebs[0] if VERBOSE > 1 : logger.debug(' Looking for a delta for %s from %s to %s ', p.name, installed_version, candidate_version) delta_uri_base=delta_uri_from_config(config, Origin=origin.origin, Label=origin.label, Site=origin.site, Archive=origin.archive, PackageName=p.name) if delta_uri_base == None: if 's' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) continue a=urlparse(delta_uri_base) assert(a[0] in ('http', 'https')) #delta name delta_name = delta_base_name(p.shortname, installed_version, candidate_version, arch) uri=delta_uri_base+'/'+os.path.dirname(deb_path)+'/'+delta_name #download first part of delta abs_delta_name= DEB_DIR+'/'+delta_name #maybe it is already here if os.path.exists(abs_delta_name): a = abs_delta_name else: a = DEB_DIR+'/partial/'+delta_name if not os.path.exists(a): a = None if a: l=os.path.getsize(a) if VERBOSE > 1 : logger.debug(' Already here: %r',abs_delta_name) s=get_info_fast(a) if s: params_of_delta[p.name]=info_2_db(s) available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, a, True) ) continue #if not, download its first part if DO_PROGRESS: sys.stderr.write(('%2.1f%% ' % (float(progress_count) * 100.0 / len(cache)) + \ _('Downloading head of %s...') % p.name).ljust(terminalcolumns) + '\r') deltas_down_time-=time.time() status, tempname, l, complete = download_10k_uri(uri,abs_delta_name) deltas_down_time+=time.time() #some strange error in remote server? #FIXME this does not support ftp delta repositories if status != 200 and status != 206 and status != 404: logger.warning('Delta is not downloadable (%s):%s', http_responses_get(status), uri) continue if status == 404: not_available_deltas.append(p.name) # check if delta is too big if uri.startswith('http') and not proxies and 200 == test_uri(uri+'-too-big'): # FIXME support ftp or proxies logger.info(_('Delta is too big:')+' '+delta_name) if 'b' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : logger.debug(' No deb-policy "b", no download of %r',deb_uri) continue # check if delta is queued in the server but not yet done if uri.startswith('http') and not proxies and 200 == test_uri(uri+'-queued'): # FIXME support ftp or proxies logger.info(_('Delta is not yet ready in the server:')+' '+delta_name) if 'q' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : logger.debug(' No deb-policy "q", no download of %r',deb_uri) continue #check if delta failed upstream if uri.startswith('http') and not proxies and 200 == test_uri(uri+'-fails'): # FIXME support ftp or proxies logger.info(_('Delta missing, server failed to create it:')+' '+delta_name) if 'f' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : logger.debug(' No deb-policy "f", no download of %r',deb_uri) continue ### FIXME the server is not generating these stamps ! #if uri.startswith('http') and not proxies and newsize <= 2 * MIN_DEB_SIZE : # check only on small packages # smallstatus = test_uri(uri+'-smalldeb') # if smallstatus == 200: ETC ETC ### # packages smaller than MIN_DEB_SIZE are ignored by the server. if newsize <= MIN_DEB_SIZE : # check only on small packages logger.info(_('Delta was not created since new package is too small:')+' '+delta_name) if 't' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : logger.debug(' No deb-policy "t", no download of %r',deb_uri) continue if DEBUG and VERBOSE: logger.info(_('Delta is not present:')+' '+uri) else: logger.info(_('Delta is not present:')+' '+delta_name) if 'u' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : logger.debug(' No deb-policy "u", no download of %r',deb_uri) continue if VERBOSE > 1: logger.debug('Delta is present: %r %r', delta_name, tempname) elif DO_PROGRESS: sys.stderr.write(('%2.1f%%' % (float(progress_count) * 100.0 / len(cache)) + \ _('Downloaded head of %s.') % p.name).ljust(terminalcolumns) + '\r') if os.path.isfile(tempname): deltas_down_size+=os.path.getsize(tempname) #parse file and save info try: s=get_info_fast(tempname) except DebDeltaError: e=sys.exc_info()[1] logger.error("!!"+str(e)+'\n') logger.error("!! (renamed to "+tempname+'~~NOT~A~DELTA~~ )\n') os.rename(tempname,tempname+'~~NOT~A~DELTA~~') if proxies: logger.error("!!maybe a proxy is returning an error page??\n") else: logger.error("!!damaged delta??\n") continue if s: params_of_delta[p.name]=info_2_db(s) s=patch_check_tmp_space(params_of_delta[p.name], '/') if s != True: logger.warning('%r : sorry %r', p.name, s) #neither download deb nor delta.. #the user may wish to free space and retry continue #FIXME may check that parameters are conformant to what we expect if complete: patching_queue.put( (p.name, abs_delta_name ,newdeb, deb_uri ) ) else: available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, tempname, complete) ) ## end of first merry-go-round available_deltas.sort() if DEBUG or VERBOSE: if DO_PROGRESS: sys.stderr.write(' ' * (terminalcolumns) +'\r') logger.info(' '+ _('Deltas: %(present)d present and %(absent)d not,') % \ {'present' : len(available_deltas), 'absent' : len(not_available_deltas)}) logger.info(' '+ _('downloaded so far: time %(time).2fsec, size %(size)s, speed %(speed)4s/sec.') % \ {'time' : deltas_down_time, 'size' : SizeToKibiStr(deltas_down_size), 'speed' : SizeToKibiStr(deltas_down_size/float(deltas_down_time+0.001))}) if available_deltas: logger.info(' '+_('Need to get %s of deltas.') , SizeToKibiStr(sum([a[0] for a in available_deltas]))) ###################### check available space a=freespace('/var/cache/apt/archives') / 1024 b=sum([int(s.get('NEW/Installed-Size','0')) for s in params_of_delta.values()]) c=sum([int(s.get('NEW/Size','0')) for s in params_of_delta.values()]) / 1024 if DEB_FORMAT == 'deb' and a < c: logger.warning('**'+_('Very low disk space, need %(need)d kB have %(have)d kB'), {'need':c,'have':a}) if DEB_FORMAT == 'unzipped' and a < b: logger.warning('**'+_('Very low disk space, need %(need)d kB have %(have)d kB'), {'need':b,'have':a}) ###################### start progress thread def print_progress(common_db): while sys and 'STOP' not in common_db: sys.stderr.write(progress_string(common_db)+'\r') time.sleep(0.2) if DO_PROGRESS and terminalcolumns > 4: progress_thread=threading.Thread(target=print_progress, args=( thread_returns, ) ) progress_thread.daemon=True progress_thread.start() else: progress_thread=None ################ second merry-go-round, download rest of available deltas , queue them for delta_len, name, uri, abs_delta_name , newdeb, deb_uri, tempname, complete in available_deltas : ## this seems to create problems.... #if not os.path.exists(abs_delta_name) and os.path.exists(tempname) and os.path.getsize(tempname) == delta_len: # print 'Just rename:',name #this actually should never happen, but , who knows... # os.rename(tempname,abs_delta_name) # tempname=abs_delta_name if name in params_of_delta: s=patch_check_tmp_space(params_of_delta[name], '/') if s != True: logger.warning('%r : sorry, %r', name, s) #argh, we ran out of space in meantime continue if not os.path.exists(abs_delta_name): thread_returns['downloaduri']=os.path.basename(uri) r=download_uri(uri, abs_delta_name, deltas_down_time, deltas_down_size, thread_returns) del thread_returns['downloaduri'] if r == None or isinstance(r, HTTPException) : if VERBOSE : logger.info(' '+_('You may wish to rerun, to get also:')+' '+uri) continue else: deltas_down_time = r[0] deltas_down_size = r[1] #queue to apply delta if os.path.exists(abs_delta_name): #append to queue patching_queue.put( (name, abs_delta_name ,newdeb, deb_uri ) ) else: if 'u' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : logger.debug(' No deb-policy "u", no download of %r',deb_uri) ## end of second merry-go-round #terminate queue patching_queue.put(None) #do something useful in the meantime debs_down_size=0 debs_down_time=0 if patching_thread.is_alive() and no_delta and VERBOSE > 1 : logger.info(' Downloading deltas done, downloading debs while waiting for patching thread.') while patching_thread.is_alive() or ('a' in DEB_POLICY and no_delta): if no_delta: uri, newdeb = no_delta.pop() thread_returns['downloaduri']=os.path.basename(uri) r=download_uri(uri, newdeb, debs_down_time, debs_down_size, thread_returns) del thread_returns['downloaduri'] if r == None or isinstance(r, HTTPException) : if VERBOSE : logger.debug(' You may wish to rerun, to get also: %r',uri) continue if r: debs_down_time = r[0] debs_down_size = r[1] if not no_delta : time.sleep(0.1) for i in http_conns: if http_conns[i] != None : http_conns[i].close() while patching_thread.is_alive(): time.sleep(0.1) #terminate progress report thread_returns['STOP']=True while progress_thread != None and progress_thread.is_alive(): time.sleep(0.1) if DO_PROGRESS: sys.stderr.write(' ' * terminalcolumns +'\r') total_time += time.time() logger.info(_('Delta-upgrade statistics:')) msg=_("size %(size)s time %(time)dsec speed %(speed)s/sec") if VERBOSE: if deltas_down_time : a=float(deltas_down_size) t=deltas_down_time logger.info(' ' + _('downloaded deltas, ') + msg % \ {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )}) if thread_returns['debs_time'] : a=float(thread_returns['debs_size']) t=thread_returns['debs_time'] logger.info(' ' + _('patching to debs, ') + msg % \ {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )}) if debs_down_time : a=float(debs_down_size) t=debs_down_time logger.info(' ' + _('downloaded debs, ') + msg % \ {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )}) if total_time: a=float(debs_down_size + thread_returns['debs_size']) t=total_time logger.info(' ' + _('total resulting debs, size %(size)s time %(time)dsec virtual speed %(speed)s/sec') % \ {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )}) if forensics: forensic_send(forensics) return max(mainexitcodes) ################################################# main program, do stuff def act(): "fake function that marks where the action starts" pass if action == 'patch': if INFO : if len(argv) > 1 and VERBOSE : sys.stderr.write(' (printing info - extra arguments are ignored)\n') elif len(argv) == 0 : sys.stderr.write(_('Need a filename; try --help.')+'\n') raise SystemExit(3) try: delta=abspath(argv[0]) check_is_delta(delta) info=get_info(delta) for s in info: if s: logger.debug(' info: %r',s) except KeyboardInterrupt: puke('debpatch exited by keyboard interrupt') raise SystemExit(5) except DebDeltaError: s=sys.exc_info()[1] puke('debpatch',s) raise SystemExit(s.exitcode) except Exception: s=sys.exc_info()[1] puke("debpatch",s) raise SystemExit(4) raise SystemExit(0) #really patch if len(argv) != 3 : sys.stderr.write(_('Need 3 filenames; try --help.')+'\n') raise SystemExit(3) newdeb=abspath(argv[2]) if newdeb == '/dev/null': newdeb = None try: do_patch(abspath(argv[0]), abspath(argv[1]), newdeb) except KeyboardInterrupt: puke('debpatch exited by keyboard interrupt') raise SystemExit(5) except DebDeltaError: s=sys.exc_info()[1] puke('debpatch',s) if s.logs: forensic_send([s.logs]) raise SystemExit(s.exitcode) except Exception: s=sys.exc_info()[1] puke('debpatch',s) raise SystemExit(4) raise SystemExit(0) elif action == 'delta' : if len(argv) != 3 : sys.stderr.write(_('Need 3 filenames; try --help.')+'\n') raise SystemExit(3) delta=abspath(argv[2]) try: r = do_delta(abspath(argv[0]), abspath(argv[1]), delta) except KeyboardInterrupt: puke('debdelta exited by keyboard interrupt') raise SystemExit(5) except DebDeltaError: s=sys.exc_info()[1] puke('debdelta',s) raise SystemExit(s.exitcode) except Exception: s=sys.exc_info()[1] puke('debdelta',s) raise SystemExit(4) raise SystemExit(0) elif action == 'deltas' : for v in argv: if not (os.path.isfile(v) or os.path.isdir(v)) : sys.stderr.write(_("Error: argument is not a directory or a regular file:")+' '+v) raise SystemExit(3) try: exitcode=do_deltas(argv) except KeyboardInterrupt: puke('debdeltas exited by keyboard interrupt') raise SystemExit(5) except DebDeltaError: s=sys.exc_info()[1] puke('debdeltas',s) raise SystemExit(s.exitcode) except Exception: s=sys.exc_info()[1] puke('debdeltas',s) raise SystemExit(4) raise SystemExit(exitcode) elif action == 'delta-upgrade': import warnings warnings.simplefilter("ignore",FutureWarning) try: exitcode=delta_upgrade_(argv) except KeyboardInterrupt: puke('debdelta-upgrade exited due to keyboard interrupt') raise SystemExit(5) except DebDeltaError: s=sys.exc_info()[1] puke('debdelta-upgrade',s) raise SystemExit(s.exitcode) except Exception: s=sys.exc_info()[1] puke('debdelta-upgrade',s) raise SystemExit(4) raise SystemExit(exitcode) elif action == 'patch-url': config=ConfigParser() config.read(['/etc/debdelta/sources.conf', expanduser('~/.debdelta/sources.conf') ]) try: import apt_pkg except ImportError: logger.error('ERROR!!! python module "apt_pkg" is missing. Please install python-apt') raise SystemExit(1) try: import apt except ImportError: logger.error('ERROR!!! python module "apt" is missing. Please install a newer version of python-apt (newer than 0.6.12)') raise SystemExit(1) apt_pkg.init() cache=apt.Cache() cache.upgrade(True) for a in argv: logger.info(_('Lookup %s') % a) p = cache[a] candidate=p.candidate origin = p.candidate.origins[0] arch=candidate.architecture if not candidate.uris : logger.warning(_("Sorry, cannot find an URI to download the debian package of `%s'.") % a) continue deb_uri = candidate.uri installed_version=p.installed.version candidate_version=p.candidate.version deb_path=str.split(deb_uri,'/') deb_path=str.join('/',deb_path[(deb_path.index('pool')):]) delta_uri_base=delta_uri_from_config(config, Origin=origin.origin, Label=origin.label, Site=origin.site, Archive=origin.archive, PackageName=p.name) if delta_uri_base == None: logger.warning(_("Sorry, no source is available to upgrade `%s'.") % a) continue if installed_version == candidate_version: logger.warning(_("Sorry, the package `%s' is already at its newest version.") % a) continue #delta name delta_name=p.shortname+'_'+version_mangle(installed_version)+\ '_'+ version_mangle(candidate_version)+'_'+\ arch+'.debdelta' delta_name = delta_base_name(p.shortname, installed_version, candidate_version, arch) uri=delta_uri_base+'/'+os.path.dirname(deb_path)+'/'+delta_name logger.info(a+' : '+uri) raise SystemExit(0) ##################################################### apt method ### still work in progress if os.path.dirname(sys.argv[0]) == '/usr/lib/apt/methods' : import os,sys, select, fcntl, apt, thread, threading, time apt_cache=apt.Cache() log=open('/tmp/log','a') log.write(' --- here we go\n') ( hi, ho , he) = os.popen3('/usr/lib/apt/methods/http.distrib','b',2) nthreads=3 class cheat_apt_gen: def __init__(self): self.uri=None self.filename=None self.acquire=False def process(self,cmd): if self.uri: self.filename=cmd[10:-1] log.write(' download %s for %s\n' % (repr(self.uri),repr(self.filename))) self.uri=None self.filename=None self.acquire=False return cmd elif self.acquire: self.uri=cmd[5:-1] return cmd elif cmd[:3] == '600' : self.acquire=True else: return cmd def copyin(): bufin='' while 1: #print ' o' s=os.read(ho.fileno(),1) bufin += s if log and bufin and (s == '' or s == '\n') : log.write( ' meth ' +repr(bufin)+'\n' ) bufin='' if s == '': thread.interrupt_main( ) global nthreads if nthreads: nthreads-=1 #log.write( ' in closed \n' ) #return os.write(1,s) def copyerr(): buferr='' while 1: s=os.read(he.fileno(),1) buferr += s if log and buferr and (s == '' or s == '\n') : log.write( ' err ' +repr(buferr)+'\n' ) buferr='' if s == '': thread.interrupt_main( ) global nthreads if nthreads: nthreads-=1 log.write( ' err closed \n' ) #return os.write(2,s) def copyout(): gen=cheat_apt_gen() bufout='' while 1: s=os.read(0,1) bufout += s if log and bufout and (s == '' or s == '\n') : log.write( ' apt ' +repr(bufout)+'\n' ) bufout=gen.process(bufout) bufout='' if s == '': thread.interrupt_main() global nthreads if nthreads: nthreads-=1 #log.write( ' out closed \n' ) #return os.write(hi.fileno(),(s)) tin=thread.start_new_thread(copyin,()) tout=thread.start_new_thread(copyout,()) terr=thread.start_new_thread(copyerr,()) while nthreads>0 : log.write( ' nthreads %d \n' % nthreads ) try: while nthreads>0 : time.sleep(1) except KeyboardInterrupt: pass raise SystemExit(0)