| 1 |
#!/usr/bin/python3 |
| 2 |
|
| 3 |
# Copyright (C) 2006-2020 Andrea Mennucci. |
| 4 |
# License: GNU Library General Public License, version 2 or later |
| 5 |
|
| 6 |
EMAIL="[email protected]" |
| 7 |
|
| 8 |
#### messages and translations |
| 9 |
# Early errors (such as errors in args parsing) are printed on stderr. |
| 10 |
# Subsequent errors,warnings,messages are printed on stdout,using the 'logging' facility |
| 11 |
# but the progress bars are on stderr |
| 12 |
|
| 13 |
# Messages printed at verbosity 0 or 1 are translated, higher verbosities are not |
| 14 |
# Errors are sometimes translated, sometimes not... |
| 15 |
# obscure error messages that would be printed only in very rare cases are not translated |
| 16 |
# e.g.: a malformed http header, a gnupg unexptected error, damaged input files... |
| 17 |
# more common error messages are translated, |
| 18 |
# e.g.: out of disk space while using debdelta-upgrade , file does not exist... |
| 19 |
|
| 20 |
import sys |
| 21 |
|
| 22 |
try: |
| 23 |
import gettext |
| 24 |
gettext.bindtextdomain('debdelta','/usr/share/locale') |
| 25 |
gettext.textdomain('debdelta') |
| 26 |
_ = gettext.gettext |
| 27 |
except Exception: |
| 28 |
a=sys.exc_info[1] |
| 29 |
sys.stderr.write('Could not initialize "gettext", translations will be unavailable\n'+str(a)) |
| 30 |
def _(x): return x |
| 31 |
|
| 32 |
doc={} |
| 33 |
doc['delta']=_("""\ |
| 34 |
Usage: debdelta [ option... ] fromfile tofile delta |
| 35 |
Computes the difference of two deb files, from fromfile to tofile, and writes it to delta |
| 36 |
|
| 37 |
Options: |
| 38 |
--signing-key KEY |
| 39 |
gnupg key used to sign the delta |
| 40 |
--no-md5 do not include MD5 info in delta |
| 41 |
--needsold create a delta that can only be used if the old deb is available |
| 42 |
-M Mb maximum memory to use (for 'bsdiff' or 'xdelta') |
| 43 |
--delta-algo ALGO |
| 44 |
use a specific backend for computing binary diffs |
| 45 |
""") |
| 46 |
|
| 47 |
|
| 48 |
doc['deltas']=_("""\ |
| 49 |
Usage: debdeltas [ option... ] [deb files and dirs, or 'Packages' files] |
| 50 |
Computes all missing deltas for deb files. |
| 51 |
It orders by version number and produce deltas to the newest version |
| 52 |
|
| 53 |
Options: |
| 54 |
--signing-key KEY |
| 55 |
key used to sign the deltas (using GnuPG) |
| 56 |
--dir DIR force saving of deltas in this DIR |
| 57 |
(otherwise they go in the dir of the newer deb_file) |
| 58 |
--old ARGS 'Packages' files containing list of old versions of debs |
| 59 |
--alt ARGS for any cmdline argument, search for debs also in this place |
| 60 |
-n N how many deltas to produce for each deb (default unlimited) |
| 61 |
--no-md5 do not include MD5 info in delta |
| 62 |
--needsold create a delta that can only be used if the old .deb is available |
| 63 |
--delta-algo ALGO |
| 64 |
use a specific backend for computing binary diffs; |
| 65 |
possible values are: xdelta xdelta-bzip xdelta3 bsdiff |
| 66 |
-M Mb maximum memory to use (for 'bsdiff' or 'xdelta') |
| 67 |
--clean-deltas delete deltas if newer deb is not in archive |
| 68 |
--cache cache parsed version of Packages.bz2 as Packages.debdelta_cache |
| 69 |
""") |
| 70 |
|
| 71 |
## implement : --search search in the directory of the above debs for older versions |
| 72 |
|
| 73 |
doc['patch']=_("""\ |
| 74 |
Usage: debpatch [ option... ] delta fromfile tofile |
| 75 |
Applies delta to fromfile and produces a reconstructed version of tofile. |
| 76 |
|
| 77 |
(When using 'debpatch' and the old .deb is not available, |
| 78 |
use the unpack directory, usually '/', for the fromfile.) |
| 79 |
|
| 80 |
Usage: debpatch --info delta |
| 81 |
Write info on delta. |
| 82 |
|
| 83 |
Options: |
| 84 |
--no-md5 do not verify MD5 (if found in info in delta) |
| 85 |
-A accept unsigned deltas |
| 86 |
--format FORMAT |
| 87 |
format of created deb |
| 88 |
""") |
| 89 |
|
| 90 |
doc['delta-upgrade']=_("""\ |
| 91 |
Usage: debdelta-upgrade [package names] |
| 92 |
Downloads all deltas and apply them to create the debs |
| 93 |
that are needed by 'apt-get upgrade'. |
| 94 |
|
| 95 |
Options: |
| 96 |
--dir DIR directory where to save results |
| 97 |
--deb-policy POLICY |
| 98 |
policy to decide which debs to download, |
| 99 |
-A accept unsigned deltas |
| 100 |
--format FORMAT |
| 101 |
format of created debs |
| 102 |
--timeout SECONDS |
| 103 |
adjust timeout for connections, default is |
| 104 |
15 seconds |
| 105 |
""") |
| 106 |
|
| 107 |
doc['patch-url']=_("""\ |
| 108 |
Usage: debpatch-url [package names] |
| 109 |
Show URL wherefrom to downloads all deltas that may be used to upgrade the given package names |
| 110 |
""") |
| 111 |
|
| 112 |
doc_common=_("""\ |
| 113 |
-v verbose (can be added multiple times) |
| 114 |
--no-act do not do that (whatever it is!) |
| 115 |
-d add extra debugging checks |
| 116 |
-k keep temporary files (use for debugging) |
| 117 |
--gpg-home HOME |
| 118 |
specify a different home for GPG |
| 119 |
|
| 120 |
See man page for more options and details. |
| 121 |
""") |
| 122 |
|
| 123 |
minigzip='/usr/lib/debdelta/minigzip' |
| 124 |
minibzip2='/usr/lib/debdelta/minibzip2' |
| 125 |
|
| 126 |
|
| 127 |
#################################################################### |
| 128 |
|
| 129 |
import os , tempfile , getopt , tarfile , shutil , time, traceback, subprocess, time, tarfile, stat, hashlib, random, gzip |
| 130 |
|
| 131 |
|
| 132 |
|
| 133 |
try: |
| 134 |
import debian.deb822 |
| 135 |
debian_deb822 = debian.deb822 |
| 136 |
except ImportError: |
| 137 |
debian_deb822 = None |
| 138 |
|
| 139 |
try: |
| 140 |
import cPickle as pickle |
| 141 |
except ImportError: |
| 142 |
import pickle |
| 143 |
|
| 144 |
from stat import ST_SIZE, ST_MTIME, ST_MODE, S_IMODE, S_IRUSR, S_IWUSR, S_IXUSR |
| 145 |
from os.path import abspath, expanduser |
| 146 |
from copy import copy |
| 147 |
|
| 148 |
from types import FunctionType, LambdaType |
| 149 |
|
| 150 |
import logging, logging.handlers |
| 151 |
if __name__ == '__main__': |
| 152 |
logging.basicConfig(format='%(message)s', stream=sys.stdout, level=logging.INFO) |
| 153 |
logger=logging.getLogger(__name__) |
| 154 |
|
| 155 |
if sys.version_info.major == 2: |
| 156 |
string_types = (str, unicode) # python2 |
| 157 |
# flags for subprocess.Popen |
| 158 |
Popen_args = {} |
| 159 |
# |
| 160 |
from ConfigParser import SafeConfigParser as ConfigParser |
| 161 |
from tarfile import filemode as stat_filemode |
| 162 |
# |
| 163 |
def str2bytes(a): |
| 164 |
return a |
| 165 |
def bytes2str(b): |
| 166 |
return b |
| 167 |
def is_bytes(b): |
| 168 |
return False |
| 169 |
# see https://docs.python.org/2/library/tarfile.html#unicode-issues |
| 170 |
if sys.getfilesystemencoding() != 'UTF-8': |
| 171 |
logger.warning('The filesystem encoding is %r, but this program assumes `UTF-8`.'%\ |
| 172 |
(sys.getfilesystemencoding(),)) |
| 173 |
else: |
| 174 |
string_types = (str, bytes) # python3 |
| 175 |
intern = sys.intern |
| 176 |
# flags for subprocess.Popen to open stdin,stdout as text streams |
| 177 |
Popen_args = {'text':True} |
| 178 |
# |
| 179 |
from configparser import ConfigParser |
| 180 |
from stat import filemode as stat_filemode |
| 181 |
# |
| 182 |
long = int |
| 183 |
def str2bytes(a): |
| 184 |
return a.encode('UTF-8',errors='surrogateescape') |
| 185 |
def bytes2str(b): |
| 186 |
return b.decode('UTF-8',errors='surrogateescape') |
| 187 |
def is_bytes(b): |
| 188 |
return isinstance(b,bytes) |
| 189 |
# see https://docs.python.org/3/library/tarfile.html#unicode-issues |
| 190 |
if sys.getfilesystemencoding() != 'utf-8': |
| 191 |
logger.warning('The filesystem encoding is %r, but this program assumes `utf-8`.'%\ |
| 192 |
(sys.getfilesystemencoding(),)) |
| 193 |
elif hasattr(sys,'getfilesystemencodeerrors') and \ |
| 194 |
sys.getfilesystemencodeerrors() != 'surrogateescape': |
| 195 |
logger.warning('The filesystem encoding is `utf-8` with %r, but this program uses `surrogateescape`.'%\ |
| 196 |
(sys.getfilesystemencodeerrors(),)) |
| 197 |
|
| 198 |
def get_termsize(): |
| 199 |
import termios, fcntl, struct |
| 200 |
s = struct.pack("HHHH", 0, 0, 0, 0) |
| 201 |
fd_stdout = sys.stdout.fileno() |
| 202 |
x = fcntl.ioctl(fd_stdout, termios.TIOCGWINSZ, s) |
| 203 |
return struct.unpack("HHHH", x)[:2] |
| 204 |
|
| 205 |
try: |
| 206 |
(terminalrows , terminalcolumns) = get_termsize() |
| 207 |
except: |
| 208 |
(terminalrows , terminalcolumns) = (None, None) #(24, 80) |
| 209 |
|
| 210 |
################################################# main program, read options |
| 211 |
|
| 212 |
#target of: maximum memory that bsdiff will use |
| 213 |
MAXMEMORY = 1024 * 1024 * 50 |
| 214 |
|
| 215 |
#this is +-10% , depending on the package size |
| 216 |
MAX_DELTA_PERCENT = 70 |
| 217 |
|
| 218 |
#min size of .deb that debdelta will consider |
| 219 |
#very small packages cannot be effectively delta-ed |
| 220 |
MIN_DEB_SIZE = 10 * 1024 |
| 221 |
|
| 222 |
|
| 223 |
N_DELTAS=None |
| 224 |
|
| 225 |
USE_DELTA_ALGO = 'bsdiff' |
| 226 |
|
| 227 |
TIMEOUT = 15 |
| 228 |
DEBUG = 0 |
| 229 |
VERBOSE = 0 |
| 230 |
KEEP = False |
| 231 |
INFO = False |
| 232 |
NEEDSOLD= False |
| 233 |
DIR = None |
| 234 |
ALT = [] |
| 235 |
OLD = [] |
| 236 |
ACT = True |
| 237 |
DO_MD5 = True |
| 238 |
DEB_POLICY = ['b','s','e','t','f','q'] |
| 239 |
DO_PROGRESS = terminalcolumns != None and __name__ == '__main__' |
| 240 |
|
| 241 |
#where/how debpatch/debdelta-upgrade will send forensic data, when patching fails |
| 242 |
#possible values: |
| 243 |
# False : do not send them |
| 244 |
# do : compute forensic but not send them, just list them |
| 245 |
# mail : automatically send by email to default address (uses `mutt`) |
| 246 |
# mail:user@domain : automatically send by email to address (uses `mutt`) |
| 247 |
# mutt : as above |
| 248 |
# mutt:user@domain : as above |
| 249 |
# http://domain/cgi : send them automatically thru a CGI script |
| 250 |
#Warning: the above is mostly TODO |
| 251 |
FORENSIC='do' |
| 252 |
|
| 253 |
FORENSIC_EMAIL='[email protected]' |
| 254 |
|
| 255 |
#directory tree where forensic info are stored by 'debdeltas' |
| 256 |
FORENSICDIR=None |
| 257 |
|
| 258 |
DEB_FORMAT='deb' |
| 259 |
DEB_FORMAT_LIST=('deb','unzipped','preunpacked') #not yet implemented on patching side : (,'piped') |
| 260 |
|
| 261 |
#for debdeltas: test patches internally |
| 262 |
DO_TEST = False |
| 263 |
|
| 264 |
DO_GPG = True #this is changed a few lines below |
| 265 |
GPG_SIGNING_KEY = None |
| 266 |
if os.getuid() == 0: |
| 267 |
GPG_HOME="/etc/debdelta/gnupg" |
| 268 |
else: |
| 269 |
GPG_HOME=None |
| 270 |
GPG_MASTER_PUB_KEYRING="/usr/share/keyrings/debian-debdelta-archive-keyring.gpg" |
| 271 |
|
| 272 |
GPG_CMD='gpg' |
| 273 |
if os.path.exists('/usr/bin/gpg2'): |
| 274 |
GPG_CMD='/usr/bin/gpg2' |
| 275 |
|
| 276 |
CLEAN_DELTAS = False |
| 277 |
CLEAN_DELTAS_MTIME = 2 # two days grace period |
| 278 |
CLEAN_ALT = False |
| 279 |
|
| 280 |
DO_PREDICTOR = False |
| 281 |
|
| 282 |
DO_CACHE = False #cache parsed version of Packages.bz2 as Packages.debdelta_cache |
| 283 |
|
| 284 |
#see README.features |
| 285 |
DISABLEABLE_FEATURES=['xz', 'lzma', 'xdelta3-fifo'] |
| 286 |
DISABLED_FEATURES=[] |
| 287 |
|
| 288 |
HTTP_USER_AGENT={'User-Agent': ('Debian debdelta-upgrade' ) } |
| 289 |
|
| 290 |
|
| 291 |
DPKG_MULTIARCH=( 0 == os.system('dpkg --assert-multi-arch 2> /dev/null') ) |
| 292 |
|
| 293 |
|
| 294 |
if __name__ != "__main__": |
| 295 |
action = None |
| 296 |
elif os.path.dirname(sys.argv[0]) == '/usr/lib/apt/methods' : |
| 297 |
action = None |
| 298 |
else: |
| 299 |
action=(os.path.basename(sys.argv[0]))[3:] |
| 300 |
actions = ('delta','patch','deltas','delta-upgrade', 'patch-url') |
| 301 |
# |
| 302 |
if action not in actions: |
| 303 |
sys.stderr.write('wrong filename: should be "deb" + '+repr(actions)) |
| 304 |
raise SystemExit(4) |
| 305 |
|
| 306 |
__doc__ = doc[action] + doc_common |
| 307 |
|
| 308 |
|
| 309 |
|
| 310 |
#GPG signatures are required for debdelta-upgrade and debpatch |
| 311 |
DO_GPG = action in ( "delta-upgrade", "patch") |
| 312 |
|
| 313 |
try: |
| 314 |
( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:n:A' , |
| 315 |
('help','info','needsold','dir=','no-act','alt=','old=','delta-algo=', |
| 316 |
'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug','forensicdir=','forensic=', |
| 317 |
'signing-key=', "accept-unsigned", "gpg-home=", "disable-feature=", "test", "format=", "cache", "timeout=") ) |
| 318 |
except getopt.GetoptError: |
| 319 |
a=sys.exc_info()[1] |
| 320 |
sys.stderr.write(sys.argv[0] +': '+ str(a)+'\n') |
| 321 |
raise SystemExit(3) |
| 322 |
|
| 323 |
for o , v in opts : |
| 324 |
if o == '-v' : |
| 325 |
VERBOSE += 1 |
| 326 |
if VERBOSE > 1 : |
| 327 |
logger.setLevel(logging.DEBUG) |
| 328 |
elif o == '-d' or o == '--debug' : |
| 329 |
DEBUG += 1 |
| 330 |
logger.setLevel(logging.DEBUG) |
| 331 |
elif o == '-k' : KEEP = True |
| 332 |
elif o == '--no-act': ACT=False |
| 333 |
elif o == '--no-md5': DO_MD5=False |
| 334 |
elif o == '--clean-deltas' : CLEAN_DELTAS = True |
| 335 |
elif o == '--clean-alt' : |
| 336 |
CLEAN_ALT = True |
| 337 |
sys.stderr.write(_('Warning, currently --clean-alt does nothing.')+'\n') |
| 338 |
elif o == '--needsold' : NEEDSOLD = True |
| 339 |
elif o == '--delta-algo': USE_DELTA_ALGO=v |
| 340 |
elif o == '--max-percent': MAX_DELTA_PERCENT=int(v) |
| 341 |
elif o == '--deb-policy' : DEB_POLICY = [j[0] for j in v.split(',') if j] |
| 342 |
elif o == '--timeout' : |
| 343 |
if int(v) < 0: |
| 344 |
sys.stderr.write(_('Error: "--timeout %s" is too small.') % v + '\n') |
| 345 |
raise SystemExit(3) |
| 346 |
TIMEOUT = int(v) |
| 347 |
elif o == '-M' : |
| 348 |
if int(v) <= 1: |
| 349 |
sys.stderr.write(_('Error: "-M %s" is too small.') % v + '\n') |
| 350 |
raise SystemExit(3) |
| 351 |
if int(v) <= 12: |
| 352 |
sys.stderr.write(_('Warning: "-M %s" is quite small.') % v + '\n') |
| 353 |
MAXMEMORY = 1024 * 1024 * int(v) |
| 354 |
elif o == '-n' : |
| 355 |
N_DELTAS = int(v) |
| 356 |
if N_DELTAS < 0: |
| 357 |
sys.stderr.write(_('Error: -n value is negative.')+'\n') |
| 358 |
raise SystemExit(3) |
| 359 |
elif o == '--test' and action == 'deltas' : DO_TEST = True |
| 360 |
elif o == '--info' and action == 'patch' : INFO = True |
| 361 |
elif o == '--dir' : |
| 362 |
DIR = abspath(expanduser(v)) |
| 363 |
if v[-2:] == '//': |
| 364 |
DIR += '//' |
| 365 |
if not os.path.isdir(DIR): |
| 366 |
sys.stderr.write( _("Error: argument of --dir is not a directory:") +' '+ DIR +'\n') |
| 367 |
raise SystemExit(3) |
| 368 |
|
| 369 |
elif o == '--forensicdir' : |
| 370 |
FORENSICDIR = abspath(expanduser(v)) |
| 371 |
if v[-2:] == '//': |
| 372 |
FORENSICDIR += '//' |
| 373 |
if not os.path.isdir(FORENSICDIR): |
| 374 |
sys.stderr.write( _("Error: argument of --forensicdir is not a directory:") +' '+ FORENSICDIR +'\n') |
| 375 |
raise SystemExit(3) |
| 376 |
|
| 377 |
elif o == '--forensic' : |
| 378 |
FORENSIC = v |
| 379 |
if FORENSIC.startswith('http'): |
| 380 |
try: |
| 381 |
import poster |
| 382 |
except: |
| 383 |
sys.stderr.write(_('To use the http forensic, you must install the package "python-poster".')+'\n') |
| 384 |
raise SystemExit(3) |
| 385 |
if FORENSIC[:4] in ('mutt','mail') and not os.path.exists('/usr/bin/mutt'): |
| 386 |
sys.stderr.write(_('To use this forensic, you must install the package "mutt".')+'\n') |
| 387 |
raise SystemExit(3) |
| 388 |
elif o == '--alt' : |
| 389 |
if not (os.path.isfile(v) or os.path.isdir(v)) : |
| 390 |
sys.stderr.write(_('Error: argument of --alt is not a directory or a regular file:')+' '+v +'\n') |
| 391 |
raise SystemExit(3) |
| 392 |
ALT.append(v) |
| 393 |
elif o == '--old' and action == 'deltas' : |
| 394 |
if not (os.path.isfile(v) or os.path.isdir(v)) : |
| 395 |
sys.stderr.write(_("Error: argument of --old is not a directory or a regular file:")+' '+v +'\n') |
| 396 |
raise SystemExit(3) |
| 397 |
OLD.append(v) |
| 398 |
elif o == '--help' or o == '-h': |
| 399 |
sys.stderr.write(__doc__) |
| 400 |
raise SystemExit(0) |
| 401 |
elif (o == '--disable-feature') and action in ("delta", "deltas"): |
| 402 |
DISABLED_FEATURES += v.split(',') |
| 403 |
elif (o == '--signing-key') and action in ("delta", "deltas"): |
| 404 |
GPG_SIGNING_KEY=v |
| 405 |
DO_GPG=True |
| 406 |
elif (o == '--accept-unsigned' or o == '-A') and action in ("delta-upgrade", "patch"): |
| 407 |
DO_GPG=False |
| 408 |
elif (o == '--gpg-home'): |
| 409 |
GPG_HOME=abspath(expanduser(v)) |
| 410 |
if not os.path.isdir(GPG_HOME): |
| 411 |
sys.stderr.write(_("Error: --gpg-home `%s' does not exist.") % GPG_HOME) |
| 412 |
raise SystemExit(3) |
| 413 |
elif o == '--format' : # maybe, and action in ("delta-upgrade", "patch"): |
| 414 |
if v not in DEB_FORMAT_LIST: |
| 415 |
sys.stderr.write(_("Error: output format `%s' is unknown.") % v + '\n') |
| 416 |
raise SystemExit(3) |
| 417 |
DEB_FORMAT=v |
| 418 |
elif o == '--cache': DO_CACHE=True |
| 419 |
else: |
| 420 |
sys.stderr.write(_("Error: option `%s' is unknown, try --help") % o + '\n') |
| 421 |
raise SystemExit(3) |
| 422 |
|
| 423 |
for i in DISABLED_FEATURES: |
| 424 |
if i not in DISABLEABLE_FEATURES: |
| 425 |
sys.stderr.write(_("Error: feature `%s' cannot be disabled.") % i + '\n') |
| 426 |
raise SystemExit(3) |
| 427 |
|
| 428 |
try: |
| 429 |
a = subprocess.Popen(b'grep bogomips /proc/cpuinfo',shell=True, stdout=subprocess.PIPE).stdout.read() |
| 430 |
a = bytes2str(a) |
| 431 |
BOGOMIPS=float(a.split(':')[-1]) |
| 432 |
except: |
| 433 |
if VERBOSE: |
| 434 |
logger.exception(' Warning, /proc not mounted, using bogus BOGOMIPS\n') |
| 435 |
BOGOMIPS=3000.0 |
| 436 |
|
| 437 |
SP=subprocess.Popen(['hostname','-f'], shell=False, stdout=subprocess.PIPE) |
| 438 |
HOSTID=hashlib.md5( SP.stdout.read() ).hexdigest() |
| 439 |
SP.wait() |
| 440 |
del SP |
| 441 |
|
| 442 |
TMPDIR = ( os.getenv('TMPDIR') or '/tmp' ).rstrip('/') |
| 443 |
|
| 444 |
if KEEP: |
| 445 |
def unlink(a): |
| 446 |
if VERBOSE > 2: logger.debug(' -k: would unlink '+repr(a)) |
| 447 |
def rmdir(a): |
| 448 |
if VERBOSE > 2: logger.debug(' -k: would rmdir '+repr(a)) |
| 449 |
def rmtree(a): |
| 450 |
if VERBOSE > 2: logger.debug(' -k: would rm -r '+repr(a)) |
| 451 |
else: |
| 452 |
def __wrap__(a,cmd): |
| 453 |
c=cmd.__name__+"("+a+")" |
| 454 |
if a[ : len(TMPDIR)+9 ] != TMPDIR+'/debdelta' : |
| 455 |
raise DebDeltaError('Internal error! refuse to '+c) |
| 456 |
try: |
| 457 |
cmd(a) |
| 458 |
except OSError: |
| 459 |
logger.exception(' Warning! when trying to '+repr(c)+' got OSError ') |
| 460 |
raise |
| 461 |
|
| 462 |
def unlink(a): |
| 463 |
return __wrap__(a,os.unlink) |
| 464 |
def rmdir(a): |
| 465 |
return __wrap__(a,os.rmdir) |
| 466 |
def rmtree(a): |
| 467 |
return __wrap__(a,shutil.rmtree) |
| 468 |
|
| 469 |
################################################################## check debian.deb822 |
| 470 |
|
| 471 |
_fake_Packages="""Package: coreutils |
| 472 |
Version: 8.30-3 |
| 473 |
Essential: yes |
| 474 |
Installed-Size: 15719 |
| 475 |
Maintainer: Michael Stone <[email protected]> |
| 476 |
Architecture: amd64 |
| 477 |
Pre-Depends: libacl1 (>= 2.2.23), libattr1 (>= 1:2.4.44), libc6 (>= 2.28), libselinux1 (>= 2.1.13) |
| 478 |
Description: GNU core utilities |
| 479 |
Multi-Arch: foreign |
| 480 |
Homepage: http://gnu.org/software/coreutils |
| 481 |
Description-md5: d0d975dec3625409d24be1238cede238 |
| 482 |
Tag: admin::configuring, implemented-in::c, interface::commandline, |
| 483 |
role::program, scope::utility, suite::gnu, works-with::file |
| 484 |
Section: utils |
| 485 |
Priority: required |
| 486 |
Filename: pool/main/c/coreutils/coreutils_8.30-3_amd64.deb |
| 487 |
Size: 2707932 |
| 488 |
MD5sum: 4a658a39c2810530df6642d22f65ddd1 |
| 489 |
SHA256: ae6e5cd6e9aaf74d66edded3931a7a6c916625b8b890379189c75574f6856bf4 |
| 490 |
|
| 491 |
Package: bash |
| 492 |
Version: 5.0-4 |
| 493 |
Essential: yes |
| 494 |
Installed-Size: 6439 |
| 495 |
Maintainer: Matthias Klose <[email protected]> |
| 496 |
Architecture: amd64 |
| 497 |
Replaces: bash-completion (<< 20060301-0), bash-doc (<= 2.05-1) |
| 498 |
Depends: base-files (>= 2.1.12), debianutils (>= 2.15) |
| 499 |
Pre-Depends: libc6 (>= 2.15), libtinfo6 (>= 6) |
| 500 |
Recommends: bash-completion (>= 20060301-0) |
| 501 |
Suggests: bash-doc |
| 502 |
Conflicts: bash-completion (<< 20060301-0) |
| 503 |
Description: GNU Bourne Again SHell |
| 504 |
Multi-Arch: foreign |
| 505 |
Homepage: http://tiswww.case.edu/php/chet/bash/bashtop.html |
| 506 |
Description-md5: 3522aa7b4374048d6450e348a5bb45d9 |
| 507 |
Tag: admin::TODO, devel::TODO, devel::interpreter, implemented-in::c, |
| 508 |
interface::shell, interface::text-mode, role::program, |
| 509 |
scope::application, suite::gnu, uitoolkit::ncurses |
| 510 |
Section: shells |
| 511 |
Priority: required |
| 512 |
Filename: pool/main/b/bash/bash_5.0-4_amd64.deb |
| 513 |
Size: 1377592 |
| 514 |
MD5sum: 6f118d96e29a5208da9033bf0b2c772c |
| 515 |
SHA256: d07f3528c615545bb182269362926bc171adab565dade3e8c35cb3bae8ad9e5e |
| 516 |
|
| 517 |
""" |
| 518 |
|
| 519 |
def test_deb822(debian_deb822, shared_storage): |
| 520 |
F = tempfile.NamedTemporaryFile() |
| 521 |
open(F.name,'w').write(_fake_Packages) |
| 522 |
I=debian_deb822.Packages.iter_paragraphs(sequence=open(F.name,'rb'),shared_storage=shared_storage) |
| 523 |
c=next(I) |
| 524 |
assert c['Package'] == 'coreutils' |
| 525 |
assert c['Size'] == '2707932' |
| 526 |
b=next(I) |
| 527 |
assert b['Package'] == 'bash' |
| 528 |
assert c['Size'] == '2707932' |
| 529 |
b['foo']='bar' |
| 530 |
c['foo']='tock' |
| 531 |
assert b['foo']=='bar' |
| 532 |
# Deb822 only accepts strings |
| 533 |
#b['foo']=(1,None) |
| 534 |
|
| 535 |
|
| 536 |
if debian_deb822 is not None: |
| 537 |
try: |
| 538 |
test_deb822(debian_deb822,False) |
| 539 |
test_deb822(debian_deb822,True) |
| 540 |
except: |
| 541 |
logger.exception("While testing debian.822") |
| 542 |
|
| 543 |
#################################################### various routines |
| 544 |
|
| 545 |
def my_popen_read(cmd, ** args): |
| 546 |
" opens a pipe (for text mode, pass `**Popen_args` )" |
| 547 |
return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=open(os.devnull), |
| 548 |
close_fds=True, **args).stdout |
| 549 |
|
| 550 |
def freespace(w): |
| 551 |
assert(os.path.exists(w)) |
| 552 |
try: |
| 553 |
a=os.statvfs(w) |
| 554 |
freespace= long(a[0]) * long(a[4]) |
| 555 |
except Exception: |
| 556 |
logger.exception('Statvfs error on %r', w) |
| 557 |
freespace=None |
| 558 |
return freespace |
| 559 |
|
| 560 |
dpkg_keeps_controls = ( |
| 561 |
'conffiles','config','list','md5sums','postinst', |
| 562 |
'postrm','preinst','prerm','shlibs','templates') |
| 563 |
|
| 564 |
def parse_dist(f,d): |
| 565 |
a=f.readline() |
| 566 |
p={} |
| 567 |
while a: |
| 568 |
if a[:4] in ('Pack','Vers','Arch','Stat','Inst','File','Size','MD5s'): |
| 569 |
a=de_n(a) |
| 570 |
i=a.index(':') |
| 571 |
assert(a[i:i+2] == ': ') |
| 572 |
p[a[:i]] = a[i+2:] |
| 573 |
elif a == '\n': |
| 574 |
d[p['Package']] = p |
| 575 |
p={} |
| 576 |
a=f.readline() |
| 577 |
|
| 578 |
|
| 579 |
def scan_control(p,params=None,prefix=None,info=None): |
| 580 |
if prefix == None: |
| 581 |
prefix = '' |
| 582 |
else: |
| 583 |
prefix += '/' |
| 584 |
a=p.readline() |
| 585 |
while a: |
| 586 |
a=de_n(a) |
| 587 |
if a[:4] in ('Pack','Vers','Arch','Stat','Inst','File'): |
| 588 |
if info != None : |
| 589 |
info.append(prefix+a) |
| 590 |
if params != None: |
| 591 |
i=a.index(':') |
| 592 |
assert(a[i:i+2] == ': ') |
| 593 |
params[prefix+a[:i]] = a[i+2:] |
| 594 |
a=p.readline() |
| 595 |
|
| 596 |
def append_info(delta,info): |
| 597 |
"insert into the delta (that is an AR archive) the info file, as a first element, possibly removing a previous occurrence" |
| 598 |
#new style : special info file |
| 599 |
TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR)) |
| 600 |
infofile=open(TD+'/info','w') |
| 601 |
for i in info: |
| 602 |
infofile.write(i+'\n') |
| 603 |
infofile.close() |
| 604 |
if DO_GPG: |
| 605 |
r=_compute_hashes_(TD+"/info") |
| 606 |
else: |
| 607 |
r=None |
| 608 |
system(['ar','rSi','0',delta, 'info'], TD) |
| 609 |
rmtree(TD) |
| 610 |
return r |
| 611 |
|
| 612 |
def de_n(a): |
| 613 |
assert isinstance(a, str) |
| 614 |
if a and a[-1:] == '\n' : |
| 615 |
a = a[:-1] |
| 616 |
return a |
| 617 |
|
| 618 |
def de_n_b(a): |
| 619 |
assert sys.version_info.major == 2 or isinstance(a, bytes) |
| 620 |
if a and a[-1:] == b'\n' : |
| 621 |
a = a[:-1] |
| 622 |
return a |
| 623 |
|
| 624 |
def de_bar(a): |
| 625 |
assert isinstance(a, str) |
| 626 |
if a and a[:2] == './' : |
| 627 |
a=a[2:] |
| 628 |
elif a == '/.' : |
| 629 |
a='' |
| 630 |
elif a and a[0] == '/' : |
| 631 |
a=a[1:] |
| 632 |
return a |
| 633 |
|
| 634 |
def de_bar_b(a): |
| 635 |
assert sys.version_info.major == 2 or isinstance(a, bytes) |
| 636 |
if a and a[:2] == b'./' : |
| 637 |
a=a[2:] |
| 638 |
elif a == b'/.' : |
| 639 |
a='' |
| 640 |
elif a and a[0] == b'/' : |
| 641 |
a=a[1:] |
| 642 |
return a |
| 643 |
|
| 644 |
|
| 645 |
def list_ar(f): |
| 646 |
assert(os.path.exists(f)) |
| 647 |
ar_list = [] |
| 648 |
p=my_popen_read(['ar','t', f]) |
| 649 |
while 1: |
| 650 |
a=p.readline() |
| 651 |
if not a : break |
| 652 |
a=de_n_b(a) |
| 653 |
ar_list.append(a) |
| 654 |
p.close() |
| 655 |
return ar_list |
| 656 |
|
| 657 |
def list_tar(f): |
| 658 |
" returns content of the tar file `f`, as a list of `bytes`" |
| 659 |
assert(os.path.exists(f)) |
| 660 |
ar_list = [] |
| 661 |
p=my_popen_read(['tar','t',f]) |
| 662 |
while 1: |
| 663 |
a=p.readline() |
| 664 |
if not a : break |
| 665 |
a=de_n_b(a) |
| 666 |
ar_list.append(a) |
| 667 |
p.close() |
| 668 |
return ar_list |
| 669 |
|
| 670 |
def untar_control_in_deb(ar_ls): |
| 671 |
" ar_ls is usually list_ar(n) ; it returns (c , b, z) \ |
| 672 |
where c 'name of control' , b 'option for tar' , z 'unzip command' " |
| 673 |
c = None |
| 674 |
for a in ar_ls : |
| 675 |
if a[:12]==b'control.tar.': |
| 676 |
c=a |
| 677 |
assert c |
| 678 |
# command for tar |
| 679 |
bdb={b'.gz':'-z', b'.bz2':'-j', b'.lzma':'--lzma', b'.xz':'-J'} |
| 680 |
zdb={b'.gz' : ['gzip','-cd'], b'.bz2': ['bzip2','-cd'], |
| 681 |
b'.lzma':['unlzma','-c'], b'.xz': ['unxz','-c'] } |
| 682 |
return c, bdb.get(c[11:]), zdb.get(c[11:]) |
| 683 |
|
| 684 |
class cache_sequence(object): |
| 685 |
cache_filename=None |
| 686 |
cache_file=None |
| 687 |
cache=None |
| 688 |
exists=None |
| 689 |
broken=None |
| 690 |
suffix='.debdelta_cache' |
| 691 |
def __init__(self, filename, cache_filename=None): |
| 692 |
"""manages a cache file that store a sequence of python object; |
| 693 |
'filename' is a reference file that is related to the data being cached, |
| 694 |
and is used to create the name of the cache file, unless 'cache_filename' |
| 695 |
is provided |
| 696 |
""" |
| 697 |
self.cache_filename=os.path.splitext(filename)[0]+self.suffix \ |
| 698 |
if cache_filename==None else cache_filename |
| 699 |
self.cache_file=None |
| 700 |
self.cache=None |
| 701 |
self.broken=None |
| 702 |
self.exists=os.path.isfile(self.cache_filename) and \ |
| 703 |
os.path.getmtime(filename) <= os.path.getmtime(self.cache_filename) |
| 704 |
|
| 705 |
def __iter__(self): |
| 706 |
assert self.exists and not self.cache |
| 707 |
self.cache=gzip.GzipFile(self.cache_filename) |
| 708 |
return self |
| 709 |
|
| 710 |
def __next__(self): |
| 711 |
assert self.cache |
| 712 |
try: |
| 713 |
return pickle.load(self.cache) |
| 714 |
except EOFError: |
| 715 |
self.cache=None |
| 716 |
raise StopIteration |
| 717 |
except Exception: |
| 718 |
logger.exception('Cache file is broken, deleting %r' % self.cache_filename) |
| 719 |
if ACT: os.unlink(self.cache_filename) |
| 720 |
self.cache=None |
| 721 |
self.broken=True |
| 722 |
# do not kill program |
| 723 |
raise StopIteration |
| 724 |
|
| 725 |
def next(self): |
| 726 |
return self.__next__() |
| 727 |
|
| 728 |
def __prepare_for_write__(self): |
| 729 |
if not self.cache: |
| 730 |
if DEBUG: logger.debug('Creating cache file : %r', self.cache_filename) |
| 731 |
self.cache_file=open(self.cache_filename+'~tmp~', 'wb') |
| 732 |
self.cache=gzip.GzipFile(filename='', fileobj=self.cache_file, mode='wb') # 'mtime=0' needs python2.7 or higher |
| 733 |
|
| 734 |
def close(self): |
| 735 |
try: |
| 736 |
if self.cache: |
| 737 |
self.cache.close() |
| 738 |
self.cache=None |
| 739 |
if self.cache_file: |
| 740 |
self.cache_file.close() |
| 741 |
self.cache_file=None |
| 742 |
os.rename(self.cache_filename+'~tmp~', self.cache_filename) |
| 743 |
except Exception: |
| 744 |
logger.exception('Cannot close the cache file %r' % self.cache_filename) |
| 745 |
self.broken=True |
| 746 |
|
| 747 |
__del__=close |
| 748 |
|
| 749 |
def write(self,s): |
| 750 |
" write one object" |
| 751 |
assert not self.exists |
| 752 |
if self.broken: |
| 753 |
return |
| 754 |
self.__prepare_for_write__() |
| 755 |
try: |
| 756 |
self.cache.write(pickle.dumps(s)) |
| 757 |
except Exception: |
| 758 |
logger.exception('Cannot write to cache file, deleting %r', self.cache_filename) |
| 759 |
self.close() |
| 760 |
if ACT: os.unlink(self.cache_filename) |
| 761 |
self.broken=True |
| 762 |
|
| 763 |
class cache_same_dict(cache_sequence): |
| 764 |
"cache occurrences of a dict that uses always the same keys; omit the keys to optimize" |
| 765 |
def __init__(self, filename, keys): |
| 766 |
super(cache_same_dict, self).__init__(filename) |
| 767 |
self.keys=keys |
| 768 |
|
| 769 |
def write(self, s): |
| 770 |
n=[s[k] for k in self.keys] |
| 771 |
super(cache_same_dict, self).write(n) |
| 772 |
|
| 773 |
def __next__(self): |
| 774 |
n=(super(cache_same_dict, self)).__next__() |
| 775 |
return dict(map(lambda x,y: (x,y) , self.keys, n)) # dict comprehension may be used instead |
| 776 |
|
| 777 |
##################################################################### |
| 778 |
|
| 779 |
ALLOWED = '<>()[]{}.,;:!_-+/ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' |
| 780 |
ALLOWED_SET = set(ord(j) for j in ALLOWED) |
| 781 |
NUMBERS_SET = set(ord(j) for j in '0123456789') |
| 782 |
|
| 783 |
ECHO_TEST = r"""c='\0151\0141' |
| 784 |
E='echo -ne' |
| 785 |
if test c`$E 'i'"$c" `o = ciiao ; then |
| 786 |
: |
| 787 |
else |
| 788 |
E='echo -n' |
| 789 |
if test c`$E 'i'"$c" `o = ciiao ; then |
| 790 |
: |
| 791 |
else |
| 792 |
#echo WARNING : BUILTIN echo DOES NOT WORK OK |
| 793 |
E='/bin/echo -ne' |
| 794 |
test c`$E 'i'"$c" `o = ciiao |
| 795 |
fi |
| 796 |
fi |
| 797 |
""" |
| 798 |
|
| 799 |
def prepare_for_echo_py2_(s): |
| 800 |
assert ( type (s) in string_types ) |
| 801 |
r='' |
| 802 |
shortquoted=False |
| 803 |
for a in s: |
| 804 |
if a in ALLOWED : |
| 805 |
r += a |
| 806 |
shortquoted = False |
| 807 |
elif a in '0123456789' : |
| 808 |
if shortquoted : |
| 809 |
a = "\\" + ('000' +oct(ord(a)))[-4:] |
| 810 |
shortquoted = False |
| 811 |
r += a |
| 812 |
elif a == 0: |
| 813 |
r += '\\0' |
| 814 |
shortquoted = True |
| 815 |
else: |
| 816 |
a = "\\" + oct(ord(a)) |
| 817 |
r += a |
| 818 |
shortquoted = len(a) < 5 |
| 819 |
return r |
| 820 |
|
| 821 |
def prepare_for_echo_py3_(s): |
| 822 |
" `s` may be a `str` or `bytes` ; returns `str`" |
| 823 |
#assert isinstance(s,bytes) |
| 824 |
if isinstance(s,str): |
| 825 |
s = str2bytes(s) |
| 826 |
r='' |
| 827 |
O = lambda x : oct(x)[2:] |
| 828 |
shortquoted=False |
| 829 |
for a in s: |
| 830 |
#j = ord(a) |
| 831 |
if a in ALLOWED_SET : |
| 832 |
r += chr(a) |
| 833 |
shortquoted = False |
| 834 |
elif a in NUMBERS_SET : |
| 835 |
if shortquoted : |
| 836 |
a = "\\" + ('000' + O(a) )[-4:] |
| 837 |
r += a |
| 838 |
shortquoted = False |
| 839 |
else: |
| 840 |
r += chr(a) |
| 841 |
elif a == 0: |
| 842 |
r += '\\0' |
| 843 |
shortquoted = True |
| 844 |
else: |
| 845 |
a = "\\0" + O(a) |
| 846 |
r += a |
| 847 |
shortquoted = len(a) < 5 |
| 848 |
return r |
| 849 |
|
| 850 |
if sys.version_info.major == 2: |
| 851 |
prepare_for_echo__ = prepare_for_echo_py2_ |
| 852 |
else: |
| 853 |
prepare_for_echo__ = prepare_for_echo_py3_ |
| 854 |
|
| 855 |
|
| 856 |
def apply_prepare_for_echo(shell,repres): |
| 857 |
a=ECHO_TEST + " $E '" + repres + "' \n exit " |
| 858 |
p = subprocess.Popen([shell], stdin=subprocess.PIPE, stdout=subprocess.PIPE, |
| 859 |
close_fds=True) #, ** Popen_args) |
| 860 |
(i, o) = (p.stdout, p.stdin) |
| 861 |
o.write(str2bytes(a)) |
| 862 |
o.close() |
| 863 |
a=i.read() |
| 864 |
i.close() |
| 865 |
return a |
| 866 |
|
| 867 |
#ack! I wanted to use 'dash' as preferred shell, but bug 379227 stopped me |
| 868 |
SHELL = '/bin/bash' |
| 869 |
#check my code |
| 870 |
s=b'\x00'+b'1ciao88\n77\r566'+b'\x00'+b'99\n' |
| 871 |
r=prepare_for_echo__(s) |
| 872 |
a=apply_prepare_for_echo(SHELL,r) |
| 873 |
if a != s : |
| 874 |
sys.stderr.write( 'string='+repr(s)+'\n') |
| 875 |
sys.stderr.write( 'repres='+repr(r)+'\n') |
| 876 |
sys.stderr.write( 'shell='+repr(SHELL)+'\n') |
| 877 |
sys.stderr.write( 'output='+repr(a)+'\n') |
| 878 |
sys.stderr.write( 'Errror in prepare_for_echo.\n') |
| 879 |
raise SystemExit(4) |
| 880 |
del r,s,a |
| 881 |
|
| 882 |
### |
| 883 |
|
| 884 |
def prepare_for_echo(s): |
| 885 |
r=prepare_for_echo__(s) |
| 886 |
if DEBUG > 2 : |
| 887 |
a=apply_prepare_for_echo(SHELL,r) |
| 888 |
if a != s: |
| 889 |
z = 'Error in prepare_for_echo()\n' |
| 890 |
z += 'string='+repr(s)+'\n' |
| 891 |
z += 'repres='+repr(r)+'\n' |
| 892 |
z += 'shell='+repr(SHELL)+'\n' |
| 893 |
z += 'output='+repr(a)+'\n' |
| 894 |
raise DebDeltaError(z, exitcode=4) |
| 895 |
return r |
| 896 |
|
| 897 |
##################################################################### |
| 898 |
|
| 899 |
|
| 900 |
def version_mangle(v): |
| 901 |
if ':' in v : |
| 902 |
return str.join('%3a',v.split(':')) |
| 903 |
else: |
| 904 |
return v |
| 905 |
|
| 906 |
def version_demangle(v): |
| 907 |
if '%' in v : |
| 908 |
return str.join(':',v.split('%3a')) |
| 909 |
else: |
| 910 |
return v |
| 911 |
|
| 912 |
def delta_base_name(pkg, old_ver, new_ver, arch, ext='.debdelta'): |
| 913 |
assert(':' not in pkg) |
| 914 |
return pkg + \ |
| 915 |
'_' + version_mangle(old_ver) + \ |
| 916 |
'_' + version_mangle(new_ver) + \ |
| 917 |
'_' + arch + ext |
| 918 |
|
| 919 |
def tempo(): |
| 920 |
TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR)) |
| 921 |
for i in 'OLD','NEW','PATCH' : |
| 922 |
os.mkdir(TD+'/'+i) |
| 923 |
if VERBOSE > 2 or KEEP : logger.debug('Temporary in '+TD) |
| 924 |
return TD |
| 925 |
|
| 926 |
########## |
| 927 |
|
| 928 |
|
| 929 |
class DebDeltaError(Exception): #should derive from (Exception):http://docs.python.org/dev/whatsnew/pep-352.html |
| 930 |
# Subclasses that define an __init__ must call Exception.__init__ |
| 931 |
# or define self.args. Otherwise, str() will fail. |
| 932 |
def __init__(self,s,retriable=False,exitcode=None,logs=None): |
| 933 |
assert(type(s) in string_types) |
| 934 |
self.retriable = retriable |
| 935 |
if retriable: |
| 936 |
self.args=(s + ' (retriable) ',) |
| 937 |
else: |
| 938 |
self.args=(s + ' (non retriable) ',) |
| 939 |
if exitcode == None: |
| 940 |
if retriable: |
| 941 |
exitcode = 1 |
| 942 |
else: |
| 943 |
exitcode = 2 |
| 944 |
self.exitcode=exitcode |
| 945 |
self.logs=logs |
| 946 |
|
| 947 |
def die(s): |
| 948 |
#if s : sys.stderr.write(s+'\n') |
| 949 |
assert type(s) in string_types |
| 950 |
raise DebDeltaError(s) |
| 951 |
|
| 952 |
class PopenPipe(object): |
| 953 |
def __init__(self, a, **dictargs): |
| 954 |
#special code for pipes http://docs.python.org/library/subprocess.html#replacing-shell-pipeline |
| 955 |
old_stdin=dictargs.pop('stdin',None) |
| 956 |
old_stdout=dictargs.pop('stdout',None) |
| 957 |
self.pros=[] |
| 958 |
while '|' in a: |
| 959 |
l=a.index('|') ; a1=a[:l] ; a=a[l+1:] |
| 960 |
p=subprocess.Popen(args=a1, stdin=old_stdin, stdout=subprocess.PIPE, **dictargs) |
| 961 |
self.pros.append( p ) |
| 962 |
old_stdin=p.stdout |
| 963 |
final_pro= subprocess.Popen(args=a, stdin=old_stdin, stdout=old_stdout, **dictargs) |
| 964 |
for p in self.pros: |
| 965 |
p.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits. |
| 966 |
self.pros.append(final_pro) |
| 967 |
# |
| 968 |
self.returncode=None |
| 969 |
self.stdout=self.pros[-1].stdout |
| 970 |
self.stdin=self.pros[0].stdin |
| 971 |
self.read=getattr(self.pros[-1].stdout,'read',None) |
| 972 |
self.write=getattr(self.pros[0].stdin,'write',None) |
| 973 |
def __set_returncode(self): |
| 974 |
if any([ (p.returncode<0) for p in self.pros]): |
| 975 |
self.returncode = min([ p.returncode for p in self.pros]) |
| 976 |
else: |
| 977 |
self.returncode = max([ p.returncode for p in self.pros]) |
| 978 |
return self.returncode |
| 979 |
def poll(self): |
| 980 |
for p in self.pros: |
| 981 |
p.poll() |
| 982 |
return self.__set_returncode() |
| 983 |
def wait(self): |
| 984 |
for p in self.pros: |
| 985 |
p.wait() |
| 986 |
return self.__set_returncode() |
| 987 |
def terminate(self): |
| 988 |
self.pros[0].poll() |
| 989 |
if self.pros[0].returncode is None: |
| 990 |
self.pros[0].terminate() |
| 991 |
def close(self): |
| 992 |
self.terminate() |
| 993 |
self.wait() |
| 994 |
|
| 995 |
|
| 996 |
def system(a,TD,saveargs=None,ignore_output=False,return_output=False): |
| 997 |
"a must be a tuple, TD the temporary directory ; if return_output , it will return (stdout,stderr,exitcode) regardless" |
| 998 |
assert type(a) in (list, tuple) |
| 999 |
# mvo: compat with python2.5 where tuple does not have index |
| 1000 |
a = list(a) |
| 1001 |
if VERBOSE and TD[: (len(TMPDIR)+9) ] != TMPDIR+'/debdelta' : |
| 1002 |
logger.debug(' Warning "system()" in '+TD+' for '+repr(a)) |
| 1003 |
(temp_fd, temp_name) = tempfile.mkstemp(prefix="debdelta_out_system") |
| 1004 |
(temp_err_fd, temp_err_name) = tempfile.mkstemp(prefix="debdelta_err_system") |
| 1005 |
|
| 1006 |
MPP=PopenPipe(a,stdin=open(os.devnull), stdout=temp_fd, stderr=temp_err_fd, cwd=TD, close_fds=True) |
| 1007 |
ret=MPP.wait() |
| 1008 |
os.close(temp_fd) |
| 1009 |
os.close(temp_err_fd) |
| 1010 |
if VERBOSE > 3 : logger.debug(' system(%r)=%d', a, ret) |
| 1011 |
if ignore_output==False and (os.stat(temp_name)[ST_SIZE] > 0 or os.stat(temp_err_name)[ST_SIZE] > 0 ): |
| 1012 |
logger.debug(' command "%s" returned %d and produced output as follows' % (a,ret)) |
| 1013 |
for i in open(temp_name): |
| 1014 |
logger.debug('stdout: '+repr(i)) |
| 1015 |
for i in open(temp_err_name): |
| 1016 |
logger.debug('stderr: '+repr(i)) |
| 1017 |
if return_output: |
| 1018 |
return temp_name, temp_err_name, ret |
| 1019 |
os.unlink(temp_err_name) |
| 1020 |
os.unlink(temp_name) |
| 1021 |
if ret == 0: |
| 1022 |
return |
| 1023 |
elif ret != 1 or a[0] != 'xdelta' : |
| 1024 |
s='Error , non zero return status '+str(ret)+' for command "'+repr(a)+'"' |
| 1025 |
try: |
| 1026 |
if DEBUG and saveargs: |
| 1027 |
T=abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR)) |
| 1028 |
open(T+'/command','w').write(repr(a)) |
| 1029 |
for l in saveargs: |
| 1030 |
if l[0] != '/': |
| 1031 |
l = TD+'/'+l |
| 1032 |
if os.path.exists(l): |
| 1033 |
shutil.copy2(l,T) |
| 1034 |
s=s+'\n saved argument '+l+' in '+T |
| 1035 |
else: |
| 1036 |
s=s+'\n did not find argument '+l |
| 1037 |
except OSError: |
| 1038 |
o=sys.exc_info()[1] |
| 1039 |
s=s+'\n (there was an additional OSError "'+str(o)+'" when trying to save arguments)' |
| 1040 |
die(s) |
| 1041 |
|
| 1042 |
def check_deb(f, die_ = True): |
| 1043 |
if not os.path.exists(f) : |
| 1044 |
die(_("Error: the file `%s' does not exist.") % f) |
| 1045 |
if not os.path.isfile(f) : |
| 1046 |
die(_("Error: `%s' is not a regular file.") % f) |
| 1047 |
p=open(f,'rb') |
| 1048 |
if p.read(21) != b"!<arch>\ndebian-binary" : |
| 1049 |
if die_: |
| 1050 |
die(_("Error: `%s' does not seem to be a Debian package.") % f) |
| 1051 |
else: |
| 1052 |
return False |
| 1053 |
p.close() |
| 1054 |
return True |
| 1055 |
|
| 1056 |
def check_is_delta(f): |
| 1057 |
if not os.path.exists(f) : |
| 1058 |
die(_("Error: the file `%s' does not exist.") % f) |
| 1059 |
if not os.path.isfile(f) : |
| 1060 |
die(_("Error: `%s' is not a regular file.") % f) |
| 1061 |
p=open(f,'rb') |
| 1062 |
if p.read(8) != b"!<arch>\n" : |
| 1063 |
die(_("Error: `%s' does not seem to be a Debian delta.") % f) |
| 1064 |
p.close() |
| 1065 |
|
| 1066 |
def puke(s,e=None): |
| 1067 |
""" write informations on stderr; if DEBUG also prints traceback; |
| 1068 |
if DEBUG>1 and `e` is given, rereaise the exception""" |
| 1069 |
(typ, value, trace)=sys.exc_info() |
| 1070 |
if e == None or len(str(e)) < 2: |
| 1071 |
logger.error(str(s)+' : '+str(e)+' '+str(typ)+' '+str(value)) |
| 1072 |
else: |
| 1073 |
logger.error(str(s)+' : '+str(e)) |
| 1074 |
if DEBUG > 1 and e is not None and trace: |
| 1075 |
raise |
| 1076 |
elif DEBUG and trace and traceback.print_tb(trace): |
| 1077 |
logger.error( traceback.print_tb(trace)) |
| 1078 |
|
| 1079 |
|
| 1080 |
################################################################### GPG |
| 1081 |
|
| 1082 |
def gpg_base_commandline(): |
| 1083 |
if GPG_HOME: |
| 1084 |
GPG_BASE_CMD_LINE=[GPG_CMD,"--homedir",GPG_HOME] |
| 1085 |
else: |
| 1086 |
GPG_BASE_CMD_LINE=[GPG_CMD,"--keyring",GPG_MASTER_PUB_KEYRING] |
| 1087 |
|
| 1088 |
if VERBOSE < 1 : |
| 1089 |
GPG_BASE_CMD_LINE+=['--quiet'] |
| 1090 |
|
| 1091 |
return GPG_BASE_CMD_LINE |
| 1092 |
|
| 1093 |
def gpg_sign_command(): |
| 1094 |
return gpg_base_commandline()+["--batch","--armor","--clearsign","--default-key",GPG_SIGNING_KEY,"--sign"] |
| 1095 |
|
| 1096 |
def compute_md5_up_to_len(o, length): |
| 1097 |
"hash initial part of a file using MD5. 'o' may be a string (in which case the file is opened) or a file type; returns MD5 and bytes effectively read" |
| 1098 |
assert type(length) in (int,long) and length >= 0 |
| 1099 |
if type(o) in string_types: |
| 1100 |
o = open(o,'rb') |
| 1101 |
m=hashlib.md5() |
| 1102 |
a=True |
| 1103 |
l=length |
| 1104 |
while l>0 and a: |
| 1105 |
a=o.read(min(1024,l)) |
| 1106 |
l -= len(a) |
| 1107 |
m.update(a) |
| 1108 |
return m.hexdigest(), length-l |
| 1109 |
|
| 1110 |
|
| 1111 |
def compute_md5_len(o): |
| 1112 |
"hash the file using MD5. 'o' may be a string (in which case the file is opened) or a file type; returns MD5 and length" |
| 1113 |
if type(o) in string_types: |
| 1114 |
o = open(o,'rb') |
| 1115 |
m=hashlib.md5() |
| 1116 |
a=o.read(1024) |
| 1117 |
l=0 |
| 1118 |
while a: |
| 1119 |
l+=len(a) |
| 1120 |
m.update(a) |
| 1121 |
a=o.read(1024) |
| 1122 |
return m.hexdigest(), l |
| 1123 |
|
| 1124 |
def compute_md5(o): |
| 1125 |
"hash the file using MD5. 'o' may be a string (in which case the file is opened) or a file type; returns MD5 (as a string of hexes)" |
| 1126 |
return compute_md5_len(o)[0] |
| 1127 |
|
| 1128 |
def _compute_hashes_(na): |
| 1129 |
"hash the file" |
| 1130 |
o = open(na,'rb') |
| 1131 |
m=hashlib.md5() |
| 1132 |
s=hashlib.sha1() |
| 1133 |
a=o.read(1024) |
| 1134 |
while a: |
| 1135 |
m.update(a) |
| 1136 |
s.update(a) |
| 1137 |
a=o.read(1024) |
| 1138 |
r = ( m.hexdigest(), s.hexdigest(), os.stat(na)[ST_SIZE]) |
| 1139 |
return r |
| 1140 |
|
| 1141 |
def _compute_hashes_db_(li,DIR): |
| 1142 |
db={} |
| 1143 |
for na in li: |
| 1144 |
db[na] = _compute_hashes_(DIR+'/'+na) |
| 1145 |
return db |
| 1146 |
|
| 1147 |
def verify_signature(signature, DIR): |
| 1148 |
a=b"-----BEGIN PGP SIGNED MESSAGE-----\n" |
| 1149 |
if open(signature,'rb').read(len(a)) != a: |
| 1150 |
return ('BAD_FORMAT',signature) |
| 1151 |
|
| 1152 |
role=os.path.basename(signature) |
| 1153 |
assert role[:4] == "_gpg" |
| 1154 |
role=role[4:] |
| 1155 |
|
| 1156 |
(temp_fd, temp_name) = tempfile.mkstemp(prefix="debdelta_gpg_verified") |
| 1157 |
#(read_end, write_end) = os.pipe() |
| 1158 |
p=subprocess.Popen(gpg_base_commandline() + ['--batch','--status-fd',"2",'--output',"-",signature], |
| 1159 |
stdout=subprocess.PIPE, stderr=temp_fd, stdin=open(os.devnull), close_fds=True, |
| 1160 |
**Popen_args) |
| 1161 |
r=_verify_signature_no_gpg(p.stdout, DIR, role) |
| 1162 |
p.wait() |
| 1163 |
|
| 1164 |
os.close(temp_fd) |
| 1165 |
|
| 1166 |
if VERBOSE > 2 or p.returncode: |
| 1167 |
for j in open(temp_name): |
| 1168 |
logger.debug(' GPG> '+j.rstrip()) |
| 1169 |
|
| 1170 |
os.unlink(temp_name) |
| 1171 |
|
| 1172 |
if p.returncode: |
| 1173 |
return ('GPG_VERIFY_FAILED',signature) |
| 1174 |
|
| 1175 |
return r |
| 1176 |
|
| 1177 |
def _verify_signature_no_gpg(signature, DIR, role): |
| 1178 |
#list stuff, skipping signatures |
| 1179 |
dir_list = [a for a in os.listdir(DIR) if a[:4] != '_gpg'] |
| 1180 |
#compute signatures |
| 1181 |
hashes = _compute_hashes_db_(dir_list, DIR) |
| 1182 |
#scan hashes file (GPG already verified) |
| 1183 |
if type(signature) in string_types: |
| 1184 |
f=open(signature) |
| 1185 |
elif hasattr(signature,'readline'): |
| 1186 |
f=signature |
| 1187 |
else: raise AssertionError |
| 1188 |
a=f.readline() |
| 1189 |
if a != "Version: 4\n": |
| 1190 |
return ("UNSUPPORTED_VERSION",a) |
| 1191 |
a=f.readline() |
| 1192 |
while a: |
| 1193 |
if a[:5] == "Role:": |
| 1194 |
if a[5:].strip() != role : |
| 1195 |
return ("ROLE_MISMATCH",a) |
| 1196 |
a=f.readline() |
| 1197 |
elif a[:6] == "Files:" : |
| 1198 |
#parse files |
| 1199 |
a=f.readline() |
| 1200 |
while a and a[0] in ( '\t' , ' ') : |
| 1201 |
a=a.rstrip('\n') |
| 1202 |
a=a.lstrip() |
| 1203 |
a=a.split(' ') |
| 1204 |
if VERBOSE > 3 : logger.debug(' checking hashes '+repr(a)) |
| 1205 |
(md5,sha1,le,na)=a |
| 1206 |
if na not in dir_list: |
| 1207 |
return ('ABSENT',na) |
| 1208 |
(cmd5,csha1,cle)=hashes[na] |
| 1209 |
if int(le) != cle: |
| 1210 |
return ('SIZE',na) |
| 1211 |
# check hashes |
| 1212 |
if md5 != cmd5 : |
| 1213 |
return ('MD5',na) |
| 1214 |
if sha1 != csha1 : |
| 1215 |
return ('SHA1',na) |
| 1216 |
dir_list.remove(na) |
| 1217 |
a=f.readline() |
| 1218 |
elif VERBOSE > 2 : |
| 1219 |
logger.debug(' signature header ignored: '+ a) |
| 1220 |
a=f.readline() |
| 1221 |
else: |
| 1222 |
a=f.readline() |
| 1223 |
#end parsing |
| 1224 |
if dir_list: |
| 1225 |
return ("UNCHECKED",dir_list) |
| 1226 |
return True |
| 1227 |
|
| 1228 |
def _write_signature(db,filename,role): |
| 1229 |
"starting from a database of hashes, see _compute_hashes_, it writes a signature file" |
| 1230 |
f=open(filename,mode='w') |
| 1231 |
##this is the format of dpkg-sig, but is redundant, since the "date" and "signer" |
| 1232 |
##are already available thru the gpg signature |
| 1233 |
#f.write("Version: 4\nSigner: \nDate: %s\nRole: %s\nFiles: \n" % (time.ctime(),role)) |
| 1234 |
##and actually dpkg-sig will validate also a simpler file, so, lets save a few bytes |
| 1235 |
f.write("Version: 4\nRole: %s\nFiles:\n" % (role,)) |
| 1236 |
for a in db: |
| 1237 |
(m,s,l) = db[a] |
| 1238 |
f.write('\t'+m+" "+s+" "+str(l)+" "+a+"\n") |
| 1239 |
f.close() |
| 1240 |
|
| 1241 |
def sign_delta(delta, db, role="maker"): |
| 1242 |
TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR)) |
| 1243 |
try: |
| 1244 |
_write_signature(db,TD+'/_temp',role) |
| 1245 |
p=subprocess.Popen(gpg_sign_command() +['--output',TD+'/_gpg'+role,TD+'/_temp'], stdin=open(os.devnull), close_fds=True) |
| 1246 |
p.wait() |
| 1247 |
if p.returncode==0: |
| 1248 |
r=system(("ar","qS",delta,TD+"/_gpg"+role),TD) |
| 1249 |
except: |
| 1250 |
rmtree(TD) |
| 1251 |
raise |
| 1252 |
rmtree(TD) |
| 1253 |
if p.returncode: |
| 1254 |
raise DebDeltaError('GnuPG fails to sign') |
| 1255 |
if r: |
| 1256 |
raise DebDeltaError('ar fails to add the signature') |
| 1257 |
|
| 1258 |
|
| 1259 |
|
| 1260 |
|
| 1261 |
#################################################################### apply patch |
| 1262 |
|
| 1263 |
########### info auxiliary routines |
| 1264 |
|
| 1265 |
def _info_patch_unzip_(TD): |
| 1266 |
"unzip info and patch.sh" |
| 1267 |
if os.path.exists(TD+'PATCH/info.gz'): |
| 1268 |
system(('gunzip','PATCH/info.gz'),TD) |
| 1269 |
if os.path.exists(TD+'PATCH/patch.sh.gz'): |
| 1270 |
system(('gunzip','PATCH/patch.sh.gz'),TD) |
| 1271 |
elif os.path.exists(TD+'PATCH/patch.sh.bz2'): |
| 1272 |
system(('bunzip2','PATCH/patch.sh.bz2'),TD) |
| 1273 |
elif os.path.exists(TD+'PATCH/patch.sh.lzma'): |
| 1274 |
if not os.path.exists('/usr/bin/unlzma'): |
| 1275 |
raise DebDeltaError('This patch needs lzma. Please install the Debian package "lzma".',retriable=True) |
| 1276 |
system(('unlzma','PATCH/patch.sh.lzma'),TD) |
| 1277 |
elif os.path.exists(TD+'PATCH/patch.sh.xz'): |
| 1278 |
if not os.path.exists('/usr/bin/unxz'): |
| 1279 |
raise DebDeltaError('This patch needs xz. Please install the Debian package "xz-utils".',retriable=True) |
| 1280 |
system(('unxz','PATCH/patch.sh.xz'),TD) |
| 1281 |
|
| 1282 |
def get_info_slow(delta,T=None): |
| 1283 |
if T: |
| 1284 |
TD=T |
| 1285 |
else: |
| 1286 |
TD=tempo() |
| 1287 |
if TD[-1] != '/': |
| 1288 |
TD = TD + '/' |
| 1289 |
delta=abspath(expanduser(delta)) |
| 1290 |
system(('ar','x',delta,'info','info.gz','patch.sh','patch.sh.gz','patch.sh.bz2','patch.sh.lzma','patch.sh.xz'), |
| 1291 |
TD+'/PATCH', ignore_output=True) |
| 1292 |
_info_patch_unzip_(TD) |
| 1293 |
info = _scan_delta_info_(TD) |
| 1294 |
if T == None: |
| 1295 |
rmtree(TD) |
| 1296 |
return info |
| 1297 |
|
| 1298 |
def get_info_fast(delta): |
| 1299 |
f=open(delta,'rb') |
| 1300 |
s=f.readline() |
| 1301 |
if b"!<arch>\n" != s : |
| 1302 |
raise DebDeltaError('This is not a delta file: '+delta) |
| 1303 |
s = f.read(60) |
| 1304 |
if len(s) != 60 : |
| 1305 |
logger.warning('(Warning, cannot get info from truncated: '+delta+' )') |
| 1306 |
return None |
| 1307 |
if s[:4] != b'info': |
| 1308 |
#old style debdelta, with info in patch.sh |
| 1309 |
if VERBOSE > 1 : logger.warning(' (Warning, cannot get info from old style: '+delta+' )') |
| 1310 |
return None |
| 1311 |
##parse ar segment |
| 1312 |
## see /usr/include/ar.h |
| 1313 |
if s[-2:] != b'`\n' : |
| 1314 |
logger.warning('(Warning, cannot get info from '+delta+' , format not known)') |
| 1315 |
return None |
| 1316 |
l=int(s[ -12:-2 ]) |
| 1317 |
s=bytes2str(f.read(l)) |
| 1318 |
if len(s) != l : |
| 1319 |
logger.warning('(Warning, cannot get info from truncated: '+delta+' )') |
| 1320 |
return None |
| 1321 |
info= s.split('\n') |
| 1322 |
f.close() |
| 1323 |
return info |
| 1324 |
|
| 1325 |
def get_info(delta,TD=None): |
| 1326 |
info=get_info_fast(delta) |
| 1327 |
if info == None: |
| 1328 |
info=get_info_slow(delta,TD) |
| 1329 |
return info |
| 1330 |
|
| 1331 |
def _scan_delta_info_(TD): |
| 1332 |
info=[] |
| 1333 |
if os.path.isfile(TD+'PATCH/info'): |
| 1334 |
#new style debdelta, with info file |
| 1335 |
p=open(TD+'PATCH/info') |
| 1336 |
info=p.read().split('\n') |
| 1337 |
p.close() |
| 1338 |
if info[-1] == '': info.pop() |
| 1339 |
else: |
| 1340 |
#old style debdelta, with info in patch.sh |
| 1341 |
p=open(TD+'PATCH/patch.sh') |
| 1342 |
s=p.readline() |
| 1343 |
s=p.readline() |
| 1344 |
while s: |
| 1345 |
if s[0] == '#' : |
| 1346 |
s=de_n(s) |
| 1347 |
info.append(s[1:]) |
| 1348 |
s=p.readline() |
| 1349 |
p.close() |
| 1350 |
return info |
| 1351 |
|
| 1352 |
def info_2_db(info): |
| 1353 |
params={} |
| 1354 |
for s in info: |
| 1355 |
if ':' in s: |
| 1356 |
i=s.index(':') |
| 1357 |
params[s[:i]] = s[i+2:] |
| 1358 |
elif s: |
| 1359 |
params[s] = True |
| 1360 |
return params |
| 1361 |
|
| 1362 |
########### other auxiliary routines |
| 1363 |
|
| 1364 |
def patch_check_tmp_space(params,olddeb): |
| 1365 |
if type(params) != dict: |
| 1366 |
params=info_2_db(params) |
| 1367 |
if 'NEW/Installed-Size' not in params or 'OLD/Installed-Size' not in params: |
| 1368 |
logger.warning('(Warning... Installed size unknown...)') |
| 1369 |
return True |
| 1370 |
free=freespace(TMPDIR) |
| 1371 |
if free == None : return True |
| 1372 |
free = free / 1024 |
| 1373 |
if os.path.isdir(olddeb): |
| 1374 |
instsize=int(params['NEW/Installed-Size']) |
| 1375 |
#the last action of the script is to gzip the data.tar, so |
| 1376 |
if 'NEW/Size' in params : |
| 1377 |
instsize += int(params['NEW/Size']) / 1024 |
| 1378 |
else: |
| 1379 |
instsize = instsize * 1.8 |
| 1380 |
else: |
| 1381 |
instsize=int(params['NEW/Installed-Size'])+int(params['OLD/Installed-Size']) |
| 1382 |
instsize += 2**13 |
| 1383 |
if free < instsize : |
| 1384 |
return _('not enough disk space (%(free)dkB) in directory %(dir)s for applying delta (needs %(size)dkB)') % \ |
| 1385 |
{'free' : int(free), 'dir' : TMPDIR, 'size' : instsize} |
| 1386 |
else: |
| 1387 |
return True |
| 1388 |
|
| 1389 |
|
| 1390 |
def scan_diversions(): |
| 1391 |
f=open('/var/lib/dpkg/diversions') |
| 1392 |
d={} |
| 1393 |
|
| 1394 |
a=1 |
| 1395 |
while 1: |
| 1396 |
a=f.readline() |
| 1397 |
if not a: break |
| 1398 |
a=de_n(a) |
| 1399 |
b=de_n(f.readline()) |
| 1400 |
p=de_n(f.readline()) |
| 1401 |
d[a]=(b,p) |
| 1402 |
f.close() |
| 1403 |
return d |
| 1404 |
|
| 1405 |
###################################################### debforensic extract |
| 1406 |
|
| 1407 |
#in base-passwd 3.5.11 |
| 1408 |
#/usr/share/base-passwd/passwd.master |
| 1409 |
base_passwd="""root::0:0:root:/root:/bin/bash |
| 1410 |
daemon:*:1:1:daemon:/usr/sbin:/bin/sh |
| 1411 |
bin:*:2:2:bin:/bin:/bin/sh |
| 1412 |
sys:*:3:3:sys:/dev:/bin/sh |
| 1413 |
sync:*:4:65534:sync:/bin:/bin/sync |
| 1414 |
games:*:5:60:games:/usr/games:/bin/sh |
| 1415 |
man:*:6:12:man:/var/cache/man:/bin/sh |
| 1416 |
lp:*:7:7:lp:/var/spool/lpd:/bin/sh |
| 1417 |
mail:*:8:8:mail:/var/mail:/bin/sh |
| 1418 |
news:*:9:9:news:/var/spool/news:/bin/sh |
| 1419 |
uucp:*:10:10:uucp:/var/spool/uucp:/bin/sh |
| 1420 |
proxy:*:13:13:proxy:/bin:/bin/sh |
| 1421 |
www-data:*:33:33:www-data:/var/www:/bin/sh |
| 1422 |
backup:*:34:34:backup:/var/backups:/bin/sh |
| 1423 |
list:*:38:38:Mailing List Manager:/var/list:/bin/sh |
| 1424 |
irc:*:39:39:ircd:/var/run/ircd:/bin/sh |
| 1425 |
gnats:*:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/bin/sh |
| 1426 |
nobody:*:65534:65534:nobody:/nonexistent:/bin/sh""" |
| 1427 |
base_passwd_db={} |
| 1428 |
base_passwd_anti_db={} |
| 1429 |
for a in base_passwd.split('\n'): |
| 1430 |
a=a.split(':') |
| 1431 |
base_passwd_db[a[0]]=int(a[2]) |
| 1432 |
base_passwd_anti_db[int(a[2])]=a[0] |
| 1433 |
|
| 1434 |
base_group="""root:*:0: |
| 1435 |
daemon:*:1: |
| 1436 |
bin:*:2: |
| 1437 |
sys:*:3: |
| 1438 |
adm:*:4: |
| 1439 |
tty:*:5: |
| 1440 |
disk:*:6: |
| 1441 |
lp:*:7: |
| 1442 |
mail:*:8: |
| 1443 |
news:*:9: |
| 1444 |
uucp:*:10: |
| 1445 |
man:*:12: |
| 1446 |
proxy:*:13: |
| 1447 |
kmem:*:15: |
| 1448 |
dialout:*:20: |
| 1449 |
fax:*:21: |
| 1450 |
voice:*:22: |
| 1451 |
cdrom:*:24: |
| 1452 |
floppy:*:25: |
| 1453 |
tape:*:26: |
| 1454 |
sudo:*:27: |
| 1455 |
audio:*:29: |
| 1456 |
dip:*:30: |
| 1457 |
www-data:*:33: |
| 1458 |
backup:*:34: |
| 1459 |
operator:*:37: |
| 1460 |
list:*:38: |
| 1461 |
irc:*:39: |
| 1462 |
src:*:40: |
| 1463 |
gnats:*:41: |
| 1464 |
shadow:*:42: |
| 1465 |
utmp:*:43: |
| 1466 |
video:*:44: |
| 1467 |
sasl:*:45: |
| 1468 |
plugdev:*:46: |
| 1469 |
staff:*:50: |
| 1470 |
games:*:60: |
| 1471 |
users:*:100: |
| 1472 |
nogroup:*:65534:""" |
| 1473 |
|
| 1474 |
base_group_db={} |
| 1475 |
base_group_anti_db={} |
| 1476 |
for a in base_group.split('\n'): |
| 1477 |
a=a.split(':') |
| 1478 |
base_group_db[a[0]]=int(a[2]) |
| 1479 |
base_group_anti_db[int(a[2])]=a[0] |
| 1480 |
|
| 1481 |
# all code following return name,mode,tartype,uid,gid,uname,gname |
| 1482 |
|
| 1483 |
#adapted from tarfile.py, a Python module |
| 1484 |
def stat_to_tar(name): |
| 1485 |
"returns name,mode,tartype,uid,gid,uname,gname,data" |
| 1486 |
statres = os.lstat(name) |
| 1487 |
stmd = statres.st_mode |
| 1488 |
data = None |
| 1489 |
if stat.S_ISREG(stmd): |
| 1490 |
tartype = tarfile.REGTYPE |
| 1491 |
# here ideally we should SHA1 the file ; |
| 1492 |
# but this is done elsewhere for performance, |
| 1493 |
# and to have multi_hash in the future |
| 1494 |
elif stat.S_ISDIR(stmd): |
| 1495 |
tartype = tarfile.DIRTYPE |
| 1496 |
elif stat.S_ISFIFO(stmd): |
| 1497 |
tartype = tarfile.FIFOTYPE |
| 1498 |
elif stat.S_ISLNK(stmd): |
| 1499 |
tartype = tarfile.SYMTYPE |
| 1500 |
data = os.readlink(name) |
| 1501 |
elif stat.S_ISCHR(stmd): |
| 1502 |
tartype = tarfile.CHRTYPE |
| 1503 |
elif stat.S_ISBLK(stmd): |
| 1504 |
tartype = tarfile.BLKTYPE |
| 1505 |
elif stat.S_ISSOCK(stmd): |
| 1506 |
tartype = 'SOCKET' #SOCKETs are not supported in tar files |
| 1507 |
else: raise TypeError |
| 1508 |
|
| 1509 |
if tartype in (tarfile.CHRTYPE, tarfile.BLKTYPE): |
| 1510 |
data = str(os.major(statres.st_rdev))+' '+str( os.minor(statres.st_rdev)) |
| 1511 |
|
| 1512 |
uid,gid = statres.st_uid, statres.st_gid |
| 1513 |
|
| 1514 |
if uid in base_passwd_anti_db : |
| 1515 |
uname = base_passwd_anti_db[uid] |
| 1516 |
else: |
| 1517 |
import pwd |
| 1518 |
try: |
| 1519 |
uname = pwd.getpwuid(uid)[0] |
| 1520 |
except KeyError: |
| 1521 |
uname = None |
| 1522 |
|
| 1523 |
if gid in base_group_anti_db : |
| 1524 |
gname = base_group_anti_db[gid] |
| 1525 |
else: |
| 1526 |
import grp |
| 1527 |
try: |
| 1528 |
gname = grp.getgrgid(gid)[0] |
| 1529 |
except KeyError: |
| 1530 |
gname = None |
| 1531 |
|
| 1532 |
#07777 is used in tarfile.TarInfo.tobuf |
| 1533 |
return name.lstrip('/'), stmd & 0o7777, tartype, uid, gid, uname, gname, data |
| 1534 |
|
| 1535 |
|
| 1536 |
def tarinfo_to_ls(tartype,tarmode): |
| 1537 |
"returns a string -rwxrwxrwx such as what ls -l prints " |
| 1538 |
if ord(tartype) == 0 : |
| 1539 |
a='_' |
| 1540 |
else: |
| 1541 |
if tartype >= b'0' and tartype <= b'6' : |
| 1542 |
a="-hlcbdp"[ord(tartype) - ord('0')] |
| 1543 |
else: |
| 1544 |
a='?' |
| 1545 |
return a + stat_filemode(tarmode)[1:] |
| 1546 |
|
| 1547 |
|
| 1548 |
def sha1_hash_file(f): |
| 1549 |
"returns hex digest of file" |
| 1550 |
s=hashlib.sha1() |
| 1551 |
if type(f) in string_types: |
| 1552 |
f=open(f,'rb') |
| 1553 |
a=f.read(1024) |
| 1554 |
while a: |
| 1555 |
s.update(a) |
| 1556 |
a=f.read(1024) |
| 1557 |
f.close() |
| 1558 |
return s.hexdigest() |
| 1559 |
|
| 1560 |
def forensics_rfc(o,db,bytar,controlfiles,files,conffiles,diverted=[],diversions={},localepurged=[],prelink_u_failed=[]): |
| 1561 |
" this is invoked by do_patch_() as well as do_delta_() ; in the former case, by_tar=False" |
| 1562 |
assert type(diversions) == dict |
| 1563 |
if type(db) == dict: |
| 1564 |
for a in sorted(db.keys()): |
| 1565 |
assert isinstance(a,str) |
| 1566 |
if a[:3] == 'OLD': |
| 1567 |
o.write(a[4:]+': '+db[a]+'\n') |
| 1568 |
else: |
| 1569 |
for a in sorted(db): |
| 1570 |
assert isinstance(a,str) |
| 1571 |
if a[:3] == 'OLD': |
| 1572 |
o.write(a[4:]+'\n') |
| 1573 |
if diverted: |
| 1574 |
o.write("Diversions:\n") |
| 1575 |
for a in sorted(diverted): |
| 1576 |
b,p = diversions[a] |
| 1577 |
o.write(" From: "+a+'\n') |
| 1578 |
o.write(" To: "+b+'\n') |
| 1579 |
o.write(" By: "+p+'\n') |
| 1580 |
if conffiles: |
| 1581 |
o.write("Conffiles:\n") |
| 1582 |
for a in sorted(conffiles): |
| 1583 |
o.write(' '+a+'\n') |
| 1584 |
for L,N in ((controlfiles,"Control"),(files,"Files")): |
| 1585 |
o.write(N+":\n") |
| 1586 |
for l in sorted(L): |
| 1587 |
if bytar: |
| 1588 |
name,mode,tartype,uid,gid,uname,gname,data=l |
| 1589 |
tmpcopy=None |
| 1590 |
divert=None |
| 1591 |
else: |
| 1592 |
name,divert,tmpcopy=l |
| 1593 |
if os.path.exists(divert): |
| 1594 |
fullname,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(divert) |
| 1595 |
else: |
| 1596 |
fullname,mode,tartype,uid,gid,uname,gname,data='',0,'?',0,0,'?','?','?' |
| 1597 |
if tartype == tarfile.REGTYPE: |
| 1598 |
if tmpcopy and os.path.exists(tmpcopy): |
| 1599 |
data=sha1_hash_file(tmpcopy) |
| 1600 |
elif os.path.exists(divert): |
| 1601 |
data=sha1_hash_file(divert) |
| 1602 |
if name in ('.', '/', './', '/.') and tartype == tarfile.DIRTYPE: #skip root |
| 1603 |
continue |
| 1604 |
if uname == None: uname=str(uid) |
| 1605 |
if gname == None: gname=str(gid) |
| 1606 |
name=de_bar(name) |
| 1607 |
o.write(' '+tarinfo_to_ls(tartype,mode)+" "+uname+' '+gname) |
| 1608 |
if N == "Files" and tartype == tarfile.REGTYPE and name in conffiles: |
| 1609 |
o.write(" [conffile]") |
| 1610 |
if N == "Files" and tartype == tarfile.REGTYPE and name in localepurged: |
| 1611 |
o.write(" [localpurged]") |
| 1612 |
if N == "Files" and tartype == tarfile.REGTYPE and name in prelink_u_failed: |
| 1613 |
o.write(" [prelink-u failed]") |
| 1614 |
if divert and not os.path.exists(divert): |
| 1615 |
o.write(" [missing file %r]" % divert) |
| 1616 |
if tmpcopy: |
| 1617 |
o.write(" [prelink-u]") |
| 1618 |
o.write("\n "+name+"\n") |
| 1619 |
if data!=None: |
| 1620 |
o.write(" "+data+"\n") |
| 1621 |
else: |
| 1622 |
o.write(" \n") |
| 1623 |
|
| 1624 |
def tar_those(f): |
| 1625 |
" tar multiple files in one tar (all in the same base directory!). Note that f may be a list of lists or strings or mixed." |
| 1626 |
temptar=tempfile.mktemp(suffix='.tgz') |
| 1627 |
tar=tarfile.open(name=temptar,mode='w:gz') |
| 1628 |
for z in f: |
| 1629 |
if type(z) in (list,tuple): |
| 1630 |
for j in z: |
| 1631 |
tar.add(j,arcname=os.path.basename(j)) |
| 1632 |
elif type(z) in string_types: |
| 1633 |
tar.add(z,arcname=os.path.basename(z)) |
| 1634 |
else: DebDeltaError(' internal error m92ksy') |
| 1635 |
tar.close() |
| 1636 |
return temptar |
| 1637 |
|
| 1638 |
def forensic_send(f,forensic=FORENSIC): |
| 1639 |
" note that f must be a list of lists (or None)" |
| 1640 |
assert type(f) == list |
| 1641 |
if forensic is False or forensic == 'False' or forensic == '': |
| 1642 |
if f: |
| 1643 |
logger.error(_('(Faulty delta. Please consider retrying with the option "--forensic=do" ).')+'\n') |
| 1644 |
return |
| 1645 |
if not f: |
| 1646 |
return |
| 1647 |
if all([(z == None) for z in f]): |
| 1648 |
logger.warning('Sorry, no forensic logs were generated') |
| 1649 |
return |
| 1650 |
if forensic[:4] in ('mutt','mail') or forensic[:7] == 'icedove' or forensic[:10]=='thunderbird': |
| 1651 |
email=FORENSIC_EMAIL |
| 1652 |
if ':' in forensic: |
| 1653 |
a=forensic.find(':') |
| 1654 |
email == forensic[a:] |
| 1655 |
forensic=forensic[:a] |
| 1656 |
logger.warning(_("There were faulty deltas.")+' '+_("Now invoking the mail sender to send the logs.")) |
| 1657 |
if forensic in ('mutt','mail'): |
| 1658 |
if sys.version_info.major == 2: |
| 1659 |
raw_input( _('(hit any key)') ) |
| 1660 |
else: |
| 1661 |
input( _('(hit any key)') ) |
| 1662 |
args=[] |
| 1663 |
for z in f: |
| 1664 |
if z: |
| 1665 |
for j in z: |
| 1666 |
args+=['-a',j] |
| 1667 |
subprocess.call(['mutt',email,'-s','delta_failures']+args) |
| 1668 |
else: |
| 1669 |
temptar=tempfile.mktemp(suffix='.tgz') |
| 1670 |
tar=tarfile.open(name=temptar,mode='w:gz') |
| 1671 |
for z in f: |
| 1672 |
if z: |
| 1673 |
for j in z: |
| 1674 |
tar.add(j,arcname=os.path.basename(j)) |
| 1675 |
tar.close() |
| 1676 |
args="to=%s,subject=delta_failures,attachment='file:///%s'" % (email,temptar) |
| 1677 |
subprocess.call([forensic,'-compose',args]) |
| 1678 |
return |
| 1679 |
elif forensic.startswith('http'): |
| 1680 |
logger.warning(_("There were faulty deltas.")+' '+_('Sending logs to server.')) |
| 1681 |
temptar=tempfile.mktemp(suffix='.tgz') |
| 1682 |
tar=tarfile.open(name=temptar,mode='w:gz') |
| 1683 |
for z in f: |
| 1684 |
if z: |
| 1685 |
for j in z: |
| 1686 |
tar.add(j,arcname=os.path.basename(j)) |
| 1687 |
tar.close() |
| 1688 |
#http://atlee.ca/software/poster |
| 1689 |
## FIXME this will not work with python3 |
| 1690 |
import urllib, urllib2, httplib, poster |
| 1691 |
poster.streaminghttp.register_openers() |
| 1692 |
datagen, headers = poster.encode.multipart_encode({'auth_userid':'debdelta','auth_password':'slartibartfast',"thefile": open(temptar, "rb")}) |
| 1693 |
# Create the Request object |
| 1694 |
request = urllib2.Request("http://debdelta.debian.net:7890/receive", datagen, headers) |
| 1695 |
# Actually do the request, and get the response |
| 1696 |
logger.info(' '+_('Server answers:'),repr(urllib2.urlopen(request).read())) |
| 1697 |
return |
| 1698 |
else: |
| 1699 |
logger.warning(_('Faulty delta. Please send by email to %s the following files:\n') % EMAIL) |
| 1700 |
for z in f: |
| 1701 |
if z: |
| 1702 |
logger.warning(' '+str.join(' ',z)+'\n') |
| 1703 |
return |
| 1704 |
logger.warning(_('(Faulty delta. Please consider retrying with the option "--forensic=do" ).')+'\n') |
| 1705 |
|
| 1706 |
def elf_info(f): |
| 1707 |
"returns (is_elf, ei_class, ei_data, ei_osabi, e_type)" |
| 1708 |
import struct |
| 1709 |
elfheader=open(f,'rb').read(32) |
| 1710 |
if len(elfheader) == 32: |
| 1711 |
#parse as specified in /usr/include/elf.h from libelf-dev |
| 1712 |
EI_CLASS={1:'ELFCLASS32', |
| 1713 |
2:'ELFCLASS64'} |
| 1714 |
EI_DATA={1:'ELFDATA2LSB', # 2's complement, little endian |
| 1715 |
2:'ELFDATA2MSB'} # 2's complement, big endian |
| 1716 |
EI_OSABI={0:'ELFOSABI_SYSV',# UNIX System V ABI |
| 1717 |
1:'ELFOSABI_HPUX', |
| 1718 |
2:'ELFOSABI_NETBSD', |
| 1719 |
3:'ELFOSABI_LINUX', |
| 1720 |
#fixme insert other values |
| 1721 |
9:'ELFOSABI_FREEBSD', |
| 1722 |
12:'ELFOSABI_OPENBSD', |
| 1723 |
97:'ELFOSABI_ARM'} |
| 1724 |
#fixme what is ET_LOOS , ET_HIOS , ET_LOPROC, ET_HIPROC ?? |
| 1725 |
ET_TYPE={1:'ET_REL', #Relocatable file |
| 1726 |
2:'ET_EXEC', #Executable file |
| 1727 |
3:'ET_DYN', #Shared object file |
| 1728 |
4:'ET_CORE'} #Core file |
| 1729 |
|
| 1730 |
ei_magic, ei_class, ei_data, ei_version, ei_osabi, ei_abiversion = \ |
| 1731 |
struct.unpack_from('4sBBBBB',elfheader) |
| 1732 |
e_type, e_machine, e_version = struct.unpack_from('HHI',elfheader,16) |
| 1733 |
#FIXME I think I am getting ei_osabi wrong.. it is always 0 |
| 1734 |
is_elf = b'\x7fELF' == ei_magic |
| 1735 |
#and ei_class in (1,2) and \ |
| 1736 |
# ei_version == 1 and \ |
| 1737 |
# ei_data in (1,2) and e_type>0 and e_machine>0 and e_version>0 |
| 1738 |
return is_elf, EI_CLASS.get(ei_class), \ |
| 1739 |
EI_DATA.get(ei_data), EI_OSABI.get(ei_osabi), ET_TYPE.get(e_type) |
| 1740 |
#, e_machine, e_version |
| 1741 |
else: |
| 1742 |
return False, 0, 0, 0, 0 |
| 1743 |
|
| 1744 |
def parse_prelink_conf(): |
| 1745 |
" fixme , currently unused and incomplete " |
| 1746 |
prelinked_dirs=[] |
| 1747 |
prelinked_blacklist=[] |
| 1748 |
prelinked_blacklist_glob=[] |
| 1749 |
for a in open('/etc/prelink.conf'): |
| 1750 |
if a[0] == '#': |
| 1751 |
continue |
| 1752 |
a=a.strip() |
| 1753 |
b=a.split() |
| 1754 |
if len(b) != 2: |
| 1755 |
logger.warning(' (sorry this line of /etc/prelink.conf cannot be parsed currently: "'+a+'")') |
| 1756 |
continue |
| 1757 |
if '-b' == b[0]: |
| 1758 |
if '/' in b[1]: |
| 1759 |
prelinked_blacklist.append(b[1]) |
| 1760 |
else: |
| 1761 |
prelinked_blacklist_glob.append(b[1]) |
| 1762 |
elif '-l' == b[0]: |
| 1763 |
prelinked_dirs.append(b[1]) |
| 1764 |
|
| 1765 |
|
| 1766 |
############ do_patch |
| 1767 |
|
| 1768 |
def do_patch(delta,olddeb,newdeb, info=None, diversions=None, do_gpg=DO_GPG): |
| 1769 |
runtime={} |
| 1770 |
T=None |
| 1771 |
try: |
| 1772 |
T=tempo() |
| 1773 |
r=do_patch_(delta,olddeb,newdeb, T, runtime, |
| 1774 |
info=info, diversions=diversions, do_gpg=do_gpg) |
| 1775 |
except: |
| 1776 |
if T : rmtree(T) |
| 1777 |
if newdeb and os.path.exists(newdeb): |
| 1778 |
os.unlink(newdeb) |
| 1779 |
raise |
| 1780 |
rmtree(T) |
| 1781 |
return r |
| 1782 |
|
| 1783 |
def do_patch_(delta, olddeb, newdeb, TD, runtime, |
| 1784 |
info=None, diversions=None, do_gpg=DO_GPG, do_progress=DO_PROGRESS): |
| 1785 |
|
| 1786 |
import threading |
| 1787 |
|
| 1788 |
if TD[-1] != '/': |
| 1789 |
TD = TD + '/' |
| 1790 |
|
| 1791 |
HAVE_PRELINK=os.path.exists('/usr/sbin/prelink') |
| 1792 |
# some people purge locales w/o using 'localepurge' , see e.g. http://bugs.debian.org/619086 |
| 1793 |
#HAVE_LOCALEPURGE=os.path.exists('/etc/locale.nopurge') or os.path.exists('/usr/sbin/localepurge') |
| 1794 |
|
| 1795 |
delta=abspath(delta) |
| 1796 |
newdebshortname='-' |
| 1797 |
if newdeb: |
| 1798 |
newdebshortname=newdeb |
| 1799 |
newdeb=abspath(newdeb) |
| 1800 |
if not os.path.isdir(olddeb): |
| 1801 |
olddeb=abspath(olddeb) |
| 1802 |
elif diversions == None: |
| 1803 |
diversions=scan_diversions() |
| 1804 |
|
| 1805 |
start_sec = time.time() |
| 1806 |
runtime['patchprogress']=0 |
| 1807 |
|
| 1808 |
check_is_delta(delta) |
| 1809 |
|
| 1810 |
if not os.path.isdir(olddeb): |
| 1811 |
check_deb(olddeb) |
| 1812 |
|
| 1813 |
temp_name, temp_err_name, ret=system(('ar','xvo',delta), TD+'/PATCH', return_output=True, ignore_output=True) |
| 1814 |
if ret : |
| 1815 |
raise DebDeltaError('Cannot extract from '+delta) |
| 1816 |
ar_list_delta=[a[4:] for a in open(temp_name).read().split('\n') if a] |
| 1817 |
os.unlink(temp_name) |
| 1818 |
os.unlink(temp_err_name) |
| 1819 |
|
| 1820 |
runtime['patchprogress']=1 |
| 1821 |
|
| 1822 |
is_signed=False |
| 1823 |
for a in ar_list_delta: |
| 1824 |
if a[:4] == '_gpg': |
| 1825 |
r = verify_signature(TD+'/PATCH/'+a,TD+'/PATCH') |
| 1826 |
if True != r: |
| 1827 |
die(delta+": the signature file "+a+" fails as follows: "+repr(r)) |
| 1828 |
is_signed=True |
| 1829 |
if VERBOSE > 1 : logger.debug(' The signature by "'+a[4:]+'" is correctly verified for '+delta) |
| 1830 |
if not is_signed: |
| 1831 |
if do_gpg: |
| 1832 |
die(_("Delta is not signed:")+' '+delta) |
| 1833 |
elif do_gpg != None: |
| 1834 |
logger.warning(_("WARNING, delta is not signed:")+' '+delta) |
| 1835 |
|
| 1836 |
runtime['patchprogress']=2 |
| 1837 |
|
| 1838 |
_info_patch_unzip_(TD) |
| 1839 |
|
| 1840 |
if not os.path.isfile(TD+'PATCH/patch.sh'): |
| 1841 |
die("Error. File `%s' is not a delta file." % delta) |
| 1842 |
|
| 1843 |
os.symlink(minigzip,TD+'minigzip') |
| 1844 |
os.symlink(minibzip2,TD+'minibzip2') |
| 1845 |
|
| 1846 |
#lets scan parameters, to see what it does and what it requires |
| 1847 |
if info == None : |
| 1848 |
info=_scan_delta_info_(TD) |
| 1849 |
params=info_2_db(info) |
| 1850 |
|
| 1851 |
runtime['patchprogress']=3 |
| 1852 |
|
| 1853 |
#this is not needed in preparing the patch, but may help in forensic |
| 1854 |
conf_files=[] |
| 1855 |
z='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles' |
| 1856 |
if FORENSIC and os.path.isfile(z): |
| 1857 |
#note that filenames do not have leading / |
| 1858 |
conf_files=[de_bar(p) for p in open(z).read().split('\n') if p] |
| 1859 |
del z |
| 1860 |
|
| 1861 |
### |
| 1862 |
s=patch_check_tmp_space(params,olddeb) |
| 1863 |
if s != True: |
| 1864 |
raise DebDeltaError('Sorry, '+s, True ) |
| 1865 |
|
| 1866 |
if not os.path.isdir(olddeb): |
| 1867 |
os.symlink(olddeb,TD+'/OLD.file') |
| 1868 |
ar_list_old= list_ar(TD+'OLD.file') |
| 1869 |
#unpack the old control structure, if available |
| 1870 |
os.mkdir(TD+'/OLD/CONTROL') |
| 1871 |
#unpack control.tar.gz |
| 1872 |
f,b,z = untar_control_in_deb(ar_list_old) |
| 1873 |
system(('ar','p',TD+'OLD.file',f,'|','tar',b,'-xp','-f','-','-C',TD+'OLD/CONTROL'),TD) |
| 1874 |
#then we check for the conformance |
| 1875 |
if not os.path.isdir(olddeb) and 'OLD/Size' in params: |
| 1876 |
olddebsize = os.stat(olddeb)[ST_SIZE] |
| 1877 |
if olddebsize != int(params['OLD/Size']): |
| 1878 |
raise DebDeltaError('Old deb size is '+str(olddebsize)+' instead of '+params['OLD/Size']) |
| 1879 |
|
| 1880 |
runtime['patchprogress']=4 |
| 1881 |
|
| 1882 |
if DEBUG > 1 : |
| 1883 |
#this is currently disabled, since 'dpkg -s' is vey slow (~ 1.6 sec) |
| 1884 |
dpkg_params={} |
| 1885 |
b=params['OLD/Package'] |
| 1886 |
if os.path.isdir(olddeb): |
| 1887 |
p=my_popen_read(['env', '-i', 'dpkg', '-s',b], **Popen_args) |
| 1888 |
else: |
| 1889 |
p=open(TD+'OLD/CONTROL/control') |
| 1890 |
scan_control(p,params=dpkg_params,prefix='OLD') |
| 1891 |
p.close() |
| 1892 |
if os.path.isdir(olddeb): |
| 1893 |
if 'OLD/Status' not in dpkg_params: |
| 1894 |
die('Error: package %s is not known to dpkg.' % b) |
| 1895 |
if dpkg_params['OLD/Status'] != 'install ok installed' : |
| 1896 |
die('Error: package %s is not installed, status is %s.' |
| 1897 |
% ( b , dpkg_params['OLD/Status'] ) ) |
| 1898 |
for a in params: |
| 1899 |
if a[:3] == 'OLD' and a != 'OLD/Installed-Size' and a != 'OLD/Size': |
| 1900 |
if a not in dpkg_params: |
| 1901 |
die('Error parsing old control file , parameter %s not found' % a) |
| 1902 |
elif params[a] != dpkg_params[a] : |
| 1903 |
die( 'Error : in delta , '+a+' = ' +params[a] +\ |
| 1904 |
'\nin old/installed deb, '+a+' = ' +dpkg_params[a]) |
| 1905 |
del b,p #cannot delete 'a', python raise a SyntaxError |
| 1906 |
|
| 1907 |
runtime['patchprogress']=5 |
| 1908 |
|
| 1909 |
### some auxiliary routines, separated to make code more readable |
| 1910 |
|
| 1911 |
def dpkg_L_faster(pa,ar,diversions): |
| 1912 |
"Scan dpkg -L . 'diversions' must be prepared by scan_diversions() . Returns list of pairs of files ,and list of diverted files. " |
| 1913 |
s=[] |
| 1914 |
diverted=[] |
| 1915 |
n='/var/lib/dpkg/info/'+pa+':'+ar+'.list' |
| 1916 |
if not DPKG_MULTIARCH or not os.path.exists(n): |
| 1917 |
n='/var/lib/dpkg/info/'+pa+'.list' |
| 1918 |
f=open(n) |
| 1919 |
while 1: |
| 1920 |
a=f.readline() |
| 1921 |
if not a: break |
| 1922 |
a=de_n(a) |
| 1923 |
if a in diversions: |
| 1924 |
b,p= diversions[a] |
| 1925 |
if p != pa: |
| 1926 |
s.append((a,b)) |
| 1927 |
diverted.append(a) |
| 1928 |
else: |
| 1929 |
s.append((a,a)) |
| 1930 |
else: s.append((a,a)) |
| 1931 |
f.close() |
| 1932 |
return s,diverted |
| 1933 |
|
| 1934 |
def dpkg_L(pa,ar): |
| 1935 |
"Scan dpkg -L . Currently unused, see previous function." |
| 1936 |
#sys.stderr.write('INTERNAL WARNING: USING OBSOLETE dpkg_L\n') |
| 1937 |
s=[] |
| 1938 |
diverted=[] |
| 1939 |
if DPKG_MULTIARCH: |
| 1940 |
p=my_popen_read(['env', '-i', 'dpkg-query', '-L', pa+':'+ar]) |
| 1941 |
else: |
| 1942 |
p=my_popen_read(['env', '-i', 'dpkg-query', '-L', pa]) |
| 1943 |
a=p.readline() |
| 1944 |
while a: |
| 1945 |
a=de_n(a) |
| 1946 |
#support diversions |
| 1947 |
if a[:26] == 'package diverts others to:': |
| 1948 |
continue |
| 1949 |
if s and a[:11] == 'diverted by' or a[:20] == 'locally diverted to:': |
| 1950 |
orig,divert=s.pop() |
| 1951 |
i = a.index(':') |
| 1952 |
divert = a[i+2:] |
| 1953 |
s.append( (orig,divert) ) |
| 1954 |
diverted.append(orig) |
| 1955 |
else: |
| 1956 |
s.append( (a,a) ) |
| 1957 |
a=p.readline() |
| 1958 |
p.close() |
| 1959 |
return s, diverted |
| 1960 |
|
| 1961 |
def _symlink_data_tree(pa,ar,TD,diversions,runtime): |
| 1962 |
localepurged=[] |
| 1963 |
prelink_u_failed=[] |
| 1964 |
file_triples=[] |
| 1965 |
prelink_time = 0 |
| 1966 |
prelink_datasize = 0 |
| 1967 |
if diversions: |
| 1968 |
s,diverted=dpkg_L_faster(pa,ar,diversions) |
| 1969 |
else: |
| 1970 |
s,diverted=dpkg_L(pa,ar) |
| 1971 |
progressline=0 |
| 1972 |
progresslen=float(len(s)) |
| 1973 |
for orig,divert in s: |
| 1974 |
progressline+=1 |
| 1975 |
progress=6.0 + 6.0 * float(progressline) / progresslen |
| 1976 |
runtime['patchprogress']=progress |
| 1977 |
if do_progress: |
| 1978 |
sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) |
| 1979 |
if os.path.isfile(divert) and not os.path.islink(divert) : |
| 1980 |
tmpcopy=TD+'OLD/DATA'+orig |
| 1981 |
d=os.path.dirname(tmpcopy) |
| 1982 |
if not os.path.exists(d): |
| 1983 |
os.makedirs(d) |
| 1984 |
#the following code idea was provided by [email protected] |
| 1985 |
unprelink=False |
| 1986 |
if HAVE_PRELINK : |
| 1987 |
prelink_time -= time.time() |
| 1988 |
is_elf, ei_class, ei_data, ei_osabi, e_type = elf_info(divert) |
| 1989 |
#according to prelink-0.0.20090925/src/main.c |
| 1990 |
unprelink = is_elf and e_type in ('ET_DYN','ET_EXEC') |
| 1991 |
prelink_time += time.time() |
| 1992 |
if unprelink: |
| 1993 |
prelink_time -= time.time() |
| 1994 |
prelink_datasize += os.path.getsize(divert) |
| 1995 |
if VERBOSE > 3 : |
| 1996 |
logger.debug(' copying/unprelinking '+divert+' to '+ tmpcopy) |
| 1997 |
#unfortunately 'prelink -o' sometimes alters files, see http://bugs.debian.org/627932 |
| 1998 |
shutil.copy2(divert, tmpcopy) |
| 1999 |
proc=subprocess.Popen(["/usr/sbin/prelink","-u",tmpcopy],stdin=open(os.devnull),\ |
| 2000 |
stdout=subprocess.PIPE,stderr=subprocess.STDOUT, |
| 2001 |
close_fds=True, **Popen_args) |
| 2002 |
out=proc.stdout.read().strip() |
| 2003 |
proc.wait() |
| 2004 |
if proc.returncode: |
| 2005 |
if not os.path.exists(tmpcopy): |
| 2006 |
if VERBOSE > 4 : logger.debug(' (prelink failed, symlinking '+divert+' to '+tmpcopy+')') |
| 2007 |
os.symlink(divert, tmpcopy) |
| 2008 |
prelink_u_failed.append(de_bar(orig)) |
| 2009 |
unprelink=False |
| 2010 |
elif VERBOSE > 4 : logger.debug(' (prelink failed, but file was copied)') |
| 2011 |
thestat = os.statvfs(tmpcopy) |
| 2012 |
if out[-39:] == 'does not have .gnu.prelink_undo section': |
| 2013 |
if DEBUG > 1: |
| 2014 |
logger.debug(' '+repr(out)) |
| 2015 |
elif (thestat.f_bsize * thestat.f_bavail / 1024) < 50000 : |
| 2016 |
logger.warning('!!Prelink -u failed, it needs at least 50000KB of free disk space\n') |
| 2017 |
prelink_u_failed.append(de_bar(orig)) |
| 2018 |
unprelink=False |
| 2019 |
else: |
| 2020 |
logger.warning('!!Prelink -u failed on %r : %r\n', tmpcopy, out) |
| 2021 |
prelink_u_failed.append(de_bar(orig)) |
| 2022 |
unprelink=False |
| 2023 |
prelink_time += time.time() |
| 2024 |
else: |
| 2025 |
if VERBOSE > 3 : logger.debug(' symlinking '+divert+' to '+a) |
| 2026 |
os.symlink(divert, tmpcopy) |
| 2027 |
if unprelink and FORENSIC: |
| 2028 |
#unfortunately the script will delete the 'tmpcopy', so we hardlink it |
| 2029 |
z=tempfile.mktemp(prefix=TD) |
| 2030 |
os.link(tmpcopy,z) |
| 2031 |
file_triples.append((orig,divert,z)) |
| 2032 |
else: |
| 2033 |
file_triples.append((orig,divert,None)) |
| 2034 |
elif not os.path.exists(divert) and os.path.islink(divert): |
| 2035 |
file_triples.append((orig,divert,None)) |
| 2036 |
if VERBOSE > 1 : logger.debug(' Broken symlink? %r',divert) |
| 2037 |
elif not os.path.exists(divert): |
| 2038 |
file_triples.append((orig,divert,None)) |
| 2039 |
if VERBOSE : logger.debug(' Disappeared file? %r',divert) |
| 2040 |
for z in ('locale','man','gnome/help','omf','doc/kde/HTML'): |
| 2041 |
w='/usr/share/'+z |
| 2042 |
if orig[:len(w)] == w: |
| 2043 |
localepurged.append(de_bar(orig)) |
| 2044 |
else: |
| 2045 |
file_triples.append((orig,divert,None)) |
| 2046 |
if VERBOSE > 3 : logger.debug(' not symlinking %r to %r',divert,orig) |
| 2047 |
return file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize |
| 2048 |
|
| 2049 |
def chmod_add(n,m): |
| 2050 |
"same as 'chmod ...+... n '" |
| 2051 |
om=S_IMODE(os.stat(n)[ST_MODE]) |
| 2052 |
nm=om | m |
| 2053 |
if nm != om : |
| 2054 |
if VERBOSE > 2 : logger.debug(' Performing chmod %s %s %s',n,oct(om),oct(nm)) |
| 2055 |
os.chmod(n,nm) |
| 2056 |
|
| 2057 |
def _fix_data_tree_(TD): |
| 2058 |
for (dirpath, dirnames, filenames) in os.walk(TD+'OLD/DATA'): |
| 2059 |
chmod_add(dirpath, S_IRUSR | S_IWUSR| S_IXUSR ) |
| 2060 |
for i in filenames: |
| 2061 |
i=os.path.join(dirpath,i) |
| 2062 |
if os.path.isfile(i): |
| 2063 |
chmod_add(i, S_IRUSR | S_IWUSR ) |
| 2064 |
for i in dirnames: |
| 2065 |
i=os.path.join(dirpath,i) |
| 2066 |
chmod_add(i, S_IRUSR | S_IWUSR| S_IXUSR ) |
| 2067 |
|
| 2068 |
#initialize, just in case |
| 2069 |
control_file_triples=[] |
| 2070 |
file_triples=[] |
| 2071 |
localepurged=[] |
| 2072 |
prelink_u_failed=[] |
| 2073 |
diverted=[] |
| 2074 |
prelink_time=0 |
| 2075 |
prelink_datasize=0 |
| 2076 |
|
| 2077 |
###see into parameters: the patch may need extra info and data |
| 2078 |
|
| 2079 |
runtime['patchprogress']=6 |
| 2080 |
|
| 2081 |
prelink_time=None |
| 2082 |
|
| 2083 |
for a in params: |
| 2084 |
if 'needs-old' == a: |
| 2085 |
if os.path.isdir(olddeb): |
| 2086 |
die('This patch needs the old version Debian package') |
| 2087 |
elif 'old-data-tree' == a : |
| 2088 |
os.mkdir(TD+'/OLD/DATA') |
| 2089 |
if os.path.isdir(olddeb): |
| 2090 |
file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize=\ |
| 2091 |
_symlink_data_tree(params['OLD/Package'],params['OLD/Architecture'],TD,diversions,runtime) |
| 2092 |
else: |
| 2093 |
ar_list_old= list_ar(TD+'OLD.file') |
| 2094 |
if b'data.tar.bz2' in ar_list_old: |
| 2095 |
system(('ar','p',TD+'OLD.file','data.tar.bz2','|','tar','-xp','--bzip2','-f','-','-C',TD+'OLD/DATA'), TD) |
| 2096 |
elif b'data.tar.gz' in ar_list_old: |
| 2097 |
system(('ar','p',TD+'OLD.file','data.tar.gz','|','tar','-xp','-z','-f','-','-C',TD+'OLD/DATA'), TD) |
| 2098 |
elif b'data.tar.lzma' in ar_list_old: |
| 2099 |
if not os.path.exists('/usr/bin/lzma'): |
| 2100 |
raise DebDeltaError('This patch needs lzma. Please install the Debian package "lzma".',retriable=True) |
| 2101 |
system(('ar','p',TD+'OLD.file','data.tar.lzma','|','unlzma','-c','|','tar','-xpf','-','-C',TD+'OLD/DATA'), TD) |
| 2102 |
elif b'data.tar.xz' in ar_list_old: |
| 2103 |
if not os.path.exists('/usr/bin/xz'): |
| 2104 |
raise DebDeltaError('This patch needs xz. Please install the Debian package "xz-utils".',retriable=True) |
| 2105 |
system(('ar','p',TD+'OLD.file','data.tar.xz','|','unxz','-c','|','tar','-xpf','-','-C',TD+'OLD/DATA'), TD) |
| 2106 |
else: assert(0) |
| 2107 |
_fix_data_tree_(TD) |
| 2108 |
elif 'old-control-tree' == a: |
| 2109 |
if os.path.isdir(olddeb): |
| 2110 |
if not os.path.isdir(TD+'OLD/CONTROL'): |
| 2111 |
os.mkdir(TD+'OLD/CONTROL') |
| 2112 |
p=params['OLD/Package'] |
| 2113 |
a=params['OLD/Architecture'] |
| 2114 |
for b in dpkg_keeps_controls : |
| 2115 |
z='/var/lib/dpkg/info/' + p + ':' + a + '.'+b |
| 2116 |
if not DPKG_MULTIARCH or not os.path.exists(z): |
| 2117 |
z='/var/lib/dpkg/info/' + p +'.'+b |
| 2118 |
if os.path.exists(z): |
| 2119 |
os.symlink(z,TD+'OLD/CONTROL/'+b) |
| 2120 |
control_file_triples.append((b,z,None)) |
| 2121 |
del z,p #cannot delete 'a', python raise a SyntaxError |
| 2122 |
#else... we always unpack the control of a .deb |
| 2123 |
elif 'needs-xdelta3' == a: |
| 2124 |
if not os.path.exists('/usr/bin/xdelta3'): |
| 2125 |
raise DebDeltaError('This patch needs xdelta3. Please install the Debian package "xdelta3".',retriable=True) |
| 2126 |
elif 'needs-xdelta3-fifo' == a: |
| 2127 |
# not doing a specific check, I am using debian/control Conflicts |
| 2128 |
if not os.path.exists('/usr/bin/xdelta3'): |
| 2129 |
raise DebDeltaError('This patch needs xdelta3, at least version 3.0y. Please install the Debian package "xdelta3".',retriable=True) |
| 2130 |
elif 'needs-xdelta' == a: |
| 2131 |
if not os.path.exists('/usr/bin/xdelta'): |
| 2132 |
raise DebDeltaError('This patch needs xdelta. Please install the Debian package "xdelta".',retriable=True) |
| 2133 |
elif 'needs-bsdiff' == a: |
| 2134 |
if not os.path.exists('/usr/bin/bsdiff'): |
| 2135 |
raise DebDeltaError('This patch needs bsdiff. Please install the Debian package "bsdiff".',retriable=True) |
| 2136 |
elif 'needs-lzma' == a: |
| 2137 |
if not os.path.exists('/usr/bin/lzma'): |
| 2138 |
raise DebDeltaError('This patch needs lzma. Please install the Debian package "lzma".',retriable=True) |
| 2139 |
elif 'needs-xz' == a: |
| 2140 |
if not os.path.exists('/usr/bin/xz'): |
| 2141 |
raise DebDeltaError('This patch needs xz. Please install the Debian package "xz-utils".',retriable=True) |
| 2142 |
elif 'needs-minibzip2' == a: |
| 2143 |
pass #its your lucky day |
| 2144 |
elif a[:6] == 'needs-': |
| 2145 |
raise DebDeltaError('patch says "'+a+"' and this is unsupported. Get a newer debdelta.",retriable=True) |
| 2146 |
elif params[a] == True: |
| 2147 |
logger.warning('WARNING patch says %r and this is unsupported. Get a newer debdelta.',a) |
| 2148 |
|
| 2149 |
if localepurged and not DEBUG: |
| 2150 |
#actually we cannot be 100% sure that the delta really needs those files, but it is quite plausible |
| 2151 |
raise DebDeltaError('Error, '+str(len(localepurged))+' locale files are absent.') |
| 2152 |
|
| 2153 |
runtime['patchprogress']=12 |
| 2154 |
|
| 2155 |
script_time = - time.time() |
| 2156 |
|
| 2157 |
this_deb_format=DEB_FORMAT |
| 2158 |
if this_deb_format != 'deb' and 'NEW/data.tar' not in params: |
| 2159 |
#this patch does not support streaming |
| 2160 |
logger.warning('Warning, created as standard deb: '+newdeb) |
| 2161 |
this_deb_format='deb' |
| 2162 |
|
| 2163 |
if this_deb_format == 'preunpacked' : #need a fifo |
| 2164 |
os.mkfifo(TD+'data.pipe') |
| 2165 |
|
| 2166 |
temp_err_name_fd, temp_err_name = tempfile.mkstemp(prefix='debdeltaE') |
| 2167 |
temp_name_fd, temp_name = tempfile.mkstemp(prefix='debdeltaO') |
| 2168 |
cmd=[SHELL,'-e','PATCH/patch.sh'] |
| 2169 |
if DEBUG > 2 : |
| 2170 |
cmd=[SHELL,'-evx','PATCH/patch.sh'] |
| 2171 |
if this_deb_format == 'unzipped': |
| 2172 |
cmd+=['unzipped'] |
| 2173 |
elif this_deb_format == 'preunpacked': |
| 2174 |
cmd+=['piped'] |
| 2175 |
|
| 2176 |
env={'PATH':os.getenv('PATH')} |
| 2177 |
F=subprocess.Popen(cmd, cwd=TD, |
| 2178 |
bufsize=4096,close_fds=True, |
| 2179 |
stdin=open(os.devnull),env=env, |
| 2180 |
stderr=temp_err_name_fd, stdout=temp_name_fd) |
| 2181 |
|
| 2182 |
### data used by the preunpacked method |
| 2183 |
data_md5=None # md5 of uncompressed data.tar |
| 2184 |
tar_status=[] # should be [True] if 'preunpacked' went fine |
| 2185 |
md5_status=[] # idem |
| 2186 |
#this list contains tuples of (unpacked_temporary_filename, real_filename, owner, group, tartype, mode, mtime, linkname) |
| 2187 |
preunpacked_filelist=[] |
| 2188 |
def do_cleanup(): |
| 2189 |
for a in preunpacked_filelist: |
| 2190 |
pass |
| 2191 |
##CHEAT we are not (yet) writing anything to disk |
| 2192 |
#if a[0]: os.unlink(a[0]) |
| 2193 |
|
| 2194 |
if this_deb_format == 'preunpacked' : #do progress reporting and unpacking in filesystem |
| 2195 |
def do_extension(): |
| 2196 |
return '_'+str(random.randint(1,9999)).rjust(4,'0')+'_debdelta_preunpacked' |
| 2197 |
|
| 2198 |
def do_pipe_md5(i,o,rm,ms,ts): |
| 2199 |
try: |
| 2200 |
a=i.read(1024) |
| 2201 |
while a and (ts == [] or ts == [True]) : |
| 2202 |
rm.update(a) |
| 2203 |
o.write(a) |
| 2204 |
a=i.read(1024) |
| 2205 |
#TODO implement progress reporting here as well |
| 2206 |
o.close() |
| 2207 |
ms.append(True) |
| 2208 |
except: |
| 2209 |
import sys |
| 2210 |
ms.append(sys.exc_info()) |
| 2211 |
if DEBUG: logger.debug(' do_pipe_md5 crashed: '+repr(ms)) |
| 2212 |
|
| 2213 |
def do_tar(i,fl,s): |
| 2214 |
try: |
| 2215 |
dt=tarfile.open(mode='r|',fileobj=i) |
| 2216 |
for tarmember in dt: |
| 2217 |
if tarmember.isreg(): |
| 2218 |
n='/'+tarmember.name+do_extension() |
| 2219 |
while os.path.exists(n) : #wont overwrite existing stuff, never ever |
| 2220 |
n='/'+tarmember.name+do_extension() |
| 2221 |
else: |
| 2222 |
n='' |
| 2223 |
a=(n, tarmember.name, tarmember.uname, tarmember.gname,\ |
| 2224 |
tarmember.type, tarmember.mode, tarmember.mtime, tarmember.linkname) |
| 2225 |
fl.append(a) |
| 2226 |
if n: |
| 2227 |
pass |
| 2228 |
##CHEAT we are not writing anything today! |
| 2229 |
#datatar.extract(tarmember,path=n) |
| 2230 |
#successfully untarred! |
| 2231 |
s.append(True) |
| 2232 |
except:#catch problems such as "out of disk space" or corrupted data |
| 2233 |
import sys |
| 2234 |
s.append(sys.exc_info()) |
| 2235 |
if DEBUG: logger.debug(' do_tar crashed: '+repr(s)) |
| 2236 |
#flush input (note that do_pipe_md5 will soon stop writing) no it seems useless #i.read() |
| 2237 |
|
| 2238 |
try: |
| 2239 |
datapipe=open(TD+'data.pipe') |
| 2240 |
rolling_md5=hashlib.md5() |
| 2241 |
(piper,pipew)=os.pipe() |
| 2242 |
md5_thread=threading.Thread(target=do_pipe_md5, |
| 2243 |
args=(open(TD+'data.pipe'),os.fdopen(pipew,'w'),rolling_md5,\ |
| 2244 |
md5_status,tar_status)) |
| 2245 |
tar_thread=threading.Thread(target=do_tar, |
| 2246 |
args=(os.fdopen(piper),preunpacked_filelist,tar_status)) |
| 2247 |
#yeah maybe using two threads is more complex than strictly needed |
| 2248 |
md5_thread.daemon=True |
| 2249 |
md5_thread.start() |
| 2250 |
tar_thread.daemon=True |
| 2251 |
tar_thread.start() |
| 2252 |
#join back the md5 |
| 2253 |
md5_thread.join() |
| 2254 |
data_md5=rolling_md5.hexdigest() |
| 2255 |
if md5_status == [True] and params['NEW/data.tar'][:32] == data_md5: |
| 2256 |
tar_thread.join() |
| 2257 |
if tar_status == [True]: |
| 2258 |
F.wait() |
| 2259 |
#write data_list |
| 2260 |
a=open(TD+'data_list','w') |
| 2261 |
a.write('Files:\n') |
| 2262 |
for (unpacked_temporary_filename, real_filename, owner,\ |
| 2263 |
group, tartype, mode, mtime, linkname) in preunpacked_filelist: |
| 2264 |
## to convert 'tar' type into 'ls' type |
| 2265 |
if tartype >= '0' and tartype <= '6' : |
| 2266 |
tartype="-hlcbdp"[ord(tartype) - ord('0')] |
| 2267 |
else: |
| 2268 |
tartype='?' |
| 2269 |
logger.debug('WARNING unsupported tar type '+repr(tartype)+' for: '+repr(real_filename)) |
| 2270 |
#if tartype < '0' or tartype > '6' : |
| 2271 |
# tartype='?' |
| 2272 |
# print 'WARNING unsupported tar type '+repr(tartype)+' for: '+repr(real_filename) |
| 2273 |
mode=oct(mode).rjust(4,'0') |
| 2274 |
a.write(' %s %s %s %s %s\n %s\n %s\n %s\n' % (tartype,mode,owner,group,mtime,\ |
| 2275 |
unpacked_temporary_filename,real_filename,linkname)) |
| 2276 |
a.close() |
| 2277 |
#append it into deb |
| 2278 |
system(['ar','q','NEW.file', 'data_list'], TD) |
| 2279 |
except: |
| 2280 |
do_cleanup() |
| 2281 |
raise |
| 2282 |
else: #progress reporting for deb_format != 'preunpacked' |
| 2283 |
runtime['patchprogress']=12 |
| 2284 |
if 'NEW/Size' in params and do_progress: |
| 2285 |
NEW_size=int(params['NEW/Size']) |
| 2286 |
while None == F.poll(): |
| 2287 |
if os.path.exists(TD+'NEW.file'): |
| 2288 |
a=os.path.getsize(TD+'NEW.file') |
| 2289 |
progress=(int(12.0 + 84.0 * a / NEW_size)) |
| 2290 |
else: |
| 2291 |
progress=12 |
| 2292 |
runtime['patchprogress']=progress |
| 2293 |
time.sleep(0.1) |
| 2294 |
if do_progress: |
| 2295 |
sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) |
| 2296 |
F.wait() |
| 2297 |
if do_progress and terminalcolumns: #clean up |
| 2298 |
sys.stderr.write(' ' * terminalcolumns + '\r') |
| 2299 |
ret=F.returncode |
| 2300 |
os.close(temp_err_name_fd) |
| 2301 |
os.close(temp_name_fd) |
| 2302 |
|
| 2303 |
script_time += time.time() # for --format='preunpacked' this time also include data MD5 |
| 2304 |
runtime['patchprogress']=97 |
| 2305 |
|
| 2306 |
#helper for debugging |
| 2307 |
def tempos(f): |
| 2308 |
if os.path.getsize(temp_name): |
| 2309 |
f.append(temp_name) |
| 2310 |
if os.path.getsize(temp_err_name): |
| 2311 |
f.append(temp_err_name) |
| 2312 |
|
| 2313 |
if not FORENSIC: |
| 2314 |
def fore(): |
| 2315 |
return None |
| 2316 |
elif not os.path.isdir(olddeb): |
| 2317 |
def fore(): |
| 2318 |
f=[delta,olddeb] |
| 2319 |
tempos(f) |
| 2320 |
return f |
| 2321 |
else: |
| 2322 |
def fore(): |
| 2323 |
temp_fore_name='' |
| 2324 |
f=[] |
| 2325 |
tempos(f) |
| 2326 |
try: |
| 2327 |
(temp_fd,temp_fore_name) = tempfile.mkstemp(prefix="debforensic_"+params['NEW/Package']+"_") |
| 2328 |
temp_file=os.fdopen(temp_fd,'w') |
| 2329 |
temp_file.write('Delta: '+delta+'\n') |
| 2330 |
temp_file.write('DeltaSHA1: ' + sha1_hash_file(delta) + '\n') |
| 2331 |
temp_file.write('LocalePurgedFilesN: '+str(len(localepurged))+'\n') |
| 2332 |
temp_file.write('PrelinkUFailedN: '+str(len(prelink_u_failed))+'\n') |
| 2333 |
if ret: |
| 2334 |
temp_file.write('PatchExitCode: '+str(ret)+'\n') |
| 2335 |
forensics_rfc(temp_file,params,False,control_file_triples,file_triples,conf_files, |
| 2336 |
diverted,diversions,localepurged,prelink_u_failed) |
| 2337 |
#copy short content here and remove from list |
| 2338 |
for i in copy(f): |
| 2339 |
if os.path.getsize(i) < 2000: |
| 2340 |
f.remove(i) |
| 2341 |
temp_file.write('PatchLogFile_'+str(i)+'_content:\n') |
| 2342 |
for ll in open(i): |
| 2343 |
temp_file.write(' '+repr(ll)+'\n') |
| 2344 |
else: temp_file.write('PatchLogFileIs: '+str(i)+'\n') |
| 2345 |
temp_file.close() |
| 2346 |
except OSError: #Exception,s: |
| 2347 |
die('!!While creating forensic '+temp_fore_name+' error:'+str(s)+'\n') |
| 2348 |
f.append(temp_fore_name) |
| 2349 |
return f |
| 2350 |
|
| 2351 |
if ret: |
| 2352 |
if localepurged: |
| 2353 |
raise DebDeltaError('"debdelta" is incompatible with "localepurge".') |
| 2354 |
else: |
| 2355 |
f=fore() |
| 2356 |
raise DebDeltaError('error in patch.sh.',logs=f) |
| 2357 |
|
| 2358 |
#then we check for the conformance |
| 2359 |
if this_deb_format == 'deb': |
| 2360 |
if 'NEW/Size' in params: |
| 2361 |
newdebsize = os.stat(TD+'NEW.file')[ST_SIZE] |
| 2362 |
if newdebsize != int(params['NEW/Size']): |
| 2363 |
f=fore() |
| 2364 |
raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size'],logs=f) |
| 2365 |
|
| 2366 |
if DO_MD5: |
| 2367 |
if 'NEW/MD5sum' in params: |
| 2368 |
if VERBOSE > 1 : logger.debug(' verifying MD5 for %r',os.path.basename(newdeb or delta)) |
| 2369 |
m= compute_md5(open(TD+'NEW.file','rb')) |
| 2370 |
if params['NEW/MD5sum'] != m : |
| 2371 |
f=fore() |
| 2372 |
raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) , logs=f) |
| 2373 |
else: logger.warning(' Warning! no MD5 was verified for %r',os.path.basename(newdeb or delta)) |
| 2374 |
elif this_deb_format == 'unzipped' : |
| 2375 |
if DO_MD5: |
| 2376 |
c,b,z=untar_control_in_deb(list_ar(TD+'NEW.file')) |
| 2377 |
p=PopenPipe(['ar','p',(TD+'NEW.file'),c,'|']+z, stdout=subprocess.PIPE) |
| 2378 |
m=compute_md5(p.stdout) |
| 2379 |
p.wait() |
| 2380 |
if params['NEW/control.tar'][:32] != m: |
| 2381 |
f=fore() |
| 2382 |
raise DebDeltaError('MD5 mismatch for control.tar' , logs=f) |
| 2383 |
p=subprocess.Popen(['ar','p',(TD+'NEW.file'),'data.tar'], stdout=subprocess.PIPE) |
| 2384 |
m=compute_md5(p.stdout) |
| 2385 |
p.wait() |
| 2386 |
if params['NEW/data.tar'][:32] != m: |
| 2387 |
f=fore() |
| 2388 |
raise DebDeltaError('MD5 mismatch for data.tar', logs=f) |
| 2389 |
elif this_deb_format == 'preunpacked' : |
| 2390 |
if tar_status != [True]: |
| 2391 |
f=fore() |
| 2392 |
do_cleanup() |
| 2393 |
raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1]), logs=f) #todo format me better |
| 2394 |
if md5_status != [True]: |
| 2395 |
f=fore() |
| 2396 |
do_cleanup() |
| 2397 |
raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1]), logs=f) #todo format me better |
| 2398 |
#if DO_MD5: #actually we always do MD5 |
| 2399 |
m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'), |
| 2400 |
stdout=subprocess.PIPE,shell=True).stdout) |
| 2401 |
if params['NEW/control.tar'][:32] != m: |
| 2402 |
f=fore() |
| 2403 |
do_cleanup() |
| 2404 |
raise DebDeltaError('MD5 mismatch for control.tar', logs=f) |
| 2405 |
if params['NEW/data.tar'][:32] != data_md5: |
| 2406 |
f=fore() |
| 2407 |
do_cleanup() |
| 2408 |
raise DebDeltaError('MD5 mismatch for data.tar', logs=f) |
| 2409 |
else: assert('unimplemented'=='') |
| 2410 |
|
| 2411 |
os.unlink(temp_name) |
| 2412 |
os.unlink(temp_err_name) |
| 2413 |
|
| 2414 |
runtime['patchprogress']=99 |
| 2415 |
|
| 2416 |
if newdeb: |
| 2417 |
shutil.move(TD+'NEW.file',newdeb) |
| 2418 |
|
| 2419 |
end_sec = time.time() |
| 2420 |
elaps=(end_sec - start_sec) |
| 2421 |
|
| 2422 |
if VERBOSE : |
| 2423 |
if newdeb: |
| 2424 |
debsize = os.stat(newdeb)[ST_SIZE] |
| 2425 |
else: |
| 2426 |
debsize = os.stat(olddeb)[ST_SIZE] |
| 2427 |
#this printout uses kibibytes, and not SizeToStr, to ease statistics |
| 2428 |
out = ' ' + _('Patching done, time %(time).2fsec, speed %(speed)dk/sec') % \ |
| 2429 |
{'time' : elaps, 'speed' : (debsize / 1024 / (elaps+.001))} |
| 2430 |
out += ' ' + _('(script %(time).2fsec %(speed)dk/sec)') % \ |
| 2431 |
{'time' : script_time, 'speed' : (debsize / 1024 / (script_time+.001))} |
| 2432 |
if prelink_time != None and prelink_time > 0 : |
| 2433 |
prelink_datasize=prelink_datasize/1024 |
| 2434 |
out += _('(prelink %(time).2fsec, %(size)dk, %(speed)dk/s)') % \ |
| 2435 |
{'time' : prelink_time, 'size' : prelink_datasize, 'speed' : prelink_datasize / (prelink_time + .001)} |
| 2436 |
out += _('(unaccounted %.2fsec)') % (elaps-prelink_time-script_time) |
| 2437 |
else: |
| 2438 |
out += _('(unaccounted %.2fsec)') % (elaps-script_time) |
| 2439 |
logger.info(out) |
| 2440 |
## this is useless, if 'debpatch' it is in the command line, if 'debdelta-upgrade' |
| 2441 |
## it is printed elsewhere |
| 2442 |
#if newdeb != None: |
| 2443 |
# print 'result: '+os.path.basename(newdeb), |
| 2444 |
return (newdeb,elaps) |
| 2445 |
|
| 2446 |
##################################################### compute delta |
| 2447 |
def do_delta(olddeb,newdeb,delta): |
| 2448 |
T=None |
| 2449 |
try: |
| 2450 |
T=tempo() |
| 2451 |
if os.path.exists(delta+'_tmp_') : |
| 2452 |
os.unlink(delta+'_tmp_') |
| 2453 |
r=do_delta_(olddeb,newdeb,delta+'_tmp_',TD=T) |
| 2454 |
(deltatmp, percent, elaps, info, gpg_hashes) = r |
| 2455 |
info_hashes=append_info(deltatmp,info) |
| 2456 |
if DO_GPG: |
| 2457 |
gpg_hashes['info']=info_hashes |
| 2458 |
sign_delta(deltatmp,gpg_hashes) |
| 2459 |
if os.path.exists(delta) : |
| 2460 |
os.rename(delta,delta+'~') |
| 2461 |
os.rename(deltatmp,delta) |
| 2462 |
except: |
| 2463 |
if delta and os.path.exists(delta): |
| 2464 |
os.unlink(delta) |
| 2465 |
if delta and os.path.exists(delta+'_tmp_'): |
| 2466 |
os.unlink(delta+'_tmp_') |
| 2467 |
if T : rmtree(T) |
| 2468 |
raise |
| 2469 |
else: |
| 2470 |
if T : rmtree(T) |
| 2471 |
return r |
| 2472 |
|
| 2473 |
def do_delta_(olddeb, newdeb, delta, TD, forensic_file=None, info=[]): |
| 2474 |
"""This function creates a delta. The delta is 'ar' archive (see 'man ar'). |
| 2475 |
The delta contains data, a script, and optional gpg signatures. |
| 2476 |
The script recreates the new deb. Note that the deb is (again) an 'ar' archive, |
| 2477 |
and has multiple members. |
| 2478 |
Simple deb members are managed directly. |
| 2479 |
The complex members are 'data.tar.xxx' and 'control.tar.gz' ; |
| 2480 |
these are studied in the delta_tar() function, that in turn uses |
| 2481 |
the Script() class. |
| 2482 |
(Note that there is just one object instanced from Script(), in variable 'script'). |
| 2483 |
To recreate one of the complex members, there are two main steps in the script: |
| 2484 |
(1) rebuild the member, uncompressed |
| 2485 |
(2) recompress the member |
| 2486 |
The part (2) is managed by script.zip_piped(). |
| 2487 |
The part (1) is more complex: |
| 2488 |
(1a) there is a first subshell where data from the old version of the deb |
| 2489 |
are piped to stdout (moreover some gzipped files may be transparently |
| 2490 |
unzipped, delta-ed and re-gzipped, see delta_gzipped_files) |
| 2491 |
(1b) the former (1a) is "piped" into the delta backend, to transform the old data into new. |
| 2492 |
If the delta-backend is 'bsdiff' then the above process (1a,b) is repeated |
| 2493 |
in chunks (indeed bsdiff cannot manage large files w/o exausting all your memory!) |
| 2494 |
All the (chunks of) steps (1a,b) are in a subshell, and its result stdout is piped in (2). |
| 2495 |
""" |
| 2496 |
if TD[-1] != '/': |
| 2497 |
TD = TD + '/' |
| 2498 |
|
| 2499 |
import fnmatch |
| 2500 |
|
| 2501 |
start_sec = time.time() |
| 2502 |
|
| 2503 |
#I do not like global variables but I do not know of another solution |
| 2504 |
global bsdiff_time, bsdiff_datasize |
| 2505 |
bsdiff_time = 0 |
| 2506 |
bsdiff_datasize = 0 |
| 2507 |
|
| 2508 |
olddeb=abspath(olddeb) |
| 2509 |
check_deb(olddeb) |
| 2510 |
os.symlink(olddeb,TD+'/OLD.file') |
| 2511 |
olddebsize = os.stat(olddeb)[ST_SIZE] |
| 2512 |
|
| 2513 |
newdeb=abspath(newdeb) |
| 2514 |
check_deb(newdeb) |
| 2515 |
os.symlink(newdeb,TD+'/NEW.file') |
| 2516 |
newdebsize = os.stat(newdeb)[ST_SIZE] |
| 2517 |
|
| 2518 |
#process all contents of old vs new .deb |
| 2519 |
ar_list_old= list_ar(TD+'OLD.file') |
| 2520 |
ar_list_new= list_ar(TD+'NEW.file') |
| 2521 |
|
| 2522 |
free=freespace(TD) |
| 2523 |
if free and free < newdebsize : |
| 2524 |
raise DebDeltaError('Error: not enough disk space in '+TD, True) |
| 2525 |
|
| 2526 |
delta=abspath(delta) |
| 2527 |
|
| 2528 |
#generater for numbered files |
| 2529 |
def a_numb_file_gen(): |
| 2530 |
deltacount = 0 |
| 2531 |
while 1: |
| 2532 |
yield str(deltacount) |
| 2533 |
deltacount+=1 |
| 2534 |
a_numb_file=a_numb_file_gen() |
| 2535 |
a_numb_patch=a_numb_file_gen() |
| 2536 |
|
| 2537 |
####################### class Script ######################## |
| 2538 |
class Script: |
| 2539 |
"""This class helps create the script 'patch.sh' that is the core of the delta. |
| 2540 |
The script recreates the new deb. See documentation of do_delta_() for details. |
| 2541 |
""" |
| 2542 |
def __init__(self, delta_uses_infifo): |
| 2543 |
#start writing script |
| 2544 |
self.fd=open(TD+'PATCH/patch.sh','w') |
| 2545 |
self.fd.write('#!/bin/bash -e\n') |
| 2546 |
self.member=None |
| 2547 |
self.current_chunk_name=None |
| 2548 |
self.delta_uses_infifo=delta_uses_infifo |
| 2549 |
if delta_uses_infifo : #create the fifo as input for xdelta3 |
| 2550 |
self.the_fifo=next(a_numb_file) |
| 2551 |
self.fd.write("mkfifo %s\n" % self.the_fifo) |
| 2552 |
else: |
| 2553 |
self.the_fifo=None |
| 2554 |
#this is used when recompressing data.tar.zx |
| 2555 |
self.xz_parameters=None |
| 2556 |
#this is used when recompressing data.tar.gz |
| 2557 |
self.gz_command=None |
| 2558 |
|
| 2559 |
def write(self,s): |
| 2560 |
"verbatim write in the script" |
| 2561 |
self.fd.write(s) |
| 2562 |
def close(self): |
| 2563 |
if self.the_fifo: |
| 2564 |
self.fd.write("rm %s\n" % self.the_fifo) |
| 2565 |
self.fd.close() |
| 2566 |
|
| 2567 |
def zip(self, n, cn, newhead=None): |
| 2568 |
"""inverts the unzip() function ; optionally, forces .gz header (to fight changes in libz) |
| 2569 |
This is obsolete, not efficient, left as a compatibility layer.""" |
| 2570 |
self.fd.write('cat "'+n+'" | ') |
| 2571 |
self.zip_piped(cn, newhead) |
| 2572 |
self.fd.write(" > '"+n+cn+"' && rm '"+n+"'\n") |
| 2573 |
|
| 2574 |
def zip_piped(self, cn, newhead=None): |
| 2575 |
"inverts the unzip() function, with piped behaviour" |
| 2576 |
if cn == '.gz' : |
| 2577 |
cmd=str.join(' ',self.gz_command) |
| 2578 |
if newhead: |
| 2579 |
s=prepare_for_echo(newhead) |
| 2580 |
self.fd.write("($E '"+ s +"' && "+cmd+" | tail -c +"+str(len(newhead)+1)+')') |
| 2581 |
else: |
| 2582 |
self.fd.write(cmd) |
| 2583 |
elif cn == '.bz2' : |
| 2584 |
info_append('needs-minibzip2') |
| 2585 |
self.fd.write('./minibzip2 -9') |
| 2586 |
elif cn == '.lzma' : |
| 2587 |
info_append('needs-lzma') |
| 2588 |
self.fd.write('lzma -9') |
| 2589 |
elif cn == '.xz' : |
| 2590 |
info_append('needs-xz') |
| 2591 |
if self.xz_parameters==None: |
| 2592 |
self.fd.write('xz -c') |
| 2593 |
else: |
| 2594 |
self.fd.write('xz -c '+self.xz_parameters) |
| 2595 |
else: assert(0) |
| 2596 |
|
| 2597 |
def start_member(self, ar_line, newname, extrachar): |
| 2598 |
"start a new 'ar' member" |
| 2599 |
self.member=newname |
| 2600 |
self.ar_line=ar_line |
| 2601 |
self.extrachar=extrachar |
| 2602 |
assert(self.current_chunk_name==None) |
| 2603 |
self.fd.write('{\n') |
| 2604 |
|
| 2605 |
def end_member(self): |
| 2606 |
assert(self.member) |
| 2607 |
self.member=None |
| 2608 |
self.fd.write('}\n') |
| 2609 |
|
| 2610 |
def start_rebuilding(self): |
| 2611 |
"starts the first part of the delta/recompressing pipe (for a 'ar' member)" |
| 2612 |
self.fd.write('(') |
| 2613 |
def end_rebuilding(self): |
| 2614 |
"ends the first part of the delta/recompressing pipe" |
| 2615 |
self.fd.write(')|') #pipe the delta/recompressing pipe |
| 2616 |
|
| 2617 |
def recompressing(self,new_filename,new_filename_ext,new_file_zip_head): |
| 2618 |
self.fd.write('(') |
| 2619 |
append_NEW_file(self.ar_line) |
| 2620 |
script.zip_piped(new_filename_ext,new_file_zip_head) |
| 2621 |
self.fd.write(' >> NEW.file\n') #end delta tar |
| 2622 |
if self.extrachar : |
| 2623 |
append_NEW_file(self.extrachar) |
| 2624 |
self.fd.write(')\n') |
| 2625 |
|
| 2626 |
def recompressing_by_arg(self,new_filename,new_filename_ext,new_file_zip_head,new_file_size): |
| 2627 |
"flexible recompressing for data.tar , depending on first argument passed to the script" |
| 2628 |
self.fd.write('( if test "$1" = "" \n then\n') |
| 2629 |
self.recompressing(new_filename,new_filename_ext,new_file_zip_head) |
| 2630 |
self.fd.write('elif test "$1" = "unzipped" \n then\n') |
| 2631 |
#http://en.wikipedia.org/wiki/Ar_(Unix) |
| 2632 |
assert(new_filename == 'NEW/data.tar') |
| 2633 |
ar_line_unzipped=b'data.tar'.ljust(16)+self.ar_line[16:48]+str2bytes(str(new_file_size)).ljust(10)+b'`\n' |
| 2634 |
append_NEW_file(ar_line_unzipped) #there is no extra char, tar is 512b blocks |
| 2635 |
self.fd.write('cat >> NEW.file \n elif test "$1" = "piped" \n then cat >> data.pipe \n fi )\n') |
| 2636 |
|
| 2637 |
def start_chunk(self, current_chunk_name): |
| 2638 |
"start the pipe to create the chunk. The chunk is always piped" |
| 2639 |
self.fd.write('(') |
| 2640 |
self.current_chunk_name=current_chunk_name |
| 2641 |
def end_chunk(self, current_chunk_name): |
| 2642 |
"""this ends the 'data part' of a chunk, and writes the old data somewhere; |
| 2643 |
a successive script code (generated by delta_files() ) |
| 2644 |
will then delta it to transform old data into new data.""" |
| 2645 |
assert(self.current_chunk_name==current_chunk_name) |
| 2646 |
if self.the_fifo: |
| 2647 |
self.fd.write(') > '+self.the_fifo+'&\n') #write to fifo, background |
| 2648 |
else: |
| 2649 |
self.fd.write(') > '+current_chunk_name+'\n') #write chunk |
| 2650 |
self.current_chunk_name=None |
| 2651 |
|
| 2652 |
def md5_check_file(self, n, md5=None): |
| 2653 |
"add a md5 check in the script (this is done only if a lot -d are passed on cmdline)" |
| 2654 |
if md5==None: |
| 2655 |
assert(os.path.isfile(TD+n)) |
| 2656 |
md5=compute_md5(TD+n) |
| 2657 |
logger.debug(" adding extra MD5 for %r",n) |
| 2658 |
self.fd.write('if ! echo "%s %s" | md5sum -c --quiet ; then echo "%s is currupt!"; exit 1; fi\n' % (md5,n,n)) |
| 2659 |
|
| 2660 |
####################### end of class Script ######################## |
| 2661 |
|
| 2662 |
##### unpack control.tar.gz, scan control, write parameters |
| 2663 |
# |
| 2664 |
def info_append(s): |
| 2665 |
"smart appending that avoids duplicate entries" |
| 2666 |
if s not in info: |
| 2667 |
info.append(s) |
| 2668 |
|
| 2669 |
for o,l in ('OLD',ar_list_old) , ('NEW',ar_list_new): |
| 2670 |
os.mkdir(TD+o+'/CONTROL') |
| 2671 |
#unpack control.tar.gz |
| 2672 |
f=None |
| 2673 |
for a in l : |
| 2674 |
if a[:12] == b'control.tar.': |
| 2675 |
f=a |
| 2676 |
b={b'.gz':'-z', b'.xz':'-J', b'.bz2':'-j'}.get(f[11:]) |
| 2677 |
assert b != None |
| 2678 |
system(('ar','p',TD+o+'.file',f,'|','tar',b,'-xf','-','-C',TD+o+'/CONTROL'),TD) |
| 2679 |
## scan control |
| 2680 |
p=open(TD+'/'+o+'/CONTROL/control') |
| 2681 |
s=[] |
| 2682 |
scan_control(p,params=None,prefix=o,info=s) |
| 2683 |
p.close() |
| 2684 |
if VERBOSE : logger.debug(' '+o+': '+str.join(' ',[o[4:] for o in s])) |
| 2685 |
info = info + s |
| 2686 |
info.append('OLD/Size: '+str(olddebsize)) |
| 2687 |
info.append('NEW/Size: '+str(newdebsize)) |
| 2688 |
params=info_2_db(info) |
| 2689 |
|
| 2690 |
#scan debdelta.conf to find any special requirement |
| 2691 |
debdelta_conf=ConfigParser() |
| 2692 |
debdelta_conf.read(['/etc/debdelta/debdelta.conf', expanduser('~/.debdelta/debdelta.conf') ]) |
| 2693 |
|
| 2694 |
debdelta_conf_skip=[] |
| 2695 |
for s in debdelta_conf.sections(): |
| 2696 |
if fnmatch.fnmatch(params['OLD/Package'],s): |
| 2697 |
opt=debdelta_conf.options(s) |
| 2698 |
if 'skip' in opt: |
| 2699 |
debdelta_conf_skip += debdelta_conf.get(s,'skip').split(';') |
| 2700 |
break |
| 2701 |
|
| 2702 |
if VERBOSE > 1 : logger.debug(' debdelta.conf says we will skip: %r', debdelta_conf_skip) |
| 2703 |
|
| 2704 |
gpg_hashes = {} |
| 2705 |
|
| 2706 |
if DO_MD5 : |
| 2707 |
# compute a MD5 of NEW deb |
| 2708 |
newdeb_md5sum=compute_md5(TD+'NEW.file') |
| 2709 |
info.append('NEW/MD5sum: '+ newdeb_md5sum) |
| 2710 |
else: |
| 2711 |
newdeb_md5sum=None |
| 2712 |
|
| 2713 |
if NEEDSOLD : |
| 2714 |
#this delta needs the old deb |
| 2715 |
info.append('needs-old') |
| 2716 |
else: |
| 2717 |
info.append('old-data-tree') |
| 2718 |
info.append('old-control-tree') |
| 2719 |
|
| 2720 |
#do we use a fifo as input for xdelta3 |
| 2721 |
delta_uses_infifo = ('xdelta3-fifo' not in DISABLED_FEATURES) and (USE_DELTA_ALGO == 'xdelta3') |
| 2722 |
|
| 2723 |
#Note that there is just one object instanced from class Script() |
| 2724 |
script=Script(delta_uses_infifo) |
| 2725 |
|
| 2726 |
a=USE_DELTA_ALGO |
| 2727 |
if a == 'xdelta-bzip': |
| 2728 |
a='xdelta' |
| 2729 |
if not os.path.exists('/usr/bin/'+a): |
| 2730 |
raise DebDeltaError('please install the package "'+a+'".', retriable=True) |
| 2731 |
if delta_uses_infifo: |
| 2732 |
info.append('needs-xdelta3-fifo') |
| 2733 |
else: |
| 2734 |
info.append('needs-'+a) |
| 2735 |
del a |
| 2736 |
|
| 2737 |
#### check for disk space |
| 2738 |
if 'NEW/Installed-Size' in params and 'OLD/Installed-Size' in params: |
| 2739 |
free=freespace(TD) |
| 2740 |
instsize=int(params['NEW/Installed-Size']) + int(params['OLD/Installed-Size']) |
| 2741 |
if free and free < ( instsize * 1024 + + 2**23 + MAXMEMORY / 6 ) : |
| 2742 |
raise DebDeltaError(' Not enough disk space (%dkB) for creating delta (needs %dkB).' % \ |
| 2743 |
( int(free/1024) , instsize ) , True ) |
| 2744 |
|
| 2745 |
|
| 2746 |
############# check for conffiles |
| 2747 |
a=TD+'/OLD/CONTROL/conffiles' |
| 2748 |
if os.path.exists(a): |
| 2749 |
p=open(a) |
| 2750 |
#files do not have leading / |
| 2751 |
old_conffiles=[ de_bar(a) for a in p.read().split('\n') if a] |
| 2752 |
p.close() |
| 2753 |
else: |
| 2754 |
old_conffiles=[] |
| 2755 |
|
| 2756 |
## a=TD+'/OLD/CONTROL/list' |
| 2757 |
## if os.path.exists(a): |
| 2758 |
## p=open(a) |
| 2759 |
## for a in p: |
| 2760 |
## a=de_bar(de_n(a)) |
| 2761 |
## for j in debdelta_conf_skip: |
| 2762 |
## if fnmatch(a,j): |
| 2763 |
## old_conffiles.append(a) #OK, this abuses the name of the var a bit |
| 2764 |
## print ' REPR skip ',repr(a) |
| 2765 |
## else: |
| 2766 |
## print ' The old debian package ',olddeb,' does not contain a file list?!?' |
| 2767 |
|
| 2768 |
def shell_not_allowed(name): |
| 2769 |
"Strings that I do not trust to inject into the shell script; maybe I am a tad too paranoid..." |
| 2770 |
#FIXME should use it , by properly quoting for the shell script |
| 2771 |
return '"' in name or "'" in name or '\\' in name or '`' in name |
| 2772 |
|
| 2773 |
# uses MD5 to detect identical files (even when renamed) |
| 2774 |
def scan_md5(n): |
| 2775 |
md5={} |
| 2776 |
f=open(n) |
| 2777 |
a=de_n(f.readline()) |
| 2778 |
while a: |
| 2779 |
m , n = a[:32] , de_bar( a[34:] ) |
| 2780 |
md5[n]=m |
| 2781 |
a=de_n(f.readline()) |
| 2782 |
f.close() |
| 2783 |
return md5 |
| 2784 |
|
| 2785 |
|
| 2786 |
new_md5=None |
| 2787 |
if os.path.exists(TD+'/NEW/CONTROL/md5sums'): |
| 2788 |
new_md5=scan_md5(TD+'/NEW/CONTROL/md5sums') |
| 2789 |
|
| 2790 |
old_md5=None |
| 2791 |
if os.path.exists(TD+'/OLD/CONTROL/md5sums') : |
| 2792 |
old_md5=scan_md5(TD+'/OLD/CONTROL/md5sums') |
| 2793 |
|
| 2794 |
############### some routines to prepare delta of two files |
| 2795 |
|
| 2796 |
def patch_append(f): |
| 2797 |
#possibly GPG |
| 2798 |
if DO_GPG: |
| 2799 |
gpg_hashes[f] = _compute_hashes_(TD+'PATCH/'+f) |
| 2800 |
if VERBOSE > 2 : |
| 2801 |
a=os.stat(TD+'PATCH/'+f)[ST_SIZE] |
| 2802 |
logger.debug(' appending %r of size %d to delta, %3.2f%% of new .deb',f,a,( a * 100. / newdebsize )) |
| 2803 |
system(['ar','qSc', delta,f], TD+'/PATCH') |
| 2804 |
unlink(TD+'PATCH/'+f) |
| 2805 |
|
| 2806 |
def verbatim(f): |
| 2807 |
pp=next(a_numb_patch) |
| 2808 |
p = 'PATCH/'+pp |
| 2809 |
if VERBOSE > 1 : logger.debug(' including %r verbatim in patch', name) |
| 2810 |
os.rename(TD+f,TD+p) |
| 2811 |
patch_append(pp) |
| 2812 |
return p |
| 2813 |
|
| 2814 |
def unzip(f): |
| 2815 |
c='' |
| 2816 |
if f[-3:] == '.gz' : |
| 2817 |
system(('gunzip',f),TD) |
| 2818 |
f=f[:-3] |
| 2819 |
c='.gz' |
| 2820 |
elif f[-4:] == '.bz2' : |
| 2821 |
system(('bunzip2',f),TD) |
| 2822 |
f=f[:-4] |
| 2823 |
c='.bz2' |
| 2824 |
elif f[-5:] == '.lzma' : |
| 2825 |
info_append('needs-lzma') |
| 2826 |
system(('unlzma',f),TD) |
| 2827 |
f=f[:-5] |
| 2828 |
c='.lzma' |
| 2829 |
elif f[-3:] == '.xz' : |
| 2830 |
info_append('needs-xz') |
| 2831 |
system(('unxz',f),TD) |
| 2832 |
f=f[:-3] |
| 2833 |
c='.xz' |
| 2834 |
else: raise NotImplementedError(' dont know how to decompress '+repr(f)) |
| 2835 |
return (f,c) |
| 2836 |
|
| 2837 |
|
| 2838 |
|
| 2839 |
def delta_files__(o, n, p, algo, outpiped, infifo): |
| 2840 |
"delta of file 'o' to 'n' using/producing patch 'p' ; xdelta3 can also pipe" |
| 2841 |
this_delta_outpiped=False |
| 2842 |
#bdiff |
| 2843 |
#http://www.webalice.it/g_pochini/bdiff/ |
| 2844 |
if algo == 'bdiff': |
| 2845 |
system(('~/debdelta/bdiff-1.0.5/bdiff','-q','-nooldmd5','-nonewmd5','-d',o,n,p),TD) |
| 2846 |
script.write('~/debdelta/bdiff-1.0.5/bdiff -p '+o+' '+p+' '+n+'\n') |
| 2847 |
#zdelta |
| 2848 |
#http://cis.poly.edu/zdelta/ |
| 2849 |
elif algo == 'zdelta': |
| 2850 |
system(('~/debdelta/zdelta-2.1/zdc',o,n,p),TD) |
| 2851 |
script.write('~/debdelta/zdelta-2.1/zdu '+o+' '+p+' '+n+'\n') |
| 2852 |
#bdelta |
| 2853 |
#http://deltup.sf.net |
| 2854 |
elif algo == 'bdelta': |
| 2855 |
system(('~/debdelta/bdelta-0.1.0/bdelta',o,n,p),TD) |
| 2856 |
script.write('~/debdelta/bdelta-0.1.0/bpatch '+o+' '+n+' '+p+'\n') |
| 2857 |
#diffball |
| 2858 |
#http://developer.berlios.de/projects/diffball/ |
| 2859 |
elif algo == 'diffball': |
| 2860 |
system(('~/debdelta/diffball-0.7.2/differ',o,n,p),TD) |
| 2861 |
script.write('~/debdelta/diffball-0.7.2/patcher '+o+' '+p+' '+n+'\n') |
| 2862 |
#rdiff |
| 2863 |
elif algo == 'rdiff': |
| 2864 |
system(('rdiff','signature',o,'sign_file.tmp'),TD) |
| 2865 |
system(('rdiff','delta','sign_file.tmp',n,p),TD) |
| 2866 |
script.write('rdiff patch '+o+' '+p+' '+n+'\n') |
| 2867 |
#xdelta3 |
| 2868 |
elif algo == 'xdelta3' : |
| 2869 |
system(('xdelta3','-9','-R','-D','-n','-S','djw','-s',o,n,p),TD) |
| 2870 |
if infifo : o = infifo #use fifo as input |
| 2871 |
if outpiped: |
| 2872 |
this_delta_outpiped=True |
| 2873 |
script.write('xdelta3 -d -R -D -c -s '+o+' '+p+'\n') |
| 2874 |
else: |
| 2875 |
script.write('xdelta3 -d -R -D -s '+o+' '+p+' '+n+'\n') |
| 2876 |
## according to the man page, |
| 2877 |
## bsdiff uses memory equal to 17 times the size of oldfile |
| 2878 |
## but , in my experiments, this number is more like 12. |
| 2879 |
##But bsdiff is sooooo slow! |
| 2880 |
elif algo == 'bsdiff' : # not ALLOW_XDELTA or ( osize < (MAXMEMORY / 12)): |
| 2881 |
system(('bsdiff',o,n,p),TD) |
| 2882 |
script.write('bspatch '+o+' '+n+' '+p+'\n') |
| 2883 |
#seems that 'xdelta' is buggy on 64bit and different-endian machines |
| 2884 |
#xdelta does not deal with different endianness! |
| 2885 |
elif algo == 'xdelta-bzip' : |
| 2886 |
system(('xdelta','delta','--pristine','--noverify','-0','-m'+str(int(MAXMEMORY/1024))+'k',o,n,p),TD) |
| 2887 |
system('bzip2 -9 '+p,TD,(p,)) |
| 2888 |
script.write('bunzip2 '+p+'.bz2 ; xdelta patch '+p+' '+o+' '+n+'\n') |
| 2889 |
p += '.bz2' |
| 2890 |
elif algo == 'xdelta' : |
| 2891 |
system(('xdelta','delta','--pristine','--noverify','-9','-m'+str(int(MAXMEMORY/1024))+'k',o,n,p),TD) |
| 2892 |
script.write('xdelta patch '+p+' '+o+' '+n+'\n') |
| 2893 |
elif algo == 'jojodiff' : |
| 2894 |
system(('~/debdelta/jdiff06/src/jdiff','-b',o,n,p),TD) |
| 2895 |
script.write('~/debdelta/jdiff06/src/jpatch '+o+' '+p+' '+n+'\n') |
| 2896 |
else: raise AssertionError(' unsupported delta algo ') |
| 2897 |
return p, this_delta_outpiped |
| 2898 |
|
| 2899 |
def delta_files(o, n, outpiped=None, infifo=None): |
| 2900 |
" compute delta of two files , and prepare the script consequently" |
| 2901 |
nsize = os.path.getsize(TD+n) |
| 2902 |
osize = os.path.getsize(TD+o) |
| 2903 |
if VERBOSE > 1 : logger.debug(' compute delta for %s (%dkB) and %s (%dkB)', |
| 2904 |
o,osize/1024,n,nsize/1024) |
| 2905 |
# |
| 2906 |
p = 'PATCH/'+next(a_numb_patch) |
| 2907 |
tim = -time.time() |
| 2908 |
# |
| 2909 |
if DEBUG > 3 : script.md5_check_file(o) |
| 2910 |
# |
| 2911 |
if USE_DELTA_ALGO == 'bsdiff' and osize > ( 1.1 * (MAXMEMORY / 12)) and VERBOSE : |
| 2912 |
logger.debug(' Warning, memory usage by bsdiff on the order of %dMb', (12 * osize / 2**20)) |
| 2913 |
# |
| 2914 |
p, this_delta_outpiped = delta_files__(o, n, p, USE_DELTA_ALGO, outpiped, infifo) |
| 2915 |
#script.write(s) |
| 2916 |
# |
| 2917 |
if DEBUG > 2 and not this_delta_outpiped : script.md5_check_file(n) |
| 2918 |
# |
| 2919 |
tim += time.time() |
| 2920 |
# |
| 2921 |
global bsdiff_time, bsdiff_datasize |
| 2922 |
bsdiff_time += tim |
| 2923 |
bsdiff_datasize += nsize |
| 2924 |
# |
| 2925 |
if infifo: |
| 2926 |
script.write('rm '+p+'\n') |
| 2927 |
else: |
| 2928 |
script.write('rm '+o+' '+p+'\n') |
| 2929 |
## how did we fare ? |
| 2930 |
deltasize = os.path.getsize(TD+p) |
| 2931 |
if VERBOSE > 1 : |
| 2932 |
logger.debug(' delta is %3.2f%% of %s, speed: %dkB /sec', |
| 2933 |
( deltasize * 100. / nsize ) , n, (nsize / 1024. / ( tim + 0.001 ))) |
| 2934 |
#save it |
| 2935 |
patch_append(p[6:]) |
| 2936 |
#clean up |
| 2937 |
unlink(TD+o) |
| 2938 |
return this_delta_outpiped |
| 2939 |
|
| 2940 |
def cmp_gz(o,n): |
| 2941 |
"compare gzip files, ignoring header; returns first different byte (+-10), or True if equal" |
| 2942 |
of=open(o,'rb') |
| 2943 |
nf=open(n,'rb') |
| 2944 |
oa=of.read(10) |
| 2945 |
na=nf.read(10) |
| 2946 |
myord = ord if sys.version_info.major == 2 else (lambda x:x) |
| 2947 |
if na[:3] != b'\037\213\010' : |
| 2948 |
logger.warning(' Warning: was not created with gzip: %r',n) |
| 2949 |
nf.close() ; of.close() |
| 2950 |
return 0 |
| 2951 |
if oa[:3] != b'\037\213\010' : |
| 2952 |
logger.warning(' Warning: was not created with gzip: %r',o) |
| 2953 |
nf.close() ; of.close() |
| 2954 |
return 0 |
| 2955 |
oflag=myord(oa[3]) |
| 2956 |
if oflag & 0xf7: |
| 2957 |
logger.warning(' Warning: unsupported .gz flags: %r %r',oct(oflag),o) |
| 2958 |
if oflag & 8 : #skip orig name |
| 2959 |
oa=of.read(1) |
| 2960 |
while oa and myord(oa) != 0: |
| 2961 |
oa=of.read(1) |
| 2962 |
if not oa: |
| 2963 |
logger.warning(' Warning: premature EOF: %r',o) |
| 2964 |
nf.close() ; of.close() |
| 2965 |
return 0 |
| 2966 |
l=10 |
| 2967 |
nflag=myord(na[3]) |
| 2968 |
if nflag & 0xf7: |
| 2969 |
logger.warning(' Warning: unsupported .gz flags: %r %r',oct(nflag),n) |
| 2970 |
if nflag & 8 : #skip orig name |
| 2971 |
na=nf.read(1) |
| 2972 |
#s=na |
| 2973 |
while na and myord(na) != 0: |
| 2974 |
na=nf.read(1) |
| 2975 |
#s+=na |
| 2976 |
if not na: |
| 2977 |
logger.warning(' Warning: premature EOF: %r',n) |
| 2978 |
nf.close() ; of.close() |
| 2979 |
return 0 |
| 2980 |
l+=len(s) |
| 2981 |
#print repr(s) |
| 2982 |
while oa and na: |
| 2983 |
oa=of.read(2) |
| 2984 |
na=nf.read(2) |
| 2985 |
if oa != na: |
| 2986 |
return l |
| 2987 |
l+=2 |
| 2988 |
if oa or na: return l |
| 2989 |
return True |
| 2990 |
|
| 2991 |
def parse_gzip_header(n): |
| 2992 |
" n may be a file name, a function returning a stream, or a stream" |
| 2993 |
myord = ord if sys.version_info.major == 2 else (lambda x:x) |
| 2994 |
if type(n) in string_types: |
| 2995 |
f=open(TD+n,'rb') |
| 2996 |
a=f.read(10) |
| 2997 |
f.close() |
| 2998 |
elif type(n) in (FunctionType,LambdaType): |
| 2999 |
f=n() |
| 3000 |
a=f.read(10) |
| 3001 |
if isinstance(f,PopenPipe): f.close() |
| 3002 |
else: |
| 3003 |
a=n.read(10) |
| 3004 |
if a[:3] != b'\037\213\010' : |
| 3005 |
logger.debug(' Warning: was not created with gzip: %r',n) |
| 3006 |
return |
| 3007 |
flag=myord(a[3]) # mostly ignored :-> |
| 3008 |
orig_name='-n' |
| 3009 |
if flag & 8: |
| 3010 |
orig_name='-N' |
| 3011 |
if flag & 0xf7: |
| 3012 |
logger.debug(' Warning: unsupported .gz flags: %r %r',oct(flag),n) |
| 3013 |
#a[4:8] #mtime ! ignored ! FIXME will be changed... |
| 3014 |
#from deflate.c in gzip source code |
| 3015 |
format=myord(a[8]) |
| 3016 |
FAST=4 |
| 3017 |
SLOW=2 #unfortunately intermediate steps are lost.... |
| 3018 |
pack_level=6 |
| 3019 |
if format == 0 : |
| 3020 |
pass |
| 3021 |
elif format == FAST : |
| 3022 |
pack_level = 1 |
| 3023 |
elif format == SLOW : |
| 3024 |
pack_level = 9 |
| 3025 |
else: |
| 3026 |
logger.debug(' Warning: unsupported compression .gz format: %r %r',oct(format),n) |
| 3027 |
return |
| 3028 |
if myord(a[9]) != 3 : |
| 3029 |
if VERBOSE : logger.debug(' Warning: unknown OS in .gz format: %r %r',oct(myord(a[9])),n) |
| 3030 |
pack_list=[1,2,3,4,5,6,7,8,9] |
| 3031 |
del pack_list[pack_level-1] |
| 3032 |
pack_list.append(pack_level) |
| 3033 |
pack_list.reverse() |
| 3034 |
#print 'format ', repr(format), type(format), format==FAST, pack_list |
| 3035 |
return pack_list,orig_name |
| 3036 |
|
| 3037 |
def delta_gzipped_files(o,n): |
| 3038 |
"delta o and n, replace o with n" |
| 3039 |
assert(o[-3:] == '.gz' and n[-3:] == '.gz') |
| 3040 |
before=cmp_gz(TD+o,TD+n) |
| 3041 |
if before == True: |
| 3042 |
if VERBOSE > 3: logger.debug(' equal but for header: %r',n) |
| 3043 |
return |
| 3044 |
#compare the cost of leaving as is , VS the minimum cost of delta |
| 3045 |
newsize=os.path.getsize(TD+n) |
| 3046 |
if ( newsize - before + 10 ) < 200 : |
| 3047 |
if VERBOSE > 3: logger.debug(' not worthwhile gunzipping: %r',n) |
| 3048 |
return |
| 3049 |
z=parse_gzip_header(n) |
| 3050 |
if z==None: |
| 3051 |
return |
| 3052 |
pack_list,orig_name=z |
| 3053 |
pack_level=pack_list[0] |
| 3054 |
#OK, it seems we can play our trick |
| 3055 |
p='_tmp_' |
| 3056 |
#unzip new file |
| 3057 |
pro=subprocess.Popen(('zcat',TD+n),stdout=open(TD+p+'.new','wb'), cwd=TD) |
| 3058 |
pro.wait() |
| 3059 |
if pro.returncode: raise DebDeltaError('argghh, zcat failed on us') |
| 3060 |
#test our ability of recompressing |
| 3061 |
best_r=0 |
| 3062 |
best_flag=None |
| 3063 |
for i in pack_list: |
| 3064 |
#force -n ... no problem with timestamps |
| 3065 |
gzip_flags="-"+str(i) |
| 3066 |
pro=subprocess.Popen(("gzip",'-c','-n',gzip_flags,TD+p+'.new'),stdout=open(TD+p+'.faked.gz','wb')) |
| 3067 |
pro.wait() |
| 3068 |
if pro.returncode: DebDeltaError('Argh, gzip failed on us') |
| 3069 |
r=cmp_gz(TD+n,TD+p+'.faked.gz') |
| 3070 |
if r > best_r : |
| 3071 |
best_r = r |
| 3072 |
best_flag = gzip_flags |
| 3073 |
if r == True: |
| 3074 |
break |
| 3075 |
if i == pack_level and VERBOSE > 3: |
| 3076 |
logger.debug(' warning: wrong guess to re-gzip to equal file: %r %r %r',gzip_flags,r,n) |
| 3077 |
if r != True: |
| 3078 |
if VERBOSE > 1 : logger.debug(' warning: cannot re-gzip to equal file, best was %d / %d , "%s" : %s ',best_r,newsize,best_flag,n) |
| 3079 |
os.unlink(TD+p+".new") ; os.unlink(TD+p+'.faked.gz') |
| 3080 |
return |
| 3081 |
#actual delta of decompressed files |
| 3082 |
pro=subprocess.Popen(('zcat',TD+o),stdout=open(TD+p+'.old','wb'), cwd=TD) |
| 3083 |
pro.wait() |
| 3084 |
if pro.returncode: raise DebDeltaError('argh, zcat failed on us') |
| 3085 |
script.write("zcat '"+o+"' > "+p+".old ; rm '"+o+"' \n") |
| 3086 |
if VERBOSE > 2 : logger.debug(' '+n[9:]+(' (= to %d%%): ' % (100*before/newsize))) |
| 3087 |
delta_files(p+'.old',p+'.new') |
| 3088 |
script.write("gzip -c -n "+gzip_flags+" < "+p+".new > '"+o+"' ; rm "+p+".new\n") |
| 3089 |
#replace the old file with the best that we can do re-gzipping the new file |
| 3090 |
#this is important in the rest of the delta-ing process |
| 3091 |
os.rename(TD+p+'.faked.gz',TD+o) |
| 3092 |
if DEBUG > 1 : |
| 3093 |
script.md5_check_file(o, compute_md5(TD+o)) |
| 3094 |
|
| 3095 |
def guess_gz_command(o, check=True): |
| 3096 |
"""tries to guess the parameters used to compress, returns a command vector |
| 3097 |
if it fails, returns False |
| 3098 |
o may be a file name, a function returning a stream, or a stream |
| 3099 |
""" |
| 3100 |
h=parse_gzip_header(o) |
| 3101 |
if h==None: |
| 3102 |
return False |
| 3103 |
if type(o) in (FunctionType,LambdaType): |
| 3104 |
o=o() |
| 3105 |
elif type(o) in string_types: |
| 3106 |
o=open(o,'rb') |
| 3107 |
z=tempfile.NamedTemporaryFile(suffix='.gz',delete=False) |
| 3108 |
shutil.copyfileobj(o,z) |
| 3109 |
z.flush() |
| 3110 |
if isinstance(o,PopenPipe): o.close() |
| 3111 |
pack_list,orig_name=h |
| 3112 |
pack_list.reverse() |
| 3113 |
cmd_list=[(['gzip','-nc'],l) for l in pack_list] |
| 3114 |
cmd_list.append(([minigzip],9)) ##old method up to dpkg-deb in 2014-01-15 |
| 3115 |
wcmd=False #fixme, there is no way to distinguish output of minigzip or gzip from header |
| 3116 |
if check: |
| 3117 |
redo=True |
| 3118 |
while redo and cmd_list: |
| 3119 |
cmd,par=cmd_list.pop() |
| 3120 |
wcmd=copy(cmd) |
| 3121 |
wcmd.append('-'+str(par)) |
| 3122 |
if VERBOSE > 2 : logger.debug(' Testing command %r',wcmd) |
| 3123 |
w=PopenPipe(['zcat',z.name,'|']+wcmd+['|','cmp','-',z.name],\ |
| 3124 |
stdout=open(os.devnull,'w'),stderr=open(os.devnull,'wb'),close_fds=True) |
| 3125 |
redo=False |
| 3126 |
w.wait() |
| 3127 |
if w.returncode : |
| 3128 |
redo=True |
| 3129 |
if VERBOSE or DEBUG : logger.debug(' Tried gzip options but failed: %r',wcmd) |
| 3130 |
elif VERBOSE > 2 : logger.debug(' Success with command %r',wcmd) |
| 3131 |
if redo and not cmd_list: |
| 3132 |
logger.debug(' NO MORE OPTIONS !') |
| 3133 |
os.unlink(z.name) |
| 3134 |
return False |
| 3135 |
os.unlink(z.name) |
| 3136 |
return wcmd |
| 3137 |
|
| 3138 |
def guess_xz_parameters(o, check=True): |
| 3139 |
"""tries to guess the parameters used to compress, returns a a string of options |
| 3140 |
if it fails, returns False |
| 3141 |
o may be a file name, a function returning a stream, or a stream |
| 3142 |
""" |
| 3143 |
par='' |
| 3144 |
crc='' |
| 3145 |
thread='' |
| 3146 |
if type(o) in (FunctionType,LambdaType): |
| 3147 |
o=o() |
| 3148 |
elif type(o) in string_types: |
| 3149 |
o=open(o,'rb') |
| 3150 |
z=tempfile.NamedTemporaryFile(suffix='.xz',delete=False) |
| 3151 |
#unfortunately 'xz --list' does not work on pipes! |
| 3152 |
shutil.copyfileobj(o,z) |
| 3153 |
z.flush() |
| 3154 |
if isinstance(o,PopenPipe): o.close() |
| 3155 |
b=subprocess.Popen(['xz','-vv','--robot','--list',z.name],stdout=subprocess.PIPE, |
| 3156 |
**Popen_args) |
| 3157 |
for a in b.stdout: |
| 3158 |
a=a.rstrip('\n') |
| 3159 |
a=str.split(a,'\t') |
| 3160 |
if a[0]=='block': |
| 3161 |
if crc and crc != a[9]: |
| 3162 |
logger.warning(" warning : this xz -- compressed file was compressed with variable blocks crc ?! '%s' != '%s'",crc,a[9]) |
| 3163 |
crc=a[9] |
| 3164 |
if par and par != a[15]: |
| 3165 |
logger.warning(" warning : this xz -- compressed file was compressed with variable blocks options ?! '%s' != '%s'",par,a[15]) |
| 3166 |
par=a[15] |
| 3167 |
if not thread and a[12]=='cu': |
| 3168 |
thread='-T2' |
| 3169 |
#print ' guessed par crc ',par,crc |
| 3170 |
if crc: |
| 3171 |
crc=crc.lower() |
| 3172 |
if crc=='sha-256': crc='sha256' |
| 3173 |
if crc not in ('crc32','crc64','sha256'): |
| 3174 |
logger.warning(' Unknown XZ crc %r',crc) |
| 3175 |
crc='' |
| 3176 |
PARS=['-6e','-9','-9e'] |
| 3177 |
if par: |
| 3178 |
PARS.append(par) |
| 3179 |
if par == '--lzma2=dict=1MiB': |
| 3180 |
# dbgsym deb files are compressed with -1e |
| 3181 |
PARS.append('-1') |
| 3182 |
PARS.append('-1e') |
| 3183 |
if check: |
| 3184 |
redo=True |
| 3185 |
while redo and PARS: |
| 3186 |
par=PARS.pop() |
| 3187 |
w=['xz','-c'] |
| 3188 |
if par: |
| 3189 |
w.append(par) |
| 3190 |
if crc: |
| 3191 |
w+=['-C',crc] |
| 3192 |
if thread: |
| 3193 |
w.append(thread) |
| 3194 |
w.append('-') |
| 3195 |
if VERBOSE > 2 : logger.debug(' Testing XZ options %r',w) |
| 3196 |
redo=False |
| 3197 |
c=PopenPipe(['unxz','-c',z.name,'|']+w+['|','cmp','-',z.name],\ |
| 3198 |
stdout=open(os.devnull,'wb'),stderr=open(os.devnull,'wb'),close_fds=True) |
| 3199 |
c.wait() |
| 3200 |
if c.returncode : |
| 3201 |
redo=True |
| 3202 |
if VERBOSE or DEBUG : logger.debug(' Tried XZ options but failed: %r %r %r',par,crc,thread) |
| 3203 |
if redo and not PARS: |
| 3204 |
#print ' HO FINITO LE OPZIONI !' |
| 3205 |
os.unlink(z.name) |
| 3206 |
return False |
| 3207 |
if crc: |
| 3208 |
crc=' -C '+crc |
| 3209 |
if thread: |
| 3210 |
thread=' '+thread |
| 3211 |
os.unlink(z.name) |
| 3212 |
return par+crc+thread |
| 3213 |
|
| 3214 |
########### helper sh functions for script, for delta_tar() |
| 3215 |
|
| 3216 |
import difflib, re |
| 3217 |
re_numbers=re.compile(r'^[0-9][0-9]*$') |
| 3218 |
|
| 3219 |
def file_similarity_premangle(fp): |
| 3220 |
fps=fp.split('/') |
| 3221 |
bns=fps[-1].split('.') |
| 3222 |
j=len(bns)-1 #search first "non numeric" extension, and put it last |
| 3223 |
while j >= 0 and re_numbers.match(bns[j]): |
| 3224 |
j-=1 |
| 3225 |
if j >= 0: |
| 3226 |
a=bns.pop(j) |
| 3227 |
r = fps[:-1] + bns + [a,] |
| 3228 |
else: |
| 3229 |
r = fps[:-1] + bns |
| 3230 |
return r |
| 3231 |
|
| 3232 |
def files_similarity_score__noext__(oo,nn): |
| 3233 |
ln=len(nn) |
| 3234 |
lo=len(oo) |
| 3235 |
l=0 |
| 3236 |
while oo and nn: |
| 3237 |
while oo and nn and oo[-1] == nn[-1]: |
| 3238 |
oo=oo[:-1] |
| 3239 |
nn=nn[:-1] |
| 3240 |
if not oo or not nn: break |
| 3241 |
while oo and nn and oo[0] == nn[0]: |
| 3242 |
oo=oo[1:] |
| 3243 |
nn=nn[1:] |
| 3244 |
if not oo or not nn: break |
| 3245 |
if len(nn) > 1 and oo[0] == nn[1]: |
| 3246 |
l+=1 |
| 3247 |
nn=nn[1:] |
| 3248 |
if len(oo) > 1 and oo[1] == nn[0]: |
| 3249 |
l+=1 |
| 3250 |
oo=oo[1:] |
| 3251 |
if not oo or not nn: break |
| 3252 |
if oo[-1] != nn[-1]: |
| 3253 |
oo=oo[:-1] |
| 3254 |
nn=nn[:-1] |
| 3255 |
l+=2 |
| 3256 |
if not oo or not nn: break |
| 3257 |
if oo[0] != nn[0]: |
| 3258 |
oo=oo[1:] |
| 3259 |
nn=nn[1:] |
| 3260 |
l+=2 |
| 3261 |
return (l +len(oo) + len(nn)) * 2.0 / float(ln+lo) |
| 3262 |
|
| 3263 |
def files_similarity_score__(oo,nn): |
| 3264 |
oo=copy(oo) |
| 3265 |
nn=copy(nn) |
| 3266 |
if oo.pop() != nn.pop() : |
| 3267 |
return 0.2 + files_similarity_score__noext__(oo,nn) |
| 3268 |
else: |
| 3269 |
return files_similarity_score__noext__(oo,nn) |
| 3270 |
|
| 3271 |
def files_similarity_score__difflib__(oo,nn): |
| 3272 |
"compute similarity by difflib. Too slow." |
| 3273 |
if oo == nn : |
| 3274 |
return 0 |
| 3275 |
d=difflib.context_diff(oo,nn,'','','','',0,'') |
| 3276 |
d=[a for a in tuple(d) if a and a[:3] != '---' and a[:3] != '***' ] |
| 3277 |
if oo[-1] != nn[-1] : #penalty for wrong extension |
| 3278 |
return 0.2+float(len(d)) * 2.0 / float(len(oo)+len(nn)) |
| 3279 |
else: |
| 3280 |
return float(len(d)) * 2.0 / float(len(oo)+len(nn)) |
| 3281 |
|
| 3282 |
def files_similarity_score(oo,nn): |
| 3283 |
if oo == nn : |
| 3284 |
return 0 |
| 3285 |
if type(oo) in string_types: |
| 3286 |
oo=file_similarity_premangle(oo) |
| 3287 |
if type(nn) in string_types: |
| 3288 |
nn=file_similarity_premangle(nn) |
| 3289 |
return files_similarity_score__(oo,nn) |
| 3290 |
|
| 3291 |
def fake_tar_header_2nd(): |
| 3292 |
" returns the second part of a tar header , for regular files and dirs" |
| 3293 |
# The following code was contributed by Detlef Lannert. |
| 3294 |
# into /usr/lib/python2.3/tarfile.py |
| 3295 |
MAGIC = b"ustar" # magic tar string |
| 3296 |
VERSION = b"00" # version number |
| 3297 |
NUL = b"\0" # the null character |
| 3298 |
parts = [] |
| 3299 |
for value, fieldsize in ( |
| 3300 |
(b"", 100), |
| 3301 |
# unfortunately this is not what DPKG does |
| 3302 |
#(MAGIC, 6), |
| 3303 |
#(VERSION, 2), |
| 3304 |
# this is what DPKG does |
| 3305 |
(b'ustar \x00',8), |
| 3306 |
(b"root", 32), |
| 3307 |
(b"root", 32), |
| 3308 |
(b"%07o" % 0, 8), |
| 3309 |
(b"%07o" % 0, 8), |
| 3310 |
(b"", 155) |
| 3311 |
): |
| 3312 |
l = len(value) |
| 3313 |
parts.append(value + (fieldsize - l) * NUL) |
| 3314 |
buf = b"".join(parts) |
| 3315 |
return buf |
| 3316 |
|
| 3317 |
fake_tar_2nd=fake_tar_header_2nd() |
| 3318 |
fake_tar_2nd_echo=prepare_for_echo(fake_tar_2nd) |
| 3319 |
script.write("FTH='"+fake_tar_2nd_echo+"'\n") |
| 3320 |
script.write("E='echo -ne'\n") |
| 3321 |
|
| 3322 |
global time_corr |
| 3323 |
time_corr=0 |
| 3324 |
|
| 3325 |
#################### vvv delta_tar vvv ########################### |
| 3326 |
def delta_tar(old_filename, new_filename, CWD, old_forensic,\ |
| 3327 |
skip=[], old_md5={}, new_md5={},\ |
| 3328 |
chunked_p=(not delta_uses_infifo) ,debdelta_conf_skip=()): |
| 3329 |
" compute delta of two tar files, and prepare the script consequently" |
| 3330 |
assert( type(old_filename) in string_types or type(old_filename) == FunctionType ) |
| 3331 |
|
| 3332 |
script.write('ECR () { $E "$1" ; $E "${FTH}" ; cat OLD/'+CWD+'/"$1" ; rm OLD/'+CWD+'/"$1" ;}\n') |
| 3333 |
script.write('EC () { $E "$1" ; $E "${FTH}" ; cat OLD/'+CWD+'/"$1" ;}\n') |
| 3334 |
|
| 3335 |
###### uncompress and scan the old tar file, extract regular files |
| 3336 |
if type(old_filename) in string_types : |
| 3337 |
(old_filename,old_filename_ext) = unzip(old_filename) |
| 3338 |
oldtar = tarfile.open(TD+old_filename, "r") |
| 3339 |
elif type(old_filename) in (FunctionType,LambdaType) : |
| 3340 |
old_filename_ext=None |
| 3341 |
oldfileobj = old_filename() |
| 3342 |
oldtar = tarfile.open(mode="r|", fileobj=oldfileobj) |
| 3343 |
else: |
| 3344 |
old_filename_ext=None |
| 3345 |
oldtar = tarfile.open(mode="r|", fileobj=old_filename) |
| 3346 |
oldnames = [] |
| 3347 |
oldtarinfos = {} |
| 3348 |
for oldtarinfo in oldtar: |
| 3349 |
oldname = de_bar(oldtarinfo.name) |
| 3350 |
if old_forensic != None: |
| 3351 |
#fixme : devices are not supported (but debian policy does not allow them) |
| 3352 |
old_forensic.append([oldtarinfo.name,oldtarinfo.mode,oldtarinfo.type,\ |
| 3353 |
oldtarinfo.uid,oldtarinfo.gid,oldtarinfo.uname,oldtarinfo.gname,oldtarinfo.linkname]) |
| 3354 |
#this always happens |
| 3355 |
#if VERBOSE > 3 and oldname != de_bar(oldname): |
| 3356 |
# print ' filename in old tar has weird ./ in front: ' , oldname |
| 3357 |
|
| 3358 |
if not oldtarinfo.isreg(): |
| 3359 |
if VERBOSE > 2 : logger.debug(' skipping old non-regular %r',oldname) |
| 3360 |
continue |
| 3361 |
|
| 3362 |
if oldtarinfo.size == 0: |
| 3363 |
if VERBOSE > 2 : logger.debug(' skipping old empty %r',oldname) |
| 3364 |
continue |
| 3365 |
|
| 3366 |
if shell_not_allowed(oldname): |
| 3367 |
if VERBOSE > 2 : logger.debug(' skipping non-allowed-name %r',oldname) |
| 3368 |
continue |
| 3369 |
|
| 3370 |
for j in debdelta_conf_skip: |
| 3371 |
if fnmatch.fnmatch(oldname,j): |
| 3372 |
if VERBOSE > 2 : logger.debug(' skipping following as per rule %r',j) |
| 3373 |
skip.append(oldname) |
| 3374 |
break |
| 3375 |
|
| 3376 |
if oldname in skip: |
| 3377 |
if VERBOSE > 2 : logger.debug(' skipping %r',oldname) |
| 3378 |
if old_forensic != None: |
| 3379 |
oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) |
| 3380 |
old_forensic.append(old_forensic.pop()[:-1] + \ |
| 3381 |
[sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname))]) |
| 3382 |
continue |
| 3383 |
|
| 3384 |
oldnames.append(oldname) |
| 3385 |
oldtarinfos[oldname] = oldtarinfo |
| 3386 |
oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) |
| 3387 |
if old_forensic != None: |
| 3388 |
old_forensic.append(old_forensic.pop()[:-1] + \ |
| 3389 |
[sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname))]) |
| 3390 |
|
| 3391 |
oldtar.close() |
| 3392 |
if type(old_filename) in string_types : |
| 3393 |
unlink(TD+old_filename) |
| 3394 |
else: |
| 3395 |
while oldfileobj.read(512): |
| 3396 |
pass |
| 3397 |
###### scan the new tarfile, save info regarding regular files therein |
| 3398 |
#save header part of new_filename, since it changes in newer versions |
| 3399 |
f=open(TD+new_filename,'rb') |
| 3400 |
new_file_zip_head=f.read(20) |
| 3401 |
f.close() |
| 3402 |
(new_filename,new_filename_ext) = unzip(new_filename) |
| 3403 |
new_file_md5=compute_md5(TD+new_filename) |
| 3404 |
new_file_size=os.path.getsize(TD+new_filename) |
| 3405 |
info_append(new_filename+': '+new_file_md5+' '+str(new_file_size)) |
| 3406 |
#### scan the new tarfile, compare to the old tar contents |
| 3407 |
assert(0 == (new_file_size % 512)) |
| 3408 |
newtar = tarfile.open(TD+new_filename, "r") |
| 3409 |
# dpkg lately uses PAX format, dunno the difference, debdelta seems to work fine |
| 3410 |
logger.log(logging.DEBUG if newtar.format in (tarfile.GNU_FORMAT , tarfile.PAX_FORMAT) else logging.WARNING, |
| 3411 |
' tar file format %r for %r'%(newtar.format,new_filename)) |
| 3412 |
newnames = [] |
| 3413 |
newtarinfos = {} |
| 3414 |
for newtarinfo in newtar: |
| 3415 |
newname = newtarinfo.name |
| 3416 |
#just curious to know |
| 3417 |
t=newtarinfo.type |
| 3418 |
a=newtarinfo.mode |
| 3419 |
if VERBOSE and (( t == '2' and a != 0o777 ) or \ |
| 3420 |
( t == '0' and ( (a & 0o400 ) == 0 )) or \ |
| 3421 |
( t == '5' and ( (a & 0o500 ) == 0 ))): |
| 3422 |
logger.debug(' weird permission %r %r %r', newname, oct(a), repr(newtarinfo.type)) |
| 3423 |
### |
| 3424 |
if not newtarinfo.isreg(): |
| 3425 |
continue |
| 3426 |
if VERBOSE > 3 and newname != de_bar(newname): |
| 3427 |
logger.debug(' filename in new tar has weird ./ in front: %r' , newname) |
| 3428 |
newname = de_bar(newname) |
| 3429 |
newnames.append(newname) |
| 3430 |
newtarinfos[newname] = newtarinfo |
| 3431 |
|
| 3432 |
old_used={} |
| 3433 |
correspondence={} |
| 3434 |
|
| 3435 |
###### find correspondences between old tar and new tar contents |
| 3436 |
global time_corr |
| 3437 |
time_corr=-time.time() |
| 3438 |
|
| 3439 |
if VERBOSE > 2 : logger.debug(' finding correspondences for %r', new_filename) |
| 3440 |
|
| 3441 |
reverse_old_md5={} |
| 3442 |
if old_md5: |
| 3443 |
for o in old_md5: |
| 3444 |
if o in oldnames: |
| 3445 |
reverse_old_md5[old_md5[o]] = o |
| 3446 |
else: |
| 3447 |
#would you believe? many packages contain MD5 for files they do not ship... |
| 3448 |
if VERBOSE > 1 and o not in skip: logger.debug(' hmmm... there is a md5 but not a file: %r',o) |
| 3449 |
|
| 3450 |
#database of databases of premangled old names , by "extension" and name |
| 3451 |
oldnames_premangle={} |
| 3452 |
for o in oldnames: |
| 3453 |
om=file_similarity_premangle(o) |
| 3454 |
a=om[-1] # "extension" |
| 3455 |
if a not in oldnames_premangle: |
| 3456 |
oldnames_premangle[a]={} |
| 3457 |
oldnames_premangle[a][o]=om |
| 3458 |
|
| 3459 |
for newname in newnames: |
| 3460 |
newtarinfo=newtarinfos[newname] |
| 3461 |
oldname=None |
| 3462 |
#ignore empty files |
| 3463 |
if newtarinfo.size == 0: |
| 3464 |
continue |
| 3465 |
#try correspondence by MD5 |
| 3466 |
if new_md5 and newname in new_md5: |
| 3467 |
md5=new_md5[newname] |
| 3468 |
if md5 in reverse_old_md5: |
| 3469 |
oldname=reverse_old_md5[md5] |
| 3470 |
if VERBOSE > 2 : |
| 3471 |
if oldname == newname : |
| 3472 |
logger.debug(' use identical old file: %r',newname) |
| 3473 |
else: |
| 3474 |
logger.debug(' use identical old file: %r %r',oldname, newname) |
| 3475 |
#try correspondence by file name |
| 3476 |
if oldname == None and newname in oldnames: |
| 3477 |
oldname=newname |
| 3478 |
if VERBOSE > 2 : logger.debug(' use same name old file: %r',newname) |
| 3479 |
#try correspondence by file name and len similarity |
| 3480 |
np=file_similarity_premangle(newname) |
| 3481 |
ne=np[-1] # "extension" |
| 3482 |
if oldname == None and ne in oldnames_premangle: |
| 3483 |
basescore=1.6 |
| 3484 |
nl=newtarinfo.size |
| 3485 |
for o in oldnames_premangle[ne]: |
| 3486 |
op=oldnames_premangle[ne][o] |
| 3487 |
l=oldtarinfos[o].size |
| 3488 |
sfile=files_similarity_score__noext__(op,np) |
| 3489 |
slen = abs(float(l - nl))/float(l+nl) |
| 3490 |
s=slen+sfile |
| 3491 |
if VERBOSE > 3 : logger.debug(' name/len diff %.2f+%.2f=%.2f %r',slen,sfile,s, o) |
| 3492 |
if s < basescore: |
| 3493 |
oldname=o |
| 3494 |
basescore=s |
| 3495 |
if oldname and VERBOSE > 2 : logger.debug(' best similar %.3f %r %r',basescore,newname,oldname) |
| 3496 |
if not oldname: |
| 3497 |
if VERBOSE > 2 : logger.debug(' no correspondence for: %r',newname) |
| 3498 |
continue |
| 3499 |
#we have correspondence, lets store |
| 3500 |
if oldname not in old_used: |
| 3501 |
old_used[oldname]=[] |
| 3502 |
old_used[oldname].append(newname) |
| 3503 |
correspondence[newname]=oldname |
| 3504 |
|
| 3505 |
time_corr+=time.time() |
| 3506 |
if VERBOSE > 1 : logger.debug(' time lost so far in finding correspondence %.2f',time_corr) |
| 3507 |
|
| 3508 |
######### final pass : scan new tar, extract regular files, prepare deltas |
| 3509 |
if VERBOSE > 2 : logger.debug(' scanning %r',new_filename) |
| 3510 |
|
| 3511 |
script.start_rebuilding() |
| 3512 |
|
| 3513 |
current_chunk_name=next(a_numb_file) |
| 3514 |
script.start_chunk(current_chunk_name) |
| 3515 |
mega_cat=open(TD+'/'+current_chunk_name,'wb') |
| 3516 |
|
| 3517 |
#helper function |
| 3518 |
def _append_(p, w, rm): |
| 3519 |
mega_cat.write(str2bytes(w)+fake_tar_2nd) |
| 3520 |
f=open(TD+p+'/'+w,'rb') |
| 3521 |
a=f.read(1024) |
| 3522 |
while a: |
| 3523 |
try: |
| 3524 |
mega_cat.write(a) |
| 3525 |
except OSError: |
| 3526 |
s=sys.exc_info()[1] |
| 3527 |
raise DebDeltaError(' OSError (at _a_) while writing: '+str(s), True) |
| 3528 |
a=f.read(1024) |
| 3529 |
f.close() |
| 3530 |
if rm: |
| 3531 |
script.write("ECR '"+w+"'\n") |
| 3532 |
unlink(TD+p+'/'+w) |
| 3533 |
else: |
| 3534 |
script.write("EC '"+w+"'\n") |
| 3535 |
|
| 3536 |
global something_backgrounded #FIXME I hate using globals for this :-( |
| 3537 |
something_backgrounded=False |
| 3538 |
#helper function |
| 3539 |
def mega_cat_chunk(oldoffset,newoffset, background=True): |
| 3540 |
global something_backgrounded |
| 3541 |
p = next(a_numb_file) |
| 3542 |
f=open(TD+new_filename,'rb') |
| 3543 |
f.seek(oldoffset) |
| 3544 |
of=open(TD+p,'wb') |
| 3545 |
l=oldoffset |
| 3546 |
while l<newoffset: |
| 3547 |
s=f.read(512) |
| 3548 |
l+=len(s) |
| 3549 |
assert(len(s)) |
| 3550 |
try: |
| 3551 |
of.write(s) |
| 3552 |
except OSError: |
| 3553 |
s=sys.exc_info()[1] |
| 3554 |
raise DebDeltaError(' OSError (at MCK) while writing: '+str(s), True) |
| 3555 |
f.close() |
| 3556 |
of.close() |
| 3557 |
#do delta, in background there |
| 3558 |
if something_backgrounded: |
| 3559 |
script.write('wait\n') |
| 3560 |
if background: |
| 3561 |
script.write('(') |
| 3562 |
this_delta_piped=delta_files(current_chunk_name, p, True, script.the_fifo) |
| 3563 |
if not this_delta_piped: |
| 3564 |
script.write('cat '+p+'\n') |
| 3565 |
if not this_delta_piped: |
| 3566 |
script.write('rm '+p+'\n') |
| 3567 |
if background: |
| 3568 |
script.write(')&\n') #the delta+cat is backgrounded |
| 3569 |
something_backgrounded=True |
| 3570 |
os.unlink(TD+p) |
| 3571 |
|
| 3572 |
#there may be files that have been renamed and edited... |
| 3573 |
def some_old_file_gen(): |
| 3574 |
for oldname in oldnames : |
| 3575 |
if (oldname in skip) or (oldname in old_used ) : |
| 3576 |
continue |
| 3577 |
if VERBOSE > 2 : logger.debug(' provide also old file %r', oldname) |
| 3578 |
yield oldname |
| 3579 |
while 1: |
| 3580 |
yield None |
| 3581 |
|
| 3582 |
some_old_file=some_old_file_gen() |
| 3583 |
one_old_file=next(some_old_file) |
| 3584 |
|
| 3585 |
max_chunk_size = MAXMEMORY / 12 |
| 3586 |
chunk_discount = 0.3 |
| 3587 |
|
| 3588 |
progressive_new_offset=0 |
| 3589 |
|
| 3590 |
for newtarinfo in newtar: |
| 3591 |
#progressive mega_cat |
| 3592 |
a=mega_cat.tell() |
| 3593 |
if chunked_p and ((a >= max_chunk_size * chunk_discount) or \ |
| 3594 |
(a >= max_chunk_size * chunk_discount * 0.9 and one_old_file ) or \ |
| 3595 |
(a>0 and (a+newtarinfo.size) >= max_chunk_size * chunk_discount )): |
| 3596 |
#provide some old unused files, if any |
| 3597 |
while one_old_file: |
| 3598 |
_append_("OLD/"+CWD, one_old_file, False) |
| 3599 |
if mega_cat.tell() >= max_chunk_size * chunk_discount : |
| 3600 |
break |
| 3601 |
one_old_file=next(some_old_file) |
| 3602 |
#write the chunk into a temporary |
| 3603 |
mega_cat.close() |
| 3604 |
script.end_chunk(current_chunk_name) |
| 3605 |
#delta the chunk |
| 3606 |
mega_cat_chunk(progressive_new_offset, newtarinfo.offset ) |
| 3607 |
#start a new chunk |
| 3608 |
current_chunk_name=next(a_numb_file) |
| 3609 |
script.start_chunk(current_chunk_name) |
| 3610 |
mega_cat=open(TD+'/'+current_chunk_name,'wb') |
| 3611 |
# |
| 3612 |
progressive_new_offset=newtarinfo.offset |
| 3613 |
chunk_discount = min( 1. , chunk_discount * 1.2 ) |
| 3614 |
# |
| 3615 |
name = de_bar( newtarinfo.name ) |
| 3616 |
|
| 3617 |
if newtarinfo.isdir(): |
| 3618 |
#recreate also parts of the tar headers |
| 3619 |
mega_cat.write(str2bytes(newtarinfo.name)+fake_tar_2nd) |
| 3620 |
script.write("$E '"+prepare_for_echo(str2bytes(newtarinfo.name))+"'\"${FTH}\"\n") |
| 3621 |
if VERBOSE > 2 : logger.debug(' directory in new : %r', name) |
| 3622 |
continue |
| 3623 |
|
| 3624 |
if not newtarinfo.isreg(): |
| 3625 |
#recreate also parts of the tar headers |
| 3626 |
mega_cat.write(str2bytes(newtarinfo.name)+fake_tar_2nd) |
| 3627 |
script.write("$E '"+prepare_for_echo(newtarinfo.name)+"'\"${FTH}\"\n") |
| 3628 |
if VERBOSE > 2 : logger.debug(' not regular in new : %r', name) |
| 3629 |
continue |
| 3630 |
|
| 3631 |
if newtarinfo.size == 0: |
| 3632 |
#recreate also parts of the tar headers |
| 3633 |
mega_cat.write(str2bytes(newtarinfo.name)+fake_tar_2nd) |
| 3634 |
script.write("$E '"+prepare_for_echo(newtarinfo.name)+"'\"${FTH}\"\n") |
| 3635 |
if VERBOSE > 2 : logger.debug(' empty new file : %r', name) |
| 3636 |
continue |
| 3637 |
|
| 3638 |
if name not in correspondence: |
| 3639 |
#recreate also parts of the tar headers |
| 3640 |
mega_cat.write(str2bytes(newtarinfo.name)+fake_tar_2nd) |
| 3641 |
script.write("$E '"+prepare_for_echo(newtarinfo.name)+"'\"${FTH}\"\n") |
| 3642 |
if VERBOSE > 2: logger.debug(' no corresponding fil: %r', name) |
| 3643 |
continue |
| 3644 |
|
| 3645 |
oldname = correspondence[name] |
| 3646 |
|
| 3647 |
mul=len( old_used[oldname]) > 1 #multiple usage |
| 3648 |
|
| 3649 |
if not mul and oldname == name and oldname[-3:] == '.gz' and \ |
| 3650 |
newtarinfo.size > 120 and \ |
| 3651 |
not ( new_md5 and name in new_md5 and old_md5 and name in old_md5 and \ |
| 3652 |
new_md5[name] == old_md5[name]): |
| 3653 |
newtar.extract(newtarinfo,TD+"NEW/"+CWD ) |
| 3654 |
delta_gzipped_files("OLD/"+CWD+'/'+name,"NEW/"+CWD+'/'+name) |
| 3655 |
|
| 3656 |
if VERBOSE > 2 : logger.debug(' adding reg file: %r %r', oldname, mul and '(multiple)' or '') |
| 3657 |
_append_("OLD/"+CWD, oldname , not mul ) |
| 3658 |
old_used[oldname].pop() |
| 3659 |
#end of for loop |
| 3660 |
|
| 3661 |
#write the chunk into a temporary |
| 3662 |
mega_cat.close() |
| 3663 |
script.end_chunk(current_chunk_name) |
| 3664 |
if os.path.exists(TD+'/OLD/'+CWD): |
| 3665 |
rmtree(TD+'/OLD/'+CWD) |
| 3666 |
if os.path.getsize(TD+'/'+current_chunk_name) > 0 : |
| 3667 |
mega_cat_chunk(progressive_new_offset, os.path.getsize(TD+new_filename), background=False) |
| 3668 |
else: |
| 3669 |
#the (tail of the) new tar did not match anything in the old tar, nothing to delta |
| 3670 |
p=verbatim(new_filename) |
| 3671 |
script.write("cat '"+p+"'\n") |
| 3672 |
script.end_rebuilding() #pipes the rebuilding part into the recompressing part |
| 3673 |
if new_filename == 'NEW/data.tar': |
| 3674 |
script.recompressing_by_arg(new_filename,new_filename_ext,new_file_zip_head,new_file_size) |
| 3675 |
else: |
| 3676 |
script.recompressing(new_filename,new_filename_ext,new_file_zip_head) |
| 3677 |
|
| 3678 |
#################### ^^^^ delta_tar ^^^^ ########################### |
| 3679 |
|
| 3680 |
############ start computing deltas |
| 3681 |
def append_NEW_file(s): |
| 3682 |
'appends some data to NEW.file' |
| 3683 |
s=prepare_for_echo(s) |
| 3684 |
script.write("$E '"+ s +"' >> NEW.file\n") |
| 3685 |
|
| 3686 |
#this following is actually |
| 3687 |
#def delta_debs_using_old(old,new): |
| 3688 |
|
| 3689 |
### start scanning the new deb |
| 3690 |
newdeb_file=open(newdeb,'rb') |
| 3691 |
# pop the "!<arch>\n" |
| 3692 |
s = newdeb_file.readline() |
| 3693 |
assert( b"!<arch>\n" == s) |
| 3694 |
append_NEW_file(s) |
| 3695 |
|
| 3696 |
|
| 3697 |
if forensic_file==None: |
| 3698 |
control_forensic=None |
| 3699 |
data_forensic=None |
| 3700 |
else: |
| 3701 |
control_forensic=[] |
| 3702 |
data_forensic=[] |
| 3703 |
|
| 3704 |
def decompress_pipe_from_ar(oldnew, name, p=None): |
| 3705 |
" use p=False to not decompress" |
| 3706 |
assert oldnew in ('OLD','NEW') |
| 3707 |
if p==None: |
| 3708 |
p=os.path.splitext(name)[1] |
| 3709 |
db={b'.gz' : ['gzip','-cd'], b'.bz2': ['bzip2','-cd'], |
| 3710 |
b'.lzma':['unlzma','-c'], b'.xz': ['unxz','-c'] } |
| 3711 |
assert p==False or p == b'.tar' or p in db |
| 3712 |
if p==b'.tar' or p==False: |
| 3713 |
def x(): |
| 3714 |
return PopenPipe(['ar','p', oldnew+'.file', name] , stdout=subprocess.PIPE, cwd=TD) |
| 3715 |
else: |
| 3716 |
def x(): |
| 3717 |
return PopenPipe(['ar','p', oldnew+'.file', name,'|']+db[p] , stdout=subprocess.PIPE, cwd=TD) |
| 3718 |
return x |
| 3719 |
|
| 3720 |
for name in ar_list_new : |
| 3721 |
newname = 'NEW/'+bytes2str(name) |
| 3722 |
system(('ar','x',TD+'NEW.file',name), TD+'/NEW/') |
| 3723 |
script.xz_parameters=None |
| 3724 |
newsize = os.stat(TD+newname)[ST_SIZE] |
| 3725 |
if VERBOSE > 1: logger.debug(' studying %r of len %dkB' , name, (newsize/1024)) |
| 3726 |
#add 'ar' structure |
| 3727 |
ar_line = newdeb_file.read(60) |
| 3728 |
if VERBOSE > 3: logger.debug(' ar line: %r',ar_line) |
| 3729 |
assert( ar_line[:len(name)] == name and ar_line[-2:] == b'`\n' ) |
| 3730 |
#sometimes there is an extra \n, depending if the previous was odd length |
| 3731 |
newdeb_file.seek(newsize ,1) |
| 3732 |
if newsize & 1 : |
| 3733 |
extrachar = newdeb_file.read(1) |
| 3734 |
else: |
| 3735 |
extrachar = '' |
| 3736 |
#add file to delta |
| 3737 |
if newsize < 128: #file is too short to compute a delta, |
| 3738 |
append_NEW_file(ar_line) |
| 3739 |
p=open(TD+newname,'rb') |
| 3740 |
append_NEW_file( p.read(newsize)) |
| 3741 |
p.close() |
| 3742 |
unlink(TD+newname) |
| 3743 |
#pad new deb |
| 3744 |
if extrachar : |
| 3745 |
append_NEW_file(extrachar) |
| 3746 |
elif not NEEDSOLD and ( name[:11] == b'control.tar' or name[:8] == b'data.tar'): |
| 3747 |
script.start_member(ar_line, newname, extrachar) |
| 3748 |
basename,ext=os.path.splitext(name) |
| 3749 |
xd=decompress_pipe_from_ar('NEW',name,False) |
| 3750 |
#delta it |
| 3751 |
if b'.lzma' == ext : |
| 3752 |
info_append('needs-lzma') |
| 3753 |
elif b'.xz' == ext : |
| 3754 |
info_append('needs-xz') |
| 3755 |
script.xz_parameters=guess_xz_parameters(xd) |
| 3756 |
if script.xz_parameters==False: |
| 3757 |
raise DebDeltaError('Cannot guess XZ parameters for new %r' % name) |
| 3758 |
elif b'.gz' == ext : |
| 3759 |
script.gz_command=guess_gz_command(xd) |
| 3760 |
if script.gz_command==False: |
| 3761 |
raise DebDeltaError('Cannot guess GZ parameters for new %r' % name) |
| 3762 |
del xd |
| 3763 |
x=None |
| 3764 |
for oldext in (b'', b'.gz', b'.bz2', b'.lzma', b'.xz'): |
| 3765 |
if basename+oldext in ar_list_old: |
| 3766 |
x=decompress_pipe_from_ar('OLD',basename+oldext) |
| 3767 |
break |
| 3768 |
assert x |
| 3769 |
if name[:11] == b'control.tar': |
| 3770 |
skip=[] |
| 3771 |
##avoid using strange files that dpkg may not install in /var...info/ |
| 3772 |
for a in os.listdir(TD+'OLD/CONTROL') : |
| 3773 |
if a not in dpkg_keeps_controls: |
| 3774 |
skip.append(a) |
| 3775 |
delta_tar(x,newname,'CONTROL',control_forensic,skip) |
| 3776 |
else: |
| 3777 |
delta_tar(x,newname,'DATA',data_forensic,old_conffiles,old_md5,new_md5,\ |
| 3778 |
debdelta_conf_skip=debdelta_conf_skip) |
| 3779 |
del x, basename, oldext |
| 3780 |
script.end_member() |
| 3781 |
if DEBUG>3 and name[:11] == b'control.tar': |
| 3782 |
length=newdeb_file.tell() |
| 3783 |
md5_len=compute_md5_up_to_len(newdeb, length) |
| 3784 |
assert md5_len[1] == length |
| 3785 |
script.md5_check_file('NEW.file', md5_len[0]) |
| 3786 |
elif not NEEDSOLD or name not in ar_list_old : #or it is not in old deb |
| 3787 |
append_NEW_file(ar_line) |
| 3788 |
patchname=verbatim(newname) |
| 3789 |
script.write('cat '+patchname+' >> NEW.file ; rm '+patchname+'\n') |
| 3790 |
#pad new deb |
| 3791 |
if extrachar : |
| 3792 |
append_NEW_file(extrachar) |
| 3793 |
elif NEEDSOLD : |
| 3794 |
append_NEW_file(ar_line) |
| 3795 |
#file is long, and has old version ; lets compute a delta |
| 3796 |
oldname = 'OLD/'+name |
| 3797 |
system(('ar','x',TD+'OLD.file',name), TD+'OLD') |
| 3798 |
script.write('ar p OLD.file '+name+' >> '+oldname+'\n') |
| 3799 |
(oldname,co) = unzip(oldname) |
| 3800 |
(newname,cn) = unzip(newname) |
| 3801 |
delta_files(oldname,newname) |
| 3802 |
script.zip(newname,cn) |
| 3803 |
script.write('cat '+newname+cn+' >> NEW.file ; rm '+newname+cn+'\n') |
| 3804 |
unlink(TD+newname) |
| 3805 |
#pad new deb |
| 3806 |
if extrachar : |
| 3807 |
append_NEW_file(extrachar) |
| 3808 |
del co,cn |
| 3809 |
else: |
| 3810 |
die('internal error j98') |
| 3811 |
# put in script any leftover |
| 3812 |
s = newdeb_file.read() |
| 3813 |
if s: |
| 3814 |
if VERBOSE > 2: logger.debug(' ar leftover character: %r',s) |
| 3815 |
append_NEW_file(s) |
| 3816 |
|
| 3817 |
#this is done already from the receiving end |
| 3818 |
if DEBUG > 2 and newdeb_md5sum : |
| 3819 |
script.md5_check_file("NEW.file",md5=newdeb_md5sum) |
| 3820 |
|
| 3821 |
#script is done |
| 3822 |
script.close() |
| 3823 |
|
| 3824 |
if forensic_file: |
| 3825 |
forensics_rfc(forensic_file,info,True,control_forensic,data_forensic,old_conffiles) |
| 3826 |
|
| 3827 |
patchsize = os.stat(TD+'PATCH/patch.sh')[ST_SIZE] |
| 3828 |
patch_files = [] |
| 3829 |
if 'lzma' not in DISABLED_FEATURES and os.path.exists('/usr/bin/lzma'): |
| 3830 |
system(('lzma','-q','-9','-k','PATCH/patch.sh'), TD) |
| 3831 |
patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.lzma'), 'lzma', 'patch.sh.lzma')) |
| 3832 |
if 'xz' not in DISABLED_FEATURES and os.path.exists('/usr/bin/xz'): |
| 3833 |
system(('xz','-q','-9','-k','PATCH/patch.sh'), TD) |
| 3834 |
patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.xz'), 'xz', 'patch.sh.xz')) |
| 3835 |
system(('bzip2','-q','--keep','-9','PATCH/patch.sh'), TD) |
| 3836 |
patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.bz2'), 'bzip2', 'patch.sh.bz2')) |
| 3837 |
system(('gzip','-q','-9','-n','PATCH/patch.sh'), TD) |
| 3838 |
patch_files.append((os.path.getsize(TD+'PATCH/patch.sh.gz'), 'gzip', 'patch.sh.gz')) |
| 3839 |
|
| 3840 |
# Use the smallest compressed patch.sh |
| 3841 |
patch_files.sort() |
| 3842 |
if VERBOSE > 1 : logger.debug(' '+patch_files[0][1]+' wins on patch.sh') |
| 3843 |
if patch_files[0][1] == 'lzma': |
| 3844 |
info_append('needs-lzma') |
| 3845 |
if patch_files[0][1] == 'xz': |
| 3846 |
info_append('needs-xz') |
| 3847 |
patch_append(patch_files[0][2]) |
| 3848 |
del patch_files |
| 3849 |
|
| 3850 |
#OK, OK... this is not yet correct, since I will add the info file later on |
| 3851 |
elaps = time.time() - start_sec |
| 3852 |
info.append('DeltaTime: %.2f' % elaps) |
| 3853 |
deltasize = os.stat(delta)[ST_SIZE] + 60 + sum(map(len,info)) |
| 3854 |
percent = deltasize * 100. / newdebsize |
| 3855 |
info.append('Ratio: %.4f' % (float(deltasize) / float(newdebsize)) ) |
| 3856 |
|
| 3857 |
if VERBOSE: |
| 3858 |
#note that sizes are written as kB but are actually kibybytes, that is 1024 bytes |
| 3859 |
logger.info(' ' + _('delta is %(perc)3.1f%% of deb; that is, %(save)dkB are saved, on a total of %(tot)dkB.'), |
| 3860 |
{'perc' : percent , 'save' : (( newdebsize -deltasize ) / 1024), 'tot' : ( newdebsize/ 1024)}) |
| 3861 |
logger.info(' ' + _('delta time %(time).2f sec, speed %(speed)dkB /sec, (%(algo)s time %(algotime).2fsec speed %(algospeed)dkB /sec) (corr %(corrtime).2f sec)') % \ |
| 3862 |
{'time' : elaps, 'speed' : newdebsize / 1024. / (elaps+0.001), |
| 3863 |
'algo' : USE_DELTA_ALGO, 'algotime' : bsdiff_time, |
| 3864 |
'algospeed' : bsdiff_datasize / 1024. / (bsdiff_time + 0.001) , 'corrtime' : time_corr }) |
| 3865 |
return (delta, percent, elaps, info, gpg_hashes) |
| 3866 |
|
| 3867 |
|
| 3868 |
##################################################### compute many deltas |
| 3869 |
|
| 3870 |
def info_by_pack_arch_add(f, info_by_pack_arch): |
| 3871 |
pack = f['Package'] |
| 3872 |
arch = f['Architecture'] |
| 3873 |
if (pack,arch) not in info_by_pack_arch : |
| 3874 |
info_by_pack_arch[ (pack,arch) ]=[] |
| 3875 |
info_by_pack_arch[ (pack,arch) ].append( f ) |
| 3876 |
|
| 3877 |
|
| 3878 |
def iterate_Packages(packages, use_debian_822=True): |
| 3879 |
fields=('Package','Architecture','Version','Filename') |
| 3880 |
for f in fields: intern(f) |
| 3881 |
|
| 3882 |
packages=abspath(packages) |
| 3883 |
assert os.path.isfile(packages) |
| 3884 |
assert os.path.basename(packages) in ('Packages', 'Packages.gz','Packages.bz2','Packages.xz') |
| 3885 |
dir=os.path.dirname(packages) |
| 3886 |
dir=dir.split('/') |
| 3887 |
try: |
| 3888 |
a=dir.index('dists') |
| 3889 |
except ValueError: |
| 3890 |
logger.error('Error: pathname "%s" does not contain "dists"\n' % packages) |
| 3891 |
return |
| 3892 |
base = str.join('/',dir[:a]) |
| 3893 |
# |
| 3894 |
cache=cache_same_dict(packages, fields) |
| 3895 |
if DO_CACHE and cache.exists: |
| 3896 |
for i in cache: |
| 3897 |
i['Basepath'] = base |
| 3898 |
yield i |
| 3899 |
if not cache.broken: |
| 3900 |
return |
| 3901 |
# |
| 3902 |
if packages[-3:] == '.gz': |
| 3903 |
import gzip |
| 3904 |
F=gzip.GzipFile(packages) |
| 3905 |
SP=None |
| 3906 |
elif packages[-4:] == '.bz2': |
| 3907 |
import bz2 |
| 3908 |
F=bz2.BZ2File(packages) |
| 3909 |
SP=None |
| 3910 |
elif packages[-3:] == '.xz': |
| 3911 |
SP=subprocess.Popen(["xzcat",packages], stdout=subprocess.PIPE, stdin=open(os.devnull), close_fds=True) |
| 3912 |
F=SP.stdout |
| 3913 |
else: |
| 3914 |
F=open(packages,'rb') |
| 3915 |
SP=None |
| 3916 |
# |
| 3917 |
if debian_deb822 and use_debian_822: #use faster implementation |
| 3918 |
#P=debian_deb822.Packages(F,fields=fields) |
| 3919 |
for a in debian_deb822.Packages.iter_paragraphs(sequence=F,shared_storage=False,fields=fields): |
| 3920 |
if DO_CACHE and not cache.exists: cache.write(a) |
| 3921 |
a['Basepath'] = base |
| 3922 |
yield a |
| 3923 |
if SP: |
| 3924 |
F.read() |
| 3925 |
SP.wait() |
| 3926 |
return |
| 3927 |
# |
| 3928 |
of,pack,vers,arch=None,None,None,None |
| 3929 |
for l in F: |
| 3930 |
l = bytes2str(l) |
| 3931 |
l=l.rstrip('\n') |
| 3932 |
if l[:9] == 'Package: ': |
| 3933 |
pack=l[9:] |
| 3934 |
elif l[:14] == 'Architecture: ': |
| 3935 |
arch = l[14:] |
| 3936 |
elif l[:9] == 'Version: ': |
| 3937 |
vers = l[9:] |
| 3938 |
elif l[:10] == 'Filename: ': |
| 3939 |
of=l[10:] |
| 3940 |
elif l == '': |
| 3941 |
if of == None or pack == None or vers == None or arch == None: |
| 3942 |
logger.warning('Warning, skipping incomplete record in index: %r %r %r %r',of,pack,vers,arch) |
| 3943 |
continue |
| 3944 |
if of[-4:] == '.udeb': |
| 3945 |
if VERBOSE > 2 : logger.debug(' skip udeb') |
| 3946 |
continue |
| 3947 |
a={} |
| 3948 |
a['Filename'] = of |
| 3949 |
a['Package'] = pack |
| 3950 |
a['Architecture'] = arch |
| 3951 |
a['Version'] = vers |
| 3952 |
if DO_CACHE and not cache.exists: cache.write(a) |
| 3953 |
a['Basepath'] = base |
| 3954 |
yield a |
| 3955 |
of,pack,vers,arch=None,None,None,None |
| 3956 |
if SP: |
| 3957 |
F.read() |
| 3958 |
SP.wait() |
| 3959 |
|
| 3960 |
def scan_deb_dir(d, debname, label, lazy, info_by_pack_arch, info_by_file): |
| 3961 |
assert (debname == None or type(debname) in string_types) and type(label) in string_types |
| 3962 |
if not os.path.isdir(d): |
| 3963 |
logger.debug('Error, skip non dir: %r',d) |
| 3964 |
return |
| 3965 |
if lazy: |
| 3966 |
scan=scan_deb_byfile_lazy |
| 3967 |
else: |
| 3968 |
scan=scan_deb_byfile |
| 3969 |
for n in os.listdir(d): |
| 3970 |
if n[-4:] != '.deb': |
| 3971 |
continue |
| 3972 |
if debname != None and debname != n.split('_')[0]: |
| 3973 |
continue |
| 3974 |
if not check_deb(os.path.join(d,n), die_ = False): |
| 3975 |
logger.warning('Is not a deb file: %r',n) |
| 3976 |
continue |
| 3977 |
a=scan(os.path.join(d,n), info_by_file) |
| 3978 |
a['Label'] = label |
| 3979 |
info_by_pack_arch_add(a, info_by_pack_arch) |
| 3980 |
|
| 3981 |
def scan_deb_byfile_lazy(f, info_by_file): |
| 3982 |
n=os.path.basename(f).split('_') |
| 3983 |
a={} |
| 3984 |
a['File']=f |
| 3985 |
a['Package']=n[0] |
| 3986 |
#version cannot be trusted, due to epochs (deleted in filenames) |
| 3987 |
#it is read using scan_deb_bydict, later on |
| 3988 |
a['Architecture']=n[2][:-4] |
| 3989 |
return a |
| 3990 |
|
| 3991 |
def scan_deb_byfile(f, info_by_file): |
| 3992 |
#DEBUG: assert( os.path.isfile(f) ) |
| 3993 |
if f in info_by_file and 'Version' in info_by_file[f]: |
| 3994 |
#already scanned in non-lazy mode |
| 3995 |
return info_by_file[f] |
| 3996 |
a={} |
| 3997 |
a['File']=f |
| 3998 |
c,b,z = untar_control_in_deb( list_ar(f) ) |
| 3999 |
p=PopenPipe(['ar','p',f,c,'|','tar',b,'-xp','-f','-','-O','./control'], |
| 4000 |
stdout = subprocess.PIPE, ** Popen_args) |
| 4001 |
scan_control(p.stdout, params=a) |
| 4002 |
p.close() |
| 4003 |
return a |
| 4004 |
|
| 4005 |
def scan_deb_bydict(a): |
| 4006 |
c,b,z = untar_control_in_deb( list_ar(a['File']) ) |
| 4007 |
p=PopenPipe(['ar','p',a['File'],c,'|','tar',b,'-xp','-f','-','-O','./control'], |
| 4008 |
stdout = subprocess.PIPE) |
| 4009 |
scan_control(p.stdout, params=a) |
| 4010 |
p.close() |
| 4011 |
return a |
| 4012 |
|
| 4013 |
def scan_delta_dir(d, debname, scanned_delta_dirs, old_deltas_by_pack_arch): |
| 4014 |
if (d,debname) in scanned_delta_dirs or (d,None) in scanned_delta_dirs : |
| 4015 |
return |
| 4016 |
if not os.path.isdir(d) : |
| 4017 |
if VERBOSE > 2 and DEBUG : logger.debug(' No such delta dir: %r',d) |
| 4018 |
scanned_delta_dirs.add((d,None)) #trick, if aint there no need to retry |
| 4019 |
return |
| 4020 |
assert debname == None or type(debname) in string_types |
| 4021 |
scanned_delta_dirs.add((d,debname)) |
| 4022 |
for n in os.listdir(d): |
| 4023 |
if debname != None and debname != n.split('_')[0]: |
| 4024 |
continue |
| 4025 |
scan_delta( os.path.join(d,n) , old_deltas_by_pack_arch) |
| 4026 |
|
| 4027 |
def scan_delta(f, old_deltas_by_pack_arch): |
| 4028 |
assert( os.path.isfile(f) ) |
| 4029 |
if f[-9:] == '.debdelta' : |
| 4030 |
a=f[:-9] |
| 4031 |
elif f[-17:] == '.debdelta-too-big' : |
| 4032 |
a=f[:-17] |
| 4033 |
elif f[-15:] == '.debdelta-fails' : |
| 4034 |
a=f[:-15] |
| 4035 |
else: return |
| 4036 |
a=os.path.basename(a) |
| 4037 |
a=a.split('_') |
| 4038 |
pa=a[0] |
| 4039 |
ar=a[3] |
| 4040 |
if (pa,ar) not in old_deltas_by_pack_arch: |
| 4041 |
old_deltas_by_pack_arch[ (pa,ar) ]=[] |
| 4042 |
ov=version_demangle(a[1]) |
| 4043 |
nv=version_demangle(a[2]) |
| 4044 |
if (f,ov,nv) not in old_deltas_by_pack_arch[ (pa,ar) ]: |
| 4045 |
old_deltas_by_pack_arch[ (pa,ar) ].append( (f, ov, nv ) ) |
| 4046 |
|
| 4047 |
def delta_dirname(f,altdir): |
| 4048 |
"f=directory, altdir=ALT or DIR with // convention. Returns augmented dirname" |
| 4049 |
if altdir != None: |
| 4050 |
if altdir[-2:] == '//' : |
| 4051 |
if '../' in f: |
| 4052 |
logger.warning('Warning: cannot combine .. and // ! Saving in %r', altdir) |
| 4053 |
return altdir |
| 4054 |
#os.path.join has a weird behaviour with absolute paths! |
| 4055 |
f=f.lstrip('/') |
| 4056 |
return os.path.join(altdir[:-2], f) |
| 4057 |
else: |
| 4058 |
return altdir |
| 4059 |
else: |
| 4060 |
#this happens when DIR is not set, and the delta goes in the same directory as the deb |
| 4061 |
return f |
| 4062 |
|
| 4063 |
def go_fishing(deb,others, loglevel=logging.ERROR): |
| 4064 |
"Find a package filepath if missing, save it in deb['File']. deb = dict representing package ; others: list of such dicts." |
| 4065 |
if 'File' in deb: |
| 4066 |
assert os.path.isfile(deb['File']) |
| 4067 |
return deb['File'] |
| 4068 |
of=deb['Filename'] |
| 4069 |
ob=os.path.basename(of) |
| 4070 |
#try to build it from its own info |
| 4071 |
if 'Basepath' in deb: |
| 4072 |
f=os.path.join(deb['Basepath'],of) |
| 4073 |
if os.path.exists(f): |
| 4074 |
deb['File']=f |
| 4075 |
if DEBUG > 1 : logger.debug('Fish! %r',deb) |
| 4076 |
return f |
| 4077 |
#try to build it from others info |
| 4078 |
others=[a for a in others if id(a) != id(deb) ] |
| 4079 |
for new in others: |
| 4080 |
if 'File' in new: |
| 4081 |
f=new['File'] |
| 4082 |
if os.path.basename(f) == ob: |
| 4083 |
deb['File']=f |
| 4084 |
if DEBUG > 1 : logger.debug('Fish! %r %r',deb,new) |
| 4085 |
return f |
| 4086 |
if 'Basepath' in new: |
| 4087 |
for a in of,ob: |
| 4088 |
f=os.path.join(new['Basepath'],a) |
| 4089 |
if os.path.exists(f): |
| 4090 |
deb['File']=f |
| 4091 |
if DEBUG > 1 : logger.debug('Fish! %r %r',deb,new) |
| 4092 |
return f |
| 4093 |
logger.log(loglevel, 'bad fishing for %r %r', deb, others) |
| 4094 |
return False |
| 4095 |
|
| 4096 |
def order_by_version(a, b, VersionCompare): |
| 4097 |
# lazy packages do not have versions; but are always ALT, so we sort them at bottom |
| 4098 |
if 'Version' not in a: |
| 4099 |
return -1 |
| 4100 |
if 'Version' not in b: |
| 4101 |
return 1 |
| 4102 |
return VersionCompare( a['Version'] , b['Version'] ) |
| 4103 |
|
| 4104 |
|
| 4105 |
def iter_deltas_one_pack_arch(pa,ar,info_pack,thedir,theforensicdir,VersionCompare, |
| 4106 |
loglevel=logging.ERROR): |
| 4107 |
" iterate deltas to be created for package pa, architecture ar, by analyzing info_pack" |
| 4108 |
if all([('CMDLINE' != a['Label']) for a in info_pack ]): |
| 4109 |
#this happens a lot, e.g. when we are scanning non-free/Packages.gz, |
| 4110 |
# all free packages in the trash are to be ignored |
| 4111 |
if DEBUG > 2 or VERBOSE > 3 : logger.debug(' No cmdline for: %r %r',pa,ar) |
| 4112 |
return |
| 4113 |
|
| 4114 |
#do we need the versions of --alt packages ? seems not |
| 4115 |
# for a in info_pack: |
| 4116 |
# #scan all lazy packages |
| 4117 |
# if 'Version' not in a: |
| 4118 |
# assert a['Label'] == 'ALT' |
| 4119 |
# scan_deb_bydict(a) |
| 4120 |
|
| 4121 |
import functools |
| 4122 |
mykey = functools.cmp_to_key(lambda x,y : order_by_version(x, y, VersionCompare)) |
| 4123 |
info_pack.sort(key = mykey) |
| 4124 |
|
| 4125 |
how_many= len(info_pack) |
| 4126 |
|
| 4127 |
if how_many <= 1 : |
| 4128 |
if VERBOSE > 3 : logger.debug(' Only one version: %r %r', pa,ar) |
| 4129 |
return |
| 4130 |
|
| 4131 |
newest = how_many -1 |
| 4132 |
while newest >= 0 : |
| 4133 |
new=info_pack[newest] |
| 4134 |
if new['Label'] != 'CMDLINE' : |
| 4135 |
if VERBOSE > 3 : logger.debug(' Newest version deb was not in cmdline, skip down one: %r', new) |
| 4136 |
else: |
| 4137 |
break |
| 4138 |
newest -= 1 |
| 4139 |
|
| 4140 |
if newest <= 0 : |
| 4141 |
if VERBOSE > 3 : logger.debug(' No older versions: %r', new) |
| 4142 |
return |
| 4143 |
|
| 4144 |
if not go_fishing(new,[], loglevel): |
| 4145 |
logger.log(loglevel, 'Cannot locate new file: %r',new) |
| 4146 |
return |
| 4147 |
|
| 4148 |
if VERBOSE > 2 : |
| 4149 |
logger.debug(' Package: %r %r Versions: %r', pa, ar, [ (o.get('Version'),o['Label'],o.get('Packages'),o.get('File')) for o in info_pack]) |
| 4150 |
|
| 4151 |
newdebsize=os.path.getsize(new['File']) |
| 4152 |
#very small packages cannot be effectively delta-ed |
| 4153 |
if newdebsize <= MIN_DEB_SIZE : |
| 4154 |
if VERBOSE > 1 : logger.debug(' Skip , too small: %r', new['File']) |
| 4155 |
return |
| 4156 |
|
| 4157 |
oldn = newest |
| 4158 |
generated=0 |
| 4159 |
seen_versions=[] |
| 4160 |
while (oldn>0) : |
| 4161 |
oldn -= 1 |
| 4162 |
|
| 4163 |
old=info_pack[oldn] |
| 4164 |
|
| 4165 |
if old['Label'] != 'OLD': |
| 4166 |
if VERBOSE > 2 : logger.debug(' Not old, skip: %r',old) |
| 4167 |
continue |
| 4168 |
|
| 4169 |
if old['Version'] == new['Version'] : |
| 4170 |
if VERBOSE > 3 and old != new : logger.debug(' Identical versions: %r %r', old, new) |
| 4171 |
continue |
| 4172 |
|
| 4173 |
assert( old['Package'] == pa and pa == new['Package'] ) |
| 4174 |
deltabasename = delta_base_name(pa, old['Version'], new['Version'], ar) |
| 4175 |
|
| 4176 |
if 'Filename' in new: |
| 4177 |
deltadirname=delta_dirname(os.path.dirname(new['Filename']),thedir) |
| 4178 |
elif 'File' in new: |
| 4179 |
deltadirname=delta_dirname(os.path.dirname(new['File']),thedir) |
| 4180 |
else: |
| 4181 |
assert(0) |
| 4182 |
if deltadirname == '' : |
| 4183 |
deltadirname = '.' |
| 4184 |
|
| 4185 |
delta=os.path.join(deltadirname,deltabasename) |
| 4186 |
|
| 4187 |
generated+=1 #count also those already generated |
| 4188 |
if N_DELTAS != None and (generated > N_DELTAS): |
| 4189 |
continue |
| 4190 |
|
| 4191 |
if os.path.exists(delta): |
| 4192 |
if VERBOSE > 1 : logger.debug(' Skip , already exists: %r',delta) |
| 4193 |
continue |
| 4194 |
|
| 4195 |
if old['Package'] in seen_versions: |
| 4196 |
if VERBOSE > 3 : logger.debug(' Skip , already considered: %r',delta) |
| 4197 |
continue |
| 4198 |
|
| 4199 |
if os.path.exists(delta+'-too-big'): |
| 4200 |
if VERBOSE > 1 : logger.debug(' Skip , tried and too big: %r',delta) |
| 4201 |
continue |
| 4202 |
|
| 4203 |
if os.path.exists(delta+'-fails'): |
| 4204 |
if VERBOSE > 1 : logger.debug(' Skip , tried and fails: %r',delta) |
| 4205 |
continue |
| 4206 |
|
| 4207 |
if not go_fishing(old,info_pack, loglevel): |
| 4208 |
logger.log(loglevel, 'Cannot locate old file: %r',old) |
| 4209 |
continue |
| 4210 |
|
| 4211 |
forensicfile=None |
| 4212 |
if theforensicdir: |
| 4213 |
if 'Filename' in new: |
| 4214 |
forensicdirname=delta_dirname(os.path.dirname(new['Filename']),theforensicdir) |
| 4215 |
elif 'File' in new: |
| 4216 |
forensicdirname=delta_dirname(os.path.dirname(new['File']),theforensicdir) |
| 4217 |
else: |
| 4218 |
assert(0) |
| 4219 |
forensicbasename = pa +'_'+ version_mangle(old['Version']) +'_'+ar+'.forensic' |
| 4220 |
forensicfile=os.path.join(forensicdirname,forensicbasename) |
| 4221 |
|
| 4222 |
seen_versions.append(old['Package']) |
| 4223 |
|
| 4224 |
yield (old,new, delta, forensicfile) |
| 4225 |
|
| 4226 |
|
| 4227 |
def do_delta_and_test(old_File, new_File, delta, forensicfile): |
| 4228 |
" returns (error_code, delta_size, delta_elaps, patch_elaps, info_delta, error_string) : for error_code, see EXIT STATUS in 'man debdelta' " |
| 4229 |
exitstatus=0 |
| 4230 |
if VERBOSE : logger.info('Creating: %r',delta) |
| 4231 |
|
| 4232 |
deltatmp=delta+'_tmp_' |
| 4233 |
ret= None |
| 4234 |
tdir=tempo() |
| 4235 |
ret = None # data returned from calls |
| 4236 |
elaps = None # delta time |
| 4237 |
delta_size = None |
| 4238 |
p_elaps = None # patching time |
| 4239 |
info_delta = [] # informations about created delta |
| 4240 |
error_string = '' # error when delta fails |
| 4241 |
|
| 4242 |
deltadirname=os.path.dirname(delta) |
| 4243 |
if not os.path.exists(deltadirname): #FIXME this does not respect --no-act |
| 4244 |
os.makedirs(deltadirname) |
| 4245 |
|
| 4246 |
free=freespace(deltadirname) |
| 4247 |
newdebsize=os.path.getsize(new_File) |
| 4248 |
if free and (free < (newdebsize /2 + 2**15)) : |
| 4249 |
if VERBOSE : logger.warning(' '+(_("Not enough disk space for storing `%s'.")), delta) |
| 4250 |
return (1, None, None, None, [], 'not_enough_disk_space') |
| 4251 |
|
| 4252 |
if forensicfile and not os.path.exists(forensicfile): |
| 4253 |
forensicdirname=os.path.dirname(forensicfile) |
| 4254 |
if not os.path.exists(forensicdirname): #FIXME this does not respect --no-act |
| 4255 |
os.makedirs(forensicdirname) |
| 4256 |
forensic_file=open(forensicfile,'w') |
| 4257 |
else: |
| 4258 |
forensic_file=None |
| 4259 |
|
| 4260 |
try: |
| 4261 |
ret=do_delta_(old_File,new_File, deltatmp, TD=tdir, forensic_file=forensic_file, info=info_delta) |
| 4262 |
(deltatmp_, percent, elaps, info_delta, gpg_hashes) = ret |
| 4263 |
if os.path.exists(deltatmp): |
| 4264 |
delta_size = os.path.getsize(deltatmp) |
| 4265 |
except (KeyboardInterrupt, SystemExit): |
| 4266 |
if os.path.exists(deltatmp): |
| 4267 |
os.unlink(deltatmp) |
| 4268 |
rmtree(tdir) |
| 4269 |
raise |
| 4270 |
except DebDeltaError: |
| 4271 |
s=sys.exc_info()[1] |
| 4272 |
error_string=str(s) |
| 4273 |
if not VERBOSE : logger.info(_('Creating:')+' '+repr(delta)) |
| 4274 |
logger.info(' Creation of delta failed, reason: %r', error_string) |
| 4275 |
if os.path.exists(deltatmp): |
| 4276 |
if KEEP: |
| 4277 |
logger.info(' '+_('You may want to examine:')+' '+str(deltatmp)) |
| 4278 |
else: |
| 4279 |
os.unlink(deltatmp) |
| 4280 |
if not s.retriable : |
| 4281 |
open(delta+'-fails','w').close() |
| 4282 |
exitstatus=max(exitstatus, s.exitcode) |
| 4283 |
ret = None |
| 4284 |
except: |
| 4285 |
exitstatus=4 |
| 4286 |
if os.path.exists(deltatmp): |
| 4287 |
os.unlink(deltatmp) |
| 4288 |
puke( " *** Error while creating delta "+delta) |
| 4289 |
open(delta+'-fails','w').close() |
| 4290 |
ret = None |
| 4291 |
|
| 4292 |
rmtree(tdir) |
| 4293 |
|
| 4294 |
if ret == None: |
| 4295 |
return (exitstatus, delta_size, elaps, p_elaps, info_delta, error_string) |
| 4296 |
|
| 4297 |
info_delta.append('ServerID: '+HOSTID) |
| 4298 |
info_delta.append('ServerBogomips: '+str(BOGOMIPS)) |
| 4299 |
|
| 4300 |
if MAX_DELTA_PERCENT and percent > MAX_DELTA_PERCENT: |
| 4301 |
os.unlink(deltatmp) |
| 4302 |
if VERBOSE : logger.info(' Warning, too big!') |
| 4303 |
open(delta+'-too-big','w').close() |
| 4304 |
return (exitstatus, delta_size, elaps, p_elaps, info_delta, 'too_big') |
| 4305 |
# |
| 4306 |
if DO_TEST : |
| 4307 |
##patch test |
| 4308 |
pret=None |
| 4309 |
try: |
| 4310 |
#test, ignoring gpg, that is added later on |
| 4311 |
pret=do_patch(deltatmp,old_File,None , info=info_delta, do_gpg=None) |
| 4312 |
except DebDeltaError: |
| 4313 |
s=sys.exc_info()[1] |
| 4314 |
logger.warning(' '+_('Error: testing of delta failed:')+' '+str(s)) |
| 4315 |
if os.path.exists(deltatmp): |
| 4316 |
if KEEP: |
| 4317 |
logger.warning(' '+_('You may want to examine:')+' '+str(deltatmp)) |
| 4318 |
else: |
| 4319 |
os.unlink(deltatmp) |
| 4320 |
if not s.retriable : |
| 4321 |
open(delta+'-fails','w').close() |
| 4322 |
except (KeyboardInterrupt, SystemExit): |
| 4323 |
if os.path.exists(deltatmp): |
| 4324 |
os.unlink(deltatmp) |
| 4325 |
raise |
| 4326 |
except Exception: |
| 4327 |
s=sys.exc_info()[1] |
| 4328 |
exitstatus=max(exitstatus,4) |
| 4329 |
puke(" *** Error while testing delta "+delta,s) |
| 4330 |
if os.path.exists(deltatmp): |
| 4331 |
if KEEP: |
| 4332 |
logger.warning(' '+_('You may want to examine:')+' '+str(deltatmp)) |
| 4333 |
else: |
| 4334 |
os.unlink(deltatmp) |
| 4335 |
open(delta+'-fails','w').close() |
| 4336 |
if pret == None: |
| 4337 |
return (max(exitstatus, 4), delta_size, elaps, p_elaps, info_delta, 'patch_fails') |
| 4338 |
|
| 4339 |
(newdeb_,p_elaps)=pret |
| 4340 |
info_delta.append('PatchTime: %.2f' % p_elaps) |
| 4341 |
##end patch test |
| 4342 |
#complete the delta file prepending to it the info |
| 4343 |
try: |
| 4344 |
hashes_info=append_info(deltatmp,info_delta) |
| 4345 |
# sign the delta |
| 4346 |
if DO_GPG: |
| 4347 |
gpg_hashes['info']=hashes_info |
| 4348 |
sign_delta(deltatmp,gpg_hashes) |
| 4349 |
except: |
| 4350 |
puke('debdeltas signing') |
| 4351 |
if os.path.exists(deltatmp): |
| 4352 |
os.unlink(deltatmp) |
| 4353 |
|
| 4354 |
#eventually, put in its place |
| 4355 |
if os.path.isfile(deltatmp): |
| 4356 |
# update delta size , since info and signatures were added |
| 4357 |
delta_size = os.path.getsize(deltatmp) |
| 4358 |
os.rename(deltatmp,delta) |
| 4359 |
return (exitstatus, delta_size, elaps, p_elaps, info_delta, '') |
| 4360 |
else: |
| 4361 |
return (max(exitstatus, 4), delta_size, elaps, p_elaps, info_delta, 'no_delta') |
| 4362 |
|
| 4363 |
|
| 4364 |
def init_apt_return_VersionCompare(): |
| 4365 |
try: |
| 4366 |
import apt_pkg |
| 4367 |
if 'init_system' in dir(apt_pkg): |
| 4368 |
apt_pkg.init_system() |
| 4369 |
elif 'InitSystem' in dir(apt_pkg): |
| 4370 |
apt_pkg.InitSystem() |
| 4371 |
else: assert 'fegrwq' == 0 |
| 4372 |
if 'VersionCompare' in dir(apt_pkg): |
| 4373 |
from apt_pkg import VersionCompare |
| 4374 |
elif 'version_compare' in dir(apt_pkg): |
| 4375 |
from apt_pkg import version_compare as VersionCompare |
| 4376 |
else: assert 'lopadf' == 0 |
| 4377 |
except ImportError: |
| 4378 |
raise DebDeltaError('python module "apt_pkg" is missing. Please install python-apt', retriable=True) |
| 4379 |
return VersionCompare |
| 4380 |
|
| 4381 |
|
| 4382 |
############ |
| 4383 |
|
| 4384 |
def do_deltas(debs): |
| 4385 |
exitstatus=0 |
| 4386 |
|
| 4387 |
|
| 4388 |
VersionCompare = init_apt_return_VersionCompare() |
| 4389 |
|
| 4390 |
if not debs: |
| 4391 |
logger.warning(_('Warning, no non-option arguments, debdeltas does nothing.')) |
| 4392 |
if not DEBUG: |
| 4393 |
return |
| 4394 |
elif not OLD and not CLEAN_DELTAS: |
| 4395 |
logger.warning(_('Warning, no --old arguments, debdeltas will not generate any deltas.')) |
| 4396 |
if not DEBUG: |
| 4397 |
return |
| 4398 |
|
| 4399 |
original_cwd = os.getcwd() |
| 4400 |
start_time = time.time() |
| 4401 |
|
| 4402 |
info_by_pack_arch={} |
| 4403 |
|
| 4404 |
info_by_file = {} #cache to speed up scanning file packages |
| 4405 |
|
| 4406 |
scanned_delta_dirs=set() |
| 4407 |
|
| 4408 |
# contains list of triples (filename,oldversion,newversion) |
| 4409 |
old_deltas_by_pack_arch={} |
| 4410 |
|
| 4411 |
ALT_DOUBLEDIR=[] # directories ending in // |
| 4412 |
for arg in ALT: |
| 4413 |
if os.path.isfile(arg): |
| 4414 |
if os.path.basename(arg) in ('Packages', 'Packages.gz','Packages.bz2'): |
| 4415 |
for a in iterate_Packages(arg): |
| 4416 |
a['Label'] = 'ALT' |
| 4417 |
#DEBUG a['Packages'] = arg |
| 4418 |
info_by_pack_arch_add(a, info_by_pack_arch) |
| 4419 |
elif arg[-4: ] == '.deb' : |
| 4420 |
a=scan_deb_byfile_lazy(arg, info_by_file) |
| 4421 |
if a: |
| 4422 |
a['Label'] = 'ALT' |
| 4423 |
info_by_pack_arch_add(a, info_by_pack_arch) |
| 4424 |
else: logger.error('Error! skipping file --alt %r',arg) |
| 4425 |
elif os.path.isdir(arg): |
| 4426 |
if arg[-2:] != '//' : |
| 4427 |
scan_deb_dir(arg, None, 'ALT', True, info_by_pack_arch) |
| 4428 |
else: |
| 4429 |
ALT_DOUBLEDIR.append(arg) |
| 4430 |
else: logger.error('Error, ignored --alt %r',arg) |
| 4431 |
|
| 4432 |
for arg in OLD: |
| 4433 |
if os.path.isfile(arg): |
| 4434 |
if os.path.basename(arg) in ('Packages', 'Packages.gz','Packages.bz2'): |
| 4435 |
for a in iterate_Packages(arg): |
| 4436 |
a['Label'] = 'OLD' |
| 4437 |
#DEBUG a['Packages'] = arg |
| 4438 |
info_by_pack_arch_add(a, info_by_pack_arch) |
| 4439 |
elif arg[-4: ] == '.deb' : |
| 4440 |
a=scan_deb_byfile(arg, info_by_file) |
| 4441 |
if a: |
| 4442 |
a['Label'] = 'OLD' |
| 4443 |
info_by_pack_arch_add(a, info_by_pack_arch) |
| 4444 |
else: logger.error('Error! skipping file --old %r',arg) |
| 4445 |
elif os.path.isdir(arg): |
| 4446 |
# no // support yet |
| 4447 |
scan_deb_dir(arg, None, 'OLD', False, info_by_pack_arch,info_by_file) |
| 4448 |
else: logger.error('Error, ignored --old %r',arg) |
| 4449 |
|
| 4450 |
#scan cmdline arguments and prepare list of debs and deltas |
| 4451 |
for arg in debs: |
| 4452 |
if os.path.isfile(arg): |
| 4453 |
if os.path.basename(arg) in ('Packages', 'Packages.gz','Packages.bz2'): |
| 4454 |
for a in iterate_Packages(arg): |
| 4455 |
a['Label'] = 'CMDLINE' #note that, if a file is scanned more than once, the last label is CMDLINE |
| 4456 |
#DEBUG a['Packages'] = arg |
| 4457 |
ofd = os.path.dirname(a['Filename']) |
| 4458 |
pa = a['Package'] |
| 4459 |
info_by_pack_arch_add(a, info_by_pack_arch) |
| 4460 |
for alt in ALT_DOUBLEDIR: |
| 4461 |
scan_deb_dir(delta_dirname(ofd,alt), pa , 'ALT', True, info_by_pack_arch) |
| 4462 |
if CLEAN_DELTAS: |
| 4463 |
scan_delta_dir(delta_dirname(ofd,DIR), pa, scanned_delta_dirs, old_deltas_by_pack_arch) |
| 4464 |
elif arg[-4: ] == '.deb': |
| 4465 |
a=scan_deb_byfile(arg, info_by_file) |
| 4466 |
if a : |
| 4467 |
a['Label'] = 'CMDLINE' |
| 4468 |
info_by_pack_arch_add(a, info_by_pack_arch) |
| 4469 |
ofd = os.path.dirname(arg) |
| 4470 |
for alt in ALT_DOUBLEDIR: |
| 4471 |
scan_deb_dir(delta_dirname(ofd,alt), a['Package'], 'ALT', True, info_by_pack_arch, info_by_file) |
| 4472 |
if CLEAN_DELTAS: |
| 4473 |
scan_delta_dir(delta_dirname(ofd,DIR), a['Package'], scanned_delta_dirs, old_deltas_by_pack_arch) |
| 4474 |
else: logger.error('Error! skipping cmdline argument: %r',arg) |
| 4475 |
elif os.path.isdir(arg): |
| 4476 |
scan_deb_dir(arg, None, 'CMDLINE', False, info_by_pack_arch, info_by_file) |
| 4477 |
for alt in ALT_DOUBLEDIR: |
| 4478 |
scan_deb_dir(delta_dirname(arg,alt), None, 'ALT', True, info_by_pack_arch, info_by_file) |
| 4479 |
if CLEAN_DELTAS: |
| 4480 |
scan_delta_dir(delta_dirname(arg,DIR), None, scanned_delta_dirs, old_deltas_by_pack_arch) |
| 4481 |
else: |
| 4482 |
logger.error('Error, skipping cmd line argument: %r',arg) |
| 4483 |
|
| 4484 |
if VERBOSE > 1 : logger.info(' total parsing time: %.1f ', ( -start_time + time.time())) |
| 4485 |
|
| 4486 |
for pa,ar in info_by_pack_arch : |
| 4487 |
info_pack=info_by_pack_arch[ (pa,ar) ] |
| 4488 |
assert(info_pack) |
| 4489 |
# delete old deltas |
| 4490 |
if CLEAN_DELTAS : |
| 4491 |
if (pa,ar) in old_deltas_by_pack_arch : |
| 4492 |
versions=[(o['Version']) for o in info_pack if (o['Label'] == 'CMDLINE')] |
| 4493 |
timegrace=time.time() - CLEAN_DELTAS_MTIME * 24 * 3600 |
| 4494 |
#DEBUG print pa,ar,versions |
| 4495 |
for f_d,o_d,n_d in old_deltas_by_pack_arch[ (pa,ar) ] : |
| 4496 |
if n_d not in versions : |
| 4497 |
if os.path.exists(f_d): |
| 4498 |
if os.stat(f_d)[ST_MTIME] < timegrace: |
| 4499 |
if VERBOSE: logger.debug(' Removing: %r', f_d) |
| 4500 |
if ACT: os.unlink(f_d) |
| 4501 |
elif VERBOSE > 1 : logger.debug(' Obsolete but still young, graced: %r',f_d) |
| 4502 |
else: |
| 4503 |
logger.debug('bellandata %r',f_d) |
| 4504 |
del versions |
| 4505 |
elif DEBUG > 1: |
| 4506 |
logger.debug('No deltas where found for: %r %r',pa,ar) |
| 4507 |
if N_DELTAS != None and 0 == N_DELTAS: |
| 4508 |
#debdeltas was invoked just to clean the archive |
| 4509 |
pass |
| 4510 |
else: |
| 4511 |
for old, new, delta, forensicfile in \ |
| 4512 |
iter_deltas_one_pack_arch(pa,ar,info_pack, DIR, FORENSICDIR, VersionCompare): |
| 4513 |
if not ACT: |
| 4514 |
logger.info('Would create: %r',delta) |
| 4515 |
else: |
| 4516 |
e=do_delta_and_test(old['File'], new['File'], delta, forensicfile) |
| 4517 |
exitstatus=max(exitstatus,e[0]) |
| 4518 |
|
| 4519 |
if VERBOSE : logger.info(' '+ ( _('Total running time: %.1f')),( -start_time + time.time())) |
| 4520 |
return exitstatus |
| 4521 |
|
| 4522 |
|
| 4523 |
##################################################### delta-upgrade |
| 4524 |
|
| 4525 |
class Predictor: |
| 4526 |
package_stats = None |
| 4527 |
upgrade_stats = None |
| 4528 |
def __init__(self): |
| 4529 |
import shelve |
| 4530 |
#self.shelve=shelve |
| 4531 |
if os.getuid() == 0: |
| 4532 |
basedir='/var/lib/debdelta' |
| 4533 |
else: |
| 4534 |
if not os.path.exists(os.path.expanduser('~/')): |
| 4535 |
logger.warning('(home directory does not exists, Predictor disabled)') |
| 4536 |
return |
| 4537 |
basedir=os.path.expanduser('~/.debdelta') |
| 4538 |
|
| 4539 |
s=os.path.join(basedir,'upgrade.db') |
| 4540 |
if not os.path.exists(basedir): |
| 4541 |
logger.info('Creating: %r',basedir) |
| 4542 |
os.makedirs(basedir) |
| 4543 |
self.upgrade_stats=shelve.open(s,flag='c') |
| 4544 |
|
| 4545 |
s=os.path.join(basedir,'packages_stats.db') |
| 4546 |
|
| 4547 |
if os.path.exists(s) or DEBUG > 1 : |
| 4548 |
self.package_stats=shelve.open(s,flag='c') |
| 4549 |
|
| 4550 |
self.patch_time_predictor=self.patch_time_predictor_math |
| 4551 |
|
| 4552 |
##### predictor for patching time |
| 4553 |
def patch_time_predictor_simple(self,p): |
| 4554 |
if 'ServerBogomips' in p and 'PatchTime' in p: |
| 4555 |
return (float(p[ 'PatchTime']) / BOGOMIPS * float(p['ServerBogomips']) ) |
| 4556 |
else: |
| 4557 |
return None |
| 4558 |
|
| 4559 |
def update(self,p,t): |
| 4560 |
#save delta info |
| 4561 |
if self.package_stats != None : |
| 4562 |
n=p['NEW/Package'] |
| 4563 |
d=copy(p) |
| 4564 |
d['LocalDeltaTime']=t |
| 4565 |
try: |
| 4566 |
self.package_stats[n]=d |
| 4567 |
except Exception: |
| 4568 |
logger.exception('ERROR:Cannot update package_stats') |
| 4569 |
|
| 4570 |
if self.upgrade_stats == None : |
| 4571 |
return |
| 4572 |
|
| 4573 |
s='ServerID' |
| 4574 |
if s not in p : |
| 4575 |
return |
| 4576 |
s=s+':'+p[s] |
| 4577 |
if s not in self.upgrade_stats: |
| 4578 |
r=1 |
| 4579 |
if 'ServerBogomips' in p : |
| 4580 |
r= float(p['ServerBogomips']) / BOGOMIPS |
| 4581 |
try: |
| 4582 |
self.upgrade_stats[s]={ 'PatchSpeedRatio' : r } |
| 4583 |
except Exception: |
| 4584 |
logger.exception('ERROR:Cannot update upgrade_stats') |
| 4585 |
|
| 4586 |
if 'PatchTime' not in p: |
| 4587 |
return |
| 4588 |
ut=float(p[ 'PatchTime']) |
| 4589 |
|
| 4590 |
r=self.upgrade_stats[s]['PatchSpeedRatio'] |
| 4591 |
|
| 4592 |
nr = 0.95 * r + 0.05 * ( t / ut ) |
| 4593 |
a=self.upgrade_stats[s] |
| 4594 |
a['PatchSpeedRatio'] = nr |
| 4595 |
try: |
| 4596 |
self.upgrade_stats[s]=a |
| 4597 |
except Exception: |
| 4598 |
logger.exception('ERROR:Cannot update upgrade_stats') |
| 4599 |
if VERBOSE > 1 : |
| 4600 |
logger.debug(' Upstream %r PatchSpeedRatio from %r to %r , %r', ut, r, nr, self.upgrade_stats[s]['PatchSpeedRatio']) |
| 4601 |
|
| 4602 |
def patch_time_predictor_math(self,p): |
| 4603 |
"Predicts time to patch." |
| 4604 |
if 'PatchTime' not in p: |
| 4605 |
return None |
| 4606 |
ut=float(p[ 'PatchTime']) |
| 4607 |
# |
| 4608 |
s='ServerID' |
| 4609 |
if s not in p : |
| 4610 |
return self.patch_time_predictor_simple(p) |
| 4611 |
s=s+':'+p[s] |
| 4612 |
if s not in self.upgrade_stats: |
| 4613 |
return self.patch_time_predictor_simple(p) |
| 4614 |
|
| 4615 |
r=self.upgrade_stats[s]['PatchSpeedRatio'] |
| 4616 |
return r * ut |
| 4617 |
|
| 4618 |
def delta_uri_from_config(config, **dictio): |
| 4619 |
secs=config.sections() |
| 4620 |
for s in secs: |
| 4621 |
opt=config.options(s) |
| 4622 |
if 'delta_uri' not in opt: |
| 4623 |
raise DebDeltaError('sources.conf section '+repr(s)+'does not contain delta_uri',exitcode=3) |
| 4624 |
match=True |
| 4625 |
for a in dictio: |
| 4626 |
#damn it, ConfigParser changes everything to lowercase ! |
| 4627 |
if ( a.lower() in opt ) and ( dictio[a] != config.get( s, a) ) : |
| 4628 |
#print '!!',a, repr(dictio[a]) , ' != ',repr(config.get( s, a)) |
| 4629 |
match=False |
| 4630 |
break |
| 4631 |
if match: |
| 4632 |
return config.get( s, 'delta_uri' ) |
| 4633 |
if VERBOSE : logger.warning(' '+_("(sources.conf does not provide a server for `%s')"), repr(dictio['PackageName'])) |
| 4634 |
|
| 4635 |
|
| 4636 |
|
| 4637 |
def delta_upgrade_(args): |
| 4638 |
# a list of all error exitcodes that derive from downloading and applying |
| 4639 |
mainexitcodes = [0] |
| 4640 |
|
| 4641 |
original_cwd = os.getcwd() |
| 4642 |
|
| 4643 |
import threading, fcntl, atexit, signal, socket |
| 4644 |
if sys.version_info.major == 2: |
| 4645 |
import Queue, urllib2, thread |
| 4646 |
from httplib import HTTPConnection, HTTPException |
| 4647 |
def http_responses_get(j): |
| 4648 |
import httplib |
| 4649 |
return httplib.responses.get(j,str(j)) |
| 4650 |
from urlparse import urlparse, urlunparse |
| 4651 |
from urllib2 import quote as urlquote |
| 4652 |
proxies=urllib2.getproxies() |
| 4653 |
else: |
| 4654 |
import queue as Queue |
| 4655 |
import _thread as thread |
| 4656 |
from http.client import HTTPConnection, HTTPException |
| 4657 |
#FIXME |
| 4658 |
http_responses_get = lambda j : str(j) |
| 4659 |
from urllib.parse import urlparse, urlunparse |
| 4660 |
from urllib.parse import quote as urlquote |
| 4661 |
#FIXME |
| 4662 |
proxies = [] |
| 4663 |
|
| 4664 |
try: |
| 4665 |
import requests |
| 4666 |
requests_session = requests.Session() |
| 4667 |
except ImportError: |
| 4668 |
if (sys.version_info.major == 2): |
| 4669 |
logger.info('You may want to install `requests`\n# sudo apt install python-requests\n') |
| 4670 |
else: |
| 4671 |
logger.warning('You should install `requests`\n# sudo apt install python3-requests\n') |
| 4672 |
requests=None |
| 4673 |
requests_session=None |
| 4674 |
|
| 4675 |
if VERBOSE and proxies: |
| 4676 |
#note that this message is indented, I dont know what's best in translations |
| 4677 |
logger.warning(_(' Proxy settings detected in the environment; using "urllib2" for downloading; but\n this disables some features and is in general slower and buggier. See man page.')) |
| 4678 |
#for example, urllib2 transforms http response "401" into "404" , and "302" into "200" |
| 4679 |
|
| 4680 |
config=ConfigParser() |
| 4681 |
a=config.read(['/etc/debdelta/sources.conf', expanduser('~/.debdelta/sources.conf') ]) |
| 4682 |
# FIXME this does not work as documented in Python |
| 4683 |
#if VERBOSE > 1 : print 'Read config files: ',repr(a) |
| 4684 |
|
| 4685 |
import warnings |
| 4686 |
warnings.simplefilter("ignore",FutureWarning) |
| 4687 |
|
| 4688 |
if DO_PROGRESS: |
| 4689 |
sys.stderr.write((_('Initializing APT cache...')).ljust(terminalcolumns)+'\r') |
| 4690 |
elif VERBOSE: |
| 4691 |
logger.debug(_('Initializing APT cache...')) |
| 4692 |
|
| 4693 |
try: |
| 4694 |
import apt_pkg |
| 4695 |
except ImportError: |
| 4696 |
raise DebDeltaError('python module "apt_pkg" is missing. Please install python-apt',True) |
| 4697 |
|
| 4698 |
try: |
| 4699 |
import apt |
| 4700 |
except ImportError: |
| 4701 |
raise DebDeltaError('python module "apt" is missing. Please install a newer version of python-apt (newer than 0.6.12).',True) |
| 4702 |
|
| 4703 |
apt_pkg.init() |
| 4704 |
|
| 4705 |
#from apt import SizeToStr |
| 4706 |
## Return a string describing the size in a human-readable manner using |
| 4707 |
## SI prefix and base-10 units, e.g. '1k' for 1000, '1M' for 1000000, etc. |
| 4708 |
|
| 4709 |
def SizeToKibiStr(a): |
| 4710 |
"this uses kibibytes (altough the program prints them as kilobytes)" |
| 4711 |
if a < 8096 : |
| 4712 |
return str(int(a))+'B' |
| 4713 |
elif a < 8096*1024: |
| 4714 |
return str(int(a/1024))+'kB' |
| 4715 |
else: |
| 4716 |
return str(int(a/1024/1024))+'MB' |
| 4717 |
|
| 4718 |
if DO_PROGRESS: |
| 4719 |
sys.stderr.write((_('Upgrading APT cache...')).ljust(terminalcolumns)+'\r') |
| 4720 |
elif VERBOSE: |
| 4721 |
logger.debug(_('Upgrading APT cache...')) |
| 4722 |
|
| 4723 |
cache=apt.Cache() |
| 4724 |
try: |
| 4725 |
cache.upgrade(True) |
| 4726 |
except: |
| 4727 |
if DO_PROGRESS: |
| 4728 |
sys.stderr.write((_('Failed! Safe upgrading APT cache...')).ljust(terminalcolumns)+'\r') |
| 4729 |
elif VERBOSE: |
| 4730 |
logger.debug(_('Failed! Safe upgrading APT cache...')) |
| 4731 |
cache.upgrade(False) |
| 4732 |
|
| 4733 |
if DO_PROGRESS: |
| 4734 |
sys.stderr.write((_('Upgraded APT cache.')).ljust(terminalcolumns)+'\r') |
| 4735 |
elif VERBOSE: |
| 4736 |
logger.debug(_('Upgraded APT cache.')) |
| 4737 |
|
| 4738 |
diversions=scan_diversions() |
| 4739 |
|
| 4740 |
if DIR == None: |
| 4741 |
if os.getuid() == 0: |
| 4742 |
DEB_DIR='/var/cache/apt/archives' |
| 4743 |
else: |
| 4744 |
DEB_DIR='/tmp/archives' |
| 4745 |
else: |
| 4746 |
DEB_DIR=DIR |
| 4747 |
if not os.path.exists(DEB_DIR): |
| 4748 |
os.mkdir(DEB_DIR) |
| 4749 |
if not os.path.exists(DEB_DIR+'/partial'): |
| 4750 |
os.mkdir(DEB_DIR+'/partial') |
| 4751 |
|
| 4752 |
try: |
| 4753 |
##APT does (according to strace) |
| 4754 |
#open("/var/cache/apt/archives/lock", O_RDWR|O_CREAT|O_TRUNC, 0640) = 17 |
| 4755 |
#fcntl64(17, F_SETFD, FD_CLOEXEC) = 0 |
| 4756 |
#fcntl64(17, F_SETLK, {type=F_WRLCK, whence=SEEK_SET, start=0, len=0}) = 0 |
| 4757 |
##so |
| 4758 |
a=os.open(DEB_DIR+'/lock', os.O_RDWR | os.O_TRUNC | os.O_CREAT, 0o640) |
| 4759 |
fcntl.fcntl(a, fcntl.F_SETFD, fcntl.FD_CLOEXEC) |
| 4760 |
# synopsis lockf( fd, operation, [length, [start, [whence]]]) |
| 4761 |
fcntl.lockf(a, fcntl.LOCK_EX | fcntl.LOCK_NB, 0,0,0) |
| 4762 |
except IOError: |
| 4763 |
s=sys.exc_info()[1] |
| 4764 |
from errno import EAGAIN |
| 4765 |
if s.errno == EAGAIN : |
| 4766 |
a=' already locked!' |
| 4767 |
else: |
| 4768 |
a=str(s) |
| 4769 |
if DEB_DIR == '/var/cache/apt/archives' : |
| 4770 |
a=a+' (is APT running?)' |
| 4771 |
raise DebDeltaError('could not lock dir: '+DEB_DIR+' '+a, retriable=True) |
| 4772 |
|
| 4773 |
if VERBOSE or DEB_DIR != '/var/cache/apt/archives' : |
| 4774 |
logger.info(_('Recreated debs are saved in the directory %s'), DEB_DIR) |
| 4775 |
|
| 4776 |
#these are the packages that do not have a delta |
| 4777 |
no_delta = [] |
| 4778 |
|
| 4779 |
total_time = -time.time() |
| 4780 |
|
| 4781 |
##### predictor for patching time |
| 4782 |
if DO_PREDICTOR: |
| 4783 |
predictor = Predictor() |
| 4784 |
|
| 4785 |
#this is a dictonary (key is package name) of parameters of deltas |
| 4786 |
#(to add some math in the future) |
| 4787 |
params_of_delta={} |
| 4788 |
|
| 4789 |
patching_queue=Queue.Queue() |
| 4790 |
thread_returns={} |
| 4791 |
######################## thread_do_patch |
| 4792 |
def thread_do_patch(que, no_delta, returns, exitcodes, forensics): |
| 4793 |
if VERBOSE > 1 : logger.info(' Patching thread started. ') |
| 4794 |
debs_size=0 |
| 4795 |
debs_time=0 |
| 4796 |
# in case of early exception |
| 4797 |
returns['debs_size']=debs_size |
| 4798 |
returns['debs_time']=debs_time |
| 4799 |
while 1: |
| 4800 |
a = que.get() |
| 4801 |
if a == None: |
| 4802 |
break |
| 4803 |
(name, delta , newdeb, deb_uri) = a |
| 4804 |
debs_time -= time.time() |
| 4805 |
TD=tempo() |
| 4806 |
if not ACT: |
| 4807 |
logger.info('Would create: %r ',newdeb) |
| 4808 |
else: |
| 4809 |
if VERBOSE >= 2 : logger.debug(' Now patching for: %r',name) |
| 4810 |
try: |
| 4811 |
#start_time=time.time() |
| 4812 |
returns['patchname']=os.path.basename(newdeb) |
| 4813 |
ret=do_patch_(delta, '/', newdeb , TD, returns, |
| 4814 |
diversions=diversions, do_progress=False) |
| 4815 |
del returns['patchname'] |
| 4816 |
l = os.path.getsize(newdeb) |
| 4817 |
#a=time.time() - start_time |
| 4818 |
a=ret[1] |
| 4819 |
#dear translator, please align this line with the line saying 'Downloaded, ...' |
| 4820 |
msg=_("Created, time %(time)5.2fsec, speed %(speed)4s/sec, %(name)s") |
| 4821 |
msgdata={'time' : a, 'speed' : SizeToKibiStr(l / (a+0.001)), 'name' : os.path.basename(newdeb)} |
| 4822 |
if DO_PROGRESS: |
| 4823 |
sys.stderr.write((msg % msgdata).ljust(terminalcolumns)+'\n') |
| 4824 |
else: |
| 4825 |
logger.info(msg % msgdata) |
| 4826 |
except KeyboardInterrupt: |
| 4827 |
thread.interrupt_main() |
| 4828 |
rmtree(TD) |
| 4829 |
return |
| 4830 |
except DebDeltaError: |
| 4831 |
s=sys.exc_info()[1] |
| 4832 |
puke(' Error: applying of delta for '+name+' failed: ',s) |
| 4833 |
if 'e' in DEB_POLICY: |
| 4834 |
no_delta.append( (deb_uri, newdeb) ) |
| 4835 |
elif VERBOSE > 1 : logger.debug(' No deb-policy "e", no download of %r',deb_uri) |
| 4836 |
forensics.append(s.logs) |
| 4837 |
exitcodes.append(s.exitcode) |
| 4838 |
except: |
| 4839 |
if puke == None: return |
| 4840 |
puke( " *** Error while applying delta for "+name+": ") |
| 4841 |
if 'e' in DEB_POLICY: |
| 4842 |
no_delta.append( (deb_uri, newdeb) ) |
| 4843 |
elif VERBOSE > 1 : logger.debug(' No deb-policy "e", no download of %r',deb_uri) |
| 4844 |
exitcodes.append(4) |
| 4845 |
else: |
| 4846 |
if name in params_of_delta : |
| 4847 |
p= params_of_delta[name] |
| 4848 |
name,elaps=ret |
| 4849 |
if DO_PREDICTOR: |
| 4850 |
predictor.update(p,elaps) |
| 4851 |
if VERBOSE > 1 : |
| 4852 |
t=predictor.patch_time_predictor(p) |
| 4853 |
if t: logger.debug(' (Predicted %.3f sec )', t) |
| 4854 |
debs_size += os.path.getsize(newdeb) |
| 4855 |
if os.path.exists(delta): |
| 4856 |
os.unlink(delta) |
| 4857 |
rmtree(TD) |
| 4858 |
debs_time += time.time() |
| 4859 |
returns['debs_size']=debs_size |
| 4860 |
returns['debs_time']=debs_time |
| 4861 |
if VERBOSE > 1 : logger.debug(' Patching thread ended , bye bye. ') |
| 4862 |
|
| 4863 |
##################################### |
| 4864 |
|
| 4865 |
def progress_string(statusdb): |
| 4866 |
download='' |
| 4867 |
if 'downloaduri' in statusdb: |
| 4868 |
download="D %2d%% (%4s/s) %s " % \ |
| 4869 |
(statusdb.get('downloadprogress',-1), |
| 4870 |
statusdb.get('downloadspeed','-'), |
| 4871 |
statusdb['downloaduri']) |
| 4872 |
patch='' |
| 4873 |
if 'patchname' in statusdb: |
| 4874 |
patch='P %2d%% %s'% (statusdb.get('patchprogress',-1), statusdb['patchname']) |
| 4875 |
if terminalcolumns == None: |
| 4876 |
return download+' ; '+patch |
| 4877 |
if not patch: |
| 4878 |
return download[:terminalcolumns] |
| 4879 |
if not download: |
| 4880 |
return patch[:terminalcolumns] |
| 4881 |
ld=len(download) |
| 4882 |
lp=len(patch) |
| 4883 |
b=ld + lp |
| 4884 |
if b < terminalcolumns - 3 : |
| 4885 |
return download+' ; '+patch |
| 4886 |
a=float(terminalcolumns-5)/float(b) |
| 4887 |
ld=int(ld*a) |
| 4888 |
lp=int(lp*a) |
| 4889 |
return download[:ld] + ' ; ' + patch[:lp] + '..' |
| 4890 |
|
| 4891 |
|
| 4892 |
#################### manage connections |
| 4893 |
#keeps a cache of all connections, by URL |
| 4894 |
http_conns={} |
| 4895 |
|
| 4896 |
def conn_by_url(url): |
| 4897 |
url=urlparse(url)[1] |
| 4898 |
if url not in http_conns: |
| 4899 |
if VERBOSE > 1 : logger.debug(' Opening connection to: %r',url) |
| 4900 |
http_conns[url] = HTTPConnection(url, timeout=TIMEOUT) |
| 4901 |
return http_conns[url] |
| 4902 |
|
| 4903 |
def conn_close(url,fatal=False): |
| 4904 |
url=urlparse(url)[1] |
| 4905 |
conn=http_conns.get(url) |
| 4906 |
if fatal: |
| 4907 |
http_conns[url] = None |
| 4908 |
else: |
| 4909 |
del http_conns[url] |
| 4910 |
if conn != None : |
| 4911 |
if VERBOSE > 1 : logger.debug(' Closing connection to: %r',url) |
| 4912 |
conn.close() |
| 4913 |
|
| 4914 |
#### |
| 4915 |
|
| 4916 |
def _connect(uri, headers): |
| 4917 |
"""connects for a GET ; returns (filetype, statuscode, servermessage, getheaders) |
| 4918 |
in case of errors returns (exception, statuscode, None, None) |
| 4919 |
""" |
| 4920 |
uri_p=urlparse(uri) |
| 4921 |
###### python3 code |
| 4922 |
if requests and uri_p.scheme in ('http','https') : |
| 4923 |
try: |
| 4924 |
req = requests_session.get(uri, headers=headers, stream = True) |
| 4925 |
req.read = req.raw.read |
| 4926 |
return req, req.raw.status, getattr(req.raw,'msg','(no message)'), req.raw.info() |
| 4927 |
except requests.exceptions.ConnectionError: |
| 4928 |
e=sys.exc_info()[1] |
| 4929 |
puke( 'Connection error (fatal): %r'%(uri,),e) |
| 4930 |
return e, None, None, None |
| 4931 |
except requests.exceptions.RequestException: |
| 4932 |
e=sys.exc_info()[1] |
| 4933 |
puke( 'Error in http request (fatal): %r'%(uri,),e) |
| 4934 |
return e, None, None, None |
| 4935 |
elif sys.version_info.major == 3: |
| 4936 |
import urllib.request |
| 4937 |
try: |
| 4938 |
req = urllib.request.urlopen(uri,) |
| 4939 |
return req, getattr(req,'code',None), getattr(req,'msg','(no message)'), req.info() |
| 4940 |
except urllib.request.HTTPError: |
| 4941 |
e=sys.exc_info()[1] |
| 4942 |
return e, e.code, None, None |
| 4943 |
except Exception: |
| 4944 |
e=sys.exc_info()[1] |
| 4945 |
#puke( 'Exception while accessing',uri) |
| 4946 |
return DebDeltaError('for %s : %s'%(uri,e)), None, None, None |
| 4947 |
###### python2 code |
| 4948 |
if uri_p.scheme in ('http','https') and not proxies: |
| 4949 |
#use persistent http connections |
| 4950 |
conn=conn_by_url(uri) |
| 4951 |
if conn == None : |
| 4952 |
return None, None, None, None |
| 4953 |
try: |
| 4954 |
conn.request("GET", urlquote(uri_p[2]),headers=headers) |
| 4955 |
r = conn.getresponse() |
| 4956 |
return r, r.status, r.reason, r.msg |
| 4957 |
except (HTTPException, socket.error): |
| 4958 |
e=sys.exc_info()[1] |
| 4959 |
if VERBOSE : puke( ' Connection error (retrying): %r'%(uri_p[1],),e) |
| 4960 |
conn_close(uri) |
| 4961 |
try: |
| 4962 |
conn=conn_by_url(uri) |
| 4963 |
conn.request("GET", urlquote(uri_p[2]),headers=headers) |
| 4964 |
r = conn.getresponse() |
| 4965 |
return r, r.status, r.reason, r.msg |
| 4966 |
except (HTTPException, socket.error): |
| 4967 |
e=sys.exc_info()[1] |
| 4968 |
puke( 'Connection error (fatal): ',uri_p[1]) |
| 4969 |
mainexitcodes.append(1) |
| 4970 |
try: |
| 4971 |
conn_close(uri,fatal=True) |
| 4972 |
except: pass |
| 4973 |
mainexitcodes.append(1) |
| 4974 |
return e, None, None, None |
| 4975 |
else: #use urllib2 |
| 4976 |
if uri_p.scheme in ('http','https'): |
| 4977 |
a=[copy(z) for z in uri_p] |
| 4978 |
a[2]=urlquote(uri_p[2]) |
| 4979 |
uri=urlunparse(a) |
| 4980 |
import urllib2 |
| 4981 |
try: |
| 4982 |
req = urllib2.Request(uri, headers=headers) |
| 4983 |
r = urllib2.urlopen(req) |
| 4984 |
#print r.info(),dir(r),r.code |
| 4985 |
return r, getattr(r,'code',None), getattr(r,'msg','(no message)'), r.info() |
| 4986 |
except urllib2.HTTPError: |
| 4987 |
e=sys.exc_info()[1] |
| 4988 |
return e.code, None, None, None |
| 4989 |
except (HTTPException,socket.error,urllib2.URLError): |
| 4990 |
e=sys.exc_info()[1] |
| 4991 |
puke( 'Connection error (fatal): %r'%(uri,), e) |
| 4992 |
mainexitcodes.append(1) |
| 4993 |
return e, None, None, None |
| 4994 |
|
| 4995 |
################################################# various HTTP facilities |
| 4996 |
|
| 4997 |
def _parse_ContentRange(s): |
| 4998 |
#bytes 0-1023/25328 |
| 4999 |
if not s or s[:6] != "bytes " : |
| 5000 |
logger.warning("Malformed Content-Range %r", s) |
| 5001 |
return |
| 5002 |
a=s[6:].split('/') |
| 5003 |
if len(a) != 2 : |
| 5004 |
logger.warning("Malformed Content-Range %r", s) |
| 5005 |
return |
| 5006 |
b=a[0].split('-') |
| 5007 |
if len(b) != 2 : |
| 5008 |
logger.warning("Malformed Content-Range %r", s) |
| 5009 |
return |
| 5010 |
return int(b[0]),int(b[1]),int(a[1]) |
| 5011 |
###################################### test_uri |
| 5012 |
def test_uri(uri): |
| 5013 |
conn=conn_by_url(uri) |
| 5014 |
if conn == None: return None |
| 5015 |
uri_p=urlparse(uri) |
| 5016 |
assert(uri_p[0] in ('http','https')) |
| 5017 |
conn.request("HEAD", urlquote(uri_p[2]),headers=HTTP_USER_AGENT) |
| 5018 |
r = conn.getresponse() |
| 5019 |
r.read() |
| 5020 |
r.close() |
| 5021 |
return r.status |
| 5022 |
|
| 5023 |
###################################### download_10k_uri |
| 5024 |
def download_10k_uri(uri,outname): |
| 5025 |
"in case of connection error, returns the (error, None, None, None) ; otherwise returns (status,len,outname,complete)" |
| 5026 |
#download |
| 5027 |
uri_p=urlparse(uri) |
| 5028 |
assert(uri_p[0] in ('http', 'https')) |
| 5029 |
re=copy(HTTP_USER_AGENT) |
| 5030 |
re["Range"] = "bytes=0-10239" |
| 5031 |
complete=False |
| 5032 |
r, status, msg, responseheaders=_connect(uri, re) |
| 5033 |
if not hasattr(r,'read') and responseheaders==None: |
| 5034 |
return r, None, None, None |
| 5035 |
if status == 301 and 'location' in responseheaders: |
| 5036 |
r.read() |
| 5037 |
r.close() |
| 5038 |
return download_10k_uri(responseheaders['location'], outname) |
| 5039 |
elif status == 206: |
| 5040 |
outnametemp=os.path.join(os.path.dirname(outname),'partial',os.path.basename(outname)) |
| 5041 |
try: |
| 5042 |
l = _parse_ContentRange(responseheaders['Content-Range'])[2] |
| 5043 |
except (KeyError, ValueError): |
| 5044 |
l = None |
| 5045 |
elif status == 200: |
| 5046 |
outnametemp=outname |
| 5047 |
complete=True |
| 5048 |
try: |
| 5049 |
l=long(responseheaders.get('Content-Length')) |
| 5050 |
except: |
| 5051 |
l=None |
| 5052 |
else: #FIXME how do we deal with a FTP mirror of deltas ? |
| 5053 |
r.read() |
| 5054 |
r.close() |
| 5055 |
return status, None, None, None |
| 5056 |
s=r.read() |
| 5057 |
r.close() |
| 5058 |
#(maybe I did not understand the status 206 ?) |
| 5059 |
if not complete and l != None and len(s) >= l: |
| 5060 |
outnametemp=outname |
| 5061 |
complete=True |
| 5062 |
if os.path.exists(outnametemp) and os.path.getsize(outnametemp) >= len(s) : |
| 5063 |
#do not truncate preexisting file |
| 5064 |
complete=(os.path.getsize(outnametemp) >= l) |
| 5065 |
return status, outnametemp, l, complete |
| 5066 |
out=open(outnametemp,'wb') |
| 5067 |
out.write(s) |
| 5068 |
out.close() |
| 5069 |
return status, outnametemp, l, complete |
| 5070 |
|
| 5071 |
###################################### download_uri |
| 5072 |
def download_uri(uri, outname, conn_time, len_downloaded, statusdb): |
| 5073 |
"returns `(time_elapsed, bytes_downloaded)`; or `None` in case of error, " |
| 5074 |
outnametemp=os.path.join(os.path.dirname(outname),'partial',os.path.basename(outname)) |
| 5075 |
re=copy(HTTP_USER_AGENT) |
| 5076 |
#content range |
| 5077 |
l=None |
| 5078 |
if os.path.exists(outnametemp): |
| 5079 |
#shamelessly adapted from APT, methods/http.cc |
| 5080 |
s=os.stat(outnametemp) |
| 5081 |
l=s[ST_SIZE] |
| 5082 |
#t=s[ST_MTIME] |
| 5083 |
### unfortunately these do not yet work |
| 5084 |
#thank god for http://docs.python.org/lib/module-time.html |
| 5085 |
#actually APT does |
| 5086 |
#t=time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(t)) |
| 5087 |
##re["If-Range"] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(t)) |
| 5088 |
####re["If-Range"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(t)) |
| 5089 |
re["Range"] = "bytes=%li-" % ( (long(l)-1) ) |
| 5090 |
#start downloading |
| 5091 |
start_time=time.time() |
| 5092 |
r, status, message, responseheaders = _connect(uri, re) |
| 5093 |
if not hasattr(r,'read') and responseheaders==None: |
| 5094 |
return |
| 5095 |
if status in (301,302) and 'location' in responseheaders: |
| 5096 |
r.read() |
| 5097 |
r.close() |
| 5098 |
if VERBOSE > 1 : logger.debug(_(' Redirect to:')+' '+repr(responseheaders['location'])) |
| 5099 |
return download_uri(responseheaders['location'], outname, conn_time, len_downloaded, statusdb) |
| 5100 |
if not ( status == None or status == 200 or ( status == 206 and l != None ) ): |
| 5101 |
if VERBOSE : |
| 5102 |
logger.warning('Connection problem, status %r msg: %r uri: %r', status, message, uri) |
| 5103 |
r.read() |
| 5104 |
r.close() |
| 5105 |
return |
| 5106 |
#print 'ooK Content-Range', r.getheader('Content-Range') #HACK |
| 5107 |
if l and status == 200 : |
| 5108 |
logger.warning(' Hmmm... our HTTP range request failed, %r %r %r',re,status,message) |
| 5109 |
if status == 200 : |
| 5110 |
out=open(outnametemp,'wb') |
| 5111 |
try: |
| 5112 |
total_len = long(responseheaders['Content-Length']) |
| 5113 |
except (KeyError,ValueError): |
| 5114 |
total_len=None |
| 5115 |
elif status == 206 : |
| 5116 |
#APT does scanf of "bytes %lu-%*u/%lu",&StartPos,&Size |
| 5117 |
#first-byte-pos "-" last-byte-pos "/" instance-length |
| 5118 |
out=open(outnametemp,'ab') |
| 5119 |
try: |
| 5120 |
a,b,total_len =_parse_ContentRange(responseheaders['Content-Range']) |
| 5121 |
except (KeyError,ValueError): |
| 5122 |
e=sys.exc_info()[1] |
| 5123 |
logger.warning('! problem, http response [206], Content Range %s , error %s , uri %s\n' %\ |
| 5124 |
(responseheaders.get('Content-Range'),e,uri)) |
| 5125 |
return |
| 5126 |
out.seek(a) |
| 5127 |
out.truncate() |
| 5128 |
else: |
| 5129 |
out=open(outnametemp,'wb') |
| 5130 |
try: |
| 5131 |
total_len=long(responseheaders.get('Content-length')) |
| 5132 |
except ValueError: |
| 5133 |
total_len=None |
| 5134 |
|
| 5135 |
free=freespace(os.path.dirname(outname)) |
| 5136 |
if total_len != None and free and (free + 2**14 ) < total_len : |
| 5137 |
logger.warning(_('Not enough disk space to download:')+' '+os.path.basename(uri)) |
| 5138 |
r.read() |
| 5139 |
r.close() |
| 5140 |
mainexitcodes.append(1) |
| 5141 |
return |
| 5142 |
j=0 |
| 5143 |
|
| 5144 |
s=r.read(1024) |
| 5145 |
while s and (total_len == None or out.tell() < total_len ): |
| 5146 |
j+=len(s) |
| 5147 |
out.write(s) |
| 5148 |
if total_len: |
| 5149 |
statusdb['downloadprogress']=99.9 * out.tell() / total_len |
| 5150 |
a=time.time() + conn_time - start_time |
| 5151 |
if a > 0.5 : |
| 5152 |
statusdb['downloadspeed']=SizeToKibiStr(float(j+len_downloaded)/a) |
| 5153 |
s=r.read(1024) |
| 5154 |
out.close() |
| 5155 |
r.close() |
| 5156 |
#end of download |
| 5157 |
a=time.time() - start_time |
| 5158 |
#if total_len == None: |
| 5159 |
# total_len = os.path.getsize(outnametemp) |
| 5160 |
|
| 5161 |
#dear translator, please align this line with the line saying 'Created, ...' |
| 5162 |
msg=_("Downloaded, time %(time)5.2fsec, speed %(speed)4s/sec, %(name)s") |
| 5163 |
msgdata={'time' : a , 'speed' : SizeToKibiStr(j / (a+0.001)) , 'name' : os.path.basename(uri)} |
| 5164 |
if DO_PROGRESS: |
| 5165 |
sys.stderr.write((msg % msgdata).ljust(terminalcolumns)+'\n') |
| 5166 |
else: |
| 5167 |
logger.info(msg % msgdata) |
| 5168 |
|
| 5169 |
os.rename(outnametemp,outname) |
| 5170 |
#FIXME this is incorrect by 1024 bytes |
| 5171 |
return conn_time + a, (j+len_downloaded) |
| 5172 |
|
| 5173 |
###################################### end of HTTP stuff |
| 5174 |
|
| 5175 |
################### start patching thread |
| 5176 |
forensics=[] |
| 5177 |
|
| 5178 |
patching_thread=threading.Thread( |
| 5179 |
target=thread_do_patch , |
| 5180 |
args=(patching_queue, no_delta, thread_returns, mainexitcodes, forensics) ) |
| 5181 |
patching_thread.daemon=True |
| 5182 |
patching_thread.start() |
| 5183 |
|
| 5184 |
################### first merry-go-round |
| 5185 |
|
| 5186 |
deltas_down_size=0 |
| 5187 |
deltas_down_time=0 |
| 5188 |
|
| 5189 |
#this is a list of tuples of ..... |
| 5190 |
available_deltas=[] |
| 5191 |
|
| 5192 |
not_available_deltas=[] |
| 5193 |
|
| 5194 |
if hasattr(apt.package.Package,'is_installed'): |
| 5195 |
is_installed=lambda p : p.is_installed |
| 5196 |
elif hasattr(apt.package.Package,'isInstalled'): |
| 5197 |
is_installed=lambda p : p.isInstalled |
| 5198 |
else: assert 0 |
| 5199 |
|
| 5200 |
if hasattr(apt.package.Package,'marked_upgrade'): |
| 5201 |
marked_upgrade=lambda p : p.marked_upgrade |
| 5202 |
elif hasattr(apt.package.Package,'markedUpgrade'): |
| 5203 |
marked_upgrade=lambda p : p.markedUpgrade |
| 5204 |
else: assert 0 |
| 5205 |
|
| 5206 |
progress_count=0 |
| 5207 |
|
| 5208 |
## first merry-go-round, use package cache to fill available_deltas, download 10kB of each delta |
| 5209 |
for p in cache : |
| 5210 |
#print progress |
| 5211 |
if DO_PROGRESS : |
| 5212 |
progress_count+=1 |
| 5213 |
if 0 == (progress_count & 2047): |
| 5214 |
sys.stderr.write(('%2.1f%%' % (float(progress_count) * 100.0 / len(cache))).ljust(terminalcolumns) + '\r') |
| 5215 |
|
| 5216 |
if is_installed(p) and marked_upgrade(p) : |
| 5217 |
if args and p.name not in args: |
| 5218 |
continue |
| 5219 |
#thanks a lot to Julian Andres Klode |
| 5220 |
candidate=p.candidate |
| 5221 |
origin = p.candidate.origins[0] |
| 5222 |
arch=candidate.architecture |
| 5223 |
deb_uri=candidate.uri |
| 5224 |
installed_version=p.installed.version |
| 5225 |
candidate_version=p.candidate.version |
| 5226 |
newsize=p.candidate.size |
| 5227 |
deb_path=str.split(deb_uri,'/') |
| 5228 |
try: |
| 5229 |
thepoolindex=deb_path.index('pool') |
| 5230 |
except ValueError: |
| 5231 |
logger.warning('! Package "%s" (version %s) does not have "pool" in the uri %s \n' % (p.name, candidate_version, deb_uri)) |
| 5232 |
continue |
| 5233 |
deb_path=str.join('/',deb_path[(thepoolindex):]) |
| 5234 |
|
| 5235 |
#try all possible variants of the filename |
| 5236 |
newdebs=[p.shortname+'_'+candidate_version+'_'+arch+'.deb', |
| 5237 |
os.path.basename(deb_uri)] |
| 5238 |
if ':' in candidate_version: |
| 5239 |
a=candidate_version.split(':') |
| 5240 |
newdebs.append(p.shortname+'_'+a[1]+'_'+arch+'.deb') |
| 5241 |
newdebs.append(p.shortname+'_'+a[0]+'%3A'+a[1]+'_'+arch+'.deb') |
| 5242 |
newdebs.append(p.shortname+'_'+a[0]+'%3a'+a[1]+'_'+arch+'.deb') |
| 5243 |
|
| 5244 |
for newdeb in newdebs: |
| 5245 |
if os.path.exists(DEB_DIR+'/'+newdeb) or \ |
| 5246 |
os.path.exists('/var/cache/apt/archives/'+newdeb): |
| 5247 |
if VERBOSE > 1 : logger.debug(' Already downloaded %r %r as %r',p.name,candidate_version,newdeb) |
| 5248 |
newdeb=None |
| 5249 |
break |
| 5250 |
if newdeb == None: |
| 5251 |
continue |
| 5252 |
newdeb = DEB_DIR+'/'+newdebs[0] |
| 5253 |
|
| 5254 |
if VERBOSE > 1 : logger.debug(' Looking for a delta for %s from %s to %s ', p.name, installed_version, candidate_version) |
| 5255 |
delta_uri_base=delta_uri_from_config(config, |
| 5256 |
Origin=origin.origin, |
| 5257 |
Label=origin.label, |
| 5258 |
Site=origin.site, |
| 5259 |
Archive=origin.archive, |
| 5260 |
PackageName=p.name) |
| 5261 |
if delta_uri_base == None: |
| 5262 |
if 's' in DEB_POLICY: |
| 5263 |
no_delta.append( (deb_uri, newdeb) ) |
| 5264 |
continue |
| 5265 |
|
| 5266 |
a=urlparse(delta_uri_base) |
| 5267 |
assert(a[0] in ('http', 'https')) |
| 5268 |
|
| 5269 |
#delta name |
| 5270 |
delta_name = delta_base_name(p.shortname, installed_version, |
| 5271 |
candidate_version, arch) |
| 5272 |
|
| 5273 |
uri=delta_uri_base+'/'+os.path.dirname(deb_path)+'/'+delta_name |
| 5274 |
|
| 5275 |
#download first part of delta |
| 5276 |
abs_delta_name= DEB_DIR+'/'+delta_name |
| 5277 |
|
| 5278 |
#maybe it is already here |
| 5279 |
if os.path.exists(abs_delta_name): |
| 5280 |
a = abs_delta_name |
| 5281 |
else: |
| 5282 |
a = DEB_DIR+'/partial/'+delta_name |
| 5283 |
if not os.path.exists(a): |
| 5284 |
a = None |
| 5285 |
if a: |
| 5286 |
l=os.path.getsize(a) |
| 5287 |
if VERBOSE > 1 : logger.debug(' Already here: %r',abs_delta_name) |
| 5288 |
s=get_info_fast(a) |
| 5289 |
if s: |
| 5290 |
params_of_delta[p.name]=info_2_db(s) |
| 5291 |
available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, a, True) ) |
| 5292 |
continue |
| 5293 |
#if not, download its first part |
| 5294 |
|
| 5295 |
if DO_PROGRESS: |
| 5296 |
sys.stderr.write(('%2.1f%% ' % (float(progress_count) * 100.0 / len(cache)) + \ |
| 5297 |
_('Downloading head of %s...') % p.name).ljust(terminalcolumns) + '\r') |
| 5298 |
deltas_down_time-=time.time() |
| 5299 |
status, tempname, l, complete = download_10k_uri(uri,abs_delta_name) |
| 5300 |
deltas_down_time+=time.time() |
| 5301 |
|
| 5302 |
#some strange error in remote server? |
| 5303 |
#FIXME this does not support ftp delta repositories |
| 5304 |
if status != 200 and status != 206 and status != 404: |
| 5305 |
logger.warning('Delta is not downloadable (%s):%s', http_responses_get(status), uri) |
| 5306 |
continue |
| 5307 |
|
| 5308 |
if status == 404: |
| 5309 |
not_available_deltas.append(p.name) |
| 5310 |
# check if delta is too big |
| 5311 |
if uri.startswith('http') and not proxies and 200 == test_uri(uri+'-too-big'): # FIXME support ftp or proxies |
| 5312 |
logger.info(_('Delta is too big:')+' '+delta_name) |
| 5313 |
if 'b' in DEB_POLICY: |
| 5314 |
no_delta.append( (deb_uri, newdeb) ) |
| 5315 |
elif VERBOSE > 1 : logger.debug(' No deb-policy "b", no download of %r',deb_uri) |
| 5316 |
continue |
| 5317 |
# check if delta is queued in the server but not yet done |
| 5318 |
if uri.startswith('http') and not proxies and 200 == test_uri(uri+'-queued'): # FIXME support ftp or proxies |
| 5319 |
logger.info(_('Delta is not yet ready in the server:')+' '+delta_name) |
| 5320 |
if 'q' in DEB_POLICY: |
| 5321 |
no_delta.append( (deb_uri, newdeb) ) |
| 5322 |
elif VERBOSE > 1 : logger.debug(' No deb-policy "q", no download of %r',deb_uri) |
| 5323 |
continue |
| 5324 |
#check if delta failed upstream |
| 5325 |
if uri.startswith('http') and not proxies and 200 == test_uri(uri+'-fails'): # FIXME support ftp or proxies |
| 5326 |
logger.info(_('Delta missing, server failed to create it:')+' '+delta_name) |
| 5327 |
if 'f' in DEB_POLICY: |
| 5328 |
no_delta.append( (deb_uri, newdeb) ) |
| 5329 |
elif VERBOSE > 1 : logger.debug(' No deb-policy "f", no download of %r',deb_uri) |
| 5330 |
continue |
| 5331 |
### FIXME the server is not generating these stamps ! |
| 5332 |
#if uri.startswith('http') and not proxies and newsize <= 2 * MIN_DEB_SIZE : # check only on small packages |
| 5333 |
# smallstatus = test_uri(uri+'-smalldeb') |
| 5334 |
# if smallstatus == 200: ETC ETC |
| 5335 |
### |
| 5336 |
# packages smaller than MIN_DEB_SIZE are ignored by the server. |
| 5337 |
if newsize <= MIN_DEB_SIZE : # check only on small packages |
| 5338 |
logger.info(_('Delta was not created since new package is too small:')+' '+delta_name) |
| 5339 |
if 't' in DEB_POLICY: |
| 5340 |
no_delta.append( (deb_uri, newdeb) ) |
| 5341 |
elif VERBOSE > 1 : logger.debug(' No deb-policy "t", no download of %r',deb_uri) |
| 5342 |
continue |
| 5343 |
if DEBUG and VERBOSE: |
| 5344 |
logger.info(_('Delta is not present:')+' '+uri) |
| 5345 |
else: |
| 5346 |
logger.info(_('Delta is not present:')+' '+delta_name) |
| 5347 |
if 'u' in DEB_POLICY: |
| 5348 |
no_delta.append( (deb_uri, newdeb) ) |
| 5349 |
elif VERBOSE > 1 : logger.debug(' No deb-policy "u", no download of %r',deb_uri) |
| 5350 |
continue |
| 5351 |
|
| 5352 |
if VERBOSE > 1: |
| 5353 |
logger.debug('Delta is present: %r %r', delta_name, tempname) |
| 5354 |
elif DO_PROGRESS: |
| 5355 |
sys.stderr.write(('%2.1f%%' % (float(progress_count) * 100.0 / len(cache)) + \ |
| 5356 |
_('Downloaded head of %s.') % p.name).ljust(terminalcolumns) + '\r') |
| 5357 |
|
| 5358 |
if os.path.isfile(tempname): |
| 5359 |
deltas_down_size+=os.path.getsize(tempname) |
| 5360 |
|
| 5361 |
#parse file and save info |
| 5362 |
try: |
| 5363 |
s=get_info_fast(tempname) |
| 5364 |
except DebDeltaError: |
| 5365 |
e=sys.exc_info()[1] |
| 5366 |
logger.error("!!"+str(e)+'\n') |
| 5367 |
logger.error("!! (renamed to "+tempname+'~~NOT~A~DELTA~~ )\n') |
| 5368 |
os.rename(tempname,tempname+'~~NOT~A~DELTA~~') |
| 5369 |
if proxies: |
| 5370 |
logger.error("!!maybe a proxy is returning an error page??\n") |
| 5371 |
else: |
| 5372 |
logger.error("!!damaged delta??\n") |
| 5373 |
continue |
| 5374 |
if s: |
| 5375 |
params_of_delta[p.name]=info_2_db(s) |
| 5376 |
s=patch_check_tmp_space(params_of_delta[p.name], '/') |
| 5377 |
if s != True: |
| 5378 |
logger.warning('%r : sorry %r', p.name, s) |
| 5379 |
#neither download deb nor delta.. |
| 5380 |
#the user may wish to free space and retry |
| 5381 |
continue |
| 5382 |
#FIXME may check that parameters are conformant to what we expect |
| 5383 |
|
| 5384 |
if complete: |
| 5385 |
patching_queue.put( (p.name, abs_delta_name ,newdeb, deb_uri ) ) |
| 5386 |
else: |
| 5387 |
available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, tempname, complete) ) |
| 5388 |
## end of first merry-go-round |
| 5389 |
|
| 5390 |
available_deltas.sort() |
| 5391 |
|
| 5392 |
if DEBUG or VERBOSE: |
| 5393 |
if DO_PROGRESS: |
| 5394 |
sys.stderr.write(' ' * (terminalcolumns) +'\r') |
| 5395 |
logger.info(' '+ _('Deltas: %(present)d present and %(absent)d not,') % \ |
| 5396 |
{'present' : len(available_deltas), 'absent' : len(not_available_deltas)}) |
| 5397 |
logger.info(' '+ _('downloaded so far: time %(time).2fsec, size %(size)s, speed %(speed)4s/sec.') % \ |
| 5398 |
{'time' : deltas_down_time, 'size' : SizeToKibiStr(deltas_down_size), 'speed' : SizeToKibiStr(deltas_down_size/float(deltas_down_time+0.001))}) |
| 5399 |
if available_deltas: |
| 5400 |
logger.info(' '+_('Need to get %s of deltas.') , SizeToKibiStr(sum([a[0] for a in available_deltas]))) |
| 5401 |
|
| 5402 |
###################### check available space |
| 5403 |
a=freespace('/var/cache/apt/archives') / 1024 |
| 5404 |
b=sum([int(s.get('NEW/Installed-Size','0')) for s in params_of_delta.values()]) |
| 5405 |
c=sum([int(s.get('NEW/Size','0')) for s in params_of_delta.values()]) / 1024 |
| 5406 |
if DEB_FORMAT == 'deb' and a < c: |
| 5407 |
logger.warning('**'+_('Very low disk space, need %(need)d kB have %(have)d kB'), |
| 5408 |
{'need':c,'have':a}) |
| 5409 |
if DEB_FORMAT == 'unzipped' and a < b: |
| 5410 |
logger.warning('**'+_('Very low disk space, need %(need)d kB have %(have)d kB'), |
| 5411 |
{'need':b,'have':a}) |
| 5412 |
|
| 5413 |
###################### start progress thread |
| 5414 |
|
| 5415 |
def print_progress(common_db): |
| 5416 |
while sys and 'STOP' not in common_db: |
| 5417 |
sys.stderr.write(progress_string(common_db)+'\r') |
| 5418 |
time.sleep(0.2) |
| 5419 |
|
| 5420 |
if DO_PROGRESS and terminalcolumns > 4: |
| 5421 |
progress_thread=threading.Thread(target=print_progress, args=( thread_returns, ) ) |
| 5422 |
progress_thread.daemon=True |
| 5423 |
progress_thread.start() |
| 5424 |
else: |
| 5425 |
progress_thread=None |
| 5426 |
|
| 5427 |
################ second merry-go-round, download rest of available deltas , queue them |
| 5428 |
for delta_len, name, uri, abs_delta_name , newdeb, deb_uri, tempname, complete in available_deltas : |
| 5429 |
## this seems to create problems.... |
| 5430 |
#if not os.path.exists(abs_delta_name) and os.path.exists(tempname) and os.path.getsize(tempname) == delta_len: |
| 5431 |
# print 'Just rename:',name #this actually should never happen, but , who knows... |
| 5432 |
# os.rename(tempname,abs_delta_name) |
| 5433 |
# tempname=abs_delta_name |
| 5434 |
|
| 5435 |
if name in params_of_delta: |
| 5436 |
s=patch_check_tmp_space(params_of_delta[name], '/') |
| 5437 |
if s != True: |
| 5438 |
logger.warning('%r : sorry, %r', name, s) |
| 5439 |
#argh, we ran out of space in meantime |
| 5440 |
continue |
| 5441 |
|
| 5442 |
if not os.path.exists(abs_delta_name): |
| 5443 |
thread_returns['downloaduri']=os.path.basename(uri) |
| 5444 |
r=download_uri(uri, abs_delta_name, deltas_down_time, deltas_down_size, thread_returns) |
| 5445 |
del thread_returns['downloaduri'] |
| 5446 |
if r == None or isinstance(r, HTTPException) : |
| 5447 |
if VERBOSE : logger.info(' '+_('You may wish to rerun, to get also:')+' '+uri) |
| 5448 |
continue |
| 5449 |
else: |
| 5450 |
deltas_down_time = r[0] |
| 5451 |
deltas_down_size = r[1] |
| 5452 |
|
| 5453 |
#queue to apply delta |
| 5454 |
if os.path.exists(abs_delta_name): |
| 5455 |
#append to queue |
| 5456 |
patching_queue.put( (name, abs_delta_name ,newdeb, deb_uri ) ) |
| 5457 |
else: |
| 5458 |
if 'u' in DEB_POLICY: |
| 5459 |
no_delta.append( (deb_uri, newdeb) ) |
| 5460 |
elif VERBOSE > 1 : logger.debug(' No deb-policy "u", no download of %r',deb_uri) |
| 5461 |
## end of second merry-go-round |
| 5462 |
|
| 5463 |
#terminate queue |
| 5464 |
patching_queue.put(None) |
| 5465 |
|
| 5466 |
#do something useful in the meantime |
| 5467 |
debs_down_size=0 |
| 5468 |
debs_down_time=0 |
| 5469 |
if patching_thread.is_alive() and no_delta and VERBOSE > 1 : |
| 5470 |
logger.info(' Downloading deltas done, downloading debs while waiting for patching thread.') |
| 5471 |
while patching_thread.is_alive() or ('a' in DEB_POLICY and no_delta): |
| 5472 |
if no_delta: |
| 5473 |
uri, newdeb = no_delta.pop() |
| 5474 |
thread_returns['downloaduri']=os.path.basename(uri) |
| 5475 |
r=download_uri(uri, newdeb, debs_down_time, debs_down_size, thread_returns) |
| 5476 |
del thread_returns['downloaduri'] |
| 5477 |
if r == None or isinstance(r, HTTPException) : |
| 5478 |
if VERBOSE : logger.debug(' You may wish to rerun, to get also: %r',uri) |
| 5479 |
continue |
| 5480 |
if r: |
| 5481 |
debs_down_time = r[0] |
| 5482 |
debs_down_size = r[1] |
| 5483 |
if not no_delta : |
| 5484 |
time.sleep(0.1) |
| 5485 |
|
| 5486 |
for i in http_conns: |
| 5487 |
if http_conns[i] != None : |
| 5488 |
http_conns[i].close() |
| 5489 |
|
| 5490 |
while patching_thread.is_alive(): |
| 5491 |
time.sleep(0.1) |
| 5492 |
|
| 5493 |
#terminate progress report |
| 5494 |
thread_returns['STOP']=True |
| 5495 |
while progress_thread != None and progress_thread.is_alive(): |
| 5496 |
time.sleep(0.1) |
| 5497 |
|
| 5498 |
if DO_PROGRESS: |
| 5499 |
sys.stderr.write(' ' * terminalcolumns +'\r') |
| 5500 |
|
| 5501 |
total_time += time.time() |
| 5502 |
logger.info(_('Delta-upgrade statistics:')) |
| 5503 |
msg=_("size %(size)s time %(time)dsec speed %(speed)s/sec") |
| 5504 |
if VERBOSE: |
| 5505 |
if deltas_down_time : |
| 5506 |
a=float(deltas_down_size) |
| 5507 |
t=deltas_down_time |
| 5508 |
logger.info(' ' + _('downloaded deltas, ') + msg % \ |
| 5509 |
{'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )}) |
| 5510 |
if thread_returns['debs_time'] : |
| 5511 |
a=float(thread_returns['debs_size']) |
| 5512 |
t=thread_returns['debs_time'] |
| 5513 |
logger.info(' ' + _('patching to debs, ') + msg % \ |
| 5514 |
{'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )}) |
| 5515 |
if debs_down_time : |
| 5516 |
a=float(debs_down_size) |
| 5517 |
t=debs_down_time |
| 5518 |
logger.info(' ' + _('downloaded debs, ') + msg % \ |
| 5519 |
{'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )}) |
| 5520 |
if total_time: |
| 5521 |
a=float(debs_down_size + thread_returns['debs_size']) |
| 5522 |
t=total_time |
| 5523 |
logger.info(' ' + _('total resulting debs, size %(size)s time %(time)dsec virtual speed %(speed)s/sec') % \ |
| 5524 |
{'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )}) |
| 5525 |
|
| 5526 |
if forensics: |
| 5527 |
forensic_send(forensics) |
| 5528 |
return max(mainexitcodes) |
| 5529 |
|
| 5530 |
################################################# main program, do stuff |
| 5531 |
|
| 5532 |
def act(): |
| 5533 |
"fake function that marks where the action starts" |
| 5534 |
pass |
| 5535 |
|
| 5536 |
if action == 'patch': |
| 5537 |
if INFO : |
| 5538 |
if len(argv) > 1 and VERBOSE : |
| 5539 |
sys.stderr.write(' (printing info - extra arguments are ignored)\n') |
| 5540 |
elif len(argv) == 0 : |
| 5541 |
sys.stderr.write(_('Need a filename; try --help.')+'\n') |
| 5542 |
raise SystemExit(3) |
| 5543 |
try: |
| 5544 |
delta=abspath(argv[0]) |
| 5545 |
check_is_delta(delta) |
| 5546 |
info=get_info(delta) |
| 5547 |
for s in info: |
| 5548 |
if s: |
| 5549 |
logger.debug(' info: %r',s) |
| 5550 |
except KeyboardInterrupt: |
| 5551 |
puke('debpatch exited by keyboard interrupt') |
| 5552 |
raise SystemExit(5) |
| 5553 |
except DebDeltaError: |
| 5554 |
s=sys.exc_info()[1] |
| 5555 |
puke('debpatch',s) |
| 5556 |
raise SystemExit(s.exitcode) |
| 5557 |
except Exception: |
| 5558 |
s=sys.exc_info()[1] |
| 5559 |
puke("debpatch",s) |
| 5560 |
raise SystemExit(4) |
| 5561 |
raise SystemExit(0) |
| 5562 |
#really patch |
| 5563 |
if len(argv) != 3 : |
| 5564 |
sys.stderr.write(_('Need 3 filenames; try --help.')+'\n') |
| 5565 |
raise SystemExit(3) |
| 5566 |
|
| 5567 |
newdeb=abspath(argv[2]) |
| 5568 |
if newdeb == '/dev/null': |
| 5569 |
newdeb = None |
| 5570 |
|
| 5571 |
try: |
| 5572 |
do_patch(abspath(argv[0]), abspath(argv[1]), newdeb) |
| 5573 |
except KeyboardInterrupt: |
| 5574 |
puke('debpatch exited by keyboard interrupt') |
| 5575 |
raise SystemExit(5) |
| 5576 |
except DebDeltaError: |
| 5577 |
s=sys.exc_info()[1] |
| 5578 |
puke('debpatch',s) |
| 5579 |
if s.logs: |
| 5580 |
forensic_send([s.logs]) |
| 5581 |
raise SystemExit(s.exitcode) |
| 5582 |
except Exception: |
| 5583 |
s=sys.exc_info()[1] |
| 5584 |
puke('debpatch',s) |
| 5585 |
raise SystemExit(4) |
| 5586 |
raise SystemExit(0) |
| 5587 |
|
| 5588 |
elif action == 'delta' : |
| 5589 |
if len(argv) != 3 : |
| 5590 |
sys.stderr.write(_('Need 3 filenames; try --help.')+'\n') |
| 5591 |
raise SystemExit(3) |
| 5592 |
|
| 5593 |
delta=abspath(argv[2]) |
| 5594 |
try: |
| 5595 |
r = do_delta(abspath(argv[0]), abspath(argv[1]), delta) |
| 5596 |
except KeyboardInterrupt: |
| 5597 |
puke('debdelta exited by keyboard interrupt') |
| 5598 |
raise SystemExit(5) |
| 5599 |
except DebDeltaError: |
| 5600 |
s=sys.exc_info()[1] |
| 5601 |
puke('debdelta',s) |
| 5602 |
raise SystemExit(s.exitcode) |
| 5603 |
except Exception: |
| 5604 |
s=sys.exc_info()[1] |
| 5605 |
puke('debdelta',s) |
| 5606 |
raise SystemExit(4) |
| 5607 |
raise SystemExit(0) |
| 5608 |
|
| 5609 |
elif action == 'deltas' : |
| 5610 |
for v in argv: |
| 5611 |
if not (os.path.isfile(v) or os.path.isdir(v)) : |
| 5612 |
sys.stderr.write(_("Error: argument is not a directory or a regular file:")+' '+v) |
| 5613 |
raise SystemExit(3) |
| 5614 |
try: |
| 5615 |
exitcode=do_deltas(argv) |
| 5616 |
except KeyboardInterrupt: |
| 5617 |
puke('debdeltas exited by keyboard interrupt') |
| 5618 |
raise SystemExit(5) |
| 5619 |
except DebDeltaError: |
| 5620 |
s=sys.exc_info()[1] |
| 5621 |
puke('debdeltas',s) |
| 5622 |
raise SystemExit(s.exitcode) |
| 5623 |
except Exception: |
| 5624 |
s=sys.exc_info()[1] |
| 5625 |
puke('debdeltas',s) |
| 5626 |
raise SystemExit(4) |
| 5627 |
raise SystemExit(exitcode) |
| 5628 |
|
| 5629 |
elif action == 'delta-upgrade': |
| 5630 |
import warnings |
| 5631 |
warnings.simplefilter("ignore",FutureWarning) |
| 5632 |
try: |
| 5633 |
exitcode=delta_upgrade_(argv) |
| 5634 |
except KeyboardInterrupt: |
| 5635 |
puke('debdelta-upgrade exited due to keyboard interrupt') |
| 5636 |
raise SystemExit(5) |
| 5637 |
except DebDeltaError: |
| 5638 |
s=sys.exc_info()[1] |
| 5639 |
puke('debdelta-upgrade',s) |
| 5640 |
raise SystemExit(s.exitcode) |
| 5641 |
except Exception: |
| 5642 |
s=sys.exc_info()[1] |
| 5643 |
puke('debdelta-upgrade',s) |
| 5644 |
raise SystemExit(4) |
| 5645 |
raise SystemExit(exitcode) |
| 5646 |
|
| 5647 |
elif action == 'patch-url': |
| 5648 |
config=ConfigParser() |
| 5649 |
config.read(['/etc/debdelta/sources.conf', expanduser('~/.debdelta/sources.conf') ]) |
| 5650 |
|
| 5651 |
try: |
| 5652 |
import apt_pkg |
| 5653 |
except ImportError: |
| 5654 |
logger.error('ERROR!!! python module "apt_pkg" is missing. Please install python-apt') |
| 5655 |
raise SystemExit(1) |
| 5656 |
|
| 5657 |
try: |
| 5658 |
import apt |
| 5659 |
except ImportError: |
| 5660 |
logger.error('ERROR!!! python module "apt" is missing. Please install a newer version of python-apt (newer than 0.6.12)') |
| 5661 |
raise SystemExit(1) |
| 5662 |
|
| 5663 |
apt_pkg.init() |
| 5664 |
|
| 5665 |
cache=apt.Cache() |
| 5666 |
cache.upgrade(True) |
| 5667 |
|
| 5668 |
for a in argv: |
| 5669 |
logger.info(_('Lookup %s') % a) |
| 5670 |
p = cache[a] |
| 5671 |
candidate=p.candidate |
| 5672 |
origin = p.candidate.origins[0] |
| 5673 |
arch=candidate.architecture |
| 5674 |
if not candidate.uris : |
| 5675 |
logger.warning(_("Sorry, cannot find an URI to download the debian package of `%s'.") % a) |
| 5676 |
continue |
| 5677 |
deb_uri = candidate.uri |
| 5678 |
installed_version=p.installed.version |
| 5679 |
candidate_version=p.candidate.version |
| 5680 |
deb_path=str.split(deb_uri,'/') |
| 5681 |
deb_path=str.join('/',deb_path[(deb_path.index('pool')):]) |
| 5682 |
|
| 5683 |
delta_uri_base=delta_uri_from_config(config, |
| 5684 |
Origin=origin.origin, |
| 5685 |
Label=origin.label, |
| 5686 |
Site=origin.site, |
| 5687 |
Archive=origin.archive, |
| 5688 |
PackageName=p.name) |
| 5689 |
|
| 5690 |
if delta_uri_base == None: |
| 5691 |
logger.warning(_("Sorry, no source is available to upgrade `%s'.") % a) |
| 5692 |
continue |
| 5693 |
|
| 5694 |
if installed_version == candidate_version: |
| 5695 |
logger.warning(_("Sorry, the package `%s' is already at its newest version.") % a) |
| 5696 |
continue |
| 5697 |
|
| 5698 |
#delta name |
| 5699 |
delta_name=p.shortname+'_'+version_mangle(installed_version)+\ |
| 5700 |
'_'+ version_mangle(candidate_version)+'_'+\ |
| 5701 |
arch+'.debdelta' |
| 5702 |
|
| 5703 |
delta_name = delta_base_name(p.shortname, installed_version, |
| 5704 |
candidate_version, arch) |
| 5705 |
|
| 5706 |
uri=delta_uri_base+'/'+os.path.dirname(deb_path)+'/'+delta_name |
| 5707 |
|
| 5708 |
logger.info(a+' : '+uri) |
| 5709 |
raise SystemExit(0) |
| 5710 |
|
| 5711 |
##################################################### apt method |
| 5712 |
|
| 5713 |
### still work in progress |
| 5714 |
if os.path.dirname(sys.argv[0]) == '/usr/lib/apt/methods' : |
| 5715 |
import os,sys, select, fcntl, apt, thread, threading, time |
| 5716 |
|
| 5717 |
apt_cache=apt.Cache() |
| 5718 |
|
| 5719 |
log=open('/tmp/log','a') |
| 5720 |
log.write(' --- here we go\n') |
| 5721 |
|
| 5722 |
( hi, ho , he) = os.popen3('/usr/lib/apt/methods/http.distrib','b',2) |
| 5723 |
|
| 5724 |
nthreads=3 |
| 5725 |
|
| 5726 |
class cheat_apt_gen: |
| 5727 |
def __init__(self): |
| 5728 |
self.uri=None |
| 5729 |
self.filename=None |
| 5730 |
self.acquire=False |
| 5731 |
def process(self,cmd): |
| 5732 |
if self.uri: |
| 5733 |
self.filename=cmd[10:-1] |
| 5734 |
log.write(' download %s for %s\n' % (repr(self.uri),repr(self.filename))) |
| 5735 |
self.uri=None |
| 5736 |
self.filename=None |
| 5737 |
self.acquire=False |
| 5738 |
return cmd |
| 5739 |
elif self.acquire: |
| 5740 |
self.uri=cmd[5:-1] |
| 5741 |
return cmd |
| 5742 |
elif cmd[:3] == '600' : |
| 5743 |
self.acquire=True |
| 5744 |
else: |
| 5745 |
return cmd |
| 5746 |
|
| 5747 |
def copyin(): |
| 5748 |
bufin='' |
| 5749 |
while 1: |
| 5750 |
#print ' o' |
| 5751 |
s=os.read(ho.fileno(),1) |
| 5752 |
bufin += s |
| 5753 |
if log and bufin and (s == '' or s == '\n') : |
| 5754 |
log.write( ' meth ' +repr(bufin)+'\n' ) |
| 5755 |
bufin='' |
| 5756 |
if s == '': |
| 5757 |
thread.interrupt_main( ) |
| 5758 |
global nthreads |
| 5759 |
if nthreads: |
| 5760 |
nthreads-=1 |
| 5761 |
#log.write( ' in closed \n' ) |
| 5762 |
#return |
| 5763 |
os.write(1,s) |
| 5764 |
|
| 5765 |
|
| 5766 |
def copyerr(): |
| 5767 |
buferr='' |
| 5768 |
while 1: |
| 5769 |
s=os.read(he.fileno(),1) |
| 5770 |
buferr += s |
| 5771 |
if log and buferr and (s == '' or s == '\n') : |
| 5772 |
log.write( ' err ' +repr(buferr)+'\n' ) |
| 5773 |
buferr='' |
| 5774 |
if s == '': |
| 5775 |
thread.interrupt_main( ) |
| 5776 |
global nthreads |
| 5777 |
if nthreads: |
| 5778 |
nthreads-=1 |
| 5779 |
log.write( ' err closed \n' ) |
| 5780 |
#return |
| 5781 |
os.write(2,s) |
| 5782 |
|
| 5783 |
def copyout(): |
| 5784 |
gen=cheat_apt_gen() |
| 5785 |
bufout='' |
| 5786 |
while 1: |
| 5787 |
s=os.read(0,1) |
| 5788 |
bufout += s |
| 5789 |
if log and bufout and (s == '' or s == '\n') : |
| 5790 |
log.write( ' apt ' +repr(bufout)+'\n' ) |
| 5791 |
|
| 5792 |
bufout=gen.process(bufout) |
| 5793 |
|
| 5794 |
bufout='' |
| 5795 |
if s == '': |
| 5796 |
thread.interrupt_main() |
| 5797 |
global nthreads |
| 5798 |
if nthreads: |
| 5799 |
nthreads-=1 |
| 5800 |
#log.write( ' out closed \n' ) |
| 5801 |
#return |
| 5802 |
os.write(hi.fileno(),(s)) |
| 5803 |
|
| 5804 |
|
| 5805 |
tin=thread.start_new_thread(copyin,()) |
| 5806 |
tout=thread.start_new_thread(copyout,()) |
| 5807 |
terr=thread.start_new_thread(copyerr,()) |
| 5808 |
while nthreads>0 : |
| 5809 |
log.write( ' nthreads %d \n' % nthreads ) |
| 5810 |
try: |
| 5811 |
while nthreads>0 : |
| 5812 |
time.sleep(1) |
| 5813 |
except KeyboardInterrupt: |
| 5814 |
pass |
| 5815 |
raise SystemExit(0) |
| 5816 |
|