| 1 |
#!/usr/bin/env python2 |
| 2 |
#emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- |
| 3 |
#ex: set sts=4 ts=4 sw=4 noet: |
| 4 |
""" |
| 5 |
Simple script to simulate an annex repository given a list of files. |
| 6 |
|
| 7 |
Not sure why I wrote it in Python, since in bash it would be more natural and shorter ;) |
| 8 |
|
| 9 |
COPYRIGHT: Yaroslav Halchenko 2014 |
| 10 |
|
| 11 |
LICENSE: MIT |
| 12 |
|
| 13 |
Permission is hereby granted, free of charge, to any person obtaining a copy |
| 14 |
of this software and associated documentation files (the "Software"), to deal |
| 15 |
in the Software without restriction, including without limitation the rights |
| 16 |
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 17 |
copies of the Software, and to permit persons to whom the Software is |
| 18 |
furnished to do so, subject to the following conditions: |
| 19 |
|
| 20 |
The above copyright notice and this permission notice shall be included in |
| 21 |
all copies or substantial portions of the Software. |
| 22 |
|
| 23 |
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 24 |
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 25 |
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 26 |
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 27 |
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 28 |
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
| 29 |
THE SOFTWARE. |
| 30 |
""" |
| 31 |
|
| 32 |
__author__ = 'Yaroslav Halchenko' |
| 33 |
__copyright__ = 'Copyright (c) 2014 Yaroslav Halchenko' |
| 34 |
__license__ = 'MIT' |
| 35 |
|
| 36 |
import fileinput |
| 37 |
import os |
| 38 |
import sys |
| 39 |
|
| 40 |
import commands |
| 41 |
|
| 42 |
i = 0 |
| 43 |
addurl = "http://some.example.com/prefix/" |
| 44 |
|
| 45 |
def run_command(cmd): |
| 46 |
global i |
| 47 |
st, output = commands.getstatusoutput(cmd) |
| 48 |
if st != 0: |
| 49 |
raise RuntimeError("E: run of {cmd} failed with status={st} output={output}".format(**locals())) |
| 50 |
return st, output |
| 51 |
|
| 52 |
def init_git_annex(path): |
| 53 |
if os.path.exists(path): |
| 54 |
raise RuntimeError("path {path} exists already".format(**locals())) |
| 55 |
run_command('mkdir -p {path}; cd {path}; git init; git annex init'.format(**locals())) |
| 56 |
print("I: initialized in {}".format(path)) |
| 57 |
|
| 58 |
def populate_git_annex(list_, path='.'): |
| 59 |
count = 0 |
| 60 |
i = 0 |
| 61 |
for l in fileinput.FileInput(list_, openhook=fileinput.hook_compressed): |
| 62 |
if not l: break |
| 63 |
i += 1 |
| 64 |
items = l.rstrip().split(None, 3) |
| 65 |
s3filename = items[-1] |
| 66 |
if s3filename.endswith('/'): |
| 67 |
continue |
| 68 |
if not s3filename.startswith('s3://'): |
| 69 |
print "ERROR: %i: %s is not starting with s3://" % (i, s3filename) |
| 70 |
import pdb; pdb.set_trace() |
| 71 |
# create a dummy file with content being just a filename |
| 72 |
filename_ = s3filename[5:] |
| 73 |
filename = os.path.join(path, filename_) |
| 74 |
dirname = os.path.dirname(filename) |
| 75 |
if not os.path.exists(dirname): |
| 76 |
os.makedirs(dirname) |
| 77 |
with open(filename, 'w') as fout: |
| 78 |
fout.write(filename) |
| 79 |
count += 1 |
| 80 |
if not (i % 100): |
| 81 |
sys.stdout.write('.') |
| 82 |
sys.stdout.flush()#print("D: ran {cmd}. Got output {output}".format(**locals())) |
| 83 |
# TODO |
| 84 |
if addurl: |
| 85 |
# we need to add a url for each file, thus first we just annex add |
| 86 |
run_command('cd {path}; git annex add {filename_}'.format(**locals())) |
| 87 |
# and then add a bogus url |
| 88 |
url = addurl + filename_ |
| 89 |
run_command('cd {path}; git annex addurl --relaxed --file={filename_} {url}'.format(**locals())) |
| 90 |
if not addurl: |
| 91 |
print() |
| 92 |
print("I: adding to annex {count} files after processing {i} lines".format(**locals())) |
| 93 |
run_command('cd {path}; git annex add *'.format(**locals())) |
| 94 |
print("I: committing") |
| 95 |
run_command('cd {path}; git commit -m "final commit"'.format(**locals())) |
| 96 |
print "DONE. Created {} files".format(count) |
| 97 |
|
| 98 |
def git_repack(path): |
| 99 |
print("Repacking {}".format(path)) |
| 100 |
run_command("cd {path}; git repack -a -d -f --window=100".format(path=path)) |
| 101 |
|
| 102 |
def du(path): |
| 103 |
st, output = commands.getstatusoutput('du -sk {path}'.format(path=path)) |
| 104 |
print "du for {path}: {output}".format(**locals()) |
| 105 |
|
| 106 |
if __name__ == '__main__': |
| 107 |
list_ = sys.argv[1] |
| 108 |
path = sys.argv[2] |
| 109 |
init_git_annex(path) |
| 110 |
populate_git_annex(list_, path) |
| 111 |
du(path) |
| 112 |
git_repack(path) |
| 113 |
du(path) |
| 114 |
|
| 115 |
# let's now clone |
| 116 |
run_command("git clone --no-hardlinks {path} {path}.cloned".format(path=path)) |
| 117 |
du(path+".cloned") |