Debian Codegraph @

private/online-build-po-corpus

raw
1 #!/usr/bin/env python3
2 # encoding=UTF-8
3
4 # Copyright © 2012-2025 Jakub Wilk <[email protected]>
5 #
6 # Permission is hereby granted, free of charge, to any person obtaining a copy
7 # of this software and associated documentation files (the “Software”), to deal
8 # in the Software without restriction, including without limitation the rights
9 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 # copies of the Software, and to permit persons to whom the Software is
11 # furnished to do so, subject to the following conditions:
12 #
13 # The above copyright notice and this permission notice shall be included in
14 # all copies or substantial portions of the Software.
15 #
16 # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 # SOFTWARE.
23
24 import argparse
25 import contextlib
26 import gzip
27 import os
28 import re
29 import subprocess
30 import sys
31 import tarfile
32 import tempfile
33 import urllib.parse
34
35 import apt
36 import apt_pkg
37
38 int(0_0) # Python >= 3.6 is required
39
40 def is_po_path(path):
41 return path.endswith(('.po', '.pot'))
42
43 class Fetcher:
44
45 def __init__(self, urlbase):
46 self._urlbase = urlbase
47 self._progress = apt.progress.text.AcquireProgress()
48 self._acquire = apt_pkg.Acquire(self._progress)
49 self._files = []
50 self._tmpdir = tempfile.TemporaryDirectory(prefix='i18nspector.private.') # pylint: disable=consider-using-with
51
52 def add(self, url, name, urlbase=None):
53 if urlbase is None:
54 urlbase = self._urlbase
55 url = urllib.parse.urljoin(urlbase, url)
56 self._files += [apt_pkg.AcquireFile(
57 self._acquire,
58 uri=url,
59 descr=url,
60 destfile=os.path.join(self._tmpdir.name, name)
61 )]
62
63 def run(self):
64 acq = self._acquire
65 rc = acq.run()
66 if rc != acq.RESULT_CONTINUE:
67 raise RuntimeError('fetching failed')
68 for acqfile in self._files:
69 if acqfile.status != acqfile.STAT_DONE:
70 raise RuntimeError('fetching failed')
71 yield acqfile.destfile
72
73 def __enter__(self):
74 return self
75
76 def __exit__(self, exc, value, tb):
77 self.close()
78
79 def close(self):
80 self._tmpdir.cleanup()
81
82 _package_name_re = re.compile(r'\A[a-z0-9][a-z0-9.+-]+\Z')
83 def validate_package_name(pkg):
84 if _package_name_re.match(pkg):
85 pass
86 else:
87 raise RuntimeError(f'invalid package name: {pkg!r}')
88
89 @contextlib.contextmanager
90 def chdir(new_cwd):
91 orig_cwd = os.getcwd()
92 try:
93 os.chdir(new_cwd)
94 yield
95 finally:
96 os.chdir(orig_cwd)
97
98 default_mirror = 'https://deb.debian.org/debian'
99 default_dist = 'stable'
100 default_areas = 'main'
101
102 def main():
103 ap = argparse.ArgumentParser()
104 ap.add_argument('--mirror', metavar='URL', default=default_mirror,
105 help=f'Debian mirror to use (default: {default_mirror})'
106 )
107 ap.add_argument('--contents-mirror', metavar='URL', default=None,
108 help='Debian mirror of Contents-* files to use (default: same as --mirror)'
109 )
110 ap.add_argument('--distribution', metavar='DIST', default=default_dist,
111 help=f'Debian distribution to use (default: {default_dist})'
112 )
113 ap.add_argument('--areas', metavar='AREA[,AREA...]', default=default_areas,
114 help=f'archive areas to use (default: {default_areas})'
115 )
116 ap.add_argument('--output', metavar='TARFILE', required=True,
117 help='output tar file'
118 )
119 options = ap.parse_args()
120 output = options.output
121 tarmode = 'w|'
122 for ext in 'gz', 'bz2', 'xz':
123 if options.output.endswith('.' + ext):
124 tarmode += ext
125 break
126 tar = tarfile.open(output, mode=tarmode) # pylint: disable=consider-using-with
127 mirror = options.mirror
128 cnts_mirror = options.contents_mirror
129 if cnts_mirror is None:
130 cnts_mirror = mirror
131 dist = options.distribution
132 areas = options.areas.split(',')
133 def tarfilter(tarinfo):
134 tarinfo.uid = tarinfo.gid = 0
135 tarinfo.uname = tarinfo.gname = 'root'
136 tarinfo.mode &= 0o700
137 tarinfo.mode |= 0o644
138 if tarinfo.mode & 0o100:
139 tarinfo.mode |= 0o111
140 prefix = f'po-corpus-{dist}/'
141 tarinfo.name = prefix + tarinfo.name
142 if tarinfo.linkname:
143 tarinfo.linkname = prefix + tarinfo.linkname
144 return tarinfo
145 subprocess.check_call(['dpkg-source', '--version'], stdout=subprocess.DEVNULL)
146 for area in areas:
147 urlbase = f'{mirror}/dists/{dist}/{area}/'
148 cnts_urlbase = f'{cnts_mirror}/dists/{dist}/{area}/'
149 with Fetcher(urlbase=urlbase) as fetcher:
150 fetcher.add('Contents-source.gz', 'Contents.gz', urlbase=cnts_urlbase)
151 fetcher.add('source/Sources.xz', 'Sources.xz')
152 [contents_path, sources_path] = fetcher.run()
153 interesting_packages = set()
154 with gzip.open(contents_path, 'rt', encoding='UTF-8', newline='\n') as file:
155 for line in file:
156 try:
157 path, packages = line.rsplit('\t', 1)
158 except ValueError:
159 raise RuntimeError(f'malformed line: {line!r}')
160 if is_po_path(path):
161 packages = packages.rstrip('\n').split(',')
162 for pkg in packages:
163 validate_package_name(pkg)
164 interesting_packages.update(packages)
165 for para in apt_pkg.TagFile(sources_path):
166 pkg = para['Package']
167 if pkg not in interesting_packages:
168 continue
169 fileinfo = para['Files'].splitlines()
170 dscname = None
171 names = set()
172 for line in fileinfo:
173 _, name = line.rsplit(' ', 1)
174 ok = (
175 name.startswith(pkg + '_') and
176 '/' not in name and
177 name not in names
178 )
179 if not ok:
180 raise RuntimeError(f'Package {pkg}: bad Files line: {line!r}')
181 names.add(name)
182 if name.endswith('.dsc'):
183 if dscname is not None:
184 raise RuntimeError(f'Package {pkg}: duplicate .dsc file')
185 dscname = name
186 if dscname is None:
187 raise RuntimeError(f'Package {pkg}: missing .dsc file')
188 names = list(names)
189 names.sort(key=dscname.__eq__)
190 pkgdir = para['Directory']
191 pkgurlbase = f'{mirror}/{pkgdir}/'
192 with Fetcher(urlbase=pkgurlbase) as pkgfetcher:
193 for name in names:
194 pkgfetcher.add(name, name)
195 for path in pkgfetcher.run():
196 pass
197 dscpath = path
198 unpacked = dscpath[:-4]
199 subprocess.check_call(['dpkg-source', '-x', dscpath, unpacked], stdout=sys.stderr)
200 with chdir(os.path.dirname(unpacked)):
201 for unp_root, unp_dirs, unp_files in os.walk(os.path.basename(unpacked)):
202 try:
203 unp_dirs.remove('.pc')
204 except ValueError:
205 pass
206 for unp_file in unp_files:
207 if is_po_path(unp_file):
208 unp_path = os.path.join(unp_root, unp_file)
209 tar.add(unp_path, filter=tarfilter)
210 tar.close()
211
212 if __name__ == '__main__':
213 main()
214
215 # vim:ts=4 sts=4 sw=4 et