Debian Codegraph @

benchmarks/common.py

raw
1 # ex: set sts=4 ts=4 sw=4 noet:
2 # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
3 #
4 # See COPYING file distributed along with the datalad package for the
5 # copyright and license terms.
6 #
7 # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
8 """Helpers for benchmarks of DataLad"""
9
10 import os
11 import os.path as op
12 import sys
13 import tarfile
14 import tempfile
15 import timeit
16 from glob import glob
17
18 from datalad.api import (
19 Dataset,
20 create_test_dataset,
21 )
22 from datalad.utils import (
23 get_tempfile_kwargs,
24 getpwd,
25 rmtree,
26 )
27
28 ############
29 # Monkey patches
30
31 # Robust is_interactive. Should be not needed since 0.11.4
32 # https://github.com/datalad/datalad/pull/3268
33 def _is_stream_tty(stream):
34 try:
35 # TODO: check on windows if hasattr check would work correctly and
36 # add value:
37 return stream.isatty()
38 except ValueError as exc:
39 # Who knows why it is a ValueError, but let's try to be specific
40 # If there is a problem with I/O - non-interactive, otherwise reraise
41 if "I/O" in str(exc):
42 return False
43 raise
44
45
46 def is_interactive():
47 """Return True if all in/outs are tty"""
48 return all(_is_stream_tty(s) for s in (sys.stdin, sys.stdout, sys.stderr))
49
50
51 class SuprocBenchmarks(object):
52 # manually set a number since otherwise takes way too long!
53 # see https://github.com/spacetelescope/asv/issues/497
54 #number = 3
55 # although seems to work ok with a timer which accounts for subprocesses
56
57 # custom timer so we account for subprocess times
58 timer = timeit.default_timer
59
60 _monkey_patched = False
61
62 def __init__(self):
63 if not self._monkey_patched:
64 # monkey patch things if needed
65 # ASV started to close one of the std streams since some point
66 # which caused our is_interactive to fail. We need to provide
67 # more robust version
68 from datalad.support.external_versions import external_versions
69
70 # comparing to 0.12.1 since the returned version is "loose"
71 # so fails correctly identify rc as pre .0
72 if external_versions['datalad'] < '0.12.1':
73 from datalad import utils
74 from datalad.api import ls
75 utils.is_interactive = is_interactive
76 ls.is_interactive = is_interactive
77 SuprocBenchmarks._monkey_patched = True
78 self.remove_paths = []
79
80 def _cleanup(self):
81 if not self.remove_paths:
82 return # Nothing TODO
83 self.log("Cleaning up %d paths", len(self.remove_paths))
84 while self.remove_paths:
85 path = self.remove_paths.pop()
86 if op.lexists(path):
87 rmtree(path)
88
89 def teardown(self):
90 self._cleanup()
91
92 def __del__(self):
93 # We will at least try
94 try:
95 self._cleanup()
96 except:
97 pass
98
99 def log(self, msg, *args):
100 """Consistent benchmarks logging"""
101 print("BM: "+ str(msg % tuple(args)))
102
103
104 class SampleSuperDatasetBenchmarks(SuprocBenchmarks):
105 """
106 Setup a sample hierarchy of datasets to be used
107 """
108
109 timeout = 3600
110 # need to assure that we are working in a different repository now
111 # see https://github.com/datalad/datalad/issues/1512
112 # might not be sufficient due to side effects between tests and
113 # thus getting into the same situation
114 ds_count = 0
115
116 # Creating in CWD so things get removed when ASV is done
117 # https://asv.readthedocs.io/en/stable/writing_benchmarks.html
118 # that is where it would be run and cleaned up after
119
120 dsname = 'testds1'
121 tarfile = 'testds1.tar'
122
123 def setup_cache(self):
124 ds_path = create_test_dataset(
125 self.dsname
126 , spec='2/-2/-2'
127 , seed=0
128 )[0]
129 self.log("Setup cache ds path %s. CWD: %s", ds_path, getpwd())
130 # Will store into a tarfile since otherwise install -r is way too slow
131 # to be invoked for every benchmark
132 # Store full path since apparently setup is not ran in that directory
133 self.tarfile = op.realpath(SampleSuperDatasetBenchmarks.tarfile)
134 with tarfile.open(self.tarfile, "w") as tar:
135 # F.CK -- Python tarfile can't later extract those because key dirs are
136 # read-only. For now just a workaround - make it all writeable
137 from datalad.utils import rotree
138 rotree(self.dsname, ro=False, chmod_files=False)
139 tar.add(self.dsname, recursive=True)
140 rmtree(self.dsname)
141
142 def setup(self):
143 self.log("Setup ran in %s, existing paths: %s", getpwd(), glob('*'))
144
145 tempdir = tempfile.mkdtemp(
146 **get_tempfile_kwargs({}, prefix="bm")
147 )
148 self.remove_paths.append(tempdir)
149 with tarfile.open(self.tarfile) as tar:
150 # note: not a concern for CVE-2007-4559 since we are the ones mastering
151 # content for the tar here. See https://github.com/datalad/datalad/pull/7104
152 # for more information.
153 tar.extractall(tempdir)
154
155 # TODO -- remove this abomination after https://github.com/datalad/datalad/issues/1512 is fixed
156 epath = op.join(tempdir, 'testds1')
157 epath_unique = epath + str(self.__class__.ds_count)
158 os.rename(epath, epath_unique)
159 self.__class__.ds_count += 1
160 self.ds = Dataset(epath_unique)
161 self.repo = self.ds.repo
162 self.log("Finished setup for %s", tempdir)