| 1 |
# ex: set sts=4 ts=4 sw=4 noet: |
| 2 |
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## |
| 3 |
# |
| 4 |
# See COPYING file distributed along with the datalad package for the |
| 5 |
# copyright and license terms. |
| 6 |
# |
| 7 |
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## |
| 8 |
"""Helpers for benchmarks of DataLad""" |
| 9 |
|
| 10 |
import os |
| 11 |
import os.path as op |
| 12 |
import sys |
| 13 |
import tarfile |
| 14 |
import tempfile |
| 15 |
import timeit |
| 16 |
from glob import glob |
| 17 |
|
| 18 |
from datalad.api import ( |
| 19 |
Dataset, |
| 20 |
create_test_dataset, |
| 21 |
) |
| 22 |
from datalad.utils import ( |
| 23 |
get_tempfile_kwargs, |
| 24 |
getpwd, |
| 25 |
rmtree, |
| 26 |
) |
| 27 |
|
| 28 |
############ |
| 29 |
# Monkey patches |
| 30 |
|
| 31 |
# Robust is_interactive. Should be not needed since 0.11.4 |
| 32 |
# https://github.com/datalad/datalad/pull/3268 |
| 33 |
def _is_stream_tty(stream): |
| 34 |
try: |
| 35 |
# TODO: check on windows if hasattr check would work correctly and |
| 36 |
# add value: |
| 37 |
return stream.isatty() |
| 38 |
except ValueError as exc: |
| 39 |
# Who knows why it is a ValueError, but let's try to be specific |
| 40 |
# If there is a problem with I/O - non-interactive, otherwise reraise |
| 41 |
if "I/O" in str(exc): |
| 42 |
return False |
| 43 |
raise |
| 44 |
|
| 45 |
|
| 46 |
def is_interactive(): |
| 47 |
"""Return True if all in/outs are tty""" |
| 48 |
return all(_is_stream_tty(s) for s in (sys.stdin, sys.stdout, sys.stderr)) |
| 49 |
|
| 50 |
|
| 51 |
class SuprocBenchmarks(object): |
| 52 |
# manually set a number since otherwise takes way too long! |
| 53 |
# see https://github.com/spacetelescope/asv/issues/497 |
| 54 |
#number = 3 |
| 55 |
# although seems to work ok with a timer which accounts for subprocesses |
| 56 |
|
| 57 |
# custom timer so we account for subprocess times |
| 58 |
timer = timeit.default_timer |
| 59 |
|
| 60 |
_monkey_patched = False |
| 61 |
|
| 62 |
def __init__(self): |
| 63 |
if not self._monkey_patched: |
| 64 |
# monkey patch things if needed |
| 65 |
# ASV started to close one of the std streams since some point |
| 66 |
# which caused our is_interactive to fail. We need to provide |
| 67 |
# more robust version |
| 68 |
from datalad.support.external_versions import external_versions |
| 69 |
|
| 70 |
# comparing to 0.12.1 since the returned version is "loose" |
| 71 |
# so fails correctly identify rc as pre .0 |
| 72 |
if external_versions['datalad'] < '0.12.1': |
| 73 |
from datalad import utils |
| 74 |
from datalad.api import ls |
| 75 |
utils.is_interactive = is_interactive |
| 76 |
ls.is_interactive = is_interactive |
| 77 |
SuprocBenchmarks._monkey_patched = True |
| 78 |
self.remove_paths = [] |
| 79 |
|
| 80 |
def _cleanup(self): |
| 81 |
if not self.remove_paths: |
| 82 |
return # Nothing TODO |
| 83 |
self.log("Cleaning up %d paths", len(self.remove_paths)) |
| 84 |
while self.remove_paths: |
| 85 |
path = self.remove_paths.pop() |
| 86 |
if op.lexists(path): |
| 87 |
rmtree(path) |
| 88 |
|
| 89 |
def teardown(self): |
| 90 |
self._cleanup() |
| 91 |
|
| 92 |
def __del__(self): |
| 93 |
# We will at least try |
| 94 |
try: |
| 95 |
self._cleanup() |
| 96 |
except: |
| 97 |
pass |
| 98 |
|
| 99 |
def log(self, msg, *args): |
| 100 |
"""Consistent benchmarks logging""" |
| 101 |
print("BM: "+ str(msg % tuple(args))) |
| 102 |
|
| 103 |
|
| 104 |
class SampleSuperDatasetBenchmarks(SuprocBenchmarks): |
| 105 |
""" |
| 106 |
Setup a sample hierarchy of datasets to be used |
| 107 |
""" |
| 108 |
|
| 109 |
timeout = 3600 |
| 110 |
# need to assure that we are working in a different repository now |
| 111 |
# see https://github.com/datalad/datalad/issues/1512 |
| 112 |
# might not be sufficient due to side effects between tests and |
| 113 |
# thus getting into the same situation |
| 114 |
ds_count = 0 |
| 115 |
|
| 116 |
# Creating in CWD so things get removed when ASV is done |
| 117 |
# https://asv.readthedocs.io/en/stable/writing_benchmarks.html |
| 118 |
# that is where it would be run and cleaned up after |
| 119 |
|
| 120 |
dsname = 'testds1' |
| 121 |
tarfile = 'testds1.tar' |
| 122 |
|
| 123 |
def setup_cache(self): |
| 124 |
ds_path = create_test_dataset( |
| 125 |
self.dsname |
| 126 |
, spec='2/-2/-2' |
| 127 |
, seed=0 |
| 128 |
)[0] |
| 129 |
self.log("Setup cache ds path %s. CWD: %s", ds_path, getpwd()) |
| 130 |
# Will store into a tarfile since otherwise install -r is way too slow |
| 131 |
# to be invoked for every benchmark |
| 132 |
# Store full path since apparently setup is not ran in that directory |
| 133 |
self.tarfile = op.realpath(SampleSuperDatasetBenchmarks.tarfile) |
| 134 |
with tarfile.open(self.tarfile, "w") as tar: |
| 135 |
# F.CK -- Python tarfile can't later extract those because key dirs are |
| 136 |
# read-only. For now just a workaround - make it all writeable |
| 137 |
from datalad.utils import rotree |
| 138 |
rotree(self.dsname, ro=False, chmod_files=False) |
| 139 |
tar.add(self.dsname, recursive=True) |
| 140 |
rmtree(self.dsname) |
| 141 |
|
| 142 |
def setup(self): |
| 143 |
self.log("Setup ran in %s, existing paths: %s", getpwd(), glob('*')) |
| 144 |
|
| 145 |
tempdir = tempfile.mkdtemp( |
| 146 |
**get_tempfile_kwargs({}, prefix="bm") |
| 147 |
) |
| 148 |
self.remove_paths.append(tempdir) |
| 149 |
with tarfile.open(self.tarfile) as tar: |
| 150 |
# note: not a concern for CVE-2007-4559 since we are the ones mastering |
| 151 |
# content for the tar here. See https://github.com/datalad/datalad/pull/7104 |
| 152 |
# for more information. |
| 153 |
tar.extractall(tempdir) |
| 154 |
|
| 155 |
# TODO -- remove this abomination after https://github.com/datalad/datalad/issues/1512 is fixed |
| 156 |
epath = op.join(tempdir, 'testds1') |
| 157 |
epath_unique = epath + str(self.__class__.ds_count) |
| 158 |
os.rename(epath, epath_unique) |
| 159 |
self.__class__.ds_count += 1 |
| 160 |
self.ds = Dataset(epath_unique) |
| 161 |
self.repo = self.ds.repo |
| 162 |
self.log("Finished setup for %s", tempdir) |