Debian Codegraph @

benchmarks/plugins/addurls.py

raw
1 # Import functions to be tested with _ suffix and name the suite after the
2 # original function so we could easily benchmark it e.g. by
3 # asv run --python=same -b get_parent_paths
4 # without need to discover what benchmark to use etc
5
6 import os
7 import tempfile
8 from pathlib import (
9 Path,
10 PurePosixPath,
11 )
12
13 import datalad.api as dl
14 from datalad import lgr
15 from datalad.utils import (
16 get_tempfile_kwargs,
17 rmtree,
18 )
19
20 from ..common import SuprocBenchmarks
21
22
23 class addurls1(SuprocBenchmarks):
24
25 # Try with excluding autometa and not
26 params = [None, '*']
27 param_names = ['exclude_metadata']
28
29
30 def setup(self, exclude_metadata):
31 self.nfiles = 20
32 self.temp = Path(
33 tempfile.mkdtemp(
34 **get_tempfile_kwargs({}, prefix='bm_addurls1')))
35
36 self.ds = dl.create(self.temp / "ds")
37 self.ds.config.set('annex.security.allowed-url-schemes', 'file', scope='local')
38
39 # populate list.csv and files
40 srcpath = PurePosixPath(self.temp)
41
42 rows = ["url,filename,bogus1,bogus2"]
43 for i in range(self.nfiles):
44 (self.temp / str(i)).write_text(str(i))
45 rows.append(
46 "file://{}/{},{},pocus,focus"
47 .format(srcpath, i, i)
48 )
49
50 self.listfile = self.temp / "list.csv"
51 self.listfile.write_text(os.linesep.join(rows))
52
53 def teardown(self, exclude_metadata):
54 # would make no sense if doesn't work correctly
55 # IIRC we cannot provide custom additional depends so cannot import nose
56 # assert_repo_status(self.ds.path)
57 status = self.ds.status()
58 assert all(r['state'] == 'clean' for r in status)
59 assert len(status) >= self.nfiles
60 rmtree(self.temp)
61
62 def time_addurls(self, exclude_autometa):
63 lgr.warning("CSV: " + self.listfile.read_text())
64 ret = dl.addurls(
65 str(self.listfile), '{url}', '{filename}',
66 dataset=self.ds,
67 exclude_autometa=exclude_autometa
68 )
69 assert not any(r['status'] == 'error' for r in ret)