"""
Note that attributes 'network' and 'local_irods' are ignored for CI.
To run 'local_irods' tests, you need an iRODS server on localhost (default port,
user 'rods', password 'rods') and a zone /tempZone/home/rods. See
UtilsForLocalIRODSTest.
"""
import json
import shutil
import os
import tempfile
import logging
import stat
import pytest
from unittest.mock import patch
from irods.session import iRODSSession
from biomaj_core.config import BiomajConfig
from biomaj_core.utils import Utils
from biomaj_download.download.interface import DownloadInterface
from biomaj_download.download.curl import CurlDownload, HTTPParse
from biomaj_download.download.direct import DirectFTPDownload, DirectHTTPDownload
from biomaj_download.download.localcopy import LocalDownload
from biomaj_download.download.rsync import RSYNCDownload
from biomaj_download.download.protocolirods import IRODSDownload
import unittest
import tenacity
class UtilsForTest():
"""
Copy properties files to a temp directory and update properties to
use a temp directory
"""
def __init__(self):
"""
Setup the temp dirs and files.
"""
self.global_properties = None
self.bank_properties = None
self.test_dir = tempfile.mkdtemp('biomaj')
self.conf_dir =os.path.join(self.test_dir,'conf')
if not os.path.exists(self.conf_dir):
os.makedirs(self.conf_dir)
self.data_dir =os.path.join(self.test_dir,'data')
if not os.path.exists(self.data_dir):
os.makedirs(self.data_dir)
self.log_dir =os.path.join(self.test_dir,'log')
if not os.path.exists(self.log_dir):
os.makedirs(self.log_dir)
self.process_dir =os.path.join(self.test_dir,'process')
if not os.path.exists(self.process_dir):
os.makedirs(self.process_dir)
self.lock_dir =os.path.join(self.test_dir,'lock')
if not os.path.exists(self.lock_dir):
os.makedirs(self.lock_dir)
self.cache_dir =os.path.join(self.test_dir,'cache')
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
if self.global_properties is None:
self.__copy_global_properties()
if self.bank_properties is None:
self.__copy_test_bank_properties()
# Create an invalid archive file (empty file). This is deleted by clean().
# See TestBiomajRSYNCDownload.test_rsync_download_skip_check_uncompress.
self.invalid_archive = os.path.join(self.test_dir, 'invalid.gz')
open(self.invalid_archive, 'w').close()
def clean(self):
"""
Deletes temp directory
"""
shutil.rmtree(self.test_dir)
def __copy_test_bank_properties(self):
if self.bank_properties is not None:
return
# Copy bank configuration (those bank use external resources so there is no tuning to do)
self.bank_properties = ['alu', 'testhttp', 'directhttp', 'multi']
curdir = os.path.dirname(os.path.realpath(__file__))
for b in self.bank_properties:
from_file = os.path.join(curdir, b+'.properties')
to_file = os.path.join(self.conf_dir, b+'.properties')
shutil.copyfile(from_file, to_file)
# Copy bank process
self.bank_process = ['test.sh']
curdir = os.path.dirname(os.path.realpath(__file__))
procdir = os.path.join(curdir, 'bank/process')
for proc in self.bank_process:
from_file = os.path.join(procdir, proc)
to_file = os.path.join(self.process_dir, proc)
shutil.copyfile(from_file, to_file)
os.chmod(to_file, stat.S_IRWXU)
# Copy and adapt bank configuration that use local resources: we use the "bank" dir in current test directory as remote
properties = ['local', 'localprocess', 'computed', 'computed2', 'sub1', 'sub2', 'computederror', 'error']
for prop in properties:
from_file = os.path.join(curdir, prop+'.properties')
to_file = os.path.join(self.conf_dir, prop+'.properties')
fout = open(to_file,'w')
with open(from_file,'r') as fin:
for line in fin:
if line.startswith('remote.dir'):
fout.write("remote.dir="+os.path.join(curdir,'bank')+"\n")
elif line.startswith('remote.files'):
fout.write(line.replace('/tmp', os.path.join(curdir,'bank')))
else:
fout.write(line)
fout.close()
def __copy_global_properties(self):
if self.global_properties is not None:
return
self.global_properties = os.path.join(self.conf_dir,'global.properties')
curdir = os.path.dirname(os.path.realpath(__file__))
global_template = os.path.join(curdir,'global.properties')
fout = open(self.global_properties,'w')
# Adapt directories in global configuration to the current test directory
with open(global_template,'r') as fin:
for line in fin:
if line.startswith('conf.dir'):
fout.write("conf.dir="+self.conf_dir+"\n")
elif line.startswith('log.dir'):
fout.write("log.dir="+self.log_dir+"\n")
elif line.startswith('data.dir'):
fout.write("data.dir="+self.data_dir+"\n")
elif line.startswith('process.dir'):
fout.write("process.dir="+self.process_dir+"\n")
elif line.startswith('lock.dir'):
fout.write("lock.dir="+self.lock_dir+"\n")
else:
fout.write(line)
fout.close()
class UtilsForLocalIRODSTest(UtilsForTest):
"""
This class is used to prepare 'local_irods' tests.
"""
SERVER = "localhost"
PORT = 1247
ZONE = "tempZone"
USER = "rods"
PASSWORD = "rods"
COLLECTION = os.path.join("/" + ZONE, "home/rods/") # Don't remove or add /
def __init__(self):
super(UtilsForLocalIRODSTest, self).__init__()
self._session = iRODSSession(host=self.SERVER, port=self.PORT,
user=self.USER, password=self.PASSWORD,
zone=self.ZONE)
self.curdir = os.path.dirname(os.path.realpath(__file__))
# Copy some valid archives (bank/test.fasta.gz)
file_ = os.path.join(self.curdir, "bank/test.fasta.gz")
self._session.data_objects.put(file_, self.COLLECTION)
# Copy invalid.gz
self._session.data_objects.put(self.invalid_archive, self.COLLECTION)
def clean(self):
super(UtilsForLocalIRODSTest, self).clean()
# Remove files on iRODS (use force otherwise the files are put in trash)
# Remove test.fasta.gz
self._session.data_objects.unlink(os.path.join(self.COLLECTION, "test.fasta.gz"), force=True)
# Remove invalid.gz
self._session.data_objects.unlink(os.path.join(self.COLLECTION, "invalid.gz"), force=True)
class TestDownloadInterface():
"""
Test of the interface.
"""
def test_retry_parsing(self):
"""
Test parsing of stop and wait conditions.
"""
downloader = DownloadInterface()
# Test some garbage
d = dict(stop_condition="stop_after_attempts") # no param
with pytest.raises(ValueError):
downloader.set_options(d)
d = dict(stop_condition="stop_after_attempts(5) & 1") # not a stop_condition
with pytest.raises(ValueError):
downloader.set_options(d)
# Test some garbage
d = dict(wait_policy="wait_random") # no param
with pytest.raises(ValueError):
downloader.set_options(d)
d = dict(wait_policy="I love python") # not a wait_condition
with pytest.raises(ValueError):
downloader.set_options(d)
#d = dict(wait_policy="wait_random(5) + 3") # not a wait_condition
#self.assertRaises(ValueError, downloader.set_options, d)
# Test operators
d = dict(stop_condition="stop_never | stop_after_attempt(5)",
wait_policy="wait_none + wait_random(1, 2)")
downloader.set_options(d)
# Test wait_combine, wait_chain
d = dict(wait_policy="wait_combine(wait_fixed(3), wait_random(1, 2))")
downloader.set_options(d)
d = dict(wait_policy="wait_chain(wait_fixed(3), wait_random(1, 2))")
downloader.set_options(d)
# Test stop_any and stop_all
stop_condition = "stop_any(stop_after_attempt(5), stop_after_delay(10))"
d = dict(stop_condition=stop_condition)
downloader.set_options(d)
stop_condition = "stop_all(stop_after_attempt(5), stop_after_delay(10))"
d = dict(stop_condition=stop_condition)
downloader.set_options(d)
class TestBiomajLocalDownload():
"""
Test Local downloader
"""
def setup_method(self, m):
self.utils = UtilsForTest()
self.curdir = os.path.dirname(os.path.realpath(__file__))
self.examples = os.path.join(self.curdir,'bank') + '/'
BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
def teardown_method(self, m):
self.utils.clean()
def test_local_list(self):
locald = LocalDownload(self.examples)
(file_list, dir_list) = locald.list()
locald.close()
assert (len(file_list) > 1)
def test_local_list_error(self):
locald = LocalDownload("/tmp/foo/")
# Check that we raise an exception and log a message
with pytest.raises(Exception):
(file_list, dir_list) = locald.list()
locald.close()
def test_local_download(self):
locald = LocalDownload(self.examples)
(file_list, dir_list) = locald.list()
locald.match([r'^test.*\.gz$'], file_list, dir_list)
locald.download(self.utils.data_dir)
locald.close()
assert (len(locald.files_to_download) == 1)
def test_local_download_in_subdir(self):
locald = LocalDownload(self.curdir+'/')
(file_list, dir_list) = locald.list()
locald.match([r'^/bank/test.*\.gz$'], file_list, dir_list)
locald.download(self.utils.data_dir)
locald.close()
assert (len(locald.files_to_download) == 1)
def test_local_download_hardlinks(self):
"""
Test download with hardlinks: we download a file from conf/ to data_dir.
This should work unless /tmp don't accept hardlinks so the last assert is
optional.
"""
test_file = "conf/global.properties"
locald = LocalDownload(self.utils.test_dir, use_hardlinks=True)
(file_list, dir_list) = locald.list()
locald.match([r'^/' + test_file + '$'], file_list, dir_list)
locald.download(self.utils.data_dir)
locald.close()
assert (len(locald.files_to_download) == 1)
# Test if data/conf/global.properties is a hard link to
# conf/global.properties
local_global_properties = os.path.join(self.utils.test_dir, test_file)
copy_global_properties = os.path.join(self.utils.data_dir, test_file)
try:
assert (
os.path.samefile(local_global_properties, copy_global_properties)
)
except Exception:
msg = "In %s: copy worked but hardlinks were not used." % self.id()
logging.info(msg)
@pytest.mark.skipif(
os.environ.get('NETWORK', 1) == '0',
reason='network tests disabled'
)
class TestBiomajHTTPDownload():
"""
Test HTTP downloader
"""
def setup_method(self, m):
self.utils = UtilsForTest()
BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
# Create an HTTPParse object used for most tests from the config file testhttp
self.config = BiomajConfig('testhttp')
self.http_parse = HTTPParse(
self.config.get('http.parse.dir.line'),
self.config.get('http.parse.file.line'),
int(self.config.get('http.group.dir.name')),
int(self.config.get('http.group.dir.date')),
int(self.config.get('http.group.file.name')),
int(self.config.get('http.group.file.date')),
self.config.get('http.group.file.date_format'),
int(self.config.get('http.group.file.size'))
)
def teardown_method(self, m):
self.utils.clean()
def test_http_list(self):
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/', self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.close()
assert (len(file_list) == 1)
def test_http_list_error(self):
"""
Test that errors in list are correctly caught.
"""
# Test access to non-existent directory
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/foo/', self.http_parse)
# Check that we raise an exception and log a message
with pytest.raises(Exception):
(file_list, dir_list) = httpd.list()
def test_http_list_dateregexp(self):
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/', self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.close()
assert (len(file_list) == 1)
def test_http_download_no_size(self):
# Create a custom http_parse without size
http_parse = HTTPParse(self.config.get('http.parse.dir.line'),
self.config.get('http.parse.file.line'),
int(self.config.get('http.group.dir.name')),
int(self.config.get('http.group.dir.date')),
int(self.config.get('http.group.file.name')),
int(self.config.get('http.group.file.date')),
self.config.get('http.group.file.date_format'),
-1
)
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/', http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^README$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
assert (len(httpd.files_to_download) == 1)
def test_http_download_no_date(self):
# Create a custom http_parse without date
http_parse = HTTPParse(self.config.get('http.parse.dir.line'),
self.config.get('http.parse.file.line'),
int(self.config.get('http.group.dir.name')),
int(self.config.get('http.group.dir.date')),
int(self.config.get('http.group.file.name')),
-1,
None,
int(self.config.get('http.group.file.size'))
)
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/', http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^README$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
assert (len(httpd.files_to_download) == 1)
def test_http_download(self):
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/', self.http_parse)
(file_list, dir_list) = httpd.list()
print(str(file_list))
httpd.match([r'^README$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
assert (len(httpd.files_to_download) == 1)
def test_http_download_in_subdir(self):
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/', self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^dists/README$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
assert (len(httpd.files_to_download) == 1)
def test_redirection(self):
"""
Test HTTP redirections
"""
# The site used in this test redirects to https (see #33).
http_parse = HTTPParse(
r'
[\s][\s][\w\-\.]+.tar.gz)<\/a>[\s]+([0-9]{2}-[A-Za-z]{3}-[0-9]{4}[\s][0-9]{2}:[0-9]{2})[\s]+([0-9]+[A-Za-z])",
"([\w\-\.]+.tar.gz)<\/a>[\s]+([0-9]{2}-[A-Za-z]{3}-[0-9]{4}[\s][0-9]{2}:[0-9]{2})[\s]+([0-9]+[A-Za-z])",
1,
2,
1,
2,
"%%d-%%b-%%Y %%H:%%M",
3
)
httpd = CurlDownload('https', 'mirrors.edge.kernel.org', '/pub/software/scm/git/debian/', http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^git-core-0.99.6.tar.gz$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
assert (len(httpd.files_to_download) == 1)
@pytest.mark.skipif(
os.environ.get('NETWORK', 1) == '0',
reason='network tests disabled'
)
class TestBiomajSFTPDownload():
"""
Test SFTP downloader
"""
PROTOCOL = "sftp"
def setup_method(self, m):
self.utils = UtilsForTest()
# Temporary host key file in test dir (so this is cleaned)
(_, self.khfile) = tempfile.mkstemp(dir=self.utils.test_dir)
def teardown_method(self, m):
self.utils.clean()
def test_list_error(self):
"""
Test that errors in list are correctly caught.
"""
# Test access to non-existent directory
sftpd = CurlDownload(self.PROTOCOL, "test.rebex.net", "/toto")
sftpd.set_credentials("demo:password")
# Check that we raise an exception and log a message
with pytest.raises(Exception):
(file_list, dir_list) = sftpd.list()
sftpd.close()
# Test with wrong password
sftpd = CurlDownload(self.PROTOCOL, "test.rebex.net", "/")
sftpd.set_credentials("demo:badpassword")
# Check that we raise an exception and log a message
with pytest.raises(Exception):
(file_list, dir_list) = sftpd.list()
sftpd.close()
def test_download(self):
sftpd = CurlDownload(self.PROTOCOL, "test.rebex.net", "/")
sftpd.set_credentials("demo:password")
sftpd.set_options({
"ssh_hosts_file": self.khfile,
"ssh_new_host": "add"
})
(file_list, dir_list) = sftpd.list()
sftpd.match([r'^readme.txt$'], file_list, dir_list)
sftpd.download(self.utils.data_dir)
sftpd.close()
assert (len(sftpd.files_to_download) == 1)
@pytest.mark.skipif(
os.environ.get('NETWORK', 1) == '0',
reason='network tests disabled'
)
class TestBiomajDirectFTPDownload():
"""
Test DirectFTP downloader
"""
def setup_method(self):
self.utils = UtilsForTest()
def teardown_method(self, m):
self.utils.clean()
def test_ftp_list(self):
file_list = ['/debian/doc/mailing-lists.txt']
ftpd = DirectFTPDownload('ftp', 'ftp.fr.debian.org', '')
ftpd.set_files_to_download(file_list)
(file_list, dir_list) = ftpd.list()
ftpd.close()
assert (len(file_list) == 1)
def test_ftp_list_error(self):
"""
Test that errors in list are correctly caught.
"""
# Test access to non-existent directory
file_list = ['/toto/debian/doc/mailing-lists.txt']
ftpd = DirectFTPDownload('ftp', 'ftp.fr.debian.org', '')
ftpd.set_files_to_download(file_list)
# Check that we raise an exception and log a message
with pytest.raises(Exception):
(file_list, dir_list) = ftpd.list()
ftpd.close()
def test_download(self):
file_list = ['/debian/doc/mailing-lists.txt']
ftpd = DirectFTPDownload('ftp', 'ftp.fr.debian.org', '')
ftpd.set_files_to_download(file_list)
(file_list, dir_list) = ftpd.list()
ftpd.download(self.utils.data_dir, False)
ftpd.close()
assert (os.path.exists(os.path.join(self.utils.data_dir,'mailing-lists.txt')))
@pytest.mark.skipif(
os.environ.get('NETWORK', 1) == '0',
reason='network tests disabled'
)
class TestBiomajDirectFTPSDownload():
"""
Test DirectFTP downloader with FTPS.
"""
def setup_method(self, m):
self.utils = UtilsForTest()
def teardown_method(self, m):
self.utils.clean()
def test_ftps_list(self):
file_list = ['/readme.txt']
ftpd = DirectFTPDownload('ftps', 'test.rebex.net', '')
ftpd.set_credentials('demo:password')
ftpd.set_files_to_download(file_list)
(file_list, dir_list) = ftpd.list()
ftpd.close()
assert (len(file_list) == 1)
def test_download(self):
file_list = ['/readme.txt']
ftpd = DirectFTPDownload('ftps', 'test.rebex.net', '')
ftpd.set_credentials('demo:password')
ftpd.set_files_to_download(file_list)
(file_list, dir_list) = ftpd.list()
ftpd.download(self.utils.data_dir, False)
ftpd.close()
assert (os.path.exists(os.path.join(self.utils.data_dir,'readme.txt')))
@pytest.mark.skipif(
os.environ.get('NETWORK', 1) == '0',
reason='network tests disabled'
)
class TestBiomajDirectHTTPDownload():
"""
Test DirectFTP downloader
"""
def setup_method(self, m):
self.utils = UtilsForTest()
def teardown_method(self, m):
self.utils.clean()
def test_http_list(self):
file_list = ['/debian/README.html']
ftpd = DirectHTTPDownload('http', 'ftp2.fr.debian.org', '')
ftpd.set_files_to_download(file_list)
fday = ftpd.files_to_download[0]['day']
fmonth = ftpd.files_to_download[0]['month']
fyear = ftpd.files_to_download[0]['year']
(file_list, dir_list) = ftpd.list()
ftpd.close()
assert (len(file_list) == 1)
assert (file_list[0]['size']!=0)
assert not (fyear == ftpd.files_to_download[0]['year'] and fmonth == ftpd.files_to_download[0]['month'] and fday == ftpd.files_to_download[0]['day'])
def test_http_list_error(self):
"""
Test that errors in list are correctly caught.
"""
# Test access to non-existent directory
file_list = ['/toto/debian/README.html']
ftpd = DirectHTTPDownload('http', 'ftp2.fr.debian.org', '')
ftpd.set_files_to_download(file_list)
# Check that we raise an exception and log a message
with pytest.raises(Exception):
(file_list, dir_list) = ftpd.list()
ftpd.close()
def test_download(self):
file_list = ['/debian/README.html']
ftpd = DirectHTTPDownload('http', 'ftp2.fr.debian.org', '')
ftpd.set_files_to_download(file_list)
(file_list, dir_list) = ftpd.list()
ftpd.download(self.utils.data_dir, False)
ftpd.close()
assert (os.path.exists(os.path.join(self.utils.data_dir,'README.html')))
def test_download_get_params_save_as(self):
file_list = ['/get']
ftpd = DirectHTTPDownload('http', 'httpbin.org', '')
ftpd.set_files_to_download(file_list)
ftpd.param = { 'key1': 'value1', 'key2': 'value2'}
ftpd.save_as = 'test.json'
(file_list, dir_list) = ftpd.list()
ftpd.download(self.utils.data_dir, False)
ftpd.close()
assert (os.path.exists(os.path.join(self.utils.data_dir,'test.json')))
with open(os.path.join(self.utils.data_dir,'test.json'), 'r') as content_file:
content = content_file.read()
my_json = json.loads(content)
assert (my_json['args']['key1'] == 'value1')
def test_download_save_as(self):
file_list = ['/debian/README.html']
ftpd = DirectHTTPDownload('http', 'ftp2.fr.debian.org', '')
ftpd.set_files_to_download(file_list)
ftpd.save_as = 'test.html'
(file_list, dir_list) = ftpd.list()
ftpd.download(self.utils.data_dir, False)
ftpd.close()
assert (os.path.exists(os.path.join(self.utils.data_dir,'test.html')))
def test_download_post_params(self):
#file_list = ['/debian/README.html']
file_list = ['/post']
ftpd = DirectHTTPDownload('http', 'httpbin.org', '')
ftpd.set_files_to_download(file_list)
ftpd.param = { 'key1': 'value1', 'key2': 'value2'}
ftpd.save_as = 'test.json'
ftpd.method = 'POST'
(file_list, dir_list) = ftpd.list()
ftpd.download(self.utils.data_dir, False)
ftpd.close()
assert (os.path.exists(os.path.join(self.utils.data_dir,'test.json')))
with open(os.path.join(self.utils.data_dir,'test.json'), 'r') as content_file:
content = content_file.read()
my_json = json.loads(content)
assert (my_json['form']['key1'] == 'value1')
def test_redirection(self):
"""
Test HTTP redirections
"""
# The site used in this test redirects to https (see #33).
# First test: allow redirections
httpd = DirectHTTPDownload('http', 'plasmodb.org', '/common/downloads/Current_Release/')
httpd.set_files_to_download(['Build_number'])
httpd.download(self.utils.data_dir)
httpd.download(self.utils.data_dir)
httpd.close()
assert (len(httpd.files_to_download) == 1)
# Second test: block redirections
httpd = DirectHTTPDownload('http', 'plasmodb.org', '/common/downloads/Current_Release/')
httpd.set_files_to_download(['Build_number'])
httpd.set_options({
"allow_redirections": False
})
with pytest.raises(Exception):
httpd.download(self.utils.data_dir)
httpd.close()
@pytest.mark.skipif(
os.environ.get('NETWORK', 1) == '0',
reason='network tests disabled'
)
@pytest.mark.skipif(
os.environ.get('FTP', 1) == '0',
reason='network tests disabled'
)
class TestBiomajFTPDownload():
"""
Test FTP downloader
"""
def setup_method(self, m):
self.utils = UtilsForTest()
def teardown_method(self, m):
self.utils.clean()
def test_ftp_list(self):
ftpd = CurlDownload('ftp', 'speedtest.tele2.net', '/')
(file_list, dir_list) = ftpd.list()
ftpd.close()
assert (len(file_list) > 1)
def test_ftp_list_error(self):
"""
Test that errors in list are correctly caught.
"""
# Test access to non-existent directory
ftpd = CurlDownload("ftp", "test.rebex.net", "/toto")
ftpd.set_credentials("demo:password")
# Check that we raise an exception and log a message
with pytest.raises(Exception):
(file_list, dir_list) = ftpd.list()
ftpd.close()
# Test with wrong password
ftpd = CurlDownload("ftp", "test.rebex.net", "/")
ftpd.set_credentials("demo:badpassword")
# Check that we raise an exception and log a message
with pytest.raises(Exception):
(file_list, dir_list) = ftpd.list()
ftpd.close()
def test_download(self):
ftpd = CurlDownload('ftp', 'speedtest.tele2.net', '/')
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^1.*KB\.zip$'], file_list, dir_list)
# This tests fails because the zip file is fake. We intercept the failure
# and continue.
# See test_download_skip_check_uncompress
try:
ftpd.download(self.utils.data_dir)
except Exception:
assert (1==1)
else:
# In case it works, this is the real assertion
assert (len(ftpd.files_to_download) == 2)
ftpd.close()
def test_download_skip_check_uncompress(self):
# This test is similar to test_download but we skip test of zip file.
ftpd = CurlDownload('ftp', 'speedtest.tele2.net', '/')
ftpd.set_options(dict(skip_check_uncompress=True))
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^1.*KB\.zip$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
assert (len(ftpd.files_to_download) == 2)
def test_download_in_subdir(self):
ftpd = CurlDownload('ftp', 'ftp.fr.debian.org', '/debian/')
(file_list, dir_list) = ftpd.list()
try:
ftpd.match([r'^doc/mailing-lists.txt$'], file_list, dir_list)
except Exception as e:
print("Error: " + str(e))
self.skipTest("Skipping test due to remote server error")
ftpd.download(self.utils.data_dir)
ftpd.close()
assert (len(ftpd.files_to_download) == 1)
def test_download_or_copy(self):
ftpd = CurlDownload('ftp', 'ftp.fr.debian.org', '/debian/')
ftpd.files_to_download = [
{'name':'/test1', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test2', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test/test1', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test/test11', 'year': '2013', 'month': '11', 'day': '10', 'size': 10}
]
available_files = [
{'name':'/test1', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test12', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test3', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test/test1', 'year': '2013', 'month': '11', 'day': '10', 'size': 20},
{'name':'/test/test11', 'year': '2013', 'month': '11', 'day': '10', 'size': 10}
]
ftpd.download_or_copy(available_files, '/biomaj', False)
ftpd.close()
assert (len(ftpd.files_to_download)==2)
assert (len(ftpd.files_to_copy)==2)
def test_download_or_copy_directhttp(self):
ftpd = DirectHTTPDownload('https', 'ftp.fr.debian.org', '/debian/')
ftpd.files_to_download = [
{'name':'/test1', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
]
available_files = [
{'name':'/test1', 'year': '2020', 'month': '11', 'day': '10', 'size': 10},
# {"root": "/", "permissions": "", "group": "", "user": "", "size": 23723408, "month": 6, "day": 19, "year": 2018, "name": "/common/downloads/release-38/Pfalciparum3D7/fasta/data/PlasmoDB-38_Pfalciparum3D7_Genome.fasta", "hash": "e58669a71eacff7a9dcceed04a8ecdd1", "save_as": "PlasmoDB-38_Pfalciparum3D7_Genome.fasta", "url": "https://plasmodb.org"}
]
ftpd.download_or_copy(available_files, '/biomaj', False)
ftpd.close()
assert (len(ftpd.files_to_download)==1)
assert (len(ftpd.files_to_copy)==0)
def test_get_more_recent_file(self):
files = [
{'name':'/test1', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test2', 'year': '2013', 'month': '11', 'day': '12', 'size': 10},
{'name':'/test/test1', 'year': '1988', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test/test11', 'year': '2013', 'month': '9', 'day': '23', 'size': 10}
]
release = Utils.get_more_recent_file(files)
assert (release['year']=='2013')
assert (release['month']=='11')
assert (release['day']=='12')
def test_download_retry(self):
"""
Try to download fake files to test retry.
"""
n_attempts = 5
ftpd = CurlDownload("ftp", "speedtest.tele2.net", "/")
# Download a fake file
ftpd.set_files_to_download([
{'name': 'TOTO.zip', 'year': '2016', 'month': '02', 'day': '19',
'size': 1, 'save_as': 'TOTO1KB'}
])
ftpd.set_options(dict(stop_condition=tenacity.stop.stop_after_attempt(n_attempts),
wait_condition=tenacity.wait.wait_none()))
with pytest.raises(Exception):
ftpd.download(self.utils.data_dir)
logging.debug(ftpd.retryer.statistics)
assert (len(ftpd.files_to_download) == 1)
assert (ftpd.retryer.statistics["attempt_number"] == n_attempts)
# Try to download another file to ensure that it retryies
ftpd.set_files_to_download([
{'name': 'TITI.zip', 'year': '2016', 'month': '02', 'day': '19',
'size': 1, 'save_as': 'TOTO1KB'}
])
with pytest.raises(Exception):
ftpd.download(self.utils.data_dir)
assert (len(ftpd.files_to_download) == 1)
assert (ftpd.retryer.statistics["attempt_number"] == n_attempts)
ftpd.close()
def test_ms_server(self):
ftpd = CurlDownload("ftp", "test.rebex.net", "/")
ftpd.set_credentials("demo:password")
(file_list, dir_list) = ftpd.list()
ftpd.match(["^readme.txt$"], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
assert (len(ftpd.files_to_download) == 1)
def test_download_tcp_keepalive(self):
"""
Test setting tcp_keepalive (it probably doesn't change anything here but
we test that there is no obvious mistake in the code).
"""
ftpd = CurlDownload("ftp", "test.rebex.net", "/")
ftpd.set_options(dict(tcp_keepalive=10))
ftpd.set_credentials("demo:password")
(file_list, dir_list) = ftpd.list()
ftpd.match(["^readme.txt$"], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
assert (len(ftpd.files_to_download) == 1)
def test_download_ftp_method(self):
"""
Test setting ftp_method (it probably doesn't change anything here but we
test that there is no obvious mistake in the code).
"""
ftpd = CurlDownload("ftp", "test.rebex.net", "/")
ftpd.set_options(dict(ftp_method="nocwd"))
ftpd.set_credentials("demo:password")
(file_list, dir_list) = ftpd.list()
ftpd.match(["^readme.txt$"], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
assert (len(ftpd.files_to_download) == 1)
@pytest.mark.skipif(
os.environ.get('NETWORK', 1) == '0',
reason='network tests disabled'
)
@pytest.mark.skipif(
os.environ.get('FTP', 1) == '0',
reason='network tests disabled'
)
class TestBiomajFTPSDownload():
"""
Test FTP downloader with FTPS.
"""
PROTOCOL = "ftps"
def setup_method(self, m):
self.utils = UtilsForTest()
def teardown_method(self, m):
self.utils.clean()
def test_ftps_list(self):
ftpd = CurlDownload(self.PROTOCOL, "test.rebex.net", "/")
ftpd.set_credentials("demo:password")
(file_list, dir_list) = ftpd.list()
ftpd.close()
assert (len(file_list) == 1)
def test_ftps_list_error(self):
"""
Test that errors in list are correctly caught.
"""
# Test access to non-existent directory
ftpd = CurlDownload("ftps", "test.rebex.net", "/toto")
ftpd.set_credentials("demo:password")
# Check that we raise an exception and log a message
with pytest.raises(Exception):
(file_list, dir_list) = ftpd.list()
ftpd.close()
# Test with wrong password
ftpd = CurlDownload("ftps", "test.rebex.net", "/")
ftpd.set_credentials("demo:badpassword")
# Check that we raise an exception and log a message
with pytest.raises(Exception):
(file_list, dir_list) = ftpd.list()
ftpd.close()
def test_download(self):
ftpd = CurlDownload(self.PROTOCOL, "test.rebex.net", "/")
ftpd.set_credentials("demo:password")
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^readme.txt$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
assert (len(ftpd.files_to_download) == 1)
def test_ftps_list_no_ssl(self):
# This server is misconfigured hence we disable all SSL verification
SERVER = "demo.wftpserver.com"
DIRECTORY = "/download/"
CREDENTIALS = "demo:demo"
ftpd = CurlDownload(self.PROTOCOL, SERVER, DIRECTORY)
ftpd.set_options(dict(ssl_verifyhost="False", ssl_verifypeer="False"))
ftpd.set_credentials(CREDENTIALS)
(file_list, dir_list) = ftpd.list()
ftpd.close()
assert (len(file_list) > 1)
def test_download_no_ssl(self):
# This server is misconfigured hence we disable all SSL verification
SERVER = "demo.wftpserver.com"
DIRECTORY = "/download/"
CREDENTIALS = "demo:demo"
ftpd = CurlDownload(self.PROTOCOL, SERVER, DIRECTORY)
ftpd.set_options(dict(ssl_verifyhost="False", ssl_verifypeer="False"))
ftpd.set_credentials(CREDENTIALS)
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^manual_en.pdf$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
assert (len(ftpd.files_to_download) == 1)
def test_download_ssl_certificate(self):
# This server is misconfigured but we use its certificate
# The hostname is wrong so we disable host verification
SERVER = "demo.wftpserver.com"
DIRECTORY = "/download/"
CREDENTIALS = "demo:demo"
ftpd = CurlDownload(self.PROTOCOL, SERVER, DIRECTORY)
curdir = os.path.dirname(os.path.realpath(__file__))
cert_file = os.path.join(curdir, "caert.demo.wftpserver.com.pem")
ftpd.set_options(dict(ssl_verifyhost="False", ssl_server_cert=cert_file))
ftpd.set_credentials(CREDENTIALS)
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^manual_en.pdf$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
assert (len(ftpd.files_to_download) == 1)
@pytest.mark.skipif(
os.environ.get('LOCAL', 1) == '0',
reason='network tests disabled'
)
class TestBiomajRSYNCDownload():
'''
Test RSYNC downloader
'''
def setup_method(self, m):
self.utils = UtilsForTest()
self.curdir = os.path.dirname(os.path.realpath(__file__)) + '/'
self.examples = os.path.join(self.curdir,'bank') + '/'
BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
def teardown_method(self, m):
self.utils.clean()
def test_rsync_list(self):
rsyncd = RSYNCDownload(self.examples, "")
(files_list, dir_list) = rsyncd.list()
assert (len(files_list) != 0)
def test_rsync_list_error(self):
# Access a non-existent directory
rsyncd = RSYNCDownload("/tmp/foo/", "")
with pytest.raises(Exception):
(file_list, dir_list) = rsyncd.list()
def test_rsync_match(self):
rsyncd = RSYNCDownload(self.examples, "")
(files_list, dir_list) = rsyncd.list()
rsyncd.match([r'^test.*\.gz$'], files_list, dir_list, prefix='', submatch=False)
assert (len(rsyncd.files_to_download) != 0)
def test_rsync_download(self):
rsyncd = RSYNCDownload(self.examples, "")
rfile = {
"name": "test2.fasta",
"root": self.examples
}
error = rsyncd._download(self.utils.data_dir, rfile)
assert not (error)
def test_rsync_general_download(self):
rsyncd = RSYNCDownload(self.examples, "")
(files_list, dir_list) = rsyncd.list()
rsyncd.match([r'^test.*\.gz$'],files_list,dir_list, prefix='')
download_files=rsyncd.download(self.curdir)
assert (len(download_files)==1)
def test_rsync_download_or_copy(self):
rsyncd = RSYNCDownload(self.examples, "")
(file_list, dir_list) = rsyncd.list()
rsyncd.match([r'^test.*\.gz$'], file_list, dir_list, prefix='')
files_to_download_prev = rsyncd.files_to_download
rsyncd.download_or_copy(rsyncd.files_to_download, self.examples, check_exists=True)
assert (files_to_download_prev != rsyncd.files_to_download)
def test_rsync_download_in_subdir(self):
rsyncd = RSYNCDownload(self.curdir, "")
(file_list, dir_list) = rsyncd.list()
rsyncd.match([r'^/bank/test*'], file_list, dir_list, prefix='')
rsyncd.download(self.utils.data_dir)
assert (len(rsyncd.files_to_download) == 3)
def test_rsync_download_skip_check_uncompress(self):
"""
Download the fake archive file with RSYNC but skip check.
"""
rsyncd = RSYNCDownload(self.utils.test_dir + '/', "")
rsyncd.set_options(dict(skip_check_uncompress=True))
(file_list, dir_list) = rsyncd.list()
rsyncd.match([r'invalid.gz'], file_list, dir_list, prefix='')
rsyncd.download(self.utils.data_dir)
assert (len(rsyncd.files_to_download) == 1)
def test_rsync_download_retry(self):
"""
Try to download fake files to test retry.
"""
n_attempts = 5
rsyncd = RSYNCDownload(self.utils.test_dir + '/', "")
rsyncd.set_options(dict(skip_check_uncompress=True))
# Download a fake file
rsyncd.set_files_to_download([
{'name': 'TOTO.zip', 'year': '2016', 'month': '02', 'day': '19',
'size': 1, 'save_as': 'TOTO1KB'}
])
rsyncd.set_options(dict(stop_condition=tenacity.stop.stop_after_attempt(n_attempts),
wait_condition=tenacity.wait.wait_none()))
with pytest.raises(Exception):
rsyncd.download(self.utils.data_dir)
logging.debug(rsyncd.retryer.statistics)
assert (len(rsyncd.files_to_download) == 1)
assert (rsyncd.retryer.statistics["attempt_number"] == n_attempts)
# Try to download another file to ensure that it retryies
rsyncd.set_files_to_download([
{'name': 'TITI.zip', 'year': '2016', 'month': '02', 'day': '19',
'size': 1, 'save_as': 'TOTO1KB'}
])
with pytest.raises(Exception):
rsyncd.download(self.utils.data_dir)
assert (len(rsyncd.files_to_download) == 1)
assert (rsyncd.retryer.statistics["attempt_number"] == n_attempts)
rsyncd.close()
class iRodsResult(object):
def __init__(self, collname, dataname, datasize, owner, modify):
self.Collname = 'tests/'
self.Dataname = 'test.fasta.gz'
self.Datasize = 45
self.Dataowner_name = 'biomaj'
self.Datamodify_time = '2017-04-10 00:00:00'
def __getitem__(self, index):
from irods.models import Collection, DataObject, User
if index.icat_id == DataObject.modify_time.icat_id:
return self.Datamodify_time
elif "DATA_SIZE" in str(index):
return self.Datasize
elif "DATA_NAME" in str(index):
return 'test.fasta.gz'
elif "COLL_NAME" in str(index):
return self.Collname
elif "D_OWNER_NAME" in str(index):
return self.Dataowner_name
class MockiRODSSession(object):
'''
Simulation of python irods client
for result in session.query(Collection.name, DataObject.name, DataObject.size, DataObject.owner_name, DataObject.modify_time).filter(User.name == self.user).get_results():
'''
def __init__(self):
self.Collname="1"
self.Dataname="2"
self.Datasize="3"
self.Dataowner_name="4"
self.Datamodify_time="5"
self.Collid=""
def __getitem__(self, index):
from irods.data_object import iRODSDataObject
from irods.models import Collection, DataObject, User
print(index)
if "COLL_ID" in str(index):
return self.Collid
if "COLL_NAME" in str(index):
return self.Collname
def configure(self):
return MockiRODSSession()
def query(self,Collname, Dataname, Datasize, Dataowner_name, Datamodify_time):
return self
def all(self):
return self
def one(self):
return self
def filter(self,boo):
return self
def get_results(self):
get_result_dict= iRodsResult('tests/', 'test.fasta.gz', 45, 'biomaj', '2017-04-10 00:00:00')
return [get_result_dict]
def cleanup(self):
return self
def open(self,r):
my_test_file = open("tests/test.fasta.gz", "r+")
return(my_test_file)
@pytest.mark.skipif(
os.environ.get('NETWORK', 1) == '0',
reason='network tests disabled'
)
@pytest.mark.skipif(
os.environ.get('IRODS', 1) == '0',
reason='iods tests disabled'
)
class TestBiomajIRODSDownload(unittest.TestCase):
'''
Test IRODS downloader
'''
def setup_method(self, m):
self.utils = UtilsForTest()
self.curdir = os.path.dirname(os.path.realpath(__file__))
self.examples = os.path.join(self.curdir,'bank') + '/'
BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
def teardown_method(self, m):
self.utils.clean()
@patch('irods.session.iRODSSession.configure')
@patch('irods.session.iRODSSession.query')
@patch('irods.session.iRODSSession.cleanup')
def test_irods_list(self,initialize_mock, query_mock,cleanup_mock):
mock_session=MockiRODSSession()
initialize_mock.return_value=mock_session.configure()
query_mock.return_value = mock_session.query(None,None,None,None,None)
cleanup_mock.return_value = mock_session.cleanup()
irodsd = IRODSDownload(self.examples, "")
(files_list, dir_list) = irodsd.list()
assert (len(files_list) != 0)
@pytest.mark.skipif(
os.environ.get('NETWORK', 1) == '0',
reason='network tests disabled'
)
@pytest.mark.skipif(
os.environ.get('LOCAL_IRODS', 1) == '0',
reason='irods tests disabled'
)
class TestBiomajLocalIRODSDownload(unittest.TestCase):
"""
Test with a local iRODS server.
"""
def setup_method(self, m):
self.utils = UtilsForLocalIRODSTest()
self.curdir = os.path.dirname(os.path.realpath(__file__))
self.examples = os.path.join(self.curdir,'bank') + '/'
BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
def teardown_method(self, m):
self.utils.clean()
def test_irods_download(self):
irodsd = IRODSDownload(self.utils.SERVER, self.utils.COLLECTION)
irodsd.set_param(dict(
user=self.utils.USER,
password=self.utils.PASSWORD,
))
(file_list, dir_list) = irodsd.list()
irodsd.match([r'^test.*\.gz$'], file_list, dir_list, prefix='')
irodsd.download(self.utils.data_dir)
assert (len(irodsd.files_to_download) == 1)
def test_irods_download_skip_check_uncompress(self):
"""
Download the fake archive file with iRODS but skip check.
"""
irodsd = IRODSDownload(self.utils.SERVER, self.utils.COLLECTION)
irodsd.set_options(dict(skip_check_uncompress=True))
irodsd.set_param(dict(
user=self.utils.USER,
password=self.utils.PASSWORD,
))
(file_list, dir_list) = irodsd.list()
irodsd.match([r'invalid.gz$'], file_list, dir_list, prefix='')
irodsd.download(self.utils.data_dir)
assert (len(irodsd.files_to_download) == 1)
def test_irods_download_retry(self):
"""
Try to download fake files to test retry.
"""
n_attempts = 5
irodsd = IRODSDownload(self.utils.SERVER, self.utils.COLLECTION)
irodsd.set_options(dict(skip_check_uncompress=True))
irodsd.set_param(dict(
user=self.utils.USER,
password=self.utils.PASSWORD,
))
# Download a fake file
irodsd.set_files_to_download([
{'name': 'TOTO.zip', 'year': '2016', 'month': '02', 'day': '19',
'size': 1, 'save_as': 'TOTO1KB'}
])
irodsd.set_options(dict(stop_condition=tenacity.stop.stop_after_attempt(n_attempts),
wait_condition=tenacity.wait.wait_none()))
with pytest.raises(Exception):
irodsd.download(self.utils.data_dir)
logging.debug(irodsd.retryer.statistics)
assert (len(irodsd.files_to_download) == 1)
assert (irodsd.retryer.statistics["attempt_number"] == n_attempts)
# Try to download another file to ensure that it retryies
irodsd.set_files_to_download([
{'name': 'TITI.zip', 'year': '2016', 'month': '02', 'day': '19',
'size': 1, 'save_as': 'TOTO1KB'}
])
with pytest.raises(Exception):
irodsd.download(self.utils.data_dir)
assert (len(irodsd.files_to_download) == 1)
assert (irodsd.retryer.statistics["attempt_number"] == n_attempts)
irodsd.close()
def test_irods_list_error(self):
# Non-existing collection
irodsd = IRODSDownload(self.utils.SERVER, "fake_collection")
irodsd.set_param(dict(
user=self.utils.USER,
password=self.utils.PASSWORD,
))
with pytest.raises(Exception):
(file_list, dir_list) = irodsd.list()
# Test with wrong password
irodsd = IRODSDownload(self.utils.SERVER, self.utils.COLLECTION)
irodsd.set_param(dict(
user=self.utils.USER,
password="badpassword",
))
with pytest.raises(Exception):
(file_list, dir_list) = irodsd.list()