From: Yaroslav Halchenko Subject: Disable unicode strings in commands to be executed in tests As you could see largely it is about executing a command with unicode, or later logging it. Whenvever Python2 seems to do it automagical conversions without blowing up, on Python3 I found no reliable way to achieve desired -- logger would not accept bytes, but would puke upon attempt to encode unicode into 'ascii', etc Problems go away if UTF-8 locale is configured and set (instead of C or POSIX) Last-Update: 2018-06-05 --- a/datalad/tests/test_config.py +++ b/datalad/tests/test_config.py @@ -63,7 +63,7 @@ novalue empty = myint = 3 -[onemore "complicated の beast with.dot"] +[onemore "complicated nounicode beast with.dot"] findme = 5.0 """ @@ -138,16 +138,16 @@ def test_something(path=None, new_home=N assert_true(cfg.has_section('something')) assert_false(cfg.has_section('somethingelse')) assert_equal(sorted(cfg.sections()), - [u'onemore.complicated の beast with.dot', 'something']) + [u'onemore.complicated nounicode beast with.dot', 'something']) assert_true(cfg.has_option('something', 'user')) assert_false(cfg.has_option('something', 'us?er')) assert_false(cfg.has_option('some?thing', 'user')) assert_equal(sorted(cfg.options('something')), ['empty', 'myint', 'novalue', 'user']) - assert_equal(cfg.options(u'onemore.complicated の beast with.dot'), ['findme']) + assert_equal(cfg.options(u'onemore.complicated nounicode beast with.dot'), ['findme']) assert_equal( sorted(cfg.items()), - [(u'onemore.complicated の beast with.dot.findme', '5.0'), + [(u'onemore.complicated nounicode beast with.dot.findme', '5.0'), ('something.empty', ''), ('something.myint', '3'), ('something.novalue', None), @@ -167,7 +167,7 @@ def test_something(path=None, new_home=N cfg.get('something.user', get_all=True), ('name=Jane Doe', 'email=jd@example.com')) assert_raises(KeyError, cfg.__getitem__, 'somedthing.user') - assert_equal(cfg.getfloat(u'onemore.complicated の beast with.dot', 'findme'), 5.0) + assert_equal(cfg.getfloat(u'onemore.complicated nounicode beast with.dot', 'findme'), 5.0) assert_equal(cfg.getint('something', 'myint'), 3) assert_equal(cfg.getbool('something', 'myint'), True) # git demands a key without value at all to be used as a flag, thus True @@ -186,8 +186,8 @@ def test_something(path=None, new_home=N assert_raises(KeyError, cfg.get_value, 'doesnot', 'exist', default=None) # modification follows - cfg.add('something.new', 'の') - assert_equal(cfg.get('something.new'), u'の') + cfg.add('something.new', 'nounicode') + assert_equal(cfg.get('something.new'), u'nounicode') # sections are added on demand cfg.add('unheard.of', 'fame') assert_true(cfg.has_section('unheard.of')) --- a/datalad/core/local/tests/test_run.py +++ b/datalad/core/local/tests/test_run.py @@ -184,29 +184,29 @@ def test_py2_unicode_command(path=None): touch_cmd = "import sys; open(sys.argv[1], 'w').write('')" cmd_str = u"{} -c \"{}\" {}".format(sys.executable, touch_cmd, - u"bβ0.dat") + u"bB0.dat") ds.run(cmd_str) assert_repo_status(ds.path) - ok_exists(op.join(path, u"bβ0.dat")) + ok_exists(op.join(path, u"bB0.dat")) # somewhat desperate attempt to detect our own Github CI tests on a # crippled filesystem (VFAT) that is so crippled that it doesn't handle # what is needed here. It just goes mad with encoded bytestrings: - # CommandError: ''python -c '"'"'import sys; open(sys.argv[1], '"'"'"'"'"'"'"'"'w'"'"'"'"'"'"'"'"').write('"'"'"'"'"'"'"'"''"'"'"'"'"'"'"'"')'"'"' '"'"' β1 '"'"''' failed with exitcode 1 under /crippledfs/ + # CommandError: ''python -c '"'"'import sys; open(sys.argv[1], '"'"'"'"'"'"'"'"'w'"'"'"'"'"'"'"'"').write('"'"'"'"'"'"'"'"''"'"'"'"'"'"'"'"')'"'"' '"'"' B1 '"'"''' failed with exitcode 1 under /crippledfs/ if not on_windows and os.environ.get('TMPDIR', None) != '/crippledfs': # FIXME - ds.run([sys.executable, "-c", touch_cmd, u"bβ1.dat"]) + ds.run([sys.executable, "-c", touch_cmd, u"bB1.dat"]) assert_repo_status(ds.path) - ok_exists(op.join(path, u"bβ1.dat")) + ok_exists(op.join(path, u"bB1.dat")) # Send in a list of byte-strings to mimic a py2 command-line # invocation. ds.run([s.encode("utf-8") - for s in [sys.executable, "-c", touch_cmd, u" β1 "]]) + for s in [sys.executable, "-c", touch_cmd, u" B1 "]]) assert_repo_status(ds.path) - ok_exists(op.join(path, u" β1 ")) + ok_exists(op.join(path, u" B1 ")) assert_in_results( - ds.run(u"bβ2.dat", result_renderer=None, on_failure="ignore"), + ds.run(u"bB2.dat", result_renderer=None, on_failure="ignore"), status="error", action="run") --- a/datalad/runner/tests/test_witless_runner.py +++ b/datalad/runner/tests/test_witless_runner.py @@ -65,7 +65,7 @@ result_counter = 0 @with_tempfile def test_runner(tempfile: str = "") -> None: runner = Runner() - content = 'Testing real run' if on_windows else 'Testing äöü東 real run' + content = 'Testing real run' cmd = 'echo %s > %s' % (content, tempfile) res = runner.run(cmd) assert isinstance(res, dict) --- a/datalad/tests/utils_pytest.py +++ b/datalad/tests/utils_pytest.py @@ -1434,7 +1434,7 @@ with_parametric_batch = pytest.mark.para OBSCURE_PREFIX = os.getenv('DATALAD_TESTS_OBSCURE_PREFIX', '') # Those will be tried to be added to the base name if filesystem allows OBSCURE_FILENAME_PARTS = [' ', '/', '|', ';', '&', '%b5', '{}', "'", '"', '<', '>'] -UNICODE_FILENAME = u"ΔЙקم๗あ" +UNICODE_FILENAME = u"" # OSX is exciting -- some I guess FS might be encoding differently from decoding # so Й might get recoded