| 1 |
# (C) Copyright 2018- ECMWF. |
| 2 |
# This software is licensed under the terms of the Apache Licence Version 2.0 |
| 3 |
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 4 |
# In applying this licence, ECMWF does not waive the privileges and immunities |
| 5 |
# granted to it by virtue of its status as an intergovernmental organisation |
| 6 |
# nor does it submit to any jurisdiction. |
| 7 |
|
| 8 |
from pathlib import Path |
| 9 |
import os |
| 10 |
import time |
| 11 |
import pytest |
| 12 |
import numpy as np |
| 13 |
|
| 14 |
from loki import Subroutine |
| 15 |
from loki.jit_build import Obj, Lib, Builder, run_isolated, jit_compile_and_run |
| 16 |
from loki.jit_build.compiler import ( |
| 17 |
Compiler, GNUCompiler, NvidiaCompiler, get_compiler_from_env, |
| 18 |
_default_compiler |
| 19 |
) |
| 20 |
|
| 21 |
|
| 22 |
@pytest.fixture(scope='module', name='here') |
| 23 |
def fixture_here(): |
| 24 |
return Path(__file__).parent |
| 25 |
|
| 26 |
|
| 27 |
@pytest.fixture(scope='function', name='builder') |
| 28 |
def fixture_builder(here, tmp_path): |
| 29 |
yield Builder(source_dirs=here, build_dir=tmp_path) |
| 30 |
Obj.clear_cache() |
| 31 |
|
| 32 |
|
| 33 |
@pytest.fixture(scope='module', name='testdir') |
| 34 |
def fixture_testdir(here): |
| 35 |
return here.parent.parent/'tests' |
| 36 |
|
| 37 |
|
| 38 |
def _isolated_add(a, b): |
| 39 |
return a + b |
| 40 |
|
| 41 |
|
| 42 |
def _isolated_write_file(path): |
| 43 |
Path(path).write_text('child process wrote this') |
| 44 |
|
| 45 |
|
| 46 |
def _isolated_raise(): |
| 47 |
raise ValueError('isolated failure') |
| 48 |
|
| 49 |
|
| 50 |
def _isolated_exit(status): |
| 51 |
os._exit(status) # pylint: disable=protected-access |
| 52 |
|
| 53 |
|
| 54 |
def _isolated_sleep(seconds): |
| 55 |
time.sleep(seconds) |
| 56 |
|
| 57 |
|
| 58 |
def test_run_isolated_returns_result(): |
| 59 |
""" |
| 60 |
Test that run_isolated returns picklable target results. |
| 61 |
""" |
| 62 |
assert run_isolated(_isolated_add, 2, 5) == 7 |
| 63 |
|
| 64 |
|
| 65 |
def test_run_isolated_process_side_effect(tmp_path): |
| 66 |
""" |
| 67 |
Test that run_isolated executes the target in a child process. |
| 68 |
""" |
| 69 |
path = tmp_path/'isolated.txt' |
| 70 |
assert run_isolated(_isolated_write_file, path) is None |
| 71 |
assert path.read_text() == 'child process wrote this' |
| 72 |
|
| 73 |
|
| 74 |
def test_run_isolated_exit_after_result(): |
| 75 |
""" |
| 76 |
Test that run_isolated can bypass child interpreter shutdown after success. |
| 77 |
""" |
| 78 |
assert run_isolated(_isolated_add, 2, 5, exit_after_result=True) == 7 |
| 79 |
|
| 80 |
|
| 81 |
def test_run_isolated_exception(): |
| 82 |
""" |
| 83 |
Test that run_isolated reports Python exceptions from the child process. |
| 84 |
""" |
| 85 |
with pytest.raises(RuntimeError) as excinfo: |
| 86 |
run_isolated(_isolated_raise) |
| 87 |
assert 'ValueError: isolated failure' in str(excinfo.value) |
| 88 |
|
| 89 |
|
| 90 |
def test_run_isolated_nonzero_exit(): |
| 91 |
""" |
| 92 |
Test that run_isolated reports child process crashes or explicit exits. |
| 93 |
""" |
| 94 |
with pytest.raises(RuntimeError) as excinfo: |
| 95 |
run_isolated(_isolated_exit, 7) |
| 96 |
assert 'exit code 7' in str(excinfo.value) |
| 97 |
|
| 98 |
|
| 99 |
def test_run_isolated_timeout(): |
| 100 |
""" |
| 101 |
Test that run_isolated terminates child processes that exceed a timeout. |
| 102 |
""" |
| 103 |
with pytest.raises(RuntimeError) as excinfo: |
| 104 |
run_isolated(_isolated_sleep, 5, timeout=0.1) |
| 105 |
assert 'timed out after 0.1 seconds' in str(excinfo.value) |
| 106 |
|
| 107 |
|
| 108 |
def test_jit_compile_and_run_in_process(tmp_path): |
| 109 |
""" |
| 110 |
Test that jit_compile_and_run supports explicit in-process execution. |
| 111 |
""" |
| 112 |
source = """ |
| 113 |
subroutine fill_array(a, n) |
| 114 |
integer, intent(inout) :: a(n) |
| 115 |
integer, intent(in) :: n |
| 116 |
integer :: j |
| 117 |
do j=1,n |
| 118 |
a(j) = j |
| 119 |
end do |
| 120 |
end subroutine fill_array |
| 121 |
""" |
| 122 |
routine = Subroutine.from_source(source) |
| 123 |
n = 5 |
| 124 |
a = np.zeros(shape=(n,), dtype=np.int32) |
| 125 |
jit_compile_and_run(routine, filepath=tmp_path/'fill_array.F90', isolated=False, a=a, n=n) |
| 126 |
assert np.all(a == range(1, n+1)) |
| 127 |
|
| 128 |
|
| 129 |
def test_jit_compile_and_run_isolated(tmp_path): |
| 130 |
""" |
| 131 |
Test that jit_compile_and_run isolates compile-and-execute cycles by default. |
| 132 |
""" |
| 133 |
source = """ |
| 134 |
subroutine fill_array(a, n) |
| 135 |
integer, intent(inout) :: a(n) |
| 136 |
integer, intent(in) :: n |
| 137 |
integer :: j |
| 138 |
do j=1,n |
| 139 |
a(j) = j |
| 140 |
end do |
| 141 |
end subroutine fill_array |
| 142 |
""" |
| 143 |
routine = Subroutine.from_source(source) |
| 144 |
n = 5 |
| 145 |
a = np.zeros(shape=(n,), dtype=np.int32) |
| 146 |
result = jit_compile_and_run( |
| 147 |
routine, filepath=tmp_path/'fill_array_isolated.F90', a=a, n=n |
| 148 |
) |
| 149 |
assert result is None |
| 150 |
assert np.all(a == range(1, n+1)) |
| 151 |
|
| 152 |
|
| 153 |
def test_build_clean(builder): |
| 154 |
""" |
| 155 |
Test basic `make clean`-style functionality. |
| 156 |
""" |
| 157 |
# Mess up the build dir before cleaning it... |
| 158 |
(builder.build_dir/'xxx_a.o').touch(exist_ok=True) |
| 159 |
(builder.build_dir/'xxx_b.o').touch(exist_ok=True) |
| 160 |
(builder.build_dir/'xxx_a.mod').touch(exist_ok=True) |
| 161 |
(builder.build_dir/'xxx_a.so').touch(exist_ok=True) |
| 162 |
(builder.build_dir/'f90wrap_xxx_a.f90').touch(exist_ok=True) |
| 163 |
|
| 164 |
builder.clean('*.o *.mod *.so f90wrap*.f90') |
| 165 |
for f in builder.build_dir.iterdir(): |
| 166 |
assert 'xxx' not in str(f) |
| 167 |
|
| 168 |
|
| 169 |
def test_build_object(here, testdir, builder): |
| 170 |
""" |
| 171 |
Test basic object compilation and wrapping via f90wrap. |
| 172 |
""" |
| 173 |
obj = Obj(source_path=here/'base.f90') |
| 174 |
obj.build(builder=builder) |
| 175 |
assert (builder.build_dir/'base.o').exists() |
| 176 |
|
| 177 |
base = obj.wrap(builder=builder, kind_map=testdir/'kind_map') |
| 178 |
assert base.Base.a_times_b_plus_c(a=2, b=3, c=1) == 7 |
| 179 |
|
| 180 |
|
| 181 |
@pytest.mark.parametrize('workers', [None, 1, 3]) |
| 182 |
def test_build_lib(here, tmp_path, testdir, workers): |
| 183 |
""" |
| 184 |
Test basic library compilation and wrapping via f90wrap |
| 185 |
from a specific list of source objects. |
| 186 |
""" |
| 187 |
builder = Builder(source_dirs=here, build_dir=tmp_path, workers=workers) |
| 188 |
|
| 189 |
# Create library with explicit dependencies |
| 190 |
base = Obj(source_path=here/'base.f90') |
| 191 |
extension = Obj(source_path=here/'extension.f90') |
| 192 |
# Note: Need to compile statically to avoid LD_LIBRARY_PATH lookup |
| 193 |
lib = Lib(name='library', objs=[base, extension], shared=False) |
| 194 |
lib.build(builder=builder) |
| 195 |
assert (builder.build_dir/'liblibrary.a').exists() |
| 196 |
|
| 197 |
test = lib.wrap(modname='test', sources=[here/'extension.f90'], builder=builder, |
| 198 |
kind_map=testdir/'kind_map') |
| 199 |
assert test.extended_fma(2., 3., 1.) == 7. |
| 200 |
|
| 201 |
|
| 202 |
def test_build_lib_with_c(here, testdir, builder): |
| 203 |
""" |
| 204 |
Test basic library compilation and wrapping via f90wrap |
| 205 |
from a specific list of source objects. |
| 206 |
""" |
| 207 |
# Create library with explicit dependencies |
| 208 |
# objects = ['wrapper.f90', 'c_util.c'] |
| 209 |
wrapper = Obj(source_path=here/'wrapper.f90') |
| 210 |
c_util = Obj(source_path=here/'c_util.c') |
| 211 |
lib = Lib(name='library', objs=[wrapper, c_util], shared=False) |
| 212 |
lib.build(builder=builder) |
| 213 |
assert (builder.build_dir/'liblibrary.a').exists() |
| 214 |
|
| 215 |
wrap = lib.wrap(modname='wrap', sources=[here/'wrapper.f90'], builder=builder, |
| 216 |
kind_map=testdir/'kind_map') |
| 217 |
assert wrap.wrapper.mult_add_external(2., 3., 1.) == 7. |
| 218 |
|
| 219 |
|
| 220 |
def test_build_obj_dependencies(): |
| 221 |
""" |
| 222 |
Test dependency resolution in a non-trivial module tree. |
| 223 |
""" |
| 224 |
# # Wrap obj without specifying dependencies |
| 225 |
# test = builder.Obj('extension.f90').wrap() |
| 226 |
# assert test.library_test(1, 2, 3) == 12 |
| 227 |
|
| 228 |
|
| 229 |
def test_build_binary(builder): |
| 230 |
""" |
| 231 |
Test basic binary compilation from objects and libs. |
| 232 |
""" |
| 233 |
assert builder |
| 234 |
|
| 235 |
|
| 236 |
@pytest.mark.parametrize('env,cls,attrs', [ |
| 237 |
# Overwrite completely custom |
| 238 |
( |
| 239 |
{'CC': 'my-weird-compiler', 'FC': 'my-other-weird-compiler', 'F90': 'weird-fortran', 'FCFLAGS': '-my-flag '}, |
| 240 |
Compiler, |
| 241 |
{'CC': 'my-weird-compiler', 'FC': 'my-other-weird-compiler', 'F90': 'weird-fortran', 'FCFLAGS': ['-my-flag']}, |
| 242 |
), |
| 243 |
# GNUCompiler |
| 244 |
({'CC': 'gcc'}, GNUCompiler, {'CC': 'gcc', 'FC': 'gfortran', 'F90': 'gfortran'}), |
| 245 |
({'CC': 'gcc-13'}, GNUCompiler, None), |
| 246 |
({'CC': '/path/to/my/gcc'}, GNUCompiler, None), |
| 247 |
({'CC': '../../relative/path/to/my/gcc-11'}, GNUCompiler, None), |
| 248 |
({'CC': 'C:\\windows\\path\\to\\gcc'}, GNUCompiler, None), |
| 249 |
({'FC': 'gfortran'}, GNUCompiler, None), |
| 250 |
({'FC': 'gfortran-13', 'FCFLAGS': '-O3 -g'}, GNUCompiler, {'FC': 'gfortran-13', 'FCFLAGS': ['-O3', '-g']}), |
| 251 |
({'FC': '/path/to/my/gfortran'}, GNUCompiler, None), |
| 252 |
({'FC': '../../relative/path/to/my/gfortran'}, GNUCompiler, None), |
| 253 |
({'FC': 'C:\\windows\\path\\to\\gfortran'}, GNUCompiler, None), |
| 254 |
# NvidiaCompiler |
| 255 |
({'FC': 'nvfortran'}, NvidiaCompiler, {'CC': 'nvc', 'FC': 'nvfortran', 'F90': 'nvfortran'}), |
| 256 |
({'CC': 'nvc'}, NvidiaCompiler, None), |
| 257 |
({'CC': '/path/to/my/nvc'}, NvidiaCompiler, None), |
| 258 |
({'CC': '../../relative/path/to/my/nvc'}, NvidiaCompiler, None), |
| 259 |
({'CC': 'C:\\windows\\path\\to\\nvc'}, NvidiaCompiler, None), |
| 260 |
({'FC': 'pgf90'}, NvidiaCompiler, None), |
| 261 |
({'FC': 'pgf95'}, NvidiaCompiler, None), |
| 262 |
({'FC': 'pgfortran'}, NvidiaCompiler, None), |
| 263 |
({'FC': '/path/to/my/nvfortran'}, NvidiaCompiler, None), |
| 264 |
({'FC': '../../relative/path/to/my/pgfortran'}, NvidiaCompiler, None), |
| 265 |
({'FC': 'C:\\windows\\path\\to\\nvfortran'}, NvidiaCompiler, None), |
| 266 |
]) |
| 267 |
def test_get_compiler_from_env(env, cls, attrs): |
| 268 |
compiler = get_compiler_from_env(env) |
| 269 |
assert type(compiler) == cls # pylint: disable=unidiomatic-typecheck |
| 270 |
for attr, expected_value in (attrs or env).items(): |
| 271 |
# NB: We are comparing the lower-case attribute |
| 272 |
# because that contains the runtime value |
| 273 |
assert getattr(compiler, attr.lower()) == expected_value |
| 274 |
|
| 275 |
|
| 276 |
def test_default_compiler(): |
| 277 |
# Check that _default_compiler corresponds to a call with None |
| 278 |
compiler = get_compiler_from_env() |
| 279 |
assert type(compiler) == type(_default_compiler) # pylint: disable=unidiomatic-typecheck |
| 280 |
|
| 281 |
|
| 282 |
def test_obj_dependencies(tmp_path): |
| 283 |
fcode = """ |
| 284 |
module import_mod |
| 285 |
use module_mod |
| 286 |
implicit none |
| 287 |
contains |
| 288 |
subroutine proc1 |
| 289 |
use , non_intrinsic :: iso_fortran_env, only: int8, int16 |
| 290 |
use , intrinsic :: iso_c_binding |
| 291 |
use other_Mod |
| 292 |
use :: third_mod |
| 293 |
use fourth_mod , only:some |
| 294 |
use,non_intrinsic::very_condensed |
| 295 |
end subroutine proc1 |
| 296 |
subroutine proc2 |
| 297 |
use, intrinsic :: iso_fortran_env, only: int8, int16 |
| 298 |
use::fifth_mod |
| 299 |
end subroutine proc2 |
| 300 |
end module import_mod |
| 301 |
""".strip() |
| 302 |
|
| 303 |
filepath = tmp_path/'import_mod.f90' |
| 304 |
filepath.write_text(fcode) |
| 305 |
|
| 306 |
obj = Obj(name='import_mod', source_path=filepath) |
| 307 |
assert obj.dependencies == ( |
| 308 |
'module_mod', 'iso_fortran_env', 'other_Mod', 'third_mod', 'fourth_mod', |
| 309 |
'very_condensed', 'fifth_mod' |
| 310 |
) |
| 311 |
Obj.clear_cache() |