diff --git a/.pylintrc-local.yml b/.pylintrc-local.yml deleted file mode 100644 index b3478b184e32ba7a1b2f612628f8c8bbc23bee6f..0000000000000000000000000000000000000000 --- a/.pylintrc-local.yml +++ /dev/null @@ -1,8 +0,0 @@ -- arg: ignore - val: - - firedrake - - to_firedrake.py - - from_firedrake.py - - test_firedrake_interop.py -- arg: extension-pkg-whitelist - val: mayavi diff --git a/arraycontext/__init__.py b/arraycontext/__init__.py index ac013212e879e85fdfe26593b2d2eccbcf5c3b80..3e0092db314c733339fbefbffcb140f37c32dc8e 100644 --- a/arraycontext/__init__.py +++ b/arraycontext/__init__.py @@ -53,8 +53,10 @@ from .container.traversal import ( from .impl.pyopencl import PyOpenCLArrayContext from .impl.pytato import PytatoPyOpenCLArrayContext -from .pytest import (pytest_generate_tests_for_array_contexts, - pytest_generate_tests_for_pyopencl_array_context) +from .pytest import ( + PytestPyOpenCLArrayContextFactory, + pytest_generate_tests_for_array_contexts, + pytest_generate_tests_for_pyopencl_array_context) from .loopy import make_loopy_program @@ -83,6 +85,7 @@ __all__ = ( "make_loopy_program", + "PytestPyOpenCLArrayContextFactory", "pytest_generate_tests_for_array_contexts", "pytest_generate_tests_for_pyopencl_array_context" ) diff --git a/arraycontext/fake_numpy.py b/arraycontext/fake_numpy.py index f19f6ff32c32cca463639a0a2439bcc0df090d0c..cb1b3fc63e01af3c297a8f950fb5fb3d63dfd509 100644 --- a/arraycontext/fake_numpy.py +++ b/arraycontext/fake_numpy.py @@ -170,6 +170,21 @@ class BaseFakeNumpyNamespace: # {{{ BaseFakeNumpyLinalgNamespace +def _scalar_list_norm(ary, ord): + if ord is None: + ord = 2 + + from numbers import Number + if ord == np.inf: + return max(ary) + elif ord == -np.inf: + return min(ary) + elif isinstance(ord, Number) and ord > 0: + return sum(iary**ord for iary in ary)**(1/ord) + else: + raise NotImplementedError(f"unsupported value of 'ord': {ord}") + + class BaseFakeNumpyLinalgNamespace: def __init__(self, array_context): self._array_context = array_context @@ -180,6 +195,8 @@ class BaseFakeNumpyLinalgNamespace: if isinstance(ary, Number): return abs(ary) + actx = self._array_context + try: from meshmode.dof_array import DOFArray, flat_norm except ImportError: @@ -193,19 +210,17 @@ class BaseFakeNumpyLinalgNamespace: "This will stop working in 2022. " "Use meshmode.dof_array.flat_norm instead.", DeprecationWarning, stacklevel=2) + return flat_norm(ary, ord=ord) if is_array_container(ary): - import numpy.linalg as la - return la.norm( - [self.norm(subary, ord=ord) - for _, subary in serialize_container(ary)], - ord=ord) + return _scalar_list_norm([ + self.norm(subary, ord=ord) + for _, subary in serialize_container(ary) + ], ord=ord) if ord is None: - return self.norm(self._array_context.np.ravel(ary, order="A"), 2) - - assert ord is not None + return self.norm(actx.np.ravel(ary, order="A"), 2) if len(ary.shape) != 1: raise NotImplementedError("only vector norms are implemented") @@ -215,6 +230,8 @@ class BaseFakeNumpyLinalgNamespace: if ord == np.inf: return self._array_context.np.max(abs(ary)) + elif ord == -np.inf: + return self._array_context.np.min(abs(ary)) elif isinstance(ord, Number) and ord > 0: return self._array_context.np.sum(abs(ary)**ord)**(1/ord) else: diff --git a/arraycontext/impl/pyopencl/__init__.py b/arraycontext/impl/pyopencl/__init__.py index ca5b2dd725c3a121e87256859d5de37f68e01a11..aa0743925e8faa20ee3a65be4c7e759ee8f447c7 100644 --- a/arraycontext/impl/pyopencl/__init__.py +++ b/arraycontext/impl/pyopencl/__init__.py @@ -2,6 +2,7 @@ .. currentmodule:: arraycontext .. autoclass:: PyOpenCLArrayContext """ + __copyright__ = """ Copyright (C) 2020-1 University of Illinois Board of Trustees """ @@ -27,7 +28,7 @@ THE SOFTWARE. """ from warnings import warn -from typing import Sequence, Union +from typing import Dict, List, Sequence, Optional, Union, TYPE_CHECKING import numpy as np @@ -38,6 +39,11 @@ from arraycontext.context import ArrayContext from numbers import Number +if TYPE_CHECKING: + import pyopencl + import loopy as lp + + # {{{ PyOpenCLArrayContext class PyOpenCLArrayContext(ArrayContext): @@ -63,7 +69,11 @@ class PyOpenCLArrayContext(ArrayContext): as the allocator can help avoid this cost. """ - def __init__(self, queue, allocator=None, wait_event_queue_length=None): + def __init__(self, + queue: "pyopencl.CommandQueue", + allocator: Optional["pyopencl.tools.AllocatorInterface"] = None, + wait_event_queue_length: Optional[int] = None, + force_device_scalars: bool = False) -> None: r""" :arg wait_event_queue_length: The length of a queue of :class:`~pyopencl.Event` objects that are maintained by the @@ -84,19 +94,31 @@ class PyOpenCLArrayContext(ArrayContext): For now, *wait_event_queue_length* should be regarded as an experimental feature that may change or disappear at any minute. + + :arg force_device_scalars: if *True*, scalar results returned from + reductions in :attr:`ArrayContext.np` will be kept on the device. + If *False*, the equivalent of :meth:`~ArrayContext.freeze` and + :meth:`~ArrayContext.to_numpy` is applied to transfer the results + to the host. """ + if not force_device_scalars: + warn("Returning host scalars from the array context is deprecated. " + "To return device scalars set 'force_device_scalars=True'. " + "Support for returning host scalars will be removed in 2022.", + DeprecationWarning, stacklevel=2) + import pyopencl as cl super().__init__() self.context = queue.context self.queue = queue self.allocator = allocator if allocator else None - if wait_event_queue_length is None: wait_event_queue_length = 10 + self._force_device_scalars = force_device_scalars self._wait_event_queue_length = wait_event_queue_length - self._kernel_name_to_wait_event_queue = {} + self._kernel_name_to_wait_event_queue: Dict[str, List[cl.Event]] = {} if queue.device.type & cl.device_type.GPU: if allocator is None: @@ -111,7 +133,8 @@ class PyOpenCLArrayContext(ArrayContext): "are running Python in debug mode. Use 'python -O' for " "a noticeable speed improvement.") - self._loopy_transform_cache = {} + self._loopy_transform_cache: \ + Dict["lp.TranslationUnit", "lp.TranslationUnit"] = {} def _get_fake_numpy_namespace(self): from arraycontext.impl.pyopencl.fake_numpy import PyOpenCLFakeNumpyNamespace @@ -134,8 +157,9 @@ class PyOpenCLArrayContext(ArrayContext): return cl_array.to_device(self.queue, array, allocator=self.allocator) def to_numpy(self, array): - if isinstance(array, Number): + if not self._force_device_scalars and np.isscalar(array): return array + return array.get(queue=self.queue) def call_loopy(self, t_unit, **kwargs): @@ -232,7 +256,9 @@ class PyOpenCLArrayContext(ArrayContext): return array def clone(self): - return type(self)(self.queue, self.allocator, self._wait_event_queue_length) + return type(self)(self.queue, self.allocator, + wait_event_queue_length=self._wait_event_queue_length, + force_device_scalars=self._force_device_scalars) @property def permits_inplace_modification(self): diff --git a/arraycontext/impl/pyopencl/fake_numpy.py b/arraycontext/impl/pyopencl/fake_numpy.py index e2bfe00df269b62b435f1d8d3e8a6c57bd8c4da0..20e0d48484859e0eeba2fc1cd61bc5e8d0e84a23 100644 --- a/arraycontext/impl/pyopencl/fake_numpy.py +++ b/arraycontext/impl/pyopencl/fake_numpy.py @@ -104,14 +104,25 @@ class PyOpenCLFakeNumpyNamespace(BaseFakeNumpyNamespace): return rec_multimap_array_container(where_inner, criterion, then, else_) def sum(self, a, dtype=None): - return cl_array.sum( - a, dtype=dtype, queue=self._array_context.queue).get()[()] + result = cl_array.sum(a, dtype=dtype, queue=self._array_context.queue) + if not self._array_context._force_device_scalars: + result = result.get()[()] + + return result def min(self, a): - return cl_array.min(a, queue=self._array_context.queue).get()[()] + result = cl_array.min(a, queue=self._array_context.queue) + if not self._array_context._force_device_scalars: + result = result.get()[()] + + return result def max(self, a): - return cl_array.max(a, queue=self._array_context.queue).get()[()] + result = cl_array.max(a, queue=self._array_context.queue) + if not self._array_context._force_device_scalars: + result = result.get()[()] + + return result def stack(self, arrays, axis=0): return rec_multimap_array_container( diff --git a/arraycontext/pytest.py b/arraycontext/pytest.py index a9c06eba4eee6610bc00ad66c45cd0e26bb050ee..a939b05c585713c291524b60b50f98284719b11b 100644 --- a/arraycontext/pytest.py +++ b/arraycontext/pytest.py @@ -1,9 +1,12 @@ """ .. currentmodule:: arraycontext + +.. autoclass:: PytestPyOpenCLArrayContextFactory + .. autofunction:: pytest_generate_tests_for_array_contexts +.. autofunction:: pytest_generate_tests_for_pyopencl_array_context """ - __copyright__ = """ Copyright (C) 2020-1 University of Illinois Board of Trustees """ @@ -28,119 +31,226 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +from typing import Any, Callable, Dict, Sequence, Type, Union -# {{{ pytest integration - -from arraycontext.impl.pyopencl import PyOpenCLArrayContext -from arraycontext.impl.pytato import PytatoPyOpenCLArrayContext import pyopencl as cl -from pyopencl.tools import _ContextFactory -from typing import List +from arraycontext.context import ArrayContext -class _PyOpenCLArrayContextFactory(_ContextFactory): - def __call__(self): - ctx = super().__call__() - from arraycontext.impl.pyopencl import PyOpenCLArrayContext - return PyOpenCLArrayContext(cl.CommandQueue(ctx)) +# {{{ array context factories - def __str__(self): - return ("" - % (self.device.name.strip(), - self.device.platform.name.strip())) +class PytestPyOpenCLArrayContextFactory: + """ + .. automethod:: __init__ + .. automethod:: __call__ + """ + + def __init__(self, device): + """ + :arg device: a :class:`pyopencl.Device`. + """ + self.device = device + + def get_command_queue(self): + # Get rid of leftovers from past tests. + # CL implementations are surprisingly limited in how many + # simultaneous contexts they allow... + from pyopencl.tools import clear_first_arg_caches + clear_first_arg_caches() + + from gc import collect + collect() + + ctx = cl.Context([self.device]) + return cl.CommandQueue(ctx) + def __call__(self) -> ArrayContext: + raise NotImplementedError + + +class _PyOpenCLArrayContextFactory(PytestPyOpenCLArrayContextFactory): + force_device_scalars = True -class _PytatoPyOpenCLArrayContextFactory(_ContextFactory): def __call__(self): - ctx = super().__call__() - from arraycontext.impl.pytato import PytatoPyOpenCLArrayContext - return PytatoPyOpenCLArrayContext(cl.CommandQueue(ctx)) + from arraycontext import PyOpenCLArrayContext + return PyOpenCLArrayContext( + self.get_command_queue(), + force_device_scalars=self.force_device_scalars) def __str__(self): - return ("" - % (self.device.name.strip(), + return ("" % + (self.device.name.strip(), self.device.platform.name.strip())) -types_to_factories = {PyOpenCLArrayContext: _PyOpenCLArrayContextFactory, - PytatoPyOpenCLArrayContext: _PytatoPyOpenCLArrayContextFactory} +class _DeprecatedPyOpenCLArrayContextFactory(_PyOpenCLArrayContextFactory): + force_device_scalars = False -def pytest_generate_tests_for_array_contexts(metafunc, actx_list=None) -> None: - """Parametrize tests for pytest to use a - :class:`~arraycontext.ArrayContext`. +_ARRAY_CONTEXT_FACTORY_REGISTRY: \ + Dict[str, Type[PytestPyOpenCLArrayContextFactory]] = { + "pyopencl": _PyOpenCLArrayContextFactory, + "pyopencl-deprecated": _DeprecatedPyOpenCLArrayContextFactory, + } - Performs device enumeration analogously to - :func:`pyopencl.tools.pytest_generate_tests_for_pyopencl`. - Using the line: +def register_array_context_factory( + name: str, + factory: Type[PytestPyOpenCLArrayContextFactory]) -> None: + if name in _ARRAY_CONTEXT_FACTORY_REGISTRY: + raise ValueError(f"factory '{name}' already exists") + + _ARRAY_CONTEXT_FACTORY_REGISTRY[name] = factory + +# }}} + + +# {{{ pytest integration + +def pytest_generate_tests_for_array_contexts( + factories: Sequence[Union[str, Type[PytestPyOpenCLArrayContextFactory]]], *, + factory_arg_name: str = "actx_factory", + ) -> Callable[[Any], None]: + """Parametrize tests for pytest to use an :class:`~arraycontext.ArrayContext`. + + Using this function in :mod:`pytest` test scripts allows you to use the + argument *factory_arg_name*, which is a callable that returns a + :class:`~arraycontext.ArrayContext`. All test functions will automatically + be run once for each implemented array context. To select specific array + context implementations explicitly define, for example, .. code-block:: python - from arraycontext import pytest_generate_tests_for_array_contexts - as pytest_generate_tests + pytest_generate_tests = pytest_generate_tests_for_array_context([ + "pyopencl", + ]) - in your pytest test scripts allows you to use the argument ``actx_factory``, - in your test functions, and they will automatically be - run once for each OpenCL device/platform in the system, as appropriate, - with an argument-less function that returns an - :class:`~arraycontext.ArrayContext` when called. + to use the :mod:`pyopencl`-based array context. For :mod:`pyopencl`-based + contexts :func:`pyopencl.tools.pytest_generate_tests_for_pyopencl` is used + as a backend, which allows specifying the ``PYOPENCL_TEST`` environment + variable for device selection. - It also allows you to specify the ``PYOPENCL_TEST`` environment variable - for device selection. - """ + The environment variable ``ARRAYCONTEXT_TEST`` can also be used to + overwrite any chosen implementations through *factories*. This is a + comma-separated list of known array contexts. - if actx_list is None: - actx_list = [PyOpenCLArrayContext, PytatoPyOpenCLArrayContext] + Current supported implementations include: - actx_factories = [types_to_factories[a] for a in actx_list] + * ``"pyopencl"``, which creates a :class:`~arraycontext.PyOpenCLArrayContext` + with ``force_device_scalars=True``. + * ``"pyopencl-deprecated"``, which creates a + :class:`~arraycontext.PyOpenCLArrayContext` with + ``force_device_scalars=False``. - import pyopencl.tools as cl_tools - arg_names = cl_tools.get_pyopencl_fixture_arg_names( - metafunc, extra_arg_names=["actx_factory"]) + :arg factories: a list of identifiers or + :class:`PytestPyOpenCLArrayContextFactory` classes (not instances) + for which to generate test fixtures. + """ - if not arg_names: - return + # {{{ get all requested array context factories - arg_values, ids = cl_tools.get_pyopencl_fixture_arg_values() - if "actx_factory" in arg_names: - if "ctx_factory" in arg_names or "ctx_getter" in arg_names: - raise RuntimeError("Cannot use both an 'actx_factory' and a " - "'ctx_factory' / 'ctx_getter' as arguments.") + import os + env_factory_string = os.environ.get("ARRAYCONTEXT_TEST", None) - for arg_dict in arg_values: - dev = arg_dict["device"] - extra_factories: List[str] = [] + if env_factory_string is not None: + unique_factories = set(env_factory_string.split(",")) + else: + unique_factories = set(factories) # type: ignore[arg-type] - for factory in actx_factories: - if "actx_factory" in arg_dict: - arg_dict["actx_factory_"+str(factory)] = factory(dev) - extra_factories += ("actx_factory_"+str(factory),) - else: - arg_dict["actx_factory"] = factory(dev) + if not unique_factories: + raise ValueError("no array context factories were selected") - arg_values_out = [ - tuple(arg_dict[name] for name in arg_names) - for arg_dict in arg_values + unknown_factories = [ + factory for factory in unique_factories + if (isinstance(factory, str) + and factory not in _ARRAY_CONTEXT_FACTORY_REGISTRY) ] - for extra_factory in extra_factories: - arg_values_out += [ - tuple((arg_dict[extra_factory],)) - for arg_dict in arg_values + if unknown_factories: + if env_factory_string is not None: + raise RuntimeError( + "unknown array context factories passed through environment " + f"variable 'ARRAYCONTEXT_TEST': {unknown_factories}") + else: + raise ValueError(f"unknown array contexts: {unknown_factories}") + + unique_factories = set([ + _ARRAY_CONTEXT_FACTORY_REGISTRY.get(factory, factory) # type: ignore[misc] + for factory in unique_factories]) + + # }}} + + def inner(metafunc): + # {{{ get pyopencl devices + + import pyopencl.tools as cl_tools + arg_names = cl_tools.get_pyopencl_fixture_arg_names( + metafunc, extra_arg_names=[factory_arg_name]) + + if not arg_names: + return + + arg_values, ids = cl_tools.get_pyopencl_fixture_arg_values() + + # }}} + + # {{{ add array context factory to arguments + + if factory_arg_name in arg_names: + if "ctx_factory" in arg_names or "ctx_getter" in arg_names: + raise RuntimeError( + f"Cannot use both an '{factory_arg_name}' and a " + "'ctx_factory' / 'ctx_getter' as arguments.") + + arg_values_with_actx = [] + for arg_dict in arg_values: + arg_values_with_actx.extend([ + {factory_arg_name: factory(arg_dict["device"]), **arg_dict} + for factory in unique_factories + ]) + else: + arg_values_with_actx = arg_values + + arg_value_tuples = [ + tuple(arg_dict[name] for name in arg_names) + for arg_dict in arg_values_with_actx ] - metafunc.parametrize(arg_names, arg_values_out, ids=ids) + # }}} + + metafunc.parametrize(arg_names, arg_value_tuples, ids=ids) + + return inner def pytest_generate_tests_for_pyopencl_array_context(metafunc) -> None: - from warnings import warn - warn("'pytato.pytest_generate_tests_for_pyopencl_array_context' " - "is deprecated, use 'pytato.pytest_generate_tests_for_array_contexts' " - "instead. Only tests with PyOpenCLArrayContext will be run.", - DeprecationWarning) - pytest_generate_tests_for_array_contexts(metafunc, - actx_list=[PyOpenCLArrayContext]) + """Parametrize tests for pytest to use a + :class:`~arraycontext.ArrayContext`. + + Performs device enumeration analogously to + :func:`pyopencl.tools.pytest_generate_tests_for_pyopencl`. + + Using the line: + + .. code-block:: python + + from arraycontext import ( + pytest_generate_tests_for_pyopencl_array_context + as pytest_generate_tests) + + in your pytest test scripts allows you to use the argument ``actx_factory``, + in your test functions, and they will automatically be + run once for each OpenCL device/platform in the system, as appropriate, + with an argument-less function that returns an + :class:`~arraycontext.ArrayContext` when called. + + It also allows you to specify the ``PYOPENCL_TEST`` environment variable + for device selection. + """ + + pytest_generate_tests_for_array_contexts([ + "pyopencl-deprecated", + ], factory_arg_name="actx_factory")(metafunc) # }}} diff --git a/run-mypy.sh b/run-mypy.sh index 75f2cff8df6301dac507ac81dda9ab8fa60d81c8..52241aea914adc6ac43daed938b4a01e5a32fcbd 100755 --- a/run-mypy.sh +++ b/run-mypy.sh @@ -1,3 +1,3 @@ #!/bin/bash -python -m mypy arraycontext/ examples/ test/ +python -m mypy --show-error-codes arraycontext examples test diff --git a/test/test_arraycontext.py b/test/test_arraycontext.py index fce673fc027c81addb02706b05464002e24f9324..e68914d2f97f9885b08909d0f4f5cf6506ab6f14 100644 --- a/test/test_arraycontext.py +++ b/test/test_arraycontext.py @@ -33,14 +33,18 @@ from arraycontext import ( freeze, thaw, FirstAxisIsElementsTag, ArrayContainer) from arraycontext import ( # noqa: F401 - pytest_generate_tests_for_array_contexts - as pytest_generate_tests, + pytest_generate_tests_for_array_contexts, _acf) import logging logger = logging.getLogger(__name__) +pytest_generate_tests = pytest_generate_tests_for_array_contexts([ + "pyopencl", "pyopencl-deprecated", + ]) + + # {{{ stand-in DOFArray implementation @with_container_arithmetic( @@ -409,8 +413,13 @@ def test_dof_array_reductions_same_as_numpy(actx_factory, op): np_red = getattr(np, op)(ary) actx_red = getattr(actx.np, op)(actx.from_numpy(ary)) actx_red = actx.to_numpy(actx_red) + + if actx._force_device_scalars: + assert actx_red.shape == () + else: + assert isinstance(actx_red, Number) - assert np.allclose(np_red, actx_red) + assert np.allclose(np_red, actx.to_numpy(actx_red)) # }}} @@ -730,10 +739,10 @@ def test_norm_complex(actx_factory, norm_ord): @pytest.mark.parametrize("ndim", [1, 2, 3, 4, 5]) def test_norm_ord_none(actx_factory, ndim): - from numpy.random import default_rng - actx = actx_factory() + from numpy.random import default_rng + rng = default_rng() shape = tuple(rng.integers(2, 7, ndim)) @@ -742,9 +751,7 @@ def test_norm_ord_none(actx_factory, ndim): norm_a_ref = np.linalg.norm(a, ord=None) norm_a = actx.np.linalg.norm(actx.from_numpy(a), ord=None) - norm_a = actx.to_numpy(norm_a) - - np.testing.assert_allclose(norm_a, norm_a_ref) + np.testing.assert_allclose(actx.to_numpy(norm_a), norm_a_ref) # {{{ test_actx_compile helpers