From b87bf6f197ce4249ede1a1204d875ce610835cd8 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Sat, 9 Jan 2021 19:45:02 -0600 Subject: [PATCH 1/6] adds support for loopy kernel callables branch --- pytato/codegen.py | 40 ++++++++++++++++++++++++++++++---------- pytato/program.py | 16 +++++++++++++--- pytato/target.py | 8 ++++---- 3 files changed, 47 insertions(+), 17 deletions(-) diff --git a/pytato/codegen.py b/pytato/codegen.py index fa17706..26931dd 100644 --- a/pytato/codegen.py +++ b/pytato/codegen.py @@ -450,9 +450,9 @@ class CodeGenState: The (global) namespace - .. attribute:: kernel + .. attribute:: _program - The partial :class:`loopy.LoopKernel` being built. + The partial :class:`loopy.LoopKernel` or :class:`loopy.Program` being built. .. attribute:: results @@ -465,22 +465,40 @@ class CodeGenState: .. automethod:: update_kernel """ namespace: Mapping[str, Array] - _kernel: lp.LoopKernel + _program: Union["lp.Program", lp.LoopKernel] results: Dict[Array, ImplementedResult] var_name_gen: pytools.UniqueNameGenerator = dataclasses.field(init=False) insn_id_gen: pytools.UniqueNameGenerator = dataclasses.field(init=False) def __post_init__(self) -> None: - self.var_name_gen = self._kernel.get_var_name_generator() - self.insn_id_gen = self._kernel.get_instruction_id_generator() + if isinstance(self._program, lp.LoopKernel): + self.var_name_gen = self._program.get_var_name_generator() + self.insn_id_gen = self._program.get_instruction_id_generator() + else: + self.var_name_gen = self._program["_pt_kernel"].get_var_name_generator() + self.insn_id_gen = ( + self._program["_pt_kernel"].get_instruction_id_generator()) + + @property + def program(self) -> Union["lp.Program", lp.LoopKernel]: + return self._program @property def kernel(self) -> lp.LoopKernel: - return self._kernel + """ + Returns the entry kernel of the loopy program being built. + """ + if isinstance(self._program, lp.LoopKernel): + return self._program + else: + return self._program["_pt_kernel"] def update_kernel(self, kernel: lp.LoopKernel) -> None: - self._kernel = kernel + if isinstance(self._program, lp.LoopKernel): + self._program = kernel + else: + self._program = self._program.with_kernel(kernel) class CodeGenMapper(Mapper): @@ -846,12 +864,13 @@ def normalize_outputs(result: Union[Array, DictOfNamedArrays, def get_initial_codegen_state(namespace: Namespace, target: Target, options: Optional[lp.Options]) -> CodeGenState: kernel = lp.make_kernel("{:}", [], + name="_pt_kernel", target=target.get_loopy_target(), options=options, lang_version=lp.MOST_RECENT_LANGUAGE_VERSION) return CodeGenState(namespace=namespace, - _kernel=kernel, + _program=kernel, results=dict()) @@ -908,7 +927,8 @@ def generate_loopy(result: Union[Array, DictOfNamedArrays, Dict[str, Array]], :param result: Outputs of the computation. :param target: Code generation target. :param options: Code generation options for the kernel. - :returns: A wrapped generated :mod:`loopy` kernel + :returns: A :class:`pytato.program.BoundProgram` wrapping the generated + :mod:`loopy` program. """ orig_outputs: DictOfNamedArrays = normalize_outputs(result) del result @@ -944,5 +964,5 @@ def generate_loopy(result: Union[Array, DictOfNamedArrays, Dict[str, Array]], state.results[expr] = StoredResult(name, expr.ndim, frozenset([insn_id])) return target.bind_program( - program=state.kernel, + program=state.program, bound_arguments=preproc_result.bound_arguments) diff --git a/pytato/program.py b/pytato/program.py index 2894332..408ef15 100644 --- a/pytato/program.py +++ b/pytato/program.py @@ -34,7 +34,7 @@ Generated Executable Programs from dataclasses import dataclass import typing -from typing import Any, Mapping, Optional +from typing import Any, Mapping, Optional, Union import loopy @@ -52,7 +52,7 @@ class BoundProgram: .. attribute:: program - The underlying :class:`loopy.LoopKernel`. + The underlying :class:`loopy.Program`. .. attribute:: target @@ -65,13 +65,20 @@ class BoundProgram: .. automethod:: __call__ """ - program: "loopy.LoopKernel" + program: Union["loopy.LoopKernel", "loopy.Program"] bound_arguments: Mapping[str, Any] target: "pytato.target.Target" def __call__(self, *args: Any, **kwargs: Any) -> Any: raise NotImplementedError + @property + def kernel(self) -> "loopy.LoopKernel": + if isinstance(self.program, loopy.LoopKernel): + return self.program + else: + return self.program["_pt_kernel"] + @dataclass(init=True, repr=False, eq=False) class BoundPyOpenCLProgram(BoundProgram): @@ -92,6 +99,9 @@ class BoundPyOpenCLProgram(BoundProgram): updated_kwargs = dict(self.bound_arguments) updated_kwargs.update(kwargs) + if not isinstance(self. program, loopy.LoopKernel): + updated_kwargs.setdefault("entrypoint", "_pt_kernel") + return self.program(self.queue, *args, **updated_kwargs) # vim: foldmethod=marker diff --git a/pytato/target.py b/pytato/target.py index 72e819b..162591f 100644 --- a/pytato/target.py +++ b/pytato/target.py @@ -32,7 +32,7 @@ Code Generation Targets .. autoclass:: PyOpenCLTarget """ -from typing import Any, Mapping, Optional +from typing import Any, Mapping, Optional, Union from pytato.program import BoundProgram, BoundPyOpenCLProgram @@ -51,11 +51,11 @@ class Target: """Return the corresponding :mod:`loopy` target.""" raise NotImplementedError - def bind_program(self, program: "loopy.LoopKernel", + def bind_program(self, program: Union["loopy.Program", "loopy.LoopKernel"], bound_arguments: Mapping[str, Any]) -> BoundProgram: """Create a :class:`pytato.program.BoundProgram` for this code generation target. - :param program: the :mod:`loopy` kernel + :param program: the :mod:`loopy` program :param bound_arguments: a mapping from argument names to outputs """ raise NotImplementedError @@ -79,7 +79,7 @@ class PyOpenCLTarget(Target): device = self.queue.device return lp.PyOpenCLTarget(device) - def bind_program(self, program: "loopy.LoopKernel", + def bind_program(self, program: Union["loopy.Program", "loopy.LoopKernel"], bound_arguments: Mapping[str, Any]) -> BoundProgram: return BoundPyOpenCLProgram(program=program, queue=self.queue, -- GitLab From 8950f6eef6380b8f2112c6325eb53f5417afa195 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Sat, 9 Jan 2021 19:45:52 -0600 Subject: [PATCH 2/6] account for syntax change for getting the entrypoint kernel --- test/test_codegen.py | 4 ++-- test/test_pytato.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_codegen.py b/test/test_codegen.py index 7d389b0..f41bbd3 100755 --- a/test/test_codegen.py +++ b/test/test_codegen.py @@ -439,7 +439,7 @@ def test_dict_of_named_array_codegen_avoids_recomputation(): yz = pt.DictOfNamedArrays({"y": y, "z": z}) - knl = pt.generate_loopy(yz).program + knl = pt.generate_loopy(yz).kernel assert ("y" in knl.id_to_insn["z_store"].read_dependency_names()) @@ -470,7 +470,7 @@ def test_only_deps_as_knl_args(): y = pt.make_placeholder(ns, name="y", shape=(10, 4), dtype=float) # noqa:F841 z = 2*x - knl = pt.generate_loopy(z).program + knl = pt.generate_loopy(z).kernel assert "x" in knl.arg_dict assert "y" not in knl.arg_dict diff --git a/test/test_pytato.py b/test/test_pytato.py index 77050e1..6804b0a 100755 --- a/test/test_pytato.py +++ b/test/test_pytato.py @@ -119,7 +119,7 @@ def test_make_placeholder_noname(): x = pt.make_placeholder(ns, shape=(10, 4), dtype=float) y = 2*x - knl = pt.generate_loopy(y).program + knl = pt.generate_loopy(y).kernel assert x.name in knl.arg_dict assert x.name in knl.get_read_variables() @@ -132,7 +132,7 @@ def test_zero_length_arrays(): assert y.shape == (0, 4) - knl = pt.generate_loopy(y).program + knl = pt.generate_loopy(y).kernel assert all(dom.is_empty() for dom in knl.domains if dom.total_dim() != 0) -- GitLab From 3dee694bd7f11ce98d4dd7907e3458f9aa2128cf Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Sat, 9 Jan 2021 19:46:12 -0600 Subject: [PATCH 3/6] test both branches of loopy --- .github/workflows/ci.yml | 16 ++++++++++++++-- .gitlab-ci.yml | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78e6f24..f0e1e42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,8 +55,8 @@ jobs: curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/prepare-and-run-mypy.sh . ./prepare-and-run-mypy.sh python3 mypy - pytest: - name: Conda Pytest + pytest-loopymaster: + name: Conda Pytest (Loopy master) runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -66,6 +66,18 @@ jobs: curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-and-test-py-project-within-miniconda.sh . ./build-and-test-py-project-within-miniconda.sh + pytest-loopycallables: + name: Conda Pytest (Loopy Kernel Callables) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: "Main Script" + run: | + sed -i "s/loopy.git/loopy.git@kernel_callables_v3-edit2/g" requirements.txt + CONDA_ENVIRONMENT=.test-conda-env-py3.yml + curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-and-test-py-project-within-miniconda.sh + . ./build-and-test-py-project-within-miniconda.sh + examples: name: Conda Examples runs-on: ubuntu-latest diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index da24af2..aa851ea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,24 @@ Python 3 POCL: junit: test/pytest.xml +Python 3 POCL (Loopy callables): + script: + - sed -i "s/loopy.git/loopy.git@kernel_callables_v3-edit2/g" requirements.txt + - export PY_EXE=python3 + - export PYOPENCL_TEST=portable:pthread + - export EXTRA_INSTALL="pyopencl" + - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-and-test-py-project.sh + - ". ./build-and-test-py-project.sh" + tags: + - python3 + - pocl + except: + - tags + artifacts: + reports: + junit: test/pytest.xml + + Python 3 POCL Examples: script: - export PY_EXE=python3 -- GitLab From dc3b5f0236cfae97ebf1c5bf345e0bd66ef696a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Sat, 9 Jan 2021 23:36:00 -0600 Subject: [PATCH 4/6] Add genpy to requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7548340..9de0916 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ git+https://github.com/inducer/pytools.git#egg=pytools +git+https://github.com/inducer/genpy.git#egg=genpy git+https://github.com/inducer/loopy.git#egg=loopy -- GitLab From 199c9c5a42b44e99bbcbb948b1d571c58a4f6c5c Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Sun, 10 Jan 2021 12:43:40 -0600 Subject: [PATCH 5/6] github CI: select loopy branch via a test matrix --- .github/workflows/ci.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0e1e42..94e11a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,25 +55,17 @@ jobs: curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/prepare-and-run-mypy.sh . ./prepare-and-run-mypy.sh python3 mypy - pytest-loopymaster: - name: Conda Pytest (Loopy master) + pytest: + name: Conda Pytest runs-on: ubuntu-latest + strategy: + matrix: + loopy-branch: [master, kernel_callables_v3-edit2] steps: - uses: actions/checkout@v2 - name: "Main Script" run: | - CONDA_ENVIRONMENT=.test-conda-env-py3.yml - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-and-test-py-project-within-miniconda.sh - . ./build-and-test-py-project-within-miniconda.sh - - pytest-loopycallables: - name: Conda Pytest (Loopy Kernel Callables) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: "Main Script" - run: | - sed -i "s/loopy.git/loopy.git@kernel_callables_v3-edit2/g" requirements.txt + sed -i "s/loopy.git/loopy.git@${{ matrix.loopy-branch }}/g" requirements.txt CONDA_ENVIRONMENT=.test-conda-env-py3.yml curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-and-test-py-project-within-miniconda.sh . ./build-and-test-py-project-within-miniconda.sh @@ -81,10 +73,14 @@ jobs: examples: name: Conda Examples runs-on: ubuntu-latest + strategy: + matrix: + loopy-branch: [master, kernel_callables_v3-edit2] steps: - uses: actions/checkout@v2 - name: "Main Script" run: | + sed -i "s/loopy.git/loopy.git@${{ matrix.loopy-branch }}/g" requirements.txt CONDA_ENVIRONMENT=.test-conda-env-py3.yml curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/ci-support.sh . ci-support.sh -- GitLab From d2170e516c277dbc5ae237a88a4fabe36bbe963a Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Sun, 10 Jan 2021 13:04:28 -0600 Subject: [PATCH 6/6] gitlab CI: use a test matrix for selecting loopy branches (see https://docs.gitlab.com/ee/ci/yaml/#parallel-matrix-jobs) --- .gitlab-ci.yml | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aa851ea..c5bbb7d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,6 @@ Python 3 POCL: script: + - sed -i "s/loopy.git/loopy.git@$LOOPY_BRANCH/g" requirements.txt - export PY_EXE=python3 - export PYOPENCL_TEST=portable:pthread - export EXTRA_INSTALL="pyopencl" @@ -14,27 +15,15 @@ Python 3 POCL: reports: junit: test/pytest.xml - -Python 3 POCL (Loopy callables): - script: - - sed -i "s/loopy.git/loopy.git@kernel_callables_v3-edit2/g" requirements.txt - - export PY_EXE=python3 - - export PYOPENCL_TEST=portable:pthread - - export EXTRA_INSTALL="pyopencl" - - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-and-test-py-project.sh - - ". ./build-and-test-py-project.sh" - tags: - - python3 - - pocl - except: - - tags - artifacts: - reports: - junit: test/pytest.xml + parallel: + matrix: + - LOOPY_BRANCH: master + - LOOPY_BRANCH: kernel_callables_v3-edit2 Python 3 POCL Examples: script: + - sed -i "s/loopy.git/loopy.git@$LOOPY_BRANCH/g" requirements.txt - export PY_EXE=python3 - export PYOPENCL_TEST=portable:pthread - export EXTRA_INSTALL="pyopencl" @@ -46,6 +35,10 @@ Python 3 POCL Examples: - large-node except: - tags + parallel: + matrix: + - LOOPY_BRANCH: master + - LOOPY_BRANCH: kernel_callables_v3-edit2 Pylint: -- GitLab