From 2e03c0c4fbb583f71999dab1c7dd735c6ccefd9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Thu, 16 Aug 2018 23:13:11 -0400 Subject: [PATCH 01/10] [WIP] Try to fix CIs on distributed-fmm-global branch --- .gitlab-ci.yml | 18 ++++++++++++++---- test/test_distributed.py | 4 ++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b99942..40adf72 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,6 @@ Python 3.5 K40: - nvidia-k40 except: - tags - - distributed-fmm-global Python 2.7 POCL: script: @@ -38,7 +37,6 @@ Python 2.7 POCL: - pocl except: - tags - - distributed-fmm-global Python 3.5 POCL: script: @@ -52,7 +50,6 @@ Python 3.5 POCL: - pocl except: - tags - - distributed-fmm-global Python 3.6 POCL: script: @@ -66,7 +63,20 @@ Python 3.6 POCL: - pocl except: - tags - - distributed-fmm-global + +Python 3.6 POCL MPI: + script: + - export PY_EXE=python3.6 + - export PYOPENCL_TEST=portable + - export EXTRA_INSTALL="numpy mako mpi4py" + - export PYTEST_ADDOPTS="-k mpi" + - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-and-test-py-project.sh + - ". ./build-and-test-py-project.sh" + tags: + - python3.6 + - pocl + except: + - tags Documentation: script: diff --git a/test/test_distributed.py b/test/test_distributed.py index b320c0c..333e77e 100644 --- a/test/test_distributed.py +++ b/test/test_distributed.py @@ -1,6 +1,5 @@ import numpy as np import pyopencl as cl -from mpi4py import MPI from boxtree.distributed.calculation import DistributedFMMLibExpansionWrangler from boxtree.distributed import DistributedFMMInfo import numpy.linalg as la @@ -15,7 +14,8 @@ logging.getLogger("boxtree.distributed").setLevel(logging.INFO) def _test_against_shared(dims, nsources, ntargets, dtype): - + from mpi4py import MPI + # Get the current rank comm = MPI.COMM_WORLD rank = comm.Get_rank() -- GitLab From b357b38384f446f88f48f636b04d73e69fb97337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Thu, 16 Aug 2018 23:20:26 -0400 Subject: [PATCH 02/10] Use setuptools.find_packages() in setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 32cca6a..eb8bbbb 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ def main(): - from setuptools import setup + from setuptools import setup, find_packages version_dict = {} init_filename = "boxtree/version.py" @@ -39,7 +39,7 @@ def main(): 'Topic :: Utilities', ], - packages=["boxtree"], + packages=find_packages(), install_requires=[ "pytools>=2018.4", "pyopencl>=2013.1", -- GitLab From 8886b41fbf3aac4ec03690e1135b4e17e1120109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Thu, 16 Aug 2018 23:23:36 -0400 Subject: [PATCH 03/10] Flake8 fixes --- test/test_distributed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_distributed.py b/test/test_distributed.py index 333e77e..6a1f5ff 100644 --- a/test/test_distributed.py +++ b/test/test_distributed.py @@ -15,7 +15,7 @@ logging.getLogger("boxtree.distributed").setLevel(logging.INFO) def _test_against_shared(dims, nsources, ntargets, dtype): from mpi4py import MPI - + # Get the current rank comm = MPI.COMM_WORLD rank = comm.Get_rank() @@ -274,6 +274,7 @@ class ConstantOneExpansionWrangler(object): def _test_constantone(dims, nsources, ntargets, dtype): + from mpi4py import MPI # Get the current rank comm = MPI.COMM_WORLD -- GitLab From 8d75e57515e9521d5f75e4c8ce8824bced336e37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Thu, 16 Aug 2018 23:34:03 -0400 Subject: [PATCH 04/10] Localize a few more imports in test_distributed.py --- test/test_distributed.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test_distributed.py b/test/test_distributed.py index 6a1f5ff..b79082d 100644 --- a/test/test_distributed.py +++ b/test/test_distributed.py @@ -1,7 +1,5 @@ import numpy as np import pyopencl as cl -from boxtree.distributed.calculation import DistributedFMMLibExpansionWrangler -from boxtree.distributed import DistributedFMMInfo import numpy.linalg as la from boxtree.pyfmmlib_integration import FMMLibExpansionWrangler import logging @@ -71,9 +69,12 @@ def _test_against_shared(dims, nsources, ntargets, dtype): # Compute FMM using distributed memory parallelism def distributed_expansion_wrangler_factory(tree): + from boxtree.distributed.calculation import DistributedFMMLibExpansionWrangler + return DistributedFMMLibExpansionWrangler( queue, tree, helmholtz_k, fmm_level_to_nterms=fmm_level_to_nterms) + from boxtree.distributed import DistributedFMMInfo distribued_fmm_info = DistributedFMMInfo( queue, trav, distributed_expansion_wrangler_factory, comm=comm) pot_dfmm = distribued_fmm_info.drive_dfmm(sources_weights) -- GitLab From a33ba810bc03d2da1f4d9618999f5d5ee59c28e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Thu, 16 Aug 2018 23:39:18 -0400 Subject: [PATCH 05/10] Flake 8 fix --- test/test_distributed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_distributed.py b/test/test_distributed.py index b79082d..300cef9 100644 --- a/test/test_distributed.py +++ b/test/test_distributed.py @@ -69,7 +69,8 @@ def _test_against_shared(dims, nsources, ntargets, dtype): # Compute FMM using distributed memory parallelism def distributed_expansion_wrangler_factory(tree): - from boxtree.distributed.calculation import DistributedFMMLibExpansionWrangler + from boxtree.distributed.calculation import \ + DistributedFMMLibExpansionWrangler return DistributedFMMLibExpansionWrangler( queue, tree, helmholtz_k, fmm_level_to_nterms=fmm_level_to_nterms) -- GitLab From 1de75ce3b9c865927f63c9533d5b1c7db8549e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Fri, 17 Aug 2018 01:31:29 -0400 Subject: [PATCH 06/10] Add PY_EXTRA_FLAGS="-m mpi4py.run" to help avoid stuck MPI-based CIs --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 40adf72..3494946 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,6 +70,7 @@ Python 3.6 POCL MPI: - export PYOPENCL_TEST=portable - export EXTRA_INSTALL="numpy mako mpi4py" - export PYTEST_ADDOPTS="-k mpi" + - export PY_EXTRA_FLAGS="-m mpi4py.run" - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-and-test-py-project.sh - ". ./build-and-test-py-project.sh" tags: -- GitLab From 2446df2291c52e953457a299206f38286aaf1838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Fri, 17 Aug 2018 01:50:38 -0400 Subject: [PATCH 07/10] Use -m mpi4py.run in spawning subprocesses --- .gitlab-ci.yml | 1 - test/test_distributed.py | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3494946..40adf72 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,7 +70,6 @@ Python 3.6 POCL MPI: - export PYOPENCL_TEST=portable - export EXTRA_INSTALL="numpy mako mpi4py" - export PYTEST_ADDOPTS="-k mpi" - - export PY_EXTRA_FLAGS="-m mpi4py.run" - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-and-test-py-project.sh - ". ./build-and-test-py-project.sh" tags: diff --git a/test/test_distributed.py b/test/test_distributed.py index 300cef9..aa50100 100644 --- a/test/test_distributed.py +++ b/test/test_distributed.py @@ -105,7 +105,8 @@ def test_against_shared(num_processes, dims, nsources, ntargets): subprocess.run([ "mpiexec", "-np", str(num_processes), "-x", "PYTEST", "-x", "dims", "-x", "nsources", "-x", "ntargets", - sys.executable, __file__], + # https://mpi4py.readthedocs.io/en/stable/mpi4py.run.html + sys.executable, "-m", "mpi4py.run", __file__], env=newenv, check=True ) @@ -349,7 +350,8 @@ def test_constantone(num_processes, dims, nsources, ntargets): subprocess.run([ "mpiexec", "-np", str(num_processes), "-x", "PYTEST", "-x", "dims", "-x", "nsources", "-x", "ntargets", - sys.executable, __file__], + # https://mpi4py.readthedocs.io/en/stable/mpi4py.run.html + sys.executable, "-m", "mpi4py.run", __file__], env=newenv, check=True ) -- GitLab From c92c81003994b25a337bafc12b041966e61f13c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Fri, 17 Aug 2018 01:58:19 -0400 Subject: [PATCH 08/10] Turn off output capture (for now) on MPI test --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 40adf72..358c7fe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -69,7 +69,7 @@ Python 3.6 POCL MPI: - export PY_EXE=python3.6 - export PYOPENCL_TEST=portable - export EXTRA_INSTALL="numpy mako mpi4py" - - export PYTEST_ADDOPTS="-k mpi" + - export PYTEST_ADDOPTS="-k mpi --capture=no" - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-and-test-py-project.sh - ". ./build-and-test-py-project.sh" tags: -- GitLab From d0ddf01ec13d9094dda9d443b5f5494f0f8373d5 Mon Sep 17 00:00:00 2001 From: Hao Gao Date: Fri, 17 Aug 2018 23:24:16 -0500 Subject: [PATCH 09/10] Add default_perf_model.json to package_data --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb8bbbb..9b424e1 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,12 @@ def main(): "pytest>=2.3", "cgen>=2013.1.2", "six", - ]) + ], + package_data={ + "boxtree": [ + "distributed/default_perf_model.json" + ] + }) if __name__ == '__main__': -- GitLab From 5d511f6072315af1f42e59b3a76fc3c8208d7f4e Mon Sep 17 00:00:00 2001 From: Hao Gao Date: Fri, 17 Aug 2018 23:31:20 -0500 Subject: [PATCH 10/10] Force OpenMP to use 1 thread in test cases --- test/test_distributed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_distributed.py b/test/test_distributed.py index 8ec032a..c4a2210 100644 --- a/test/test_distributed.py +++ b/test/test_distributed.py @@ -101,6 +101,7 @@ def test_against_shared(num_processes, dims, nsources, ntargets): newenv["dims"] = str(dims) newenv["nsources"] = str(nsources) newenv["ntargets"] = str(ntargets) + newenv["OMP_NUM_THREADS"] = "1" import subprocess import sys @@ -140,7 +141,6 @@ def _test_constantone(dims, nsources, ntargets, dtype): import pyopencl as cl ctx = cl.create_some_context() queue = cl.CommandQueue(ctx) - print(queue.context.devices) if rank == 0: @@ -193,6 +193,7 @@ def test_constantone(num_processes, dims, nsources, ntargets): newenv["dims"] = str(dims) newenv["nsources"] = str(nsources) newenv["ntargets"] = str(ntargets) + newenv["OMP_NUM_THREADS"] = "1" import subprocess import sys -- GitLab