From d3d4833a68132ca6b454b7734f7ad42a6292408b Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 13 Jul 2020 16:26:20 -0500 Subject: [PATCH 01/16] Fixes for api changes --- experiments/volume_potential_3d_derivatives.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/experiments/volume_potential_3d_derivatives.py b/experiments/volume_potential_3d_derivatives.py index 023871c..db78c3a 100644 --- a/experiments/volume_potential_3d_derivatives.py +++ b/experiments/volume_potential_3d_derivatives.py @@ -323,9 +323,11 @@ local_expn_class = expn_factory.get_local_expansion_class(knl) mpole_expn_class = expn_factory.get_multipole_expansion_class(knl) exclude_self = True -from volumential.expansion_wrangler_interface import ExpansionWranglerCodeContainer +from volumential.expansion_wrangler_fpnd import ( + FPNDExpansionWranglerCodeContainer, + FPNDExpansionWrangler) -wcc = ExpansionWranglerCodeContainer( +wcc = FPNDExpansionWranglerCodeContainer( ctx, partial(mpole_expn_class, knl), partial(local_expn_class, knl), @@ -339,8 +341,6 @@ if exclude_self: else: self_extra_kwargs = {} -from volumential.expansion_wrangler_fpnd import FPNDExpansionWrangler - wrangler = FPNDExpansionWrangler( code_container=wcc, queue=queue, -- GitLab From c1c27ad45f6b459b8b9f869a89cab388532ef130 Mon Sep 17 00:00:00 2001 From: xywei Date: Tue, 14 Jul 2020 14:00:57 -0500 Subject: [PATCH 02/16] Fix random polynomial generation in test suits --- test/test_interpolation.py | 4 +++- volumential/tools.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/test/test_interpolation.py b/test/test_interpolation.py index 07fffa7..bcc74a4 100644 --- a/test/test_interpolation.py +++ b/test/test_interpolation.py @@ -50,8 +50,10 @@ def random_polynomial_func(dim, degree, seed=None): npts = pts.shape[1] res = np.zeros(npts) for deg in np.ndindex(coefs.shape): + mono = np.ones(npts) for iaxis in range(dim): - res += coefs[deg] * pts[iaxis, :]**deg[iaxis] + mono += pts[iaxis, :]**deg[iaxis] + res += coefs[deg] * mono return res return poly_func diff --git a/volumential/tools.py b/volumential/tools.py index 6b24224..b189e26 100644 --- a/volumential/tools.py +++ b/volumential/tools.py @@ -528,6 +528,18 @@ class DiscreteLegendreTransform(BoxSpecificMap): # }}} End discrete Legendre transform +# {{{ inverse discrete Legendre transform + + +class InverseDiscreteLegendreTransform(BoxSpecificMap): + """ + Box-specific transform that maps box-local modal coefficients + to nodal values. Inverse of :class:`DiscreteLegendreTransform`. + """ + pass + +# }}} End inverse discrete Legendre transform + # }}} End box-specific maps # {{{ box-specific reductions -- GitLab From 42159294434cff02b7b9bad5880213c811f7d933 Mon Sep 17 00:00:00 2001 From: xywei Date: Tue, 14 Jul 2020 15:45:15 -0500 Subject: [PATCH 03/16] Formatting --- volumential/tools.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volumential/tools.py b/volumential/tools.py index b189e26..146c6e1 100644 --- a/volumential/tools.py +++ b/volumential/tools.py @@ -496,14 +496,13 @@ class DiscreteLegendreTransform(BoxSpecificMap): """ if filtering is None: - filter_multiplier = 1 + cl.array.zeros(queue, - self.degree**self.dim, np.float64) + filter_multiplier = 1 + cl.array.zeros( + queue, self.degree**self.dim, np.float64) elif isinstance(filtering, cl.array.Array): assert filtering.shape == (self.degree**self.dim,) filter_multiplier = filtering else: - raise RuntimeError("Invalid filtering argument: %s" - % str(filtering)) + raise RuntimeError(f"Invalid filtering argument: {str(filtering)}") knl = self.get_cached_optimized_kernel() -- GitLab From 68b684e943a46395e8b9a618cebaf66825c22289 Mon Sep 17 00:00:00 2001 From: xywei Date: Tue, 14 Jul 2020 20:16:31 -0500 Subject: [PATCH 04/16] Conda deps: +openmpi-mpicxx --- .test-conda-env-py3-macos.yml | 1 + .test-conda-env-py3.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.test-conda-env-py3-macos.yml b/.test-conda-env-py3-macos.yml index 54ea6a4..3c0ef9c 100644 --- a/.test-conda-env-py3-macos.yml +++ b/.test-conda-env-py3-macos.yml @@ -26,6 +26,7 @@ dependencies: - libgfortran>=3.0.1 - clangdev - openmp + - openmpi-mpicxx - pip - pip: diff --git a/.test-conda-env-py3.yml b/.test-conda-env-py3.yml index 37f7c40..2e208de 100644 --- a/.test-conda-env-py3.yml +++ b/.test-conda-env-py3.yml @@ -7,6 +7,7 @@ dependencies: - c-compiler - binutils - openmp + - openmpi-mpicxx - scipy - conda-forge::numpy - conda-forge::sympy -- GitLab From bdcd3412b51a3132dd4582131dc7dada5d22e72f Mon Sep 17 00:00:00 2001 From: xywei Date: Tue, 14 Jul 2020 20:51:10 -0500 Subject: [PATCH 05/16] Add conda deps: +fftw --- .test-conda-env-py3-macos.yml | 2 ++ .test-conda-env-py3.yml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.test-conda-env-py3-macos.yml b/.test-conda-env-py3-macos.yml index 3c0ef9c..29d474a 100644 --- a/.test-conda-env-py3-macos.yml +++ b/.test-conda-env-py3-macos.yml @@ -26,7 +26,9 @@ dependencies: - libgfortran>=3.0.1 - clangdev - openmp + # for pypvfmm - openmpi-mpicxx + - fftw - pip - pip: diff --git a/.test-conda-env-py3.yml b/.test-conda-env-py3.yml index 2e208de..f676d3c 100644 --- a/.test-conda-env-py3.yml +++ b/.test-conda-env-py3.yml @@ -7,7 +7,6 @@ dependencies: - c-compiler - binutils - openmp - - openmpi-mpicxx - scipy - conda-forge::numpy - conda-forge::sympy @@ -24,6 +23,9 @@ dependencies: - dealii - gmsh - python-gmsh + # for pypvfmm + - openmpi-mpicxx + - fftw - pip - pip: -- GitLab From 26982622d627c192857f67fe05dab0066f94bd86 Mon Sep 17 00:00:00 2001 From: xywei Date: Tue, 14 Jul 2020 20:53:45 -0500 Subject: [PATCH 06/16] Remove table cache (might be from previously failed tests) before testing --- test/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index dcd9a6a..272d662 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -65,6 +65,8 @@ def pytest_sessionstart(session): # can be easily turned off to run individual tests that do not require # the table cache. if 1: + import subprocess + subprocess.call(['rm', '-f', '/tmp/volumential-tests.hdf5']) with NearFieldInteractionTableManager( "/tmp/volumential-tests.hdf5") as tm: table, _ = tm.get_table( -- GitLab From 03059d1ab06bc9518b8ecda801f4424c865aa7a5 Mon Sep 17 00:00:00 2001 From: xywei Date: Tue, 14 Jul 2020 21:38:51 -0500 Subject: [PATCH 07/16] Pre-compute 5 orders on test sessionstart --- test/conftest.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 272d662..fa4a483 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -61,17 +61,18 @@ def pytest_sessionstart(session): import subprocess subprocess.call(['rm', '-f', '/tmp/volumential-tests.hdf5']) - # pre-compute a basic table that is re-used in many tests. - # can be easily turned off to run individual tests that do not require - # the table cache. + # Pre-compute tables that is re-used in the tests. + # Updating the table file from different tests may result in file + # corruption. if 1: import subprocess subprocess.call(['rm', '-f', '/tmp/volumential-tests.hdf5']) - with NearFieldInteractionTableManager( - "/tmp/volumential-tests.hdf5") as tm: - table, _ = tm.get_table( - 2, "Laplace", - q_order=1, force_recompute=False, queue=None) + for q_order in [1, 2, 3, 4, 5]: + with NearFieldInteractionTableManager( + "/tmp/volumential-tests.hdf5") as tm: + table, _ = tm.get_table( + 2, "Laplace", q_order=q_order, + force_recompute=False, queue=None) def pytest_sessionfinish(session, exitstatus): -- GitLab From 3c712826ce317c512fc7b620314b5b7a96ae4744 Mon Sep 17 00:00:00 2001 From: xywei Date: Tue, 14 Jul 2020 21:47:47 -0500 Subject: [PATCH 08/16] Add deps: +autotools, +pytest-cov --- .test-conda-env-py3-macos.yml | 7 ++++++- .test-conda-env-py3.yml | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.test-conda-env-py3-macos.yml b/.test-conda-env-py3-macos.yml index 29d474a..1735795 100644 --- a/.test-conda-env-py3-macos.yml +++ b/.test-conda-env-py3-macos.yml @@ -5,6 +5,8 @@ channels: dependencies: - git - c-compiler + - pytest + - pytest-cov - binutils - scipy - conda-forge::numpy @@ -29,11 +31,14 @@ dependencies: # for pypvfmm - openmpi-mpicxx - fftw + - autoconf + - automake + - libtool - pip - pip: - git+https://gitlab.tiker.net/inducer/loopy - - git+https://www.github.com/xywei/gmsh_interop + - git+https://gitlab.tiker.net/inducer/gmsh_interop - git+https://gitlab.tiker.net/xywei/boxtree - git+https://gitlab.tiker.net/inducer/modepy - git+https://gitlab.tiker.net/inducer/meshmode diff --git a/.test-conda-env-py3.yml b/.test-conda-env-py3.yml index f676d3c..d579709 100644 --- a/.test-conda-env-py3.yml +++ b/.test-conda-env-py3.yml @@ -10,6 +10,8 @@ dependencies: - scipy - conda-forge::numpy - conda-forge::sympy + - pytest + - pytest-cov - pocl - islpy - pyopencl @@ -30,7 +32,7 @@ dependencies: - pip - pip: - git+https://gitlab.tiker.net/inducer/loopy - - git+https://www.github.com/xywei/gmsh_interop + - git+https://gitlab.tiker.net/inducer/gmsh_interop - git+https://gitlab.tiker.net/xywei/boxtree - git+https://gitlab.tiker.net/inducer/modepy - git+https://gitlab.tiker.net/inducer/meshmode -- GitLab From b064559ee8f3ac08a58d0bd996cc7c1511c11ff0 Mon Sep 17 00:00:00 2001 From: xywei Date: Tue, 14 Jul 2020 22:24:08 -0500 Subject: [PATCH 09/16] Remove pypvfmm in CI tests --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1a2bf60..d435fe1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ --index-url https://pypi.python.org/simple/ -git+https://gitlab.tiker.net/xywei/pypvfmm.git#egg=pypvfmm +# git+https://gitlab.tiker.net/xywei/pypvfmm.git#egg=pypvfmm -e git+https://gitlab.tiker.net/xywei/boxtree.git#egg=boxtree -e git+https://gitlab.tiker.net/inducer/gmsh_interop.git#egg=gmsh_interop -- GitLab From eb8cca8e5e287306d27aa94b27b03355f37ef364 Mon Sep 17 00:00:00 2001 From: xywei Date: Tue, 14 Jul 2020 23:55:52 -0500 Subject: [PATCH 10/16] Cut back on precomputation after disabling pypvfmm tests --- test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index fa4a483..f6dc07d 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -67,7 +67,7 @@ def pytest_sessionstart(session): if 1: import subprocess subprocess.call(['rm', '-f', '/tmp/volumential-tests.hdf5']) - for q_order in [1, 2, 3, 4, 5]: + for q_order in [1, ]: with NearFieldInteractionTableManager( "/tmp/volumential-tests.hdf5") as tm: table, _ = tm.get_table( -- GitLab From f78b2fe8fe3fcd6fc57aa44a52517f99fa432af4 Mon Sep 17 00:00:00 2001 From: xywei Date: Wed, 15 Jul 2020 08:22:23 -0500 Subject: [PATCH 11/16] Make copies of the precomputed table to avoid racing (due to pytest-xdist) --- test/test_table_manager.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_table_manager.py b/test/test_table_manager.py index e78660f..4087d13 100644 --- a/test/test_table_manager.py +++ b/test/test_table_manager.py @@ -22,16 +22,21 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import os import numpy as np import pyopencl as cl import pytest import volumential as vm +from shutil import copyfile from volumential.table_manager import ( NearFieldInteractionTableManager as NFTable) def get_table(queue, q_order=1, dim=2): - with NFTable("/tmp/volumential-tests.hdf5") as table_manager: + pid = os.getpid() + copyfile("/tmp/volumential-tests.hdf5", + f"/tmp/volumential-tests-{pid}.hdf5") + with NFTable(f"/tmp/volumential-tests-{pid}.hdf5") as table_manager: table, _ = table_manager.get_table( dim, "Laplace", q_order=q_order, force_recompute=False, queue=queue) -- GitLab From e165b997b2a3235f1d33e586b2bc21d566f8406c Mon Sep 17 00:00:00 2001 From: xywei Date: Wed, 15 Jul 2020 09:10:00 -0500 Subject: [PATCH 12/16] Reorganize table caches during testing, and add progress_bar=True/False option to table manager --- test/conftest.py | 27 +++-------------- ...test_nearfield_interaction_completeness.py | 23 +++++++------- test/test_nearfield_potential_table.py | 10 ++++--- test/test_table_manager.py | 21 +++++++++++-- test/test_volume_fmm.py | 22 ++++++-------- volumential/nearfield_potential_table.py | 30 ++++++++++++------- 6 files changed, 66 insertions(+), 67 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index f6dc07d..5268293 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -22,6 +22,8 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import subprocess +import glob import pytest # noqa: F401 # setup the ctx_factory fixture @@ -54,28 +56,7 @@ def requires_pypvfmm(request): pytest.skip("needs pypvfmm to run") -def pytest_sessionstart(session): - from volumential.table_manager import NearFieldInteractionTableManager - - # clean the table file, in case there was an aborted previous test run - import subprocess - subprocess.call(['rm', '-f', '/tmp/volumential-tests.hdf5']) - - # Pre-compute tables that is re-used in the tests. - # Updating the table file from different tests may result in file - # corruption. - if 1: - import subprocess - subprocess.call(['rm', '-f', '/tmp/volumential-tests.hdf5']) - for q_order in [1, ]: - with NearFieldInteractionTableManager( - "/tmp/volumential-tests.hdf5") as tm: - table, _ = tm.get_table( - 2, "Laplace", q_order=q_order, - force_recompute=False, queue=None) - - def pytest_sessionfinish(session, exitstatus): # remove table caches - import subprocess - subprocess.call(['rm', '-f', '/tmp/volumential-tests.hdf5']) + for table_file in glob.glob("*.hdf5"): + subprocess.call(['rm', '-f', table_file]) diff --git a/test/test_nearfield_interaction_completeness.py b/test/test_nearfield_interaction_completeness.py index 32fafa6..edc959e 100644 --- a/test/test_nearfield_interaction_completeness.py +++ b/test/test_nearfield_interaction_completeness.py @@ -119,17 +119,17 @@ def drive_test_completeness(ctx, queue, dim, q_order): from volumential.table_manager import NearFieldInteractionTableManager - subprocess.check_call(['rm', '-f', 'nft.hdf5']) + subprocess.check_call(['rm', '-f', 'nft-test-completeness.hdf5']) - with NearFieldInteractionTableManager("nft.hdf5") as tm: + with NearFieldInteractionTableManager("nft-test-completeness.hdf5", + progress_bar=False) as tm: - nft, _ = tm.get_table(dim, "Constant", q_order, - queue=queue, n_levels=1, alpha=0, - compute_method="DrosteSum", - n_brick_quad_points=50, - adaptive_level=False, use_symmetry=True) + nft, _ = tm.get_table( + dim, "Constant", q_order, queue=queue, n_levels=1, alpha=0, + compute_method="DrosteSum", n_brick_quad_points=50, + adaptive_level=False, use_symmetry=True) - subprocess.check_call(['rm', 'nft.hdf5']) + subprocess.check_call(['rm', 'nft-test-completeness.hdf5']) # {{{ expansion wrangler @@ -166,11 +166,8 @@ def drive_test_completeness(ctx, queue, dim, q_order): # }}} End sumpy expansion for laplace kernel pot = wrangler.eval_direct( - trav.target_boxes, - trav.neighbor_source_boxes_starts, - trav.neighbor_source_boxes_lists, - mode_coefs=source_vals, - ) + trav.target_boxes, trav.neighbor_source_boxes_starts, + trav.neighbor_source_boxes_lists, mode_coefs=source_vals) pot = pot[0] for p in pot[0]: assert(abs(p - 2**dim) < 1e-8) diff --git a/test/test_nearfield_potential_table.py b/test/test_nearfield_potential_table.py index c7a6dd4..ec87c38 100644 --- a/test/test_nearfield_potential_table.py +++ b/test/test_nearfield_potential_table.py @@ -28,14 +28,16 @@ from numpy.polynomial.chebyshev import chebval, chebval2d, chebval3d def test_const_order_1(): - table = npt.NearFieldInteractionTable(quad_order=1, build_method='Transform') + table = npt.NearFieldInteractionTable( + quad_order=1, build_method='Transform', progress_bar=False) table.build_table() for ary in table.data: assert np.allclose(ary, 1) def test_const_order_2(longrun): - table = npt.NearFieldInteractionTable(quad_order=2, build_method='Transform') + table = npt.NearFieldInteractionTable( + quad_order=2, build_method='Transform', progress_bar=False) table.build_table() for ary in table.data: assert np.allclose(ary, 0.25) @@ -43,7 +45,7 @@ def test_const_order_2(longrun): def interp_modes(q_order): table = npt.NearFieldInteractionTable( - quad_order=q_order, build_method='Transform') + quad_order=q_order, build_method='Transform', progress_bar=False) modes = [table.get_mode(i) for i in range(table.n_q_points)] @@ -88,7 +90,7 @@ def drive_test_modes_cheb_coeffs(dim, q, cheb_order): raise RuntimeError('Insufficient cheb_order to fully resolve the modes') sample_mode = np.random.randint(q**dim) - table = npt.NearFieldInteractionTable(quad_order=q, dim=dim) + table = npt.NearFieldInteractionTable(quad_order=q, dim=dim, progress_bar=False) ccoefs = table.get_mode_cheb_coeffs(sample_mode, cheb_order) shape = (cheb_order, ) * dim ccoefs = ccoefs.reshape(*shape) diff --git a/test/test_table_manager.py b/test/test_table_manager.py index 4087d13..48b77d7 100644 --- a/test/test_table_manager.py +++ b/test/test_table_manager.py @@ -23,6 +23,7 @@ THE SOFTWARE. """ import os +import subprocess import numpy as np import pyopencl as cl import pytest @@ -34,12 +35,26 @@ from volumential.table_manager import ( def get_table(queue, q_order=1, dim=2): pid = os.getpid() - copyfile("/tmp/volumential-tests.hdf5", - f"/tmp/volumential-tests-{pid}.hdf5") - with NFTable(f"/tmp/volumential-tests-{pid}.hdf5") as table_manager: + + # copy from shared cache + if os.path.exists("nft-test-table-manager.hdf5"): + copyfile("nft-test-table-manager.hdf5", + f"nft-test-table-manager-{pid}.hdf5") + + subprocess.check_call(['rm', '-f', f'nft-test-table-manager-{pid}.hdf5']) + with NFTable(f"nft-test-table-manager-{pid}.hdf5", + progress_bar=False) as table_manager: table, _ = table_manager.get_table( dim, "Laplace", q_order=q_order, force_recompute=False, queue=queue) + + # save to shared cache + if not os.path.exists("nft-test-table-manager.hdf5"): + copyfile(f"nft-test-table-manager-{pid}.hdf5", + "nft-test-table-manager.hdf5") + + subprocess.check_call(['rm', f'nft-test-table-manager-{pid}.hdf5']) + return table diff --git a/test/test_volume_fmm.py b/test/test_volume_fmm.py index 486432b..739baa5 100644 --- a/test/test_volume_fmm.py +++ b/test/test_volume_fmm.py @@ -22,26 +22,18 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -# import volumential.volume_fmm as vfmm +import subprocess +import logging +import pytest import numpy as np import pyopencl as cl - -# import pyopencl.array -# import boxtree as bt -# import sumpy as sp -# import volumential as vm +import volumential.meshgen as mg from functools import partial -import logging - logger = logging.getLogger(__name__) -import volumential.meshgen as mg - -import pytest - # {{{ make sure context getter works @@ -211,9 +203,13 @@ def laplace_problem(ctx_factory): from volumential.table_manager import NearFieldInteractionTableManager - tm = NearFieldInteractionTableManager("../nft.hdf5") + subprocess.check_call(['rm', '-f', 'nft-test-volume-fmm.hdf5']) + + tm = NearFieldInteractionTableManager("nft-test-volume-fmm.hdf5") nftable, _ = tm.get_table(dim, "Laplace", q_order) + subprocess.check_call(['rm', 'nft-test-volume-fmm.hdf5']) + # }}} End build near field potential table # {{{ sumpy expansion for laplace kernel diff --git a/volumential/nearfield_potential_table.py b/volumential/nearfield_potential_table.py index d4a526d..cb0bcbd 100644 --- a/volumential/nearfield_potential_table.py +++ b/volumential/nearfield_potential_table.py @@ -212,6 +212,7 @@ class NearFieldInteractionTable(object): source_box_extent=1, dtype=np.float64, inverse_droste=False, + progress_bar=True, **kwargs ): """ @@ -330,9 +331,11 @@ class NearFieldInteractionTable(object): total_evals = len(self.data) + self.n_q_points - from pytools import ProgressBar - - self.pb = ProgressBar("Building table:", total_evals) + if progress_bar: + from pytools import ProgressBar + self.pb = ProgressBar("Building table:", total_evals) + else: + self.pb = None self.is_built = False @@ -738,7 +741,8 @@ class NearFieldInteractionTable(object): else: pool = None - self.pb.draw() + if self.pb is not None: + self.pb.draw() self.build_normalizer_table(pool, pb=self.pb) self.has_normalizers = True @@ -750,18 +754,19 @@ class NearFieldInteractionTable(object): ] if 0: - # multiprocess disabled for py2 compatibility - # (also drops dependency on dill/multiprocess) + # multiprocess disabled to remove dependency on dill/multiprocess for entry_id, entry_val in pool.imap_unordered( self.compute_table_entry, invariant_entry_ids ): self.data[entry_id] = entry_val - self.pb.progress(1) + if self.pb is not None: + self.pb.progress(1) else: for entry_id in invariant_entry_ids: _, entry_val = self.compute_table_entry(entry_id) self.data[entry_id] = entry_val - self.pb.progress(1) + if self.pb is not None: + self.pb.progress(1) if 0: # Then complete the table via symmetry lookup @@ -771,7 +776,8 @@ class NearFieldInteractionTable(object): if centry_id == entry_id: continue self.data[entry_id] = self.data[centry_id] - self.pb.progress(1) + if self.pb is not None: + self.pb.progress(1) else: for entry_id in range(len(self.data)): _, centry_id = self.lookup_by_symmetry(entry_id) @@ -779,9 +785,11 @@ class NearFieldInteractionTable(object): if centry_id == entry_id: continue self.data[entry_id] = self.data[centry_id] - self.pb.progress(1) + if self.pb is not None: + self.pb.progress(1) - self.pb.finished() + if self.pb is not None: + self.pb.finished() for entry in self.data: assert not np.isnan(entry) -- GitLab From 5d226235b5a2cea4ccc38cda58e18b8fff48034b Mon Sep 17 00:00:00 2001 From: xywei Date: Wed, 15 Jul 2020 09:56:11 -0500 Subject: [PATCH 13/16] Skip writing to shared cahce from test sessions --- test/test_table_manager.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/test_table_manager.py b/test/test_table_manager.py index 48b77d7..af62e8a 100644 --- a/test/test_table_manager.py +++ b/test/test_table_manager.py @@ -36,23 +36,19 @@ from volumential.table_manager import ( def get_table(queue, q_order=1, dim=2): pid = os.getpid() - # copy from shared cache + # copy from shared cache if exists if os.path.exists("nft-test-table-manager.hdf5"): copyfile("nft-test-table-manager.hdf5", f"nft-test-table-manager-{pid}.hdf5") subprocess.check_call(['rm', '-f', f'nft-test-table-manager-{pid}.hdf5']) + with NFTable(f"nft-test-table-manager-{pid}.hdf5", progress_bar=False) as table_manager: table, _ = table_manager.get_table( dim, "Laplace", q_order=q_order, force_recompute=False, queue=queue) - # save to shared cache - if not os.path.exists("nft-test-table-manager.hdf5"): - copyfile(f"nft-test-table-manager-{pid}.hdf5", - "nft-test-table-manager.hdf5") - subprocess.check_call(['rm', f'nft-test-table-manager-{pid}.hdf5']) return table -- GitLab From b22ac1b95bff240bb7c543180653d2f9f4aeea65 Mon Sep 17 00:00:00 2001 From: xywei Date: Wed, 15 Jul 2020 10:04:55 -0500 Subject: [PATCH 14/16] Use sessionfinish to clean up tables --- ...test_nearfield_interaction_completeness.py | 3 --- test/test_table_manager.py | 2 -- test/test_volume_fmm.py | 21 ++++++++----------- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/test/test_nearfield_interaction_completeness.py b/test/test_nearfield_interaction_completeness.py index edc959e..c1f566c 100644 --- a/test/test_nearfield_interaction_completeness.py +++ b/test/test_nearfield_interaction_completeness.py @@ -120,7 +120,6 @@ def drive_test_completeness(ctx, queue, dim, q_order): from volumential.table_manager import NearFieldInteractionTableManager subprocess.check_call(['rm', '-f', 'nft-test-completeness.hdf5']) - with NearFieldInteractionTableManager("nft-test-completeness.hdf5", progress_bar=False) as tm: @@ -129,8 +128,6 @@ def drive_test_completeness(ctx, queue, dim, q_order): compute_method="DrosteSum", n_brick_quad_points=50, adaptive_level=False, use_symmetry=True) - subprocess.check_call(['rm', 'nft-test-completeness.hdf5']) - # {{{ expansion wrangler from sumpy.kernel import LaplaceKernel diff --git a/test/test_table_manager.py b/test/test_table_manager.py index af62e8a..1278a0f 100644 --- a/test/test_table_manager.py +++ b/test/test_table_manager.py @@ -49,8 +49,6 @@ def get_table(queue, q_order=1, dim=2): dim, "Laplace", q_order=q_order, force_recompute=False, queue=queue) - subprocess.check_call(['rm', f'nft-test-table-manager-{pid}.hdf5']) - return table diff --git a/test/test_volume_fmm.py b/test/test_volume_fmm.py index 739baa5..df0ca1e 100644 --- a/test/test_volume_fmm.py +++ b/test/test_volume_fmm.py @@ -95,8 +95,8 @@ def laplace_problem(ctx_factory): rratio_bot = 0.5 # bounding box - a = -1 - b = 1 + a = -1. + b = 1. m_order = 15 # multipole order @@ -115,7 +115,7 @@ def laplace_problem(ctx_factory): # {{{ generate quad points - mesh = mg.MeshGen2D(q_order, n_levels) + mesh = mg.MeshGen2D(q_order, n_levels, a, b) iloop = 0 while mesh.n_active_cells() < refined_n_cells: iloop += 1 @@ -204,12 +204,9 @@ def laplace_problem(ctx_factory): from volumential.table_manager import NearFieldInteractionTableManager subprocess.check_call(['rm', '-f', 'nft-test-volume-fmm.hdf5']) - tm = NearFieldInteractionTableManager("nft-test-volume-fmm.hdf5") nftable, _ = tm.get_table(dim, "Laplace", q_order) - subprocess.check_call(['rm', 'nft-test-volume-fmm.hdf5']) - # }}} End build near field potential table # {{{ sumpy expansion for laplace kernel @@ -270,8 +267,8 @@ def laplace_problem(ctx_factory): @pytest.mark.skipif( - mg.provider != "meshgen_dealii", reason="Adaptive mesh module is not available" -) + mg.provider != "meshgen_dealii", + reason="Adaptive mesh module is not available") def test_volume_fmm_laplace(laplace_problem): trav, wrangler, source_vals, q_weights = laplace_problem @@ -279,12 +276,12 @@ def test_volume_fmm_laplace(laplace_problem): from volumential.volume_fmm import drive_volume_fmm direct_pot, = drive_volume_fmm( - trav, wrangler, source_vals * q_weights, source_vals, direct_evaluation=True - ) + trav, wrangler, source_vals * q_weights, + source_vals, direct_evaluation=True) fmm_pot, = drive_volume_fmm( - trav, wrangler, source_vals * q_weights, source_vals, direct_evaluation=False - ) + trav, wrangler, source_vals * q_weights, + source_vals, direct_evaluation=False) assert max(abs(direct_pot - fmm_pot)) < 1e-6 -- GitLab From b63c9fb78ed4c1b65e2a81cf83fe61ab7dd6c00f Mon Sep 17 00:00:00 2001 From: xywei Date: Wed, 15 Jul 2020 12:15:16 -0500 Subject: [PATCH 15/16] Cache 2d order1 table with pytest fixture --- requirements.txt | 2 + test/conftest.py | 25 ++++++++ test/test_table_manager.py | 113 ++++++++++++++++++++++--------------- 3 files changed, 94 insertions(+), 46 deletions(-) diff --git a/requirements.txt b/requirements.txt index d435fe1..be9ec06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ --index-url https://pypi.python.org/simple/ +filelock + # git+https://gitlab.tiker.net/xywei/pypvfmm.git#egg=pypvfmm -e git+https://gitlab.tiker.net/xywei/boxtree.git#egg=boxtree diff --git a/test/conftest.py b/test/conftest.py index 5268293..6857159 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -24,6 +24,7 @@ THE SOFTWARE. import subprocess import glob +from filelock import FileLock import pytest # noqa: F401 # setup the ctx_factory fixture @@ -31,6 +32,9 @@ from pyopencl.tools import ( # NOQA pytest_generate_tests_for_pyopencl as pytest_generate_tests, ) +from volumential.table_manager import \ + NearFieldInteractionTableManager as NFTManager + def pytest_addoption(parser): """Add extra command line options. @@ -56,6 +60,27 @@ def requires_pypvfmm(request): pytest.skip("needs pypvfmm to run") +@pytest.fixture(scope='session') +def table_2d_order1(tmp_path_factory, worker_id): + if not worker_id: + # not executing in with multiple workers, just produce the data and let + # pytest's fixture caching do its job + with NFTManager("nft.hdf5", progress_bar=True) as table_manager: + table, _ = table_manager.get_table(2, "Laplace", q_order=1) + subprocess.check_call(['rm', '-f', 'nft.hdf5']) + return table + + # get the temp directory shared by all workers + root_tmp_dir = tmp_path_factory.getbasetemp().parent + + fn = root_tmp_dir / "nft.hdf5" + with FileLock(str(fn) + ".lock"): + with NFTManager(str(fn), progress_bar=True) as table_manager: + table, _ = table_manager.get_table(2, "Laplace", q_order=1) + return table + return table + + def pytest_sessionfinish(session, exitstatus): # remove table caches for table_file in glob.glob("*.hdf5"): diff --git a/test/test_table_manager.py b/test/test_table_manager.py index 1278a0f..58e4b77 100644 --- a/test/test_table_manager.py +++ b/test/test_table_manager.py @@ -37,14 +37,14 @@ def get_table(queue, q_order=1, dim=2): pid = os.getpid() # copy from shared cache if exists - if os.path.exists("nft-test-table-manager.hdf5"): - copyfile("nft-test-table-manager.hdf5", + if os.path.exists("nft.hdf5"): + copyfile("nft.hdf5", f"nft-test-table-manager-{pid}.hdf5") subprocess.check_call(['rm', '-f', f'nft-test-table-manager-{pid}.hdf5']) with NFTable(f"nft-test-table-manager-{pid}.hdf5", - progress_bar=False) as table_manager: + progress_bar=True) as table_manager: table, _ = table_manager.get_table( dim, "Laplace", q_order=q_order, force_recompute=False, queue=queue) @@ -52,23 +52,24 @@ def get_table(queue, q_order=1, dim=2): return table -def test_case_id(ctx_factory): +def test_case_id(ctx_factory, table_2d_order1): cl_ctx = ctx_factory() queue = cl.CommandQueue(cl_ctx) - table = get_table(queue) + table = table_2d_order1 case_same_box = len(table.interaction_case_vecs) // 2 assert list(table.interaction_case_vecs[case_same_box]) == [0, 0] -def test_get_table(ctx_factory): - cl_ctx = ctx_factory() - queue = cl.CommandQueue(cl_ctx) - table = get_table(queue) +def test_get_table_2d_order1(table_2d_order1): + table = table_2d_order1 assert table.dim == 2 -def laplace_const_source_same_box(queue, q_order, dim=2): - nft = get_table(queue, q_order, dim) +def laplace_const_source_same_box(table_2d_order1, queue, q_order, dim=2): + if q_order == 1: + nft = table_2d_order1 + else: + nft = get_table(queue, q_order, dim) n_pairs = nft.n_pairs n_q_points = nft.n_q_points @@ -87,8 +88,12 @@ def laplace_const_source_same_box(queue, q_order, dim=2): return pot -def laplace_cons_source_neighbor_box(queue, q_order, case_id, dim=2): - nft = get_table(queue, q_order, dim) +def laplace_cons_source_neighbor_box( + table_2d_order1, queue, q_order, case_id, dim=2): + if q_order == 1: + nft = table_2d_order1 + else: + nft = get_table(queue, q_order, dim) n_pairs = nft.n_pairs n_q_points = nft.n_q_points @@ -105,15 +110,19 @@ def laplace_cons_source_neighbor_box(queue, q_order, case_id, dim=2): return pot -def test_lcssb_1(ctx_factory): +def test_lcssb_1(ctx_factory, table_2d_order1): cl_ctx = ctx_factory() queue = cl.CommandQueue(cl_ctx) - u = laplace_const_source_same_box(queue, 1) + u = laplace_const_source_same_box(table_2d_order1, queue, 1) assert len(u) == 1 -def interp_func(queue, q_order, coef, dim=2): - nft = get_table(queue, q_order, dim) +def interp_func(table_2d_order1, queue, q_order, coef, dim=2): + if q_order == 1: + nft = table_2d_order1 + else: + nft = get_table(queue, q_order, dim) + assert dim == 2 modes = [nft.get_mode(i) for i in range(nft.n_q_points)] @@ -128,7 +137,7 @@ def interp_func(queue, q_order, coef, dim=2): return func -def test_interp_func(longrun, ctx_factory): +def test_interp_func(longrun, ctx_factory, table_2d_order1): cl_ctx = ctx_factory() queue = cl.CommandQueue(cl_ctx) q_order = 3 @@ -137,7 +146,7 @@ def test_interp_func(longrun, ctx_factory): h = 0.1 xx = yy = np.arange(-1.0, 1.0, h) xi, yi = np.meshgrid(xx, yy) - func = interp_func(queue, q_order, coef) + func = interp_func(table_2d_order1, queue, q_order, coef) zi = func(xi, yi) @@ -156,10 +165,14 @@ def direct_quad(source_func, target_point, dim=2): return integral -def drive_test_direct_quad_same_box(queue, q_order, dim=2): - u = laplace_const_source_same_box(queue, q_order) - func = interp_func(queue, q_order, u) - nft = get_table(queue, q_order, dim) +def drive_test_direct_quad_same_box(table_2d_order1, queue, q_order, dim=2): + u = laplace_const_source_same_box(table_2d_order1, queue, q_order) + func = interp_func(table_2d_order1, queue, q_order, u) + + if q_order == 1: + nft = table_2d_order1 + else: + nft = get_table(queue, q_order, dim) def const_one_source_func(x, y): return 1 @@ -184,23 +197,21 @@ def drive_test_direct_quad_same_box(queue, q_order, dim=2): @pytest.mark.parametrize("q_order", [1, ]) -def test_direct_quad(q_order, ctx_factory): +def test_direct_quad(q_order, ctx_factory, table_2d_order1): cl_ctx = ctx_factory() queue = cl.CommandQueue(cl_ctx) - drive_test_direct_quad_same_box(queue, q_order) + drive_test_direct_quad_same_box(table_2d_order1, queue, q_order) @pytest.mark.parametrize("q_order", [2, 3, 4, 5]) -def test_direct_quad_longrun(longrun, ctx_factory, q_order): +def test_direct_quad_longrun(longrun, ctx_factory, q_order, table_2d_order1): cl_ctx = ctx_factory() queue = cl.CommandQueue(cl_ctx) - drive_test_direct_quad_same_box(queue, q_order) + drive_test_direct_quad_same_box(table_2d_order1, queue, q_order) -def test_case_ids(ctx_factory): - cl_ctx = ctx_factory() - queue = cl.CommandQueue(cl_ctx) - table = get_table(queue) +def test_case_ids(ctx_factory, table_2d_order1): + table = table_2d_order1 for i in range(len(table.interaction_case_vecs)): code = table.case_encode( table.interaction_case_vecs[i]) @@ -223,10 +234,8 @@ def get_target_point(case_id, target_id, table): return target_point -def test_get_neighbor_target_point(ctx_factory): - cl_ctx = ctx_factory() - queue = cl.CommandQueue(cl_ctx) - table = get_table(queue) +def test_get_neighbor_target_point(ctx_factory, table_2d_order1): + table = table_2d_order1 case_same_box = len(table.interaction_case_vecs) // 2 for cid in range(len(table.interaction_case_vecs)): if cid == case_same_box: @@ -237,8 +246,12 @@ def test_get_neighbor_target_point(ctx_factory): assert np.allclose(pt, pt2) -def laplace_const_source_neighbor_box(queue, q_order, case_id, dim=2): - nft = get_table(queue, q_order, dim) +def laplace_const_source_neighbor_box( + table_2d_order1, queue, q_order, case_id, dim=2): + if q_order == 1: + nft = table_2d_order1 + else: + nft = get_table(queue, q_order, dim) n_pairs = nft.n_pairs n_q_points = nft.n_q_points @@ -252,9 +265,14 @@ def laplace_const_source_neighbor_box(queue, q_order, case_id, dim=2): return pot -def drive_test_direct_quad_neighbor_box(queue, q_order, case_id, dim=2): - u = laplace_const_source_neighbor_box(queue, q_order, case_id) - nft = get_table(queue, q_order, dim) +def drive_test_direct_quad_neighbor_box( + table_2d_order1, queue, q_order, case_id, dim=2): + u = laplace_const_source_neighbor_box( + table_2d_order1, queue, q_order, case_id) + if q_order == 1: + nft = table_2d_order1 + else: + nft = get_table(queue, q_order, dim) def const_one_source_func(x, y): return 1 @@ -276,21 +294,24 @@ def drive_test_direct_quad_neighbor_box(queue, q_order, case_id, dim=2): @pytest.mark.parametrize("q_order", [1, ]) -def test_direct_quad_neighbor_box(ctx_factory, q_order): +def test_direct_quad_neighbor_box(ctx_factory, q_order, table_2d_order1): cl_ctx = ctx_factory() queue = cl.CommandQueue(cl_ctx) - table = get_table(queue) + table = table_2d_order1 for case_id in range(len(table.interaction_case_vecs)): - drive_test_direct_quad_neighbor_box(queue, q_order, case_id) + drive_test_direct_quad_neighbor_box( + table_2d_order1, queue, q_order, case_id) @pytest.mark.parametrize("q_order", [2, ]) -def test_direct_quad_neighbor_box_longrun(longrun, ctx_factory, q_order): +def test_direct_quad_neighbor_box_longrun( + longrun, ctx_factory, q_order, table_2d_order1): cl_ctx = ctx_factory() queue = cl.CommandQueue(cl_ctx) - table = get_table(queue) + table = table_2d_order1 for case_id in range(len(table.interaction_case_vecs)): - drive_test_direct_quad_neighbor_box(queue, q_order, case_id) + drive_test_direct_quad_neighbor_box( + table_2d_order1, queue, q_order, case_id) # fdm=marker:ft=pyopencl -- GitLab From 0e6830c7a07fc641bdb4ae49553342d2b48887b5 Mon Sep 17 00:00:00 2001 From: xywei Date: Wed, 15 Jul 2020 12:16:01 -0500 Subject: [PATCH 16/16] Flake8 fixes --- test/test_table_manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_table_manager.py b/test/test_table_manager.py index 58e4b77..e783cac 100644 --- a/test/test_table_manager.py +++ b/test/test_table_manager.py @@ -53,8 +53,6 @@ def get_table(queue, q_order=1, dim=2): def test_case_id(ctx_factory, table_2d_order1): - cl_ctx = ctx_factory() - queue = cl.CommandQueue(cl_ctx) table = table_2d_order1 case_same_box = len(table.interaction_case_vecs) // 2 assert list(table.interaction_case_vecs[case_same_box]) == [0, 0] -- GitLab