diff --git a/grudge/symbolic/operators.py b/grudge/symbolic/operators.py index 05a23adfccee398fe568290326b35163a0533630..a1d0f210ad3a94157af431e3610c2e3bdc577d09 100644 --- a/grudge/symbolic/operators.py +++ b/grudge/symbolic/operators.py @@ -387,14 +387,14 @@ class OppositeRankFaceSwap(Operator): if dd_in is None: # FIXME: What is FRESTR_INTERIOR_FACES? dd_in = sym.DOFDesc(sym.FRESTR_INTERIOR_FACES) - # dd_in = sym.DOFDesc(sym.BTAG_PARTITION) + # dd_in = sym.DOFDesc(BTAG_PARTITION) if dd_out is None: dd_out = dd_in # if dd_in.domain_tag is not BTAG_PARTITION: # raise ValueError("dd_in must be a rank boundary faces domain") - # if dd_out != dd_in: - # raise ValueError("dd_out and dd_in must be identical") + if dd_out != dd_in: + raise ValueError("dd_out and dd_in must be identical") super(OppositeRankFaceSwap, self).__init__(dd_in, dd_out) diff --git a/test/test_mpi_communication.py b/test/test_mpi_communication.py new file mode 100644 index 0000000000000000000000000000000000000000..a343beb0a7b81b35df4f7144fb815c4f9b60eb22 --- /dev/null +++ b/test/test_mpi_communication.py @@ -0,0 +1,104 @@ +from __future__ import division, absolute_import, print_function + +__copyright__ = """ +Copyright (C) 2017 Ellis Hoag +Copyright (C) 2017 Andreas Kloeckner +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import pytest +import os + +import logging +logger = logging.getLogger(__name__) + +import numpy as np + + +def mpi_communication_entrypoint(): + from meshmode.distributed import MPIMeshDistributor, MPIBoundaryCommunicator + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + num_parts = comm.Get_size() + + mesh_dist = MPIMeshDistributor(comm) + + if mesh_dist.is_mananger_rank(): + np.random.seed(42) + from meshmode.mesh.generation import generate_warped_rect_mesh + meshes = [generate_warped_rect_mesh(3, order=4, n=4) for _ in range(2)] + + from meshmode.mesh.processing import merge_disjoint_meshes + mesh = merge_disjoint_meshes(meshes) + + part_per_element = np.random.randint(num_parts, size=mesh.nelements) + + local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element, num_parts) + else: + local_mesh = mesh_dist.receive_mesh_part() + + from meshmode.discretization.poly_element\ + import PolynomialWarpAndBlendGroupFactory + group_factory = PolynomialWarpAndBlendGroupFactory(4) + import pyopencl as cl + cl_ctx = cl.create_some_context() + queue = cl.CommandQueue(cl_ctx) + + from meshmode.discretization import Discretization + vol_discr = Discretization(cl_ctx, local_mesh, group_factory) + + logger.debug("Rank %d exiting", rank) + + +# {{{ MPI test pytest entrypoint + +@pytest.mark.mpi +@pytest.mark.parametrize("num_partitions", [3, 4]) +def test_mpi_communication(num_partitions): + pytest.importorskip("mpi4py") + + num_ranks = num_partitions + from subprocess import check_call + import sys + newenv = os.environ.copy() + newenv["RUN_WITHIN_MPI"] = "1" + check_call([ + "mpiexec", "-np", str(num_ranks), "-x", "RUN_WITHIN_MPI", + sys.executable, __file__], + env=newenv) + +# }}} + +if __name__ == "__main__": + if "RUN_WITHIN_MPI" in os.environ: + mpi_communication_entrypoint() + else: + import sys + if len(sys.argv) > 1: + exec(sys.argv[1]) + else: + from py.test.cmdline import main + main([__file__]) + +# vim: fdm=marker