diff --git a/grudge/execution.py b/grudge/execution.py
index 9285cecc9e9fcffe2b14762c29ebd4a29609b8e7..ec0d6c6a2eae61523c107926589b94da4556a362 100644
--- a/grudge/execution.py
+++ b/grudge/execution.py
@@ -279,120 +279,8 @@ class ExecutionMapper(mappers.Evaluator,
return conn(self.queue, self.rec(field_expr)).with_queue(self.queue)
- def map_opposite_partition_face_swap(self, op, field_expr):
- raise NotImplementedError("map_opposite_partition_face_swap")
-
- # TODO: Fetch these variables
- vol_discr = None
- group_factory = None
- cl_ctx = None
- TAG_SEND_MESH = 1 # noqa
-
- from mpi4py import MPI
- comm = MPI.COMM_WORLD
- # FIXME: Assumes rank 0 is a 'central hub' and
- # i_part = rank - 1 for all other ranks
- rank = comm.Get_rank()
- num_parts = comm.Get_size() - 1
-
- i_local_part = rank - 1
- local_bdry_conns = {}
- for i_remote_part in range(num_parts):
- if i_local_part == i_remote_part:
- continue
- # Mark faces within local_mesh that are connected to remote_mesh
- from meshmode.discretization.connection import make_face_restriction
- from meshmode.mesh import BTAG_PARTITION
- # TODO: May not be necessary to compute every time
- local_bdry_conns[i_remote_part] =\
- make_face_restriction(vol_discr, group_factory,
- BTAG_PARTITION(i_remote_part))
-
- # Send boundary data
- send_reqs = []
- for i_remote_part in range(num_parts):
- if i_local_part == i_remote_part:
- continue
- bdry_nodes = local_bdry_conns[i_remote_part].to_discr.nodes()
- if bdry_nodes.size == 0:
- # local_mesh is not connected to remote_mesh; send None
- send_reqs.append(comm.isend(None,
- dest=i_remote_part+1,
- tag=TAG_SEND_MESH))
- continue
-
- # Gather information to send to other ranks
- local_bdry = local_bdry_conns[i_remote_part].to_discr
- local_mesh = local_bdry_conns[i_remote_part].from_discr.mesh
- local_adj_groups = [local_mesh.facial_adjacency_groups[i][None]
- for i in range(len(local_mesh.groups))]
- local_batches = [local_bdry_conns[i_remote_part].groups[i].batches
- for i in range(len(local_mesh.groups))]
- local_to_elem_faces = [[batch.to_element_face for batch in grp_batches]
- for grp_batches in local_batches]
- local_to_elem_indices = [[batch.to_element_indices.get(queue=self.queue)
- for batch in grp_batches]
- for grp_batches in local_batches]
-
- local_data = {'bdry_mesh': local_bdry.mesh,
- 'adj': local_adj_groups,
- 'to_elem_faces': local_to_elem_faces,
- 'to_elem_indices': local_to_elem_indices}
- send_reqs.append(comm.isend(local_data,
- dest=i_remote_part+1,
- tag=TAG_SEND_MESH))
-
- # Receive boundary data
- remote_buf = {}
- for i_remote_part in range(num_parts):
- if i_local_part == i_remote_part:
- continue
- remote_rank = i_remote_part + 1
- status = MPI.Status()
- comm.probe(source=remote_rank, tag=TAG_SEND_MESH, status=status)
- remote_buf[i_remote_part] = np.empty(status.count, dtype=bytes)
-
- recv_reqs = {}
- for i_remote_part, buf in remote_buf.items():
- remote_rank = i_remote_part + 1
- recv_reqs[i_remote_part] = comm.irecv(buf=buf,
- source=remote_rank,
- tag=TAG_SEND_MESH)
-
- remote_data = {}
- for i_remote_part, req in recv_reqs.items():
- status = MPI.Status()
- remote_data[i_remote_part] = req.wait(status=status)
- # Free the buffer
- remote_buf[i_remote_part] = None # FIXME: Is this a good idea?
- print('Rank {0}: Received rank {1} data ({2} bytes)'
- .format(rank, i_remote_part + 1, status.count))
-
- for req in send_reqs:
- req.wait()
-
- connections = []
- for i_remote_part, data in remote_data.items():
- if data is None:
- # Local mesh is not connected to remote mesh
- continue
- remote_bdry_mesh = data['bdry_mesh']
- from meshmode.discretization import Discretization
- remote_bdry = Discretization(cl_ctx, remote_bdry_mesh, group_factory)
- remote_adj_groups = data['adj']
- remote_to_elem_faces = data['to_elem_faces']
- remote_to_elem_indices = data['to_elem_indices']
- # Connect local_mesh to remote_mesh
- from meshmode.discretization.connection import make_partition_connection
- connection = make_partition_connection(local_bdry_conns[i_remote_part],
- i_local_part,
- remote_bdry,
- remote_adj_groups,
- remote_to_elem_faces,
- remote_to_elem_indices)
- connections.append(connection)
-
- return None
+ def map_opposite_rank_face_swap(self, op, field_expr):
+ raise NotImplementedError("map_opposite_rank_face_swap")
def map_opposite_interior_face_swap(self, op, field_expr):
dd = op.dd_in
diff --git a/grudge/symbolic/operators.py b/grudge/symbolic/operators.py
index dc2e4fa1afd8cf6bd91ff1faa8b8df7a34b06a46..70c43a100bc6676874f1a1485df301e7d6ca2493 100644
--- a/grudge/symbolic/operators.py
+++ b/grudge/symbolic/operators.py
@@ -379,6 +379,25 @@ class RefInverseMassOperator(RefMassOperatorBase):
# {{{ boundary-related operators
+class OppositeRankFaceSwap(Operator):
+ def __init__(self, dd_in=None, dd_out=None):
+ sym = _sym()
+
+ if dd_in is None:
+ dd_in = sym.DOFDesc(BTAG_PARTITION, None)
+ if dd_out is None:
+ dd_out = dd_in
+
+ if dd_in.domain_tag is not BTAG_PARTITION:
+ raise ValueError("dd_in must be a rank boundary faces domain")
+ if dd_out != dd_in:
+ raise ValueError("dd_out and dd_in must be identical")
+
+ super(OppositeRankFaceSwap, self).__init__(dd_in, dd_out)
+
+ mapper_method = intern("map_opposite_rank_face_swap")
+
+
class OppositeInteriorFaceSwap(Operator):
def __init__(self, dd_in=None, dd_out=None):
sym = _sym()