diff --git a/meshmode/discretization/connection/opposite_face.py b/meshmode/discretization/connection/opposite_face.py index 181e059e7402d427717abfdfedac448940914961..cccae33c183b66265110c681b0e709b6df613c77 100644 --- a/meshmode/discretization/connection/opposite_face.py +++ b/meshmode/discretization/connection/opposite_face.py @@ -408,7 +408,8 @@ def make_opposite_face_connection(volume_to_bdry_conn): # {{{ partition_connection def make_partition_connection(local_bdry_conn, i_local_part, - remote_bdry, remote_adj_groups, remote_batches): + remote_bdry, remote_adj_groups, + remote_to_elem_faces, remote_to_elem_indices): """ Connects ``local_bdry_conn`` to a neighboring partition. @@ -419,7 +420,11 @@ def make_partition_connection(local_bdry_conn, i_local_part, remote partition. :arg remote_bdry: A :class:`Discretization` of the boundary of the remote partition. - :arg remote_batches: A list of batches of the remote partition. + :arg remote_to_elem_faces: `remote_to_elem_faces[igrp][idx]` gives the face + that batch `idx` interpolates from group `igrp`. + :arg remote_to_elem_indices: `remote_to_elem_indices[igrp][idx]` gives a + :class:`np.array` of element indices that batch `idx` interpolates from + group `igrp`. :returns: A :class:`DirectDiscretizationConnection` that performs data exchange across faces from partition `i_local_part` to the remote partition. @@ -465,29 +470,29 @@ def make_partition_connection(local_bdry_conn, i_local_part, if not np.any(index_flags): continue - vbc_remote_grp_face_batch = _find_ibatch_for_face( - remote_batches[i_remote_grp], i_remote_face) + batch_idx = np.where(remote_to_elem_faces[i_remote_grp] + == i_remote_face)[0] - remote_bdry_element_indices = vbc_remote_grp_face_batch.\ - to_element_indices.get(queue=queue) + remote_bdry_indices =\ + remote_to_elem_indices[i_remote_grp][batch_idx] elems = i_local_meshwide_elems[index_flags] - elem_base faces = i_local_faces[index_flags] - local_bdry_element_indices = local_el_lookup[elems, faces] + local_bdry_indices = local_el_lookup[elems, faces] batches = _make_cross_face_batches(queue, remote_bdry, local_bdry, i_remote_grp, i_local_grp, - remote_bdry_element_indices, - local_bdry_element_indices) + remote_bdry_indices, + local_bdry_indices) part_batches[i_remote_grp].extend(batches) return DirectDiscretizationConnection( from_discr=local_bdry, to_discr=remote_bdry, - groups=[DiscretizationConnectionElementGroup(batches=batches) - for batches in part_batches], + groups=[DiscretizationConnectionElementGroup(batches=grp_batches) + for grp_batches in part_batches], is_surjective=True) # }}} diff --git a/test/test_meshmode.py b/test/test_meshmode.py index 1d2a954a299f0339115ddce81001257092f31485..f0009aa7b9da0ae1d0b360fbfa8318327f98d1a8 100644 --- a/test/test_meshmode.py +++ b/test/test_meshmode.py @@ -139,6 +139,12 @@ def test_partition_interpolation(ctx_getter, group_factory, dim, mesh_pars, for i in range(len(local_mesh.groups))] local_batches = [local_bdry_conn.groups[i].batches for i in range(len(local_mesh.groups))] + local_to_elem_faces = [[batch.to_element_face + for batch in grp_batches] + for grp_batches in local_batches] + local_to_elem_indices = [[batch.to_element_indices.get(queue=queue) + for batch in grp_batches] + for grp_batches in local_batches] remote_bdry = remote_bdry_conn.to_discr remote_mesh = remote_bdry_conn.from_discr.mesh @@ -146,20 +152,28 @@ def test_partition_interpolation(ctx_getter, group_factory, dim, mesh_pars, for i in range(len(remote_mesh.groups))] remote_batches = [remote_bdry_conn.groups[i].batches for i in range(len(remote_mesh.groups))] + remote_to_elem_faces = [[batch.to_element_face + for batch in grp_batches] + for grp_batches in remote_batches] + remote_to_elem_indices = [[batch.to_element_indices.get(queue=queue) + for batch in grp_batches] + for grp_batches in remote_batches] # Connect local_mesh to remote_mesh local_part_conn = make_partition_connection(local_bdry_conn, i_local_part, remote_bdry, remote_adj_groups, - remote_batches) + remote_to_elem_faces, + remote_to_elem_indices) # Connect remote mesh to local mesh remote_part_conn = make_partition_connection(remote_bdry_conn, i_remote_part, local_bdry, local_adj_groups, - local_batches) + local_to_elem_faces, + local_to_elem_indices) check_connection(local_part_conn) check_connection(remote_part_conn) diff --git a/testmpi.py b/testmpi.py index fbc13883857b05effc7c0cfee501fd9ca374546a..bb3c19781da08754209da4dacd6d32c85a93adc4 100644 --- a/testmpi.py +++ b/testmpi.py @@ -1,6 +1,6 @@ from mpi4py import MPI import numpy as np -import pyopencl +import pyopencl as cl comm = MPI.COMM_WORLD rank = comm.Get_rank() @@ -30,8 +30,9 @@ elif (rank - 1) in range(num_parts): mesh = comm.recv(source=0, tag=1) print('Recieved mesh') - cl_ctx = pyopencl.create_some_context() - + cl_ctx = cl.create_some_context() + queue = cl.CommandQueue(cl_ctx) + from meshmode.discretization.poly_element\ import PolynomialWarpAndBlendGroupFactory group_factory = PolynomialWarpAndBlendGroupFactory(4) @@ -68,9 +69,17 @@ elif (rank - 1) in range(num_parts): for i in range(len(local_mesh.groups))] local_batches = [local_bdry_conns[i_remote_part].groups[i].batches for i in range(len(local_mesh.groups))] + local_to_elem_faces = [[batch.to_element_face for batch in grp_batches] + for grp_batches in local_batches] + local_to_elem_indices = [[batch.to_element_indices.get(queue=queue) + for batch in grp_batches] + for grp_batches in local_batches] + + print(local_bdry.groups) local_data = {'bdry': local_bdry, 'adj': local_adj_groups, - 'batches': local_batches} + 'to_elem_faces': local_to_elem_faces, + 'to_elem_indices': local_to_elem_indices} send_reqs.append(comm.isend(local_data, dest=i_remote_part+1, tag=2)) recv_reqs = {} @@ -93,7 +102,8 @@ elif (rank - 1) in range(num_parts): continue remote_bdry = data['bdry'] remote_adj_groups =data['adj'] - remote_batches = data['batches'] + remote_to_elem_faces = data['to_elem_faces'] + remote_to_elem_indices = data['to_elem_indices'] # Connect local_mesh to remote_mesh from meshmode.discretization.connection import make_partition_connection connection[i_remote_part] =\ @@ -101,7 +111,8 @@ elif (rank - 1) in range(num_parts): i_local_part, remote_bdry, remote_adj_groups, - remote_batches) + remote_to_elem_faces, + remote_to_elem_indices) from meshmode.discretization.connection import check_connection check_connection(connection[i_remote_part])