From e26f55b63aed89c5a5599364e65675f2b327cda1 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Mon, 10 Sep 2018 19:50:48 -0500 Subject: [PATCH 1/4] Change pyfmmlib's form_locals() to utilize OpenMP. --- boxtree/pyfmmlib_integration.py | 128 +++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 43 deletions(-) diff --git a/boxtree/pyfmmlib_integration.py b/boxtree/pyfmmlib_integration.py index 33e9dec..45d54f5 100644 --- a/boxtree/pyfmmlib_integration.py +++ b/boxtree/pyfmmlib_integration.py @@ -396,6 +396,13 @@ class FMMLibExpansionWrangler(object): def _get_targets(self, pslice): return self._get_single_targets_array()[:, pslice] + @memoize_method + def _get_single_box_centers_array(self): + return np.array([ + self.tree.box_centers[idim] + for idim in range(self.dim) + ], order="F") + # }}} @log_process(logger) @@ -406,23 +413,37 @@ class FMMLibExpansionWrangler(object): def reorder_potentials(self, potentials): return potentials[self.tree.sorted_target_ids] - def get_source_kwargs(self, src_weights, pslice): + def get_source_kwargs(self, src_weights, pslice, extra_kwargs=None): + if extra_kwargs is None: + extra_kwargs = {} + if self.dipole_vec is None: - return { + result = { "charge": src_weights[pslice], } + for name, val in extra_kwargs.items(): + result["charge_%s" % name] = val + return result + + elif self.eqn_letter == "l" and self.dim == 2: + result = { + "dipstr": -src_weights[pslice] * ( + self.dipole_vec[0, pslice] + + 1j * self.dipole_vec[1, pslice]) + } + for name, val in extra_kwargs.items(): + result["dipstr_%s" % name] = val + return result + else: - if self.eqn_letter == "l" and self.dim == 2: - return { - "dipstr": -src_weights[pslice] * ( - self.dipole_vec[0, pslice] - + 1j * self.dipole_vec[1, pslice]) - } - else: - return { - "dipstr": src_weights[pslice], - "dipvec": self.dipole_vec[:, pslice], - } + result = { + "dipstr": src_weights[pslice], + "dipvec": self.dipole_vec[:, pslice], + } + for name, val in extra_kwargs.items(): + result["dipstr_%s" % name] = val + result["dipvec_%s" % name] = val + return result @log_process(logger) @return_timing_data @@ -534,7 +555,6 @@ class FMMLibExpansionWrangler(object): if tgt_pslice.stop - tgt_pslice.start == 0: continue - #tgt_result = np.zeros(tgt_pslice.stop - tgt_pslice.start, self.dtype) tgt_pot_result = 0 tgt_grad_result = 0 @@ -698,48 +718,70 @@ class FMMLibExpansionWrangler(object): target_or_target_parent_boxes, starts, lists, src_weights): local_exps = self.local_expansion_zeros() - formta = self.get_routine("%ddformta" + self.dp_suffix) + formta = self.get_routine("%ddformta" + self.dp_suffix, suffix="_imany") + + sources = self._get_single_sources_array() + sources_offsets = self.tree.box_source_starts[lists] + + nsources = self.tree.box_source_counts_nonchild + nsources_offsets = lists + + centers = self._get_single_box_centers_array() for lev in range(self.tree.nlevels): lev_start, lev_stop = \ level_start_target_or_target_parent_box_nrs[lev:lev+2] + if lev_start == lev_stop: continue - target_level_start_ibox, target_local_exps_view = \ + target_box_start, target_local_exps_view = \ self.local_expansions_view(local_exps, lev) - rscale = self.level_to_rscale(lev) - - for itgt_box, tgt_ibox in enumerate( - target_or_target_parent_boxes[lev_start:lev_stop]): - start, end = starts[lev_start+itgt_box:lev_start+itgt_box+2] - - contrib = 0 - - for src_ibox in lists[start:end]: - src_pslice = self._get_source_slice(src_ibox) - tgt_center = self.tree.box_centers[:, tgt_ibox] + centers_offsets = target_or_target_parent_boxes[lev_start:lev_stop] - if src_pslice.stop - src_pslice.start == 0: - continue - - kwargs = {} - kwargs.update(self.kernel_kwargs) - kwargs.update(self.get_source_kwargs(src_weights, src_pslice)) + """ + ier = np.zeros(lev_stop - lev_start, dtype=np.int) + expn = np.zeros( + (lev_stop - lev_start,) + + self.expansion_shape(self.level_nterms[lev]), + dtype=self.dtype) + """ - ier, mpole = formta( - rscale=rscale, - source=self._get_sources(src_pslice), - center=tgt_center, - nterms=self.level_nterms[lev], - **kwargs) - if ier: - raise RuntimeError("formta failed") + rscale = self.level_to_rscale(lev) - contrib = contrib + mpole.T + sources_starts = starts[lev_start:1 + lev_stop] + nsources_starts = sources_starts - target_local_exps_view[tgt_ibox-target_level_start_ibox] = contrib + kwargs = {} + kwargs.update(self.kernel_kwargs) + kwargs.update( + self.get_source_kwargs( + src_weights, + slice(None), + extra_kwargs={ + "starts": sources_starts, + "offsets": sources_offsets})) + + ier, expn = formta( + rscale=rscale, + sources=sources, + sources_offsets=sources_offsets, + sources_starts=sources_starts, + nsources=nsources, + nsources_starts=nsources_starts, + nsources_offsets=nsources_offsets, + centers=centers, + centers_offsets=centers_offsets, + nterms=self.level_nterms[lev], + **kwargs) + + if ier.any(): + raise RuntimeError("formta failed") + + target_local_exps_view[ + target_or_target_parent_boxes[lev_start:lev_stop] + - target_box_start] = expn.T return local_exps -- GitLab From 52103e18d031f4146a05da9663409a0f3213f4f7 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Mon, 10 Sep 2018 19:51:48 -0500 Subject: [PATCH 2/4] Point requirements.txt to pyfmmlib's formta-openmp branch. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cd0a243..cf616ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy git+https://github.com/inducer/pyopencl git+https://github.com/inducer/islpy git+https://github.com/inducer/loopy -git+https://github.com/inducer/pyfmmlib +git+https://gitlab.tiker.net/inducer/pyfmmlib@formta-openmp # only for reference values for the fmmlib test # (unable to use--circular dep) -- GitLab From 0ed226eb54630744071bc90b8380acfa5c17a396 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Tue, 11 Sep 2018 02:13:12 -0500 Subject: [PATCH 3/4] Add explanatory comments. --- boxtree/pyfmmlib_integration.py | 75 +++++++++++++++------------------ 1 file changed, 33 insertions(+), 42 deletions(-) diff --git a/boxtree/pyfmmlib_integration.py b/boxtree/pyfmmlib_integration.py index 45d54f5..dfff104 100644 --- a/boxtree/pyfmmlib_integration.py +++ b/boxtree/pyfmmlib_integration.py @@ -413,37 +413,23 @@ class FMMLibExpansionWrangler(object): def reorder_potentials(self, potentials): return potentials[self.tree.sorted_target_ids] - def get_source_kwargs(self, src_weights, pslice, extra_kwargs=None): - if extra_kwargs is None: - extra_kwargs = {} - + def get_source_kwargs(self, src_weights, pslice): if self.dipole_vec is None: - result = { + return { "charge": src_weights[pslice], } - for name, val in extra_kwargs.items(): - result["charge_%s" % name] = val - return result - - elif self.eqn_letter == "l" and self.dim == 2: - result = { - "dipstr": -src_weights[pslice] * ( - self.dipole_vec[0, pslice] - + 1j * self.dipole_vec[1, pslice]) - } - for name, val in extra_kwargs.items(): - result["dipstr_%s" % name] = val - return result - else: - result = { - "dipstr": src_weights[pslice], - "dipvec": self.dipole_vec[:, pslice], - } - for name, val in extra_kwargs.items(): - result["dipstr_%s" % name] = val - result["dipvec_%s" % name] = val - return result + if self.eqn_letter == "l" and self.dim == 2: + return { + "dipstr": -src_weights[pslice] * ( + self.dipole_vec[0, pslice] + + 1j * self.dipole_vec[1, pslice]) + } + else: + return { + "dipstr": src_weights[pslice], + "dipvec": self.dipole_vec[:, pslice], + } @log_process(logger) @return_timing_data @@ -555,6 +541,7 @@ class FMMLibExpansionWrangler(object): if tgt_pslice.stop - tgt_pslice.start == 0: continue + #tgt_result = np.zeros(tgt_pslice.stop - tgt_pslice.start, self.dtype) tgt_pot_result = 0 tgt_grad_result = 0 @@ -721,13 +708,23 @@ class FMMLibExpansionWrangler(object): formta = self.get_routine("%ddformta" + self.dp_suffix, suffix="_imany") sources = self._get_single_sources_array() + # sources_starts / sources_lists is a CSR list mapping box centers to + # lists of starting indices into the sources array. To get the starting + # source indices we have to look at box_source_starts. sources_offsets = self.tree.box_source_starts[lists] + # nsources_starts / nsources_lists is a CSR list mapping box centers to + # lists of indices into nsources, each of which represents a source + # count. nsources = self.tree.box_source_counts_nonchild nsources_offsets = lists + # centers is indexed into by values of centers_offsets, which is a list + # mapping box indices to box center indices. centers = self._get_single_box_centers_array() + source_kwargs = self.get_source_kwargs(src_weights, slice(None)) + for lev in range(self.tree.nlevels): lev_start, lev_stop = \ level_start_target_or_target_parent_box_nrs[lev:lev+2] @@ -740,14 +737,6 @@ class FMMLibExpansionWrangler(object): centers_offsets = target_or_target_parent_boxes[lev_start:lev_stop] - """ - ier = np.zeros(lev_stop - lev_start, dtype=np.int) - expn = np.zeros( - (lev_stop - lev_start,) - + self.expansion_shape(self.level_nterms[lev]), - dtype=self.dtype) - """ - rscale = self.level_to_rscale(lev) sources_starts = starts[lev_start:1 + lev_stop] @@ -755,13 +744,15 @@ class FMMLibExpansionWrangler(object): kwargs = {} kwargs.update(self.kernel_kwargs) - kwargs.update( - self.get_source_kwargs( - src_weights, - slice(None), - extra_kwargs={ - "starts": sources_starts, - "offsets": sources_offsets})) + for key, val in source_kwargs.items(): + kwargs[key] = val + # Add CSR lists mapping box centers to lists of starting positions + # in the array of source strengths. + # Since the source strengths have the same order as the sources, + # these lists are the same as those for starting position in the + # sources array. + kwargs[key + "_starts"] = sources_starts + kwargs[key + "_offsets"] = sources_offsets ier, expn = formta( rscale=rscale, -- GitLab From c3014fa98e3a9a6538268e41b40e609308144a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Tue, 11 Sep 2018 12:37:08 -0400 Subject: [PATCH 4/4] Point pyfmmlib back at master branch --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cf616ac..c3174a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy git+https://github.com/inducer/pyopencl git+https://github.com/inducer/islpy git+https://github.com/inducer/loopy -git+https://gitlab.tiker.net/inducer/pyfmmlib@formta-openmp +git+https://gitlab.tiker.net/inducer/pyfmmlib # only for reference values for the fmmlib test # (unable to use--circular dep) -- GitLab