From 398c401db46b5dd05c944241eb03e050f20fd136 Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Thu, 21 Jun 2018 19:26:06 -0500 Subject: [PATCH 1/3] Ensure that registered callback functions unpickle equally, to avoid cache collisions --- loopy/__init__.py | 24 ++++++++++++++++++++++++ loopy/tools.py | 8 +++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/loopy/__init__.py b/loopy/__init__.py index 54c3523d5..92b7fca77 100644 --- a/loopy/__init__.py +++ b/loopy/__init__.py @@ -331,18 +331,34 @@ def register_preamble_generators(kernel, preamble_generators): :returns: *kernel* with *manglers* registered """ + from loopy.tools import unpickles_equally + new_pgens = kernel.preamble_generators[:] for pgen in preamble_generators: if pgen not in new_pgens: + if not unpickles_equally(pgen): + raise LoopyError("preamble generator '%s' does not " + "compare equally after being upickled " + "and would thus disrupt loopy's caches" + % pgen) + new_pgens.insert(0, pgen) return kernel.copy(preamble_generators=new_pgens) def register_symbol_manglers(kernel, manglers): + from loopy.tools import unpickles_equally + new_manglers = kernel.symbol_manglers[:] for m in manglers: if m not in new_manglers: + if not unpickles_equally(m): + raise LoopyError("mangler '%s' does not " + "compare equally after being upickled " + "and would disrupt loopy's caches" + % m) + new_manglers.insert(0, m) return kernel.copy(symbol_manglers=new_manglers) @@ -354,9 +370,17 @@ def register_function_manglers(kernel, manglers): returning a :class:`loopy.CallMangleInfo`. :returns: *kernel* with *manglers* registered """ + from loopy.tools import unpickles_equally + new_manglers = kernel.function_manglers[:] for m in manglers: if m not in new_manglers: + if not unpickles_equally(m): + raise LoopyError("mangler '%s' does not " + "compare equally after being upickled " + "and would disrupt loopy's caches" + % m) + new_manglers.insert(0, m) return kernel.copy(function_manglers=new_manglers) diff --git a/loopy/tools.py b/loopy/tools.py index 15d2a859a..8c5d36390 100644 --- a/loopy/tools.py +++ b/loopy/tools.py @@ -1,5 +1,4 @@ from __future__ import division, absolute_import -import six __copyright__ = "Copyright (C) 2012 Andreas Kloeckner" @@ -23,6 +22,8 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import six + import collections import numpy as np from pytools import memoize_method @@ -581,6 +582,11 @@ class LazilyUnpicklingListWithEqAndPersistentHashing(LazilyUnpicklingList): # }}} +def unpickles_equally(obj): + from six.moves.cPickle import loads, dumps + return loads(dumps(obj)) == obj + + def is_interned(s): return s is None or intern(s) is s -- GitLab From 05de1b5835737601b37cae1c82f23025700efc1d Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Thu, 21 Jun 2018 19:26:39 -0500 Subject: [PATCH 2/3] Adapt test_preamble_with_separate_temporaries to make sure registered callables unpickle equally --- test/test_loopy.py | 12 ++++++++---- test/testlib.py | 26 +++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/test/test_loopy.py b/test/test_loopy.py index 07179ae14..3ceca5a75 100644 --- a/test/test_loopy.py +++ b/test/test_loopy.py @@ -2719,9 +2719,13 @@ def test_preamble_with_separate_temporaries(ctx_factory): read_only=True), lp.GlobalArg('data', shape=(data.size,), dtype=np.float64)], ) + # fixt params, and add manglers / preamble - from testlib import SeparateTemporariesPreambleTestHelper - preamble_with_sep_helper = SeparateTemporariesPreambleTestHelper( + from testlib import ( + SeparateTemporariesPreambleTestMangler, + SeparateTemporariesPreambleTestPreambleGenerator, + ) + func_info = dict( func_name='indirect', func_arg_dtypes=(np.int32, np.int32, np.int32), func_result_dtypes=(np.int32,), @@ -2730,9 +2734,9 @@ def test_preamble_with_separate_temporaries(ctx_factory): kernel = lp.fix_parameters(kernel, **{'n': n}) kernel = lp.register_preamble_generators( - kernel, [preamble_with_sep_helper.preamble_gen]) + kernel, [SeparateTemporariesPreambleTestPreambleGenerator(**func_info)]) kernel = lp.register_function_manglers( - kernel, [preamble_with_sep_helper.mangler]) + kernel, [SeparateTemporariesPreambleTestMangler(**func_info)]) print(lp.generate_code(kernel)[0]) # and call (functionality unimportant, more that it compiles) diff --git a/test/testlib.py b/test/testlib.py index 73de4199d..399b4852a 100644 --- a/test/testlib.py +++ b/test/testlib.py @@ -1,5 +1,7 @@ import loopy as lp +from collections import namedtuple + # {{{ test_barrier_in_overridden_get_grid_size_expanded_kernel @@ -17,14 +19,29 @@ class GridOverride(object): # {{{ test_preamble_with_separate_temporaries -class SeparateTemporariesPreambleTestHelper: +class SeparateTemporariesPreambleTestDataHolder: def __init__(self, func_name, func_arg_dtypes, func_result_dtypes, arr): self.func_name = func_name self.func_arg_dtypes = func_arg_dtypes self.func_result_dtypes = func_result_dtypes self.arr = arr - def mangler(self, kernel, name, arg_dtypes): + def __eq__(self, other): + import numpy as np + return ( + isinstance(other, type(self)) + and self.func_name == other.func_name + and self.func_arg_dtypes == other.func_arg_dtypes + and self.func_result_dtypes == other.func_result_dtypes + and np.array_equal(self.arr, other.arr)) + + def __ne__(self, other): + return not self.__eq__(other) + + +class SeparateTemporariesPreambleTestMangler( + SeparateTemporariesPreambleTestDataHolder): + def __call__(self, kernel, name, arg_dtypes): """ A function that will return a :class:`loopy.kernel.data.CallMangleInfo` to interface with the calling :class:`loopy.LoopKernel` @@ -61,7 +78,10 @@ class SeparateTemporariesPreambleTestHelper: self.func_result_dtypes), arg_dtypes=arg_dtypes) - def preamble_gen(self, preamble_info): + +class SeparateTemporariesPreambleTestPreambleGenerator( + SeparateTemporariesPreambleTestDataHolder): + def __call__(self, preamble_info): from loopy.kernel.data import temp_var_scope as scopes # find a function matching our name -- GitLab From 4ca061d46db6a8ff56898eb38dc5d454d17d6c85 Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Thu, 21 Jun 2018 19:43:42 -0500 Subject: [PATCH 3/3] Remove unused import --- test/testlib.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/testlib.py b/test/testlib.py index 399b4852a..ad290ee7c 100644 --- a/test/testlib.py +++ b/test/testlib.py @@ -1,7 +1,5 @@ import loopy as lp -from collections import namedtuple - # {{{ test_barrier_in_overridden_get_grid_size_expanded_kernel -- GitLab