diff --git a/loopy/codegen/__init__.py b/loopy/codegen/__init__.py index 009dadc1a0d6236f092029dbc03ad0c035c7b8f8..392e9bbecd277da7bbeb9f47b0f9818a8f17a0d2 100644 --- a/loopy/codegen/__init__.py +++ b/loopy/codegen/__init__.py @@ -507,9 +507,9 @@ def generate_code_v2(kernel): # }}} # For faster unpickling in the common case when implemented_domains isn't needed. - from loopy.tools import LazilyUnpicklingDictionary + from loopy.tools import LazyDict codegen_result = codegen_result.copy( - implemented_domains=LazilyUnpicklingDictionary( + implemented_domains=LazyDict( codegen_result.implemented_domains)) logger.info("%s: generate code: done" % kernel.name) diff --git a/loopy/kernel/__init__.py b/loopy/kernel/__init__.py index 622f5e49be1e40b4156113d92907fe8b1d9fb859..fd0b1eff0af85f5445b4219e1fcae696475f3ee8 100644 --- a/loopy/kernel/__init__.py +++ b/loopy/kernel/__init__.py @@ -1410,6 +1410,9 @@ class LoopKernel(ImmutableRecordWithoutPickling): for key in self.__class__.fields if hasattr(self, key)) + from loopy.tools import LazilyUnpicklingInstructionList + result["instructions"] = LazilyUnpicklingInstructionList(self.instructions) + result.pop("cache_manager", None) # make sure that kernels are pickled with a cached hash key in place diff --git a/loopy/kernel/instruction.py b/loopy/kernel/instruction.py index d5c388af60a39987c09092fc93325f067a8f4cf7..d66f1fa6a59c4cd5bf477693db1ce92af54d031b 100644 --- a/loopy/kernel/instruction.py +++ b/loopy/kernel/instruction.py @@ -415,10 +415,7 @@ class InstructionBase(ImmutableRecord): Only works in conjunction with :class:`loopy.tools.KeyBuilder`. """ - - # Order matters for hash forming--sort the field names - for field_name in sorted(self.fields): - key_builder.rec(key_hash, getattr(self, field_name)) + key_builder.update_for_str(self, key_hash, self.comparison_key) # }}} @@ -451,9 +448,34 @@ class InstructionBase(ImmutableRecord): self.within_inames = ( intern_frozenset_of_ids(self.within_inames)) + @property + @memoize_method + def comparison_key(self): + attrs = [] + + for fld in sorted(self.__class__.fields): + if not hasattr(self, fld): + continue + + attr = getattr(self, fld) + if isinstance(attr, (frozenset, set)): + attrs.append(fld + "=" + _repr_set(attr)) + else: + attrs.append(fld + "=" + repr(attr)) + + return "%s(%s)" % ( + self.__class__.__name__, + ", ".join(attrs)) + # }}} +def _repr_set(set_like): + return "%s({%s})" % ( + type(set_like).__name__, + ", ".join(repr(item) for item in sorted(set_like))) + + def _get_assignee_var_name(expr): from pymbolic.primitives import Variable, Subscript, Lookup from loopy.symbolic import LinearSubscript @@ -825,6 +847,9 @@ class Assignment(MultiAssignmentBase): Only works in conjunction with :class:`loopy.tools.KeyBuilder`. """ + key_builder.update_for_str(key_hash, self.comparison_key) + + """ # Order matters for hash forming--sort the fields. for field_name in sorted(self.fields): if field_name in ["assignee", "expression"]: @@ -837,6 +862,7 @@ class Assignment(MultiAssignmentBase): key_hash, pred) else: key_builder.rec(key_hash, getattr(self, field_name)) + """ # {{{ for interface uniformity with CallInstruction diff --git a/loopy/library/reduction.py b/loopy/library/reduction.py index 0e5a093b76b8d09d331edead7c69fcc2e3134601..29a604681bf96f94a61ee2e2bf22148684755836 100644 --- a/loopy/library/reduction.py +++ b/loopy/library/reduction.py @@ -120,6 +120,8 @@ class ScalarReductionOperation(ReductionOperation): return result + __repr__ = __str__ + class SumReductionOperation(ScalarReductionOperation): def neutral_element(self, dtype): diff --git a/loopy/schedule/__init__.py b/loopy/schedule/__init__.py index 4281e50bd006a3cddf5a3cae0ffffe3d78abcfac..c7b006c4d4c61129ab13e062b2496387d478d0ae 100644 --- a/loopy/schedule/__init__.py +++ b/loopy/schedule/__init__.py @@ -1956,6 +1956,7 @@ def get_one_scheduled_kernel(kernel): logger.debug("%s: schedule cache hit" % kernel.name) from_cache = True except KeyError: + print("not in cache") pass if not from_cache: diff --git a/loopy/tools.py b/loopy/tools.py index 56b673b597fc3bf43a6b03f87607ea8d3db0866a..85f57a5741b0a8f9f484c48e13ba883c64c92514 100644 --- a/loopy/tools.py +++ b/loopy/tools.py @@ -23,6 +23,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import collections import numpy as np from pytools.persistent_dict import KeyBuilder as KeyBuilderBase from loopy.symbolic import WalkMapper as LoopyWalkMapper @@ -340,23 +341,19 @@ def compute_sccs(graph): # }}} -# {{{ lazily unpickling dictionary - +# {{{ pickled container value -class _PickledObjectWrapper(object): - """ - A class meant to wrap a pickled value (for :class:`LazilyUnpicklingDictionary`). +class _PickledObject(object): + """A class meant to wrap a pickled value (for :class:`LazyDict` and + :class:`LazyList`). """ - @classmethod - def from_object(cls, obj): - if isinstance(obj, cls): - return obj - from pickle import dumps - return cls(dumps(obj)) - - def __init__(self, objstring): - self.objstring = objstring + def __init__(self, obj): + if isinstance(obj, _PickledObject): + self.objstring = obj.objstring + else: + from pickle import dumps + self.objstring = dumps(obj) def unpickle(self): from pickle import loads @@ -366,12 +363,28 @@ class _PickledObjectWrapper(object): return {"objstring": self.objstring} -import collections +class _PickledObjectWithComparisonKey(_PickledObject): + """Like :class:`_PickledObject`, with an additional `comparison_key` attribute + that can be used for comparison and persistent hasing. + """ + def __init__(self, obj, comparison_key): + _PickledObject.__init__(self, obj) + self.comparison_key = comparison_key -class LazilyUnpicklingDictionary(collections.MutableMapping): - """ - A dictionary-like object which lazily unpickles its values. + def update_persistent_hash(self, key_hash, key_builder): + key_builder.rec(key_hash, self.comparison_key) + + def __getstate__(self): + return {"objstring": self.objstring, "comparison_key": self.comparison_key} + +# }}} + + +# {{{ lazily unpickling dictionary + +class LazyDict(collections.MutableMapping): + """A dictionary-like object which lazily unpickles its values. """ def __init__(self, *args, **kwargs): @@ -379,7 +392,7 @@ class LazilyUnpicklingDictionary(collections.MutableMapping): def __getitem__(self, key): value = self._map[key] - if isinstance(value, _PickledObjectWrapper): + if isinstance(value, _PickledObject): value = self._map[key] = value.unpickle() return value @@ -397,12 +410,87 @@ class LazilyUnpicklingDictionary(collections.MutableMapping): def __getstate__(self): return {"_map": dict( - (key, _PickledObjectWrapper.from_object(val)) + (key, _PickledObject(val)) for key, val in six.iteritems(self._map))} # }}} +# {{{ lazily unpickling list + +class LazyList(collections.MutableSequence): + """A list which lazily unpickles its values.""" + + def __init__(self, *args, **kwargs): + self._list = list(*args, **kwargs) + + def __getitem__(self, key): + item = self._list[key] + if isinstance(item, _PickledObject): + item = self._list[key] = item.unpickle() + return item + + def __setitem__(self, key, value): + self._list[key] = value + + def __delitem__(self, key): + del self._list[key] + + def __len__(self): + return len(self._list) + + def insert(self, key, value): + self._list.insert(key, value) + + def __getstate__(self): + return {"_list": [ + _PickledObject(val) + for val in self._list]} + + +class LazyListWithFastComparison(LazyList): + """A list which lazily unpickles its values, and supports equality comparison of + pickled values using a comparison key. + """ + + def __init__(self, *args, comparison_key_getter=None, **kwargs): + LazyList.__init__(self, *args, **kwargs) + self.comparison_key_getter = comparison_key_getter + + def update_persistent_hash(self, key_hash, key_builder): + key_builder.update_for_list(key_hash, self._list) + + def _get_comparison_key(self, obj): + if isinstance(obj, _PickledObjectWithComparisonKey): + return obj.comparison_key + return self.comparison_key_getter(obj) + + def __eq__(self, other): + print("CMP") + if len(self) != len(other): + return False + + if isinstance(other, LazyListWithFastComparison): + other = other._list + + for a, b in zip(self._list, other): + if self._get_comparison_key(a) != self._get_comparison_key(b): + return False + + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __getstate__(self): + return {"_list": [_PickledObjectWithComparisonKey( + val, self._get_comparison_key(val)) + for val in self._list], + "comparison_key_getter": self.comparison_key_getter} + +# }}} + + def is_interned(s): return s is None or intern(s) is s diff --git a/test/test_misc.py b/test/test_misc.py index a22e424630255df4225586eeb9f0d62a03d5318f..2c06355be6e5dd846cb9f9ea265d027867e4879c 100644 --- a/test/test_misc.py +++ b/test/test_misc.py @@ -92,26 +92,36 @@ def test_SetTrie(): s.add_or_update(set([1, 4])) -class PicklableItem(object): +class PickleDetector(object): + """Contains a class attribute which flags if any instance was unpickled. + """ - flags = {"unpickled": False} + @classmethod + def reset(cls): + cls.instance_unpickled = False def __getstate__(self): - return True + return {"state": self.state} def __setstate__(self, state): - PicklableItem.flags["unpickled"] = True + self.__class__.instance_unpickled = True + self.state = state["state"] -def test_LazilyUnpicklingDictionary(): - def is_unpickled(): - return PicklableItem.flags["unpickled"] +class PickleDetectorForLazyDict(PickleDetector): + instance_unpickled = False - from loopy.tools import LazilyUnpicklingDictionary + def __init__(self): + self.state = None - mapping = LazilyUnpicklingDictionary({0: PicklableItem()}) - assert not is_unpickled() +def test_LazyDict(): + from loopy.tools import LazyDict + + cls = PickleDetectorForLazyDict + mapping = LazyDict({0: cls()}) + + assert not cls.instance_unpickled from pickle import loads, dumps @@ -120,30 +130,142 @@ def test_LazilyUnpicklingDictionary(): # {{{ test lazy loading mapping = loads(pickled_mapping) - assert not is_unpickled() + assert not cls.instance_unpickled list(mapping.keys()) - assert not is_unpickled() - assert isinstance(mapping[0], PicklableItem) - assert is_unpickled() + assert not cls.instance_unpickled + assert isinstance(mapping[0], cls) + assert cls.instance_unpickled + + # }}} + + # {{{ conversion + + cls.reset() + dict(mapping) + assert cls.instance_unpickled # }}} # {{{ test multi round trip mapping = loads(dumps(loads(pickled_mapping))) - assert isinstance(mapping[0], PicklableItem) + assert isinstance(mapping[0], cls) # }}} # {{{ test empty map - mapping = LazilyUnpicklingDictionary({}) + mapping = LazyDict({}) mapping = loads(dumps(mapping)) assert len(mapping) == 0 # }}} +class PickleDetectorForLazyList(PickleDetector): + instance_unpickled = False + + def __init__(self): + self.state = None + + +def test_LazyList(): + from loopy.tools import LazyList + + cls = PickleDetectorForLazyList + lst = LazyList([cls()]) + assert not cls.instance_unpickled + + from pickle import loads, dumps + pickled_lst = dumps(lst) + + # {{{ test lazy loading + + lst = loads(pickled_lst) + assert not cls.instance_unpickled + assert isinstance(lst[0], cls) + assert cls.instance_unpickled + + # }}} + + # {{{ conversion + + cls.reset() + list(lst) + assert cls.instance_unpickled + + # }}} + + # {{{ test multi round trip + + lst = loads(dumps(loads(dumps(lst)))) + assert isinstance(lst[0], cls) + + # }}} + + # {{{ test empty list + + lst = LazyList([]) + lst = loads(dumps(lst)) + assert len(lst) == 0 + + # }}} + + +class PickleDetectorForLazyListWithFastComparison(PickleDetector): + instance_unpickled = False + + def __init__(self, comparison_key): + self.state = comparison_key + + def __repr__(self): + return repr(self.state) + + +def test_LazyListWithFastComparison(): + from loopy.tools import LazyListWithFastComparison + + cls = PickleDetectorForLazyListWithFastComparison + from pickle import loads, dumps + + # {{{ test comparison of a pair of lazy lists + + lst0 = LazyListWithFastComparison( + [cls(0), cls(1)], + comparison_key_getter=repr) + lst1 = LazyListWithFastComparison( + [cls(0), cls(1)], + comparison_key_getter=repr) + + assert not cls.instance_unpickled + + assert lst0 == lst1 + assert not cls.instance_unpickled + + lst0 = loads(dumps(lst0)) + lst1 = loads(dumps(lst1)) + + assert lst0 == lst1 + assert not cls.instance_unpickled + + lst0.append(cls(3)) + lst1.append(cls(2)) + + assert lst0 != lst1 + + # }}} + + # {{{ comparison with plain lists + + lst = [cls(0), cls(1), cls(3)] + + assert lst == lst0 + assert lst0 == lst + assert not cls.instance_unpickled + + # }}} + + if __name__ == "__main__": if len(sys.argv) > 1: exec(sys.argv[1])