From 94100f7798662518c6ad7ec9180aac2f5f160aa9 Mon Sep 17 00:00:00 2001
From: Matt Wala <wala1@illinois.edu>
Date: Sat, 3 Jun 2017 01:41:31 -0500
Subject: [PATCH 1/3] Reduction: Make repr() deterministic (FIXME: Make sure
 it's an isomorphism).

---
 loopy/library/reduction.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/loopy/library/reduction.py b/loopy/library/reduction.py
index f9648bde7..f62c3a116 100644
--- a/loopy/library/reduction.py
+++ b/loopy/library/reduction.py
@@ -120,6 +120,8 @@ class ScalarReductionOperation(ReductionOperation):
 
         return result
 
+    __repr__ = __str__
+
 
 class SumReductionOperation(ScalarReductionOperation):
     def neutral_element(self, dtype):
-- 
GitLab


From ea40f3461ef74c9cd06ad6d1534df0f6c8f37e6e Mon Sep 17 00:00:00 2001
From: Matt Wala <wala1@illinois.edu>
Date: Sat, 3 Jun 2017 01:42:32 -0500
Subject: [PATCH 2/3] [ci skip] Implement LazilyUnpicklingInstructionList.

---
 loopy/kernel/__init__.py    |  3 ++
 loopy/kernel/instruction.py | 34 +++++++++++++--
 loopy/schedule/__init__.py  |  1 +
 loopy/tools.py              | 87 +++++++++++++++++++++++++++++++++++++
 4 files changed, 121 insertions(+), 4 deletions(-)

diff --git a/loopy/kernel/__init__.py b/loopy/kernel/__init__.py
index dccaca2ec..357baf5b2 100644
--- a/loopy/kernel/__init__.py
+++ b/loopy/kernel/__init__.py
@@ -1398,6 +1398,9 @@ class LoopKernel(ImmutableRecordWithoutPickling):
                 for key in self.__class__.fields
                 if hasattr(self, key))
 
+        from loopy.tools import LazilyUnpicklingInstructionList
+        result["instructions"] = LazilyUnpicklingInstructionList(self.instructions)
+
         result.pop("cache_manager", None)
 
         # make sure that kernels are pickled with a cached hash key in place
diff --git a/loopy/kernel/instruction.py b/loopy/kernel/instruction.py
index 0d22dbb88..4b087a5ce 100644
--- a/loopy/kernel/instruction.py
+++ b/loopy/kernel/instruction.py
@@ -415,10 +415,7 @@ class InstructionBase(ImmutableRecord):
 
         Only works in conjunction with :class:`loopy.tools.KeyBuilder`.
         """
-
-        # Order matters for hash forming--sort the field names
-        for field_name in sorted(self.fields):
-            key_builder.rec(key_hash, getattr(self, field_name))
+        key_builder.update_for_str(self, key_hash, self.comparison_key)
 
     # }}}
 
@@ -451,9 +448,34 @@ class InstructionBase(ImmutableRecord):
         self.within_inames = (
                 intern_frozenset_of_ids(self.within_inames))
 
+    @property
+    @memoize_method
+    def comparison_key(self):
+        attrs = []
+
+        for fld in sorted(self.__class__.fields):
+            if not hasattr(self, fld):
+                continue
+
+            attr = getattr(self, fld)
+            if isinstance(attr, (frozenset, set)):
+                attrs.append(fld + "=" + _repr_set(attr))
+            else:
+                attrs.append(fld + "=" + repr(attr))
+
+        return "%s(%s)" % (
+                self.__class__.__name__,
+                ", ".join(attrs))
+
 # }}}
 
 
+def _repr_set(set_like):
+    return "%s({%s})" % (
+        type(set_like).__name__,
+        ", ".join(repr(item) for item in sorted(set_like)))
+
+
 def _get_assignee_var_name(expr):
     from pymbolic.primitives import Variable, Subscript, Lookup
     from loopy.symbolic import LinearSubscript
@@ -825,6 +847,9 @@ class Assignment(MultiAssignmentBase):
         Only works in conjunction with :class:`loopy.tools.KeyBuilder`.
         """
 
+        key_builder.update_for_str(key_hash, self.comparison_key)
+
+        """
         # Order matters for hash forming--sort the fields.
         for field_name in sorted(self.fields):
             if field_name in ["assignee", "expression"]:
@@ -837,6 +862,7 @@ class Assignment(MultiAssignmentBase):
                             key_hash, pred)
             else:
                 key_builder.rec(key_hash, getattr(self, field_name))
+        """
 
     # {{{ for interface uniformity with CallInstruction
 
diff --git a/loopy/schedule/__init__.py b/loopy/schedule/__init__.py
index c078da2ec..3a6eb3685 100644
--- a/loopy/schedule/__init__.py
+++ b/loopy/schedule/__init__.py
@@ -1977,6 +1977,7 @@ def get_one_scheduled_kernel(kernel):
             logger.debug("%s: schedule cache hit" % kernel.name)
             from_cache = True
         except KeyError:
+            print("not in cache")
             pass
 
     if not from_cache:
diff --git a/loopy/tools.py b/loopy/tools.py
index 56b673b59..9d77bcbfe 100644
--- a/loopy/tools.py
+++ b/loopy/tools.py
@@ -402,6 +402,93 @@ class LazilyUnpicklingDictionary(collections.MutableMapping):
 
 # }}}
 
+# {{{ lazily unpickling instruction list
+
+
+class _PickledInstructionWrapper(object):
+    """A class meant to wrap a pickled instruction (for
+    :class:`LazilyUnpicklingInstructionList`).
+    """
+
+    @classmethod
+    def from_instruction(cls, insn):
+        if isinstance(insn, cls):
+            return insn
+        from pickle import dumps
+        return cls(dumps(insn), insn.comparison_key)
+
+    def __init__(self, objstring, comparison_key):
+        self.objstring = objstring
+        self.comparison_key = comparison_key
+
+    def unpickle(self):
+        from pickle import loads
+        return loads(self.objstring)
+
+    def __eq__(self, other):
+        from loopy.kernel.instruction import InstructionBase
+        if isinstance(other, (InstructionBase, _PickledInstructionWrapper)):
+            return self.comparison_key == other.comparison_key
+        else:
+            return False
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __getstate__(self):
+        return {"objstring": self.objstring, "comparison_key": self.comparison_key}
+
+    def update_persistent_hash(self, key_hash, key_builder):
+        key_builder.update_for_str(key_hash, self.comparison_key)
+
+
+class LazilyUnpicklingInstructionList(collections.MutableSequence):
+    """
+    A list of :mod:`loopy` instructions which lazily unpickles its values.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self._list = list(*args, **kwargs)
+
+    def update_persistent_hash(self, key_hash, key_builder):
+        key_builder.update_for_list(key_hash, self._list)
+
+    def __eq__(self, other):
+        if len(self) != len(other):
+            return False
+        if isinstance(other, LazilyUnpicklingInstructionList):
+            other = other._list
+        res = all(a == b for a, b in zip(self._list, other))
+        return res
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __getitem__(self, key):
+        item = self._list[key]
+        if isinstance(item, _PickledInstructionWrapper):
+            item = self._list[key] = item.unpickle()
+        return item
+
+    def __setitem__(self, key, value):
+        self._list[key] = value
+
+    def __delitem__(self, key):
+        del self._list[key]
+
+    def __len__(self):
+        return len(self._list)
+
+    def insert(self, key, value):
+        self._list.insert(key, value)
+
+    def __getstate__(self):
+        return {"_list": [
+            _PickledInstructionWrapper.from_instruction(val)
+            for val in self._list]}
+
+# }}}
+
 
 def is_interned(s):
     return s is None or intern(s) is s
-- 
GitLab


From dbb30dc371cc9180c3e42e8702befb2be7563671 Mon Sep 17 00:00:00 2001
From: Matt Wala <wala1@illinois.edu>
Date: Sat, 12 Aug 2017 23:11:47 -0500
Subject: [PATCH 3/3] Towards having lazily unpickling data structures. *
 Rename LazilyUnpicklingDictionary to LazyDict. * Implement LazyList and
 LazyListWithFastComparison.

---
 loopy/codegen/__init__.py |   4 +-
 loopy/tools.py            | 157 +++++++++++++++++++-------------------
 test/test_misc.py         | 154 +++++++++++++++++++++++++++++++++----
 3 files changed, 219 insertions(+), 96 deletions(-)

diff --git a/loopy/codegen/__init__.py b/loopy/codegen/__init__.py
index 009dadc1a..392e9bbec 100644
--- a/loopy/codegen/__init__.py
+++ b/loopy/codegen/__init__.py
@@ -507,9 +507,9 @@ def generate_code_v2(kernel):
     # }}}
 
     # For faster unpickling in the common case when implemented_domains isn't needed.
-    from loopy.tools import LazilyUnpicklingDictionary
+    from loopy.tools import LazyDict
     codegen_result = codegen_result.copy(
-            implemented_domains=LazilyUnpicklingDictionary(
+            implemented_domains=LazyDict(
                     codegen_result.implemented_domains))
 
     logger.info("%s: generate code: done" % kernel.name)
diff --git a/loopy/tools.py b/loopy/tools.py
index 9d77bcbfe..85f57a574 100644
--- a/loopy/tools.py
+++ b/loopy/tools.py
@@ -23,6 +23,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 """
 
+import collections
 import numpy as np
 from pytools.persistent_dict import KeyBuilder as KeyBuilderBase
 from loopy.symbolic import WalkMapper as LoopyWalkMapper
@@ -340,23 +341,19 @@ def compute_sccs(graph):
 # }}}
 
 
-# {{{ lazily unpickling dictionary
-
+# {{{ pickled container value
 
-class _PickledObjectWrapper(object):
+class _PickledObject(object):
+    """A class meant to wrap a pickled value (for :class:`LazyDict` and
+    :class:`LazyList`).
     """
-    A class meant to wrap a pickled value (for :class:`LazilyUnpicklingDictionary`).
-    """
-
-    @classmethod
-    def from_object(cls, obj):
-        if isinstance(obj, cls):
-            return obj
-        from pickle import dumps
-        return cls(dumps(obj))
 
-    def __init__(self, objstring):
-        self.objstring = objstring
+    def __init__(self, obj):
+        if isinstance(obj, _PickledObject):
+            self.objstring = obj.objstring
+        else:
+            from pickle import dumps
+            self.objstring = dumps(obj)
 
     def unpickle(self):
         from pickle import loads
@@ -366,12 +363,28 @@ class _PickledObjectWrapper(object):
         return {"objstring": self.objstring}
 
 
-import collections
+class _PickledObjectWithComparisonKey(_PickledObject):
+    """Like :class:`_PickledObject`, with an additional `comparison_key` attribute
+    that can be used for comparison and persistent hasing.
+    """
 
+    def __init__(self, obj, comparison_key):
+        _PickledObject.__init__(self, obj)
+        self.comparison_key = comparison_key
 
-class LazilyUnpicklingDictionary(collections.MutableMapping):
-    """
-    A dictionary-like object which lazily unpickles its values.
+    def update_persistent_hash(self, key_hash, key_builder):
+        key_builder.rec(key_hash, self.comparison_key)
+
+    def __getstate__(self):
+        return {"objstring": self.objstring, "comparison_key": self.comparison_key}
+
+# }}}
+
+
+# {{{ lazily unpickling dictionary
+
+class LazyDict(collections.MutableMapping):
+    """A dictionary-like object which lazily unpickles its values.
     """
 
     def __init__(self, *args, **kwargs):
@@ -379,7 +392,7 @@ class LazilyUnpicklingDictionary(collections.MutableMapping):
 
     def __getitem__(self, key):
         value = self._map[key]
-        if isinstance(value, _PickledObjectWrapper):
+        if isinstance(value, _PickledObject):
             value = self._map[key] = value.unpickle()
         return value
 
@@ -397,95 +410,83 @@ class LazilyUnpicklingDictionary(collections.MutableMapping):
 
     def __getstate__(self):
         return {"_map": dict(
-            (key, _PickledObjectWrapper.from_object(val))
+            (key, _PickledObject(val))
             for key, val in six.iteritems(self._map))}
 
 # }}}
 
-# {{{ lazily unpickling instruction list
 
+# {{{ lazily unpickling list
 
-class _PickledInstructionWrapper(object):
-    """A class meant to wrap a pickled instruction (for
-    :class:`LazilyUnpicklingInstructionList`).
-    """
+class LazyList(collections.MutableSequence):
+    """A list which lazily unpickles its values."""
 
-    @classmethod
-    def from_instruction(cls, insn):
-        if isinstance(insn, cls):
-            return insn
-        from pickle import dumps
-        return cls(dumps(insn), insn.comparison_key)
+    def __init__(self, *args, **kwargs):
+        self._list = list(*args, **kwargs)
 
-    def __init__(self, objstring, comparison_key):
-        self.objstring = objstring
-        self.comparison_key = comparison_key
+    def __getitem__(self, key):
+        item = self._list[key]
+        if isinstance(item, _PickledObject):
+            item = self._list[key] = item.unpickle()
+        return item
 
-    def unpickle(self):
-        from pickle import loads
-        return loads(self.objstring)
+    def __setitem__(self, key, value):
+        self._list[key] = value
 
-    def __eq__(self, other):
-        from loopy.kernel.instruction import InstructionBase
-        if isinstance(other, (InstructionBase, _PickledInstructionWrapper)):
-            return self.comparison_key == other.comparison_key
-        else:
-            return False
+    def __delitem__(self, key):
+        del self._list[key]
 
-    def __ne__(self, other):
-        return not self.__eq__(other)
+    def __len__(self):
+        return len(self._list)
 
-    def __getstate__(self):
-        return {"objstring": self.objstring, "comparison_key": self.comparison_key}
+    def insert(self, key, value):
+        self._list.insert(key, value)
 
-    def update_persistent_hash(self, key_hash, key_builder):
-        key_builder.update_for_str(key_hash, self.comparison_key)
+    def __getstate__(self):
+        return {"_list": [
+                _PickledObject(val)
+                for val in self._list]}
 
 
-class LazilyUnpicklingInstructionList(collections.MutableSequence):
-    """
-    A list of :mod:`loopy` instructions which lazily unpickles its values.
+class LazyListWithFastComparison(LazyList):
+    """A list which lazily unpickles its values, and supports equality comparison of
+    pickled values using a comparison key.
     """
 
-    def __init__(self, *args, **kwargs):
-        self._list = list(*args, **kwargs)
+    def __init__(self, *args, comparison_key_getter=None, **kwargs):
+        LazyList.__init__(self, *args, **kwargs)
+        self.comparison_key_getter = comparison_key_getter
 
     def update_persistent_hash(self, key_hash, key_builder):
         key_builder.update_for_list(key_hash, self._list)
 
+    def _get_comparison_key(self, obj):
+        if isinstance(obj, _PickledObjectWithComparisonKey):
+            return obj.comparison_key
+        return self.comparison_key_getter(obj)
+
     def __eq__(self, other):
+        print("CMP")
         if len(self) != len(other):
             return False
-        if isinstance(other, LazilyUnpicklingInstructionList):
-            other = other._list
-        res = all(a == b for a, b in zip(self._list, other))
-        return res
 
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
-    def __getitem__(self, key):
-        item = self._list[key]
-        if isinstance(item, _PickledInstructionWrapper):
-            item = self._list[key] = item.unpickle()
-        return item
-
-    def __setitem__(self, key, value):
-        self._list[key] = value
+        if isinstance(other, LazyListWithFastComparison):
+            other = other._list
 
-    def __delitem__(self, key):
-        del self._list[key]
+        for a, b in zip(self._list, other):
+            if self._get_comparison_key(a) != self._get_comparison_key(b):
+                return False
 
-    def __len__(self):
-        return len(self._list)
+        return True
 
-    def insert(self, key, value):
-        self._list.insert(key, value)
+    def __ne__(self, other):
+        return not self.__eq__(other)
 
     def __getstate__(self):
-        return {"_list": [
-            _PickledInstructionWrapper.from_instruction(val)
-            for val in self._list]}
+        return {"_list": [_PickledObjectWithComparisonKey(
+                        val, self._get_comparison_key(val))
+                        for val in self._list],
+                "comparison_key_getter": self.comparison_key_getter}
 
 # }}}
 
diff --git a/test/test_misc.py b/test/test_misc.py
index a22e42463..2c06355be 100644
--- a/test/test_misc.py
+++ b/test/test_misc.py
@@ -92,26 +92,36 @@ def test_SetTrie():
         s.add_or_update(set([1, 4]))
 
 
-class PicklableItem(object):
+class PickleDetector(object):
+    """Contains a class attribute which flags if any instance was unpickled.
+    """
 
-    flags = {"unpickled": False}
+    @classmethod
+    def reset(cls):
+        cls.instance_unpickled = False
 
     def __getstate__(self):
-        return True
+        return {"state": self.state}
 
     def __setstate__(self, state):
-        PicklableItem.flags["unpickled"] = True
+        self.__class__.instance_unpickled = True
+        self.state = state["state"]
 
 
-def test_LazilyUnpicklingDictionary():
-    def is_unpickled():
-        return PicklableItem.flags["unpickled"]
+class PickleDetectorForLazyDict(PickleDetector):
+    instance_unpickled = False
 
-    from loopy.tools import LazilyUnpicklingDictionary
+    def __init__(self):
+        self.state = None
 
-    mapping = LazilyUnpicklingDictionary({0: PicklableItem()})
 
-    assert not is_unpickled()
+def test_LazyDict():
+    from loopy.tools import LazyDict
+
+    cls = PickleDetectorForLazyDict
+    mapping = LazyDict({0: cls()})
+
+    assert not cls.instance_unpickled
 
     from pickle import loads, dumps
 
@@ -120,30 +130,142 @@ def test_LazilyUnpicklingDictionary():
     # {{{ test lazy loading
 
     mapping = loads(pickled_mapping)
-    assert not is_unpickled()
+    assert not cls.instance_unpickled
     list(mapping.keys())
-    assert not is_unpickled()
-    assert isinstance(mapping[0], PicklableItem)
-    assert is_unpickled()
+    assert not cls.instance_unpickled
+    assert isinstance(mapping[0], cls)
+    assert cls.instance_unpickled
+
+    # }}}
+
+    # {{{ conversion
+
+    cls.reset()
+    dict(mapping)
+    assert cls.instance_unpickled    
 
     # }}}
 
     # {{{ test multi round trip
 
     mapping = loads(dumps(loads(pickled_mapping)))
-    assert isinstance(mapping[0], PicklableItem)
+    assert isinstance(mapping[0], cls)
 
     # }}}
 
     # {{{ test empty map
 
-    mapping = LazilyUnpicklingDictionary({})
+    mapping = LazyDict({})
     mapping = loads(dumps(mapping))
     assert len(mapping) == 0
 
     # }}}
 
 
+class PickleDetectorForLazyList(PickleDetector):
+    instance_unpickled = False
+
+    def __init__(self):
+        self.state = None
+
+
+def test_LazyList():
+    from loopy.tools import LazyList
+
+    cls = PickleDetectorForLazyList
+    lst = LazyList([cls()])
+    assert not cls.instance_unpickled
+
+    from pickle import loads, dumps
+    pickled_lst = dumps(lst)
+
+    # {{{ test lazy loading
+
+    lst = loads(pickled_lst)
+    assert not cls.instance_unpickled
+    assert isinstance(lst[0], cls)
+    assert cls.instance_unpickled
+
+    # }}}
+
+    # {{{ conversion
+
+    cls.reset()
+    list(lst)
+    assert cls.instance_unpickled    
+
+    # }}}
+
+    # {{{ test multi round trip
+
+    lst = loads(dumps(loads(dumps(lst))))
+    assert isinstance(lst[0], cls)
+
+    # }}}
+
+    # {{{ test empty list
+
+    lst = LazyList([])
+    lst = loads(dumps(lst))
+    assert len(lst) == 0
+
+    # }}}
+
+
+class PickleDetectorForLazyListWithFastComparison(PickleDetector):
+    instance_unpickled = False
+
+    def __init__(self, comparison_key):
+        self.state = comparison_key
+
+    def __repr__(self):
+        return repr(self.state)
+
+
+def test_LazyListWithFastComparison():
+    from loopy.tools import LazyListWithFastComparison
+
+    cls = PickleDetectorForLazyListWithFastComparison
+    from pickle import loads, dumps
+
+    # {{{ test comparison of a pair of lazy lists
+
+    lst0 = LazyListWithFastComparison(
+            [cls(0), cls(1)],
+            comparison_key_getter=repr)
+    lst1 = LazyListWithFastComparison(
+            [cls(0), cls(1)],
+            comparison_key_getter=repr)
+
+    assert not cls.instance_unpickled
+
+    assert lst0 == lst1
+    assert not cls.instance_unpickled
+
+    lst0 = loads(dumps(lst0))
+    lst1 = loads(dumps(lst1))
+
+    assert lst0 == lst1
+    assert not cls.instance_unpickled
+
+    lst0.append(cls(3))
+    lst1.append(cls(2))
+
+    assert lst0 != lst1
+
+    # }}}
+
+    # {{{ comparison with plain lists
+
+    lst = [cls(0), cls(1), cls(3)]
+
+    assert lst == lst0
+    assert lst0 == lst
+    assert not cls.instance_unpickled
+
+    # }}}
+
+
 if __name__ == "__main__":
     if len(sys.argv) > 1:
         exec(sys.argv[1])
-- 
GitLab