From 8be8cd3aa50f0a6700f1b8d146f2316b1000c6b5 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Tue, 29 Aug 2017 20:10:48 -0500 Subject: [PATCH 1/3] Implement a lazily unpickling list data structure. This changes implements a LazyList and LazyListWithEqAndPersistentHashing data structure. These data structures lazily unpickle their contents. This change also renames LazilyUnpicklingDictionary to LazyDict. --- loopy/codegen/__init__.py | 4 +- loopy/tools.py | 151 ++++++++++++++++++++++++++++----- test/test_misc.py | 171 ++++++++++++++++++++++++++++++++++---- 3 files changed, 289 insertions(+), 37 deletions(-) diff --git a/loopy/codegen/__init__.py b/loopy/codegen/__init__.py index 009dadc1a..392e9bbec 100644 --- a/loopy/codegen/__init__.py +++ b/loopy/codegen/__init__.py @@ -507,9 +507,9 @@ def generate_code_v2(kernel): # }}} # For faster unpickling in the common case when implemented_domains isn't needed. - from loopy.tools import LazilyUnpicklingDictionary + from loopy.tools import LazyDict codegen_result = codegen_result.copy( - implemented_domains=LazilyUnpicklingDictionary( + implemented_domains=LazyDict( codegen_result.implemented_domains)) logger.info("%s: generate code: done" % kernel.name) diff --git a/loopy/tools.py b/loopy/tools.py index 56b673b59..a7e536989 100644 --- a/loopy/tools.py +++ b/loopy/tools.py @@ -23,6 +23,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import collections import numpy as np from pytools.persistent_dict import KeyBuilder as KeyBuilderBase from loopy.symbolic import WalkMapper as LoopyWalkMapper @@ -340,23 +341,19 @@ def compute_sccs(graph): # }}} -# {{{ lazily unpickling dictionary - +# {{{ pickled container value -class _PickledObjectWrapper(object): - """ - A class meant to wrap a pickled value (for :class:`LazilyUnpicklingDictionary`). +class _PickledObject(object): + """A class meant to wrap a pickled value (for :class:`LazyDict` and + :class:`LazyList`). """ - @classmethod - def from_object(cls, obj): - if isinstance(obj, cls): - return obj - from pickle import dumps - return cls(dumps(obj)) - - def __init__(self, objstring): - self.objstring = objstring + def __init__(self, obj): + if isinstance(obj, _PickledObject): + self.objstring = obj.objstring + else: + from pickle import dumps + self.objstring = dumps(obj) def unpickle(self): from pickle import loads @@ -366,12 +363,35 @@ class _PickledObjectWrapper(object): return {"objstring": self.objstring} -import collections +class _PickledObjectWithEqAndPersistentHashKeys(_PickledObject): + """Like :class:`_PickledObject`, with two additional attributes: + * `eq_key` + * `persistent_hash_key` -class LazilyUnpicklingDictionary(collections.MutableMapping): + This allows for comparison and for persistent hashing without unpickling. """ - A dictionary-like object which lazily unpickles its values. + + def __init__(self, obj, eq_key, persistent_hash_key): + _PickledObject.__init__(self, obj) + self.eq_key = eq_key + self.persistent_hash_key = persistent_hash_key + + def update_persistent_hash(self, key_hash, key_builder): + key_builder.rec(key_hash, self.persistent_hash_key) + + def __getstate__(self): + return {"objstring": self.objstring, + "eq_key": self.eq_key, + "persistent_hash_key": self.persistent_hash_key} + +# }}} + + +# {{{ lazily unpickling dictionary + +class LazyDict(collections.MutableMapping): + """A dictionary-like object which lazily unpickles its values. """ def __init__(self, *args, **kwargs): @@ -379,7 +399,7 @@ class LazilyUnpicklingDictionary(collections.MutableMapping): def __getitem__(self, key): value = self._map[key] - if isinstance(value, _PickledObjectWrapper): + if isinstance(value, _PickledObject): value = self._map[key] = value.unpickle() return value @@ -397,12 +417,105 @@ class LazilyUnpicklingDictionary(collections.MutableMapping): def __getstate__(self): return {"_map": dict( - (key, _PickledObjectWrapper.from_object(val)) + (key, _PickledObject(val)) for key, val in six.iteritems(self._map))} # }}} +# {{{ lazily unpickling list + +class LazyList(collections.MutableSequence): + """A list which lazily unpickles its values.""" + + def __init__(self, *args, **kwargs): + self._list = list(*args, **kwargs) + + def __getitem__(self, key): + item = self._list[key] + if isinstance(item, _PickledObject): + item = self._list[key] = item.unpickle() + return item + + def __setitem__(self, key, value): + self._list[key] = value + + def __delitem__(self, key): + del self._list[key] + + def __len__(self): + return len(self._list) + + def insert(self, key, value): + self._list.insert(key, value) + + def __getstate__(self): + return {"_list": [_PickledObject(val) for val in self._list]} + + +class LazyListWithEqAndPersistentHashing(LazyList): + """A list which lazily unpickles its values, and supports equality comparison + and persistent hashing without unpickling. + + Persistent hashing only works in conjunction with :class:`LoopyKeyBuilder`. + + Equality comparison and persistent hashing are implemented by supplying + functions `eq_key_getter` and `persistent_hash_key_getter` to the + constructor. These functions should return keys that can be used in place of + the original object for the respective purposes of equality comparison and + persistent hashing. + """ + + def __init__(self, *args, **kwargs): + self.eq_key_getter = kwargs.pop("eq_key_getter") + self.persistent_hash_key_getter = kwargs.pop("persistent_hash_key_getter") + LazyList.__init__(self, *args, **kwargs) + + def update_persistent_hash(self, key_hash, key_builder): + key_builder.update_for_list(key_hash, self._list) + + def _get_eq_key(self, obj): + if isinstance(obj, _PickledObjectWithEqAndPersistentHashKeys): + return obj.eq_key + return self.eq_key_getter(obj) + + def _get_persistent_hash_key(self, obj): + if isinstance(obj, _PickledObjectWithEqAndPersistentHashKeys): + return obj.persistent_hash_key + return self.persistent_hash_key_getter(obj) + + def __eq__(self, other): + if not isinstance(other, (list, LazyList)): + return NotImplemented + + if isinstance(other, LazyList): + other = other._list + + if len(self) != len(other): + return False + + for a, b in zip(self._list, other): + if self._get_eq_key(a) != self._get_eq_key(b): + return False + + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __getstate__(self): + return {"_list": [ + _PickledObjectWithEqAndPersistentHashKeys( + val, + self._get_eq_key(val), + self._get_persistent_hash_key(val)) + for val in self._list], + "eq_key_getter": self.eq_key_getter, + "persistent_hash_key_getter": self.persistent_hash_key_getter} + +# }}} + + def is_interned(s): return s is None or intern(s) is s diff --git a/test/test_misc.py b/test/test_misc.py index a22e42463..53da8ee61 100644 --- a/test/test_misc.py +++ b/test/test_misc.py @@ -92,26 +92,36 @@ def test_SetTrie(): s.add_or_update(set([1, 4])) -class PicklableItem(object): +class PickleDetector(object): + """Contains a class attribute which flags if any instance was unpickled. + """ - flags = {"unpickled": False} + @classmethod + def reset(cls): + cls.instance_unpickled = False def __getstate__(self): - return True + return {"state": self.state} def __setstate__(self, state): - PicklableItem.flags["unpickled"] = True + self.__class__.instance_unpickled = True + self.state = state["state"] -def test_LazilyUnpicklingDictionary(): - def is_unpickled(): - return PicklableItem.flags["unpickled"] +class PickleDetectorForLazyDict(PickleDetector): + instance_unpickled = False - from loopy.tools import LazilyUnpicklingDictionary + def __init__(self): + self.state = None - mapping = LazilyUnpicklingDictionary({0: PicklableItem()}) - assert not is_unpickled() +def test_LazyDict(): + from loopy.tools import LazyDict + + cls = PickleDetectorForLazyDict + mapping = LazyDict({0: cls()}) + + assert not cls.instance_unpickled from pickle import loads, dumps @@ -120,30 +130,159 @@ def test_LazilyUnpicklingDictionary(): # {{{ test lazy loading mapping = loads(pickled_mapping) - assert not is_unpickled() + assert not cls.instance_unpickled list(mapping.keys()) - assert not is_unpickled() - assert isinstance(mapping[0], PicklableItem) - assert is_unpickled() + assert not cls.instance_unpickled + assert isinstance(mapping[0], cls) + assert cls.instance_unpickled + + # }}} + + # {{{ conversion + + cls.reset() + mapping = loads(pickled_mapping) + dict(mapping) + assert cls.instance_unpickled # }}} # {{{ test multi round trip mapping = loads(dumps(loads(pickled_mapping))) - assert isinstance(mapping[0], PicklableItem) + assert isinstance(mapping[0], cls) # }}} # {{{ test empty map - mapping = LazilyUnpicklingDictionary({}) + mapping = LazyDict({}) mapping = loads(dumps(mapping)) assert len(mapping) == 0 # }}} +class PickleDetectorForLazyList(PickleDetector): + instance_unpickled = False + + def __init__(self): + self.state = None + + +def test_LazyList(): + from loopy.tools import LazyList + + cls = PickleDetectorForLazyList + lst = LazyList([cls()]) + assert not cls.instance_unpickled + + from pickle import loads, dumps + pickled_lst = dumps(lst) + + # {{{ test lazy loading + + lst = loads(pickled_lst) + assert not cls.instance_unpickled + assert isinstance(lst[0], cls) + assert cls.instance_unpickled + + # }}} + + # {{{ conversion + + cls.reset() + lst = loads(pickled_lst) + list(lst) + assert cls.instance_unpickled + + # }}} + + # {{{ test multi round trip + + lst = loads(dumps(loads(dumps(lst)))) + assert isinstance(lst[0], cls) + + # }}} + + # {{{ test empty list + + lst = LazyList([]) + lst = loads(dumps(lst)) + assert len(lst) == 0 + + # }}} + + +class PickleDetectorForLazyListWithEqAndPersistentHashing(PickleDetector): + instance_unpickled = False + + def __init__(self, comparison_key): + self.state = comparison_key + + def __repr__(self): + return repr(self.state) + + def update_persistent_hash(self, key_hash, key_builder): + key_builder.rec(key_hash, repr(self)) + + +def test_LazyListWithEqAndPersistentHashing(): + from loopy.tools import LazyListWithEqAndPersistentHashing + + cls = PickleDetectorForLazyListWithEqAndPersistentHashing + from pickle import loads, dumps + + # {{{ test comparison of a pair of lazy lists + + lst0 = LazyListWithEqAndPersistentHashing( + [cls(0), cls(1)], + eq_key_getter=repr, + persistent_hash_key_getter=repr) + lst1 = LazyListWithEqAndPersistentHashing( + [cls(0), cls(1)], + eq_key_getter=repr, + persistent_hash_key_getter=repr) + + assert not cls.instance_unpickled + + assert lst0 == lst1 + assert not cls.instance_unpickled + + lst0 = loads(dumps(lst0)) + lst1 = loads(dumps(lst1)) + + assert lst0 == lst1 + assert not cls.instance_unpickled + + lst0.append(cls(3)) + lst1.append(cls(2)) + + assert lst0 != lst1 + + # }}} + + # {{{ comparison with plain lists + + lst = [cls(0), cls(1), cls(3)] + + assert lst == lst0 + assert lst0 == lst + assert not cls.instance_unpickled + + # }}} + + # {{{ persistent hashing + + from loopy.tools import LoopyKeyBuilder + kb = LoopyKeyBuilder() + + assert kb(lst0) == kb(lst) + assert not cls.instance_unpickled + + # }}} + + if __name__ == "__main__": if len(sys.argv) > 1: exec(sys.argv[1]) -- GitLab From 678188897b68fb6537bccfe985d9b066eefd39c0 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Tue, 29 Aug 2017 20:13:23 -0500 Subject: [PATCH 2/3] Bump version for lazy dict rename. --- loopy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loopy/version.py b/loopy/version.py index 2e86b974b..f16d689bb 100644 --- a/loopy/version.py +++ b/loopy/version.py @@ -32,4 +32,4 @@ except ImportError: else: _islpy_version = islpy.version.VERSION_TEXT -DATA_MODEL_VERSION = "v65-islpy%s" % _islpy_version +DATA_MODEL_VERSION = "v66-islpy%s" % _islpy_version -- GitLab From 85397ad4632f25c61638d7cf61bda5919a2d9776 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Wed, 30 Aug 2017 00:58:27 -0500 Subject: [PATCH 3/3] s/Lazy/LazilyUnpickling --- loopy/codegen/__init__.py | 4 ++-- loopy/tools.py | 16 ++++++++-------- test/test_misc.py | 37 +++++++++++++++++++------------------ 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/loopy/codegen/__init__.py b/loopy/codegen/__init__.py index 392e9bbec..07bcdc7c6 100644 --- a/loopy/codegen/__init__.py +++ b/loopy/codegen/__init__.py @@ -507,9 +507,9 @@ def generate_code_v2(kernel): # }}} # For faster unpickling in the common case when implemented_domains isn't needed. - from loopy.tools import LazyDict + from loopy.tools import LazilyUnpicklingDict codegen_result = codegen_result.copy( - implemented_domains=LazyDict( + implemented_domains=LazilyUnpicklingDict( codegen_result.implemented_domains)) logger.info("%s: generate code: done" % kernel.name) diff --git a/loopy/tools.py b/loopy/tools.py index a7e536989..69a25b375 100644 --- a/loopy/tools.py +++ b/loopy/tools.py @@ -344,8 +344,8 @@ def compute_sccs(graph): # {{{ pickled container value class _PickledObject(object): - """A class meant to wrap a pickled value (for :class:`LazyDict` and - :class:`LazyList`). + """A class meant to wrap a pickled value (for :class:`LazilyUnpicklingDict` and + :class:`LazilyUnpicklingList`). """ def __init__(self, obj): @@ -390,7 +390,7 @@ class _PickledObjectWithEqAndPersistentHashKeys(_PickledObject): # {{{ lazily unpickling dictionary -class LazyDict(collections.MutableMapping): +class LazilyUnpicklingDict(collections.MutableMapping): """A dictionary-like object which lazily unpickles its values. """ @@ -425,7 +425,7 @@ class LazyDict(collections.MutableMapping): # {{{ lazily unpickling list -class LazyList(collections.MutableSequence): +class LazilyUnpicklingList(collections.MutableSequence): """A list which lazily unpickles its values.""" def __init__(self, *args, **kwargs): @@ -453,7 +453,7 @@ class LazyList(collections.MutableSequence): return {"_list": [_PickledObject(val) for val in self._list]} -class LazyListWithEqAndPersistentHashing(LazyList): +class LazilyUnpicklingListWithEqAndPersistentHashing(LazilyUnpicklingList): """A list which lazily unpickles its values, and supports equality comparison and persistent hashing without unpickling. @@ -469,7 +469,7 @@ class LazyListWithEqAndPersistentHashing(LazyList): def __init__(self, *args, **kwargs): self.eq_key_getter = kwargs.pop("eq_key_getter") self.persistent_hash_key_getter = kwargs.pop("persistent_hash_key_getter") - LazyList.__init__(self, *args, **kwargs) + LazilyUnpicklingList.__init__(self, *args, **kwargs) def update_persistent_hash(self, key_hash, key_builder): key_builder.update_for_list(key_hash, self._list) @@ -485,10 +485,10 @@ class LazyListWithEqAndPersistentHashing(LazyList): return self.persistent_hash_key_getter(obj) def __eq__(self, other): - if not isinstance(other, (list, LazyList)): + if not isinstance(other, (list, LazilyUnpicklingList)): return NotImplemented - if isinstance(other, LazyList): + if isinstance(other, LazilyUnpicklingList): other = other._list if len(self) != len(other): diff --git a/test/test_misc.py b/test/test_misc.py index 53da8ee61..0273948b3 100644 --- a/test/test_misc.py +++ b/test/test_misc.py @@ -108,18 +108,18 @@ class PickleDetector(object): self.state = state["state"] -class PickleDetectorForLazyDict(PickleDetector): +class PickleDetectorForLazilyUnpicklingDict(PickleDetector): instance_unpickled = False def __init__(self): self.state = None -def test_LazyDict(): - from loopy.tools import LazyDict +def test_LazilyUnpicklingDict(): + from loopy.tools import LazilyUnpicklingDict - cls = PickleDetectorForLazyDict - mapping = LazyDict({0: cls()}) + cls = PickleDetectorForLazilyUnpicklingDict + mapping = LazilyUnpicklingDict({0: cls()}) assert not cls.instance_unpickled @@ -156,25 +156,25 @@ def test_LazyDict(): # {{{ test empty map - mapping = LazyDict({}) + mapping = LazilyUnpicklingDict({}) mapping = loads(dumps(mapping)) assert len(mapping) == 0 # }}} -class PickleDetectorForLazyList(PickleDetector): +class PickleDetectorForLazilyUnpicklingList(PickleDetector): instance_unpickled = False def __init__(self): self.state = None -def test_LazyList(): - from loopy.tools import LazyList +def test_LazilyUnpicklingList(): + from loopy.tools import LazilyUnpicklingList - cls = PickleDetectorForLazyList - lst = LazyList([cls()]) + cls = PickleDetectorForLazilyUnpicklingList + lst = LazilyUnpicklingList([cls()]) assert not cls.instance_unpickled from pickle import loads, dumps @@ -207,14 +207,15 @@ def test_LazyList(): # {{{ test empty list - lst = LazyList([]) + lst = LazilyUnpicklingList([]) lst = loads(dumps(lst)) assert len(lst) == 0 # }}} -class PickleDetectorForLazyListWithEqAndPersistentHashing(PickleDetector): +class PickleDetectorForLazilyUnpicklingListWithEqAndPersistentHashing( + PickleDetector): instance_unpickled = False def __init__(self, comparison_key): @@ -227,19 +228,19 @@ class PickleDetectorForLazyListWithEqAndPersistentHashing(PickleDetector): key_builder.rec(key_hash, repr(self)) -def test_LazyListWithEqAndPersistentHashing(): - from loopy.tools import LazyListWithEqAndPersistentHashing +def test_LazilyUnpicklingListWithEqAndPersistentHashing(): + from loopy.tools import LazilyUnpicklingListWithEqAndPersistentHashing - cls = PickleDetectorForLazyListWithEqAndPersistentHashing + cls = PickleDetectorForLazilyUnpicklingListWithEqAndPersistentHashing from pickle import loads, dumps # {{{ test comparison of a pair of lazy lists - lst0 = LazyListWithEqAndPersistentHashing( + lst0 = LazilyUnpicklingListWithEqAndPersistentHashing( [cls(0), cls(1)], eq_key_getter=repr, persistent_hash_key_getter=repr) - lst1 = LazyListWithEqAndPersistentHashing( + lst1 = LazilyUnpicklingListWithEqAndPersistentHashing( [cls(0), cls(1)], eq_key_getter=repr, persistent_hash_key_getter=repr) -- GitLab