From c5427abec5605f298bf09b9b044473b5253b70d4 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Fri, 17 Feb 2017 15:51:10 -0600 Subject: [PATCH 1/7] Implement LazilyUnpicklingDictionary. --- loopy/tools.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++ test/test_misc.py | 44 ++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) diff --git a/loopy/tools.py b/loopy/tools.py index 01d0641fc..36f015a29 100644 --- a/loopy/tools.py +++ b/loopy/tools.py @@ -340,6 +340,71 @@ def compute_sccs(graph): # }}} +# {{{ lazily unpickling dictionary + + +class _PickledObjectWrapper(object): + """ + A class meant to wrap a pickled value (for :class:`LazilyUnpicklingDictionary`). + """ + + __slots__ = ["objstring"] + + @classmethod + def from_object(cls, obj): + if isinstance(obj, cls): + return obj + from pickle import dumps + return cls(dumps(obj)) + + def __init__(self, objstring): + self.objstring = objstring + + def unpickle(self): + from pickle import loads + return loads(self.objstring) + + +import collections + + +class LazilyUnpicklingDictionary(collections.MutableMapping): + """ + A dictionary-like object which lazily unpickles its values. + """ + + def __init__(self, *args, **kwargs): + self._map = dict(*args, **kwargs) + + def __getitem__(self, key): + value = self._map[key] + if isinstance(value, _PickledObjectWrapper): + value = self._map[key] = value.unpickle() + return value + + def __setitem__(self, key, value): + self._map[key] = value + + def __delitem__(self, key): + del self._map[key] + + def __len__(self): + return len(self._map) + + def __iter__(self): + return iter(self._map) + + def __getstate__(self): + return dict( + (key, _PickledObjectWrapper.from_object(val)) + for key, val in six.iteritems(self._map)) + + def __setstate__(self, state): + self._map = state + +# }}} + + def is_interned(s): return s is None or intern(s) is s diff --git a/test/test_misc.py b/test/test_misc.py index ca4aee5b8..139c337e1 100644 --- a/test/test_misc.py +++ b/test/test_misc.py @@ -92,6 +92,50 @@ def test_SetTrie(): s.add_or_update(set([1, 4])) +class PicklableItem(object): + + flags = {"unpickled": False} + + def __getstate__(self): + return () + + def __setstate__(self, state): + PicklableItem.flags["unpickled"] = True + + +def test_LazilyUnpicklingDictionary(): + def is_unpickled(): + return PicklableItem.flags["unpickled"] + + from loopy.tools import LazilyUnpicklingDictionary + + mapping = LazilyUnpicklingDictionary({0: PicklableItem()}) + + assert not is_unpickled() + + from pickle import loads, dumps + + pickled_mapping = dumps(mapping) + + # {{{ test lazy loading + + mapping = loads(pickled_mapping) + assert not is_unpickled() + list(mapping.keys()) + assert not is_unpickled() + assert isinstance(mapping[0], PicklableItem) + assert is_unpickled() + + # }}} + + # {{{ test multi round trip + + mapping = loads(dumps(loads(pickled_mapping))) + assert isinstance(mapping[0], PicklableItem) + + # }}} + + if __name__ == "__main__": if len(sys.argv) > 1: exec(sys.argv[1]) -- GitLab From cc567e7e38fe2eec0d2e061024f8381447eb39d5 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Fri, 17 Feb 2017 15:54:12 -0600 Subject: [PATCH 2/7] codegen: wrap implemented_domains in a LazilyUnpicklingDictionary. --- loopy/codegen/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/loopy/codegen/__init__.py b/loopy/codegen/__init__.py index 6f312ec79..738def915 100644 --- a/loopy/codegen/__init__.py +++ b/loopy/codegen/__init__.py @@ -397,10 +397,13 @@ def generate_code_v2(kernel): if CACHING_ENABLED: input_kernel = kernel try: + print("===trying to find kernel") result = code_gen_cache[input_kernel] + print("===FOUND") logger.debug("%s: code generation cache hit" % kernel.name) return result except KeyError: + print("===NOT FOUND") pass # }}} @@ -506,6 +509,12 @@ def generate_code_v2(kernel): # }}} + # For faster unpickling in the common case when implemented_domains isn't needed. + from loopy.tools import LazilyUnpicklingDictionary + codegen_result = codegen_result.copy( + implemented_domains=LazilyUnpicklingDictionary( + codegen_result.implemented_domains)) + logger.info("%s: generate code: done" % kernel.name) if CACHING_ENABLED: -- GitLab From 84f8f435ef8ded52336e5f0adb5f1329175c4389 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Fri, 17 Feb 2017 16:17:19 -0600 Subject: [PATCH 3/7] Bump cache version. --- loopy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loopy/version.py b/loopy/version.py index bb2181bfa..f5384a49f 100644 --- a/loopy/version.py +++ b/loopy/version.py @@ -32,4 +32,4 @@ except ImportError: else: _islpy_version = islpy.version.VERSION_TEXT -DATA_MODEL_VERSION = "v55-islpy%s" % _islpy_version +DATA_MODEL_VERSION = "v56-islpy%s" % _islpy_version -- GitLab From 500e48963ccd9f2033c422a8abd447dad70cea34 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Fri, 17 Feb 2017 16:20:12 -0600 Subject: [PATCH 4/7] Fix accidentally included print statements. --- loopy/codegen/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/loopy/codegen/__init__.py b/loopy/codegen/__init__.py index 738def915..009dadc1a 100644 --- a/loopy/codegen/__init__.py +++ b/loopy/codegen/__init__.py @@ -397,13 +397,10 @@ def generate_code_v2(kernel): if CACHING_ENABLED: input_kernel = kernel try: - print("===trying to find kernel") result = code_gen_cache[input_kernel] - print("===FOUND") logger.debug("%s: code generation cache hit" % kernel.name) return result except KeyError: - print("===NOT FOUND") pass # }}} -- GitLab From 8a648f677f788ea2778a99502cb576470361198b Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Fri, 17 Feb 2017 16:39:41 -0600 Subject: [PATCH 5/7] Fix pickleability in Py2. --- loopy/tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/loopy/tools.py b/loopy/tools.py index 36f015a29..524d2cf72 100644 --- a/loopy/tools.py +++ b/loopy/tools.py @@ -348,8 +348,6 @@ class _PickledObjectWrapper(object): A class meant to wrap a pickled value (for :class:`LazilyUnpicklingDictionary`). """ - __slots__ = ["objstring"] - @classmethod def from_object(cls, obj): if isinstance(obj, cls): @@ -364,6 +362,9 @@ class _PickledObjectWrapper(object): from pickle import loads return loads(self.objstring) + def __getstate__(self): + return {"objstring": self.objstring} + import collections -- GitLab From 3331117911f92eeb1d2c6f77347afbdb96f0eb4c Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Fri, 17 Feb 2017 16:40:04 -0600 Subject: [PATCH 6/7] Bump cache version again just in case. --- loopy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loopy/version.py b/loopy/version.py index f5384a49f..35a86a995 100644 --- a/loopy/version.py +++ b/loopy/version.py @@ -32,4 +32,4 @@ except ImportError: else: _islpy_version = islpy.version.VERSION_TEXT -DATA_MODEL_VERSION = "v56-islpy%s" % _islpy_version +DATA_MODEL_VERSION = "v57-islpy%s" % _islpy_version -- GitLab From 346fceebb7e91df7d52a422d049444f8407ec6f4 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Fri, 17 Feb 2017 17:05:04 -0600 Subject: [PATCH 7/7] Apparently, __getstate__() results are coerced to bool in Python 2 and __setstate__() doesn't run if the state is False. Ensure that we always return a non-false state (another version bump here). --- loopy/tools.py | 7 ++----- loopy/version.py | 2 +- test/test_misc.py | 10 +++++++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/loopy/tools.py b/loopy/tools.py index 524d2cf72..56b673b59 100644 --- a/loopy/tools.py +++ b/loopy/tools.py @@ -396,12 +396,9 @@ class LazilyUnpicklingDictionary(collections.MutableMapping): return iter(self._map) def __getstate__(self): - return dict( + return {"_map": dict( (key, _PickledObjectWrapper.from_object(val)) - for key, val in six.iteritems(self._map)) - - def __setstate__(self, state): - self._map = state + for key, val in six.iteritems(self._map))} # }}} diff --git a/loopy/version.py b/loopy/version.py index 35a86a995..2c15cc2da 100644 --- a/loopy/version.py +++ b/loopy/version.py @@ -32,4 +32,4 @@ except ImportError: else: _islpy_version = islpy.version.VERSION_TEXT -DATA_MODEL_VERSION = "v57-islpy%s" % _islpy_version +DATA_MODEL_VERSION = "v58-islpy%s" % _islpy_version diff --git a/test/test_misc.py b/test/test_misc.py index 139c337e1..a22e42463 100644 --- a/test/test_misc.py +++ b/test/test_misc.py @@ -97,7 +97,7 @@ class PicklableItem(object): flags = {"unpickled": False} def __getstate__(self): - return () + return True def __setstate__(self, state): PicklableItem.flags["unpickled"] = True @@ -135,6 +135,14 @@ def test_LazilyUnpicklingDictionary(): # }}} + # {{{ test empty map + + mapping = LazilyUnpicklingDictionary({}) + mapping = loads(dumps(mapping)) + assert len(mapping) == 0 + + # }}} + if __name__ == "__main__": if len(sys.argv) > 1: -- GitLab