From 9943f855cb9966656e3c999af8c2d18be77b73c8 Mon Sep 17 00:00:00 2001 From: xywei Date: Sat, 23 May 2020 19:22:52 -0500 Subject: [PATCH 01/32] Add basic contracts --- pytato/array.py | 25 +++++++++++++++++++------ pytato/contract.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 3 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 pytato/contract.py diff --git a/pytato/array.py b/pytato/array.py index 4a52ca8..beb6737 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -34,6 +34,9 @@ is referenced from :class:`DataArray`. import collections.abc from pytools import single_valued, is_single_valued +from contracts import contract +from pymbolic.primitives import Expression # noqa +from pytato.contract import c_identifier, ArrayInterface # noqa class DottedName: @@ -41,13 +44,15 @@ class DottedName: .. attribute:: name_parts A tuple of strings, each of which is a valid - Python identifier. + C identifier (non-Unicode Python identifier). The name (at least morally) exists in the name space defined by the Python module system. It need not necessarily identify an importable object. """ + + @contract(name_parts='list[>0](str,c_identifier)') def __init__(self, name_parts): self.name_parts = name_parts @@ -65,13 +70,14 @@ class Namespace: def __init__(self): self.symbol_table = {} + @contract(name='str,c_identifier', value=ArrayInterface) def assign(self, name, value): if name in self.symbol_table: raise ValueError(f"'{name}' is already assigned") self.symbol_table[name] = value -class Array: +class Array(ArrayInterface): """ A base class (abstract interface + supplemental functionality) for lazily @@ -150,6 +156,7 @@ class Array: purposefully so. """ + @contract(namespace=Namespace, name='str,c_identifier') def __init__(self, namespace, name, tags=None): if tags is None: tags = {} @@ -172,6 +179,7 @@ class Array: def ndim(self): return len(self.shape) + @contract(dotted_name=DottedName) def with_tag(self, dotted_name, args=None): """ Returns a copy of *self* tagged with *dotted_name* @@ -182,9 +190,11 @@ class Array: if args is None: pass + @contract(dotted_name=DottedName) def without_tag(self, dotted_name): pass + @contract(name='str,c_identifier') def with_name(self, name): self.namespace.assign_name(name, self) return self.copy(name=name) @@ -196,7 +206,7 @@ class Array: class DictOfNamedArrays(collections.abc.Mapping): - """A container that maps valid Python identifiers + """A container that maps valid C identifiers to instances of :class:`Array`. May occur as a result type of array computations. @@ -211,12 +221,10 @@ class DictOfNamedArrays(collections.abc.Mapping): arithmetic. """ + @contract(data='dict((str,c_identifier):$ArrayInterface)') def __init__(self, data): self._data = data - # TODO: Check that keys are valid Python identifiers - if not is_single_valued(ary.target for ary in data.values()): - raise ValueError("arrays do not have same target") if not is_single_valued(ary.namespace for ary in data.values()): raise ValueError("arrays do not have same namespace") @@ -224,9 +232,11 @@ class DictOfNamedArrays(collections.abc.Mapping): def namespace(self): return single_valued(ary.namespace for ary in self._data.values()) + @contract(name='str,c_identifier') def __contains__(self, name): return name in self._data + @contract(name='str,c_identifier') def __getitem__(self, name): return self._data[name] @@ -301,6 +311,9 @@ class Placeholder(Array): # Not tied to this, open for discussion about how to implement this. return self._shape + # TODO: proper contracts for the shape + @contract(namespace=Namespace, name='str,c_identifier', + shape=tuple, tags='None | dict($DottedName:$DottedName)') def __init__(self, namespace, name, shape, tags=None): if name is None: raise ValueError("PlaceholderArray instances must have a name") diff --git a/pytato/contract.py b/pytato/contract.py new file mode 100644 index 0000000..941bad9 --- /dev/null +++ b/pytato/contract.py @@ -0,0 +1,44 @@ +__copyright__ = """ +Copyright (C) 2020 Andreas Kloeckner +Copyright (C) 2020 Matt Wala +Copyright (C) 2020 Xiaoyu Wei +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import re +from contracts import new_contract, ContractsMeta + + +C_IDENTIFIER = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") + + +@new_contract +def c_identifier(x): + """Tests whether a string is a valid variable name in C. + """ + return re.match(C_IDENTIFIER, x) is not None + + +class ArrayInterface(): + """Abstract class for types implementing the Array interface. + """ + __metaclass__ = ContractsMeta diff --git a/setup.py b/setup.py index c56a780..153fe38 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ setup(name="pytato", install_requires=[ "loo.py", + "pycontracts", ], author="Andreas Kloeckner, Matt Wala, Xiaoyu Wei", -- GitLab From 039eb93c1a5a0c0173e66ce79394044e1c196871 Mon Sep 17 00:00:00 2001 From: xywei Date: Sat, 23 May 2020 19:45:02 -0500 Subject: [PATCH 02/32] Fixes --- pytato/array.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index beb6737..904e257 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -156,7 +156,9 @@ class Array(ArrayInterface): purposefully so. """ - @contract(namespace=Namespace, name='str,c_identifier') + @contract(namespace=Namespace, + name='None | (str,c_identifier)', + tags='None | dict($DottedName:$DottedName)') def __init__(self, namespace, name, tags=None): if tags is None: tags = {} @@ -179,7 +181,8 @@ class Array(ArrayInterface): def ndim(self): return len(self.shape) - @contract(dotted_name=DottedName) + @contract(dotted_name=DottedName, + args='(None | $DottedName)') def with_tag(self, dotted_name, args=None): """ Returns a copy of *self* tagged with *dotted_name* @@ -311,12 +314,12 @@ class Placeholder(Array): # Not tied to this, open for discussion about how to implement this. return self._shape - # TODO: proper contracts for the shape - @contract(namespace=Namespace, name='str,c_identifier', - shape=tuple, tags='None | dict($DottedName:$DottedName)') + # TODO: check for meaningfulness of the shape expressions + @contract(namespace=Namespace, + name='str,c_identifier', + shape='seq((int,>0) | (str,c_identifier) | $Expression)', + tags='None | dict($DottedName:$DottedName)') def __init__(self, namespace, name, shape, tags=None): - if name is None: - raise ValueError("PlaceholderArray instances must have a name") super().__init__( namespace=namespace, name=name, -- GitLab From a7dc0cfe383bce48061edabd45a69e7f3c9f70cf Mon Sep 17 00:00:00 2001 From: xywei Date: Sun, 24 May 2020 13:09:29 -0500 Subject: [PATCH 03/32] Re-implement without pycontract --- pytato/array.py | 66 +++++++++++++++---------------- pytato/contract.py | 44 --------------------- pytato/typing.py | 99 ++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 1 - 4 files changed, 132 insertions(+), 78 deletions(-) delete mode 100644 pytato/contract.py create mode 100644 pytato/typing.py diff --git a/pytato/array.py b/pytato/array.py index 904e257..c0379d7 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -33,13 +33,13 @@ is referenced from :class:`DataArray`. import collections.abc +import pytato.typing as ptype from pytools import single_valued, is_single_valued -from contracts import contract -from pymbolic.primitives import Expression # noqa -from pytato.contract import c_identifier, ArrayInterface # noqa +from typing import Optional, Union, List, Dict +from pymbolic.primitives import Expression -class DottedName: +class DottedName(ptype.TagInterface): """ .. attribute:: name_parts @@ -52,12 +52,13 @@ class DottedName: object. """ - @contract(name_parts='list[>0](str,c_identifier)') - def __init__(self, name_parts): + def __init__(self, name_parts: List[str]): + assert len(name_parts) > 0 + assert all(check_c_identifier(p) for p in name_parts) self.name_parts = name_parts -class Namespace: +class Namespace(ptype.NamespaceInterface): # Possible future extension: .parent attribute """ .. attribute:: symbol_table @@ -70,14 +71,14 @@ class Namespace: def __init__(self): self.symbol_table = {} - @contract(name='str,c_identifier', value=ArrayInterface) - def assign(self, name, value): + def assign(self, name: ptype.NameType, value: ptype.ArrayInterface): + assert ptype.check_name(name) if name in self.symbol_table: raise ValueError(f"'{name}' is already assigned") self.symbol_table[name] = value -class Array(ArrayInterface): +class Array(ptype.ArrayInterface): """ A base class (abstract interface + supplemental functionality) for lazily @@ -156,10 +157,12 @@ class Array(ArrayInterface): purposefully so. """ - @contract(namespace=Namespace, - name='None | (str,c_identifier)', - tags='None | dict($DottedName:$DottedName)') - def __init__(self, namespace, name, tags=None): + def __init__(self, namespace: Namespace, + name: Optional[ptype.NameType], + tags: Optional[ptype.TagsType] = None): + assert ptype.check_name(name) + assert ptype.check_tags(tags) + if tags is None: tags = {} @@ -181,9 +184,8 @@ class Array(ArrayInterface): def ndim(self): return len(self.shape) - @contract(dotted_name=DottedName, - args='(None | $DottedName)') - def with_tag(self, dotted_name, args=None): + def with_tag(self, dotted_name: DottedName, + args: Optional[DottedName] = None): """ Returns a copy of *self* tagged with *dotted_name* and arguments *args* @@ -193,12 +195,11 @@ class Array(ArrayInterface): if args is None: pass - @contract(dotted_name=DottedName) - def without_tag(self, dotted_name): + def without_tag(self, dotted_name: DottedName): pass - @contract(name='str,c_identifier') - def with_name(self, name): + def with_name(self, name: ptype.NameType): + assert ptype.check_name(name) self.namespace.assign_name(name, self) return self.copy(name=name) @@ -224,8 +225,8 @@ class DictOfNamedArrays(collections.abc.Mapping): arithmetic. """ - @contract(data='dict((str,c_identifier):$ArrayInterface)') - def __init__(self, data): + def __init__(self, data: Dict[ptype.NameType, ptype.ArrayInterface]): + assert all(ptype.check_name(key) for key in data.keys()) self._data = data if not is_single_valued(ary.namespace for ary in data.values()): @@ -235,12 +236,12 @@ class DictOfNamedArrays(collections.abc.Mapping): def namespace(self): return single_valued(ary.namespace for ary in self._data.values()) - @contract(name='str,c_identifier') - def __contains__(self, name): + def __contains__(self, name: ptype.NameType): + assert ptype.check_name(name) return name in self._data - @contract(name='str,c_identifier') - def __getitem__(self, name): + def __getitem__(self, name: ptype.NameType): + assert ptype.check_name(name) return self._data[name] def __iter__(self): @@ -314,12 +315,11 @@ class Placeholder(Array): # Not tied to this, open for discussion about how to implement this. return self._shape - # TODO: check for meaningfulness of the shape expressions - @contract(namespace=Namespace, - name='str,c_identifier', - shape='seq((int,>0) | (str,c_identifier) | $Expression)', - tags='None | dict($DottedName:$DottedName)') - def __init__(self, namespace, name, shape, tags=None): + def __init__(self, namespace: ptype.NamespaceInterface, + name: ptype.NameType, shape: ptype.ShapeType, + tags: Optional[ptype.TagsType] = None): + assert ptype.check_shape(shape) + # namespace, name and tags will be checked in super().__init__() super().__init__( namespace=namespace, name=name, diff --git a/pytato/contract.py b/pytato/contract.py deleted file mode 100644 index 941bad9..0000000 --- a/pytato/contract.py +++ /dev/null @@ -1,44 +0,0 @@ -__copyright__ = """ -Copyright (C) 2020 Andreas Kloeckner -Copyright (C) 2020 Matt Wala -Copyright (C) 2020 Xiaoyu Wei -""" - -__license__ = """ -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -""" - -import re -from contracts import new_contract, ContractsMeta - - -C_IDENTIFIER = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") - - -@new_contract -def c_identifier(x): - """Tests whether a string is a valid variable name in C. - """ - return re.match(C_IDENTIFIER, x) is not None - - -class ArrayInterface(): - """Abstract class for types implementing the Array interface. - """ - __metaclass__ = ContractsMeta diff --git a/pytato/typing.py b/pytato/typing.py new file mode 100644 index 0000000..27f07f5 --- /dev/null +++ b/pytato/typing.py @@ -0,0 +1,99 @@ +__copyright__ = """ +Copyright (C) 2020 Andreas Kloeckner +Copyright (C) 2020 Matt Wala +Copyright (C) 2020 Xiaoyu Wei +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +__doc__ = """Interface classes and type specifications. +Each type is paired with a check_* function that, when used together, achieves +contracts-like functionality.""" + +import re +from pymbolic.primitives import Expression +from abc import ABC +from typing import Optional, Union, Dict, Tuple + +# {{{ abstract classes + + +class NamespaceInterface(): + __metaclass__ = ABC + + +class TagInterface(): + __metaclass__ = ABC + + +class ArrayInterface(): + """Abstract class for types implementing the Array interface. + """ + __metaclass__ = ABC + +# }}} End abstract classes + +# {{{ name type + + +NameType = str +C_IDENTIFIER = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") + + +def check_name(name: NameType) -> bool: + assert re.match(C_IDENTIFIER, name) is not None, \ + f"{name} is not a C identifier" + return True + +# }}} End name type + +# {{{ tags type + + +TagsType = Dict[TagInterface, TagInterface] + + +def check_tags(tags: TagsType) -> bool: + # assuming TagInterface implementation gurantees correctness + return True + +# }}} End tags type + +# {{{ shape type + + +ShapeComponentType = Union[int, Expression, str] +ShapeType = Tuple[ShapeComponentType, ...] + + +def check_shape(shape: ShapeType, + ns: Optional[NamespaceInterface] = None) -> bool: + for s in shape: + if isinstance(s, int): + assert s > 0, f"size parameter must be positive (got {s})" + elif isinstance(s, str): + assert check_name(s) + elif isinstance(Expression) and ns is not None: + # TODO: check expression in namespace + pass + return True + +# }}} End shape type diff --git a/setup.py b/setup.py index 153fe38..c56a780 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,6 @@ setup(name="pytato", install_requires=[ "loo.py", - "pycontracts", ], author="Andreas Kloeckner, Matt Wala, Xiaoyu Wei", -- GitLab From f802bfa8ef672ee852ab822ec60aa0f0f8ce6d90 Mon Sep 17 00:00:00 2001 From: xywei Date: Sun, 24 May 2020 13:10:45 -0500 Subject: [PATCH 04/32] Flake8 fixes --- pytato/array.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index c0379d7..b2eefd1 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -35,8 +35,7 @@ is referenced from :class:`DataArray`. import collections.abc import pytato.typing as ptype from pytools import single_valued, is_single_valued -from typing import Optional, Union, List, Dict -from pymbolic.primitives import Expression +from typing import Optional, List, Dict class DottedName(ptype.TagInterface): @@ -54,7 +53,7 @@ class DottedName(ptype.TagInterface): def __init__(self, name_parts: List[str]): assert len(name_parts) > 0 - assert all(check_c_identifier(p) for p in name_parts) + assert all(ptype.check_name(p) for p in name_parts) self.name_parts = name_parts -- GitLab From bae7244400682543b8b93dd59bd2e5f6fc893ab5 Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 25 May 2020 10:16:54 -0500 Subject: [PATCH 05/32] Add mypy ci job --- .gitlab-ci.yml | 9 +++++++++ pytato/array.py | 11 +++++------ pytato/typing.py | 45 ++++++++++++++++++++++++++++++++++++++++++--- run-mypy.sh | 3 +++ setup.cfg | 6 ++++++ 5 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 run-mypy.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 510eba8..ad24891 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,6 +34,15 @@ Flake8: except: - tags +Mypy: + script: + - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/prepare-and-run-mypy.sh + - ". ./prepare-and-run-mypy.sh python3 mypy" + tags: + - python3 + except: + - tags + Documentation: script: - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-docs.sh diff --git a/pytato/array.py b/pytato/array.py index b2eefd1..1f6f268 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -156,11 +156,11 @@ class Array(ptype.ArrayInterface): purposefully so. """ - def __init__(self, namespace: Namespace, + def __init__(self, namespace: ptype.NamespaceInterface, name: Optional[ptype.NameType], tags: Optional[ptype.TagsType] = None): - assert ptype.check_name(name) - assert ptype.check_tags(tags) + assert (name is None) or ptype.check_name(name) + assert (tags is None) or ptype.check_tags(tags) if tags is None: tags = {} @@ -195,7 +195,7 @@ class Array(ptype.ArrayInterface): pass def without_tag(self, dotted_name: DottedName): - pass + raise NotImplementedError def with_name(self, name: ptype.NameType): assert ptype.check_name(name) @@ -235,8 +235,7 @@ class DictOfNamedArrays(collections.abc.Mapping): def namespace(self): return single_valued(ary.namespace for ary in self._data.values()) - def __contains__(self, name: ptype.NameType): - assert ptype.check_name(name) + def __contains__(self, name: object): return name in self._data def __getitem__(self, name: ptype.NameType): diff --git a/pytato/typing.py b/pytato/typing.py index 27f07f5..fbc94c5 100644 --- a/pytato/typing.py +++ b/pytato/typing.py @@ -26,11 +26,12 @@ THE SOFTWARE. __doc__ = """Interface classes and type specifications. Each type is paired with a check_* function that, when used together, achieves -contracts-like functionality.""" +contracts-like functionality. +""" import re from pymbolic.primitives import Expression -from abc import ABC +from abc import ABC, abstractmethod from typing import Optional, Union, Dict, Tuple # {{{ abstract classes @@ -39,6 +40,10 @@ from typing import Optional, Union, Dict, Tuple class NamespaceInterface(): __metaclass__ = ABC + @abstractmethod + def assign(self, name, value): + pass + class TagInterface(): __metaclass__ = ABC @@ -49,6 +54,40 @@ class ArrayInterface(): """ __metaclass__ = ABC + @property + def namespace(self): + return self._namespace + + @namespace.setter + def namespace(self, val: NamespaceInterface): + self._namespace = val + + @property + @abstractmethod + def ndim(self): + pass + + @property + @abstractmethod + def shape(self): + pass + + @abstractmethod + def copy(self, **kwargs): + pass + + @abstractmethod + def with_tag(self, tag_key, tag_val): + pass + + @abstractmethod + def without_tag(self, tag_key): + pass + + @abstractmethod + def with_name(self, name): + pass + # }}} End abstract classes # {{{ name type @@ -91,7 +130,7 @@ def check_shape(shape: ShapeType, assert s > 0, f"size parameter must be positive (got {s})" elif isinstance(s, str): assert check_name(s) - elif isinstance(Expression) and ns is not None: + elif isinstance(s, Expression) and ns is not None: # TODO: check expression in namespace pass return True diff --git a/run-mypy.sh b/run-mypy.sh new file mode 100644 index 0000000..14730b1 --- /dev/null +++ b/run-mypy.sh @@ -0,0 +1,3 @@ +#! /bin/bash + +mypy --strict pytato diff --git a/setup.cfg b/setup.cfg index 07d5743..a8663c0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,9 @@ [flake8] ignore = E126,E127,E128,E123,E226,E241,E242,E265,N802,W503,E402,N814,N817,W504 max-line-length=85 + +[mypy-pymbolic.primitives] +ignore_missing_imports = True + +[mypy-pytools] +ignore_missing_imports = True -- GitLab From 87b373ce99fb98c12e824dff178f8a8b333cb741 Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 25 May 2020 11:11:33 -0500 Subject: [PATCH 06/32] Remove infix ops, mypy fixes --- pytato/array.py | 62 +++++++++++++++-------------------------- pytato/infix.py | 58 -------------------------------------- pytato/mapper.py | 6 ++-- pytato/typing.py | 72 +++++++++++++++++++++++++++++++----------------- run-mypy.sh | 0 5 files changed, 72 insertions(+), 126 deletions(-) delete mode 100644 pytato/infix.py mode change 100644 => 100755 run-mypy.sh diff --git a/pytato/array.py b/pytato/array.py index 1f6f268..1cedb37 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -32,29 +32,9 @@ is referenced from :class:`DataArray`. """ -import collections.abc import pytato.typing as ptype from pytools import single_valued, is_single_valued -from typing import Optional, List, Dict - - -class DottedName(ptype.TagInterface): - """ - .. attribute:: name_parts - - A tuple of strings, each of which is a valid - C identifier (non-Unicode Python identifier). - - The name (at least morally) exists in the - name space defined by the Python module system. - It need not necessarily identify an importable - object. - """ - - def __init__(self, name_parts: List[str]): - assert len(name_parts) > 0 - assert all(ptype.check_name(p) for p in name_parts) - self.name_parts = name_parts +from typing import Optional, Dict, Any, Mapping, Iterator class Namespace(ptype.NamespaceInterface): @@ -67,10 +47,11 @@ class Namespace(ptype.NamespaceInterface): :class:`Array` interface. """ - def __init__(self): + def __init__(self) -> None: self.symbol_table = {} - def assign(self, name: ptype.NameType, value: ptype.ArrayInterface): + def assign(self, name: ptype.NameType, + value: ptype.ArrayInterface) -> None: assert ptype.check_name(name) if name in self.symbol_table: raise ValueError(f"'{name}' is already assigned") @@ -160,7 +141,6 @@ class Array(ptype.ArrayInterface): name: Optional[ptype.NameType], tags: Optional[ptype.TagsType] = None): assert (name is None) or ptype.check_name(name) - assert (tags is None) or ptype.check_tags(tags) if tags is None: tags = {} @@ -172,19 +152,19 @@ class Array(ptype.ArrayInterface): self.name = name self.tags = tags - def copy(self, **kwargs): + def copy(self, **kwargs: Any) -> 'Array': raise NotImplementedError @property - def shape(self): + def shape(self) -> ptype.ShapeType: raise NotImplementedError @property - def ndim(self): + def ndim(self) -> int: return len(self.shape) - def with_tag(self, dotted_name: DottedName, - args: Optional[DottedName] = None): + def with_tag(self, dotted_name: ptype.DottedName, + args: Optional[ptype.DottedName] = None) -> 'Array': """ Returns a copy of *self* tagged with *dotted_name* and arguments *args* @@ -193,13 +173,14 @@ class Array(ptype.ArrayInterface): """ if args is None: pass + return self.copy() - def without_tag(self, dotted_name: DottedName): + def without_tag(self, dotted_name: ptype.DottedName) -> 'Array': raise NotImplementedError - def with_name(self, name: ptype.NameType): + def with_name(self, name: ptype.NameType) -> 'Array': assert ptype.check_name(name) - self.namespace.assign_name(name, self) + self.namespace.assign(name, self) return self.copy(name=name) # TODO: @@ -208,7 +189,7 @@ class Array(ptype.ArrayInterface): # - naming -class DictOfNamedArrays(collections.abc.Mapping): +class DictOfNamedArrays(Mapping[ptype.NameType, ptype.ArrayInterface]): """A container that maps valid C identifiers to instances of :class:`Array`. May occur as a result type of array computations. @@ -232,20 +213,21 @@ class DictOfNamedArrays(collections.abc.Mapping): raise ValueError("arrays do not have same namespace") @property - def namespace(self): - return single_valued(ary.namespace for ary in self._data.values()) + def namespace(self) -> ptype.NamespaceInterface: + return single_valued( # type: ignore + ary.namespace for ary in self._data.values()) - def __contains__(self, name: object): + def __contains__(self, name: object) -> bool: return name in self._data - def __getitem__(self, name: ptype.NameType): + def __getitem__(self, name: ptype.NameType) -> ptype.ArrayInterface: assert ptype.check_name(name) return self._data[name] - def __iter__(self): + def __iter__(self) -> Iterator[ptype.NameType]: return iter(self._data) - def __len__(self): + def __len__(self) -> int: return len(self._data) @@ -308,7 +290,7 @@ class Placeholder(Array): """ @property - def shape(self): + def shape(self) -> ptype.ShapeType: # Matt added this to make Pylint happy. # Not tied to this, open for discussion about how to implement this. return self._shape diff --git a/pytato/infix.py b/pytato/infix.py deleted file mode 100644 index c71f936..0000000 --- a/pytato/infix.py +++ /dev/null @@ -1,58 +0,0 @@ -__copyright__ = """ -Copyright (C) 2020 Andreas Kloeckner -Copyright (C) 2020 Matt Wala -Copyright (C) 2020 Xiaoyu Wei -""" - -__license__ = """ -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -""" -# definition of an Infix operator class -# this recipe also works in jython -# calling sequence for the infix is either: -# x |op| y -# or: -# x <> y - - -class Infix: - - def __init__(self, function): - self.function = function - - def __ror__(self, other): - return Infix(lambda x, self=self, other=other: self.function(other, x)) - - def __or__(self, other): - return self.function(other) - - def __rlshift__(self, other): - return Infix(lambda x, self=self, other=other: self.function(other, x)) - - def __rshift__(self, other): - return self.function(other) - - def __call__(self, value1, value2): - return self.function(value1, value2) - - -# Examples -eq = Infix(lambda x, y: x == y) # may be x.eq(y) for arrays -print(2 |eq| 4) # noqa -print(4 |eq| 4) # noqa diff --git a/pytato/mapper.py b/pytato/mapper.py index cfd1794..afeec20 100644 --- a/pytato/mapper.py +++ b/pytato/mapper.py @@ -24,6 +24,8 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +from typing import Any + class Mapper: pass @@ -45,10 +47,10 @@ class ToLoopyMapper: # {{{ - def _stringify(self): + def _stringify(self) -> str: pass - def _generate_code(self): + def _generate_code(self) -> Any: pass # }}} diff --git a/pytato/typing.py b/pytato/typing.py index fbc94c5..8ad7e96 100644 --- a/pytato/typing.py +++ b/pytato/typing.py @@ -32,7 +32,7 @@ contracts-like functionality. import re from pymbolic.primitives import Expression from abc import ABC, abstractmethod -from typing import Optional, Union, Dict, Tuple +from typing import Optional, Union, Dict, Tuple, Any, List # {{{ abstract classes @@ -40,13 +40,18 @@ from typing import Optional, Union, Dict, Tuple class NamespaceInterface(): __metaclass__ = ABC - @abstractmethod - def assign(self, name, value): - pass + @property + def symbol_table(self) -> Dict['NameType', 'ArrayInterface']: + return self._namespace + @symbol_table.setter + def symbol_table(self, val: Dict['NameType', 'ArrayInterface']) -> None: + self._namespace = val -class TagInterface(): - __metaclass__ = ABC + @abstractmethod + def assign(self, name: 'NameType', + value: 'ArrayInterface') -> None: + pass class ArrayInterface(): @@ -55,37 +60,38 @@ class ArrayInterface(): __metaclass__ = ABC @property - def namespace(self): + def namespace(self) -> NamespaceInterface: return self._namespace @namespace.setter - def namespace(self, val: NamespaceInterface): + def namespace(self, val: NamespaceInterface) -> None: self._namespace = val @property @abstractmethod - def ndim(self): + def ndim(self) -> Any: pass @property @abstractmethod - def shape(self): + def shape(self) -> Any: pass @abstractmethod - def copy(self, **kwargs): + def copy(self, **kwargs: Any) -> 'ArrayInterface': pass @abstractmethod - def with_tag(self, tag_key, tag_val): + def with_tag(self, tag_key: Any, + tag_val: Any) -> 'ArrayInterface': pass @abstractmethod - def without_tag(self, tag_key): + def without_tag(self, tag_key: Any) -> 'ArrayInterface': pass @abstractmethod - def with_name(self, name): + def with_name(self, name: 'NameType') -> 'ArrayInterface': pass # }}} End abstract classes @@ -104,18 +110,6 @@ def check_name(name: NameType) -> bool: # }}} End name type -# {{{ tags type - - -TagsType = Dict[TagInterface, TagInterface] - - -def check_tags(tags: TagsType) -> bool: - # assuming TagInterface implementation gurantees correctness - return True - -# }}} End tags type - # {{{ shape type @@ -136,3 +130,29 @@ def check_shape(shape: ShapeType, return True # }}} End shape type + +# {{{ tags type + + +class DottedName(): + """ + .. attribute:: name_parts + + A tuple of strings, each of which is a valid + C identifier (non-Unicode Python identifier). + + The name (at least morally) exists in the + name space defined by the Python module system. + It need not necessarily identify an importable + object. + """ + + def __init__(self, name_parts: List[str]): + assert len(name_parts) > 0 + assert all(check_name(p) for p in name_parts) + self.name_parts = name_parts + + +TagsType = Dict[DottedName, DottedName] + +# }}} End tags type diff --git a/run-mypy.sh b/run-mypy.sh old mode 100644 new mode 100755 -- GitLab From 5288b3ff4d012904727b4612c3f8e656626baab7 Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 25 May 2020 15:45:46 -0500 Subject: [PATCH 07/32] Fix metaclass syntax --- pytato/typing.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pytato/typing.py b/pytato/typing.py index 8ad7e96..65ba8a1 100644 --- a/pytato/typing.py +++ b/pytato/typing.py @@ -37,8 +37,7 @@ from typing import Optional, Union, Dict, Tuple, Any, List # {{{ abstract classes -class NamespaceInterface(): - __metaclass__ = ABC +class NamespaceInterface(ABC): @property def symbol_table(self) -> Dict['NameType', 'ArrayInterface']: @@ -54,10 +53,9 @@ class NamespaceInterface(): pass -class ArrayInterface(): +class ArrayInterface(ABC): """Abstract class for types implementing the Array interface. """ - __metaclass__ = ABC @property def namespace(self) -> NamespaceInterface: -- GitLab From c604292daf86d71def310f8d5334ce8244995a55 Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 25 May 2020 15:47:10 -0500 Subject: [PATCH 08/32] Put types in double quotes --- pytato/array.py | 8 ++++---- pytato/typing.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index 014f894..b310e16 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -176,7 +176,7 @@ class Array(ptype.ArrayInterface): self.name = name self.tags = tags - def copy(self, **kwargs: Any) -> 'Array': + def copy(self, **kwargs: Any) -> "Array": raise NotImplementedError @property @@ -188,7 +188,7 @@ class Array(ptype.ArrayInterface): return len(self.shape) def with_tag(self, dotted_name: ptype.DottedName, - args: Optional[ptype.DottedName] = None) -> 'Array': + args: Optional[ptype.DottedName] = None) -> "Array": """ Returns a copy of *self* tagged with *dotted_name* and arguments *args* @@ -199,10 +199,10 @@ class Array(ptype.ArrayInterface): pass return self.copy() - def without_tag(self, dotted_name: ptype.DottedName) -> 'Array': + def without_tag(self, dotted_name: ptype.DottedName) -> "Array": raise NotImplementedError - def with_name(self, name: ptype.NameType) -> 'Array': + def with_name(self, name: ptype.NameType) -> "Array": assert ptype.check_name(name) self.namespace.assign(name, self) return self.copy(name=name) diff --git a/pytato/typing.py b/pytato/typing.py index 65ba8a1..6822809 100644 --- a/pytato/typing.py +++ b/pytato/typing.py @@ -40,16 +40,16 @@ from typing import Optional, Union, Dict, Tuple, Any, List class NamespaceInterface(ABC): @property - def symbol_table(self) -> Dict['NameType', 'ArrayInterface']: + def symbol_table(self) -> Dict["NameType", "ArrayInterface"]: return self._namespace @symbol_table.setter - def symbol_table(self, val: Dict['NameType', 'ArrayInterface']) -> None: + def symbol_table(self, val: Dict["NameType", "ArrayInterface"]) -> None: self._namespace = val @abstractmethod - def assign(self, name: 'NameType', - value: 'ArrayInterface') -> None: + def assign(self, name: "NameType", + value: "ArrayInterface") -> None: pass @@ -76,20 +76,20 @@ class ArrayInterface(ABC): pass @abstractmethod - def copy(self, **kwargs: Any) -> 'ArrayInterface': + def copy(self, **kwargs: Any) -> "ArrayInterface": pass @abstractmethod def with_tag(self, tag_key: Any, - tag_val: Any) -> 'ArrayInterface': + tag_val: Any) -> "ArrayInterface": pass @abstractmethod - def without_tag(self, tag_key: Any) -> 'ArrayInterface': + def without_tag(self, tag_key: Any) -> "ArrayInterface": pass @abstractmethod - def with_name(self, name: 'NameType') -> 'ArrayInterface': + def with_name(self, name: "NameType") -> "ArrayInterface": pass # }}} End abstract classes -- GitLab From 3a26c65445fae08b5363089949e8e6036bd60c6f Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 25 May 2020 15:54:08 -0500 Subject: [PATCH 09/32] Import DottedName from pytato.typing --- pytato/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytato/__init__.py b/pytato/__init__.py index 5f047b9..ab52ffd 100644 --- a/pytato/__init__.py +++ b/pytato/__init__.py @@ -24,8 +24,9 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +from pytato.typing import DottedName from pytato.array import ( - DottedName, Namespace, Array, DictOfNamedArrays, + Namespace, Array, DictOfNamedArrays, ) __all__ = ("DottedName", "Namespace", "Array", "DictOfNamedArrays") -- GitLab From a62366ce3295f733a5b638186fc0fdcf8c3e6f13 Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 25 May 2020 15:59:41 -0500 Subject: [PATCH 10/32] Make Array.namespace abstract and read-only --- pytato/array.py | 6 +++++- pytato/typing.py | 7 ++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index b310e16..db7fd76 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -172,13 +172,17 @@ class Array(ptype.ArrayInterface): if name is not None: namespace.assign(name, self) - self.namespace = namespace + self._namespace = namespace self.name = name self.tags = tags def copy(self, **kwargs: Any) -> "Array": raise NotImplementedError + @property + def namespace(self) -> ptype.NamespaceInterface: + return self._namespace + @property def shape(self) -> ptype.ShapeType: raise NotImplementedError diff --git a/pytato/typing.py b/pytato/typing.py index 6822809..922499e 100644 --- a/pytato/typing.py +++ b/pytato/typing.py @@ -58,12 +58,9 @@ class ArrayInterface(ABC): """ @property + @abstractmethod def namespace(self) -> NamespaceInterface: - return self._namespace - - @namespace.setter - def namespace(self, val: NamespaceInterface) -> None: - self._namespace = val + pass @property @abstractmethod -- GitLab From 3c5daab57c9538a39d35b2fd4813ac7c5c56aa8e Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 25 May 2020 16:09:46 -0500 Subject: [PATCH 11/32] Remove NameType. Use built-in str type and str.isidentifier() to check names --- pytato/array.py | 24 +++++++++++------------- pytato/typing.py | 27 ++++++--------------------- 2 files changed, 17 insertions(+), 34 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index db7fd76..d539777 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -76,9 +76,9 @@ class Namespace(ptype.NamespaceInterface): def __init__(self) -> None: self.symbol_table = {} - def assign(self, name: ptype.NameType, + def assign(self, name: str, value: ptype.ArrayInterface) -> None: - assert ptype.check_name(name) + assert str.isidentifier(name) if name in self.symbol_table: raise ValueError(f"'{name}' is already assigned") self.symbol_table[name] = value @@ -162,9 +162,9 @@ class Array(ptype.ArrayInterface): """ def __init__(self, namespace: ptype.NamespaceInterface, - name: Optional[ptype.NameType], + name: Optional[str], tags: Optional[ptype.TagsType] = None): - assert (name is None) or ptype.check_name(name) + assert (name is None) or str.isidentifier(name) if tags is None: tags = {} @@ -206,8 +206,7 @@ class Array(ptype.ArrayInterface): def without_tag(self, dotted_name: ptype.DottedName) -> "Array": raise NotImplementedError - def with_name(self, name: ptype.NameType) -> "Array": - assert ptype.check_name(name) + def with_name(self, name: str) -> "Array": self.namespace.assign(name, self) return self.copy(name=name) @@ -217,7 +216,7 @@ class Array(ptype.ArrayInterface): # - naming -class DictOfNamedArrays(Mapping[ptype.NameType, ptype.ArrayInterface]): +class DictOfNamedArrays(Mapping[str, ptype.ArrayInterface]): """A container that maps valid C identifiers to instances of :class:`Array`. May occur as a result type of array computations. @@ -233,8 +232,8 @@ class DictOfNamedArrays(Mapping[ptype.NameType, ptype.ArrayInterface]): arithmetic. """ - def __init__(self, data: Dict[ptype.NameType, ptype.ArrayInterface]): - assert all(ptype.check_name(key) for key in data.keys()) + def __init__(self, data: Dict[str, ptype.ArrayInterface]): + assert all(str.isidentifier(key) for key in data.keys()) self._data = data if not is_single_valued(ary.namespace for ary in data.values()): @@ -248,11 +247,10 @@ class DictOfNamedArrays(Mapping[ptype.NameType, ptype.ArrayInterface]): def __contains__(self, name: object) -> bool: return name in self._data - def __getitem__(self, name: ptype.NameType) -> ptype.ArrayInterface: - assert ptype.check_name(name) + def __getitem__(self, name: str) -> ptype.ArrayInterface: return self._data[name] - def __iter__(self) -> Iterator[ptype.NameType]: + def __iter__(self) -> Iterator[str]: return iter(self._data) def __len__(self) -> int: @@ -324,7 +322,7 @@ class Placeholder(Array): return self._shape def __init__(self, namespace: ptype.NamespaceInterface, - name: ptype.NameType, shape: ptype.ShapeType, + name: str, shape: ptype.ShapeType, tags: Optional[ptype.TagsType] = None): assert ptype.check_shape(shape) # namespace, name and tags will be checked in super().__init__() diff --git a/pytato/typing.py b/pytato/typing.py index 922499e..8f784ec 100644 --- a/pytato/typing.py +++ b/pytato/typing.py @@ -29,7 +29,6 @@ Each type is paired with a check_* function that, when used together, achieves contracts-like functionality. """ -import re from pymbolic.primitives import Expression from abc import ABC, abstractmethod from typing import Optional, Union, Dict, Tuple, Any, List @@ -40,15 +39,15 @@ from typing import Optional, Union, Dict, Tuple, Any, List class NamespaceInterface(ABC): @property - def symbol_table(self) -> Dict["NameType", "ArrayInterface"]: + def symbol_table(self) -> Dict[str, "ArrayInterface"]: return self._namespace @symbol_table.setter - def symbol_table(self, val: Dict["NameType", "ArrayInterface"]) -> None: + def symbol_table(self, val: Dict[str, "ArrayInterface"]) -> None: self._namespace = val @abstractmethod - def assign(self, name: "NameType", + def assign(self, name: str, value: "ArrayInterface") -> None: pass @@ -86,25 +85,11 @@ class ArrayInterface(ABC): pass @abstractmethod - def with_name(self, name: "NameType") -> "ArrayInterface": + def with_name(self, name: str) -> "ArrayInterface": pass # }}} End abstract classes -# {{{ name type - - -NameType = str -C_IDENTIFIER = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") - - -def check_name(name: NameType) -> bool: - assert re.match(C_IDENTIFIER, name) is not None, \ - f"{name} is not a C identifier" - return True - -# }}} End name type - # {{{ shape type @@ -118,7 +103,7 @@ def check_shape(shape: ShapeType, if isinstance(s, int): assert s > 0, f"size parameter must be positive (got {s})" elif isinstance(s, str): - assert check_name(s) + assert str.isidentifier(s) elif isinstance(s, Expression) and ns is not None: # TODO: check expression in namespace pass @@ -144,7 +129,7 @@ class DottedName(): def __init__(self, name_parts: List[str]): assert len(name_parts) > 0 - assert all(check_name(p) for p in name_parts) + assert all(str.isidentifier(p) for p in name_parts) self.name_parts = name_parts -- GitLab From 4e0e77683f9793cdf6433dbd5d9982c54c5e94ef Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 25 May 2020 16:11:02 -0500 Subject: [PATCH 12/32] Fix docstrings regarding names --- pytato/array.py | 4 ++-- pytato/typing.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index d539777..edb2fb9 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -69,7 +69,7 @@ class Namespace(ptype.NamespaceInterface): .. attribute:: symbol_table A mapping from strings that must be valid - C identifiers to objects implementing the + Python identifiers to objects implementing the :class:`Array` interface. """ @@ -217,7 +217,7 @@ class Array(ptype.ArrayInterface): class DictOfNamedArrays(Mapping[str, ptype.ArrayInterface]): - """A container that maps valid C identifiers + """A container that maps valid Python identifiers to instances of :class:`Array`. May occur as a result type of array computations. diff --git a/pytato/typing.py b/pytato/typing.py index 8f784ec..6fb3482 100644 --- a/pytato/typing.py +++ b/pytato/typing.py @@ -119,7 +119,7 @@ class DottedName(): .. attribute:: name_parts A tuple of strings, each of which is a valid - C identifier (non-Unicode Python identifier). + Python identifier (non-Unicode Python identifier). The name (at least morally) exists in the name space defined by the Python module system. -- GitLab From 5e57e97767f52986fec8df68e10244b69c6da691 Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 25 May 2020 16:40:58 -0500 Subject: [PATCH 13/32] Let sphinx properly parse type hints --- .gitlab-ci.yml | 1 + doc/conf.py | 1 + doc/reference.rst | 1 + pytato/array.py | 10 +++++----- pytato/typing.py | 31 ++++++++++++++++++++++++++++--- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad24891..46cf817 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -45,6 +45,7 @@ Mypy: Documentation: script: + - EXTRA_INSTALL="sphinx-autodoc-typehints" - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-docs.sh - ". ./build-docs.sh" tags: diff --git a/doc/conf.py b/doc/conf.py index b96066b..7bd8b30 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -31,6 +31,7 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.mathjax', + 'sphinx_autodoc_typehints', ] # Add any paths that contain templates here, relative to this directory. diff --git a/doc/reference.rst b/doc/reference.rst index 522bdd4..4e20bdb 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -2,3 +2,4 @@ Reference ========= .. automodule:: pytato.array +.. automodule:: pytato.typing diff --git a/pytato/array.py b/pytato/array.py index edb2fb9..9766131 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -41,11 +41,6 @@ Array Interface .. autoclass :: Array .. autoclass :: DictOfNamedArrays -Supporting Functionality ------------------------- - -.. autoclass :: DottedName - Built-in Expression Nodes ------------------------- .. currentmodule:: pytato.array @@ -78,6 +73,11 @@ class Namespace(ptype.NamespaceInterface): def assign(self, name: str, value: ptype.ArrayInterface) -> None: + """Declare a new array. + + :param name: a Python identifier + :param value: the array object + """ assert str.isidentifier(name) if name in self.symbol_table: raise ValueError(f"'{name}' is already assigned") diff --git a/pytato/typing.py b/pytato/typing.py index 6fb3482..c3275af 100644 --- a/pytato/typing.py +++ b/pytato/typing.py @@ -24,9 +24,27 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -__doc__ = """Interface classes and type specifications. -Each type is paired with a check_* function that, when used together, achieves -contracts-like functionality. +__doc__ = """ + + +Interface classes and type specifications. +Some types are paired with ``check_*`` functions that, +when used together, achieves contracts-like functionality. + +Abstract Classes +---------------- + +.. autoclass :: NamespaceInterface + +.. autoclass :: ArrayInterface + +Supporting Functionalities +-------------------------- + +.. autoclass :: DottedName + +.. autofunction :: check_shape + """ from pymbolic.primitives import Expression @@ -99,6 +117,13 @@ ShapeType = Tuple[ShapeComponentType, ...] def check_shape(shape: ShapeType, ns: Optional[NamespaceInterface] = None) -> bool: + """Checks for a shape tuple. + + :param shape: the shape tuple + + :param ns: if a namespace is given, extra checks are performed to + ensure that expressions are well-defined. + """ for s in shape: if isinstance(s, int): assert s > 0, f"size parameter must be positive (got {s})" -- GitLab From 5922d2cd1193b2bc9bfc6aad2c17a270cd680d6c Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 25 May 2020 16:45:18 -0500 Subject: [PATCH 14/32] Make symbol_table read-only --- pytato/array.py | 5 ++++- pytato/typing.py | 7 ++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index 9766131..d885ea2 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -69,7 +69,10 @@ class Namespace(ptype.NamespaceInterface): """ def __init__(self) -> None: - self.symbol_table = {} + self._symbol_table = {} + + def symbol_table(self) -> Dict[str, ptype.ArrayInterface]: + return self._symbol_table def assign(self, name: str, value: ptype.ArrayInterface) -> None: diff --git a/pytato/typing.py b/pytato/typing.py index c3275af..2abddda 100644 --- a/pytato/typing.py +++ b/pytato/typing.py @@ -57,12 +57,9 @@ from typing import Optional, Union, Dict, Tuple, Any, List class NamespaceInterface(ABC): @property + @abstractmethod def symbol_table(self) -> Dict[str, "ArrayInterface"]: - return self._namespace - - @symbol_table.setter - def symbol_table(self, val: Dict[str, "ArrayInterface"]) -> None: - self._namespace = val + pass @abstractmethod def assign(self, name: str, -- GitLab From 5deb7e9448f4e5dea2cd17e637ff41b838c55dbb Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 25 May 2020 16:49:10 -0500 Subject: [PATCH 15/32] Symbol table fixes --- pytato/array.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index d885ea2..297e5a9 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -69,7 +69,7 @@ class Namespace(ptype.NamespaceInterface): """ def __init__(self) -> None: - self._symbol_table = {} + self._symbol_table: Dict[str, ptype.ArrayInterface] = {} def symbol_table(self) -> Dict[str, ptype.ArrayInterface]: return self._symbol_table @@ -82,9 +82,9 @@ class Namespace(ptype.NamespaceInterface): :param value: the array object """ assert str.isidentifier(name) - if name in self.symbol_table: + if name in self._symbol_table: raise ValueError(f"'{name}' is already assigned") - self.symbol_table[name] = value + self._symbol_table[name] = value class Array(ptype.ArrayInterface): -- GitLab From 7133061480f3b08fc4ad6a39b5e828e7f3a61de2 Mon Sep 17 00:00:00 2001 From: xywei Date: Wed, 27 May 2020 10:56:37 -0500 Subject: [PATCH 16/32] Relocate assertions from constructors into make_* functions --- pytato/__init__.py | 4 +++- pytato/array.py | 21 +++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pytato/__init__.py b/pytato/__init__.py index ab52ffd..5ca6a9f 100644 --- a/pytato/__init__.py +++ b/pytato/__init__.py @@ -27,6 +27,8 @@ THE SOFTWARE. from pytato.typing import DottedName from pytato.array import ( Namespace, Array, DictOfNamedArrays, + Placeholder, make_placeholder, ) -__all__ = ("DottedName", "Namespace", "Array", "DictOfNamedArrays") +__all__ = ("DottedName", "Namespace", "Array", "DictOfNamedArrays", + "Placeholder", "make_placeholder") diff --git a/pytato/array.py b/pytato/array.py index 297e5a9..c2421a8 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -81,7 +81,6 @@ class Namespace(ptype.NamespaceInterface): :param name: a Python identifier :param value: the array object """ - assert str.isidentifier(name) if name in self._symbol_table: raise ValueError(f"'{name}' is already assigned") self._symbol_table[name] = value @@ -167,8 +166,6 @@ class Array(ptype.ArrayInterface): def __init__(self, namespace: ptype.NamespaceInterface, name: Optional[str], tags: Optional[ptype.TagsType] = None): - assert (name is None) or str.isidentifier(name) - if tags is None: tags = {} @@ -236,7 +233,6 @@ class DictOfNamedArrays(Mapping[str, ptype.ArrayInterface]): """ def __init__(self, data: Dict[str, ptype.ArrayInterface]): - assert all(str.isidentifier(key) for key in data.keys()) self._data = data if not is_single_valued(ary.namespace for ary in data.values()): @@ -327,8 +323,6 @@ class Placeholder(Array): def __init__(self, namespace: ptype.NamespaceInterface, name: str, shape: ptype.ShapeType, tags: Optional[ptype.TagsType] = None): - assert ptype.check_shape(shape) - # namespace, name and tags will be checked in super().__init__() super().__init__( namespace=namespace, name=name, @@ -345,3 +339,18 @@ class LoopyFunction(DictOfNamedArrays): and one that's obtained by importing a dotted name. """ + + +# {{{ end-user-facing + + +def make_placeholder(namespace: ptype.NamespaceInterface, + name: str, + shape: ptype.ShapeType, + tags: Optional[ptype.TagsType] = None + ) -> Placeholder: + assert str.isidentifier(name) + assert ptype.check_shape(shape, namespace) + return Placeholder(namespace, name, shape, tags) + +# }}} End end-user-facing -- GitLab From b6e9115a104c4af7bab76373f977f89b35b5dda3 Mon Sep 17 00:00:00 2001 From: xywei Date: Wed, 27 May 2020 11:10:27 -0500 Subject: [PATCH 17/32] More checks relocated --- pytato/array.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index c2421a8..70800e4 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -50,11 +50,18 @@ Built-in Expression Nodes .. autoclass:: DataWrapper .. autoclass:: Placeholder .. autoclass:: LoopyFunction + +User Interface for Making Nodes +------------------------------- +.. currentmodule:: pytato.array + +.. autofunction:: make_dict_of_named_arrays +.. autofunction:: make_placeholder """ import pytato.typing as ptype -from pytools import single_valued, is_single_valued +from pytools import is_single_valued from typing import Optional, Dict, Any, Mapping, Iterator @@ -235,13 +242,9 @@ class DictOfNamedArrays(Mapping[str, ptype.ArrayInterface]): def __init__(self, data: Dict[str, ptype.ArrayInterface]): self._data = data - if not is_single_valued(ary.namespace for ary in data.values()): - raise ValueError("arrays do not have same namespace") - @property def namespace(self) -> ptype.NamespaceInterface: - return single_valued( # type: ignore - ary.namespace for ary in self._data.values()) + return next(iter(self._data.values())).namespace def __contains__(self, name: object) -> bool: return name in self._data @@ -344,11 +347,30 @@ class LoopyFunction(DictOfNamedArrays): # {{{ end-user-facing +def make_dict_of_named_arrays( + data: Dict[str, ptype.ArrayInterface]) -> DictOfNamedArrays: + """Make a :class:`DictOfNamedArrays` object and ensure that all arrays + share the same namespace. + + :param data: member keys and arrays + """ + if not is_single_valued(ary.namespace for ary in data.values()): + raise ValueError("arrays do not have same namespace") + return DictOfNamedArrays(data) + + def make_placeholder(namespace: ptype.NamespaceInterface, name: str, shape: ptype.ShapeType, tags: Optional[ptype.TagsType] = None ) -> Placeholder: + """Make a :class:`Placeholder` object. + + :param namespace: namespace of the placeholder array + :param name: name of the placeholder array + :param shape: shape of the placeholder array + :param tags: implementation tags + """ assert str.isidentifier(name) assert ptype.check_shape(shape, namespace) return Placeholder(namespace, name, shape, tags) -- GitLab From 602e0ab05378e292308665c92a641731676cf8eb Mon Sep 17 00:00:00 2001 From: xywei Date: Thu, 28 May 2020 09:59:04 -0500 Subject: [PATCH 18/32] Strip forward references of quotes --- pytato/array.py | 10 ++++++---- pytato/typing.py | 14 ++++++++------ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index 297e5a9..2841a09 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -1,3 +1,5 @@ +from __future__ import annotations + __copyright__ = """ Copyright (C) 2020 Andreas Kloeckner Copyright (C) 2020 Matt Wala @@ -179,7 +181,7 @@ class Array(ptype.ArrayInterface): self.name = name self.tags = tags - def copy(self, **kwargs: Any) -> "Array": + def copy(self, **kwargs: Any) -> Array: raise NotImplementedError @property @@ -195,7 +197,7 @@ class Array(ptype.ArrayInterface): return len(self.shape) def with_tag(self, dotted_name: ptype.DottedName, - args: Optional[ptype.DottedName] = None) -> "Array": + args: Optional[ptype.DottedName] = None) -> Array: """ Returns a copy of *self* tagged with *dotted_name* and arguments *args* @@ -206,10 +208,10 @@ class Array(ptype.ArrayInterface): pass return self.copy() - def without_tag(self, dotted_name: ptype.DottedName) -> "Array": + def without_tag(self, dotted_name: ptype.DottedName) -> Array: raise NotImplementedError - def with_name(self, name: str) -> "Array": + def with_name(self, name: str) -> Array: self.namespace.assign(name, self) return self.copy(name=name) diff --git a/pytato/typing.py b/pytato/typing.py index 2abddda..7383a87 100644 --- a/pytato/typing.py +++ b/pytato/typing.py @@ -1,3 +1,5 @@ +from __future__ import annotations + __copyright__ = """ Copyright (C) 2020 Andreas Kloeckner Copyright (C) 2020 Matt Wala @@ -58,12 +60,12 @@ class NamespaceInterface(ABC): @property @abstractmethod - def symbol_table(self) -> Dict[str, "ArrayInterface"]: + def symbol_table(self) -> Dict[str, ArrayInterface]: pass @abstractmethod def assign(self, name: str, - value: "ArrayInterface") -> None: + value: ArrayInterface) -> None: pass @@ -87,20 +89,20 @@ class ArrayInterface(ABC): pass @abstractmethod - def copy(self, **kwargs: Any) -> "ArrayInterface": + def copy(self, **kwargs: Any) -> ArrayInterface: pass @abstractmethod def with_tag(self, tag_key: Any, - tag_val: Any) -> "ArrayInterface": + tag_val: Any) -> ArrayInterface: pass @abstractmethod - def without_tag(self, tag_key: Any) -> "ArrayInterface": + def without_tag(self, tag_key: Any) -> ArrayInterface: pass @abstractmethod - def with_name(self, name: str) -> "ArrayInterface": + def with_name(self, name: str) -> ArrayInterface: pass # }}} End abstract classes -- GitLab From fbe24d5657e4ba5d071b0c40068dc1a39590f314 Mon Sep 17 00:00:00 2001 From: xywei Date: Fri, 29 May 2020 11:48:09 -0500 Subject: [PATCH 19/32] Mypy fixes --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index e52dab2..b70f9b3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,6 +5,9 @@ max-line-length=85 [mypy-pymbolic] ignore_missing_imports = True +[mypy-pymbolic.primitives] +ignore_missing_imports = True + [mypy-pytools] ignore_missing_imports = True -- GitLab From f224daff4d45f29ca7c8bc187275827d5b9de8d5 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wei Date: Fri, 29 May 2020 18:50:17 +0200 Subject: [PATCH 20/32] Apply suggestion to pytato/array.py --- pytato/array.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index e5d6818..43408c6 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -69,8 +69,13 @@ Built-in Expression Nodes .. autoclass:: Placeholder .. autoclass:: LoopyFunction -User Interface for Making Nodes -------------------------------- +User-Facing Node Creation +------------------------- + +Node constructors such as :class:`Placeholder.__init__` and +:class:`DictOfNamedArrays.__init__` offer limited input validation +(in favor of faster execution). Node creation from outside +:mod:`pytato` should use the following interfaces: .. autofunction:: make_dict_of_named_arrays .. autofunction:: make_placeholder -- GitLab From e70533440d63931c32e833d145437dd329dcbf04 Mon Sep 17 00:00:00 2001 From: xywei Date: Fri, 29 May 2020 12:53:44 -0500 Subject: [PATCH 21/32] Make input checks mandatory --- pytato/array.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index 43408c6..dcde49d 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -248,12 +248,17 @@ def check_shape(shape: ShapeType, """ for s in shape: if isinstance(s, int): - assert s > 0, f"size parameter must be positive (got {s})" + if s <= 0: + raise ValueError(f"size parameter must be positive (got {s})") + elif isinstance(s, str): - assert str.isidentifier(s) + if not str.isidentifier(s): + raise ValueError(f"{s} is not a valid identifier") + elif isinstance(s, prim.Expression) and ns is not None: # TODO: check expression in namespace - pass + raise NotImplementedError + return True # }}} @@ -655,8 +660,17 @@ class LoopyFunction(DictOfNamedArrays): def make_dotted_name(name_parts: Tuple[str, ...]) -> DottedName: - assert len(name_parts) > 0 - assert all(str.isidentifier(p) for p in name_parts) + """Make a :class:`DottedName` for tagging purposes. + + :param name_parts: each name part must be a valid Python identifier + """ + if len(name_parts) == 0: + raise ValueError("empty name parts") + + for p in name_parts: + if not str.isidentifier(p): + raise ValueError(f"{p} is not a Python identifier") + return DottedName(name_parts) @@ -669,6 +683,7 @@ def make_dict_of_named_arrays( """ if not is_single_valued(ary.namespace for ary in data.values()): raise ValueError("arrays do not have same namespace") + return DictOfNamedArrays(data) @@ -684,8 +699,11 @@ def make_placeholder(namespace: Namespace, :param shape: shape of the placeholder array :param tags: implementation tags """ - assert str.isidentifier(name) - assert check_shape(shape, namespace) + if not str.isidentifier(name): + raise ValueError(f"{name} is not a Python identifier") + + check_shape(shape, namespace) + return Placeholder(namespace, name, shape, tags) # }}} End end-user-facing -- GitLab From 9549a45b51331554579b2069924dacbcd39d1cdc Mon Sep 17 00:00:00 2001 From: xywei Date: Fri, 29 May 2020 14:45:32 -0500 Subject: [PATCH 22/32] Shrink diff size --- pytato/array.py | 72 ++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index dcde49d..305fa84 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -90,10 +90,44 @@ from dataclasses import dataclass from pytools import is_single_valued from typing import Optional, Dict, Any, Mapping, Iterator, Tuple, Union +# {{{ dotted name + + +class DottedName(): + """ + .. attribute:: name_parts + + A tuple of strings, each of which is a valid + Python identifier. No name part may start with + a double underscore. + + The name (at least morally) exists in the + name space defined by the Python module system. + It need not necessarily identify an importable + object. + + .. automethod:: from_class + """ + + def __init__(self, name_parts: Tuple[str, ...]): + self.name_parts = name_parts + + @classmethod + def from_class(cls, argcls: Any) -> DottedName: + name_parts = tuple( + [str(part) for part in argcls.__module__.split(".")] + + [str(argcls.__name__)]) + if not all(not npart.startswith("__") for npart in name_parts): + raise ValueError(f"some name parts of {'.'.join(name_parts)} " + "start with double underscores") + return cls(name_parts) + +# }}} + # {{{ namespace -class Namespace(): +class Namespace: r""" Represents a mapping from :term:`identifier` strings to :term:`array expression`\ s or *None*, where *None* indicates that the name @@ -159,36 +193,6 @@ class Namespace(): # {{{ tag -class DottedName(): - """ - .. attribute:: name_parts - - A tuple of strings, each of which is a valid - Python identifier. No name part may start with - a double underscore. - - The name (at least morally) exists in the - name space defined by the Python module system. - It need not necessarily identify an importable - object. - - .. automethod:: from_class - """ - - def __init__(self, name_parts: Tuple[str, ...]): - self.name_parts = name_parts - - @classmethod - def from_class(cls, argcls: Any) -> DottedName: - name_parts = tuple( - [str(part) for part in argcls.__module__.split(".")] - + [str(argcls.__name__)]) - if not all(not npart.startswith("__") for npart in name_parts): - raise ValueError(f"some name parts of {'.'.join(name_parts)} " - "start with double underscores") - return cls(name_parts) - - tag_dataclass = dataclass(init=True, eq=True, frozen=True, repr=True) @@ -332,17 +336,13 @@ class Array: if tags is None: tags = {} - self._namespace = namespace + self.namespace = namespace self.tags = tags self.dtype: np.dtype = np.float64 # FIXME def copy(self, **kwargs: Any) -> Array: raise NotImplementedError - @property - def namespace(self) -> Namespace: - return self._namespace - @property def shape(self) -> ShapeType: raise NotImplementedError -- GitLab From 8bf01c2cd9a75073fc298ad1068bfcfec9cbe3fc Mon Sep 17 00:00:00 2001 From: xywei Date: Fri, 29 May 2020 14:51:46 -0500 Subject: [PATCH 23/32] Make diff even smaller --- pytato/array.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index 305fa84..05a9a67 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -90,10 +90,10 @@ from dataclasses import dataclass from pytools import is_single_valued from typing import Optional, Dict, Any, Mapping, Iterator, Tuple, Union -# {{{ dotted name +# {{{ dotted name -class DottedName(): +class DottedName: """ .. attribute:: name_parts @@ -124,10 +124,11 @@ class DottedName(): # }}} -# {{{ namespace +# {{{ namespace class Namespace: + # Possible future extension: .parent attribute r""" Represents a mapping from :term:`identifier` strings to :term:`array expression`\ s or *None*, where *None* indicates that the name @@ -183,15 +184,14 @@ class Namespace: if value.shape: var_ref = var_ref[tuple("_%d" % i for i in range(len(value.shape)))] - # FIXME: mypy thinks that this is Any return IndexLambda( self, expr=var_ref, shape=value.shape, dtype=value.dtype) # }}} -# {{{ tag +# {{{ tag tag_dataclass = dataclass(init=True, eq=True, frozen=True, repr=True) @@ -234,8 +234,8 @@ TagsType = Dict[DottedName, Tag] # }}} -# {{{ shape +# {{{ shape ShapeComponentType = Union[int, prim.Expression, str] ShapeType = Tuple[ShapeComponentType, ...] @@ -267,8 +267,8 @@ def check_shape(shape: ShapeType, # }}} -# {{{ array inteface +# {{{ array inteface class Array: """ @@ -372,8 +372,8 @@ class Array: # }}} -# {{{ pre-defined tag: ImplementAs +# {{{ pre-defined tag: ImplementAs @tag_dataclass class ImplementationStrategy(Tag): @@ -405,8 +405,8 @@ class ImplementAs(UniqueTag): # }}} -# {{{ pre-defined tag: CountNamed +# {{{ pre-defined tag: CountNamed @tag_dataclass class CountNamed(UniqueTag): @@ -418,8 +418,8 @@ class CountNamed(UniqueTag): # }}} -# {{{ dict of named arrays +# {{{ dict of named arrays class DictOfNamedArrays(Mapping[str, Array]): """A container that maps valid Python identifiers @@ -458,8 +458,8 @@ class DictOfNamedArrays(Mapping[str, Array]): # }}} -# {{{ index lambda +# {{{ index lambda class IndexLambda(Array): """ @@ -545,8 +545,8 @@ class IndexLambda(Array): # }}} -# {{{ einsum +# {{{ einsum class Einsum(Array): """ @@ -554,8 +554,8 @@ class Einsum(Array): # }}} -# {{{ reshape +# {{{ reshape class Reshape(Array): """ @@ -563,8 +563,8 @@ class Reshape(Array): # }}} -# {{{ data wrapper +# {{{ data wrapper class DataWrapper(Array): # TODO: Name? @@ -601,8 +601,8 @@ class DataWrapper(Array): # }}} -# {{{ placeholder +# {{{ placeholder class Placeholder(Array): """ @@ -642,8 +642,8 @@ class Placeholder(Array): # }}} -# {{{ loopy function +# {{{ loopy function class LoopyFunction(DictOfNamedArrays): """ @@ -656,8 +656,8 @@ class LoopyFunction(DictOfNamedArrays): # }}} -# {{{ end-user-facing +# {{{ end-user-facing def make_dotted_name(name_parts: Tuple[str, ...]) -> DottedName: """Make a :class:`DottedName` for tagging purposes. -- GitLab From 4dc30c5bb5e42d1ca5b40f4b2ee1eb22964bb63d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Mon, 1 Jun 2020 17:27:07 +0200 Subject: [PATCH 24/32] Apply suggestion to pytato/array.py --- pytato/array.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index 05a9a67..85ce32b 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -252,8 +252,8 @@ def check_shape(shape: ShapeType, """ for s in shape: if isinstance(s, int): - if s <= 0: - raise ValueError(f"size parameter must be positive (got {s})") + if s < 0: + raise ValueError(f"size parameter must be nonnegative (got {s})") elif isinstance(s, str): if not str.isidentifier(s): -- GitLab From 63283c724c11ff1d615b9c83589fc37c89e1fe35 Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Mon, 1 Jun 2020 11:02:18 -0500 Subject: [PATCH 25/32] Gitignore mypy cache --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index d97d175..1b2f54d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ distribute*tar.gz .ipynb_checkpoints doc/_build + +.mypy_cache -- GitLab From 56fedf90a906ce00d7d3d25a282a0339a401f627 Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Mon, 1 Jun 2020 11:02:32 -0500 Subject: [PATCH 26/32] More spacing in gitlab CI --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c0b6f9f..56cf388 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,7 @@ Python 3 POCL: reports: junit: test/pytest.xml + Pylint: script: - export PY_EXE=python3 @@ -25,6 +26,7 @@ Pylint: except: - tags + Flake8: script: - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/prepare-and-run-flake8.sh @@ -34,6 +36,7 @@ Flake8: except: - tags + Mypy: script: - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/prepare-and-run-mypy.sh @@ -43,6 +46,7 @@ Mypy: except: - tags + Documentation: script: - EXTRA_INSTALL="sphinx-autodoc-typehints" -- GitLab From e56d21aea051bcd179bf5236ce1f720021d7e8b9 Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Mon, 1 Jun 2020 11:02:50 -0500 Subject: [PATCH 27/32] Add a submodule for scalar expression handling --- pytato/scalar_expr.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 pytato/scalar_expr.py diff --git a/pytato/scalar_expr.py b/pytato/scalar_expr.py new file mode 100644 index 0000000..aa430ee --- /dev/null +++ b/pytato/scalar_expr.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +__copyright__ = """ +Copyright (C) 2020 Andreas Kloeckner +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from pymbolic.mapper import WalkMapper as WalkMapperBase + + +def parse(s): + from pymbolic.parser import Parser + return Parser()(s) + + +class WalkMapper(WalkMapperBase): + pass + + +# vim: foldmethod=marker -- GitLab From 8151e9e03378d7494e5922fc79e79a3cc026e9f8 Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Mon, 1 Jun 2020 11:03:23 -0500 Subject: [PATCH 28/32] Mypy config for more pymbolic imports --- setup.cfg | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.cfg b/setup.cfg index b70f9b3..d16b02e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,6 +8,12 @@ ignore_missing_imports = True [mypy-pymbolic.primitives] ignore_missing_imports = True +[mypy-pymbolic.mapper] +ignore_missing_imports = True + +[mypy-pymbolic.parser] +ignore_missing_imports = True + [mypy-pytools] ignore_missing_imports = True -- GitLab From 059c720cc1bff582f8147b6f0fe3f91a1eb9a61b Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Mon, 1 Jun 2020 11:04:15 -0500 Subject: [PATCH 29/32] Introduce ConvertibleToShape, normalize_shape --- pytato/array.py | 79 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index 85ce32b..c965c18 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -77,6 +77,8 @@ Node constructors such as :class:`Placeholder.__init__` and (in favor of faster execution). Node creation from outside :mod:`pytato` should use the following interfaces: +.. class:: ConvertibleToShape + .. autofunction:: make_dict_of_named_arrays .. autofunction:: make_placeholder """ @@ -85,6 +87,7 @@ Node constructors such as :class:`Placeholder.__init__` and import numpy as np import pymbolic.primitives as prim +import pytato.scalar_expr as scalar_expr from dataclasses import dataclass from pytools import is_single_valued @@ -237,33 +240,66 @@ TagsType = Dict[DottedName, Tag] # {{{ shape -ShapeComponentType = Union[int, prim.Expression, str] +ShapeComponentType = Union[int, prim.Expression] ShapeType = Tuple[ShapeComponentType, ...] +ConvertibleToShapeComponent = Union[int, prim.Expression, str] +ConvertibleToShape = Union[ + str, + prim.Expression, + Tuple[ConvertibleToShapeComponent, ...]] + + +def _check_identifier(s, ns: Optional[Namespace] = None): + if not str.isidentifier(s): + raise ValueError(f"'{s}' is not a valid identifier") + + if ns is not None: + if s not in ns: + raise ValueError(f"'{s}' is not known in the namespace") + + return True + +class _ShapeChecker(scalar_expr.WalkMapper): + def __init__(self, ns: Optional[Namespace] = None): + super().__init__() -def check_shape(shape: ShapeType, - ns: Optional[Namespace] = None) -> bool: - """Checks for a shape tuple. + def map_variable(self, expr): + _check_identifier(expr.name, self.ns) + super().map_variable(expr) - :param shape: the shape tuple +def normalize_shape( + shape: ConvertibleToShape, + ns: Optional[Namespace] = None + ) -> ShapeType: + """ :param ns: if a namespace is given, extra checks are performed to ensure that expressions are well-defined. """ - for s in shape: + from pytato.scalar_expr import parse + + def nnormalize_shape_component(s): + if isinstance(s, str): + s = parse(s) + if isinstance(s, int): - if s < 0: - raise ValueError(f"size parameter must be nonnegative (got {s})") + if s < 0: + raise ValueError(f"size parameter must be nonnegative (got '{s}')") - elif isinstance(s, str): - if not str.isidentifier(s): - raise ValueError(f"{s} is not a valid identifier") + elif isinstance(s, prim.Expression): + # TODO: check expression affine-ness + _ShapeChecker()(s) - elif isinstance(s, prim.Expression) and ns is not None: - # TODO: check expression in namespace - raise NotImplementedError + return s - return True + if isinstance(shape, str): + shape = parse(shape) + + if isinstance(shape, (int, prim.Expression)): + shape = (shape,) + + return tuple(nnormalize_shape_component(s) for s in shape) # }}} @@ -689,23 +725,22 @@ def make_dict_of_named_arrays( def make_placeholder(namespace: Namespace, name: str, - shape: ShapeType, + shape: ConvertibleToShape, tags: Optional[TagsType] = None ) -> Placeholder: """Make a :class:`Placeholder` object. - :param namespace: namespace of the placeholder array - :param name: name of the placeholder array - :param shape: shape of the placeholder array - :param tags: implementation tags + :param namespace: namespace of the placeholder array + :param shape: shape of the placeholder array + :param tags: implementation tags """ if not str.isidentifier(name): raise ValueError(f"{name} is not a Python identifier") - check_shape(shape, namespace) + shape = normalize_shape(shape, namespace) return Placeholder(namespace, name, shape, tags) -# }}} End end-user-facing +# }}} # vim: foldmethod=marker -- GitLab From 45fba6921575b3d2a5798cadc01d86923147a089 Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Mon, 1 Jun 2020 11:04:35 -0500 Subject: [PATCH 30/32] Fold make_dotted_name into DottedName constructor --- pytato/array.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index c965c18..6e3bf69 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -113,6 +113,13 @@ class DottedName: """ def __init__(self, name_parts: Tuple[str, ...]): + if len(name_parts) == 0: + raise ValueError("empty name parts") + + for p in name_parts: + if not str.isidentifier(p): + raise ValueError(f"{p} is not a Python identifier") + self.name_parts = name_parts @classmethod @@ -695,21 +702,6 @@ class LoopyFunction(DictOfNamedArrays): # {{{ end-user-facing -def make_dotted_name(name_parts: Tuple[str, ...]) -> DottedName: - """Make a :class:`DottedName` for tagging purposes. - - :param name_parts: each name part must be a valid Python identifier - """ - if len(name_parts) == 0: - raise ValueError("empty name parts") - - for p in name_parts: - if not str.isidentifier(p): - raise ValueError(f"{p} is not a Python identifier") - - return DottedName(name_parts) - - def make_dict_of_named_arrays( data: Dict[str, Array]) -> DictOfNamedArrays: """Make a :class:`DictOfNamedArrays` object and ensure that all arrays -- GitLab From d9ed7495a8905536efa37660ff3df4556638b425 Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Mon, 1 Jun 2020 11:13:59 -0500 Subject: [PATCH 31/32] Implement Array.tagged, Array.without_tag --- pytato/array.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index 6e3bf69..1391103 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -91,7 +91,7 @@ import pytato.scalar_expr as scalar_expr from dataclasses import dataclass from pytools import is_single_valued -from typing import Optional, Dict, Any, Mapping, Iterator, Tuple, Union +from typing import Optional, Dict, Any, Mapping, Iterator, Tuple, Union, FrozenSet # {{{ dotted name @@ -240,7 +240,7 @@ class UniqueTag(Tag): """ -TagsType = Dict[DottedName, Tag] +TagsType = FrozenSet[Tag] # }}} @@ -356,8 +356,7 @@ class Array: .. attribute:: tags - A :class:`dict` mapping :class:`DottedName` instances - to instances of the :class:`Tag` interface. + A :class:`tuple` of :class:`Tag` instances. Motivation: `RDF `__ @@ -377,7 +376,7 @@ class Array: def __init__(self, namespace: Namespace, tags: Optional[TagsType] = None): if tags is None: - tags = {} + tags = frozenset() self.namespace = namespace self.tags = tags @@ -404,11 +403,17 @@ class Array: tags of this type are already present, an error is raised. """ - raise NotImplementedError - return self.copy() + return self.copy(tags=self.tags | frozenset([tag])) - def without_tag(self, dotted_name: DottedName) -> Array: - raise NotImplementedError + def without_tag(self, tag: Tag, verify_existence: bool = True) -> Array: + new_tags = tuple( + t for t in self.tags + if t != tag) + + if verify_existence and len(new_tags) == len(self.tags): + raise ValueError(f"tag '{tag}' was not present") + + return self.copy(tags=new_tags) # TODO: # - codegen interface @@ -536,7 +541,7 @@ class IndexLambda(Array): self, namespace: Namespace, expr: prim.Expression, shape: ShapeType, dtype: np.dtype, bindings: Optional[Dict[str, Array]] = None, - tags: Optional[Dict[DottedName, Tag]] = None): + tags: Optional[TagsType] = None): if bindings is None: bindings = {} -- GitLab From 4dcb62e4f49d6b912f3e8c8a72ee991a9eb2f2e9 Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Mon, 1 Jun 2020 11:43:20 -0500 Subject: [PATCH 32/32] Placate linters --- pytato/array.py | 19 +++++++++++-------- pytato/scalar_expr.py | 9 ++++++++- setup.cfg | 3 +++ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/pytato/array.py b/pytato/array.py index 1391103..046955f 100644 --- a/pytato/array.py +++ b/pytato/array.py @@ -88,6 +88,7 @@ Node constructors such as :class:`Placeholder.__init__` and import numpy as np import pymbolic.primitives as prim import pytato.scalar_expr as scalar_expr +from pytato.scalar_expr import ScalarExpression from dataclasses import dataclass from pytools import is_single_valued @@ -247,16 +248,15 @@ TagsType = FrozenSet[Tag] # {{{ shape -ShapeComponentType = Union[int, prim.Expression] -ShapeType = Tuple[ShapeComponentType, ...] +ShapeType = Tuple[ScalarExpression, ...] ConvertibleToShapeComponent = Union[int, prim.Expression, str] ConvertibleToShape = Union[ str, - prim.Expression, + ScalarExpression, Tuple[ConvertibleToShapeComponent, ...]] -def _check_identifier(s, ns: Optional[Namespace] = None): +def _check_identifier(s: str, ns: Optional[Namespace] = None) -> bool: if not str.isidentifier(s): raise ValueError(f"'{s}' is not a valid identifier") @@ -270,8 +270,9 @@ def _check_identifier(s, ns: Optional[Namespace] = None): class _ShapeChecker(scalar_expr.WalkMapper): def __init__(self, ns: Optional[Namespace] = None): super().__init__() + self.ns = ns - def map_variable(self, expr): + def map_variable(self, expr: prim.Variable) -> None: _check_identifier(expr.name, self.ns) super().map_variable(expr) @@ -286,7 +287,8 @@ def normalize_shape( """ from pytato.scalar_expr import parse - def nnormalize_shape_component(s): + def normalize_shape_component( + s: ConvertibleToShapeComponent) -> ScalarExpression: if isinstance(s, str): s = parse(s) @@ -303,10 +305,11 @@ def normalize_shape( if isinstance(shape, str): shape = parse(shape) - if isinstance(shape, (int, prim.Expression)): + from numbers import Number + if isinstance(shape, (Number, prim.Expression)): shape = (shape,) - return tuple(nnormalize_shape_component(s) for s in shape) + return tuple(normalize_shape_component(s) for s in shape) # }}} diff --git a/pytato/scalar_expr.py b/pytato/scalar_expr.py index aa430ee..c675984 100644 --- a/pytato/scalar_expr.py +++ b/pytato/scalar_expr.py @@ -25,9 +25,16 @@ THE SOFTWARE. """ from pymbolic.mapper import WalkMapper as WalkMapperBase +import pymbolic.primitives as prim +from numbers import Number +from typing import Union -def parse(s): + +ScalarExpression = Union[Number, prim.Expression] + + +def parse(s: str) -> ScalarExpression: from pymbolic.parser import Parser return Parser()(s) diff --git a/setup.cfg b/setup.cfg index d16b02e..b975cd3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,6 +2,9 @@ ignore = E126,E127,E128,E123,E226,E241,E242,E265,N802,W503,E402,N814,N817,W504 max-line-length=85 +[mypy-pytato.scalar_expr] +disallow_subclassing_any = False + [mypy-pymbolic] ignore_missing_imports = True -- GitLab