From 344c1145920f5fd9870d9fddf6632b4d3d287068 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Sun, 23 Jun 2019 00:51:11 -0500 Subject: [PATCH 01/23] switch to ply parsing in query lang --- loopy/match.py | 236 ++++++++++++++++++++----------------------------- 1 file changed, 95 insertions(+), 141 deletions(-) diff --git a/loopy/match.py b/loopy/match.py index 3c047e463..c322cfb9f 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -25,13 +25,14 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from six.moves import range, intern +from six.moves import range +from loopy.diagnostic import LoopyError +import ply.lex as lex +import ply.yacc as yacc NoneType = type(None) -from pytools.lex import RE - __doc__ = """ .. autofunction:: parse_match @@ -59,58 +60,6 @@ def re_from_glob(s): return re.compile("^"+translate(s.strip())+"$") -# {{{ parsing - -# {{{ lexer data - -_and = intern("and") -_or = intern("or") -_not = intern("not") -_openpar = intern("openpar") -_closepar = intern("closepar") - -_id = intern("_id") -_tag = intern("_tag") -_writes = intern("_writes") -_reads = intern("_reads") -_iname = intern("_iname") - -_whitespace = intern("_whitespace") - -# }}} - - -_LEX_TABLE = [ - (_and, RE(r"and\b")), - (_or, RE(r"or\b")), - (_not, RE(r"not\b")), - (_openpar, RE(r"\(")), - (_closepar, RE(r"\)")), - - # TERMINALS - (_id, RE(r"id:([\w?*]+)")), - (_tag, RE(r"tag:([\w?*]+)")), - (_writes, RE(r"writes:([\w?*]+)")), - (_reads, RE(r"reads:([\w?*]+)")), - (_iname, RE(r"iname:([\w?*]+)")), - - (_whitespace, RE("[ \t]+")), - ] - - -_TERMINALS = ([_id, _tag, _writes, _reads, _iname]) - -# {{{ operator precedence - -_PREC_OR = 10 -_PREC_AND = 20 -_PREC_NOT = 30 - -# }}} - -# }}} - - # {{{ match expression class MatchExpressionBase(object): @@ -278,94 +227,99 @@ def parse_match(expr): * ``id:yoink and writes:a_temp`` * ``id:yoink and (not writes:a_temp or tag:input)`` """ - if not expr: - return All() - - def parse_terminal(pstate): - next_tag = pstate.next_tag() - if next_tag is _id: - result = Id(pstate.next_match_obj().group(1)) - pstate.advance() - return result - elif next_tag is _tag: - result = Tagged(pstate.next_match_obj().group(1)) - pstate.advance() - return result - elif next_tag is _writes: - result = Writes(pstate.next_match_obj().group(1)) - pstate.advance() - return result - elif next_tag is _reads: - result = Reads(pstate.next_match_obj().group(1)) - pstate.advance() - return result - elif next_tag is _iname: - result = Iname(pstate.next_match_obj().group(1)) - pstate.advance() - return result + + # None -> '' + expr = expr if expr else '' + + _RESERVED_TO_TYPES = { + 'id': Id, + 'iname': Iname, + 'reads': Reads, + 'writes': Writes, + 'tag': Tagged + } + + _BINARY_OPS_TO_TYPES = { + 'and': And, + 'or': Or + } + + reserved = { + 'id': 'ID', + 'iname': 'INAME', + 'reads': 'READS', + 'writes': 'WRITES', + 'tag': 'TAG', + + 'not': 'NOT', + 'or': 'OR', + 'and': 'AND'} + + tokens = ('NAME', 'COLON', 'LPAREN', 'RPAREN',) + tuple(reserved.values()) # noqa + + precedence = ( # noqa + ('left', 'OR'), + ('left', 'AND'), + ('left', 'NOT'),) + + def t_NAME(t): + r'[a-zA-Z_*][a-zA-Z0-9_*.]*' + t.type = reserved.get(t.value, 'NAME') + return t + + t_COLON = r':' # noqa + t_LPAREN = r'\(' # noqa + t_RPAREN = r'\)' # noqa + t_ignore = ' \t' # noqa + + def t_error(t): + raise RuntimeError("Illegal character '%s'." % t.value[0]) + + def p_expr_of_binary_ops(p): + '''expression : expression AND expression + | expression OR expression''' + children = [] + if type(p[1]) == _BINARY_OPS_TO_TYPES[p[2]]: + children.extend(p[1].children) else: - pstate.expected("terminal") - - def inner_parse(pstate, min_precedence=0): - pstate.expect_not_end() - - if pstate.is_next(_not): - pstate.advance() - left_query = Not(inner_parse(pstate, _PREC_NOT)) - elif pstate.is_next(_openpar): - pstate.advance() - left_query = inner_parse(pstate) - pstate.expect(_closepar) - pstate.advance() + children.append(p[1]) + if type(p[3]) == _BINARY_OPS_TO_TYPES[p[2]]: + children.extend(p[3].children) else: - left_query = parse_terminal(pstate) - - did_something = True - while did_something: - did_something = False - if pstate.is_at_end(): - return left_query - - next_tag = pstate.next_tag() - - if next_tag is _and and _PREC_AND > min_precedence: - pstate.advance() - left_query = And( - (left_query, inner_parse(pstate, _PREC_AND))) - did_something = True - elif next_tag is _or and _PREC_OR > min_precedence: - pstate.advance() - left_query = Or( - (left_query, inner_parse(pstate, _PREC_OR))) - did_something = True - - return left_query - - if isinstance(expr, MatchExpressionBase): - return expr - - from pytools.lex import LexIterator, lex, InvalidTokenError - try: - pstate = LexIterator( - [(tag, s, idx, matchobj) - for (tag, s, idx, matchobj) in lex(_LEX_TABLE, expr, - match_objects=True) - if tag is not _whitespace], expr) - except InvalidTokenError as e: - from loopy.diagnostic import LoopyError - raise LoopyError( - "invalid match expression: '{match_expr}' ({err_type}: {err_str})" - .format( - match_expr=expr, - err_type=type(e).__name__, - err_str=str(e))) - - if pstate.is_at_end(): - pstate.raise_parse_error("unexpected end of input") - - result = inner_parse(pstate) - if not pstate.is_at_end(): - pstate.raise_parse_error("leftover input after completed parse") + children.append(p[3]) + p[0] = _BINARY_OPS_TO_TYPES[p[2]](tuple(children)) + + def p_expr_of_unary_ops(p): + 'expression : NOT expression' + p[0] = Not(p[2]) + + def p_expr_of_parens(p): + 'expression : LPAREN expression RPAREN' + p[0] = p[2] + + def p_expr_as_term(p): + 'expression : term' + p[0] = p[1] + + def p_terminal(p): + '''term : ID COLON NAME + | INAME COLON NAME + | READS COLON NAME + | WRITES COLON NAME + | TAG COLON NAME''' + p[0] = _RESERVED_TO_TYPES[p[1]](p[3]) + + def p_empty_terminal(p): + 'term :' + p[0] = All() + + def p_error(p): + raise LoopyError("Syntax error at '%s'." % str(p)) + + lexer = lex.lex() # noqa + + parser = yacc.yacc() + result = parser.parse(expr) return result -- GitLab From 0852ce8acbd3dcd9cd9cb11d69efdba3e2437df9 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Sun, 23 Jun 2019 02:42:59 -0500 Subject: [PATCH 02/23] dont parse expressions which are already MatchExpressionBase --- loopy/match.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/loopy/match.py b/loopy/match.py index c322cfb9f..6f395d058 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -231,6 +231,9 @@ def parse_match(expr): # None -> '' expr = expr if expr else '' + if isinstance(expr, MatchExpressionBase): + return expr + _RESERVED_TO_TYPES = { 'id': Id, 'iname': Iname, -- GitLab From cbcc220c3a7bdbfbdbac90e7254a419dcb614566 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Sun, 23 Jun 2019 02:51:29 -0500 Subject: [PATCH 03/23] removes duplication in property matching --- loopy/match.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/loopy/match.py b/loopy/match.py index 6f395d058..9a07f5cb6 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -305,13 +305,17 @@ def parse_match(expr): p[0] = p[1] def p_terminal(p): - '''term : ID COLON NAME - | INAME COLON NAME - | READS COLON NAME - | WRITES COLON NAME - | TAG COLON NAME''' + 'term : prop COLON NAME' p[0] = _RESERVED_TO_TYPES[p[1]](p[3]) + def p_prop(p): + '''prop : ID + | INAME + | TAG + | READS + | WRITES''' + p[0] = p[1] + def p_empty_terminal(p): 'term :' p[0] = All() -- GitLab From afaf25e3d6c90a70caf89744a55d218092d24a79 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Sun, 23 Jun 2019 17:54:33 -0500 Subject: [PATCH 04/23] preliminary work on parsing calls --- loopy/match.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/loopy/match.py b/loopy/match.py index 9a07f5cb6..47f234e98 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -25,10 +25,12 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import six from six.moves import range from loopy.diagnostic import LoopyError import ply.lex as lex import ply.yacc as yacc +import re NoneType = type(None) @@ -216,6 +218,28 @@ class Iname(GlobMatchExpressionBase): return any(self.re.match(name) for name in matchable.within_inames) + +class Call(GlobMatchExpressionBase): + def __init__(self, glob, kwargs): + assert isinstance(glob, str) + if not isinstance(kwargs, dict): + kwargs_as_tuple = kwargs[:] + kwargs = dict(kwargs_as_tuple) + if len(kwargs) != len(kwargs_as_tuple): + raise LoopyError("Call '{0}' has repitition in the given " + "attributes.".format(glob)) + super(Call, self).__init__(glob) + self.kwargs = kwargs + + def __call__(self, kernel, matchable): + raise NotImplementedError() + + def __str__(self): + return "{0}({1})".format( + self.glob, + ", ".join("%s=%s" % (key, val) for key, val in + six.iteritems(self.kwargs))) + # }}} @@ -258,7 +282,7 @@ def parse_match(expr): 'or': 'OR', 'and': 'AND'} - tokens = ('NAME', 'COLON', 'LPAREN', 'RPAREN',) + tuple(reserved.values()) # noqa + tokens = ('NAME', 'COLON', 'LPAREN', 'RPAREN', 'COMMA', 'ASSIGN',) + tuple(reserved.values()) # noqa precedence = ( # noqa ('left', 'OR'), @@ -273,6 +297,8 @@ def parse_match(expr): t_COLON = r':' # noqa t_LPAREN = r'\(' # noqa t_RPAREN = r'\)' # noqa + t_COMMA = r',' # noqa + t_ASSIGN = r'=' # noqa t_ignore = ' \t' # noqa def t_error(t): @@ -296,18 +322,59 @@ def parse_match(expr): 'expression : NOT expression' p[0] = Not(p[2]) - def p_expr_of_parens(p): - 'expression : LPAREN expression RPAREN' + def p_parens(p): + '''expression : LPAREN expression RPAREN + tuple_of_names : LPAREN tuple_of_names RPAREN''' p[0] = p[2] - def p_expr_as_term(p): + def p_expr_as_terminal(p): 'expression : term' p[0] = p[1] - def p_terminal(p): + def p_terminal_as_property(p): 'term : prop COLON NAME' p[0] = _RESERVED_TO_TYPES[p[1]](p[3]) + def p_terminal_as_call(p): + 'term : NAME LPAREN kwargs RPAREN' + p[0] = Call(p[1], p[3]) + + def p_tuple(p): + '''kwargs : kwargs COMMA kwarg + names : names COMMA name''' + p[0] = p[1] + p[3] + + def p_tuple_as_element(p): + '''kwargs : kwarg + names : name''' + p[0] = p[1] + + def p_kwarg(p): + 'kwarg : NAME ASSIGN kwarg_val' + if not re.match(r'^[a-zA-Z_][a-zA-Z_0-9]*$', p[1]): + raise LoopyError('Invalid Kwarg attribute {0}. Can only contain ' + 'alphanumerics and underscore.'.format(p[1])) + p[0] = ((p[1], p[3]), ) + + def p_name(p): + 'name : NAME' + p[0] = (p[1],) + + def p_empty_tuple(p): + '''kwarg : empty + name : empty''' + p[0] = () + + def p_kwarg_val(p): + '''kwarg_val : expression + | tuple_of_names + | NAME''' + p[0] = p[1] + + def p_tuple_of_names(p): + 'tuple_of_names : LPAREN names RPAREN' + p[0] = p[2] + def p_prop(p): '''prop : ID | INAME @@ -317,9 +384,13 @@ def parse_match(expr): p[0] = p[1] def p_empty_terminal(p): - 'term :' + 'term : empty' p[0] = All() + def p_empty(p): + 'empty :' + pass + def p_error(p): raise LoopyError("Syntax error at '%s'." % str(p)) -- GitLab From 4537715cf718b7e601e180e92b81abcc7b3013c8 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Sun, 23 Jun 2019 18:50:21 -0500 Subject: [PATCH 05/23] call parsing works --- loopy/match.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/loopy/match.py b/loopy/match.py index 47f234e98..3bf7a5992 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -1,4 +1,4 @@ -"""Matching functionality for instruction ids and subsitution +"""Matching functionality for instruction ids and substitution rule invocations stacks.""" from __future__ import division, absolute_import @@ -341,29 +341,25 @@ def parse_match(expr): def p_tuple(p): '''kwargs : kwargs COMMA kwarg - names : names COMMA name''' - p[0] = p[1] + p[3] + names : names COMMA NAME''' + p[0] = p[1] + (p[3],) + + def p_tuple_ignore_comma(p): + '''kwargs : kwargs COMMA + names : names COMMA''' + p[0] = p[1] def p_tuple_as_element(p): '''kwargs : kwarg - names : name''' - p[0] = p[1] + names : NAME''' + p[0] = (p[1],) def p_kwarg(p): 'kwarg : NAME ASSIGN kwarg_val' if not re.match(r'^[a-zA-Z_][a-zA-Z_0-9]*$', p[1]): raise LoopyError('Invalid Kwarg attribute {0}. Can only contain ' 'alphanumerics and underscore.'.format(p[1])) - p[0] = ((p[1], p[3]), ) - - def p_name(p): - 'name : NAME' - p[0] = (p[1],) - - def p_empty_tuple(p): - '''kwarg : empty - name : empty''' - p[0] = () + p[0] = (p[1], p[3]) def p_kwarg_val(p): '''kwarg_val : expression -- GitLab From 4ce0418f14dd9504c64cecf944c03c19c7aca6b4 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Sun, 23 Jun 2019 21:14:34 -0500 Subject: [PATCH 06/23] done with parsing, just need some renaming --- loopy/match.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/loopy/match.py b/loopy/match.py index 3bf7a5992..67059c61b 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -324,6 +324,7 @@ def parse_match(expr): def p_parens(p): '''expression : LPAREN expression RPAREN + parened_name : LPAREN parened_name RPAREN tuple_of_names : LPAREN tuple_of_names RPAREN''' p[0] = p[2] @@ -341,7 +342,7 @@ def parse_match(expr): def p_tuple(p): '''kwargs : kwargs COMMA kwarg - names : names COMMA NAME''' + names : names COMMA parened_name''' p[0] = p[1] + (p[3],) def p_tuple_ignore_comma(p): @@ -349,11 +350,19 @@ def parse_match(expr): names : names COMMA''' p[0] = p[1] + def p_parened_name(p): + 'parened_name : NAME' + p[0] = p[1] + def p_tuple_as_element(p): '''kwargs : kwarg - names : NAME''' + names : parened_name COMMA''' p[0] = (p[1],) + def p_names_min_1(p): + 'names : parened_name COMMA parened_name' + p[0] = (p[1], p[3]) + def p_kwarg(p): 'kwarg : NAME ASSIGN kwarg_val' if not re.match(r'^[a-zA-Z_][a-zA-Z_0-9]*$', p[1]): @@ -364,7 +373,7 @@ def parse_match(expr): def p_kwarg_val(p): '''kwarg_val : expression | tuple_of_names - | NAME''' + | parened_name''' p[0] = p[1] def p_tuple_of_names(p): -- GitLab From 84d30de3152dbefc4a57990b4a623532da4ee621 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 24 Jun 2019 13:37:16 -0500 Subject: [PATCH 07/23] restructures the functions --- loopy/match.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/loopy/match.py b/loopy/match.py index 67059c61b..74993466f 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -336,30 +336,34 @@ def parse_match(expr): 'term : prop COLON NAME' p[0] = _RESERVED_TO_TYPES[p[1]](p[3]) + def p_prop(p): + '''prop : ID + | INAME + | TAG + | READS + | WRITES''' + p[0] = p[1] + def p_terminal_as_call(p): 'term : NAME LPAREN kwargs RPAREN' p[0] = Call(p[1], p[3]) - def p_tuple(p): + def p_tuple_contents(p): '''kwargs : kwargs COMMA kwarg names : names COMMA parened_name''' p[0] = p[1] + (p[3],) - def p_tuple_ignore_comma(p): + def p_tuple_contents_ignore_comma_postfix(p): '''kwargs : kwargs COMMA names : names COMMA''' p[0] = p[1] - def p_parened_name(p): - 'parened_name : NAME' - p[0] = p[1] - def p_tuple_as_element(p): '''kwargs : kwarg names : parened_name COMMA''' p[0] = (p[1],) - def p_names_min_1(p): + def p_names_tuple_of_len_2(p): 'names : parened_name COMMA parened_name' p[0] = (p[1], p[3]) @@ -376,18 +380,14 @@ def parse_match(expr): | parened_name''' p[0] = p[1] + def p_parened_name(p): + 'parened_name : NAME' + p[0] = p[1] + def p_tuple_of_names(p): 'tuple_of_names : LPAREN names RPAREN' p[0] = p[2] - def p_prop(p): - '''prop : ID - | INAME - | TAG - | READS - | WRITES''' - p[0] = p[1] - def p_empty_terminal(p): 'term : empty' p[0] = All() -- GitLab From ec75a04e19e63651a576f8f0d287e939683b9f85 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 24 Jun 2019 14:07:39 -0500 Subject: [PATCH 08/23] stops PLY from logging info to stderr --- loopy/match.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loopy/match.py b/loopy/match.py index 9a07f5cb6..7a627bd3f 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -325,7 +325,7 @@ def parse_match(expr): lexer = lex.lex() # noqa - parser = yacc.yacc() + parser = yacc.yacc(debug=False) result = parser.parse(expr) return result -- GitLab From a783b2b312a6b135aa5ae7614e29febc3c20d9b8 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 24 Jun 2019 23:32:19 -0500 Subject: [PATCH 09/23] preliminary work to parse the options in tags of instructions --- loopy/match.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/loopy/match.py b/loopy/match.py index 565ddd34f..d446319b8 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -62,6 +62,58 @@ def re_from_glob(s): return re.compile("^"+translate(s.strip())+"$") +class InstructionMatchableTagParsr(object): + tokens = ('NAME, ASSIGN, COMMA, LKWPICKLE, RKWPICKLE, LKWTUPLE,' + 'RKWTUPLE, LPAREN, RPAREN') + + t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' + t_COMMA = r',' # noqa + t_ASSIGN = r'=' # noqa + t_ignore = ' \t' # noqa + t_RKWPICKLE = r'' + t_RKWPICKLE = r'' + t_RKWTUPLE = r'' + t_RKWTUPLE = r'' + + def p_kwargs(p): + 'kwargs : kwargs COMMA kwarg' + p[0] = p[1] + (p[3],) + + def p_kwargs_as_kwarg(p): + 'kwargs : kwarg' + p[0] = (p[1],) + + def p_kwarg(p): + 'kwarg : NAME ASSIGN kwarg_val' + p[0] = (p[1], p[2]) + + def p_kwarg_val_as_pickled_obj(p): + 'kwarg_val : LKWPICKLE NAME RKWPICKLE' + p[0] = p[2] + + def p_kwarg_val_as_tuple(p): + 'kwarg_val : LKWTUPLE LPAREN names RPAREN RKWTUPLE' + p[0] = p[3] + + def p_kwarg_val_as_name(p): + 'kwarg_val : NAME' + p[0] = p[1] + + def p_tuple_defn(p): + 'names : names COMMA NAME' + p[0] = p[1] + (p[2],) + + def p_tuple_of_2_elems(p): + 'names : NAME' + p[0] = (p[1],) + + lexer = lex.lex() + parser = yacc.yacc() + + def parse(expr): + result = self.parser.parse(expr) + return result + # {{{ match expression class MatchExpressionBase(object): @@ -165,7 +217,6 @@ class GlobMatchExpressionBase(MatchExpressionBase): def __init__(self, glob): self.glob = glob - import re from fnmatch import translate self.re = re.compile("^"+translate(glob.strip())+"$") @@ -228,11 +279,26 @@ class Call(GlobMatchExpressionBase): if len(kwargs) != len(kwargs_as_tuple): raise LoopyError("Call '{0}' has repitition in the given " "attributes.".format(glob)) - super(Call, self).__init__(glob) self.kwargs = kwargs + self.glob = glob + from fnmatch import translate + self.re = re.compile("^"+translate(glob.strip()) + "(.*)$") def __call__(self, kernel, matchable): - raise NotImplementedError() + probable_tags = [tag for tag in matchable.tags if self.re.match(tag)] + for tag in probable_tags: + # extract the kwarg keys. + # if the the kwarg keys do not exactly match the kwarg keys in + # this, then leave it, this is not our guy. + # Then read the kwarg vals. + # look for the string after + # if there is in the given kwarg val, then unpickle it. + # look for the string after + # Then that is our tuple type. + # PS: Do we need a parser for this, yep? + raise NotImplementedError() + + return False def __str__(self): return "{0}({1})".format( -- GitLab From 3a8fef40df45fb311b20c110f67b3e702b916397 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Tue, 2 Jul 2019 10:10:48 -0500 Subject: [PATCH 10/23] moves instruction tag parser to kernel.instruction --- loopy/kernel/instruction.py | 61 ++++++++++++++++++++++++++++++++ loopy/match.py | 70 +++++-------------------------------- 2 files changed, 70 insertions(+), 61 deletions(-) diff --git a/loopy/kernel/instruction.py b/loopy/kernel/instruction.py index 5dee96e75..95e5aa732 100644 --- a/loopy/kernel/instruction.py +++ b/loopy/kernel/instruction.py @@ -27,6 +27,8 @@ from pytools import ImmutableRecord, memoize_method from loopy.diagnostic import LoopyError from loopy.tools import Optional from warnings import warn +import ply.lex as lex +import ply.yacc as yacc # {{{ instructions: base class @@ -1531,4 +1533,63 @@ def _check_and_fix_temp_var_type(temp_var_type, stacklevel=2): # }}} +# {{{ parsing instruction tags + +class InstructionMatchableTagParsr(object): + + tokens = ('NAME, ASSIGN, COMMA, LKWPICKLE, RKWPICKLE, LKWTUPLE,' + 'RKWTUPLE, LPAREN, RPAREN') + + t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' + t_COMMA = r',' # noqa + t_ASSIGN = r'=' # noqa + t_ignore = ' \t' # noqa + t_LKWPICKLE = r'' + t_RKWPICKLE = r'' + t_LKWTUPLE = r'' + t_RKWTUPLE = r'' + + def __init__(self): + self.lexer = lex.lex() + self.parser = yacc.yacc() + + def p_kwargs(self, p): + 'kwargs : kwargs COMMA kwarg' + p[0] = p[1] + (p[3],) + + def p_kwargs_as_kwarg(self, p): + 'kwargs : kwarg' + p[0] = (p[1],) + + def p_kwarg(self, p): + 'kwarg : NAME ASSIGN kwarg_val' + p[0] = (p[1], p[2]) + + def p_kwarg_val_as_pickled_obj(self, p): + 'kwarg_val : LKWPICKLE NAME RKWPICKLE' + p[0] = p[2] + + def p_kwarg_val_as_tuple(self, p): + 'kwarg_val : LKWTUPLE LPAREN names RPAREN RKWTUPLE' + p[0] = p[3] + + def p_kwarg_val_as_name(self, p): + 'kwarg_val : NAME' + p[0] = p[1] + + def p_tuple_defn(self, p): + 'names : names COMMA NAME' + p[0] = p[1] + (p[2],) + + def p_tuple_of_2_elems(self, p): + 'names : NAME' + p[0] = (p[1],) + + def parse(self, expr): + result = self.parser.parse(expr) + return result + +# }}} + + # vim: foldmethod=marker diff --git a/loopy/match.py b/loopy/match.py index d446319b8..4113254aa 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -62,58 +62,6 @@ def re_from_glob(s): return re.compile("^"+translate(s.strip())+"$") -class InstructionMatchableTagParsr(object): - tokens = ('NAME, ASSIGN, COMMA, LKWPICKLE, RKWPICKLE, LKWTUPLE,' - 'RKWTUPLE, LPAREN, RPAREN') - - t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' - t_COMMA = r',' # noqa - t_ASSIGN = r'=' # noqa - t_ignore = ' \t' # noqa - t_RKWPICKLE = r'' - t_RKWPICKLE = r'' - t_RKWTUPLE = r'' - t_RKWTUPLE = r'' - - def p_kwargs(p): - 'kwargs : kwargs COMMA kwarg' - p[0] = p[1] + (p[3],) - - def p_kwargs_as_kwarg(p): - 'kwargs : kwarg' - p[0] = (p[1],) - - def p_kwarg(p): - 'kwarg : NAME ASSIGN kwarg_val' - p[0] = (p[1], p[2]) - - def p_kwarg_val_as_pickled_obj(p): - 'kwarg_val : LKWPICKLE NAME RKWPICKLE' - p[0] = p[2] - - def p_kwarg_val_as_tuple(p): - 'kwarg_val : LKWTUPLE LPAREN names RPAREN RKWTUPLE' - p[0] = p[3] - - def p_kwarg_val_as_name(p): - 'kwarg_val : NAME' - p[0] = p[1] - - def p_tuple_defn(p): - 'names : names COMMA NAME' - p[0] = p[1] + (p[2],) - - def p_tuple_of_2_elems(p): - 'names : NAME' - p[0] = (p[1],) - - lexer = lex.lex() - parser = yacc.yacc() - - def parse(expr): - result = self.parser.parse(expr) - return result - # {{{ match expression class MatchExpressionBase(object): @@ -411,7 +359,8 @@ def parse_match(expr): p[0] = p[1] def p_terminal_as_call(p): - 'term : NAME LPAREN kwargs RPAREN' + '''term : NAME LPAREN kwargs RPAREN + | NAME LPAREN kwargs COMMA RPAREN''' p[0] = Call(p[1], p[3]) def p_tuple_contents(p): @@ -419,14 +368,8 @@ def parse_match(expr): names : names COMMA parened_name''' p[0] = p[1] + (p[3],) - def p_tuple_contents_ignore_comma_postfix(p): - '''kwargs : kwargs COMMA - names : names COMMA''' - p[0] = p[1] - def p_tuple_as_element(p): - '''kwargs : kwarg - names : parened_name COMMA''' + 'kwargs : kwarg' p[0] = (p[1],) def p_names_tuple_of_len_2(p): @@ -451,9 +394,14 @@ def parse_match(expr): p[0] = p[1] def p_tuple_of_names(p): - 'tuple_of_names : LPAREN names RPAREN' + '''tuple_of_names : LPAREN names RPAREN + | LPAREN names COMMA RPAREN''' p[0] = p[2] + def p_tuple_of_name_with_single_elem(p): + 'tuple_of_names : LPAREN parened_name COMMA RPAREN' + p[0] = (p[2], ) + def p_empty_terminal(p): 'term : empty' p[0] = All() -- GitLab From 686bb6d00fadd69f464027ac4efbbe99a175bbd6 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Tue, 2 Jul 2019 11:17:53 -0500 Subject: [PATCH 11/23] more progress on the instruction tag parser --- loopy/kernel/instruction.py | 65 ++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/loopy/kernel/instruction.py b/loopy/kernel/instruction.py index 95e5aa732..6939ef3da 100644 --- a/loopy/kernel/instruction.py +++ b/loopy/kernel/instruction.py @@ -1535,59 +1535,64 @@ def _check_and_fix_temp_var_type(temp_var_type, stacklevel=2): # {{{ parsing instruction tags -class InstructionMatchableTagParsr(object): - tokens = ('NAME, ASSIGN, COMMA, LKWPICKLE, RKWPICKLE, LKWTUPLE,' - 'RKWTUPLE, LPAREN, RPAREN') - - t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' - t_COMMA = r',' # noqa - t_ASSIGN = r'=' # noqa - t_ignore = ' \t' # noqa - t_LKWPICKLE = r'' - t_RKWPICKLE = r'' - t_LKWTUPLE = r'' - t_RKWTUPLE = r'' - - def __init__(self): - self.lexer = lex.lex() - self.parser = yacc.yacc() - - def p_kwargs(self, p): +def parse_instruction_tag(tag): + tokens = ('NAME', 'ASSIGN', 'COMMA', 'LKWPICKLE', 'RKWPICKLE', # noqa:F841 + 'LKWTUPLE', 'RKWTUPLE', 'LPAREN', 'RPAREN') + + t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' # noqa:F841 + t_COMMA = r',' # noqa: F841 + t_ASSIGN = r'=' # noqa:F841 + t_ignore = ' \t' # noqa:F841 + t_LPAREN = r'\(' # noqa:F841 + t_RPAREN = r'\)' # noqa:F841 + t_LKWPICKLE = r'' # noqa:F841 + t_RKWPICKLE = r'' # noqa:F841 + t_LKWTUPLE = r'' # noqa:F841 + t_RKWTUPLE = r'' # noqa:F841 + + def t_error(t): + raise RuntimeError("Illegal character '%s'." % t.value[0]) + + def p_kwargs(p): 'kwargs : kwargs COMMA kwarg' p[0] = p[1] + (p[3],) - def p_kwargs_as_kwarg(self, p): + def p_kwargs_as_kwarg(p): 'kwargs : kwarg' p[0] = (p[1],) - def p_kwarg(self, p): + def p_kwarg(p): 'kwarg : NAME ASSIGN kwarg_val' - p[0] = (p[1], p[2]) + p[0] = (p[1], p[3]) - def p_kwarg_val_as_pickled_obj(self, p): + def p_kwarg_val_as_pickled_obj(p): 'kwarg_val : LKWPICKLE NAME RKWPICKLE' p[0] = p[2] - def p_kwarg_val_as_tuple(self, p): + def p_kwarg_val_as_tuple(p): 'kwarg_val : LKWTUPLE LPAREN names RPAREN RKWTUPLE' p[0] = p[3] - def p_kwarg_val_as_name(self, p): + def p_kwarg_val_as_name(p): 'kwarg_val : NAME' p[0] = p[1] - def p_tuple_defn(self, p): + def p_tuple_defn(p): 'names : names COMMA NAME' - p[0] = p[1] + (p[2],) + p[0] = p[1] + (p[3],) - def p_tuple_of_2_elems(self, p): + def p_tuple_of_2_elems(p): 'names : NAME' p[0] = (p[1],) - def parse(self, expr): - result = self.parser.parse(expr) - return result + def p_error(p): + raise LoopyError("Syntax error at '%s'." % str(p)) + + lexer = lex.lex() # noqa:F841 + parser = yacc.yacc() + assert isinstance(tag, str) + return parser.parse(tag) # }}} -- GitLab From 41207be4f570bfdcdf24728cdfdbdb8248baa46a Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Tue, 2 Jul 2019 16:46:32 -0500 Subject: [PATCH 12/23] adds machinery for (de-)serializing an instruction(almost there to swing the wand) --- loopy/kernel/instruction.py | 82 +++++++++++++++++++++++++++++-------- 1 file changed, 66 insertions(+), 16 deletions(-) diff --git a/loopy/kernel/instruction.py b/loopy/kernel/instruction.py index 6939ef3da..07eeb0202 100644 --- a/loopy/kernel/instruction.py +++ b/loopy/kernel/instruction.py @@ -26,6 +26,7 @@ from six.moves import intern from pytools import ImmutableRecord, memoize_method from loopy.diagnostic import LoopyError from loopy.tools import Optional +import re from warnings import warn import ply.lex as lex import ply.yacc as yacc @@ -1535,21 +1536,20 @@ def _check_and_fix_temp_var_type(temp_var_type, stacklevel=2): # {{{ parsing instruction tags - def parse_instruction_tag(tag): - tokens = ('NAME', 'ASSIGN', 'COMMA', 'LKWPICKLE', 'RKWPICKLE', # noqa:F841 - 'LKWTUPLE', 'RKWTUPLE', 'LPAREN', 'RPAREN') - - t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' # noqa:F841 - t_COMMA = r',' # noqa: F841 - t_ASSIGN = r'=' # noqa:F841 - t_ignore = ' \t' # noqa:F841 - t_LPAREN = r'\(' # noqa:F841 - t_RPAREN = r'\)' # noqa:F841 - t_LKWPICKLE = r'' # noqa:F841 - t_RKWPICKLE = r'' # noqa:F841 - t_LKWTUPLE = r'' # noqa:F841 - t_RKWTUPLE = r'' # noqa:F841 + tokens = ('NAME', 'ASSIGN', 'COMMA', 'LKWINSN', 'RKWINSN', # noqa: F841,N806 + 'LKWTUPLE', 'RKWTUPLE', 'LPAREN', 'RPAREN', 'ANY') + + t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' # noqa:F841,N806 + t_COMMA = r',' # noqa: F841,N806 + t_ASSIGN = r'=' # noqa: F841,N806 + t_LPAREN = r'\(' # noqa: F841,N806 + t_RPAREN = r'\)' # noqa: F841,N806 + t_LKWINSN = r'' # noqa: F841,N806 + t_RKWINSN = r'' # noqa: F841,N806 + t_LKWTUPLE = r'' # noqa: F841,N806 + t_RKWTUPLE = r'' # noqa: F841,N806 + t_ignore = ' \t' # noqa: F841,N806 def t_error(t): raise RuntimeError("Illegal character '%s'." % t.value[0]) @@ -1567,8 +1567,8 @@ def parse_instruction_tag(tag): p[0] = (p[1], p[3]) def p_kwarg_val_as_pickled_obj(p): - 'kwarg_val : LKWPICKLE NAME RKWPICKLE' - p[0] = p[2] + 'kwarg_val : LKWINSN LPAREN names RPAREN RKWINSN' + p[0] = p[1] def p_kwarg_val_as_tuple(p): 'kwarg_val : LKWTUPLE LPAREN names RPAREN RKWTUPLE' @@ -1597,4 +1597,54 @@ def parse_instruction_tag(tag): # }}} +def serialize_instruction(insn): + + # {{{ get the options + + insn_options = 'id=%s, ' % insn.id + if insn.depends_on: + insn_options += ("dep="+":".join(insn.depends_on)+", ") + if insn.tags: + insn_options += ("tags="+":".join(insn.tags)+", ") + if insn.within_inames: + insn_options += ("inames="+":".join(insn.within_inames)+", ") + if isinstance(insn, MultiAssignmentBase): + if insn.atomicity: + insn_options += "atomic, " + elif isinstance(insn, BarrierInstruction): + insn_options += ("mem_kind=%s, " % insn.mem_kind) + + insn_options = insn_options[:-2] # remove ', ' + + # }}} + + if isinstance(insn, MultiAssignmentBase): + if insn.assignees: + insn_core = "{assignees} = ".format( + assignees=', '.join(str(assignee) for assignee in + insn.assignees)) + insn_core += str(insn.expression) + elif isinstance(insn, BarrierInstruction): + insn_core = "... {kind}barrier".format( + kind=insn.synchronization_kind[0]) + elif isinstance(insn, NoOpInstruction): + insn_core = "... nop" + else: + raise NotImplementedError() + + return "{core} {{{opts}}}".format(core=insn_core, opts=insn_options) + + +def deserialize_instruction(insn_expr): + from loopy.kernel.creation import parse_instructions + insns, inames_to_dup, substitutions = parse_instructions(insn_expr.split('\n'), {}) + if substitutions: + raise LoopyError("Received substitutions on parsing" + " {}".format(insn_expr)) + if len(insns) != 1: + raise LoopyError("Received multiple/no instructions on parsing" + " {}".format(insn_expr)) + #FIXME; Any checks on inames_to_dup? + return insns[0] + # vim: foldmethod=marker -- GitLab From f94df6b3f940a26f515588841a9d19f40121673c Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Tue, 2 Jul 2019 17:20:20 -0500 Subject: [PATCH 13/23] tag match kwargs works now --- loopy/kernel/instruction.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/loopy/kernel/instruction.py b/loopy/kernel/instruction.py index 07eeb0202..9320a86a9 100644 --- a/loopy/kernel/instruction.py +++ b/loopy/kernel/instruction.py @@ -26,8 +26,8 @@ from six.moves import intern from pytools import ImmutableRecord, memoize_method from loopy.diagnostic import LoopyError from loopy.tools import Optional -import re from warnings import warn +import re import ply.lex as lex import ply.yacc as yacc @@ -1537,16 +1537,15 @@ def _check_and_fix_temp_var_type(temp_var_type, stacklevel=2): # {{{ parsing instruction tags def parse_instruction_tag(tag): - tokens = ('NAME', 'ASSIGN', 'COMMA', 'LKWINSN', 'RKWINSN', # noqa: F841,N806 - 'LKWTUPLE', 'RKWTUPLE', 'LPAREN', 'RPAREN', 'ANY') + tokens = ('NAME', 'ASSIGN', 'COMMA', 'INSN', # noqa: F841,N806 + 'LKWTUPLE', 'RKWTUPLE', 'LPAREN', 'RPAREN') t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' # noqa:F841,N806 + t_INSN = r' [a-zA-Z0-9_={}\s\[\]\.\(\)]+ ' # noqa:F841,N806 t_COMMA = r',' # noqa: F841,N806 t_ASSIGN = r'=' # noqa: F841,N806 t_LPAREN = r'\(' # noqa: F841,N806 t_RPAREN = r'\)' # noqa: F841,N806 - t_LKWINSN = r'' # noqa: F841,N806 - t_RKWINSN = r'' # noqa: F841,N806 t_LKWTUPLE = r'' # noqa: F841,N806 t_RKWTUPLE = r'' # noqa: F841,N806 t_ignore = ' \t' # noqa: F841,N806 @@ -1567,8 +1566,9 @@ def parse_instruction_tag(tag): p[0] = (p[1], p[3]) def p_kwarg_val_as_pickled_obj(p): - 'kwarg_val : LKWINSN LPAREN names RPAREN RKWINSN' - p[0] = p[1] + 'kwarg_val : INSN' + p[0] = deserialize_instruction( + re.match("(.*)", p[1]).group(1)) def p_kwarg_val_as_tuple(p): 'kwarg_val : LKWTUPLE LPAREN names RPAREN RKWTUPLE' @@ -1637,7 +1637,8 @@ def serialize_instruction(insn): def deserialize_instruction(insn_expr): from loopy.kernel.creation import parse_instructions - insns, inames_to_dup, substitutions = parse_instructions(insn_expr.split('\n'), {}) + insns, inames_to_dup, substitutions = parse_instructions( + insn_expr.split('\n'), {}) if substitutions: raise LoopyError("Received substitutions on parsing" " {}".format(insn_expr)) -- GitLab From 217805ade86549a6e40024015b23096ca5a099a8 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 3 Jul 2019 09:52:37 -0500 Subject: [PATCH 14/23] fixes minor error in parsing insn tags; adds support for single element name tuple in insn tags --- loopy/kernel/instruction.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/loopy/kernel/instruction.py b/loopy/kernel/instruction.py index 9320a86a9..135e3f0b4 100644 --- a/loopy/kernel/instruction.py +++ b/loopy/kernel/instruction.py @@ -1571,8 +1571,13 @@ def parse_instruction_tag(tag): re.match("(.*)", p[1]).group(1)) def p_kwarg_val_as_tuple(p): - 'kwarg_val : LKWTUPLE LPAREN names RPAREN RKWTUPLE' - p[0] = p[3] + '''kwarg_val : LKWTUPLE LPAREN names RPAREN RKWTUPLE + | LKWTUPLE LPAREN names COMMA RPAREN RKWTUPLE + | LKWTUPLE LPAREN NAME COMMA RPAREN RKWTUPLE''' + if isinstance(p[3], tuple): + p[0] = p[3] + else: + p[0] = (p[3], ) def p_kwarg_val_as_name(p): 'kwarg_val : NAME' @@ -1583,8 +1588,8 @@ def parse_instruction_tag(tag): p[0] = p[1] + (p[3],) def p_tuple_of_2_elems(p): - 'names : NAME' - p[0] = (p[1],) + 'names : NAME COMMA NAME' + p[0] = (p[1], p[3]) def p_error(p): raise LoopyError("Syntax error at '%s'." % str(p)) -- GitLab From 760ed68f3f85eee3e417a2241ad4d61196851836 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 3 Jul 2019 10:09:32 -0500 Subject: [PATCH 15/23] adds the relevant tag for reduction.init --- loopy/preprocess.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/loopy/preprocess.py b/loopy/preprocess.py index 2afcd3db4..6a4171991 100644 --- a/loopy/preprocess.py +++ b/loopy/preprocess.py @@ -1032,6 +1032,8 @@ def realize_reduction(kernel, insn_id_filter=None, unknown_types_ok=True, init_id = insn_id_gen( "%s_%s_init" % (insn.id, "_".join(expr.inames))) + from loopy.kernel.instruction import serialize_instruction + init_insn = make_assignment( id=init_id, assignees=acc_vars, @@ -1039,6 +1041,11 @@ def realize_reduction(kernel, insn_id_filter=None, unknown_types_ok=True, within_inames_is_final=insn.within_inames_is_final, depends_on=init_insn_depends_on, expression=expr.operation.neutral_element(*arg_dtypes), + tags=frozenset([ + "loopy.reduction.init(insn={0}" + ", inames=({1})".format( + serialize_instruction(insn), + ','.join(expr.inames)+',')]), predicates=insn.predicates,) generated_insns.append(init_insn) -- GitLab From 140eee1e57c04619cdae88ad49022cfec2f433a6 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 3 Jul 2019 12:33:00 -0500 Subject: [PATCH 16/23] stringified reduce to be able to be parsed by loopy-lang parser --- loopy/symbolic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loopy/symbolic.py b/loopy/symbolic.py index f5cf07b0e..3d2c602d8 100644 --- a/loopy/symbolic.py +++ b/loopy/symbolic.py @@ -204,7 +204,7 @@ class StringifyMapper(StringifyMapperBase): def map_reduction(self, expr, prec): from pymbolic.mapper.stringifier import PREC_NONE - return "%sreduce(%s, [%s], %s)" % ( + return "%sreduce(%s, (%s), %s)" % ( "simul_" if expr.allow_simultaneous else "", expr.operation, ", ".join(expr.inames), self.rec(expr.expr, PREC_NONE)) -- GitLab From 5bfed0713702da980ab510401badeb52bd0c6f29 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 3 Jul 2019 12:35:31 -0500 Subject: [PATCH 17/23] minor errors/renaming vars --- loopy/kernel/instruction.py | 11 ++++++----- loopy/preprocess.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/loopy/kernel/instruction.py b/loopy/kernel/instruction.py index 135e3f0b4..787cb7808 100644 --- a/loopy/kernel/instruction.py +++ b/loopy/kernel/instruction.py @@ -1541,7 +1541,7 @@ def parse_instruction_tag(tag): 'LKWTUPLE', 'RKWTUPLE', 'LPAREN', 'RPAREN') t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' # noqa:F841,N806 - t_INSN = r' [a-zA-Z0-9_={}\s\[\]\.\(\)]+ ' # noqa:F841,N806 + t_INSN = r' [a-zA-Z0-9_={}\s\[\]\.(),]+ ' # noqa:F841,N806 t_COMMA = r',' # noqa: F841,N806 t_ASSIGN = r'=' # noqa: F841,N806 t_LPAREN = r'\(' # noqa: F841,N806 @@ -1640,16 +1640,17 @@ def serialize_instruction(insn): return "{core} {{{opts}}}".format(core=insn_core, opts=insn_options) -def deserialize_instruction(insn_expr): +def deserialize_instruction(insn): + assert isinstance(insn, str) from loopy.kernel.creation import parse_instructions insns, inames_to_dup, substitutions = parse_instructions( - insn_expr.split('\n'), {}) + insn.split('\n'), {}) if substitutions: raise LoopyError("Received substitutions on parsing" - " {}".format(insn_expr)) + " {}".format(insn)) if len(insns) != 1: raise LoopyError("Received multiple/no instructions on parsing" - " {}".format(insn_expr)) + " {}".format(insn)) #FIXME; Any checks on inames_to_dup? return insns[0] diff --git a/loopy/preprocess.py b/loopy/preprocess.py index 6a4171991..7e093d04c 100644 --- a/loopy/preprocess.py +++ b/loopy/preprocess.py @@ -1043,7 +1043,7 @@ def realize_reduction(kernel, insn_id_filter=None, unknown_types_ok=True, expression=expr.operation.neutral_element(*arg_dtypes), tags=frozenset([ "loopy.reduction.init(insn={0}" - ", inames=({1})".format( + ", inames=({1}))".format( serialize_instruction(insn), ','.join(expr.inames)+',')]), predicates=insn.predicates,) -- GitLab From e916e88aa7ca3a6422a3e6d9aa802b00bdc4e909 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 3 Jul 2019 12:40:38 -0500 Subject: [PATCH 18/23] WIP:implement matching of args in the tag to the one provided in query --- loopy/match.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/loopy/match.py b/loopy/match.py index 4113254aa..7efd8a932 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -28,6 +28,7 @@ THE SOFTWARE. import six from six.moves import range from loopy.diagnostic import LoopyError +from loopy.kernel.instruction import parse_instruction_tag import ply.lex as lex import ply.yacc as yacc import re @@ -225,16 +226,19 @@ class Call(GlobMatchExpressionBase): kwargs_as_tuple = kwargs[:] kwargs = dict(kwargs_as_tuple) if len(kwargs) != len(kwargs_as_tuple): - raise LoopyError("Call '{0}' has repitition in the given " + raise LoopyError("Call '{0}' has repetition in the given " "attributes.".format(glob)) self.kwargs = kwargs self.glob = glob from fnmatch import translate - self.re = re.compile("^"+translate(glob.strip()) + "(.*)$") + self.re = re.compile("^"+translate(glob.strip())[:-2]+r"[(](.*)[)]$") def __call__(self, kernel, matchable): probable_tags = [tag for tag in matchable.tags if self.re.match(tag)] for tag in probable_tags: + tag_kwargs = dict(parse_instruction_tag(self.re.match(tag).group(1))) + print(tag_kwargs) + 1/0 # extract the kwarg keys. # if the the kwarg keys do not exactly match the kwarg keys in # this, then leave it, this is not our guy. -- GitLab From 8ef93deacf8e2812873e2e8511f1e8abaac976a8 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 3 Jul 2019 15:58:23 -0500 Subject: [PATCH 19/23] completes the work on matching between the tag and the call node; checking left --- loopy/match.py | 72 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 14 deletions(-) diff --git a/loopy/match.py b/loopy/match.py index 7efd8a932..8b686e9bb 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -28,10 +28,12 @@ THE SOFTWARE. import six from six.moves import range from loopy.diagnostic import LoopyError -from loopy.kernel.instruction import parse_instruction_tag +from loopy.kernel.instruction import (parse_instruction_tag, + InstructionBase) import ply.lex as lex import ply.yacc as yacc import re +from itertools import permutations NoneType = type(None) @@ -234,21 +236,63 @@ class Call(GlobMatchExpressionBase): self.re = re.compile("^"+translate(glob.strip())[:-2]+r"[(](.*)[)]$") def __call__(self, kernel, matchable): - probable_tags = [tag for tag in matchable.tags if self.re.match(tag)] - for tag in probable_tags: + for tag in matchable.tags: + # any match + if not self.re.match(tag): + continue tag_kwargs = dict(parse_instruction_tag(self.re.match(tag).group(1))) print(tag_kwargs) - 1/0 - # extract the kwarg keys. - # if the the kwarg keys do not exactly match the kwarg keys in - # this, then leave it, this is not our guy. - # Then read the kwarg vals. - # look for the string after - # if there is in the given kwarg val, then unpickle it. - # look for the string after - # Then that is our tuple type. - # PS: Do we need a parser for this, yep? - raise NotImplementedError() + from fnmatch import translate + + # {{{ sanity checks on the provided kwargs + + if not(self.kwargs.keys() <= tag_kwargs.keys()): + raise LoopyError( + "Unknown attributes '{0}' provided to '{1}'.".format( + self.kwarg.keys()-tag_kwargs.keys(), + self.glob)) + # }}} + + did_match = True + for kw, match_arg in six.iteritems(self.kwargs): + tagged_arg = tag_kwargs[kw] + if isinstance(tagged_arg, InstructionBase): + if not match_arg(kernel, matchable): + did_match = False + break + elif isinstance(tagged_arg, tuple): + if not isinstance(match_arg, tuple): + match_arg = (match_arg,) + + if len(match_arg) != len(tagged_arg): + # FIXME: Should we check for matches if the number of + # the names in the mathchee tuple is less than the + # matcher? + did_match = False + break + + tuple_match = False + # Check for matches between all tuple perms + for tagged_perm, match_perm in zip( + permutations(tagged_arg), permutations(match_arg)): + if all(self.re.match("^"+translate(match_name), tagged_name) + for tagged_name, match_name in zip( + tagged_perm, match_perm)): + tuple_match = True + break + if not tuple_match: + did_match = False + break + elif isinstance(tagged_arg, str): + if not re.match("^"+translate(match_arg), tagged_arg): + did_match = False + break + else: + raise NotImplementedError("Unknown tagged arg {0}.".format( + type(tagged_arg).__name__)) + + if did_match: + return True return False -- GitLab From a81d42ee4e3a5f82732b2978c45937c88d71d682 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 3 Jul 2019 17:30:21 -0500 Subject: [PATCH 20/23] fixes minor errors; removes debug statements --- loopy/match.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/loopy/match.py b/loopy/match.py index 8b686e9bb..7f0b3113d 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -241,7 +241,6 @@ class Call(GlobMatchExpressionBase): if not self.re.match(tag): continue tag_kwargs = dict(parse_instruction_tag(self.re.match(tag).group(1))) - print(tag_kwargs) from fnmatch import translate # {{{ sanity checks on the provided kwargs @@ -257,7 +256,7 @@ class Call(GlobMatchExpressionBase): for kw, match_arg in six.iteritems(self.kwargs): tagged_arg = tag_kwargs[kw] if isinstance(tagged_arg, InstructionBase): - if not match_arg(kernel, matchable): + if not match_arg(kernel, tagged_arg): did_match = False break elif isinstance(tagged_arg, tuple): @@ -275,7 +274,7 @@ class Call(GlobMatchExpressionBase): # Check for matches between all tuple perms for tagged_perm, match_perm in zip( permutations(tagged_arg), permutations(match_arg)): - if all(self.re.match("^"+translate(match_name), tagged_name) + if all(re.match("^"+translate(match_name), tagged_name) for tagged_name, match_name in zip( tagged_perm, match_perm)): tuple_match = True -- GitLab From 89d245013d6d9e3f570e948cca367d25e2f378d0 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 3 Jul 2019 17:31:43 -0500 Subject: [PATCH 21/23] adds a test for basic transform addressing --- test/test_loopy.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/test_loopy.py b/test/test_loopy.py index 89b4f5e63..fd896435c 100644 --- a/test/test_loopy.py +++ b/test/test_loopy.py @@ -2986,6 +2986,23 @@ def test_shape_mismatch_check(ctx_factory): prg(queue, a=a, b=b) +def test_transform_addressing(): + from loopy.match import parse_match + + knl = lp.make_kernel( + "{[i]: 0<=i<10}", + """ + a = sum(i, b[i]) {id=insn} + """) + + knl = lp.realize_reduction(knl) + + within = parse_match("loopy.reduction.init(insn=id:insn, inames=(i,))") + + assert len([print(insn) + for insn in knl.instructions if within(knl, insn)]) == 1 + + if __name__ == "__main__": if len(sys.argv) > 1: exec(sys.argv[1]) -- GitLab From fe378f8b4d009dcad8d79c3e07c21bed28c59853 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 3 Jul 2019 19:41:25 -0500 Subject: [PATCH 22/23] flake8; pylint; passes tests --- loopy/kernel/instruction.py | 13 ++++++++----- loopy/match.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/loopy/kernel/instruction.py b/loopy/kernel/instruction.py index 787cb7808..bfe567996 100644 --- a/loopy/kernel/instruction.py +++ b/loopy/kernel/instruction.py @@ -1540,8 +1540,9 @@ def parse_instruction_tag(tag): tokens = ('NAME', 'ASSIGN', 'COMMA', 'INSN', # noqa: F841,N806 'LKWTUPLE', 'RKWTUPLE', 'LPAREN', 'RPAREN') - t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' # noqa:F841,N806 - t_INSN = r' [a-zA-Z0-9_={}\s\[\]\.(),]+ ' # noqa:F841,N806 + t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' # noqa: F841,N806 + t_INSN = (r' [a-zA-Z0-9_={}\s\[\]\.(),]+' # noqa: F841,N806 + ' ') t_COMMA = r',' # noqa: F841,N806 t_ASSIGN = r'=' # noqa: F841,N806 t_LPAREN = r'\(' # noqa: F841,N806 @@ -1623,17 +1624,19 @@ def serialize_instruction(insn): # }}} + insn_core = "" + if isinstance(insn, MultiAssignmentBase): if insn.assignees: - insn_core = "{assignees} = ".format( + insn_core += "{assignees} = ".format( assignees=', '.join(str(assignee) for assignee in insn.assignees)) insn_core += str(insn.expression) elif isinstance(insn, BarrierInstruction): - insn_core = "... {kind}barrier".format( + insn_core += "... {kind}barrier".format( kind=insn.synchronization_kind[0]) elif isinstance(insn, NoOpInstruction): - insn_core = "... nop" + insn_core += "... nop" else: raise NotImplementedError() diff --git a/loopy/match.py b/loopy/match.py index 7f0b3113d..9ae0cee3a 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -248,7 +248,7 @@ class Call(GlobMatchExpressionBase): if not(self.kwargs.keys() <= tag_kwargs.keys()): raise LoopyError( "Unknown attributes '{0}' provided to '{1}'.".format( - self.kwarg.keys()-tag_kwargs.keys(), + self.kwargs.keys()-tag_kwargs.keys(), self.glob)) # }}} -- GitLab From f38edb1bb1ffce3eb9a6c39cfa33ed1fd948964a Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Thu, 4 Jul 2019 00:57:01 -0500 Subject: [PATCH 23/23] different strategy for fnmatch-ing the call name --- loopy/match.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/loopy/match.py b/loopy/match.py index 9ae0cee3a..1f37b974c 100644 --- a/loopy/match.py +++ b/loopy/match.py @@ -233,14 +233,21 @@ class Call(GlobMatchExpressionBase): self.kwargs = kwargs self.glob = glob from fnmatch import translate - self.re = re.compile("^"+translate(glob.strip())[:-2]+r"[(](.*)[)]$") + self.re = re.compile("^"+translate(glob.strip())) def __call__(self, kernel, matchable): for tag in matchable.tags: - # any match - if not self.re.match(tag): + match = re.match(r'^(?P[\w.\*]+)\((?P.+)\)$', tag) + + if not match: + # no match => skip this tag + continue + + if not self.re.match(match.groupdict()['name']): + # 'name' of the call should fnmatch with the tag call name continue - tag_kwargs = dict(parse_instruction_tag(self.re.match(tag).group(1))) + + tag_kwargs = dict(parse_instruction_tag(match.groupdict()['kwargs'])) from fnmatch import translate # {{{ sanity checks on the provided kwargs -- GitLab