From ff9c629e525573b08c7f8ffab3b32914ccb35996 Mon Sep 17 00:00:00 2001 From: arghdos Date: Wed, 15 Mar 2017 18:03:57 -0400 Subject: [PATCH 01/11] add fuse instructions stream with overlaps --- pymbolic/imperative/transform.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pymbolic/imperative/transform.py b/pymbolic/imperative/transform.py index a52d446..f1bcbea 100644 --- a/pymbolic/imperative/transform.py +++ b/pymbolic/imperative/transform.py @@ -22,6 +22,20 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +# {{{ fuse overlapping instruction streams + +def fuse_instruction_streams_with_overlapping_ids(instructions_a, instructions_b, + allowed_duplicates=[]): + id_a = set([insna.id for insna in instructions_a]) + #filter b instructions + uniques_b = [x for x in instructions_b + if x.id in id_a + and x.id not in allowed_duplicates] + + return fuse_instruction_streams_with_unique_ids(instructions_a, uniques_b) + +# }}} + # {{{ fuse instruction streams -- GitLab From d2aeb6e29473855e598a5db224b03bbbf677813d Mon Sep 17 00:00:00 2001 From: arghdos Date: Wed, 15 Mar 2017 18:22:15 -0400 Subject: [PATCH 02/11] fix depends --- pymbolic/imperative/transform.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pymbolic/imperative/transform.py b/pymbolic/imperative/transform.py index f1bcbea..f9a7ae3 100644 --- a/pymbolic/imperative/transform.py +++ b/pymbolic/imperative/transform.py @@ -45,6 +45,7 @@ def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b): insn_id_gen = UniqueNameGenerator( set([insna.id for insna in new_instructions])) + a_ids = set([insna.id for insna in instructions_a]) b_unique_instructions = [] old_b_id_to_new_b_id = {} for insnb in instructions_b: @@ -55,11 +56,17 @@ def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b): b_unique_instructions.append( insnb.copy(id=new_id)) + #if this is called by fuse_instruction_streams_with_overlapping_ids + #some instructions in b may depend on those in a + #therefore we force that any dependencies are in the b map *or* + #the a id's for insnb in b_unique_instructions: new_instructions.append( insnb.copy( depends_on=frozenset( - old_b_id_to_new_b_id[dep_id] + old_b_id_to_new_b_id[dep_id] \ + if dep_id in old_b_id_to_new_b_id + else next(a_id for a_id in a_ids if a_id == dep_id) for dep_id in insnb.depends_on))) return new_instructions, old_b_id_to_new_b_id -- GitLab From 006f5a00f95f6a8cb7d021ae94dd3783f2324af0 Mon Sep 17 00:00:00 2001 From: arghdos Date: Wed, 22 Mar 2017 10:22:32 -0400 Subject: [PATCH 03/11] flake updates --- pymbolic/imperative/transform.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pymbolic/imperative/transform.py b/pymbolic/imperative/transform.py index f9a7ae3..696a81e 100644 --- a/pymbolic/imperative/transform.py +++ b/pymbolic/imperative/transform.py @@ -22,12 +22,13 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ + # {{{ fuse overlapping instruction streams def fuse_instruction_streams_with_overlapping_ids(instructions_a, instructions_b, - allowed_duplicates=[]): + allowed_duplicates=[]): id_a = set([insna.id for insna in instructions_a]) - #filter b instructions + # filter b instructions uniques_b = [x for x in instructions_b if x.id in id_a and x.id not in allowed_duplicates] @@ -56,16 +57,16 @@ def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b): b_unique_instructions.append( insnb.copy(id=new_id)) - #if this is called by fuse_instruction_streams_with_overlapping_ids - #some instructions in b may depend on those in a - #therefore we force that any dependencies are in the b map *or* - #the a id's + # if this is called by fuse_instruction_streams_with_overlapping_ids + # some instructions in b may depend on those in a + # therefore we force that any dependencies are in the b map *or* + # the a id's for insnb in b_unique_instructions: new_instructions.append( insnb.copy( depends_on=frozenset( - old_b_id_to_new_b_id[dep_id] \ - if dep_id in old_b_id_to_new_b_id + old_b_id_to_new_b_id[dep_id] + if dep_id in old_b_id_to_new_b_id else next(a_id for a_id in a_ids if a_id == dep_id) for dep_id in insnb.depends_on))) -- GitLab From 13c079600cc534fdb3d6e52a9b6a0599d8e14b1c Mon Sep 17 00:00:00 2001 From: arghdos Date: Sat, 25 Mar 2017 18:20:12 -0400 Subject: [PATCH 04/11] convert to sets, add explicit assert to raise error on missing depends --- pymbolic/imperative/transform.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/pymbolic/imperative/transform.py b/pymbolic/imperative/transform.py index 696a81e..6249c73 100644 --- a/pymbolic/imperative/transform.py +++ b/pymbolic/imperative/transform.py @@ -33,14 +33,16 @@ def fuse_instruction_streams_with_overlapping_ids(instructions_a, instructions_b if x.id in id_a and x.id not in allowed_duplicates] - return fuse_instruction_streams_with_unique_ids(instructions_a, uniques_b) + return fuse_instruction_streams_with_unique_ids(instructions_a, uniques_b, + allow_b_depend_on_a=True) # }}} # {{{ fuse instruction streams -def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b): +def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b, + allow_b_depend_on_a=False): new_instructions = list(instructions_a) from pytools import UniqueNameGenerator insn_id_gen = UniqueNameGenerator( @@ -57,18 +59,23 @@ def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b): b_unique_instructions.append( insnb.copy(id=new_id)) - # if this is called by fuse_instruction_streams_with_overlapping_ids - # some instructions in b may depend on those in a - # therefore we force that any dependencies are in the b map *or* - # the a id's for insnb in b_unique_instructions: + b_deps = set() + for dep in insnb.depends_on: + new_dep_id = old_b_id_to_new_b_id[dep_id] \ + if dep_id in old_b_id_to_new_b_id else None + if allow_b_depend_on_a: + # if this is called by + # fuse_instruction_streams_with_overlapping_ids + # some instructions in b may depend on those in a + # therefore also check if dep in a instructions + new_dep_id = dep_id if dep_id in a_ids else None + assert new_dep_id is not None, ('Instruction {} in stream b ' + 'missing dependency {}'.format(insnb.id, dep_id)) + b_deps |= dep_id + new_instructions.append( - insnb.copy( - depends_on=frozenset( - old_b_id_to_new_b_id[dep_id] - if dep_id in old_b_id_to_new_b_id - else next(a_id for a_id in a_ids if a_id == dep_id) - for dep_id in insnb.depends_on))) + insnb.copy(frozenset(b_deps))) return new_instructions, old_b_id_to_new_b_id -- GitLab From 24f241420d5a1f8aca6bde30efaa3d48c6cb2d49 Mon Sep 17 00:00:00 2001 From: arghdos Date: Sat, 25 Mar 2017 18:22:46 -0400 Subject: [PATCH 05/11] oops, wrong id --- pymbolic/imperative/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymbolic/imperative/transform.py b/pymbolic/imperative/transform.py index 6249c73..3627674 100644 --- a/pymbolic/imperative/transform.py +++ b/pymbolic/imperative/transform.py @@ -61,7 +61,7 @@ def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b, for insnb in b_unique_instructions: b_deps = set() - for dep in insnb.depends_on: + for dep_id in insnb.depends_on: new_dep_id = old_b_id_to_new_b_id[dep_id] \ if dep_id in old_b_id_to_new_b_id else None if allow_b_depend_on_a: @@ -72,7 +72,7 @@ def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b, new_dep_id = dep_id if dep_id in a_ids else None assert new_dep_id is not None, ('Instruction {} in stream b ' 'missing dependency {}'.format(insnb.id, dep_id)) - b_deps |= dep_id + b_deps |= new_dep_id new_instructions.append( insnb.copy(frozenset(b_deps))) -- GitLab From 41f9ec5118fded177fed4b56b2be439cbdfab121 Mon Sep 17 00:00:00 2001 From: arghdos Date: Sat, 25 Mar 2017 19:02:39 -0400 Subject: [PATCH 06/11] bugfixes and add tests --- pymbolic/imperative/transform.py | 6 +- test/test_pymbolic.py | 96 ++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 3 deletions(-) diff --git a/pymbolic/imperative/transform.py b/pymbolic/imperative/transform.py index 3627674..13e1026 100644 --- a/pymbolic/imperative/transform.py +++ b/pymbolic/imperative/transform.py @@ -64,7 +64,7 @@ def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b, for dep_id in insnb.depends_on: new_dep_id = old_b_id_to_new_b_id[dep_id] \ if dep_id in old_b_id_to_new_b_id else None - if allow_b_depend_on_a: + if allow_b_depend_on_a and new_dep_id is None: # if this is called by # fuse_instruction_streams_with_overlapping_ids # some instructions in b may depend on those in a @@ -72,10 +72,10 @@ def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b, new_dep_id = dep_id if dep_id in a_ids else None assert new_dep_id is not None, ('Instruction {} in stream b ' 'missing dependency {}'.format(insnb.id, dep_id)) - b_deps |= new_dep_id + b_deps.add(new_dep_id) new_instructions.append( - insnb.copy(frozenset(b_deps))) + insnb.copy(depends_on=frozenset(b_deps))) return new_instructions, old_b_id_to_new_b_id diff --git a/test/test_pymbolic.py b/test/test_pymbolic.py index fe7c8a3..0009eee 100644 --- a/test/test_pymbolic.py +++ b/test/test_pymbolic.py @@ -487,6 +487,102 @@ def test_long_sympy_mapping(): SympyToPymbolicMapper()(sp.sympify(int(10))) +# dummy instruction +class dummy_insn(object): + def __init__(self, idv, depends_on=frozenset()): + self.id = idv + self.depends_on = depends_on + + def copy(self, id=None, depends_on=None): + if id is not None: + self.id = id + if depends_on is not None: + self.depends_on = depends_on + return dummy_insn(self.id, self.depends_on.copy()) + + +def test_unique_fusion(): + from pymbolic.imperative.transform import \ + fuse_instruction_streams_with_unique_ids + + # create instructions 1-5 with depends + insns_a = [dummy_insn(str(i), [str(x) for x in range(i)]) + for i in range(5)] + + # first try feeding it 'a' twice + new_insns, b_map = fuse_instruction_streams_with_unique_ids( + insns_a, [i.copy() for i in insns_a]) + new_insn_ids = set([x.id for x in new_insns]) + + # should have a duplicate of each instruction + assert all(x.id in b_map for x in insns_a) + assert all(x.id + '_0' in new_insn_ids for x in insns_a) + + # next create independent instructions 5-10 + insns_b = [dummy_insn(str(i), [str(x) for x in range(5, i)]) + for i in range(5, 10)] + + new_insns, b_map = fuse_instruction_streams_with_unique_ids( + insns_a, insns_b) + new_insn_ids = set([x.id for x in new_insns]) + + # should have both sets of instructions + assert all(x.id in new_insn_ids for x in + insns_a + insns_b) + assert all(k == v for k, v in b_map.items()) + + # next test b depends on a without flag + insns_b[0].depends_on = frozenset(['0']) + + with pytest.raises(AssertionError): + fuse_instruction_streams_with_unique_ids( + insns_a, insns_b) + + # and with flag + new_insns, b_map = fuse_instruction_streams_with_unique_ids( + insns_a, insns_b, allow_b_depend_on_a=True) + + # should have both sets of instructions + assert all(x.id in new_insn_ids for x in + insns_a + insns_b) + assert all(k == v for k, v in b_map.items()) + + +def test_overlapping_fusion(): + from pymbolic.imperative.transform import \ + fuse_instruction_streams_with_overlapping_ids + + # create instructions 1-5 with depends + insns_a = [dummy_insn(str(i)) + for i in range(5)] + a_copy = [i.copy() for i in insns_a] + a_ids = [i.id for i in insns_a] + + # first try feeding it 'a' twice + new_insns, b_map = fuse_instruction_streams_with_overlapping_ids( + insns_a, a_copy, a_ids) + + # should have just the original instructions + assert len(new_insns) == len(insns_a) + assert all(new_insns[i] == insns_a[i] for i in range(len(new_insns))) + assert len(b_map) == 0 + + # now try with no allowed overlaps + new_insns, b_map = fuse_instruction_streams_with_overlapping_ids( + insns_a, a_copy, []) + # check all a in new instructions + new_ids = set([x.id for x in new_insns]) + # and all a in maps + assert all(insn_a.id in new_ids for insn_a in insns_a) + assert all(insn_a.id in b_map for insn_a in insns_a) + + # try with b depends on a, and the instruction overlapping + insns_b = a_copy[:] + insns_b[1].depends_on = frozenset(insns_a[0].id) + new_insns, b_map = fuse_instruction_streams_with_overlapping_ids( + insns_a, a_copy, [insns_a[0].id]) + + if __name__ == "__main__": import sys if len(sys.argv) > 1: -- GitLab From f48c7f1d594539e7bb79c5058caa4873ebd93fed Mon Sep 17 00:00:00 2001 From: arghdos Date: Sat, 25 Mar 2017 19:13:29 -0400 Subject: [PATCH 07/11] fix depends on copy for earlier python --- test/test_pymbolic.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_pymbolic.py b/test/test_pymbolic.py index 0009eee..252f871 100644 --- a/test/test_pymbolic.py +++ b/test/test_pymbolic.py @@ -488,7 +488,7 @@ def test_long_sympy_mapping(): # dummy instruction -class dummy_insn(object): +class DummyInsn(object): def __init__(self, idv, depends_on=frozenset()): self.id = idv self.depends_on = depends_on @@ -498,7 +498,7 @@ class dummy_insn(object): self.id = id if depends_on is not None: self.depends_on = depends_on - return dummy_insn(self.id, self.depends_on.copy()) + return DummyInsn(self.id, self.depends_on.copy()) def test_unique_fusion(): @@ -506,7 +506,7 @@ def test_unique_fusion(): fuse_instruction_streams_with_unique_ids # create instructions 1-5 with depends - insns_a = [dummy_insn(str(i), [str(x) for x in range(i)]) + insns_a = [DummyInsn(str(i), frozenset([str(x) for x in range(i)])) for i in range(5)] # first try feeding it 'a' twice @@ -519,7 +519,7 @@ def test_unique_fusion(): assert all(x.id + '_0' in new_insn_ids for x in insns_a) # next create independent instructions 5-10 - insns_b = [dummy_insn(str(i), [str(x) for x in range(5, i)]) + insns_b = [DummyInsn(str(i), frozenset([str(x) for x in range(5, i)])) for i in range(5, 10)] new_insns, b_map = fuse_instruction_streams_with_unique_ids( @@ -553,7 +553,7 @@ def test_overlapping_fusion(): fuse_instruction_streams_with_overlapping_ids # create instructions 1-5 with depends - insns_a = [dummy_insn(str(i)) + insns_a = [DummyInsn(str(i)) for i in range(5)] a_copy = [i.copy() for i in insns_a] a_ids = [i.id for i in insns_a] @@ -578,7 +578,7 @@ def test_overlapping_fusion(): # try with b depends on a, and the instruction overlapping insns_b = a_copy[:] - insns_b[1].depends_on = frozenset(insns_a[0].id) + insns_b[1].depends_on = frozenset([insns_a[0].id]) new_insns, b_map = fuse_instruction_streams_with_overlapping_ids( insns_a, a_copy, [insns_a[0].id]) -- GitLab From 2cb22fd5fdf05256bac9fda7fb142bc686a4eb25 Mon Sep 17 00:00:00 2001 From: arghdos Date: Mon, 27 Mar 2017 10:17:32 -0400 Subject: [PATCH 08/11] fix for python2.6 --- pymbolic/imperative/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymbolic/imperative/transform.py b/pymbolic/imperative/transform.py index 13e1026..5baeb4e 100644 --- a/pymbolic/imperative/transform.py +++ b/pymbolic/imperative/transform.py @@ -70,8 +70,8 @@ def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b, # some instructions in b may depend on those in a # therefore also check if dep in a instructions new_dep_id = dep_id if dep_id in a_ids else None - assert new_dep_id is not None, ('Instruction {} in stream b ' - 'missing dependency {}'.format(insnb.id, dep_id)) + assert new_dep_id is not None, ('Instruction {0} in stream b ' + 'missing dependency {1}'.format(insnb.id, dep_id)) b_deps.add(new_dep_id) new_instructions.append( -- GitLab From e548941ffd310a9f459553ca863260828d3a2e43 Mon Sep 17 00:00:00 2001 From: arghdos Date: Thu, 30 Mar 2017 10:06:37 -0400 Subject: [PATCH 09/11] better comments --- pymbolic/imperative/transform.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pymbolic/imperative/transform.py b/pymbolic/imperative/transform.py index 5baeb4e..9d50888 100644 --- a/pymbolic/imperative/transform.py +++ b/pymbolic/imperative/transform.py @@ -60,16 +60,20 @@ def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b, insnb.copy(id=new_id)) for insnb in b_unique_instructions: + # Now we must update the dependencies in `instructions_b` to + # the new b_id's. + # If allow_b_depend_on_a is True, `insnb` is allowed to depend on + # instructions in `instructions_a` b_deps = set() for dep_id in insnb.depends_on: - new_dep_id = old_b_id_to_new_b_id[dep_id] \ - if dep_id in old_b_id_to_new_b_id else None + # First, see if the dependency is in the updated `instructions_b` + # ids. + new_dep_id = old_b_id_to_new_b_id.get(dep_id) + # Next, if allow_b_depend_on_a, check the a_ids if allow_b_depend_on_a and new_dep_id is None: - # if this is called by - # fuse_instruction_streams_with_overlapping_ids - # some instructions in b may depend on those in a - # therefore also check if dep in a instructions new_dep_id = dep_id if dep_id in a_ids else None + # If the dependency is not found in `instructions_a` or the updated + # `instructions_b`, raise an AssertionError. assert new_dep_id is not None, ('Instruction {0} in stream b ' 'missing dependency {1}'.format(insnb.id, dep_id)) b_deps.add(new_dep_id) -- GitLab From 1bbdb9a04ba2f0aae40dbe88216a7d7696493e9d Mon Sep 17 00:00:00 2001 From: arghdos Date: Wed, 5 Apr 2017 11:32:14 -0400 Subject: [PATCH 10/11] use set.get() --- pymbolic/imperative/transform.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pymbolic/imperative/transform.py b/pymbolic/imperative/transform.py index 9d50888..0b796a1 100644 --- a/pymbolic/imperative/transform.py +++ b/pymbolic/imperative/transform.py @@ -60,18 +60,15 @@ def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b, insnb.copy(id=new_id)) for insnb in b_unique_instructions: - # Now we must update the dependencies in `instructions_b` to - # the new b_id's. - # If allow_b_depend_on_a is True, `insnb` is allowed to depend on - # instructions in `instructions_a` b_deps = set() for dep_id in insnb.depends_on: # First, see if the dependency is in the updated `instructions_b` # ids. new_dep_id = old_b_id_to_new_b_id.get(dep_id) - # Next, if allow_b_depend_on_a, check the a_ids + # Next if `allow_b_depend_on_a` is True, check for the dendency in the + # `a_ids` if allow_b_depend_on_a and new_dep_id is None: - new_dep_id = dep_id if dep_id in a_ids else None + new_dep_id = a_ids.get(dep_id) # If the dependency is not found in `instructions_a` or the updated # `instructions_b`, raise an AssertionError. assert new_dep_id is not None, ('Instruction {0} in stream b ' -- GitLab From 89fa55ee75a32d5eac3290286bd9ce5be3636ad1 Mon Sep 17 00:00:00 2001 From: arghdos Date: Wed, 5 Apr 2017 11:38:25 -0400 Subject: [PATCH 11/11] well keep the comments anyways --- pymbolic/imperative/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymbolic/imperative/transform.py b/pymbolic/imperative/transform.py index 0b796a1..c380a49 100644 --- a/pymbolic/imperative/transform.py +++ b/pymbolic/imperative/transform.py @@ -68,7 +68,7 @@ def fuse_instruction_streams_with_unique_ids(instructions_a, instructions_b, # Next if `allow_b_depend_on_a` is True, check for the dendency in the # `a_ids` if allow_b_depend_on_a and new_dep_id is None: - new_dep_id = a_ids.get(dep_id) + new_dep_id = dep_id if dep_id in a_ids else None # If the dependency is not found in `instructions_a` or the updated # `instructions_b`, raise an AssertionError. assert new_dep_id is not None, ('Instruction {0} in stream b ' -- GitLab