diff --git a/loopy/creation.py b/loopy/creation.py
index 0908498ce013d76e538ad16076595e12e491e787..848fa1aaaf2518a7b0d27f710ca116789d9f6860 100644
--- a/loopy/creation.py
+++ b/loopy/creation.py
@@ -136,6 +136,80 @@ def create_temporaries(knl):
# }}}
+# {{{ reduction iname duplication
+
+def duplicate_reduction_inames(kernel):
+
+ # {{{ helper function
+
+ newly_created_vars = set()
+
+ def duplicate_reduction_inames(reduction_expr, rec):
+ child = rec(reduction_expr.expr)
+ new_red_inames = []
+ did_something = False
+
+ for iname in reduction_expr.inames:
+ if iname.startswith("@"):
+ new_iname = kernel.make_unique_var_name(iname[1:]+"_"+insn.id,
+ newly_created_vars)
+
+ old_insn_inames.append(iname.lstrip("@"))
+ new_insn_inames.append(new_iname)
+ newly_created_vars.add(new_iname)
+ new_red_inames.append(new_iname)
+ did_something = True
+ else:
+ new_red_inames.append(iname)
+
+ if did_something:
+ from loopy.symbolic import SubstitutionMapper
+ from pymbolic.mapper.substitutor import make_subst_func
+ from pymbolic import var
+
+ subst_dict = dict(
+ (old_iname, var(new_iname))
+ for old_iname, new_iname in zip(
+ reduction_expr.untagged_inames, new_red_inames))
+ subst_map = SubstitutionMapper(make_subst_func(subst_dict))
+
+ child = subst_map(child)
+
+ from loopy.symbolic import Reduction
+ return Reduction(
+ operation=reduction_expr.operation,
+ inames=tuple(new_red_inames),
+ expr=child)
+
+ # }}}
+
+ new_domain = kernel.domain
+ new_insns = []
+
+ new_iname_to_tag = kernel.iname_to_tag.copy()
+
+ for insn in kernel.instructions:
+ old_insn_inames = []
+ new_insn_inames = []
+
+ from loopy.symbolic import ReductionCallbackMapper
+ new_insns.append(insn.copy(
+ expression=ReductionCallbackMapper(duplicate_reduction_inames)
+ (insn.expression)))
+
+ from loopy.isl_helpers import duplicate_axes
+ for old, new in zip(old_insn_inames, new_insn_inames):
+ new_domain = duplicate_axes(new_domain, [old], [new])
+ if old in kernel.iname_to_tag:
+ new_iname_to_tag[new] = kernel.iname_to_tag[old]
+
+ return kernel.copy(
+ instructions=new_insns,
+ domain=new_domain,
+ iname_to_tag=new_iname_to_tag)
+
+# }}}
+
# {{{ duplicate inames
def duplicate_inames(knl):
@@ -229,9 +303,19 @@ def make_kernel(*args, **kwargs):
check_kernel(knl)
knl = create_temporaries(knl)
- knl = expand_cses(knl)
+ knl = duplicate_reduction_inames(knl)
knl = duplicate_inames(knl)
+ # -------------------------------------------------------------------------
+ # Ordering dependency:
+ # -------------------------------------------------------------------------
+ # Must duplicate inames before expanding CSEs, otherwise inames within the
+ # scope of duplication might be CSE'd out to a different instruction and
+ # never be found by duplication.
+ # -------------------------------------------------------------------------
+
+ knl = expand_cses(knl)
+
return knl
# }}}
diff --git a/loopy/preprocess.py b/loopy/preprocess.py
index e010b2e78eee41fc5a89a26c362c9e43f3000c76..f49832ed80008533b5bcbcc9b41234e42d5180ee 100644
--- a/loopy/preprocess.py
+++ b/loopy/preprocess.py
@@ -171,80 +171,6 @@ def mark_local_temporaries(kernel):
# }}}
-# {{{ reduction iname duplication
-
-def duplicate_reduction_inames(kernel):
-
- # {{{ helper function
-
- newly_created_vars = set()
-
- def duplicate_reduction_inames(reduction_expr, rec):
- child = rec(reduction_expr.expr)
- new_red_inames = []
- did_something = False
-
- for iname in reduction_expr.inames:
- if iname.startswith("@"):
- new_iname = kernel.make_unique_var_name(iname[1:]+"_"+insn.id,
- newly_created_vars)
-
- old_insn_inames.append(iname.lstrip("@"))
- new_insn_inames.append(new_iname)
- newly_created_vars.add(new_iname)
- new_red_inames.append(new_iname)
- did_something = True
- else:
- new_red_inames.append(iname)
-
- if did_something:
- from loopy.symbolic import SubstitutionMapper
- from pymbolic.mapper.substitutor import make_subst_func
- from pymbolic import var
-
- subst_dict = dict(
- (old_iname, var(new_iname))
- for old_iname, new_iname in zip(
- reduction_expr.untagged_inames, new_red_inames))
- subst_map = SubstitutionMapper(make_subst_func(subst_dict))
-
- child = subst_map(child)
-
- from loopy.symbolic import Reduction
- return Reduction(
- operation=reduction_expr.operation,
- inames=tuple(new_red_inames),
- expr=child)
-
- # }}}
-
- new_domain = kernel.domain
- new_insns = []
-
- new_iname_to_tag = kernel.iname_to_tag.copy()
-
- for insn in kernel.instructions:
- old_insn_inames = []
- new_insn_inames = []
-
- from loopy.symbolic import ReductionCallbackMapper
- new_insns.append(insn.copy(
- expression=ReductionCallbackMapper(duplicate_reduction_inames)
- (insn.expression)))
-
- from loopy.isl_helpers import duplicate_axes
- for old, new in zip(old_insn_inames, new_insn_inames):
- new_domain = duplicate_axes(new_domain, [old], [new])
- if old in kernel.iname_to_tag:
- new_iname_to_tag[new] = kernel.iname_to_tag[old]
-
- return kernel.copy(
- instructions=new_insns,
- domain=new_domain,
- iname_to_tag=new_iname_to_tag)
-
-# }}}
-
# {{{ rewrite reduction to imperative form
def realize_reduction(kernel, insn_id_filter=None):
@@ -257,14 +183,8 @@ def realize_reduction(kernel, insn_id_filter=None):
If *insn_id_filter* is not given, all reductions in all instructions will
be realized.
-
- This routine also implicitly performs (global) reduction iname duplication,
- if requested by '@' prefixes on any reduction iname.
"""
- # Reduction iname duplication needs to happen beforehand, and it is
- # idempotent. So just call it now.
- kernel = duplicate_reduction_inames(kernel)
new_insns = []
new_temporary_variables = kernel.temporary_variables.copy()