From c4b1a594cab539b08e975029705412d1b335976a Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Tue, 12 Nov 2019 00:22:35 -0600 Subject: [PATCH 1/6] Language changes for paper These changes simplify the Dagrt language and the code builder interface. The first change is that *depends_on* becomes a computed attribute of the phase DAG. The semantics of the DAG is changed so that *all* statements get executed, instead of just those reachable from *depends_on*. This is a non-backwards compatible change that means that DAGs are no longer allowed to carry unreachable "dead" code. The language version is bumped to 2019.3. The code builder interface is changed to remove the function *reset_dep_tracking* and the attribute *phase_dependencies*. Code that is generated by the code builder should still function as previously. The dependency DAG that is generated by the code builder no longer uses Nops as placeholders, which simplifies its output. cc: @cmikida, @inducer --- dagrt/language.py | 379 ++++++++++++++--------------------- dagrt/transform.py | 3 - dagrt/version.py | 2 +- test/test_ast.py | 2 +- test/test_builtins.py | 16 +- test/test_code_builder.py | 25 +-- test/test_codegen.py | 15 +- test/test_codegen_fortran.py | 11 +- test/test_codegen_python.py | 63 +++--- 9 files changed, 202 insertions(+), 314 deletions(-) diff --git a/dagrt/language.py b/dagrt/language.py index 708b8d9..3797935 100644 --- a/dagrt/language.py +++ b/dagrt/language.py @@ -627,12 +627,20 @@ class ExecutionPhase(RecordWithoutPickling): will actually be executed. """ - def __init__(self, depends_on, next_phase, statements): + def __init__(self, next_phase, statements): super(ExecutionPhase, self).__init__( - depends_on=depends_on, next_phase=next_phase, statements=statements) + @property + @memoize_method + def depends_on(self): + # Get statement IDs which no other statement depends on. + result = set(stmt.id for stmt in self.statements) + for stmt in self.statements: + result -= set(stmt.depends_on) + return result + @property @memoize_method def id_to_stmt(self): @@ -653,24 +661,21 @@ class DAGCode(RecordWithoutPickling): """ @classmethod - def create_with_steady_phase(cls, dep_on, statements): + def create_with_steady_phase(cls, statements): phases = {'main': ExecutionPhase( - dep_on, next_phase='main', statements=statements)} + next_phase='main', statements=statements)} return cls(phases, 'main') @classmethod - def _create_with_init_and_step(cls, initialization_dep_on, - step_dep_on, statements): + def _create_with_init_and_step(cls, init_statements, main_statements): phases = {} phases['initialization'] = ExecutionPhase( - initialization_dep_on, - next_phase='primary', - statements=statements) + next_phase='main', + statements=init_statements) - phases['primary'] = ExecutionPhase( - step_dep_on, - next_phase='primary', - statements=statements) + phases['main'] = ExecutionPhase( + next_phase='main', + statements=main_statements) return cls(phases, 'initialization') @@ -813,21 +818,12 @@ class CodeBuilder(object): The set of statements generated for the phase - .. attribute:: phase_dependencies - - A list of statement names. Starting with these statements - as the root dependencies, the phase can be executed by following - the dependency list of each statement. - - .. automethod:: reset_dep_tracking .. automethod:: if_ .. automethod:: else_ - .. automethod:: __call__ - - .. method:: assign - - Alias for :func:`CodeBuilder.__call__`. + .. method:: __call__ + Alias for :func:`CodeBuilder.assign`. + .. automethod:: assign .. automethod:: fresh_var_name .. automethod:: fresh_var .. automethod:: assign_implicit @@ -838,71 +834,117 @@ class CodeBuilder(object): .. automethod:: raise_ .. automethod:: switch_phase .. automethod:: __enter__ - + .. automethod:: __exit__ """ - class Context(RecordWithoutPickling): - """ - A context represents a block of statements being built into the DAG - - .. attribute:: lead_statement_ids - - .. attribute:: introduced_condition - - .. attribute:: context_statement_ids - - .. attribute:: used_variables - - .. attribute:: definition_map - """ - def __init__(self, lead_statement_ids=[], definition_map={}, - used_variables=[], condition=True): - RecordWithoutPickling.__init__(self, - lead_statement_ids=frozenset(lead_statement_ids), - context_statement_ids=set(lead_statement_ids), - definition_map=dict(definition_map), - used_variables=set(used_variables), - condition=condition) + # This is a dummy variable name representing the "system state", which is + # used to track the dependencies for instructions having side effects. + _EXECUTION_STATE = "" def __init__(self, label="phase"): """ :arg label: The name of the phase to generate """ self.label = label - self._statement_map = {} - self._statement_count = 0 - self._contexts = [] - self._last_popped_context = None - self._all_var_names = set() - self._all_generated_var_names = set() - - def reset_dep_tracking(self): - """ - Enter a new logical block of statements. Force all prior - statements to execute before subsequent ones. - """ - self._contexts[-1] = self._make_new_context(Nop(), - additional_condition=self._contexts[-1].condition) - - def _get_active_condition(self): - def is_nontrivial_condition(cond): - return cond is not True - - conditions = list(filter(is_nontrivial_condition, - [context.condition for context in self._contexts])) - num_conditions = len(conditions) + self.statements = [] + + # Maps variables to the sequentially last instruction to write them + self._writer_map = {} + # Maps variables to the set of instructions that read them + self._reader_map = {} + # Stack of conditional expressions used to implemented nested ifs + self._conditional_expression_stack = [] + # Used to implement if/else + self._last_active_conditional_expression = None + # Set of seen variables + self._seen_var_names = set([self._EXECUTION_STATE]) + + def _add_statement(self, stmt): + stmt_id = self.next_statement_id() + + read_variables = set(stmt.get_read_variables()) + written_variables = set(stmt.get_written_variables()) + + # Add the global execution state as an implicitly read variable. + read_variables.add(self._EXECUTION_STATE) + + # Build the condition attribute. + if not self._conditional_expression_stack: + condition = True + elif len(self._conditional_expression_stack) == 1: + condition = self._conditional_expression_stack[0] + else: + from pymbolic.primitives import LogicalAnd + condition = LogicalAnd(tuple(self._conditional_expression_stack)) + + from dagrt.utils import get_variables + read_variables |= get_variables(condition) + + is_non_assignment = ( + not isinstance( + stmt, (Assign, AssignImplicit, AssignFunctionCall))) + + # We regard all non-assignments as having potential external side + # effects (i.e., writing to EXECUTION_STATE). To keep the global + # variables in a well-defined state, ensure that all updates to global + # variables have happened before a non-assignment. + if is_non_assignment: + from dagrt.utils import is_state_variable + read_variables |= set( + var for var in self._seen_var_names if + is_state_variable(var)) + written_variables.add(self._EXECUTION_STATE) + + depends_on = set() + + # Ensure this statement happens after the last write of all the + # variables it reads or writes. + for var in read_variables | written_variables: + writer = self._writer_map.get(var, None) + if writer is not None: + depends_on.add(writer) + + # Ensure this statement happens after the last read(s) of the variables + # it writes to. + for var in written_variables: + readers = self._reader_map.get(var, set()) + depends_on |= readers + # Keep the graph sparse by clearing the readers set. + readers.clear() + + for var in written_variables: + self._writer_map[var] = stmt_id + + for var in read_variables: + self._reader_map.setdefault(var, set()).add(stmt_id) + + stmt = stmt.copy( + id=stmt_id, + condition=condition, + depends_on=frozenset(depends_on)) + self.statements.append(stmt) + self._seen_var_names |= read_variables | written_variables + + def next_statement_id(self): + return "%s_%d" % (self.label, len(self.statements)) - # No conditions - trival - if num_conditions == 0: - return True + @memoize_method + def _var_name_generator(self, prefix): + from pytools import generate_unique_names + return generate_unique_names(prefix) - # Single condition - if num_conditions == 1: - return conditions[0] + def fresh_var_name(self, prefix="temp"): + """Return a variable name that is not in use also and won't be returned in the + future, regardless of use. + """ + for var_name in self._var_name_generator(prefix): + if var_name not in self._seen_var_names: + self._seen_var_names.add(var_name) + return var_name - # Conjunction of conditions - from pymbolic.primitives import LogicalAnd - return LogicalAnd(tuple(conditions)) + def fresh_var(self, prefix="temp"): + from pymbolic import var + return var(self.fresh_var_name(prefix)) @contextmanager def if_(self, *condition_arg): @@ -935,77 +977,61 @@ class CodeBuilder(object): assignee_subscript=(), expression=condition) - self._contexts.append( - self._make_new_context(cond_assignment, additional_condition=cond_var)) + self._add_statement(cond_assignment) + self._conditional_expression_stack.append(cond_var) yield - - # Pop myself from the stack. - last_context = self._contexts.pop() - self._contexts[-1] = self._make_new_context( - Nop(depends_on=last_context.context_statement_ids), - additional_condition=self._contexts[-1].condition) - - self._last_popped_if = last_context + self._conditional_expression_stack.pop() + self._last_active_conditional_expression = cond_var @contextmanager def else_(self): """ Create the "else" portion of a conditionally executed block. """ - assert self._last_popped_if + assert self._last_active_conditional_expression is not None # Create conditions for the context. from pymbolic.primitives import LogicalNot - self._contexts.append( - self._make_new_context(Nop(), - additional_condition=LogicalNot(self._last_popped_if.condition))) - - self._last_popped_if = None - + self._conditional_expression_stack.append( + LogicalNot(self._last_active_conditional_expression)) yield + self._conditional_expression_stack.pop() + self._last_active_conditional_expression = None - # Pop myself from the stack. - last_context = self._contexts.pop() - - self._contexts[-1] = self._make_new_context( - Nop(depends_on=last_context.context_statement_ids), - additional_condition=self._contexts[-1].condition) - - def _next_statement_id(self): - self._statement_count += 1 - return self.label + "_" + str(self._statement_count) - - def __call__(self, assignees, expression, loops=[]): + def assign(self, assignees, expression, loops=[]): """Generate code for an assignment. *assignees* may be a variable, a subscript (if referring to an array), or a tuple of variables. There must be exactly one assignee unless *expression* is a function call. + + *loops* is a list of tuples of the form + *(array, start_index, stop_index)*. """ from dagrt.expression import parse - def _parse_if_necessary(s): + def parse_if_necessary(s): if isinstance(s, str): return parse(s) else: return s - assignees = _parse_if_necessary(assignees) + assignees = parse_if_necessary(assignees) if isinstance(assignees, tuple): assignees = tuple( - _parse_if_necessary(s) + parse_if_necessary(s) for s in assignees) else: assignees = (assignees,) - expression = _parse_if_necessary(expression) + expression = parse_if_necessary(expression) new_loops = [] for ident, start, stop in loops: - start = _parse_if_necessary(start) - stop = _parse_if_necessary(stop) + start = parse_if_necessary(start) + stop = parse_if_necessary(stop) new_loops.append((ident, start, stop)) from pymbolic.primitives import Call, CallWithKwargs, Variable @@ -1024,7 +1050,7 @@ class CodeBuilder(object): else: kw_parameters = {} - self._add_inst_to_context(AssignFunctionCall( + self._add_statement(AssignFunctionCall( assignees=assignee_names, function_id=expression.function.name, parameters=expression.parameters, @@ -1053,96 +1079,13 @@ class CodeBuilder(object): "variable or a subscribted variable, not '%s'" % type(assignee)) - self._add_inst_to_context(Assign( + self._add_statement(Assign( assignee=aname, assignee_subscript=asub, expression=expression, loops=new_loops)) - assign = __call__ - - def _add_inst_to_context(self, inst): - inst_id = self._next_statement_id() - context = self._contexts[-1] - dependencies = set(context.lead_statement_ids) - - # Verify that assignees are not being places after uses of the - # assignees in this context. - for assignee in inst.get_written_variables(): - # Warn about potential ordering of assignments that may - # be unexpected by the user. - if assignee in context.used_variables: - raise ValueError( - "write after use of " + assignee - + " in the same block") - - if ( - assignee in context.definition_map - - # multiple assignments with subscript are OK - and not ( - isinstance(inst, Assign) - and inst.assignee_subscript is not None)): - raise ValueError("multiple assignments to " + assignee) - - # Create the set of dependencies based on the set of used - # variables. - for used_variable in inst.get_read_variables(): - if used_variable in context.definition_map: - dependencies.update(context.definition_map[used_variable]) - - for used_variable in inst.get_written_variables(): - # Make second (indexed) writes depend on initialization - for def_inst_id in context.definition_map.get(used_variable, []): - def_inst = self._statement_map[def_inst_id] - if ( - not isinstance(def_inst, Assign) - or def_inst.assignee_subscript is None): - dependencies.add(def_inst_id) - - # Add the condition to the statement. - # Update context and global information. - context.context_statement_ids.add(inst_id) - for assignee in inst.get_written_variables(): - context.definition_map.setdefault(assignee, set()).add(inst_id) - - context.used_variables |= inst.get_read_variables() - self._all_var_names |= inst.get_written_variables() - self._statement_map[inst_id] = \ - inst.copy(id=inst_id, depends_on=list(dependencies), - condition=self._get_active_condition()) - return inst_id - - def _make_new_context(self, inst, additional_condition=True): - """ - :param leading_statements: A list of lead statement ids - :conditions: A - """ - inst_id = self._next_statement_id() - context = self._contexts[-1] - new_context = CodeBuilder.Context( - lead_statement_ids=[inst_id], - used_variables=set(), - condition=additional_condition) - self._statement_map[inst_id] = \ - inst.copy(id=inst_id, - depends_on=inst.depends_on | context.context_statement_ids, - condition=self._get_active_condition()) - return new_context - - def fresh_var_name(self, prefix="temp"): - """Return a variable name that is not guaranteed not to be in - use and not to be generated in the future.""" - from pytools import generate_unique_names - for possible_var in generate_unique_names(str(prefix)): - if possible_var not in self._all_var_names \ - and possible_var not in self._all_generated_var_names: - self._all_generated_var_names.add(possible_var) - return possible_var - - def fresh_var(self, prefix="temp"): - from pymbolic import var - return var(self.fresh_var_name(prefix)) + __call__ = assign def assign_implicit_1(self, assignee, solve_component, expression, guess, solver_id=None): @@ -1153,8 +1096,9 @@ class CodeBuilder(object): def assign_implicit(self, assignees, solve_components, expressions, other_params, solver_id): - self._add_inst_to_context(AssignImplicit(assignees, solve_components, - expressions, other_params, solver_id)) + self._add_statement(AssignImplicit( + assignees, solve_components, + expressions, other_params, solver_id)) def yield_state(self, expression, component_id, time, time_id): """Yield a value.""" @@ -1164,44 +1108,29 @@ class CodeBuilder(object): if isinstance(expression, str): expression = parse(expression) - self._add_inst_to_context(YieldState( + self._add_statement(YieldState( expression=expression, component_id=component_id, time=time, time_id=time_id)) def fail_step(self): - self.reset_dep_tracking() - self._add_inst_to_context(FailStep()) + self._add_statement(FailStep()) def restart_step(self): - self.reset_dep_tracking() - self._add_inst_to_context(RestartStep()) + self._add_statement(RestartStep()) def raise_(self, error_condition, error_message=None): - self.reset_dep_tracking() - self._add_inst_to_context(Raise(error_condition, error_message)) + self._add_statement(Raise(error_condition, error_message)) def switch_phase(self, next_phase): - self.reset_dep_tracking() - self._add_inst_to_context(SwitchPhase(next_phase)) + self._add_statement(SwitchPhase(next_phase)) def __enter__(self): - self._contexts.append(CodeBuilder.Context()) return self def __exit__(self, *ignored): - self.reset_dep_tracking() - self.phase_dependencies = list(self._contexts[-1].lead_statement_ids) - self.statements = set(self._statement_map.values()) - - def __str__(self): - roots = [ - self._statement_map[stmt_id] - for ctx in self._contexts - for stmt_id in ctx.context_statement_ids] - - return "\n".join(_stringify_statements(roots, self._statement_map)) + pass def as_execution_phase(self, next_phase): """ @@ -1209,8 +1138,8 @@ class CodeBuilder(object): :arg next_phase: The name of the default next phase """ return ExecutionPhase( - depends_on=self.phase_dependencies, next_phase=next_phase, - statements=self.statements) + next_phase=next_phase, + statements=frozenset(self.statements)) # }}} diff --git a/dagrt/transform.py b/dagrt/transform.py index 95c0a29..a6c8023 100644 --- a/dagrt/transform.py +++ b/dagrt/transform.py @@ -39,9 +39,6 @@ def fuse_two_phases(phase_name, phase1, phase2): return ExecutionPhase( next_phase=phase1.next_phase, - depends_on=frozenset(phase1.depends_on) | frozenset( - old_2_id_to_new_2_id.get(id2, id2) - for id2 in phase2.depends_on), statements=new_statements ) diff --git a/dagrt/version.py b/dagrt/version.py index b8345dc..74bb455 100644 --- a/dagrt/version.py +++ b/dagrt/version.py @@ -1,2 +1,2 @@ -VERSION = (2019, 2) +VERSION = (2019, 3) VERSION_TEXT = ".".join(str(i) for i in VERSION) diff --git a/test/test_ast.py b/test/test_ast.py index f67e0a3..50db7e8 100755 --- a/test/test_ast.py +++ b/test/test_ast.py @@ -58,7 +58,7 @@ def test_create_ast(): return hash(self._state()) nop = ComparableNop(condition=x, id="nop", depends_on=()) - code = DAGCode.create_with_steady_phase(["nop"], [nop]) + code = DAGCode.create_with_steady_phase([nop]) ast = create_ast_from_phase(code, "main") assert ast == IfThen(x, StatementWrapper(nop.copy(condition=True))) diff --git a/test/test_builtins.py b/test/test_builtins.py index c0ccf03..59bbb6c 100755 --- a/test/test_builtins.py +++ b/test/test_builtins.py @@ -51,9 +51,7 @@ def test_len(python_method_impl, obj, len_): expression=var('x'), component_id='', depends_on=['assign_1'])) cbuild.commit() - code = DAGCode.create_with_steady_phase( - dep_on=['return'], - statements=cbuild.statements) + code = DAGCode.create_with_steady_phase(cbuild.statements) result = execute_and_return_single_result(python_method_impl, code) assert result == len_ @@ -69,9 +67,7 @@ def test_isnan(python_method_impl, value): expression=var('x'), component_id='', depends_on=['assign_1'])) cbuild.commit() - code = DAGCode.create_with_steady_phase( - dep_on=['return'], - statements=cbuild.statements) + code = DAGCode.create_with_steady_phase(cbuild.statements) result = execute_and_return_single_result(python_method_impl, code) assert result == np.isnan(value) @@ -102,9 +98,7 @@ def test_norm(python_method_impl, order, norm_suffix, test_vector): expression=var('n'), component_id='', depends_on=['assign_2'])) cbuild.commit() - code = DAGCode.create_with_steady_phase( - dep_on=['return'], - statements=cbuild.statements) + code = DAGCode.create_with_steady_phase(cbuild.statements) result = execute_and_return_single_result(python_method_impl, code) assert np.allclose(result, true_norm(test_vector)) @@ -121,9 +115,7 @@ def test_dot_product(python_method_impl, x, y): expression=var('x'), component_id='', depends_on=['assign_1'])) cbuild.commit() - code = DAGCode.create_with_steady_phase( - dep_on=['return'], - statements=cbuild.statements) + code = DAGCode.create_with_steady_phase(cbuild.statements) result = execute_and_return_single_result(python_method_impl, code) assert result == np.vdot(x, y) diff --git a/test/test_code_builder.py b/test/test_code_builder.py index 0641a80..faec01e 100755 --- a/test/test_code_builder.py +++ b/test/test_code_builder.py @@ -41,8 +41,7 @@ THE SOFTWARE. def test_CodeBuilder_yield(python_method_impl): with CodeBuilder() as builder: builder.yield_state(1, 'x', 0, 'final') - code = DAGCode.create_with_steady_phase( - builder.phase_dependencies, builder.statements) + code = DAGCode.create_with_steady_phase(builder.statements) result = execute_and_return_single_result(python_method_impl, code) assert result == 1 @@ -51,8 +50,7 @@ def test_CodeBuilder_assign(python_method_impl): with CodeBuilder() as builder: builder(var('x'), 1) builder.yield_state(var('x'), 'x', 0, 'final') - code = DAGCode.create_with_steady_phase( - builder.phase_dependencies, builder.statements) + code = DAGCode.create_with_steady_phase(builder.statements) result = execute_and_return_single_result(python_method_impl, code) assert result == 1 @@ -63,8 +61,7 @@ def test_CodeBuilder_condition(python_method_impl): with builder.if_(var('x'), '==', 1): builder(var('x'), 2) builder.yield_state(var('x'), 'x', 0, 'final') - code = DAGCode.create_with_steady_phase( - builder.phase_dependencies, builder.statements) + code = DAGCode.create_with_steady_phase(builder.statements) result = execute_and_return_single_result(python_method_impl, code) assert result == 2 @@ -77,8 +74,7 @@ def test_CodeBuilder_condition_with_else(python_method_impl): with builder.else_(): builder(var('x'), 3) builder.yield_state(var('x'), 'x', 0, 'final') - code = DAGCode.create_with_steady_phase( - builder.phase_dependencies, builder.statements) + code = DAGCode.create_with_steady_phase(builder.statements) result = execute_and_return_single_result(python_method_impl, code) assert result == 3 @@ -91,8 +87,7 @@ def test_CodeBuilder_condition_with_else_not_taken(python_method_impl): with builder.else_(): builder(var('x'), 3) builder.yield_state(var('x'), 'x', 0, 'final') - code = DAGCode.create_with_steady_phase( - builder.phase_dependencies, builder.statements) + code = DAGCode.create_with_steady_phase(builder.statements) result = execute_and_return_single_result(python_method_impl, code) assert result == 2 @@ -105,8 +100,7 @@ def test_CodeBuilder_nested_condition(python_method_impl): with builder.if_(var('x'), '==', 2): builder(var('x'), 3) builder.yield_state(var('x'), 'x', 0, 'final') - code = DAGCode.create_with_steady_phase( - builder.phase_dependencies, builder.statements) + code = DAGCode.create_with_steady_phase(builder.statements) result = execute_and_return_single_result(python_method_impl, code) assert result == 3 @@ -121,8 +115,7 @@ def test_CodeBuilder_nested_condition_with_else(python_method_impl): with builder.else_(): builder(var('x'), 4) builder.yield_state(var('x'), 'x', 0, 'final') - code = DAGCode.create_with_steady_phase( - builder.phase_dependencies, builder.statements) + code = DAGCode.create_with_steady_phase(builder.statements) result = execute_and_return_single_result(python_method_impl, code) assert result == 4 @@ -137,8 +130,7 @@ def test_CodeBuilder_nested_condition_with_else_not_taken(python_method_impl): with builder.else_(): builder(var('x'), 4) builder.yield_state(var('x'), 'x', 0, 'final') - code = DAGCode.create_with_steady_phase( - builder.phase_dependencies, builder.statements) + code = DAGCode.create_with_steady_phase(builder.statements) result = execute_and_return_single_result(python_method_impl, code) assert result == 3 @@ -147,7 +139,6 @@ def test_CodeBuilder_restart_step(python_method_impl): with CodeBuilder() as builder1: builder1("

x", 1) builder1.restart_step() - builder1.reset_dep_tracking() builder1("

x", 2) with CodeBuilder() as builder2: diff --git a/test/test_codegen.py b/test/test_codegen.py index 684f1bd..6051878 100755 --- a/test/test_codegen.py +++ b/test/test_codegen.py @@ -58,8 +58,8 @@ def test_circular_dependency_detection(): depends_on=['assign'])) cbuild.commit() code = DAGCode._create_with_init_and_step( - initialization_dep_on=[], - statements=cbuild.statements, step_dep_on=['return']) + init_statements=[], + main_statements=cbuild.statements) with pytest.raises(CodeGenerationError): verify_code(code) @@ -75,8 +75,8 @@ def test_missing_dependency_detection(): depends_on=['assign']) ]) code = DAGCode._create_with_init_and_step( - initialization_dep_on=[], - statements=statements, step_dep_on=['return']) + init_statements=[], + main_statements=statements) with pytest.raises(CodeGenerationError): verify_code(code) @@ -88,8 +88,7 @@ def test_missing_state_detection(): with CodeBuilder(label="state_1") as cb: cb.switch_phase("state_2") - code = DAGCode.create_with_steady_phase( - dep_on=cb.phase_dependencies, statements=cb.statements) + code = DAGCode.create_with_steady_phase(statements=cb.statements) with pytest.raises(CodeGenerationError): verify_code(code) @@ -117,8 +116,8 @@ def test_cond_detection(): depends_on=['assign2'])) cbuild.commit() code = DAGCode._create_with_init_and_step( - initialization_dep_on=[], - statements=cbuild.statements, step_dep_on=['return']) + init_statements=[], + main_statements=cbuild.statements) with pytest.raises(CodeGenerationError): verify_code(code) diff --git a/test/test_codegen_fortran.py b/test/test_codegen_fortran.py index bbca5ca..35c4522 100755 --- a/test/test_codegen_fortran.py +++ b/test/test_codegen_fortran.py @@ -52,9 +52,7 @@ def test_basic_codegen(): expression=0, component_id='state', depends_on=[])) cbuild.commit() - code = DAGCode._create_with_init_and_step( - initialization_dep_on=[], - statements=cbuild.statements, step_dep_on=['return']) + code = DAGCode.create_with_steady_phase(cbuild.statements) codegen = f.CodeGenerator("simple", user_type_map={ "state": f.ArrayType( @@ -76,21 +74,17 @@ def test_arrays_and_linalg(): cb("nodes", "`array`(n)") cb("vdm", "`array`(n*n)") cb("identity", "`array`(n*n)") - cb.reset_dep_tracking() cb("nodes[i]", "i/n", loops=[("i", 0, "n")]) cb("identity[i]", "0", loops=[("i", 0, "n*n")]) - cb.reset_dep_tracking() cb("identity[i*n + i]", "1", loops=[("i", 0, "n")]) cb("vdm[j*n + i]", "nodes[i]**j", loops=[("i", 0, "n"), ("j", 0, "n")]) - cb.reset_dep_tracking() - cb("vdm_inverse", "`linear_solve`(vdm, identity, n, n)") cb("myarray", "`matmul`(vdm, vdm_inverse, n, n)") @@ -99,8 +93,7 @@ def test_arrays_and_linalg(): with cb.if_("`norm_2`(myzero) > 10**(-8)"): cb.raise_(MatrixInversionFailure) - code = DAGCode.create_with_steady_phase( - cb.phase_dependencies, cb.statements) + code = DAGCode.create_with_steady_phase(cb.statements) codegen = f.CodeGenerator( 'arrays', diff --git a/test/test_codegen_python.py b/test/test_codegen_python.py index d2eb7ff..aeabd58 100755 --- a/test/test_codegen_python.py +++ b/test/test_codegen_python.py @@ -49,12 +49,12 @@ def test_basic_codegen(): depends_on=[])) cbuild.commit() code = DAGCode._create_with_init_and_step( - initialization_dep_on=[], - statements=cbuild.statements, step_dep_on=['return']) + init_statements=[], + main_statements=cbuild.statements) codegen = PythonCodeGenerator(class_name='Method') + print(codegen(code)) Method = codegen.get_class(code) # noqa method = Method({}) - print(codegen(code)) method.set_up(t_start=0, dt_start=0, context={}) hist = [s for s in method.run(max_steps=2)] assert len(hist) == 3 @@ -63,7 +63,7 @@ def test_basic_codegen(): assert isinstance(hist[1], method.StateComputed) assert hist[1].state_component == 0 assert isinstance(hist[2], method.StepCompleted) - assert hist[2].current_phase == 'primary' + assert hist[2].current_phase == 'main' def test_basic_conditional_codegen(): @@ -83,8 +83,8 @@ def test_basic_conditional_codegen(): depends_on=['branch'])) cbuild.commit() code = DAGCode._create_with_init_and_step( - initialization_dep_on=[], - statements=cbuild.statements, step_dep_on=['return']) + init_statements=[], + main_statements=cbuild.statements) codegen = PythonCodeGenerator(class_name='Method') Method = codegen.get_class(code) # noqa method = Method({}) @@ -117,8 +117,8 @@ def test_basic_assign_rhs_codegen(): ) cbuild.commit() code = DAGCode._create_with_init_and_step( - initialization_dep_on=[], - statements=cbuild.statements, step_dep_on=['return']) + init_statements=[], + main_statements=cbuild.statements) codegen = PythonCodeGenerator(class_name='Method') Method = codegen.get_class(code) # noqa @@ -147,8 +147,8 @@ def test_basic_raise_codegen(): cbuild.add_and_get_ids(Raise(TimeStepUnderflow, "underflow", id="raise")) cbuild.commit() code = DAGCode._create_with_init_and_step( - initialization_dep_on=[], - statements=cbuild.statements, step_dep_on=["raise"]) + init_statements=[], + main_statements=cbuild.statements) codegen = PythonCodeGenerator(class_name="Method") Method = codegen.get_class(code) # noqa method = Method({}) @@ -174,8 +174,8 @@ def test_basic_fail_step_codegen(): cbuild.add_and_get_ids(FailStep(id="fail")) cbuild.commit() code = DAGCode._create_with_init_and_step( - initialization_dep_on=[], - statements=cbuild.statements, step_dep_on=["fail"]) + init_statements=[], + main_statements=cbuild.statements) codegen = PythonCodeGenerator(class_name="Method") Method = codegen.get_class(code) # noqa method = Method({}) @@ -205,8 +205,8 @@ def test_local_name_distinctness(): component_id='y', depends_on=['assign_y^', 'assign_y*'])) cbuild.commit() code = DAGCode._create_with_init_and_step( - initialization_dep_on=[], - statements=cbuild.statements, step_dep_on=['return']) + init_statements=[], + main_statements=cbuild.statements) codegen = PythonCodeGenerator(class_name='Method') Method = codegen.get_class(code) # noqa method = Method({}) @@ -232,8 +232,8 @@ def test_global_name_distinctness(): component_id='y', depends_on=['assign_y^', 'assign_y*'])) cbuild.commit() code = DAGCode._create_with_init_and_step( - initialization_dep_on=[], - statements=cbuild.statements, step_dep_on=['return']) + init_statements=[], + main_statements=cbuild.statements) codegen = PythonCodeGenerator(class_name='Method') Method = codegen.get_class(code) # noqa method = Method({}) @@ -253,8 +253,8 @@ def test_function_name_distinctness(): component_id='y')) cbuild.commit() code = DAGCode._create_with_init_and_step( - initialization_dep_on=[], - statements=cbuild.statements, step_dep_on=['return']) + init_statements=[], + main_statements=cbuild.statements) codegen = PythonCodeGenerator(class_name='Method') Method = codegen.get_class(code) # noqa method = Method({'y^': lambda: 0, @@ -278,10 +278,10 @@ def test_switch_phases(python_method_impl): code = DAGCode( phases={ "state_1": ExecutionPhase( - builder_1.phase_dependencies, next_phase="state_1", + next_phase="state_1", statements=builder_1.statements), "state_2": ExecutionPhase( - builder_2.phase_dependencies, next_phase="state_2", + next_phase="state_2", statements=builder_2.statements) }, initial_phase="state_1") @@ -309,8 +309,7 @@ def get_IfThenElse_test_code_and_expected_result(): cb.yield_state(tuple(var("c" + str(i)) for i in range(1, 11)), "result", 0, "final") - code = DAGCode.create_with_steady_phase( - cb.phase_dependencies, cb.statements) + code = DAGCode.create_with_steady_phase(cb.statements) return (code, (0, 1, 0, 1, 0, 1, 1, 2, 1, 2)) @@ -334,14 +333,12 @@ def test_IfThenElse_expansion(python_method_impl): def test_arrays_and_looping(python_method_impl): with CodeBuilder(label="primary") as cb: cb("myarray", "`array`(20)") - cb.reset_dep_tracking() cb("myarray[i]", "i", loops=[("i", 0, 20)]) cb.yield_state("myarray[15]", "result", 0, "final") from utils import execute_and_return_single_result - code = DAGCode.create_with_steady_phase( - cb.phase_dependencies, cb.statements) + code = DAGCode.create_with_steady_phase(cb.statements) result = execute_and_return_single_result(python_method_impl, code) assert result == 15 @@ -352,21 +349,17 @@ def test_arrays_and_linalg(python_method_impl): cb("nodes", "`array`(n)") cb("vdm", "`array`(n*n)") cb("identity", "`array`(n*n)") - cb.reset_dep_tracking() cb("nodes[i]", "i/n", loops=[("i", 0, "n")]) cb("identity[i]", "0", loops=[("i", 0, "n*n")]) - cb.reset_dep_tracking() cb("identity[i*n + i]", "1", loops=[("i", 0, "n")]) cb("vdm[j*n + i]", "nodes[i]**j", loops=[("i", 0, "n"), ("j", 0, "n")]) - cb.reset_dep_tracking() - cb("vdm_inverse", "`linear_solve`(vdm, identity, n, n)") cb("myarray", "`matmul`(vdm, vdm_inverse, n, n)") @@ -376,8 +369,7 @@ def test_arrays_and_linalg(python_method_impl): from utils import execute_and_return_single_result - code = DAGCode.create_with_steady_phase( - cb.phase_dependencies, cb.statements) + code = DAGCode.create_with_steady_phase(cb.statements) result = execute_and_return_single_result(python_method_impl, code) result = result.reshape(4, 4, order="F") @@ -391,7 +383,6 @@ def test_svd(python_method_impl): cb("nodes", "`array`(n)") cb("vdm", "`array`(n*n)") cb("identity", "`array`(n*n)") - cb.reset_dep_tracking() cb("nodes[i]", "i/n", loops=[("i", 0, "n")]) @@ -399,10 +390,7 @@ def test_svd(python_method_impl): cb("vdm[j*n + i]", "nodes[i]**j", loops=[("i", 0, "n"), ("j", 0, "n")]) - cb.reset_dep_tracking() - cb("vdm_u, vdm_sigma, vdm_vt", "`svd`(vdm, n)") - cb.reset_dep_tracking() cb("vdm_usigma", "`array`(n*n)") cb("vdm_v", "`array`(n*n)") cb("vdm_usigma[i + j*n]", "vdm_u[i + j*n] * vdm_sigma[j]", @@ -419,8 +407,7 @@ def test_svd(python_method_impl): from utils import execute_and_return_single_result - code = DAGCode.create_with_steady_phase( - cb.phase_dependencies, cb.statements) + code = DAGCode.create_with_steady_phase(cb.statements) result = execute_and_return_single_result(python_method_impl, code) assert la.norm(result) < 1e-10 @@ -433,7 +420,7 @@ def test_class_preamble(): cb.assign("", " +

") cb.yield_state("f()", "f", 0, "final") - code = DAGCode.create_with_steady_phase(cb.phase_dependencies, cb.statements) + code = DAGCode.create_with_steady_phase(cb.statements) from dagrt.codegen import PythonCodeGenerator import dagrt.function_registry as freg -- GitLab From 1108f06bcfec2e8b21d03859e6416e08b009d26f Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Tue, 12 Nov 2019 00:54:02 -0600 Subject: [PATCH 2/6] Clarify comment --- dagrt/language.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dagrt/language.py b/dagrt/language.py index 3797935..66347ca 100644 --- a/dagrt/language.py +++ b/dagrt/language.py @@ -838,7 +838,8 @@ class CodeBuilder(object): """ # This is a dummy variable name representing the "system state", which is - # used to track the dependencies for instructions having side effects. + # used to track the dependencies for instructions having globally visible + # side effects. _EXECUTION_STATE = "" def __init__(self, label="phase"): -- GitLab From 015950b01c4179cd122d1e389c1ff931f00e6af6 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Tue, 12 Nov 2019 01:08:04 -0600 Subject: [PATCH 3/6] Some fixes to reader_map in the code builder --- dagrt/language.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dagrt/language.py b/dagrt/language.py index 66347ca..1753c36 100644 --- a/dagrt/language.py +++ b/dagrt/language.py @@ -851,7 +851,8 @@ class CodeBuilder(object): # Maps variables to the sequentially last instruction to write them self._writer_map = {} - # Maps variables to the set of instructions that read them + # Maps variables to the set of instructions that read them between the + # last time the var was written and the current instruction self._reader_map = {} # Stack of conditional expressions used to implemented nested ifs self._conditional_expression_stack = [] @@ -917,6 +918,10 @@ class CodeBuilder(object): self._writer_map[var] = stmt_id for var in read_variables: + # reader_map should ignore reads that happen before writes, so + # ignore if this instruction also reads *var*. + if var in written_variables: + continue self._reader_map.setdefault(var, set()).add(stmt_id) stmt = stmt.copy( -- GitLab From d74db6e62fcd7d4409741c8fe5d9bc8f6fa06570 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Tue, 12 Nov 2019 14:32:59 -0600 Subject: [PATCH 4/6] Add a test for write ordering --- test/test_code_builder.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/test_code_builder.py b/test/test_code_builder.py index faec01e..18dc239 100755 --- a/test/test_code_builder.py +++ b/test/test_code_builder.py @@ -66,6 +66,18 @@ def test_CodeBuilder_condition(python_method_impl): assert result == 2 +def test_CodeBuilder_write_ordering(python_method_impl): + with CodeBuilder() as builder: + builder("y", "1") + builder("x", "y") + builder("y", "2") + builder("z", "y") + builder.yield_state(var('x'), 'x', 0, 'final') + code = DAGCode.create_with_steady_phase(builder.statements) + result = execute_and_return_single_result(python_method_impl, code) + assert result == 1 + + def test_CodeBuilder_condition_with_else(python_method_impl): with CodeBuilder() as builder: builder(var('x'), 1) -- GitLab From cad2aa0837a9c37f883dbf47c7bd323f8eb86026 Mon Sep 17 00:00:00 2001 From: Matt Wala Date: Tue, 12 Nov 2019 14:37:42 -0600 Subject: [PATCH 5/6] last_active_conditional_expression -> last_if_block_conditional_expression --- dagrt/language.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dagrt/language.py b/dagrt/language.py index 1753c36..02114f1 100644 --- a/dagrt/language.py +++ b/dagrt/language.py @@ -857,7 +857,7 @@ class CodeBuilder(object): # Stack of conditional expressions used to implemented nested ifs self._conditional_expression_stack = [] # Used to implement if/else - self._last_active_conditional_expression = None + self._last_if_block_conditional_expression = None # Set of seen variables self._seen_var_names = set([self._EXECUTION_STATE]) @@ -988,22 +988,22 @@ class CodeBuilder(object): self._conditional_expression_stack.append(cond_var) yield self._conditional_expression_stack.pop() - self._last_active_conditional_expression = cond_var + self._last_if_block_conditional_expression = cond_var @contextmanager def else_(self): """ Create the "else" portion of a conditionally executed block. """ - assert self._last_active_conditional_expression is not None + assert self._last_if_block_conditional_expression is not None # Create conditions for the context. from pymbolic.primitives import LogicalNot self._conditional_expression_stack.append( - LogicalNot(self._last_active_conditional_expression)) + LogicalNot(self._last_if_block_conditional_expression)) yield self._conditional_expression_stack.pop() - self._last_active_conditional_expression = None + self._last_if_block_conditional_expression = None def assign(self, assignees, expression, loops=[]): """Generate code for an assignment. -- GitLab From 7f9fcdaa2d69030730b863928ee0aef24c019534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Tue, 19 Nov 2019 01:41:44 +0100 Subject: [PATCH 6/6] Fix instruction->statement in comments in dagrt.language --- dagrt/language.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dagrt/language.py b/dagrt/language.py index 02114f1..2a9955f 100644 --- a/dagrt/language.py +++ b/dagrt/language.py @@ -838,7 +838,7 @@ class CodeBuilder(object): """ # This is a dummy variable name representing the "system state", which is - # used to track the dependencies for instructions having globally visible + # used to track the dependencies for statements having globally visible # side effects. _EXECUTION_STATE = "" @@ -849,10 +849,10 @@ class CodeBuilder(object): self.label = label self.statements = [] - # Maps variables to the sequentially last instruction to write them + # Maps variables to the sequentially last statement to write them self._writer_map = {} - # Maps variables to the set of instructions that read them between the - # last time the var was written and the current instruction + # Maps variables to the set of statements that read them between the + # last time the var was written and the current statement self._reader_map = {} # Stack of conditional expressions used to implemented nested ifs self._conditional_expression_stack = [] @@ -919,7 +919,7 @@ class CodeBuilder(object): for var in read_variables: # reader_map should ignore reads that happen before writes, so - # ignore if this instruction also reads *var*. + # ignore if this statement also reads *var*. if var in written_variables: continue self._reader_map.setdefault(var, set()).add(stmt_id) -- GitLab