diff --git a/loopy/compiled.py b/loopy/compiled.py
index 563c676d5be9faea97ed89328ff23d1001386727..55613fc9aa38cde9cb40cc66ddded2f2ea19d864 100644
--- a/loopy/compiled.py
+++ b/loopy/compiled.py
@@ -493,6 +493,17 @@ def auto_test_vs_ref(ref_knl, ctx, kernel_gen, op_count=[], op_label=[], paramet
         warn("op_label should be a list", stacklevel=2)
         op_label = [op_label]
 
+    read_and_written_args = (
+            ref_knl.get_read_variables()
+            & ref_knl.get_written_variables()
+            & set(ref_knl.arg_dict))
+
+    if read_and_written_args:
+        # FIXME: In principle, that's possible to test
+        raise RuntimeError("kernel reads *and* writes argument(s) '%s' "
+                "and therefore cannot be automatically tested"
+                % ", ".join(read_and_written_args))
+
     from time import time
 
     if check_result is None:
diff --git a/loopy/kernel.py b/loopy/kernel.py
index 0d3d74d2fe1603e8b0b51d12c64479f58b1eaf32..c62eaf3cda3fb80fa31668f538de4faac1c7415d 100644
--- a/loopy/kernel.py
+++ b/loopy/kernel.py
@@ -1395,9 +1395,16 @@ class LoopKernel(Record):
 
         return result
 
+    @memoize_method
+    def get_read_variables(self):
+        result = set()
+        for insn in self.instructions:
+            result.update(insn.get_read_var_names())
+        return result
+
     @memoize_method
     def get_written_variables(self):
-        return set(
+        return frozenset(
             insn.get_assignee_var_name()
             for insn in self.instructions)