Python: Check that tests are valid

python/ql/test/experimental/dataflow/coverage/test.py

@@ -6,19 +6,27 @@
 #
 # Functions whose name ends with "_with_local_flow" will also be tested for local flow.
 #
-# All functions starting with "test_" should run and either
-#  - print a source (sources are defined in testConfig.qll).
-#  - print "Unexpected flow: " and a non-source
-# (The idea is to later write a script to autimatically confirm this.)
+# All functions starting with "test_" should run and print `"OK"`.
+# This can be checked by running validTest.py.
 
 # These are defined so that we can evaluate the test code.
 NONSOURCE = "not a source"
 SOURCE = "source"
+
+def is_source(x):
+    return x == "source" or x == b"source" or x == 42 or x == 42.0 or x == 42j
+
 def SINK(x):
-    print(x)
+    if is_source(x):
+        print("OK")
+    else:
+        print("Unexpected flow", x)
 
 def SINK_F(x):
-    print("Unexpected flow: ", x)
+    if is_source(x):
+        print("Unexpected flow", x)
+    else:
+        print("OK")
 
 def test_tuple_with_local_flow():
     x = (NONSOURCE, SOURCE)

python/ql/test/experimental/dataflow/coverage/validTest.py

@@ -0,0 +1,33 @@
+def check_output(s, f):
+    if s == "OK\n":
+        pass
+    else:
+        raise RuntimeError("Function failed", s, f)
+
+def check_test_function(f):
+    from io import StringIO
+    import sys
+
+    capturer = StringIO()
+    old_stdout = sys.stdout
+    sys.stdout = capturer
+    f()
+    sys.stdout = old_stdout
+    check_output(capturer.getvalue(), f)
+
+def check_classes_valid(testFile):
+    # import python.ql.test.experimental.dataflow.coverage.classes as tests
+    # import classes as tests
+    import importlib
+    tests = importlib.import_module(testFile)
+    for i in dir(tests):
+        # print("Considering", i)
+        if i.startswith("test_"):
+            item = getattr(tests,i)
+            if callable(item):
+                print("Checking", testFile, item)
+                check_test_function(item)
+
+if __name__ == '__main__':
+    check_classes_valid("classes")
+    check_classes_valid("test")