diff --git a/java/ql/src/Stubs/MinimalStubsFromSource.ql b/java/ql/src/Stubs/MinimalStubsFromSource.ql new file mode 100644 index 00000000000..52e37e27409 --- /dev/null +++ b/java/ql/src/Stubs/MinimalStubsFromSource.ql @@ -0,0 +1,27 @@ +/** + * Tool to generate C# stubs from a qltest snapshot. + * + * It finds all declarations used in the source code, + * and generates minimal C# stubs containing those declarations + * and their dependencies. + */ + +import java +import Stubs + +/** Declarations used by source code. */ +class UsedInSource extends GeneratedDeclaration { + UsedInSource() { + ( + this = any(Variable v | v.fromSource()).getType() + or + this = any(Expr e | e.getEnclosingCallable().fromSource()).getType() + or + this = any(RefType t | t.fromSource()) + ) + } +} + +from GeneratedTopLevel t +where not t.fromSource() +select t.getQualifiedName(), t.stubFile() diff --git a/java/ql/src/Stubs/Stubs.qll b/java/ql/src/Stubs/Stubs.qll new file mode 100644 index 00000000000..6fca20d5407 --- /dev/null +++ b/java/ql/src/Stubs/Stubs.qll @@ -0,0 +1,377 @@ +/** + * Generates java stubs for use in test code. + * + * Extend the abstract class `GeneratedDeclaration` with the declarations that should be generated. + * This will generate stubs for all the required dependencies as well. + */ + +import java + +/** A type that should be in the generated code. */ +abstract /*private*/ class GeneratedType extends RefType { + GeneratedType() { + ( + this instanceof Interface + or + this instanceof Class + ) and + not this instanceof AnonymousClass and + not this instanceof LocalClass and + not this.getPackage() instanceof ExcludedPackage + } + + private string stubKeyword() { + this instanceof Interface and result = "interface" + or + this instanceof Class and result = "class" + // or + // this instanceof Enum and result = "enum" + } + + private string stubAbstractModifier() { + if this.(Class).isAbstract() then result = "abstract " else result = "" + } + + private string stubStaticModifier() { + if this.isStatic() then result = "static " else result = "" + } + + private string stubAccessibilityModifier() { + if this.isPublic() then result = "public " else result = "" + } + + /** Gets the entire Java stub code for this type. */ + final string getStub() { + result = + this.stubAbstractModifier() + this.stubStaticModifier() + this.stubAccessibilityModifier() + + this.stubKeyword() + " " + this.getName() + stubGenericArguments(this) + + stubBaseTypesString() + "\n{\n" + stubMembers() + "}" + } + + private RefType getAnInterestingBaseType() { + result = this.getASupertype() and + not result instanceof TypeObject and + result.getSourceDeclaration() != this + } + + private string stubBaseTypesString() { + if exists(getAnInterestingBaseType()) + then + exists(string cls, string interface, string int_kw | result = cls + int_kw + interface | + ( + if exists(getAnInterestingBaseType().(Class)) + then cls = " extends " + stubTypeName(getAnInterestingBaseType()) + else cls = "" + ) and + ( + if exists(getAnInterestingBaseType().(Interface)) + then ( + (if this instanceof Class then int_kw = " implements " else int_kw = " extends ") and + interface = concat(stubTypeName(getAnInterestingBaseType().(Interface)), ", ") + ) else ( + int_kw = "" and interface = "" + ) + ) + ) + else result = "" + } + + language[monotonicAggregates] + private string stubMembers() { + result = concat(Member m | m = this.getAGeneratedMember() | stubMember(m)) + } + + private Member getAGeneratedMember() { + ( + result.getDeclaringType() = this + or + exists(NestedType nt | result = nt | + nt = nt.getSourceDeclaration() and + nt.getEnclosingType().getSourceDeclaration() = this + ) + ) and + not result.isPrivate() and + not result instanceof StaticInitializer and + not result instanceof InstanceInitializer + } + + final Type getAGeneratedType() { + result = getAnInterestingBaseType() + or + result = getAGeneratedMember().(Callable).getReturnType() + or + result = getAGeneratedMember().(Callable).getAParameter().getType() + or + result = getAGeneratedMember().(Field).getType() + or + result = getAGeneratedMember().(NestedType) + } +} + +/** + * A declaration that should be generated. + * This is extended in client code to identify the actual + * declarations that should be generated. + */ +abstract class GeneratedDeclaration extends Element { } + +private class IndirectType extends GeneratedType { + IndirectType() { + this.getASubtype() instanceof GeneratedType + or + this.(GenericType).getAParameterizedType() instanceof GeneratedType + or + exists(GeneratedType t | + this = getAContainedType(t.getAGeneratedType()).(RefType).getSourceDeclaration() + ) + or + exists(GeneratedDeclaration decl | + decl.(Member).getDeclaringType().getSourceDeclaration() = this + ) + or + this.(NestedType).getEnclosingType() instanceof GeneratedType + or + exists(NestedType nt | nt instanceof GeneratedType and this = nt.getEnclosingType()) + } +} + +private class RootGeneratedType extends GeneratedType { + RootGeneratedType() { this = any(GeneratedDeclaration decl).(RefType).getSourceDeclaration() } +} + +private Type getAContainedType(Type t) { + result = t + or + result = getAContainedType(t.(ParameterizedType).getATypeArgument()) +} + +/** + * Specify packages to exclude. + * Do not generate any types from these packages. + */ +abstract class ExcludedPackage extends Package { } + +/** Exclude types from the standard library. */ +private class DefaultLibs extends ExcludedPackage { + DefaultLibs() { this.getName().matches(["java.%", "javax.%", "jdk.%", "sun.%"]) } +} + +private string stubAccessibility(Member m) { + if m.getDeclaringType() instanceof Interface + then result = "" + else + if m.isPublic() + then result = "public " + else + if m.isProtected() + then result = "protected " + else + if m.isPrivate() + then result = "private " + else + if m.isPackageProtected() + then result = "" + else result = "unknown-accessibility" +} + +private string stubModifiers(Member m) { + result = stubAccessibility(m) + stubStaticOrFinal(m) + stubAbstract(m) +} + +private string stubStaticOrFinal(Member m) { + if m.(Modifiable).isStatic() + then result = "static " + else + if m.(Modifiable).isFinal() + then result = "final " + else result = "" +} + +private string stubAbstract(Member m) { + if m.getDeclaringType() instanceof Interface + then result = "" + else + if m.isAbstract() + then result = "abstract " + else result = "" +} + +private string stubTypeName(Type t) { + if t instanceof PrimitiveType + then result = t.getName() + else + if t instanceof VoidType + then result = "void" + else + if t instanceof TypeVariable + then result = t.getName() + else + if t instanceof Array + then result = stubTypeName(t.(Array).getElementType()) + "[]" + else + if t instanceof RefType + then + result = + stubQualifier(t) + t.(RefType).getSourceDeclaration().getName() + + stubGenericArguments(t) + else result = "" +} + +private string stubQualifier(RefType t) { + if t instanceof NestedType + then result = stubTypeName(t.(NestedType).getEnclosingType()) + "." + else result = "" +} + +language[monotonicAggregates] +private string stubGenericArguments(RefType t) { + if t instanceof GenericType + then + result = + "<" + + concat(int n | + exists(t.(GenericType).getTypeParameter(n)) + | + t.(GenericType).getTypeParameter(n).getName(), "," order by n + ) + ">" + else + if t instanceof ParameterizedType + then + result = + "<" + + concat(int n | + exists(t.(ParameterizedType).getTypeArgument(n)) + | + stubTypeName(t.(ParameterizedType).getTypeArgument(n)), "," order by n + ) + ">" + else result = "" +} + +private string stubGenericMethodParams(Method m) { + if m instanceof GenericMethod + then + result = + " <" + + concat(int n, TypeVariable param | + param = m.(GenericMethod).getTypeParameter(n) + | + param.getName(), "," order by n + ) + "> " + else result = "" +} + +private string stubImplementation(Callable c) { + if c.isAbstract() or c.getDeclaringType() instanceof Interface + then result = ";" + else + if c instanceof Constructor or c.getReturnType() instanceof VoidType + then result = "{}" + else result = "{ return " + stubDefaultValue(c.getReturnType()) + "; }" +} + +private string stubDefaultValue(Type t) { + if t instanceof RefType + then result = "null" + else + if t instanceof CharacterType + then result = "'0'" + else + if t instanceof BooleanType + then result = "false" + else + if t instanceof NumericType + then result = "0" + else result = "" +} + +private string stubParameters(Callable c) { + result = + concat(int i, Parameter param | + param = c.getParameter(i) + | + stubParameter(param), ", " order by i + ) +} + +private string stubParameter(Parameter p) { + exists(Type t, string suff | result = stubTypeName(t) + suff + " " + p.getName() | + if p.isVarargs() + then ( + t = p.getType().(Array).getElementType() and + suff = "..." + ) else ( + t = p.getType() and suff = "" + ) + ) +} + +private string stubMember(Member m) { + exists(Method c | m = c | + result = + " " + stubModifiers(c) + stubGenericMethodParams(c) + stubTypeName(c.getReturnType()) + " " + + c.getName() + "(" + stubParameters(c) + ")" + stubImplementation(c) + "\n" + ) + or + exists(Constructor c | m = c | + result = + " " + stubModifiers(m) + c.getName() + "(" + stubParameters(c) + ")" + + stubImplementation(c) + "\n" + ) + or + exists(Field f, string impl | f = m | + /* and not f instanceof EnumConstant */ + /*if f.isConst() then impl = " = throw null" else*/ impl = "" and + result = + " " + stubModifiers(m) + stubTypeName(f.getType()) + " " + f.getName() + impl + ";\n" + ) + or + exists(NestedType nt | nt = m | result = indent(nt.(GeneratedType).getStub())) +} + +bindingset[s] +private string indent(string s) { result = " " + s.replaceAll("\n", "\n ") + "\n" } + +private TopLevelType getTopLevel(RefType t) { + result = t or + result = getTopLevel(t.(NestedType).getEnclosingType()) +} + +class GeneratedTopLevel extends TopLevelType { + GeneratedTopLevel() { + this = this.getSourceDeclaration() and + this instanceof GeneratedType + } + + RefType getAReferencedType() { + exists(GeneratedType t | this = getTopLevel(t) | result = getTopLevel(t.getAGeneratedType())) + } + + private string stubAnImport() { + exists(RefType t, string pkg, string name | + t = getAReferencedType().getSourceDeclaration() and + (t instanceof Class or t instanceof Interface) and + t.hasQualifiedName(pkg, name) and + t != this and + pkg != "java.lang" + | + result = "import " + pkg + "." + name + ";\n" + ) + } + + private string stubImports() { result = concat(stubAnImport()) + "\n" } + + private string stubPackage() { + if this.getPackage().getName() != "" + then result = "package " + this.getPackage().getName() + ";\n\n" + else result = "" + } + + private string stubComment() { + result = + "// Generated automatically from " + this.getQualifiedName() + " for testing purposes\n\n" + } + + string stubFile() { + result = stubComment() + stubPackage() + stubImports() + this.(GeneratedType).getStub() + "\n" + } +} diff --git a/java/ql/src/Stubs/make_stubs.py b/java/ql/src/Stubs/make_stubs.py new file mode 100644 index 00000000000..7e7331405e0 --- /dev/null +++ b/java/ql/src/Stubs/make_stubs.py @@ -0,0 +1,155 @@ +# Tool to generate Java stubs for qltests + +import sys +import os +import subprocess +import json + + +def print_usage(exit_code=1): + print("Usage: python3 make_stubs.py testDir stubDir\n", + "testDir: the directory containing the qltest to be stubbed. Should contain an `options0` file pointing to the jars to stub, and an `options1` file pointing to `stubdir`\n", + "stubDir: the directory to output the generated stubs to") + exit(exit_code) + + +if "--help" in sys.argv or "-h" in sys.argv: + print_usage(0) + +if len(sys.argv) != 3: + print_usage() + +testDir = sys.argv[1].rstrip("/") +stubDir = sys.argv[2].rstrip("/") + + +def check_dir_exists(path): + if not os.path.isdir(path): + print("Directory", path, "does not exist") + exit(1) + + +def check_file_exists(path): + if not os.path.isfile(path): + print("File", path, "does not exist") + exit(1) + + +def copy_file(src, dest): + with open(src) as srcf: + with open(dest, "w") as destf: + destf.write(srcf.read()) + + +check_dir_exists(testDir) +check_dir_exists(stubDir) + +optionsFile = os.path.join(testDir, "options") +options0File = os.path.join(testDir, "options0") +options1File = os.path.join(testDir, "options1") + +check_file_exists(options0File) +check_file_exists(options1File) + +# Does it contain a .ql file and a .java file? + +foundJava = False +foundQL = False + +for file in os.listdir(testDir): + if file.endswith(".java"): + foundJava = True + if file.endswith(".ql") or file.endswith(".qlref"): + foundQL = True + +if not foundQL: + print("Test directory does not contain .ql files. Please specify a working qltest directory.") + exit(1) + +if not foundJava: + print("Test directory does not contain .java files. Please specify a working qltest directory.") + exit(1) + + +javaQueries = os.path.abspath(os.path.dirname(sys.argv[0])) +outputBqrsFile = os.path.join(testDir, 'output.bqrs') +outputJsonFile = os.path.join(testDir, 'output.json') + +dbDir = os.path.join(testDir, os.path.basename(testDir) + ".testproj") + + +def print_javac_output(): + logDir = os.path.join(dbDir, "log") + if not os.path.isdir(logDir): + print("No database logs found") + return + + logFile = None + for file in os.listdir(logDir): + if file.startswith("javac-output"): + logFile = os.path.join(logDir, file) + break + else: + print("No javac output found") + + print("\nJavac output:\n") + + with open(logFile) as f: + for line in f: + b1 = line.find(']') + b2 = line.find(']', b1+1) + print(line[b2+2:], end="") + + +print("Stubbing qltest in", testDir) + +copy_file(options0File, optionsFile) + +cmd = ['codeql', 'test', 'run', '--keep-databases', testDir] +print('Running ' + ' '.join(cmd)) +if subprocess.call(cmd): + print_javac_output() + print("codeql test failed. Please fix up the test before proceeding.") + exit(1) + +if not os.path.isdir(dbDir): + print("Expected database directory " + dbDir + " not found.") + exit(1) + +cmd = ['codeql', 'query', 'run', os.path.join( + javaQueries, 'MinimalStubsFromSource.ql'), '--database', dbDir, '--output', outputBqrsFile] +print('Running ' + ' '.join(cmd)) +if subprocess.call(cmd): + print('Failed to run the query to generate the stubs.') + exit(1) + +cmd = ['codeql', 'bqrs', 'decode', outputBqrsFile, + '--format=json', '--output', outputJsonFile] +print('Running ' + ' '.join(cmd)) +if subprocess.call(cmd): + print('Failed to convert ' + outputBqrsFile + ' to JSON.') + exit(1) + +with open(outputJsonFile) as f: + results = json.load(f) + +for (typ, stub) in results['#select']['tuples']: + stubFile = os.path.join(stubDir, typ.replace(".", "/") + ".java") + os.makedirs(os.path.dirname(stubFile), exist_ok=True) + with open(stubFile, "w") as f: + f.write(stub) + +print("Verifying stub correctness") + +copy_file(options1File, optionsFile) + +cmd = ['codeql', 'test', 'run', testDir] +print('Running ' + ' '.join(cmd)) +if subprocess.call(cmd): + print_javac_output() + print('\nTest failed. You may need to fix up the generated stubs.') + exit(1) + +print("\nStub generation successful!") + +exit(0)