diff --git a/java/kotlin-extractor/generate_dbscheme.py b/java/kotlin-extractor/generate_dbscheme.py new file mode 100755 index 00000000000..2f75bc12212 --- /dev/null +++ b/java/kotlin-extractor/generate_dbscheme.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +import re +import sys + +def upperFirst(string): + return string[0].upper() + string[1:] + +with open('../ql/src/config/semmlecode.dbscheme', 'r') as f: + dbscheme = f.read() + +# Remove comments +dbscheme = re.sub(r'/\*.*?\*/', '', dbscheme, flags=re.DOTALL) +dbscheme = re.sub(r'//[^\r\n]*/', '', dbscheme) + +type_hierarchy = {} + +with open('src/main/kotlin/KotlinExtractorDbScheme.kt', 'w') as kt: + kt.write('/* Generated by ' + sys.argv[0] + ': Do not edit manually. */\n') + kt.write('package com.github.codeql\n') + + kt.write('class Label(val name: Int) {\n') + kt.write(' override fun toString(): String = "#$name"\n') + kt.write('}\n') + + + # kind enums + for name, kind, body in re.findall(r'case\s+@([^.\s]*)\.([^.\s]*)\s+of\b(.*?);', + dbscheme, + flags=re.DOTALL): + for num, typ in re.findall(r'(\d+)\s*=\s*@(\S+)', body): + s = type_hierarchy.get(typ, set()) + s.add(name) + type_hierarchy[typ] = s + + # unions + for name, unions in re.findall(r'@(\w+)\s*=\s*(@\w+(?:\s*\|\s*@\w+)*)', + dbscheme, + flags=re.DOTALL): + type_hierarchy[name] = type_hierarchy.get(name, set()) + for typ in re.findall(r'@(\w+)', unions): + s = type_hierarchy.get(typ, set()) + s.add(name) + type_hierarchy[typ] = s + kt.write('\n') + + # tables + for relname, body in re.findall('\n([\w_]+)(\([^)]*\))', + dbscheme, + flags=re.DOTALL): + for db_type in re.findall(':\s*@([^\s,]+)\s*(?:,|$)', body): + type_hierarchy[db_type] = type_hierarchy.get(db_type, set()) + kt.write('fun write' + upperFirst(relname) + '(data: TrapWriter, ') + for colname, db_type in re.findall('(\S+)\s*:\s*([^\s,]+)', body): + kt.write(colname + ': ') + if db_type == 'int': + # TODO: Do something better if the column is a 'case' + kt.write('Int') + elif db_type == 'float': + kt.write('Double') + elif db_type == 'string': + kt.write('String') + elif db_type == 'date': + kt.write('String') + elif db_type == 'boolean': + kt.write('Boolean') + elif db_type[0] == '@': + kt.write('Label') + else: + raise Exception('Bad db_type: ' + db_type) + kt.write(', ') + kt.write(') {\n') + kt.write(' data.writeTrap("' + relname + '(') + comma = '' + for colname, db_type in re.findall('(\S+)\s*:\s*([^\s,]+)', body): + kt.write(comma) + if db_type == 'string' or db_type == 'date': + kt.write('\\"$' + colname + '\\"') # TODO: Escaping + else: + # TODO: Any reformatting or escaping necessary? + # e.g. float formats? + kt.write('$' + colname) + comma = ', ' + kt.write(')\\n")\n') + kt.write('}\n') + + for typ in type_hierarchy: + kt.write('sealed interface Db' + upperFirst(typ)) + names = type_hierarchy[typ] + if names: + kt.write(': ') + kt.write(', '.join(map(lambda name: 'Db' + upperFirst(name), type_hierarchy[typ]))) + kt.write(' {}\n') + diff --git a/java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt b/java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt index 5844263577f..283853ac5c9 100644 --- a/java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt +++ b/java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt @@ -36,26 +36,26 @@ fun extractorBug(msg: String) { interface TrapWriter { fun writeTrap(trap: String) - fun getLocation(startOffset: Int, endOffset: Int): Int - fun getIdFor(label: String): Int - fun getFreshId(): Int + fun getLocation(startOffset: Int, endOffset: Int): Label + fun getIdFor(label: String): Label + fun getFreshId(): Label } class RootTrapWriter: TrapWriter { override fun writeTrap(trap: String) { extractorBug("Tried to write TRAP outside a file: $trap") } - override fun getLocation(startOffset: Int, endOffset: Int): Int { + override fun getLocation(startOffset: Int, endOffset: Int): Label { extractorBug("Asked for location, but not in a file") - return 0 + return Label(0) } - override fun getIdFor(label: String): Int { + override fun getIdFor(label: String): Label { extractorBug("Asked for ID for '$label' outside a file") - return 0 + return Label(0) } - override fun getFreshId(): Int { + override fun getFreshId(): Label { extractorBug("Asked for fresh ID outside a file") - return 0 + return Label(0) } } @@ -68,30 +68,30 @@ class FileTrapWriter( override fun writeTrap(trap: String) { file.write(trap) } - override fun getLocation(startOffset: Int, endOffset: Int): Int { + override fun getLocation(startOffset: Int, endOffset: Int): Label { val startLine = fileEntry.getLineNumber(startOffset) + 1 val startColumn = fileEntry.getColumnNumber(startOffset) + 1 val endLine = fileEntry.getLineNumber(endOffset) + 1 val endColumn = fileEntry.getColumnNumber(endOffset) - val id = getFreshId() - val fileId = getIdFor(fileLabel) - writeTrap("#$id = @\"loc,{#$fileId},$startLine,$startColumn,$endLine,$endColumn\"\n") - writeTrap("locations_default(#$id, #$fileId, $startLine, $startColumn, $endLine, $endColumn)\n") + val id: Label = getFreshId() + val fileId: Label = getIdFor(fileLabel) + writeTrap("$id = @\"loc,{$fileId},$startLine,$startColumn,$endLine,$endColumn\"\n") + writeLocations_default(this, id, fileId, startLine, startColumn, endLine, endColumn) return id } - val labelMapping: MutableMap = mutableMapOf() - override fun getIdFor(label: String): Int { + val labelMapping: MutableMap> = mutableMapOf>() + override fun getIdFor(label: String): Label { val maybeId = labelMapping.get(label) if(maybeId == null) { - val id = getFreshId() + val id: Label = getFreshId() labelMapping.put(label, id) return id } else { - return maybeId + return maybeId as Label } } - override fun getFreshId(): Int { - return nextId++ + override fun getFreshId(): Label { + return Label(nextId++) } } @@ -104,16 +104,16 @@ class KotlinExtractorVisitor(val trapDir: File, val srcDir: File) : IrElementVis element.acceptChildren(this, data) } override fun visitClass(declaration: IrClass, data: TrapWriter) { - val id = data.getFreshId() - val pkgId = data.getFreshId() + val id: Label = data.getFreshId() + val pkgId: Label = data.getFreshId() val locId = data.getLocation(declaration.startOffset, declaration.endOffset) val pkg = declaration.packageFqName?.asString() ?: "" val cls = declaration.name.asString() - data.writeTrap("#$pkgId = @\"pkg;$pkg\"\n") - data.writeTrap("packages(#$pkgId, \"$pkg\")\n") - data.writeTrap("#$id = @\"class;$pkg.$cls\"\n") - data.writeTrap("classes(#$id, \"$cls\", #$pkgId, #$id)\n") - data.writeTrap("hasLocation(#$id, #$locId)\n") + data.writeTrap("$pkgId = @\"pkg;$pkg\"\n") + writePackages(data, pkgId, pkg) + data.writeTrap("$id = @\"class;$pkg.$cls\"\n") + writeClasses(data, id, cls, pkgId, id) + writeHasLocation(data, id, locId) declaration.acceptChildren(this, data) } override fun visitFile(declaration: IrFile, data: TrapWriter) { @@ -132,9 +132,9 @@ class KotlinExtractorVisitor(val trapDir: File, val srcDir: File) : IrElementVis trapFileDir.mkdirs() trapFile.bufferedWriter().use { trapFileBW -> val tw = FileTrapWriter(fileLabel, trapFileBW, declaration.fileEntry) - val id = tw.getIdFor(fileLabel) - tw.writeTrap("#$id = $fileLabel\n") - tw.writeTrap("files(#$id, \"$filePath\", \"$basename\", \"$extension\", 0)\n") + val id: Label = tw.getIdFor(fileLabel) + tw.writeTrap("$id = $fileLabel\n") + writeFiles(tw, id, filePath, basename, extension, 0) declaration.acceptChildren(this, tw) } }