From 155da0b2433e1d55c103b66b9a7fe4ead3e4934d Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Wed, 28 Aug 2024 14:17:56 +0100 Subject: [PATCH] KE2: test-kotlin2/library-tests/files now has no consistency failures --- java/kotlin-extractor2/BUILD.bazel | 35 +++++ java/kotlin-extractor2/ke2.sh | 8 +- .../src/main/kotlin/KotlinExtractor.kt | 124 +++++++++++------- .../src/main/kotlin/Label.kt | 6 - .../src/main/kotlin/TrapWriter.kt | 46 +++++-- .../src/main/kotlin/utils/Logger.kt | 29 +++- 6 files changed, 176 insertions(+), 72 deletions(-) diff --git a/java/kotlin-extractor2/BUILD.bazel b/java/kotlin-extractor2/BUILD.bazel index 0c2b184a758..1a90d638599 100644 --- a/java/kotlin-extractor2/BUILD.bazel +++ b/java/kotlin-extractor2/BUILD.bazel @@ -30,6 +30,41 @@ kt_jvm_library( srcs = [ ":generated-dbscheme", "src/main/java/com/semmle/util/unicode/UTF8Util.java", + "src/main/java/com/semmle/extractor/java/PopulateFile.java", + "src/main/java/com/semmle/util/array/ArrayUtil.java", + "src/main/java/com/semmle/util/basic/ObjectUtil.java", + "src/main/java/com/semmle/util/data/IntRef.java", + "src/main/java/com/semmle/util/data/Pair.java", + "src/main/java/com/semmle/util/data/StringUtil.java", + "src/main/java/com/semmle/util/data/Tuple1.java", + "src/main/java/com/semmle/util/data/Tuple2.java", + "src/main/java/com/semmle/util/data/TupleN.java", + "src/main/java/com/semmle/util/exception/CatastrophicError.java", + "src/main/java/com/semmle/util/exception/Exceptions.java", + "src/main/java/com/semmle/util/exception/InterruptedError.java", + "src/main/java/com/semmle/util/exception/NestedError.java", + "src/main/java/com/semmle/util/exception/ResourceError.java", + "src/main/java/com/semmle/util/exception/UserError.java", + "src/main/java/com/semmle/util/expansion/ExpansionEnvironment.java", + "src/main/java/com/semmle/util/files/FileUtil.java", + "src/main/java/com/semmle/util/files/PathMatcher.java", + "src/main/java/com/semmle/util/io/BufferedLineReader.java", + "src/main/java/com/semmle/util/io/csv/CSVParser.java", + "src/main/java/com/semmle/util/io/csv/CSVReader.java", + "src/main/java/com/semmle/util/io/csv/CSVWriter.java", + "src/main/java/com/semmle/util/io/RawStreamMuncher.java", + "src/main/java/com/semmle/util/io/StreamMuncher.java", + "src/main/java/com/semmle/util/io/StreamUtil.java", + "src/main/java/com/semmle/util/io/WholeIO.java", + "src/main/java/com/semmle/util/logging/Streams.java", + "src/main/java/com/semmle/util/process/AbstractProcessBuilder.java", + "src/main/java/com/semmle/util/process/Builder.java", + "src/main/java/com/semmle/util/process/Env.java", + "src/main/java/com/semmle/util/process/LeakPrevention.java", + "src/main/java/com/semmle/util/projectstructure/ProjectLayout.java", + "src/main/java/com/semmle/util/trap/pathtransformers/NoopTransformer.java", + "src/main/java/com/semmle/util/trap/pathtransformers/PathTransformer.java", + "src/main/java/com/semmle/util/trap/pathtransformers/ProjectLayoutTransformer.java", ] + glob(["src/main/kotlin/**/*.kt"]), javac_opts = ":javac-options", kotlinc_opts = ":kotlinc-options", diff --git a/java/kotlin-extractor2/ke2.sh b/java/kotlin-extractor2/ke2.sh index e70a570597e..1dc21922846 100755 --- a/java/kotlin-extractor2/ke2.sh +++ b/java/kotlin-extractor2/ke2.sh @@ -17,7 +17,13 @@ echo "// Invocation of Kotlin Extractor 2" >> "$INVOCATION_TRAP" echo "#compilation = *" >> "$INVOCATION_TRAP" # TODO: This should be properly escaped: echo "compilations(#compilation, 2, \"`pwd`\",\"$INVOCATION_TRAP\")" >> "$INVOCATION_TRAP" -# TODO: compilation_args +ARG_INDEX=0 +for ARG in "$@" +do + # TODO: This should be properly escaped: + echo "compilation_args(#compilation, $ARG_INDEX, \"$ARG\")" >> "$INVOCATION_TRAP" + ARG_INDEX=$(("$ARG_INDEX" + 1)) +done if [[ -n "$CODEQL_JAVA_HOME" ]]; then JAVA="$CODEQL_JAVA_HOME/bin/java" diff --git a/java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt b/java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt index 2e5cd9ee82a..e64e7aa93b2 100644 --- a/java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt +++ b/java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt @@ -133,7 +133,7 @@ OLD: KE1 srcDir.mkdirs() val globalExtensionState = KotlinExtractorGlobalState() */ - doAnalysis(kotlinArgs) + doAnalysis(dtw, compilation, invocationExtractionProblems, kotlinArgs) /* OLD: KE1 loggerBase.printLimitedDiagnosticCounts(tw) @@ -153,40 +153,7 @@ OLD: KE1 } } -fun doAnalysis(args : List) { -/* -OLD: KE1 - moduleFragment.files.mapIndexed { index: Int, file: IrFile -> - val fileExtractionProblems = FileExtractionProblems(invocationExtractionProblems) - val fileTrapWriter = tw.makeSourceFileTrapWriter(file, true) - loggerBase.setFileNumber(index) - fileTrapWriter.writeCompilation_compiling_files( - compilation, - index, - fileTrapWriter.fileId - ) - doFile( - compression, - fileExtractionProblems, - invocationTrapFile, - fileTrapWriter, - checkTrapIdentical, - loggerBase, - trapDir, - srcDir, - file, - primitiveTypeMapping, - pluginContext, - globalExtensionState - ) - fileTrapWriter.writeCompilation_compiling_files_completed( - compilation, - index, - fileExtractionProblems.extractionResult() - ) - } -*/ - +fun doAnalysis(dtw : DiagnosticTrapWriter, compilation: Label, invocationExtractionProblems : ExtractionProblems, args : List) { lateinit var sourceModule: KaSourceModule val k2args : K2JVMCompilerArguments = parseCommandLineArguments(args.toList()) @@ -215,20 +182,52 @@ OLD: KE1 } val psiFiles = session.modulesWithFiles.getValue(sourceModule) +/* +OLD: KE1 + moduleFragment.files.mapIndexed { index: Int, file: IrFile -> +*/ + var fileIndex = 0 for (psiFile in psiFiles) { if (psiFile is KtFile) { analyze(psiFile) { - val c = psiFile.getDeclarations()[0] - if (c is KtClass) { - for (d: KtDeclaration in c.getDeclarations()) { - if (d is KtFunction) { - if (d.name == "f") { - dumpFunction(d) - } - } - } - } + val fileExtractionProblems = FileExtractionProblems(invocationExtractionProblems) + val fileDiagnosticTrapWriter = dtw.makeSourceFileTrapWriter(psiFile, true) +/* +OLD: KE1 + loggerBase.setFileNumber(fileIndex) +*/ + fileDiagnosticTrapWriter.writeCompilation_compiling_files( + compilation, + fileIndex, + fileDiagnosticTrapWriter.fileId + ) + doFile( +/* +OLD: KE1 + compression, + fileExtractionProblems, + invocationTrapFile, + fileDiagnosticTrapWriter, + checkTrapIdentical, + loggerBase, + trapDir, + srcDir, +*/ + psiFile, +/* +OLD: KE1 + primitiveTypeMapping, + pluginContext, + globalExtensionState +*/ + ) + fileDiagnosticTrapWriter.writeCompilation_compiling_files_completed( + compilation, + fileIndex, + fileExtractionProblems.extractionResult() + ) } + fileIndex += 1 } else { System.out.println("Warning: Not a KtFile") } @@ -423,8 +422,6 @@ open class ExtractionProblems { } } -/* -OLD: KE1 /* The `FileExtractionProblems` is analogous to `ExtractionProblems`, except it records whether there were any problems while extracting a @@ -443,6 +440,8 @@ class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblem } } +/* +OLD: KE1 /* This function determines whether 2 TRAP files should be considered to be equivalent. It returns `true` iff all of their non-comment lines are @@ -467,23 +466,45 @@ private fun equivalentTrap(r1: BufferedReader, r2: BufferedReader): Boolean { } } } +*/ +context (KaSession) private fun doFile( +/* +OLD: KE1 compression: Compression, fileExtractionProblems: FileExtractionProblems, invocationTrapFile: String, - fileTrapWriter: FileTrapWriter, + fileDiagnosticTrapWriter: FileTrapWriter, checkTrapIdentical: Boolean, loggerBase: LoggerBase, dbTrapDir: File, dbSrcDir: File, - srcFile: IrFile, +*/ + srcFile: KtFile, +/* +OLD: KE1 primitiveTypeMapping: PrimitiveTypeMapping, pluginContext: IrPluginContext, globalExtensionState: KotlinExtractorGlobalState +*/ ) { + // TODO: Testing + val c = srcFile.getDeclarations()[0] + if (c is KtClass) { + for (d: KtDeclaration in c.getDeclarations()) { + if (d is KtFunction) { + if (d.name == "f") { + dumpFunction(d) + } + } + } + } + +/* +OLD: KE1 val srcFilePath = srcFile.path - val logger = FileLogger(loggerBase, fileTrapWriter) + val logger = FileLogger(loggerBase, fileDiagnosticTrapWriter) logger.info("Extracting file $srcFilePath") logger.flush() @@ -579,8 +600,11 @@ private fun doFile( fileExtractionProblems.setNonRecoverableProblem() } } +*/ } +/* +OLD: KE1 enum class Compression(val extension: String) { NONE("") { override fun bufferedWriter(file: File): BufferedWriter { diff --git a/java/kotlin-extractor2/src/main/kotlin/Label.kt b/java/kotlin-extractor2/src/main/kotlin/Label.kt index dabf7ee3456..65d5fe72461 100644 --- a/java/kotlin-extractor2/src/main/kotlin/Label.kt +++ b/java/kotlin-extractor2/src/main/kotlin/Label.kt @@ -2,21 +2,15 @@ package com.github.codeql /** This represents a label (`#...`) in a TRAP file. */ interface Label { -/* -OLD: KE1 fun cast(): Label { @Suppress("UNCHECKED_CAST") return this as Label } -*/ } -/* -OLD: KE1 /** The label `#i`, e.g. `#123`. Most labels we generate are of this form. */ class IntLabel(val i: Int) : Label { override fun toString(): String = "#$i" } -*/ /** * The label `#name`, e.g. `#compilation`. This is used when labels are shared between different diff --git a/java/kotlin-extractor2/src/main/kotlin/TrapWriter.kt b/java/kotlin-extractor2/src/main/kotlin/TrapWriter.kt index 35e723e67a0..fdc7c636d1d 100644 --- a/java/kotlin-extractor2/src/main/kotlin/TrapWriter.kt +++ b/java/kotlin-extractor2/src/main/kotlin/TrapWriter.kt @@ -3,13 +3,13 @@ package com.github.codeql /* OLD: KE1 import com.github.codeql.KotlinUsesExtractor.LocallyVisibleFunctionLabels -import com.semmle.extractor.java.PopulateFile */ +import com.semmle.extractor.java.PopulateFile import com.semmle.util.unicode.UTF8Util import java.io.BufferedWriter +import java.io.File /* OLD: KE1 -import java.io.File import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET import org.jetbrains.kotlin.ir.declarations.IrClass @@ -20,14 +20,13 @@ import org.jetbrains.kotlin.ir.declarations.path import org.jetbrains.kotlin.ir.expressions.IrCall import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET */ +import org.jetbrains.kotlin.psi.* /** * Each `.trap` file has a `TrapLabelManager` while we are writing it. It provides fresh TRAP label * names, and maintains a mapping from keys (`@"..."`) to labels. */ class TrapLabelManager { -/* -OLD: KE1 /** The next integer to use as a label name. */ private var nextInt: Int = 100 @@ -39,6 +38,8 @@ OLD: KE1 /** A mapping from a key (`@"..."`) to the label defined to be that key, if any. */ val labelMapping: MutableMap> = mutableMapOf>() +/* +OLD: KE1 val anonymousTypeMapping: MutableMap = mutableMapOf() val locallyVisibleFunctionLabelMapping: MutableMap = @@ -86,9 +87,12 @@ OLD: KE1 * otherwise. Most users will want to use `getLabelFor` instead, which allows non-existent * labels to be initialised. */ - fun getExistingLabelFor(key: String): Label? { +TODO: Inline this if it can remain private +*/ + private fun getExistingLabelFor(key: String): Label? { return lm.labelMapping.get(key)?.cast() } + /** * Returns the label for the given key, if one exists. Otherwise, a fresh label is bound to that * key, `initialise` is run on it, and it is returned. @@ -107,6 +111,8 @@ OLD: KE1 } } +/* +OLD: KE1 /** Returns a label for a fresh ID (i.e. a new label bound to `*`). */ fun getFreshIdLabel(): Label { val label: Label = lm.getFreshLabel() @@ -136,6 +142,7 @@ OLD: KE1 fun getExistingVariableLabelFor(v: IrVariable): Label? { return variableLabelMapping.get(v) } +*/ /** * This returns a label for the location described by its arguments. Typically users will not @@ -149,11 +156,17 @@ OLD: KE1 endLine: Int, endColumn: Int ): Label { +/* +OLD: HE1 return getLabelFor("@\"loc,{$fileId},$startLine,$startColumn,$endLine,$endColumn\"") { writeLocations_default(it, fileId, startLine, startColumn, endLine, endColumn) } +*/ + TODO() } +/* +OLD: HE1 /** * The label for the 'unknown' file ID. Users will want to use `unknownLocation` instead. This * is lazy, as we don't want to define it in a TRAP file unless the TRAP file actually contains @@ -169,6 +182,7 @@ OLD: KE1 * file unless the TRAP file actually contains something with an 'unknown' location. */ val unknownLocation: Label by lazy { getWholeFileLocation(unknownFileId) } +*/ /** * Returns the label for the file `filePath`. If `populateFileTables` is true, then this also @@ -191,6 +205,8 @@ OLD: KE1 } } +/* +OLD: HE1 /** * If you have an ID for a file, then this gets a label for the location representing the whole * of that file. @@ -271,7 +287,8 @@ OLD: KE1 * Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the * given `IrFile` for locations. */ - fun makeSourceFileTrapWriter(file: IrFile, populateFileTables: Boolean) = +*/ + fun makeSourceFileTrapWriter(file: KtFile, populateFileTables: Boolean) = SourceFileTrapWriter( loggerBase, lm, @@ -280,7 +297,6 @@ OLD: KE1 file, populateFileTables ) -*/ } /* @@ -319,6 +335,7 @@ OLD: KE1 * An ID for the file will be created, and if `populateFileTables` is true then we will also add * rows to the `files` and `folders` tables for it. */ +*/ open class FileTrapWriter( loggerBase: LoggerBase, lm: TrapLabelManager, @@ -335,6 +352,8 @@ open class FileTrapWriter( return dtw } +/* +OLD: KE1 private fun offsetMinOf(default: Int, vararg options: Int?): Int { if (default == UNDEFINED_OFFSET || default == SYNTHETIC_OFFSET) { return default @@ -400,8 +419,11 @@ open class FileTrapWriter( fun getWholeFileLocation(): Label { return getWholeFileLocation(fileId) } +*/ } +/* +OLD: KE1 /** * A `SourceFileTrapWriter` is used when not only do we know which file we are extracting entities * from, but we also have an `IrFileEntry` (from an `IrFile`) which allows us to map byte offsets to @@ -410,15 +432,17 @@ open class FileTrapWriter( * An ID for the file will be created, and if `populateFileTables` is true then we will also add * rows to the `files` and `folders` tables for it. */ +*/ class SourceFileTrapWriter( loggerBase: LoggerBase, lm: TrapLabelManager, bw: BufferedWriter, dtw: DiagnosticTrapWriter, - val irFile: IrFile, + val ktFile: KtFile, populateFileTables: Boolean -) : FileTrapWriter(loggerBase, lm, bw, dtw, irFile.path, populateFileTables) { - +) : FileTrapWriter(loggerBase, lm, bw, dtw, ktFile.virtualFilePath, populateFileTables) { +/* +OLD: KE1 /** * The file entry for the file that we are extracting from. Used to map offsets to line/column * numbers. @@ -490,5 +514,5 @@ class SourceFileTrapWriter( val endColumn = fileEntry.getColumnNumber(e.endOffset) return "file://$filePath:$startLine:$startColumn:$endLine:$endColumn" } -} */ +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt b/java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt index 5fe31b39918..f60ffc2f3ea 100644 --- a/java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt +++ b/java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt @@ -248,15 +248,21 @@ OLD: KE1 ) logStream.write(logMessage.toJsonLine()) } +*/ fun trace(dtw: DiagnosticTrapWriter, msg: String) { if (verbosity >= 4) { +/* +OLD: KE1 val logMessage = LogMessage("TRACE", msg) tw.writeComment(logMessage.toText()) logStream.write(logMessage.toJsonLine()) +*/ } } +/* +OLD: KE1 fun debug(dtw: DiagnosticTrapWriter, msg: String) { if (verbosity >= 4) { val logMessage = LogMessage("DEBUG", msg) @@ -324,8 +330,6 @@ open class Logger(val loggerBase: LoggerBase, val dtw: DiagnosticTrapWriter) { loggerBase.flush() } -/* -OLD: KE1 fun trace(msg: String) { loggerBase.trace(dtw, msg) } @@ -335,9 +339,11 @@ OLD: KE1 } fun debug(msg: String) { +/* +OLD: KE1 loggerBase.debug(dtw, msg) - } */ + } fun info(msg: String) { loggerBase.info(dtw, msg) @@ -348,27 +354,42 @@ OLD: KE1 private fun warn(msg: String, extraInfo: String?) { loggerBase.warn(dtw, msg, extraInfo) } +*/ fun warn(msg: String, exn: Throwable) { +/* +OLD: KE1 warn(msg, exn.stackTraceToString()) +*/ } +/* +OLD: KE1 fun warn(msg: String) { warn(msg, null) } +*/ +/* +OLD: KE1 private fun error(msg: String, extraInfo: String?) { loggerBase.error(dtw, msg, extraInfo) } +*/ fun error(msg: String) { +/* +OLD: KE1 error(msg, null) +*/ } fun error(msg: String, exn: Throwable) { +/* +OLD: KE1 error(msg, exn.stackTraceToString()) - } */ + } } /*