hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-05-22 07:07:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

KE2: test-kotlin2/library-tests/files now has no consistency failures

Browse Source
This commit is contained in:
Ian Lynagh
2024-08-28 14:17:56 +01:00
parent 6073180e02
commit 155da0b243
6 changed files with 176 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
java/kotlin-extractor2/BUILD.bazel
View File

@@ -30,6 +30,41 @@ kt_jvm_library(
srcs = [
":generated-dbscheme",
"src/main/java/com/semmle/util/unicode/UTF8Util.java",
"src/main/java/com/semmle/extractor/java/PopulateFile.java",
"src/main/java/com/semmle/util/array/ArrayUtil.java",
"src/main/java/com/semmle/util/basic/ObjectUtil.java",
"src/main/java/com/semmle/util/data/IntRef.java",
"src/main/java/com/semmle/util/data/Pair.java",
"src/main/java/com/semmle/util/data/StringUtil.java",
"src/main/java/com/semmle/util/data/Tuple1.java",
"src/main/java/com/semmle/util/data/Tuple2.java",
"src/main/java/com/semmle/util/data/TupleN.java",
"src/main/java/com/semmle/util/exception/CatastrophicError.java",
"src/main/java/com/semmle/util/exception/Exceptions.java",
"src/main/java/com/semmle/util/exception/InterruptedError.java",
"src/main/java/com/semmle/util/exception/NestedError.java",
"src/main/java/com/semmle/util/exception/ResourceError.java",
"src/main/java/com/semmle/util/exception/UserError.java",
"src/main/java/com/semmle/util/expansion/ExpansionEnvironment.java",
"src/main/java/com/semmle/util/files/FileUtil.java",
"src/main/java/com/semmle/util/files/PathMatcher.java",
"src/main/java/com/semmle/util/io/BufferedLineReader.java",
"src/main/java/com/semmle/util/io/csv/CSVParser.java",
"src/main/java/com/semmle/util/io/csv/CSVReader.java",
"src/main/java/com/semmle/util/io/csv/CSVWriter.java",
"src/main/java/com/semmle/util/io/RawStreamMuncher.java",
"src/main/java/com/semmle/util/io/StreamMuncher.java",
"src/main/java/com/semmle/util/io/StreamUtil.java",
"src/main/java/com/semmle/util/io/WholeIO.java",
"src/main/java/com/semmle/util/logging/Streams.java",
"src/main/java/com/semmle/util/process/AbstractProcessBuilder.java",
"src/main/java/com/semmle/util/process/Builder.java",
"src/main/java/com/semmle/util/process/Env.java",
"src/main/java/com/semmle/util/process/LeakPrevention.java",
"src/main/java/com/semmle/util/projectstructure/ProjectLayout.java",
"src/main/java/com/semmle/util/trap/pathtransformers/NoopTransformer.java",
"src/main/java/com/semmle/util/trap/pathtransformers/PathTransformer.java",
"src/main/java/com/semmle/util/trap/pathtransformers/ProjectLayoutTransformer.java",
] + glob(["src/main/kotlin/**/*.kt"]),
javac_opts = ":javac-options",
kotlinc_opts = ":kotlinc-options",

8
java/kotlin-extractor2/ke2.sh
View File

@@ -17,7 +17,13 @@ echo "// Invocation of Kotlin Extractor 2" >> "$INVOCATION_TRAP"
echo "#compilation = *" >> "$INVOCATION_TRAP"
# TODO: This should be properly escaped:
echo "compilations(#compilation, 2, \"`pwd`\",\"$INVOCATION_TRAP\")" >> "$INVOCATION_TRAP"
# TODO: compilation_args
ARG_INDEX=0
for ARG in "$@"
do
# TODO: This should be properly escaped:
echo "compilation_args(#compilation, $ARG_INDEX, \"$ARG\")" >> "$INVOCATION_TRAP"
ARG_INDEX=$(("$ARG_INDEX" + 1))
done
if [[ -n "$CODEQL_JAVA_HOME" ]]; then
JAVA="$CODEQL_JAVA_HOME/bin/java"

124
java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt
View File

@@ -133,7 +133,7 @@ OLD: KE1
srcDir.mkdirs()
val globalExtensionState = KotlinExtractorGlobalState()
*/
doAnalysis(kotlinArgs)
doAnalysis(dtw, compilation, invocationExtractionProblems, kotlinArgs)
/*
OLD: KE1
loggerBase.printLimitedDiagnosticCounts(tw)
@@ -153,40 +153,7 @@ OLD: KE1
}
}
fun doAnalysis(args : List<String>) {
/*
OLD: KE1
moduleFragment.files.mapIndexed { index: Int, file: IrFile ->
val fileExtractionProblems = FileExtractionProblems(invocationExtractionProblems)
val fileTrapWriter = tw.makeSourceFileTrapWriter(file, true)
loggerBase.setFileNumber(index)
fileTrapWriter.writeCompilation_compiling_files(
compilation,
index,
fileTrapWriter.fileId
)
doFile(
compression,
fileExtractionProblems,
invocationTrapFile,
fileTrapWriter,
checkTrapIdentical,
loggerBase,
trapDir,
srcDir,
file,
primitiveTypeMapping,
pluginContext,
globalExtensionState
)
fileTrapWriter.writeCompilation_compiling_files_completed(
compilation,
index,
fileExtractionProblems.extractionResult()
)
}
*/
fun doAnalysis(dtw : DiagnosticTrapWriter, compilation: Label<DbCompilation>, invocationExtractionProblems : ExtractionProblems, args : List<String>) {
lateinit var sourceModule: KaSourceModule
val k2args : K2JVMCompilerArguments = parseCommandLineArguments(args.toList())
@@ -215,20 +182,52 @@ OLD: KE1
}
val psiFiles = session.modulesWithFiles.getValue(sourceModule)
/*
OLD: KE1
moduleFragment.files.mapIndexed { index: Int, file: IrFile ->
*/
var fileIndex = 0
for (psiFile in psiFiles) {
if (psiFile is KtFile) {
analyze(psiFile) {
val c = psiFile.getDeclarations()[0]
if (c is KtClass) {
for (d: KtDeclaration in c.getDeclarations()) {
if (d is KtFunction) {
if (d.name == "f") {
dumpFunction(d)
}
}
}
}
val fileExtractionProblems = FileExtractionProblems(invocationExtractionProblems)
val fileDiagnosticTrapWriter = dtw.makeSourceFileTrapWriter(psiFile, true)
/*
OLD: KE1
loggerBase.setFileNumber(fileIndex)
*/
fileDiagnosticTrapWriter.writeCompilation_compiling_files(
compilation,
fileIndex,
fileDiagnosticTrapWriter.fileId
)
doFile(
/*
OLD: KE1
compression,
fileExtractionProblems,
invocationTrapFile,
fileDiagnosticTrapWriter,
checkTrapIdentical,
loggerBase,
trapDir,
srcDir,
*/
psiFile,
/*
OLD: KE1
primitiveTypeMapping,
pluginContext,
globalExtensionState
*/
)
fileDiagnosticTrapWriter.writeCompilation_compiling_files_completed(
compilation,
fileIndex,
fileExtractionProblems.extractionResult()
)
}
fileIndex += 1
} else {
System.out.println("Warning: Not a KtFile")
}
@@ -423,8 +422,6 @@ open class ExtractionProblems {
}
}
/*
OLD: KE1
/*
The `FileExtractionProblems` is analogous to `ExtractionProblems`,
except it records whether there were any problems while extracting a
@@ -443,6 +440,8 @@ class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblem
}
}
/*
OLD: KE1
/*
This function determines whether 2 TRAP files should be considered to be
equivalent. It returns `true` iff all of their non-comment lines are
@@ -467,23 +466,45 @@ private fun equivalentTrap(r1: BufferedReader, r2: BufferedReader): Boolean {
}
}
}
*/
context (KaSession)
private fun doFile(
/*
OLD: KE1
compression: Compression,
fileExtractionProblems: FileExtractionProblems,
invocationTrapFile: String,
fileTrapWriter: FileTrapWriter,
fileDiagnosticTrapWriter: FileTrapWriter,
checkTrapIdentical: Boolean,
loggerBase: LoggerBase,
dbTrapDir: File,
dbSrcDir: File,
srcFile: IrFile,
*/
srcFile: KtFile,
/*
OLD: KE1
primitiveTypeMapping: PrimitiveTypeMapping,
pluginContext: IrPluginContext,
globalExtensionState: KotlinExtractorGlobalState
*/
) {
// TODO: Testing
val c = srcFile.getDeclarations()[0]
if (c is KtClass) {
for (d: KtDeclaration in c.getDeclarations()) {
if (d is KtFunction) {
if (d.name == "f") {
dumpFunction(d)
}
}
}
}
/*
OLD: KE1
val srcFilePath = srcFile.path
val logger = FileLogger(loggerBase, fileTrapWriter)
val logger = FileLogger(loggerBase, fileDiagnosticTrapWriter)
logger.info("Extracting file $srcFilePath")
logger.flush()
@@ -579,8 +600,11 @@ private fun doFile(
fileExtractionProblems.setNonRecoverableProblem()
}
}
*/
}
/*
OLD: KE1
enum class Compression(val extension: String) {
NONE("") {
override fun bufferedWriter(file: File): BufferedWriter {

6
java/kotlin-extractor2/src/main/kotlin/Label.kt
View File

@@ -2,21 +2,15 @@ package com.github.codeql
/** This represents a label (`#...`) in a TRAP file. */
interface Label<T : AnyDbType> {
/*
OLD: KE1
fun <U : AnyDbType> cast(): Label<U> {
@Suppress("UNCHECKED_CAST") return this as Label<U>
}
*/
}
/*
OLD: KE1
/** The label `#i`, e.g. `#123`. Most labels we generate are of this form. */
class IntLabel<T : AnyDbType>(val i: Int) : Label<T> {
override fun toString(): String = "#$i"
}
*/
/**
* The label `#name`, e.g. `#compilation`. This is used when labels are shared between different

46
java/kotlin-extractor2/src/main/kotlin/TrapWriter.kt
View File

@@ -3,13 +3,13 @@ package com.github.codeql
/*
OLD: KE1
import com.github.codeql.KotlinUsesExtractor.LocallyVisibleFunctionLabels
import com.semmle.extractor.java.PopulateFile
*/
import com.semmle.extractor.java.PopulateFile
import com.semmle.util.unicode.UTF8Util
import java.io.BufferedWriter
import java.io.File
/*
OLD: KE1
import java.io.File
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
import org.jetbrains.kotlin.ir.declarations.IrClass
@@ -20,14 +20,13 @@ import org.jetbrains.kotlin.ir.declarations.path
import org.jetbrains.kotlin.ir.expressions.IrCall
import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET
*/
import org.jetbrains.kotlin.psi.*
/**
* Each `.trap` file has a `TrapLabelManager` while we are writing it. It provides fresh TRAP label
* names, and maintains a mapping from keys (`@"..."`) to labels.
*/
class TrapLabelManager {
/*
OLD: KE1
/** The next integer to use as a label name. */
private var nextInt: Int = 100
@@ -39,6 +38,8 @@ OLD: KE1
/** A mapping from a key (`@"..."`) to the label defined to be that key, if any. */
val labelMapping: MutableMap<String, Label<*>> = mutableMapOf<String, Label<*>>()
/*
OLD: KE1
val anonymousTypeMapping: MutableMap<IrClass, TypeResults> = mutableMapOf()
val locallyVisibleFunctionLabelMapping: MutableMap<IrFunction, LocallyVisibleFunctionLabels> =
@@ -86,9 +87,12 @@ OLD: KE1
* otherwise. Most users will want to use `getLabelFor` instead, which allows non-existent
* labels to be initialised.
*/
fun <T : AnyDbType> getExistingLabelFor(key: String): Label<T>? {
TODO: Inline this if it can remain private
*/
private fun <T : AnyDbType> getExistingLabelFor(key: String): Label<T>? {
return lm.labelMapping.get(key)?.cast<T>()
}
/**
* Returns the label for the given key, if one exists. Otherwise, a fresh label is bound to that
* key, `initialise` is run on it, and it is returned.
@@ -107,6 +111,8 @@ OLD: KE1
}
}
/*
OLD: KE1
/** Returns a label for a fresh ID (i.e. a new label bound to `*`). */
fun <T : AnyDbType> getFreshIdLabel(): Label<T> {
val label: Label<T> = lm.getFreshLabel()
@@ -136,6 +142,7 @@ OLD: KE1
fun getExistingVariableLabelFor(v: IrVariable): Label<out DbLocalvar>? {
return variableLabelMapping.get(v)
}
*/
/**
* This returns a label for the location described by its arguments. Typically users will not
@@ -149,11 +156,17 @@ OLD: KE1
endLine: Int,
endColumn: Int
): Label<DbLocation> {
/*
OLD: HE1
return getLabelFor("@\"loc,{$fileId},$startLine,$startColumn,$endLine,$endColumn\"") {
writeLocations_default(it, fileId, startLine, startColumn, endLine, endColumn)
}
*/
TODO()
}
/*
OLD: HE1
/**
* The label for the 'unknown' file ID. Users will want to use `unknownLocation` instead. This
* is lazy, as we don't want to define it in a TRAP file unless the TRAP file actually contains
@@ -169,6 +182,7 @@ OLD: KE1
* file unless the TRAP file actually contains something with an 'unknown' location.
*/
val unknownLocation: Label<DbLocation> by lazy { getWholeFileLocation(unknownFileId) }
*/
/**
* Returns the label for the file `filePath`. If `populateFileTables` is true, then this also
@@ -191,6 +205,8 @@ OLD: KE1
}
}
/*
OLD: HE1
/**
* If you have an ID for a file, then this gets a label for the location representing the whole
* of that file.
@@ -271,7 +287,8 @@ OLD: KE1
* Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the
* given `IrFile` for locations.
*/
fun makeSourceFileTrapWriter(file: IrFile, populateFileTables: Boolean) =
*/
fun makeSourceFileTrapWriter(file: KtFile, populateFileTables: Boolean) =
SourceFileTrapWriter(
loggerBase,
lm,
@@ -280,7 +297,6 @@ OLD: KE1
file,
populateFileTables
)
*/
}
/*
@@ -319,6 +335,7 @@ OLD: KE1
* An ID for the file will be created, and if `populateFileTables` is true then we will also add
* rows to the `files` and `folders` tables for it.
*/
*/
open class FileTrapWriter(
loggerBase: LoggerBase,
lm: TrapLabelManager,
@@ -335,6 +352,8 @@ open class FileTrapWriter(
return dtw
}
/*
OLD: KE1
private fun offsetMinOf(default: Int, vararg options: Int?): Int {
if (default == UNDEFINED_OFFSET || default == SYNTHETIC_OFFSET) {
return default
@@ -400,8 +419,11 @@ open class FileTrapWriter(
fun getWholeFileLocation(): Label<DbLocation> {
return getWholeFileLocation(fileId)
}
*/
}
/*
OLD: KE1
/**
* A `SourceFileTrapWriter` is used when not only do we know which file we are extracting entities
* from, but we also have an `IrFileEntry` (from an `IrFile`) which allows us to map byte offsets to
@@ -410,15 +432,17 @@ open class FileTrapWriter(
* An ID for the file will be created, and if `populateFileTables` is true then we will also add
* rows to the `files` and `folders` tables for it.
*/
*/
class SourceFileTrapWriter(
loggerBase: LoggerBase,
lm: TrapLabelManager,
bw: BufferedWriter,
dtw: DiagnosticTrapWriter,
val irFile: IrFile,
val ktFile: KtFile,
populateFileTables: Boolean
) : FileTrapWriter(loggerBase, lm, bw, dtw, irFile.path, populateFileTables) {
) : FileTrapWriter(loggerBase, lm, bw, dtw, ktFile.virtualFilePath, populateFileTables) {
/*
OLD: KE1
/**
* The file entry for the file that we are extracting from. Used to map offsets to line/column
* numbers.
@@ -490,5 +514,5 @@ class SourceFileTrapWriter(
val endColumn = fileEntry.getColumnNumber(e.endOffset)
return "file://$filePath:$startLine:$startColumn:$endLine:$endColumn"
}
}
*/
}

29
java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt
View File

@@ -248,15 +248,21 @@ OLD: KE1
)
logStream.write(logMessage.toJsonLine())
}
*/
fun trace(dtw: DiagnosticTrapWriter, msg: String) {
if (verbosity >= 4) {
/*
OLD: KE1
val logMessage = LogMessage("TRACE", msg)
tw.writeComment(logMessage.toText())
logStream.write(logMessage.toJsonLine())
*/
}
}
/*
OLD: KE1
fun debug(dtw: DiagnosticTrapWriter, msg: String) {
if (verbosity >= 4) {
val logMessage = LogMessage("DEBUG", msg)
@@ -324,8 +330,6 @@ open class Logger(val loggerBase: LoggerBase, val dtw: DiagnosticTrapWriter) {
loggerBase.flush()
}
/*
OLD: KE1
fun trace(msg: String) {
loggerBase.trace(dtw, msg)
}
@@ -335,9 +339,11 @@ OLD: KE1
}
fun debug(msg: String) {
/*
OLD: KE1
loggerBase.debug(dtw, msg)
}
*/
}
fun info(msg: String) {
loggerBase.info(dtw, msg)
@@ -348,27 +354,42 @@ OLD: KE1
private fun warn(msg: String, extraInfo: String?) {
loggerBase.warn(dtw, msg, extraInfo)
}
*/
fun warn(msg: String, exn: Throwable) {
/*
OLD: KE1
warn(msg, exn.stackTraceToString())
*/
}
/*
OLD: KE1
fun warn(msg: String) {
warn(msg, null)
}
*/
/*
OLD: KE1
private fun error(msg: String, extraInfo: String?) {
loggerBase.error(dtw, msg, extraInfo)
}
*/
fun error(msg: String) {
/*
OLD: KE1
error(msg, null)
*/
}
fun error(msg: String, exn: Throwable) {
/*
OLD: KE1
error(msg, exn.stackTraceToString())
}
*/
}
}
/*