hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-17 01:03:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

Kotlin: Reformat code

Browse Source 
Using:
    java -jar ktfmt-0.46-jar-with-dependencies.jar --kotlinlang-style java/kotlin-extractor/**/*.kt
This commit is contained in:
Ian Lynagh
2024-01-09 15:33:41 +00:00
parent 0bc1463ab0
commit bf611feab3
64 changed files with 7420 additions and 3219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
java/kotlin-extractor/src/main/kotlin/ExternalDeclExtractor.kt
View File

@@ -3,18 +3,26 @@ package com.github.codeql
import com.github.codeql.utils.isExternalFileClassMember
import com.semmle.extractor.java.OdasaOutput
import com.semmle.util.data.StringDigestor
import java.io.BufferedWriter
import java.io.File
import java.util.ArrayList
import java.util.HashSet
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.isFileClass
import org.jetbrains.kotlin.ir.util.packageFqName
import java.io.BufferedWriter
import java.io.File
import java.util.ArrayList
import java.util.HashSet
import java.util.zip.GZIPOutputStream
class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression, val invocationTrapFile: String, val sourceFilePath: String, val primitiveTypeMapping: PrimitiveTypeMapping, val pluginContext: IrPluginContext, val globalExtensionState: KotlinExtractorGlobalState, val diagnosticTrapWriter: DiagnosticTrapWriter) {
class ExternalDeclExtractor(
val logger: FileLogger,
val compression: Compression,
val invocationTrapFile: String,
val sourceFilePath: String,
val primitiveTypeMapping: PrimitiveTypeMapping,
val pluginContext: IrPluginContext,
val globalExtensionState: KotlinExtractorGlobalState,
val diagnosticTrapWriter: DiagnosticTrapWriter
) {
val declBinaryNames = HashMap<IrDeclaration, String>()
val externalDeclsDone = HashSet<Pair<String, String>>()
@@ -23,13 +31,17 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
val propertySignature = ";property"
val fieldSignature = ";field"
val output = OdasaOutput(false, compression, logger).also {
val output =
OdasaOutput(false, compression, logger).also {
it.setCurrentSourceFile(File(sourceFilePath))
}
fun extractLater(d: IrDeclarationWithName, signature: String): Boolean {
if (d !is IrClass && !isExternalFileClassMember(d)) {
logger.errorElement("External declaration is neither a class, nor a top-level declaration", d)
logger.errorElement(
"External declaration is neither a class, nor a top-level declaration",
d
)
return false
}
val declBinaryName = declBinaryNames.getOrPut(d) { getIrElementBinaryName(d) }
@@ -37,24 +49,39 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
if (ret) externalDeclWorkList.add(Pair(d, signature))
return ret
}
fun extractLater(c: IrClass) = extractLater(c, "")
fun writeStubTrapFile(e: IrElement, signature: String = "") {
extractElement(e, signature, true) { trapFileBW, _, _ ->
trapFileBW.write("// Trap file stubbed because this declaration was extracted from source in $sourceFilePath\n")
trapFileBW.write(
"// Trap file stubbed because this declaration was extracted from source in $sourceFilePath\n"
)
trapFileBW.write("// Part of invocation $invocationTrapFile\n")
}
}
private fun extractElement(element: IrElement, possiblyLongSignature: String, fromSource: Boolean, extractorFn: (BufferedWriter, String, OdasaOutput.TrapFileManager) -> Unit) {
// In order to avoid excessively long signatures which can lead to trap file names longer than the filesystem
private fun extractElement(
element: IrElement,
possiblyLongSignature: String,
fromSource: Boolean,
extractorFn: (BufferedWriter, String, OdasaOutput.TrapFileManager) -> Unit
) {
// In order to avoid excessively long signatures which can lead to trap file names longer
// than the filesystem
// limit, we truncate and add a hash to preserve uniqueness if necessary.
val signature = if (possiblyLongSignature.length > 100) {
possiblyLongSignature.substring(0, 92) + "#" + StringDigestor.digest(possiblyLongSignature).substring(0, 8)
} else { possiblyLongSignature }
val signature =
if (possiblyLongSignature.length > 100) {
possiblyLongSignature.substring(0, 92) +
"#" +
StringDigestor.digest(possiblyLongSignature).substring(0, 8)
} else {
possiblyLongSignature
}
output.getTrapLockerForDecl(element, signature, fromSource).useAC { locker ->
locker.trapFileManager.useAC { manager ->
val shortName = when(element) {
val shortName =
when (element) {
is IrDeclarationWithName -> element.name.asString()
is IrFile -> element.name
else -> "(unknown name)"
@@ -64,7 +91,12 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
} else {
val trapFile = manager.file
logger.info("Will write TRAP file $trapFile")
val trapTmpFile = File.createTempFile("${trapFile.nameWithoutExtension}.", ".${trapFile.extension}.tmp", trapFile.parentFile)
val trapTmpFile =
File.createTempFile(
"${trapFile.nameWithoutExtension}.",
".${trapFile.extension}.tmp",
trapFile.parentFile
)
logger.debug("Writing temporary TRAP file $trapTmpFile")
try {
compression.bufferedWriter(trapTmpFile).use {
@@ -77,7 +109,10 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
logger.info("Finished writing TRAP file $trapFile")
} catch (e: Exception) {
manager.setHasError()
logger.error("Failed to extract '$shortName'. Partial TRAP file location is $trapTmpFile", e)
logger.error(
"Failed to extract '$shortName'. Partial TRAP file location is $trapTmpFile",
e
)
}
}
}
@@ -90,37 +125,75 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
externalDeclWorkList.clear()
nextBatch.forEach { workPair ->
val (irDecl, possiblyLongSignature) = workPair
extractElement(irDecl, possiblyLongSignature, false) { trapFileBW, signature, manager ->
extractElement(irDecl, possiblyLongSignature, false) {
trapFileBW,
signature,
manager ->
val binaryPath = getIrDeclarationBinaryPath(irDecl)
if (binaryPath == null) {
logger.errorElement("Unable to get binary path", irDecl)
} else {
// We want our comments to be the first thing in the file,
// so start off with a PlainTrapWriter
val tw = PlainTrapWriter(logger.loggerBase, TrapLabelManager(), trapFileBW, diagnosticTrapWriter)
tw.writeComment("Generated by the CodeQL Kotlin extractor for external dependencies")
val tw =
PlainTrapWriter(
logger.loggerBase,
TrapLabelManager(),
trapFileBW,
diagnosticTrapWriter
)
tw.writeComment(
"Generated by the CodeQL Kotlin extractor for external dependencies"
)
tw.writeComment("Part of invocation $invocationTrapFile")
if (signature != possiblyLongSignature) {
tw.writeComment("Function signature abbreviated; full signature is: $possiblyLongSignature")
tw.writeComment(
"Function signature abbreviated; full signature is: $possiblyLongSignature"
)
}
// Now elevate to a SourceFileTrapWriter, and populate the
// file information if needed:
val ftw = tw.makeFileTrapWriter(binaryPath, true)
val fileExtractor = KotlinFileExtractor(logger, ftw, null, binaryPath, manager, this, primitiveTypeMapping, pluginContext, KotlinFileExtractor.DeclarationStack(), globalExtensionState)
val fileExtractor =
KotlinFileExtractor(
logger,
ftw,
null,
binaryPath,
manager,
this,
primitiveTypeMapping,
pluginContext,
KotlinFileExtractor.DeclarationStack(),
globalExtensionState
)
if (irDecl is IrClass) {
// Populate a location and compilation-unit package for the file. This is similar to
// the beginning of `KotlinFileExtractor.extractFileContents` but without an `IrFile`
// Populate a location and compilation-unit package for the file. This
// is similar to
// the beginning of `KotlinFileExtractor.extractFileContents` but
// without an `IrFile`
// to start from.
val pkg = irDecl.packageFqName?.asString() ?: ""
val pkgId = fileExtractor.extractPackage(pkg)
ftw.writeHasLocation(ftw.fileId, ftw.getWholeFileLocation())
ftw.writeCupackage(ftw.fileId, pkgId)
fileExtractor.extractClassSource(irDecl, extractDeclarations = !irDecl.isFileClass, extractStaticInitializer = false, extractPrivateMembers = false, extractFunctionBodies = false)
fileExtractor.extractClassSource(
irDecl,
extractDeclarations = !irDecl.isFileClass,
extractStaticInitializer = false,
extractPrivateMembers = false,
extractFunctionBodies = false
)
} else {
fileExtractor.extractDeclaration(irDecl, extractPrivateMembers = false, extractFunctionBodies = false, extractAnnotations = true)
fileExtractor.extractDeclaration(
irDecl,
extractPrivateMembers = false,
extractFunctionBodies = false,
extractAnnotations = true
)
}
}
}
@@ -128,5 +201,4 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
} while (externalDeclWorkList.isNotEmpty())
output.writeTrapSet()
}
}

46
java/kotlin-extractor/src/main/kotlin/KotlinExtractorCommandLineProcessor.kt
View File

@@ -11,7 +11,8 @@ import org.jetbrains.kotlin.config.CompilerConfigurationKey
class KotlinExtractorCommandLineProcessor : CommandLineProcessor {
override val pluginId = "kotlin-extractor"
override val pluginOptions = listOf(
override val pluginOptions =
listOf(
CliOption(
optionName = OPTION_INVOCATION_TRAP_FILE,
valueDescription = "Invocation TRAP file",
@@ -35,8 +36,10 @@ class KotlinExtractorCommandLineProcessor : CommandLineProcessor {
),
CliOption(
optionName = OPTION_EXIT_AFTER_EXTRACTION,
valueDescription = "Specify whether to call exitProcess after the extraction has completed",
description = "Specify whether to call exitProcess after the extraction has completed",
valueDescription =
"Specify whether to call exitProcess after the extraction has completed",
description =
"Specify whether to call exitProcess after the extraction has completed",
required = false,
allowMultipleOccurrences = false
)
@@ -46,19 +49,40 @@ class KotlinExtractorCommandLineProcessor : CommandLineProcessor {
option: AbstractCliOption,
value: String,
configuration: CompilerConfiguration
) = when (option.optionName) {
) =
when (option.optionName) {
OPTION_INVOCATION_TRAP_FILE -> configuration.put(KEY_INVOCATION_TRAP_FILE, value)
OPTION_CHECK_TRAP_IDENTICAL -> processBooleanOption(value, OPTION_CHECK_TRAP_IDENTICAL, KEY_CHECK_TRAP_IDENTICAL, configuration)
OPTION_EXIT_AFTER_EXTRACTION -> processBooleanOption(value, OPTION_EXIT_AFTER_EXTRACTION, KEY_EXIT_AFTER_EXTRACTION, configuration)
OPTION_CHECK_TRAP_IDENTICAL ->
processBooleanOption(
value,
OPTION_CHECK_TRAP_IDENTICAL,
KEY_CHECK_TRAP_IDENTICAL,
configuration
)
OPTION_EXIT_AFTER_EXTRACTION ->
processBooleanOption(
value,
OPTION_EXIT_AFTER_EXTRACTION,
KEY_EXIT_AFTER_EXTRACTION,
configuration
)
OPTION_COMPILATION_STARTTIME ->
when (val v = value.toLongOrNull()) {
is Long -> configuration.put(KEY_COMPILATION_STARTTIME, v)
else -> error("kotlin extractor: Bad argument $value for $OPTION_COMPILATION_STARTTIME")
else ->
error(
"kotlin extractor: Bad argument $value for $OPTION_COMPILATION_STARTTIME"
)
}
else -> error("kotlin extractor: Bad option: ${option.optionName}")
}
private fun processBooleanOption(value: String, optionName: String, configKey: CompilerConfigurationKey<Boolean>, configuration: CompilerConfiguration) =
private fun processBooleanOption(
value: String,
optionName: String,
configKey: CompilerConfigurationKey<Boolean>,
configuration: CompilerConfiguration
) =
when (value) {
"true" -> configuration.put(configKey, true)
"false" -> configuration.put(configKey, false)
@@ -69,8 +93,8 @@ class KotlinExtractorCommandLineProcessor : CommandLineProcessor {
private val OPTION_INVOCATION_TRAP_FILE = "invocationTrapFile"
val KEY_INVOCATION_TRAP_FILE = CompilerConfigurationKey<String>(OPTION_INVOCATION_TRAP_FILE)
private val OPTION_CHECK_TRAP_IDENTICAL = "checkTrapIdentical"
val KEY_CHECK_TRAP_IDENTICAL= CompilerConfigurationKey<Boolean>(OPTION_CHECK_TRAP_IDENTICAL)
val KEY_CHECK_TRAP_IDENTICAL = CompilerConfigurationKey<Boolean>(OPTION_CHECK_TRAP_IDENTICAL)
private val OPTION_COMPILATION_STARTTIME = "compilationStartTime"
val KEY_COMPILATION_STARTTIME= CompilerConfigurationKey<Long>(OPTION_COMPILATION_STARTTIME)
val KEY_COMPILATION_STARTTIME = CompilerConfigurationKey<Long>(OPTION_COMPILATION_STARTTIME)
private val OPTION_EXIT_AFTER_EXTRACTION = "exitAfterExtraction"
val KEY_EXIT_AFTER_EXTRACTION= CompilerConfigurationKey<Boolean>(OPTION_EXIT_AFTER_EXTRACTION)
val KEY_EXIT_AFTER_EXTRACTION = CompilerConfigurationKey<Boolean>(OPTION_EXIT_AFTER_EXTRACTION)

13
java/kotlin-extractor/src/main/kotlin/KotlinExtractorComponentRegistrar.kt
View File

@@ -3,12 +3,9 @@
package com.github.codeql
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
import com.intellij.mock.MockProject
import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.config.CompilerConfiguration
import com.github.codeql.Kotlin2ComponentRegistrar
class KotlinExtractorComponentRegistrar : Kotlin2ComponentRegistrar() {
override fun registerProjectComponents(
@@ -19,10 +16,14 @@ class KotlinExtractorComponentRegistrar : Kotlin2ComponentRegistrar() {
if (invocationTrapFile == null) {
throw Exception("Required argument for TRAP invocation file not given")
}
IrGenerationExtension.registerExtension(project, KotlinExtractorExtension(
IrGenerationExtension.registerExtension(
project,
KotlinExtractorExtension(
invocationTrapFile,
configuration[KEY_CHECK_TRAP_IDENTICAL] ?: false,
configuration[KEY_COMPILATION_STARTTIME],
configuration[KEY_EXIT_AFTER_EXTRACTION] ?: false))
configuration[KEY_EXIT_AFTER_EXTRACTION] ?: false
)
)
}
}

221
java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt
View File

@@ -1,15 +1,11 @@
package com.github.codeql
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.*
import org.jetbrains.kotlin.ir.IrElement
import java.io.BufferedReader
import java.io.BufferedWriter
import com.github.codeql.utils.versions.usesK2
import com.semmle.util.files.FileUtil
import java.io.BufferedInputStream
import java.io.BufferedOutputStream
import java.io.BufferedReader
import java.io.BufferedWriter
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
@@ -20,9 +16,12 @@ import java.nio.file.Files
import java.nio.file.Paths
import java.util.zip.GZIPInputStream
import java.util.zip.GZIPOutputStream
import com.github.codeql.utils.versions.usesK2
import com.semmle.util.files.FileUtil
import kotlin.system.exitProcess
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.*
/*
* KotlinExtractorExtension is the main entry point of the CodeQL Kotlin
@@ -55,8 +54,8 @@ class KotlinExtractorExtension(
// can be set to true to make the plugin terminate the kotlinc
// invocation when it has finished. This means that kotlinc will not
// write any `.class` files etc.
private val exitAfterExtraction: Boolean)
: IrGenerationExtension {
private val exitAfterExtraction: Boolean
) : IrGenerationExtension {
// This is the main entry point to the extractor.
// It will be called by kotlinc with the IR for the files being
@@ -68,7 +67,7 @@ class KotlinExtractorExtension(
// We catch Throwable rather than Exception, as we want to
// continue trying to extract everything else even if we get a
// stack overflow or an assertion failure in one file.
} catch(e: Throwable) {
} catch (e: Throwable) {
// If we get an exception at the top level, then something's
// gone very wrong. Don't try to be too fancy, but try to
// log a simple message.
@@ -80,7 +79,8 @@ class KotlinExtractorExtension(
// We use a slightly different filename pattern compared
// to normal logs. Just the existence of a `-top` log is
// a sign that something's gone very wrong.
val logFile = File.createTempFile("kotlin-extractor-top.", ".log", File(extractorLogDir))
val logFile =
File.createTempFile("kotlin-extractor-top.", ".log", File(extractorLogDir))
logFile.writeText(msg)
// Now we've got that out, let's see if we can append a stack trace too
logFile.appendText(e.stackTraceToString())
@@ -99,12 +99,18 @@ class KotlinExtractorExtension(
private fun runExtractor(moduleFragment: IrModuleFragment, pluginContext: IrPluginContext) {
val startTimeMs = System.currentTimeMillis()
val usesK2 = usesK2(pluginContext)
// This default should be kept in sync with com.semmle.extractor.java.interceptors.KotlinInterceptor.initializeExtractionContext
val trapDir = File(System.getenv("CODEQL_EXTRACTOR_JAVA_TRAP_DIR").takeUnless { it.isNullOrEmpty() } ?: "kotlin-extractor/trap")
// This default should be kept in sync with
// com.semmle.extractor.java.interceptors.KotlinInterceptor.initializeExtractionContext
val trapDir =
File(
System.getenv("CODEQL_EXTRACTOR_JAVA_TRAP_DIR").takeUnless { it.isNullOrEmpty() }
?: "kotlin-extractor/trap"
)
// The invocation TRAP file will already have been started
// before the plugin is run, so we always use no compression
// and we open it in append mode.
FileOutputStream(File(invocationTrapFile), true).bufferedWriter().use { invocationTrapFileBW ->
FileOutputStream(File(invocationTrapFile), true).bufferedWriter().use { invocationTrapFileBW
->
val invocationExtractionProblems = ExtractionProblems()
val lm = TrapLabelManager()
val logCounter = LogCounter()
@@ -113,12 +119,21 @@ class KotlinExtractorExtension(
// The interceptor has already defined #compilation = *
val compilation: Label<DbCompilation> = StringLabel("compilation")
tw.writeCompilation_started(compilation)
tw.writeCompilation_info(compilation, "Kotlin Compiler Version", KotlinCompilerVersion.getVersion() ?: "<unknown>")
val extractor_name = this::class.java.getResource("extractor.name")?.readText() ?: "<unknown>"
tw.writeCompilation_info(
compilation,
"Kotlin Compiler Version",
KotlinCompilerVersion.getVersion() ?: "<unknown>"
)
val extractor_name =
this::class.java.getResource("extractor.name")?.readText() ?: "<unknown>"
tw.writeCompilation_info(compilation, "Kotlin Extractor Name", extractor_name)
tw.writeCompilation_info(compilation, "Uses Kotlin 2", usesK2.toString())
if (compilationStartTime != null) {
tw.writeCompilation_compiler_times(compilation, -1.0, (System.currentTimeMillis()-compilationStartTime)/1000.0)
tw.writeCompilation_compiler_times(
compilation,
-1.0,
(System.currentTimeMillis() - compilationStartTime) / 1000.0
)
}
tw.flush()
val logger = Logger(loggerBase, tw)
@@ -138,23 +153,54 @@ class KotlinExtractorExtension(
// FIXME: FileUtil expects a static global logger
// which should be provided by SLF4J's factory facility. For now we set it here.
FileUtil.logger = logger
val srcDir = File(System.getenv("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR").takeUnless { it.isNullOrEmpty() } ?: "kotlin-extractor/src")
val srcDir =
File(
System.getenv("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR").takeUnless {
it.isNullOrEmpty()
} ?: "kotlin-extractor/src"
)
srcDir.mkdirs()
val globalExtensionState = KotlinExtractorGlobalState()
moduleFragment.files.mapIndexed { index: Int, file: IrFile ->
val fileExtractionProblems = FileExtractionProblems(invocationExtractionProblems)
val fileTrapWriter = tw.makeSourceFileTrapWriter(file, true)
loggerBase.setFileNumber(index)
fileTrapWriter.writeCompilation_compiling_files(compilation, index, fileTrapWriter.fileId)
doFile(compression, fileExtractionProblems, invocationTrapFile, fileTrapWriter, checkTrapIdentical, loggerBase, trapDir, srcDir, file, primitiveTypeMapping, pluginContext, globalExtensionState)
fileTrapWriter.writeCompilation_compiling_files_completed(compilation, index, fileExtractionProblems.extractionResult())
fileTrapWriter.writeCompilation_compiling_files(
compilation,
index,
fileTrapWriter.fileId
)
doFile(
compression,
fileExtractionProblems,
invocationTrapFile,
fileTrapWriter,
checkTrapIdentical,
loggerBase,
trapDir,
srcDir,
file,
primitiveTypeMapping,
pluginContext,
globalExtensionState
)
fileTrapWriter.writeCompilation_compiling_files_completed(
compilation,
index,
fileExtractionProblems.extractionResult()
)
}
loggerBase.printLimitedDiagnosticCounts(tw)
logPeakMemoryUsage(logger, "after extractor")
logger.info("Extraction completed")
logger.flush()
val compilationTimeMs = System.currentTimeMillis() - startTimeMs
tw.writeCompilation_finished(compilation, -1.0, compilationTimeMs.toDouble() / 1000, invocationExtractionProblems.extractionResult())
tw.writeCompilation_finished(
compilation,
-1.0,
compilationTimeMs.toDouble() / 1000,
invocationExtractionProblems.extractionResult()
)
tw.flush()
loggerBase.close()
}
@@ -170,13 +216,17 @@ class KotlinExtractorExtension(
try {
val compression_option_upper = compression_option.uppercase()
if (compression_option_upper == "BROTLI") {
logger.warn("Kotlin extractor doesn't support Brotli compression. Using GZip instead.")
logger.warn(
"Kotlin extractor doesn't support Brotli compression. Using GZip instead."
)
return Compression.GZIP
} else {
return Compression.valueOf(compression_option_upper)
}
} catch (e: IllegalArgumentException) {
logger.warn("Unsupported compression type (\$$compression_env_var) \"$compression_option\". Supported values are ${Compression.values().joinToString()}.")
logger.warn(
"Unsupported compression type (\$$compression_env_var) \"$compression_option\". Supported values are ${Compression.values().joinToString()}."
)
return defaultCompression
}
}
@@ -190,9 +240,16 @@ class KotlinExtractorExtension(
var nonheap: Long = 0
for (bean in beans) {
val peak = bean.getPeakUsage().getUsed()
val kind = when (bean.getType()) {
MemoryType.HEAP -> { heap += peak; "heap" }
MemoryType.NON_HEAP -> { nonheap += peak; "non-heap" }
val kind =
when (bean.getType()) {
MemoryType.HEAP -> {
heap += peak
"heap"
}
MemoryType.NON_HEAP -> {
nonheap += peak
"non-heap"
}
else -> "unknown"
}
logger.info("Peak memory: * Peak for $kind bean ${bean.getName()} is $peak")
@@ -203,9 +260,12 @@ class KotlinExtractorExtension(
}
class KotlinExtractorGlobalState {
// These three record mappings of classes, functions and fields that should be replaced wherever they are found.
// As of now these are only used to fix IR generated by the Gradle Android Extensions plugin, hence e.g. IrProperty
// doesn't have a map as that plugin doesn't generate them. If and when these are used more widely additional maps
// These three record mappings of classes, functions and fields that should be replaced wherever
// they are found.
// As of now these are only used to fix IR generated by the Gradle Android Extensions plugin,
// hence e.g. IrProperty
// doesn't have a map as that plugin doesn't generate them. If and when these are used more
// widely additional maps
// should be added here.
val syntheticToRealClassMap = HashMap<IrClass, IrClass?>()
val syntheticToRealFunctionMap = HashMap<IrFunction, IrFunction?>()
@@ -227,13 +287,15 @@ open class ExtractionProblems {
open fun setRecoverableProblem() {
recoverableProblem = true
}
open fun setNonRecoverableProblem() {
nonRecoverableProblem = true
}
fun extractionResult(): Int {
if(nonRecoverableProblem) {
if (nonRecoverableProblem) {
return 2
} else if(recoverableProblem) {
} else if (recoverableProblem) {
return 1
} else {
return 0
@@ -246,11 +308,13 @@ The `FileExtractionProblems` is analogous to `ExtractionProblems`,
except it records whether there were any problems while extracting a
particular source file.
*/
class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblems): ExtractionProblems() {
class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblems) :
ExtractionProblems() {
override fun setRecoverableProblem() {
super.setRecoverableProblem()
invocationExtractionProblems.setRecoverableProblem()
}
override fun setNonRecoverableProblem() {
super.setNonRecoverableProblem()
invocationExtractionProblems.setNonRecoverableProblem()
@@ -265,7 +329,7 @@ identical.
private fun equivalentTrap(r1: BufferedReader, r2: BufferedReader): Boolean {
r1.use { br1 ->
r2.use { br2 ->
while(true) {
while (true) {
val l1 = br1.readLine()
val l2 = br2.readLine()
if (l1 == null && l2 == null) {
@@ -294,7 +358,8 @@ private fun doFile(
srcFile: IrFile,
primitiveTypeMapping: PrimitiveTypeMapping,
pluginContext: IrPluginContext,
globalExtensionState: KotlinExtractorGlobalState) {
globalExtensionState: KotlinExtractorGlobalState
) {
val srcFilePath = srcFile.path
val logger = FileLogger(loggerBase, fileTrapWriter)
logger.info("Extracting file $srcFilePath")
@@ -311,10 +376,13 @@ private fun doFile(
val dbSrcFilePath = Paths.get("$dbSrcDir/$srcFileRelativePath")
val dbSrcDirPath = dbSrcFilePath.parent
Files.createDirectories(dbSrcDirPath)
val srcTmpFile = File.createTempFile(dbSrcFilePath.fileName.toString() + ".", ".src.tmp", dbSrcDirPath.toFile())
srcTmpFile.outputStream().use {
Files.copy(Paths.get(srcFilePath), it)
}
val srcTmpFile =
File.createTempFile(
dbSrcFilePath.fileName.toString() + ".",
".src.tmp",
dbSrcDirPath.toFile()
)
srcTmpFile.outputStream().use { Files.copy(Paths.get(srcFilePath), it) }
srcTmpFile.renameTo(dbSrcFilePath.toFile())
val trapFileName = "$dbTrapDir/$srcFileRelativePath.trap"
@@ -327,22 +395,52 @@ private fun doFile(
trapFileWriter.getTempWriter().use { trapFileBW ->
// We want our comments to be the first thing in the file,
// so start off with a mere TrapWriter
val tw = PlainTrapWriter(loggerBase, TrapLabelManager(), trapFileBW, fileTrapWriter.getDiagnosticTrapWriter())
val tw =
PlainTrapWriter(
loggerBase,
TrapLabelManager(),
trapFileBW,
fileTrapWriter.getDiagnosticTrapWriter()
)
tw.writeComment("Generated by the CodeQL Kotlin extractor for kotlin source code")
tw.writeComment("Part of invocation $invocationTrapFile")
// Now elevate to a SourceFileTrapWriter, and populate the
// file information
val sftw = tw.makeSourceFileTrapWriter(srcFile, true)
val externalDeclExtractor = ExternalDeclExtractor(logger, compression, invocationTrapFile, srcFilePath, primitiveTypeMapping, pluginContext, globalExtensionState, fileTrapWriter.getDiagnosticTrapWriter())
val externalDeclExtractor =
ExternalDeclExtractor(
logger,
compression,
invocationTrapFile,
srcFilePath,
primitiveTypeMapping,
pluginContext,
globalExtensionState,
fileTrapWriter.getDiagnosticTrapWriter()
)
val linesOfCode = LinesOfCode(logger, sftw, srcFile)
val fileExtractor = KotlinFileExtractor(logger, sftw, linesOfCode, srcFilePath, null, externalDeclExtractor, primitiveTypeMapping, pluginContext, KotlinFileExtractor.DeclarationStack(), globalExtensionState)
val fileExtractor =
KotlinFileExtractor(
logger,
sftw,
linesOfCode,
srcFilePath,
null,
externalDeclExtractor,
primitiveTypeMapping,
pluginContext,
KotlinFileExtractor.DeclarationStack(),
globalExtensionState
)
fileExtractor.extractFileContents(srcFile, sftw.fileId)
externalDeclExtractor.extractExternalClasses()
}
if (checkTrapIdentical && trapFileWriter.exists()) {
if (equivalentTrap(trapFileWriter.getTempReader(), trapFileWriter.getRealReader())) {
if (
equivalentTrap(trapFileWriter.getTempReader(), trapFileWriter.getRealReader())
) {
trapFileWriter.deleteTemp()
} else {
trapFileWriter.renameTempToDifferent()
@@ -372,17 +470,26 @@ enum class Compression(val extension: String) {
return GZIPOutputStream(file.outputStream()).bufferedWriter()
}
};
abstract fun bufferedWriter(file: File): BufferedWriter
}
private fun getTrapFileWriter(compression: Compression, logger: FileLogger, trapFileName: String): TrapFileWriter {
private fun getTrapFileWriter(
compression: Compression,
logger: FileLogger,
trapFileName: String
): TrapFileWriter {
return when (compression) {
Compression.NONE -> NonCompressedTrapFileWriter(logger, trapFileName)
Compression.GZIP -> GZipCompressedTrapFileWriter(logger, trapFileName)
}
}
private abstract class TrapFileWriter(val logger: FileLogger, trapName: String, val extension: String) {
private abstract class TrapFileWriter(
val logger: FileLogger,
trapName: String,
val extension: String
) {
private val realFile = File(trapName + extension)
private val parentDir = realFile.parentFile
lateinit private var tempFile: File
@@ -404,6 +511,7 @@ private abstract class TrapFileWriter(val logger: FileLogger, trapName: String,
}
abstract protected fun getReader(file: File): BufferedReader
abstract protected fun getWriter(file: File): BufferedWriter
fun getRealReader(): BufferedReader {
@@ -431,7 +539,8 @@ private abstract class TrapFileWriter(val logger: FileLogger, trapName: String,
}
fun renameTempToDifferent() {
val trapDifferentFile = File.createTempFile(realFile.getName() + ".", ".trap.different" + extension, parentDir)
val trapDifferentFile =
File.createTempFile(realFile.getName() + ".", ".trap.different" + extension, parentDir)
if (tempFile.renameTo(trapDifferentFile)) {
logger.warn("TRAP difference: $realFile vs $trapDifferentFile")
} else {
@@ -447,7 +556,8 @@ private abstract class TrapFileWriter(val logger: FileLogger, trapName: String,
}
}
private class NonCompressedTrapFileWriter(logger: FileLogger, trapName: String): TrapFileWriter(logger, trapName, "") {
private class NonCompressedTrapFileWriter(logger: FileLogger, trapName: String) :
TrapFileWriter(logger, trapName, "") {
override protected fun getReader(file: File): BufferedReader {
return file.bufferedReader()
}
@@ -457,12 +567,17 @@ private class NonCompressedTrapFileWriter(logger: FileLogger, trapName: String):
}
}
private class GZipCompressedTrapFileWriter(logger: FileLogger, trapName: String): TrapFileWriter(logger, trapName, ".gz") {
private class GZipCompressedTrapFileWriter(logger: FileLogger, trapName: String) :
TrapFileWriter(logger, trapName, ".gz") {
override protected fun getReader(file: File): BufferedReader {
return BufferedReader(InputStreamReader(GZIPInputStream(BufferedInputStream(FileInputStream(file)))))
return BufferedReader(
InputStreamReader(GZIPInputStream(BufferedInputStream(FileInputStream(file))))
)
}
override protected fun getWriter(file: File): BufferedWriter {
return BufferedWriter(OutputStreamWriter(GZIPOutputStream(BufferedOutputStream(FileOutputStream(file)))))
return BufferedWriter(
OutputStreamWriter(GZIPOutputStream(BufferedOutputStream(FileOutputStream(file))))
)
}
}

5404
java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt
View File

File diff suppressed because it is too large Load Diff

1351
java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt
View File

File diff suppressed because it is too large Load Diff

28
java/kotlin-extractor/src/main/kotlin/Label.kt
View File

@@ -1,31 +1,21 @@
package com.github.codeql
import java.io.PrintWriter
import java.io.StringWriter
/**
* This represents a label (`#...`) in a TRAP file.
*/
interface Label<T: AnyDbType> {
fun <U: AnyDbType> cast(): Label<U> {
@Suppress("UNCHECKED_CAST")
return this as Label<U>
/** This represents a label (`#...`) in a TRAP file. */
interface Label<T : AnyDbType> {
fun <U : AnyDbType> cast(): Label<U> {
@Suppress("UNCHECKED_CAST") return this as Label<U>
}
}
/**
* The label `#i`, e.g. `#123`. Most labels we generate are of this
* form.
*/
class IntLabel<T: AnyDbType>(val i: Int): Label<T> {
/** The label `#i`, e.g. `#123`. Most labels we generate are of this form. */
class IntLabel<T : AnyDbType>(val i: Int) : Label<T> {
override fun toString(): String = "#$i"
}
/**
* The label `#name`, e.g. `#compilation`. This is used when labels are
* shared between different components (e.g. when both the interceptor
* and the extractor need to refer to the same label).
* The label `#name`, e.g. `#compilation`. This is used when labels are shared between different
* components (e.g. when both the interceptor and the extractor need to refer to the same label).
*/
class StringLabel<T: AnyDbType>(val name: String): Label<T> {
class StringLabel<T : AnyDbType>(val name: String) : Label<T> {
override fun toString(): String = "#$name"
}

16
java/kotlin-extractor/src/main/kotlin/LinesOfCode.kt
View File

@@ -2,11 +2,7 @@ package com.github.codeql
import org.jetbrains.kotlin.ir.declarations.*
class LinesOfCode(
val logger: FileLogger,
val tw: FileTrapWriter,
val file: IrFile
) {
class LinesOfCode(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
val linesOfCodePSI = LinesOfCodePSI(logger, tw, file)
val linesOfCodeLighterAST = LinesOfCodeLighterAST(logger, tw, file)
@@ -14,7 +10,10 @@ class LinesOfCode(
val psiExtracted = linesOfCodePSI.linesOfCodeInFile(id)
val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInFile(id)
if (psiExtracted && lighterASTExtracted) {
logger.warnElement("Both PSI and LighterAST number-of-lines-in-file information for ${file.path}.", file)
logger.warnElement(
"Both PSI and LighterAST number-of-lines-in-file information for ${file.path}.",
file
)
}
}
@@ -22,7 +21,10 @@ class LinesOfCode(
val psiExtracted = linesOfCodePSI.linesOfCodeInDeclaration(d, id)
val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInDeclaration(d, id)
if (psiExtracted && lighterASTExtracted) {
logger.warnElement("Both PSI and LighterAST number-of-lines-in-file information for declaration.", d)
logger.warnElement(
"Both PSI and LighterAST number-of-lines-in-file information for declaration.",
d
)
}
}
}

15
java/kotlin-extractor/src/main/kotlin/LinesOfCodePSI.kt
View File

@@ -7,18 +7,15 @@ import com.intellij.psi.PsiWhiteSpace
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.kdoc.psi.api.KDocElement
import org.jetbrains.kotlin.psi.KtCodeFragment
import org.jetbrains.kotlin.psi.KtVisitor
class LinesOfCodePSI(
val logger: FileLogger,
val tw: FileTrapWriter,
val file: IrFile
) {
val psi2Ir = getPsi2Ir().also {
class LinesOfCodePSI(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
val psi2Ir =
getPsi2Ir().also {
if (it == null) {
logger.warn("Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
logger.warn(
"Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})"
)
}
}

299
java/kotlin-extractor/src/main/kotlin/MetaAnnotationSupport.kt
View File

@@ -3,6 +3,8 @@ package com.github.codeql
import com.github.codeql.utils.versions.copyParameterToFunction
import com.github.codeql.utils.versions.createImplicitParameterDeclarationWithWrappedDescriptor
import com.github.codeql.utils.versions.getAnnotationType
import java.lang.annotation.ElementType
import java.util.HashSet
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.builtins.StandardNames
import org.jetbrains.kotlin.config.JvmTarget
@@ -48,10 +50,12 @@ import org.jetbrains.kotlin.load.java.JvmAnnotationNames
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
import java.lang.annotation.ElementType
import java.util.HashSet
class MetaAnnotationSupport(private val logger: FileLogger, private val pluginContext: IrPluginContext, private val extractor: KotlinFileExtractor) {
class MetaAnnotationSupport(
private val logger: FileLogger,
private val pluginContext: IrPluginContext,
private val extractor: KotlinFileExtractor
) {
// Taken from AdditionalIrUtils.kt (not available in Kotlin < 1.6)
private val IrConstructorCall.annotationClass
@@ -82,8 +86,7 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
wrapAnnotationEntriesInContainer(annotationClass, containerClass, grouped)?.let {
result.add(it)
}
else
logger.warnElement("Failed to find an annotation container class", annotationClass)
else logger.warnElement("Failed to find an annotation container class", annotationClass)
}
return result
}
@@ -91,9 +94,14 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
// Adapted from RepeatedAnnotationLowering.kt
private fun getOrCreateContainerClass(annotationClass: IrClass): IrClass? {
val metaAnnotations = annotationClass.annotations
val jvmRepeatable = metaAnnotations.find { it.symbol.owner.parentAsClass.fqNameWhenAvailable == JvmAnnotationNames.REPEATABLE_ANNOTATION }
val jvmRepeatable =
metaAnnotations.find {
it.symbol.owner.parentAsClass.fqNameWhenAvailable ==
JvmAnnotationNames.REPEATABLE_ANNOTATION
}
return if (jvmRepeatable != null) {
((jvmRepeatable.getValueArgument(0) as? IrClassReference)?.symbol as? IrClassSymbol)?.owner
((jvmRepeatable.getValueArgument(0) as? IrClassReference)?.symbol as? IrClassSymbol)
?.owner
} else {
getOrCreateSyntheticRepeatableAnnotationContainer(annotationClass)
}
@@ -108,14 +116,22 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
val annotationType = annotationClass.typeWith()
val containerConstructor = containerClass.primaryConstructor
if (containerConstructor == null) {
logger.warnElement("Expected container class to have a primary constructor", containerClass)
logger.warnElement(
"Expected container class to have a primary constructor",
containerClass
)
return null
} else {
return IrConstructorCallImpl.fromSymbolOwner(containerClass.defaultType, containerConstructor.symbol).apply {
return IrConstructorCallImpl.fromSymbolOwner(
containerClass.defaultType,
containerConstructor.symbol
)
.apply {
putValueArgument(
0,
IrVarargImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
pluginContext.irBuiltIns.arrayClass.typeWith(annotationType),
annotationType,
entries
@@ -134,14 +150,19 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
// Taken from AdditionalClassAnnotationLowering.kt
private fun loadAnnotationTargets(targetEntry: IrConstructorCall): Set<KotlinTarget>? {
val valueArgument = targetEntry.getValueArgument(0) as? IrVararg ?: return null
return valueArgument.elements.filterIsInstance<IrGetEnumValue>().mapNotNull {
KotlinTarget.valueOrNull(it.symbol.owner.name.asString())
}.toSet()
return valueArgument.elements
.filterIsInstance<IrGetEnumValue>()
.mapNotNull { KotlinTarget.valueOrNull(it.symbol.owner.name.asString()) }
.toSet()
}
private val javaAnnotationTargetElementType by lazy { extractor.referenceExternalClass("java.lang.annotation.ElementType") }
private val javaAnnotationTargetElementType by lazy {
extractor.referenceExternalClass("java.lang.annotation.ElementType")
}
private val javaAnnotationTarget by lazy { extractor.referenceExternalClass("java.lang.annotation.Target") }
private val javaAnnotationTarget by lazy {
extractor.referenceExternalClass("java.lang.annotation.Target")
}
private fun findEnumEntry(c: IrClass, name: String) =
c.declarations.filterIsInstance<IrEnumEntry>().find { it.name.asString() == name }
@@ -168,7 +189,8 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
private val jvm8TargetMap by lazy {
javaAnnotationTargetElementType?.let {
jvm6TargetMap?.let { j6Map ->
j6Map + mapOf(
j6Map +
mapOf(
KotlinTarget.TYPE_PARAMETER to findEnumEntry(it, "TYPE_PARAMETER"),
KotlinTarget.TYPE to findEnumEntry(it, "TYPE_USE")
)
@@ -179,45 +201,59 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
private fun getAnnotationTargetMap() =
if (pluginContext.platform?.any { it.targetPlatformVersion == JvmTarget.JVM_1_6 } == true)
jvm6TargetMap
else
jvm8TargetMap
else jvm8TargetMap
// Adapted from AdditionalClassAnnotationLowering.kt
private fun generateTargetAnnotation(c: IrClass): IrConstructorCall? {
if (c.hasAnnotation(JvmAnnotationNames.TARGET_ANNOTATION))
return null
if (c.hasAnnotation(JvmAnnotationNames.TARGET_ANNOTATION)) return null
val elementType = javaAnnotationTargetElementType ?: return null
val targetType = javaAnnotationTarget ?: return null
val targetConstructor = targetType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null
val targetConstructor =
targetType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null
val targets = getApplicableTargetSet(c) ?: return null
val annotationTargetMap = getAnnotationTargetMap() ?: return null
val javaTargets = targets.mapNotNullTo(HashSet()) { annotationTargetMap[it] }.sortedBy {
ElementType.valueOf(it.symbol.owner.name.asString())
}
val vararg = IrVarargImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
val javaTargets =
targets
.mapNotNullTo(HashSet()) { annotationTargetMap[it] }
.sortedBy { ElementType.valueOf(it.symbol.owner.name.asString()) }
val vararg =
IrVarargImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
type = pluginContext.irBuiltIns.arrayClass.typeWith(elementType.defaultType),
varargElementType = elementType.defaultType
)
for (target in javaTargets) {
vararg.elements.add(
IrGetEnumValueImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, elementType.defaultType, target.symbol
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
elementType.defaultType,
target.symbol
)
)
}
return IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, targetConstructor.returnType, targetConstructor.symbol, 0
).apply {
putValueArgument(0, vararg)
}
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
targetConstructor.returnType,
targetConstructor.symbol,
0
)
.apply { putValueArgument(0, vararg) }
}
private val javaAnnotationRetention by lazy { extractor.referenceExternalClass("java.lang.annotation.Retention") }
private val javaAnnotationRetentionPolicy by lazy { extractor.referenceExternalClass("java.lang.annotation.RetentionPolicy") }
private val javaAnnotationRetentionPolicyRuntime by lazy { javaAnnotationRetentionPolicy?.let { findEnumEntry(it, "RUNTIME") } }
private val javaAnnotationRetention by lazy {
extractor.referenceExternalClass("java.lang.annotation.Retention")
}
private val javaAnnotationRetentionPolicy by lazy {
extractor.referenceExternalClass("java.lang.annotation.RetentionPolicy")
}
private val javaAnnotationRetentionPolicyRuntime by lazy {
javaAnnotationRetentionPolicy?.let { findEnumEntry(it, "RUNTIME") }
}
private val annotationRetentionMap by lazy {
javaAnnotationRetentionPolicy?.let {
@@ -232,62 +268,95 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
// Taken from AnnotationCodegen.kt (not available in Kotlin < 1.6.20)
private fun IrClass.getAnnotationRetention(): KotlinRetention? {
val retentionArgument =
getAnnotation(StandardNames.FqNames.retention)?.getValueArgument(0)
as? IrGetEnumValue ?: return null
getAnnotation(StandardNames.FqNames.retention)?.getValueArgument(0) as? IrGetEnumValue
?: return null
val retentionArgumentValue = retentionArgument.symbol.owner
return KotlinRetention.valueOf(retentionArgumentValue.name.asString())
}
// Taken from AdditionalClassAnnotationLowering.kt
private fun generateRetentionAnnotation(irClass: IrClass): IrConstructorCall? {
if (irClass.hasAnnotation(JvmAnnotationNames.RETENTION_ANNOTATION))
return null
if (irClass.hasAnnotation(JvmAnnotationNames.RETENTION_ANNOTATION)) return null
val retentionMap = annotationRetentionMap ?: return null
val kotlinRetentionPolicy = irClass.getAnnotationRetention()
val javaRetentionPolicy = kotlinRetentionPolicy?.let { retentionMap[it] } ?: javaAnnotationRetentionPolicyRuntime ?: return null
val javaRetentionPolicy =
kotlinRetentionPolicy?.let { retentionMap[it] }
?: javaAnnotationRetentionPolicyRuntime
?: return null
val retentionPolicyType = javaAnnotationRetentionPolicy ?: return null
val retentionType = javaAnnotationRetention ?: return null
val targetConstructor = retentionType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null
val targetConstructor =
retentionType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null
return IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, targetConstructor.returnType, targetConstructor.symbol, 0
).apply {
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
targetConstructor.returnType,
targetConstructor.symbol,
0
)
.apply {
putValueArgument(
0,
IrGetEnumValueImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, retentionPolicyType.defaultType, javaRetentionPolicy.symbol
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
retentionPolicyType.defaultType,
javaRetentionPolicy.symbol
)
)
}
}
private val javaAnnotationRepeatable by lazy { extractor.referenceExternalClass("java.lang.annotation.Repeatable") }
private val kotlinAnnotationRepeatableContainer by lazy { extractor.referenceExternalClass("kotlin.jvm.internal.RepeatableContainer") }
private val javaAnnotationRepeatable by lazy {
extractor.referenceExternalClass("java.lang.annotation.Repeatable")
}
private val kotlinAnnotationRepeatableContainer by lazy {
extractor.referenceExternalClass("kotlin.jvm.internal.RepeatableContainer")
}
// Taken from declarationBuilders.kt (not available in Kotlin < 1.6):
private fun addDefaultGetter(p: IrProperty, parentClass: IrClass) {
val field = p.backingField ?:
run { logger.warnElement("Expected property to have a backing field", p); return }
val field =
p.backingField
?: run {
logger.warnElement("Expected property to have a backing field", p)
return
}
p.addGetter {
origin = IrDeclarationOrigin.DEFAULT_PROPERTY_ACCESSOR
returnType = field.type
}.apply {
val thisReceiever = parentClass.thisReceiver ?:
run { logger.warnElement("Expected property's parent class to have a receiver parameter", parentClass); return }
}
.apply {
val thisReceiever =
parentClass.thisReceiver
?: run {
logger.warnElement(
"Expected property's parent class to have a receiver parameter",
parentClass
)
return
}
val newParam = copyParameterToFunction(thisReceiever, this)
dispatchReceiverParameter = newParam
body = factory.createBlockBody(UNDEFINED_OFFSET, UNDEFINED_OFFSET).apply({
body =
factory
.createBlockBody(UNDEFINED_OFFSET, UNDEFINED_OFFSET)
.apply({
this.statements.add(
IrReturnImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
pluginContext.irBuiltIns.nothingType,
symbol,
IrGetFieldImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
field.symbol,
field.type,
IrGetValueImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
newParam.type,
newParam.symbol
)
@@ -300,41 +369,50 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
// Taken from JvmCachedDeclarations.kt
private fun getOrCreateSyntheticRepeatableAnnotationContainer(annotationClass: IrClass) =
extractor.globalExtensionState.syntheticRepeatableAnnotationContainers.getOrPut(annotationClass) {
val containerClass = pluginContext.irFactory.buildClass {
extractor.globalExtensionState.syntheticRepeatableAnnotationContainers.getOrPut(
annotationClass
) {
val containerClass =
pluginContext.irFactory
.buildClass {
kind = ClassKind.ANNOTATION_CLASS
name = Name.identifier("Container")
}.apply {
}
.apply {
createImplicitParameterDeclarationWithWrappedDescriptor()
parent = annotationClass
superTypes = listOf(getAnnotationType(pluginContext))
}
val propertyName = Name.identifier("value")
val propertyType = pluginContext.irBuiltIns.arrayClass.typeWith(annotationClass.typeWith())
val propertyType =
pluginContext.irBuiltIns.arrayClass.typeWith(annotationClass.typeWith())
containerClass.addConstructor {
isPrimary = true
}.apply {
addValueParameter(propertyName.identifier, propertyType)
}
containerClass
.addConstructor { isPrimary = true }
.apply { addValueParameter(propertyName.identifier, propertyType) }
containerClass.addProperty {
name = propertyName
}.apply property@{
backingField = pluginContext.irFactory.buildField {
containerClass
.addProperty { name = propertyName }
.apply property@{
backingField =
pluginContext.irFactory
.buildField {
name = propertyName
type = propertyType
}.apply {
}
.apply {
parent = containerClass
correspondingPropertySymbol = this@property.symbol
}
addDefaultGetter(this, containerClass)
}
val repeatableContainerAnnotation = kotlinAnnotationRepeatableContainer?.constructors?.single()
val repeatableContainerAnnotation =
kotlinAnnotationRepeatableContainer?.constructors?.single()
containerClass.annotations = annotationClass.annotations
containerClass.annotations =
annotationClass.annotations
.filter {
it.isAnnotationWithEqualFqName(StandardNames.FqNames.retention) ||
it.isAnnotationWithEqualFqName(StandardNames.FqNames.target)
@@ -343,7 +421,11 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
listOfNotNull(
repeatableContainerAnnotation?.let {
IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, it.returnType, it.symbol, 0
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
it.returnType,
it.symbol,
0
)
}
)
@@ -352,45 +434,80 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
}
// Adapted from AdditionalClassAnnotationLowering.kt
private fun generateRepeatableAnnotation(irClass: IrClass, extractAnnotationTypeAccesses: Boolean): IrConstructorCall? {
if (!irClass.hasAnnotation(StandardNames.FqNames.repeatable) ||
private fun generateRepeatableAnnotation(
irClass: IrClass,
extractAnnotationTypeAccesses: Boolean
): IrConstructorCall? {
if (
!irClass.hasAnnotation(StandardNames.FqNames.repeatable) ||
irClass.hasAnnotation(JvmAnnotationNames.REPEATABLE_ANNOTATION)
) return null
)
return null
val repeatableConstructor = javaAnnotationRepeatable?.declarations?.firstIsInstanceOrNull<IrConstructor>() ?: return null
val repeatableConstructor =
javaAnnotationRepeatable?.declarations?.firstIsInstanceOrNull<IrConstructor>()
?: return null
val containerClass = getOrCreateSyntheticRepeatableAnnotationContainer(irClass)
// Whenever a repeatable annotation with a Kotlin-synthesised container is extracted, extract the synthetic container to the same trap file.
extractor.extractClassSource(containerClass, extractDeclarations = true, extractStaticInitializer = true, extractPrivateMembers = true, extractFunctionBodies = extractAnnotationTypeAccesses)
// Whenever a repeatable annotation with a Kotlin-synthesised container is extracted,
// extract the synthetic container to the same trap file.
extractor.extractClassSource(
containerClass,
extractDeclarations = true,
extractStaticInitializer = true,
extractPrivateMembers = true,
extractFunctionBodies = extractAnnotationTypeAccesses
)
val containerReference = IrClassReferenceImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, pluginContext.irBuiltIns.kClassClass.typeWith(containerClass.defaultType),
containerClass.symbol, containerClass.defaultType
val containerReference =
IrClassReferenceImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
pluginContext.irBuiltIns.kClassClass.typeWith(containerClass.defaultType),
containerClass.symbol,
containerClass.defaultType
)
return IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, repeatableConstructor.returnType, repeatableConstructor.symbol, 0
).apply {
putValueArgument(0, containerReference)
}
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
repeatableConstructor.returnType,
repeatableConstructor.symbol,
0
)
.apply { putValueArgument(0, containerReference) }
}
private val javaAnnotationDocumented by lazy { extractor.referenceExternalClass("java.lang.annotation.Documented") }
private val javaAnnotationDocumented by lazy {
extractor.referenceExternalClass("java.lang.annotation.Documented")
}
// Taken from AdditionalClassAnnotationLowering.kt
private fun generateDocumentedAnnotation(irClass: IrClass): IrConstructorCall? {
if (!irClass.hasAnnotation(StandardNames.FqNames.mustBeDocumented) ||
if (
!irClass.hasAnnotation(StandardNames.FqNames.mustBeDocumented) ||
irClass.hasAnnotation(JvmAnnotationNames.DOCUMENTED_ANNOTATION)
) return null
)
return null
val documentedConstructor = javaAnnotationDocumented?.declarations?.firstIsInstanceOrNull<IrConstructor>() ?: return null
val documentedConstructor =
javaAnnotationDocumented?.declarations?.firstIsInstanceOrNull<IrConstructor>()
?: return null
return IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, documentedConstructor.returnType, documentedConstructor.symbol, 0
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
documentedConstructor.returnType,
documentedConstructor.symbol,
0
)
}
fun generateJavaMetaAnnotations(c: IrClass, extractAnnotationTypeAccesses: Boolean) =
// This is essentially AdditionalClassAnnotationLowering adapted to run outside the backend.
listOfNotNull(generateTargetAnnotation(c), generateRetentionAnnotation(c), generateRepeatableAnnotation(c, extractAnnotationTypeAccesses), generateDocumentedAnnotation(c))
listOfNotNull(
generateTargetAnnotation(c),
generateRetentionAnnotation(c),
generateRepeatableAnnotation(c, extractAnnotationTypeAccesses),
generateDocumentedAnnotation(c)
)
}

68
java/kotlin-extractor/src/main/kotlin/PrimitiveTypeInfo.kt
View File

@@ -1,34 +1,36 @@
package com.github.codeql
import com.github.codeql.utils.*
import com.github.codeql.utils.versions.*
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.builtins.StandardNames
import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.ir.declarations.IrPackageFragment
import org.jetbrains.kotlin.ir.types.IrSimpleType
import org.jetbrains.kotlin.ir.types.classOrNull
import org.jetbrains.kotlin.name.FqName
import com.github.codeql.utils.*
import com.github.codeql.utils.versions.*
class PrimitiveTypeMapping(val logger: Logger, val pluginContext: IrPluginContext) {
fun getPrimitiveInfo(s: IrSimpleType) =
s.classOrNull?.let {
if ((it.owner.parent as? IrPackageFragment)?.packageFqName == StandardNames.BUILT_INS_PACKAGE_FQ_NAME)
if (
(it.owner.parent as? IrPackageFragment)?.packageFqName ==
StandardNames.BUILT_INS_PACKAGE_FQ_NAME
)
mapping[it.owner.name]
else
null
else null
}
data class PrimitiveTypeInfo(
val primitiveName: String?,
val otherIsPrimitive: Boolean,
val javaClass: IrClass,
val kotlinPackageName: String, val kotlinClassName: String
val kotlinPackageName: String,
val kotlinClassName: String
)
private fun findClass(fqName: String, fallback: IrClass): IrClass {
val symbol = getClassByFqName(pluginContext, fqName)
if(symbol == null) {
if (symbol == null) {
logger.warn("Can't find $fqName")
// Do the best we can
return fallback
@@ -37,7 +39,8 @@ class PrimitiveTypeMapping(val logger: Logger, val pluginContext: IrPluginContex
}
}
private val mapping = {
private val mapping =
{
val kotlinByte = pluginContext.irBuiltIns.byteClass.owner
val javaLangByte = findClass("java.lang.Byte", kotlinByte)
val kotlinShort = pluginContext.irBuiltIns.shortClass.owner
@@ -69,25 +72,34 @@ class PrimitiveTypeMapping(val logger: Logger, val pluginContext: IrPluginContex
val javaLangVoid = findClass("java.lang.Void", kotlinNothing)
mapOf(
StandardNames.FqNames._byte.shortName() to PrimitiveTypeInfo("byte", true, javaLangByte, "kotlin", "Byte"),
StandardNames.FqNames._short.shortName() to PrimitiveTypeInfo("short", true, javaLangShort, "kotlin", "Short"),
StandardNames.FqNames._int.shortName() to PrimitiveTypeInfo("int", true, javaLangInteger, "kotlin", "Int"),
StandardNames.FqNames._long.shortName() to PrimitiveTypeInfo("long", true, javaLangLong, "kotlin", "Long"),
StandardNames.FqNames.uByteFqName.shortName() to PrimitiveTypeInfo("byte", true, kotlinUByte, "kotlin", "UByte"),
StandardNames.FqNames.uShortFqName.shortName() to PrimitiveTypeInfo("short", true, kotlinUShort, "kotlin", "UShort"),
StandardNames.FqNames.uIntFqName.shortName() to PrimitiveTypeInfo("int", true, kotlinUInt, "kotlin", "UInt"),
StandardNames.FqNames.uLongFqName.shortName() to PrimitiveTypeInfo("long", true, kotlinULong, "kotlin", "ULong"),
StandardNames.FqNames._double.shortName() to PrimitiveTypeInfo("double", true, javaLangDouble, "kotlin", "Double"),
StandardNames.FqNames._float.shortName() to PrimitiveTypeInfo("float", true, javaLangFloat, "kotlin", "Float"),
StandardNames.FqNames._boolean.shortName() to PrimitiveTypeInfo("boolean", true, javaLangBoolean, "kotlin", "Boolean"),
StandardNames.FqNames._char.shortName() to PrimitiveTypeInfo("char", true, javaLangCharacter, "kotlin", "Char"),
StandardNames.FqNames.unit.shortName() to PrimitiveTypeInfo("void", false, kotlinUnit, "kotlin", "Unit"),
StandardNames.FqNames.nothing.shortName() to PrimitiveTypeInfo(null, true, javaLangVoid, "kotlin", "Nothing"),
StandardNames.FqNames._byte.shortName() to
PrimitiveTypeInfo("byte", true, javaLangByte, "kotlin", "Byte"),
StandardNames.FqNames._short.shortName() to
PrimitiveTypeInfo("short", true, javaLangShort, "kotlin", "Short"),
StandardNames.FqNames._int.shortName() to
PrimitiveTypeInfo("int", true, javaLangInteger, "kotlin", "Int"),
StandardNames.FqNames._long.shortName() to
PrimitiveTypeInfo("long", true, javaLangLong, "kotlin", "Long"),
StandardNames.FqNames.uByteFqName.shortName() to
PrimitiveTypeInfo("byte", true, kotlinUByte, "kotlin", "UByte"),
StandardNames.FqNames.uShortFqName.shortName() to
PrimitiveTypeInfo("short", true, kotlinUShort, "kotlin", "UShort"),
StandardNames.FqNames.uIntFqName.shortName() to
PrimitiveTypeInfo("int", true, kotlinUInt, "kotlin", "UInt"),
StandardNames.FqNames.uLongFqName.shortName() to
PrimitiveTypeInfo("long", true, kotlinULong, "kotlin", "ULong"),
StandardNames.FqNames._double.shortName() to
PrimitiveTypeInfo("double", true, javaLangDouble, "kotlin", "Double"),
StandardNames.FqNames._float.shortName() to
PrimitiveTypeInfo("float", true, javaLangFloat, "kotlin", "Float"),
StandardNames.FqNames._boolean.shortName() to
PrimitiveTypeInfo("boolean", true, javaLangBoolean, "kotlin", "Boolean"),
StandardNames.FqNames._char.shortName() to
PrimitiveTypeInfo("char", true, javaLangCharacter, "kotlin", "Char"),
StandardNames.FqNames.unit.shortName() to
PrimitiveTypeInfo("void", false, kotlinUnit, "kotlin", "Unit"),
StandardNames.FqNames.nothing.shortName() to
PrimitiveTypeInfo(null, true, javaLangVoid, "kotlin", "Nothing"),
)
}()
}

319
java/kotlin-extractor/src/main/kotlin/TrapWriter.kt
View File

@@ -1,92 +1,87 @@
package com.github.codeql
import com.github.codeql.KotlinUsesExtractor.LocallyVisibleFunctionLabels
import com.semmle.extractor.java.PopulateFile
import com.semmle.util.unicode.UTF8Util
import java.io.BufferedWriter
import java.io.File
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.path
import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.ir.declarations.IrFunction
import org.jetbrains.kotlin.ir.declarations.IrVariable
import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
import org.jetbrains.kotlin.ir.declarations.path
import org.jetbrains.kotlin.ir.expressions.IrCall
import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET
import com.semmle.extractor.java.PopulateFile
import com.semmle.util.unicode.UTF8Util
import org.jetbrains.kotlin.ir.expressions.IrCall
/**
* Each `.trap` file has a `TrapLabelManager` while we are writing it.
* It provides fresh TRAP label names, and maintains a mapping from keys
* (`@"..."`) to labels.
* Each `.trap` file has a `TrapLabelManager` while we are writing it. It provides fresh TRAP label
* names, and maintains a mapping from keys (`@"..."`) to labels.
*/
class TrapLabelManager {
/** The next integer to use as a label name. */
private var nextInt: Int = 100
/** Returns a fresh label. */
fun <T: AnyDbType> getFreshLabel(): Label<T> {
fun <T : AnyDbType> getFreshLabel(): Label<T> {
return IntLabel(nextInt++)
}
/**
* A mapping from a key (`@"..."`) to the label defined to be that
* key, if any.
*/
/** A mapping from a key (`@"..."`) to the label defined to be that key, if any. */
val labelMapping: MutableMap<String, Label<*>> = mutableMapOf<String, Label<*>>()
val anonymousTypeMapping: MutableMap<IrClass, TypeResults> = mutableMapOf()
val locallyVisibleFunctionLabelMapping: MutableMap<IrFunction, LocallyVisibleFunctionLabels> = mutableMapOf()
val locallyVisibleFunctionLabelMapping: MutableMap<IrFunction, LocallyVisibleFunctionLabels> =
mutableMapOf()
/**
* The set of labels of generic specialisations that we have extracted
* in this TRAP file.
* We can't easily avoid duplication between TRAP files, as the labels
* contain references to other labels, so we just accept this
* duplication.
* The set of labels of generic specialisations that we have extracted in this TRAP file. We
* can't easily avoid duplication between TRAP files, as the labels contain references to other
* labels, so we just accept this duplication.
*/
val genericSpecialisationsExtracted = HashSet<String>()
/**
* Sometimes, when we extract a file class we don't have the IrFile
* for it, so we are not able to give it a location. This means that
* the location is written outside of the label creation.
* This allows us to keep track of whether we've written the location
* already in this TRAP file, to avoid duplication.
* Sometimes, when we extract a file class we don't have the IrFile for it, so we are not able
* to give it a location. This means that the location is written outside of the label creation.
* This allows us to keep track of whether we've written the location already in this TRAP file,
* to avoid duplication.
*/
val fileClassLocationsExtracted = HashSet<IrFile>()
}
/**
* A `TrapWriter` is used to write TRAP to a particular TRAP file.
* There may be multiple `TrapWriter`s for the same file, as different
* instances will have different additional state, but they must all
* share the same `TrapLabelManager` and `BufferedWriter`.
* A `TrapWriter` is used to write TRAP to a particular TRAP file. There may be multiple
* `TrapWriter`s for the same file, as different instances will have different additional state, but
* they must all share the same `TrapLabelManager` and `BufferedWriter`.
*/
// TODO lm was `protected` before anonymousTypeMapping and locallyVisibleFunctionLabelMapping moved into it. Should we re-protect it and provide accessors?
abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLabelManager, private val bw: BufferedWriter) {
// TODO lm was `protected` before anonymousTypeMapping and locallyVisibleFunctionLabelMapping moved
// into it. Should we re-protect it and provide accessors?
abstract class TrapWriter(
protected val loggerBase: LoggerBase,
val lm: TrapLabelManager,
private val bw: BufferedWriter
) {
abstract fun getDiagnosticTrapWriter(): DiagnosticTrapWriter
/**
* Returns the label that is defined to be the given key, if such
* a label exists, and `null` otherwise. Most users will want to use
* `getLabelFor` instead, which allows non-existent labels to be
* initialised.
* Returns the label that is defined to be the given key, if such a label exists, and `null`
* otherwise. Most users will want to use `getLabelFor` instead, which allows non-existent
* labels to be initialised.
*/
fun <T: AnyDbType> getExistingLabelFor(key: String): Label<T>? {
fun <T : AnyDbType> getExistingLabelFor(key: String): Label<T>? {
return lm.labelMapping.get(key)?.cast<T>()
}
/**
* Returns the label for the given key, if one exists.
* Otherwise, a fresh label is bound to that key, `initialise`
* is run on it, and it is returned.
* Returns the label for the given key, if one exists. Otherwise, a fresh label is bound to that
* key, `initialise` is run on it, and it is returned.
*/
@JvmOverloads // Needed so Java can call a method with an optional argument
fun <T: AnyDbType> getLabelFor(key: String, initialise: (Label<T>) -> Unit = {}): Label<T> {
fun <T : AnyDbType> getLabelFor(key: String, initialise: (Label<T>) -> Unit = {}): Label<T> {
val maybeLabel: Label<T>? = getExistingLabelFor(key)
if(maybeLabel == null) {
if (maybeLabel == null) {
val label: Label<T> = lm.getFreshLabel()
lm.labelMapping.put(key, label)
writeTrap("$label = $key\n")
@@ -97,30 +92,24 @@ abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLab
}
}
/**
* Returns a label for a fresh ID (i.e. a new label bound to `*`).
*/
fun <T: AnyDbType> getFreshIdLabel(): Label<T> {
/** Returns a label for a fresh ID (i.e. a new label bound to `*`). */
fun <T : AnyDbType> getFreshIdLabel(): Label<T> {
val label: Label<T> = lm.getFreshLabel()
writeTrap("$label = *\n")
return label
}
/**
* It is not easy to assign keys to local variables, so they get
* given `*` IDs. However, the same variable may be referred to
* from distant places in the IR, so we need a way to find out
* which label is used for a given local variable. This information
* is stored in this mapping.
*/
private val variableLabelMapping: MutableMap<IrVariable, Label<out DbLocalvar>> = mutableMapOf<IrVariable, Label<out DbLocalvar>>()
/**
* This returns the label used for a local variable, creating one
* if none currently exists.
* It is not easy to assign keys to local variables, so they get given `*` IDs. However, the
* same variable may be referred to from distant places in the IR, so we need a way to find out
* which label is used for a given local variable. This information is stored in this mapping.
*/
private val variableLabelMapping: MutableMap<IrVariable, Label<out DbLocalvar>> =
mutableMapOf<IrVariable, Label<out DbLocalvar>>()
/** This returns the label used for a local variable, creating one if none currently exists. */
fun <T> getVariableLabelFor(v: IrVariable): Label<out DbLocalvar> {
val maybeLabel = variableLabelMapping.get(v)
if(maybeLabel == null) {
if (maybeLabel == null) {
val label = getFreshIdLabel<DbLocalvar>()
variableLabelMapping.put(v, label)
return label
@@ -134,43 +123,41 @@ abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLab
}
/**
* This returns a label for the location described by its arguments.
* Typically users will not want to call this directly, but instead
* use `unknownLocation`, or overloads of this defined by subclasses.
* This returns a label for the location described by its arguments. Typically users will not
* want to call this directly, but instead use `unknownLocation`, or overloads of this defined
* by subclasses.
*/
fun getLocation(fileId: Label<DbFile>, startLine: Int, startColumn: Int, endLine: Int, endColumn: Int): Label<DbLocation> {
fun getLocation(
fileId: Label<DbFile>,
startLine: Int,
startColumn: Int,
endLine: Int,
endColumn: Int
): Label<DbLocation> {
return getLabelFor("@\"loc,{$fileId},$startLine,$startColumn,$endLine,$endColumn\"") {
writeLocations_default(it, fileId, startLine, startColumn, endLine, endColumn)
}
}
/**
* The label for the 'unknown' file ID.
* Users will want to use `unknownLocation` instead.
* This is lazy, as we don't want to define it in a TRAP file unless
* the TRAP file actually contains something in the 'unknown' file.
* The label for the 'unknown' file ID. Users will want to use `unknownLocation` instead. This
* is lazy, as we don't want to define it in a TRAP file unless the TRAP file actually contains
* something in the 'unknown' file.
*/
protected val unknownFileId: Label<DbFile> by lazy {
val unknownFileLabel = "@\";sourcefile\""
getLabelFor(unknownFileLabel, {
writeFiles(it, "")
})
getLabelFor(unknownFileLabel, { writeFiles(it, "") })
}
/**
* The label for the 'unknown' location.
* This is lazy, as we don't want to define it in a TRAP file unless
* the TRAP file actually contains something with an 'unknown'
* location.
* The label for the 'unknown' location. This is lazy, as we don't want to define it in a TRAP
* file unless the TRAP file actually contains something with an 'unknown' location.
*/
val unknownLocation: Label<DbLocation> by lazy {
getWholeFileLocation(unknownFileId)
}
val unknownLocation: Label<DbLocation> by lazy { getWholeFileLocation(unknownFileId) }
/**
* Returns the label for the file `filePath`.
* If `populateFileTables` is true, then this also adds rows to the
* `files` and `folders` tables for this file.
* Returns the label for the file `filePath`. If `populateFileTables` is true, then this also
* adds rows to the `files` and `folders` tables for this file.
*/
fun mkFileId(filePath: String, populateFileTables: Boolean): Label<DbFile> {
// If a file is in a jar, then the Kotlin compiler gives
@@ -178,65 +165,65 @@ abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLab
// it as appropriate, to make the right file ID.
val populateFile = PopulateFile(this)
val splitFilePath = filePath.split("!/")
if(splitFilePath.size == 1) {
if (splitFilePath.size == 1) {
return populateFile.getFileLabel(File(filePath), populateFileTables)
} else {
return populateFile.getFileInJarLabel(File(splitFilePath.get(0)), splitFilePath.get(1), populateFileTables)
return populateFile.getFileInJarLabel(
File(splitFilePath.get(0)),
splitFilePath.get(1),
populateFileTables
)
}
}
/**
* If you have an ID for a file, then this gets a label for the
* location representing the whole of that file.
* If you have an ID for a file, then this gets a label for the location representing the whole
* of that file.
*/
fun getWholeFileLocation(fileId: Label<DbFile>): Label<DbLocation> {
return getLocation(fileId, 0, 0, 0, 0)
}
/**
* Write a raw string into the TRAP file. Users should call one of
* the wrapper functions instead.
* Write a raw string into the TRAP file. Users should call one of the wrapper functions
* instead.
*/
fun writeTrap(trap: String) {
bw.write(trap)
}
/**
* Write a comment into the TRAP file.
*/
/** Write a comment into the TRAP file. */
fun writeComment(comment: String) {
writeTrap("// ${comment.replace("\n", "\n// ")}\n")
}
/**
* Flush the TRAP file.
*/
/** Flush the TRAP file. */
fun flush() {
bw.flush()
}
/**
* Escape a string so that it can be used in a TRAP string literal,
* i.e. with `"` escaped as `""`.
* Escape a string so that it can be used in a TRAP string literal, i.e. with `"` escaped as
* `""`.
*/
fun escapeTrapString(str: String) = str.replace("\"", "\"\"")
/**
* TRAP string literals are limited to 1 megabyte.
*/
/** TRAP string literals are limited to 1 megabyte. */
private val MAX_STRLEN = 1.shl(20)
/**
* Truncate a string, if necessary, so that it can be used as a TRAP
* string literal. TRAP string literals are limited to 1 megabyte.
* Truncate a string, if necessary, so that it can be used as a TRAP string literal. TRAP string
* literals are limited to 1 megabyte.
*/
fun truncateString(str: String): String {
val len = str.length
val newLen = UTF8Util.encodablePrefixLength(str, MAX_STRLEN)
if (newLen < len) {
loggerBase.warn(this.getDiagnosticTrapWriter(),
loggerBase.warn(
this.getDiagnosticTrapWriter(),
"Truncated string of length $len",
"Truncated string of length $len, starting '${str.take(100)}', ending '${str.takeLast(100)}'")
"Truncated string of length $len, starting '${str.take(100)}', ending '${str.takeLast(100)}'"
)
return str.take(newLen)
} else {
return str
@@ -244,69 +231,75 @@ abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLab
}
/**
* Gets a FileTrapWriter like this one (using the same label manager,
* writer etc), but using the given `filePath` for locations.
* Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the
* given `filePath` for locations.
*/
fun makeFileTrapWriter(filePath: String, populateFileTables: Boolean) =
FileTrapWriter(loggerBase, lm, bw, this.getDiagnosticTrapWriter(), filePath, populateFileTables)
FileTrapWriter(
loggerBase,
lm,
bw,
this.getDiagnosticTrapWriter(),
filePath,
populateFileTables
)
/**
* Gets a FileTrapWriter like this one (using the same label manager,
* writer etc), but using the given `IrFile` for locations.
* Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the
* given `IrFile` for locations.
*/
fun makeSourceFileTrapWriter(file: IrFile, populateFileTables: Boolean) =
SourceFileTrapWriter(loggerBase, lm, bw, this.getDiagnosticTrapWriter(), file, populateFileTables)
SourceFileTrapWriter(
loggerBase,
lm,
bw,
this.getDiagnosticTrapWriter(),
file,
populateFileTables
)
}
/**
* A `PlainTrapWriter` has no additional context of its own.
*/
class PlainTrapWriter (
/** A `PlainTrapWriter` has no additional context of its own. */
class PlainTrapWriter(
loggerBase: LoggerBase,
lm: TrapLabelManager,
bw: BufferedWriter,
val dtw: DiagnosticTrapWriter
): TrapWriter (loggerBase, lm, bw) {
) : TrapWriter(loggerBase, lm, bw) {
override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter {
return dtw
}
}
/**
* A `DiagnosticTrapWriter` is a TrapWriter that diagnostics can be
* written to; i.e. it has the #compilation label defined. In practice,
* this means that it is a TrapWriter for the invocation TRAP file.
* A `DiagnosticTrapWriter` is a TrapWriter that diagnostics can be written to; i.e. it has
* the #compilation label defined. In practice, this means that it is a TrapWriter for the
* invocation TRAP file.
*/
class DiagnosticTrapWriter (
loggerBase: LoggerBase,
lm: TrapLabelManager,
bw: BufferedWriter
): TrapWriter (loggerBase, lm, bw) {
class DiagnosticTrapWriter(loggerBase: LoggerBase, lm: TrapLabelManager, bw: BufferedWriter) :
TrapWriter(loggerBase, lm, bw) {
override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter {
return this
}
}
/**
* A `FileTrapWriter` is used when we know which file we are extracting
* entities from, so we can at least give the right file as a location.
* A `FileTrapWriter` is used when we know which file we are extracting entities from, so we can at
* least give the right file as a location.
*
* An ID for the file will be created, and if `populateFileTables` is
* true then we will also add rows to the `files` and `folders` tables
* for it.
* An ID for the file will be created, and if `populateFileTables` is true then we will also add
* rows to the `files` and `folders` tables for it.
*/
open class FileTrapWriter (
open class FileTrapWriter(
loggerBase: LoggerBase,
lm: TrapLabelManager,
bw: BufferedWriter,
val dtw: DiagnosticTrapWriter,
val filePath: String,
populateFileTables: Boolean
): TrapWriter (loggerBase, lm, bw) {
) : TrapWriter(loggerBase, lm, bw) {
/**
* The ID for the file that we are extracting from.
*/
/** The ID for the file that we are extracting from. */
val fileId = mkFileId(filePath, populateFileTables)
override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter {
@@ -320,7 +313,12 @@ open class FileTrapWriter (
var currentMin = default
for (option in options) {
if (option != null && option != UNDEFINED_OFFSET && option != SYNTHETIC_OFFSET && option < currentMin) {
if (
option != null &&
option != UNDEFINED_OFFSET &&
option != SYNTHETIC_OFFSET &&
option < currentMin
) {
currentMin = option
}
}
@@ -344,15 +342,13 @@ open class FileTrapWriter (
return e.endOffset
}
/**
* Gets a label for the location of `e`.
*/
/** Gets a label for the location of `e`. */
fun getLocation(e: IrElement): Label<DbLocation> {
return getLocation(getStartOffset(e), getEndOffset(e))
}
/**
* Gets a label for the location corresponding to `startOffset` and
* `endOffset` within this file.
* Gets a label for the location corresponding to `startOffset` and `endOffset` within this
* file.
*/
open fun getLocation(startOffset: Int, endOffset: Int): Label<DbLocation> {
// We don't have a FileEntry to look up the offsets in, so all
@@ -360,8 +356,8 @@ open class FileTrapWriter (
return getWholeFileLocation()
}
/**
* Gets the location of `e` as a human-readable string. Only used in
* log messages and exception messages.
* Gets the location of `e` as a human-readable string. Only used in log messages and exception
* messages.
*/
open fun getLocationString(e: IrElement): String {
// We don't have a FileEntry to look up the offsets in, so all
@@ -371,50 +367,54 @@ open class FileTrapWriter (
// to be 0.
return "file://$filePath"
}
/**
* Gets a label for the location representing the whole of this file.
*/
/** Gets a label for the location representing the whole of this file. */
fun getWholeFileLocation(): Label<DbLocation> {
return getWholeFileLocation(fileId)
}
}
/**
* A `SourceFileTrapWriter` is used when not only do we know which file
* we are extracting entities from, but we also have an `IrFileEntry`
* (from an `IrFile`) which allows us to map byte offsets to line
* and column numbers.
* A `SourceFileTrapWriter` is used when not only do we know which file we are extracting entities
* from, but we also have an `IrFileEntry` (from an `IrFile`) which allows us to map byte offsets to
* line and column numbers.
*
* An ID for the file will be created, and if `populateFileTables` is
* true then we will also add rows to the `files` and `folders` tables
* for it.
* An ID for the file will be created, and if `populateFileTables` is true then we will also add
* rows to the `files` and `folders` tables for it.
*/
class SourceFileTrapWriter (
class SourceFileTrapWriter(
loggerBase: LoggerBase,
lm: TrapLabelManager,
bw: BufferedWriter,
dtw: DiagnosticTrapWriter,
val irFile: IrFile,
populateFileTables: Boolean) :
FileTrapWriter(loggerBase, lm, bw, dtw, irFile.path, populateFileTables) {
populateFileTables: Boolean
) : FileTrapWriter(loggerBase, lm, bw, dtw, irFile.path, populateFileTables) {
/**
* The file entry for the file that we are extracting from.
* Used to map offsets to line/column numbers.
* The file entry for the file that we are extracting from. Used to map offsets to line/column
* numbers.
*/
private val fileEntry = irFile.fileEntry
override fun getLocation(startOffset: Int, endOffset: Int): Label<DbLocation> {
if (startOffset == UNDEFINED_OFFSET || endOffset == UNDEFINED_OFFSET) {
if (startOffset != endOffset) {
loggerBase.warn(dtw, "Location with inconsistent offsets (start $startOffset, end $endOffset)", null)
loggerBase.warn(
dtw,
"Location with inconsistent offsets (start $startOffset, end $endOffset)",
null
)
}
return getWholeFileLocation()
}
if (startOffset == SYNTHETIC_OFFSET || endOffset == SYNTHETIC_OFFSET) {
if (startOffset != endOffset) {
loggerBase.warn(dtw, "Location with inconsistent offsets (start $startOffset, end $endOffset)", null)
loggerBase.warn(
dtw,
"Location with inconsistent offsets (start $startOffset, end $endOffset)",
null
)
}
return getWholeFileLocation()
}
@@ -428,20 +428,29 @@ class SourceFileTrapWriter (
fileEntry.getLineNumber(startOffset) + 1,
fileEntry.getColumnNumber(startOffset) + 1,
fileEntry.getLineNumber(endOffset) + 1,
fileEntry.getColumnNumber(endOffset) + endColumnOffset)
fileEntry.getColumnNumber(endOffset) + endColumnOffset
)
}
override fun getLocationString(e: IrElement): String {
if (e.startOffset == UNDEFINED_OFFSET || e.endOffset == UNDEFINED_OFFSET) {
if (e.startOffset != e.endOffset) {
loggerBase.warn(dtw, "Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})", null)
loggerBase.warn(
dtw,
"Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})",
null
)
}
return "<unknown location while processing $filePath>"
}
if (e.startOffset == SYNTHETIC_OFFSET || e.endOffset == SYNTHETIC_OFFSET) {
if (e.startOffset != e.endOffset) {
loggerBase.warn(dtw, "Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})", null)
loggerBase.warn(
dtw,
"Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})",
null
)
}
return "<synthetic location while processing $filePath>"
}

29
java/kotlin-extractor/src/main/kotlin/comments/CommentExtractor.kt
View File

@@ -8,13 +8,16 @@ import org.jetbrains.kotlin.ir.expressions.IrBody
import org.jetbrains.kotlin.ir.expressions.IrExpression
import org.jetbrains.kotlin.ir.util.parentClassOrNull
open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, protected val file: IrFile, protected val fileLabel: Label<out DbFile>) {
open class CommentExtractor(
protected val fileExtractor: KotlinFileExtractor,
protected val file: IrFile,
protected val fileLabel: Label<out DbFile>
) {
protected val tw = fileExtractor.tw
protected val logger = fileExtractor.logger
protected fun getLabel(element: IrElement): Label<out DbTop>? {
if (element == file)
return fileLabel
if (element == file) return fileLabel
if (element is IrValueParameter && element.index == -1) {
// Don't attribute comments to the implicit `this` parameter of a function.
@@ -22,12 +25,15 @@ open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, pr
}
val label: String
val existingLabel = if (element is IrVariable) {
// local variables are not named globally, so we need to get them from the variable label cache
val existingLabel =
if (element is IrVariable) {
// local variables are not named globally, so we need to get them from the variable
// label cache
label = "variable ${element.name.asString()}"
tw.getExistingVariableLabelFor(element)
} else if (element is IrFunction && element.isLocalFunction()) {
// local functions are not named globally, so we need to get them from the local function label cache
// local functions are not named globally, so we need to get them from the local
// function label cache
label = "local function ${element.name.asString()}"
fileExtractor.getExistingLocallyVisibleFunctionLabel(element)
} else {
@@ -41,7 +47,7 @@ open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, pr
return existingLabel
}
private fun getLabelForNamedElement(element: IrElement) : String? {
private fun getLabelForNamedElement(element: IrElement): String? {
when (element) {
is IrClass -> return fileExtractor.getClassLabel(element, listOf()).classLabel
is IrTypeParameter -> return fileExtractor.getTypeParameterLabel(element)
@@ -57,14 +63,14 @@ open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, pr
is IrField -> return fileExtractor.getFieldLabel(element)
is IrEnumEntry -> return fileExtractor.getEnumEntryLabel(element)
is IrTypeAlias -> return fileExtractor.getTypeAliasLabel(element)
is IrAnonymousInitializer -> {
val parentClass = element.parentClassOrNull
if (parentClass == null) {
logger.warnElement("Parent of anonymous initializer is not a class", element)
return null
}
// Assign the comment to the class. The content of the `init` blocks might be extracted in multiple constructors.
// Assign the comment to the class. The content of the `init` blocks might be
// extracted in multiple constructors.
return getLabelForNamedElement(parentClass)
}
@@ -74,7 +80,10 @@ open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, pr
// todo add others:
else -> {
logger.warnElement("Unhandled element type found during comment extraction: ${element::class}", element)
logger.warnElement(
"Unhandled element type found during comment extraction: ${element::class}",
element
)
return null
}
}

24
java/kotlin-extractor/src/main/kotlin/comments/CommentExtractorPSI.kt
View File

@@ -15,12 +15,18 @@ import org.jetbrains.kotlin.psi.KtVisitor
import org.jetbrains.kotlin.psi.psiUtil.endOffset
import org.jetbrains.kotlin.psi.psiUtil.startOffset
class CommentExtractorPSI(fileExtractor: KotlinFileExtractor, file: IrFile, fileLabel: Label<out DbFile>): CommentExtractor(fileExtractor, file, fileLabel) {
class CommentExtractorPSI(
fileExtractor: KotlinFileExtractor,
file: IrFile,
fileLabel: Label<out DbFile>
) : CommentExtractor(fileExtractor, file, fileLabel) {
// Returns true if it extracted the comments; false otherwise.
fun extract(): Boolean {
val psi2Ir = getPsi2Ir()
if (psi2Ir == null) {
logger.warn("Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
logger.warn(
"Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})"
)
return false
}
val ktFile = psi2Ir.getKtFile(file)
@@ -37,14 +43,16 @@ class CommentExtractorPSI(fileExtractor: KotlinFileExtractor, file: IrFile, file
override fun visitElement(element: PsiElement) {
element.acceptChildren(this)
// Slightly hacky, but `visitComment` doesn't seem to visit comments with `tokenType` `KtTokens.DOC_COMMENT`
if (element is PsiComment){
// Slightly hacky, but `visitComment` doesn't seem to visit comments with
// `tokenType` `KtTokens.DOC_COMMENT`
if (element is PsiComment) {
visitCommentElement(element)
}
}
private fun visitCommentElement(comment: PsiComment) {
val type: CommentType = when (comment.tokenType) {
val type: CommentType =
when (comment.tokenType) {
KtTokens.EOL_COMMENT -> {
CommentType.SingleLine
}
@@ -101,10 +109,12 @@ class CommentExtractorPSI(fileExtractor: KotlinFileExtractor, file: IrFile, file
}
}
private fun getKDocOwner(comment: KDoc) : PsiElement? {
private fun getKDocOwner(comment: KDoc): PsiElement? {
val owner = comment.owner
if (owner == null) {
logger.warn("Couldn't get owner of KDoc. The comment is extracted without an owner.")
logger.warn(
"Couldn't get owner of KDoc. The comment is extracted without an owner."
)
}
return owner
}

4
java/kotlin-extractor/src/main/kotlin/comments/CommentType.kt
View File

@@ -1,5 +1,7 @@
package com.github.codeql.comments
enum class CommentType(val value: Int) {
SingleLine(1), Block(2), Doc(3)
SingleLine(1),
Block(2),
Doc(3)
}

28
java/kotlin-extractor/src/main/kotlin/utils/AutoCloseableUse.kt
View File

@@ -1,14 +1,16 @@
package com.github.codeql
// Functions copied from stdlib/jdk7/src/kotlin/AutoCloseable.kt, which is not available within kotlinc,
// Functions copied from stdlib/jdk7/src/kotlin/AutoCloseable.kt, which is not available within
// kotlinc,
// but allows the `.use` pattern to be applied to JDK7 AutoCloseables:
/**
* Executes the given [block] function on this resource and then closes it down correctly whether an exception
* is thrown or not.
* Executes the given [block] function on this resource and then closes it down correctly whether an
* exception is thrown or not.
*
* In case if the resource is being closed due to an exception occurred in [block], and the closing also fails with an exception,
* the latter is added to the [suppressed][java.lang.Throwable.addSuppressed] exceptions of the former.
* In case if the resource is being closed due to an exception occurred in [block], and the closing
* also fails with an exception, the latter is added to the
* [suppressed][java.lang.Throwable.addSuppressed] exceptions of the former.
*
* @param block a function to process this [AutoCloseable] resource.
* @return the result of [block] function invoked on this resource.
@@ -26,12 +28,14 @@ public inline fun <T : AutoCloseable?, R> T.useAC(block: (T) -> R): R {
}
/**
* Closes this [AutoCloseable], suppressing possible exception or error thrown by [AutoCloseable.close] function when
* it's being closed due to some other [cause] exception occurred.
*
* The suppressed exception is added to the list of suppressed exceptions of [cause] exception.
*/
fun AutoCloseable?.closeFinallyAC(cause: Throwable?) = when {
* Closes this [AutoCloseable], suppressing possible exception or error thrown by
* [AutoCloseable.close] function when it's being closed due to some other [cause] exception
* occurred.
*
* The suppressed exception is added to the list of suppressed exceptions of [cause] exception.
*/
fun AutoCloseable?.closeFinallyAC(cause: Throwable?) =
when {
this == null -> {}
cause == null -> close()
else ->
@@ -40,4 +44,4 @@ fun AutoCloseable?.closeFinallyAC(cause: Throwable?) = when {
} catch (closeException: Throwable) {
cause.addSuppressed(closeException)
}
}
}

57
java/kotlin-extractor/src/main/kotlin/utils/ClassNames.kt
View File

@@ -3,30 +3,31 @@ package com.github.codeql
import com.github.codeql.utils.getJvmName
import com.github.codeql.utils.versions.*
import com.intellij.openapi.vfs.StandardFileSystems
import org.jetbrains.kotlin.fir.java.JavaBinarySourceElement
import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClass
import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement
import com.intellij.openapi.vfs.VirtualFile
import org.jetbrains.kotlin.builtins.jvm.JavaToKotlinClassMap
import org.jetbrains.kotlin.fir.java.JavaBinarySourceElement
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable
import org.jetbrains.kotlin.ir.util.parentClassOrNull
import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
import org.jetbrains.kotlin.load.kotlin.JvmPackagePartSource
import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement
import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClass
// Adapted from Kotlin's interpreter/Utils.kt function 'internalName'
// Translates class names into their JLS section 13.1 binary name,
// and declarations within them into the parent class' JLS 13.1 name as
// specified above, followed by a `$` separator and then the short name
// for `that`.
private fun getName(d: IrDeclarationWithName) = (d as? IrAnnotationContainer)?.let { getJvmName(it) } ?: d.name.asString()
private fun getName(d: IrDeclarationWithName) =
(d as? IrAnnotationContainer)?.let { getJvmName(it) } ?: d.name.asString()
fun getFileClassName(f: IrFile) =
getJvmName(f) ?:
((f.fileEntry.name.replaceFirst(Regex(""".*[/\\]"""), "")
getJvmName(f)
?: ((f.fileEntry.name
.replaceFirst(Regex(""".*[/\\]"""), "")
.replaceFirst(Regex("""\.kt$"""), "")
.replaceFirstChar { it.uppercase() }) + "Kt")
@@ -41,7 +42,8 @@ fun getIrElementBinaryName(that: IrElement): String {
return "(unknown-name)"
}
val shortName = when(that) {
val shortName =
when (that) {
is IrDeclarationWithName -> getName(that)
else -> "(unknown-name)"
}
@@ -50,7 +52,8 @@ fun getIrElementBinaryName(that: IrElement): String {
if (that !is IrClass) {
val parent = that.parent
if (parent is IrFile) {
// Note we'll fall through and do the IrPackageFragment case as well, since IrFile <: IrPackageFragment
// Note we'll fall through and do the IrPackageFragment case as well, since IrFile <:
// IrPackageFragment
internalName.insert(0, getFileClassName(parent) + "$")
}
}
@@ -59,7 +62,11 @@ fun getIrElementBinaryName(that: IrElement): String {
.forEach {
when (it) {
is IrClass -> internalName.insert(0, getName(it) + "$")
is IrPackageFragment -> it.packageFqName.asString().takeIf { fqName -> fqName.isNotEmpty() }?.let { fqName -> internalName.insert(0, "$fqName.") }
is IrPackageFragment ->
it.packageFqName
.asString()
.takeIf { fqName -> fqName.isNotEmpty() }
?.let { fqName -> internalName.insert(0, "$fqName.") }
}
}
return internalName.toString()
@@ -67,16 +74,18 @@ fun getIrElementBinaryName(that: IrElement): String {
fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
val cSource = irClass.source
// Don't emit a location for multi-file classes until we're sure we can cope with different declarations
// inside a class disagreeing about their source file. In particular this currently causes problems when
// a source-location for a declarations tries to refer to a file-id which is assumed to be declared in
// Don't emit a location for multi-file classes until we're sure we can cope with different
// declarations
// inside a class disagreeing about their source file. In particular this currently causes
// problems when
// a source-location for a declarations tries to refer to a file-id which is assumed to be
// declared in
// the class trap file.
if (irClass.origin == IrDeclarationOrigin.JVM_MULTIFILE_CLASS)
return null
when(cSource) {
if (irClass.origin == IrDeclarationOrigin.JVM_MULTIFILE_CLASS) return null
when (cSource) {
is JavaSourceElement -> {
val element = cSource.javaElement
when(element) {
when (element) {
is BinaryJavaClass -> return element.virtualFile
}
}
@@ -85,7 +94,7 @@ fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
}
is KotlinJvmBinarySourceElement -> {
val binaryClass = cSource.binaryClass
when(binaryClass) {
when (binaryClass) {
is VirtualFileKotlinClass -> return binaryClass.file
}
}
@@ -102,11 +111,11 @@ fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
private fun getRawIrClassBinaryPath(irClass: IrClass) =
getIrClassVirtualFile(irClass)?.let {
val path = it.path
if(it.fileSystem.protocol == StandardFileSystems.JRT_PROTOCOL)
// For JRT files, which we assume to be the JDK, hide the containing JAR path to match the Java extractor's behaviour.
if (it.fileSystem.protocol == StandardFileSystems.JRT_PROTOCOL)
// For JRT files, which we assume to be the JDK, hide the containing JAR path to match the
// Java extractor's behaviour.
"/${path.split("!/", limit = 2)[1]}"
else
path
else path
}
fun getIrClassBinaryPath(irClass: IrClass): String {

16
java/kotlin-extractor/src/main/kotlin/utils/ExternalDecls.kt
View File

@@ -5,7 +5,6 @@ import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
import org.jetbrains.kotlin.ir.declarations.IrExternalPackageFragment
import org.jetbrains.kotlin.ir.util.isFileClass
import org.jetbrains.kotlin.ir.util.parentClassOrNull
fun isExternalDeclaration(d: IrDeclaration): Boolean {
/*
@@ -15,9 +14,11 @@ fun isExternalDeclaration(d: IrDeclaration): Boolean {
EXPRESSION_BODY
CONST Int type=kotlin.Int value=-2147483648
*/
if (d.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB ||
if (
d.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB ||
d.origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB ||
d.origin.toString() == "FUNCTION_INTERFACE_CLASS") { // Treat kotlin.coroutines.* like ordinary library classes
d.origin.toString() == "FUNCTION_INTERFACE_CLASS"
) { // Treat kotlin.coroutines.* like ordinary library classes
return true
}
/*
@@ -39,11 +40,11 @@ fun isExternalDeclaration(d: IrDeclaration): Boolean {
return false
}
/**
* Returns true if `d` is not itself a class, but is a member of an external file class.
*/
/** Returns true if `d` is not itself a class, but is a member of an external file class. */
fun isExternalFileClassMember(d: IrDeclaration): Boolean {
if (d is IrClass) { return false }
if (d is IrClass) {
return false
}
val p = d.parent
when (p) {
is IrClass -> return p.isFileClass
@@ -53,4 +54,3 @@ fun isExternalFileClassMember(d: IrDeclaration): Boolean {
}
return false
}

12
java/kotlin-extractor/src/main/kotlin/utils/GetByFqName.kt
View File

@@ -9,10 +9,18 @@ fun getClassByFqName(pluginContext: IrPluginContext, fqName: String): IrClassSym
return getClassByFqName(pluginContext, FqName(fqName))
}
fun getFunctionsByFqName(pluginContext: IrPluginContext, pkgName: String, name: String): Collection<IrSimpleFunctionSymbol> {
fun getFunctionsByFqName(
pluginContext: IrPluginContext,
pkgName: String,
name: String
): Collection<IrSimpleFunctionSymbol> {
return getFunctionsByFqName(pluginContext, FqName(pkgName), Name.identifier(name))
}
fun getPropertiesByFqName(pluginContext: IrPluginContext, pkgName: String, name: String): Collection<IrPropertySymbol> {
fun getPropertiesByFqName(
pluginContext: IrPluginContext,
pkgName: String,
name: String
): Collection<IrPropertySymbol> {
return getPropertiesByFqName(pluginContext, FqName(pkgName), Name.identifier(name))
}

3
java/kotlin-extractor/src/main/kotlin/utils/Helpers.kt
View File

@@ -9,4 +9,5 @@ fun IrFunction.isLocalFunction(): Boolean {
return this.visibility == DescriptorVisibilities.LOCAL
}
val IrClass.isInterfaceLike get() = kind == ClassKind.INTERFACE || kind == ClassKind.ANNOTATION_CLASS
val IrClass.isInterfaceLike
get() = kind == ClassKind.INTERFACE || kind == ClassKind.ANNOTATION_CLASS

8
java/kotlin-extractor/src/main/kotlin/utils/IrVisitorLookup.kt
View File

@@ -1,6 +1,5 @@
package com.github.codeql.utils
import com.github.codeql.utils.Psi2IrFacade
import com.intellij.psi.PsiElement
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
@@ -8,8 +7,11 @@ import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.ir.util.isFakeOverride
import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
class IrVisitorLookup(private val psi2Ir: Psi2IrFacade, private val psi: PsiElement, private val file: IrFile) :
IrElementVisitor<Unit, MutableCollection<IrElement>> {
class IrVisitorLookup(
private val psi2Ir: Psi2IrFacade,
private val psi: PsiElement,
private val file: IrFile
) : IrElementVisitor<Unit, MutableCollection<IrElement>> {
private val location = psi.getLocation()
override fun visitElement(element: IrElement, data: MutableCollection<IrElement>): Unit {

7
java/kotlin-extractor/src/main/kotlin/utils/Iterable.kt
View File

@@ -3,11 +3,10 @@ package com.github.codeql
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
/**
* This behaves the same as Iterable<IrDeclaration>.find, but requires
* that the value found is of the subtype S, and it casts
* the result for you appropriately.
* This behaves the same as Iterable<IrDeclaration>.find, but requires that the value found is of
* the subtype S, and it casts the result for you appropriately.
*/
inline fun <reified S: IrDeclaration> Iterable<IrDeclaration>.findSubType(
inline fun <reified S : IrDeclaration> Iterable<IrDeclaration>.findSubType(
predicate: (S) -> Boolean
): S? {
return this.find { it is S && predicate(it) } as S?

15
java/kotlin-extractor/src/main/kotlin/utils/JvmNames.kt
View File

@@ -17,10 +17,13 @@ import org.jetbrains.kotlin.name.Name
private data class MethodKey(val className: FqName, val functionName: Name)
private fun makeDescription(className: FqName, functionName: String) = MethodKey(className, Name.guessByFirstCharacter(functionName))
private fun makeDescription(className: FqName, functionName: String) =
MethodKey(className, Name.guessByFirstCharacter(functionName))
// This essentially mirrors SpecialBridgeMethods.kt, a backend pass which isn't easily available to our extractor.
private val specialFunctions = mapOf(
// This essentially mirrors SpecialBridgeMethods.kt, a backend pass which isn't easily available to
// our extractor.
private val specialFunctions =
mapOf(
makeDescription(StandardNames.FqNames.collection, "<get-size>") to "size",
makeDescription(FqName("java.util.Collection"), "<get-size>") to "size",
makeDescription(StandardNames.FqNames.map, "<get-size>") to "size",
@@ -53,7 +56,7 @@ private val specialFunctions = mapOf(
makeDescription(FqName("java.lang.Number"), "toDouble") to "doubleValue",
makeDescription(StandardNames.FqNames.string.toSafe(), "get") to "charAt",
makeDescription(FqName("java.lang.String"), "get") to "charAt",
)
)
private val specialFunctionShortNames = specialFunctions.keys.map { it.functionName }.toSet()
@@ -71,7 +74,7 @@ private fun getSpecialJvmName(f: IrFunction): String? {
}
fun getJvmName(container: IrAnnotationContainer): String? {
for(a: IrConstructorCall in container.annotations) {
for (a: IrConstructorCall in container.annotations) {
val t = a.type
if (t is IrSimpleType && a.valueArgumentsCount == 1) {
val owner = t.classifier.owner
@@ -79,7 +82,7 @@ fun getJvmName(container: IrAnnotationContainer): String? {
if (owner is IrClass) {
val aPkg = owner.packageFqName?.asString()
val name = owner.name.asString()
if(aPkg == "kotlin.jvm" && name == "JvmName" && v is IrConst<*>) {
if (aPkg == "kotlin.jvm" && name == "JvmName" && v is IrConst<*>) {
val value = v.value
if (value is String) {
return value

6
java/kotlin-extractor/src/main/kotlin/utils/List.kt
View File

@@ -1,13 +1,13 @@
package com.github.codeql
/**
* Turns this list of nullable elements into a list of non-nullable
* elements if they are all non-null, or returns null otherwise.
* Turns this list of nullable elements into a list of non-nullable elements if they are all
* non-null, or returns null otherwise.
*/
public fun <T : Any> List<T?>.requireNoNullsOrNull(): List<T>? {
try {
return this.requireNoNulls()
} catch (e: IllegalArgumentException) {
return null;
return null
}
}

8
java/kotlin-extractor/src/main/kotlin/utils/Location.kt
View File

@@ -5,8 +5,8 @@ import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.psi.psiUtil.endOffset
import org.jetbrains.kotlin.psi.psiUtil.startOffset
data class Location(val startOffset: Int, val endOffset: Int){
fun contains(location: Location) : Boolean {
data class Location(val startOffset: Int, val endOffset: Int) {
fun contains(location: Location): Boolean {
return this.startOffset <= location.startOffset && this.endOffset >= location.endOffset
}
@@ -15,10 +15,10 @@ data class Location(val startOffset: Int, val endOffset: Int){
}
}
fun IrElement.getLocation() : Location {
fun IrElement.getLocation(): Location {
return Location(this.startOffset, this.endOffset)
}
fun PsiElement.getLocation() : Location {
fun PsiElement.getLocation(): Location {
return Location(this.startOffset, this.endOffset)
}

115
java/kotlin-extractor/src/main/kotlin/utils/Logger.kt
View File

@@ -12,8 +12,10 @@ import org.jetbrains.kotlin.ir.IrElement
class LogCounter() {
public val diagnosticInfo = mutableMapOf<String, Pair<Severity, Int>>()
public val diagnosticLimit: Int
init {
diagnosticLimit = System.getenv("CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT")?.toIntOrNull() ?: 100
diagnosticLimit =
System.getenv("CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT")?.toIntOrNull() ?: 100
}
}
@@ -35,6 +37,7 @@ enum class Severity(val sev: Int) {
class LogMessage(private val kind: String, private val message: String) {
val timestamp: String
init {
timestamp = "${SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Date())}"
}
@@ -65,25 +68,37 @@ class LogMessage(private val kind: String, private val message: String) {
}
fun toJsonLine(): String {
val kvs = listOf(Pair("origin", "CodeQL Kotlin extractor"),
val kvs =
listOf(
Pair("origin", "CodeQL Kotlin extractor"),
Pair("timestamp", timestamp),
Pair("kind", kind),
Pair("message", message))
return "{ " + kvs.map { p -> "\"${p.first}\": \"${escape(p.second)}\""}.joinToString(", ") + " }\n"
Pair("message", message)
)
return "{ " +
kvs.map { p -> "\"${p.first}\": \"${escape(p.second)}\"" }.joinToString(", ") +
" }\n"
}
}
data class ExtractorContext(val kind: String, val element: IrElement, val name: String, val loc: String)
data class ExtractorContext(
val kind: String,
val element: IrElement,
val name: String,
val loc: String
)
open class LoggerBase(val logCounter: LogCounter) {
val extractorContextStack = Stack<ExtractorContext>()
private val verbosity: Int
init {
verbosity = System.getenv("CODEQL_EXTRACTOR_KOTLIN_VERBOSITY")?.toIntOrNull() ?: 3
}
private val logStream: Writer
init {
val extractorLogDir = System.getenv("CODEQL_EXTRACTOR_JAVA_LOG_DIR")
if (extractorLogDir == null || extractorLogDir == "") {
@@ -96,8 +111,8 @@ open class LoggerBase(val logCounter: LogCounter) {
private fun getDiagnosticLocation(): String? {
val st = Exception().stackTrace
for(x in st) {
when(x.className) {
for (x in st) {
when (x.className) {
"com.github.codeql.LoggerBase",
"com.github.codeql.Logger",
"com.github.codeql.FileLogger" -> {}
@@ -117,20 +132,29 @@ open class LoggerBase(val logCounter: LogCounter) {
file_number_diagnostic_number = 0
}
fun diagnostic(dtw: DiagnosticTrapWriter, severity: Severity, msg: String, extraInfo: String?, locationString: String? = null, mkLocationId: () -> Label<DbLocation> = { dtw.unknownLocation }) {
fun diagnostic(
dtw: DiagnosticTrapWriter,
severity: Severity,
msg: String,
extraInfo: String?,
locationString: String? = null,
mkLocationId: () -> Label<DbLocation> = { dtw.unknownLocation }
) {
val diagnosticLoc = getDiagnosticLocation()
val diagnosticLocStr = if(diagnosticLoc == null) "<unknown location>" else diagnosticLoc
val diagnosticLocStr = if (diagnosticLoc == null) "<unknown location>" else diagnosticLoc
val suffix =
if(diagnosticLoc == null) {
if (diagnosticLoc == null) {
" Missing caller information.\n"
} else {
val oldInfo = logCounter.diagnosticInfo.getOrDefault(diagnosticLoc, Pair(severity, 0))
if(severity != oldInfo.first) {
val oldInfo =
logCounter.diagnosticInfo.getOrDefault(diagnosticLoc, Pair(severity, 0))
if (severity != oldInfo.first) {
// We don't want to get in a loop, so just emit this
// directly without going through the diagnostic
// counting machinery
if (verbosity >= 1) {
val message = "Severity mismatch ($severity vs ${oldInfo.first}) at $diagnosticLoc"
val message =
"Severity mismatch ($severity vs ${oldInfo.first}) at $diagnosticLoc"
emitDiagnostic(dtw, Severity.Error, "Inconsistency", message, message)
}
}
@@ -139,7 +163,8 @@ open class LoggerBase(val logCounter: LogCounter) {
logCounter.diagnosticInfo[diagnosticLoc] = newInfo
when {
logCounter.diagnosticLimit <= 0 -> ""
newCount == logCounter.diagnosticLimit -> " Limit reached for diagnostics from $diagnosticLoc.\n"
newCount == logCounter.diagnosticLimit ->
" Limit reached for diagnostics from $diagnosticLoc.\n"
newCount > logCounter.diagnosticLimit -> return
else -> ""
}
@@ -162,15 +187,36 @@ open class LoggerBase(val logCounter: LogCounter) {
emitDiagnostic(dtw, severity, diagnosticLocStr, msg, fullMsg, locationString, mkLocationId)
}
private fun emitDiagnostic(dtw: DiagnosticTrapWriter, severity: Severity, diagnosticLocStr: String, msg: String, fullMsg: String, locationString: String? = null, mkLocationId: () -> Label<DbLocation> = { dtw.unknownLocation }) {
private fun emitDiagnostic(
dtw: DiagnosticTrapWriter,
severity: Severity,
diagnosticLocStr: String,
msg: String,
fullMsg: String,
locationString: String? = null,
mkLocationId: () -> Label<DbLocation> = { dtw.unknownLocation }
) {
val locStr = if (locationString == null) "" else "At " + locationString + ": "
val kind = if (severity <= Severity.WarnHigh) "WARN" else "ERROR"
val logMessage = LogMessage(kind, "Diagnostic($diagnosticLocStr): $locStr$fullMsg")
// We don't actually make the location until after the `return` above
val locationId = mkLocationId()
val diagLabel = dtw.getFreshIdLabel<DbDiagnostic>()
dtw.writeDiagnostics(diagLabel, "CodeQL Kotlin extractor", severity.sev, "", msg, "${logMessage.timestamp} $fullMsg", locationId)
dtw.writeDiagnostic_for(diagLabel, StringLabel("compilation"), file_number, file_number_diagnostic_number++)
dtw.writeDiagnostics(
diagLabel,
"CodeQL Kotlin extractor",
severity.sev,
"",
msg,
"${logMessage.timestamp} $fullMsg",
locationId
)
dtw.writeDiagnostic_for(
diagLabel,
StringLabel("compilation"),
file_number,
file_number_diagnostic_number++
)
logStream.write(logMessage.toJsonLine())
}
@@ -203,6 +249,7 @@ open class LoggerBase(val logCounter: LogCounter) {
diagnostic(dtw, Severity.Warn, msg, extraInfo)
}
}
fun error(dtw: DiagnosticTrapWriter, msg: String, extraInfo: String?) {
if (verbosity >= 1) {
diagnostic(dtw, Severity.Error, msg, extraInfo)
@@ -210,11 +257,12 @@ open class LoggerBase(val logCounter: LogCounter) {
}
fun printLimitedDiagnosticCounts(dtw: DiagnosticTrapWriter) {
for((caller, info) in logCounter.diagnosticInfo) {
for ((caller, info) in logCounter.diagnosticInfo) {
val severity = info.first
val count = info.second
if(count >= logCounter.diagnosticLimit) {
val message = "Total of $count diagnostics (reached limit of ${logCounter.diagnosticLimit}) from $caller."
if (count >= logCounter.diagnosticLimit) {
val message =
"Total of $count diagnostics (reached limit of ${logCounter.diagnosticLimit}) from $caller."
if (verbosity >= 1) {
emitDiagnostic(dtw, severity, "Limit", message, message)
}
@@ -240,9 +288,11 @@ open class Logger(val loggerBase: LoggerBase, open val dtw: DiagnosticTrapWriter
fun trace(msg: String) {
loggerBase.trace(dtw, msg)
}
fun trace(msg: String, exn: Throwable) {
trace(msg + "\n" + exn.stackTraceToString())
}
fun debug(msg: String) {
loggerBase.debug(dtw, msg)
}
@@ -254,9 +304,11 @@ open class Logger(val loggerBase: LoggerBase, open val dtw: DiagnosticTrapWriter
private fun warn(msg: String, extraInfo: String?) {
loggerBase.warn(dtw, msg, extraInfo)
}
fun warn(msg: String, exn: Throwable) {
warn(msg, exn.stackTraceToString())
}
fun warn(msg: String) {
warn(msg, null)
}
@@ -264,24 +316,41 @@ open class Logger(val loggerBase: LoggerBase, open val dtw: DiagnosticTrapWriter
private fun error(msg: String, extraInfo: String?) {
loggerBase.error(dtw, msg, extraInfo)
}
fun error(msg: String) {
error(msg, null)
}
fun error(msg: String, exn: Throwable) {
error(msg, exn.stackTraceToString())
}
}
class FileLogger(loggerBase: LoggerBase, val ftw: FileTrapWriter): Logger(loggerBase, ftw.getDiagnosticTrapWriter()) {
class FileLogger(loggerBase: LoggerBase, val ftw: FileTrapWriter) :
Logger(loggerBase, ftw.getDiagnosticTrapWriter()) {
fun warnElement(msg: String, element: IrElement, exn: Throwable? = null) {
val locationString = ftw.getLocationString(element)
val mkLocationId = { ftw.getLocation(element) }
loggerBase.diagnostic(ftw.getDiagnosticTrapWriter(), Severity.Warn, msg, exn?.stackTraceToString(), locationString, mkLocationId)
loggerBase.diagnostic(
ftw.getDiagnosticTrapWriter(),
Severity.Warn,
msg,
exn?.stackTraceToString(),
locationString,
mkLocationId
)
}
fun errorElement(msg: String, element: IrElement, exn: Throwable? = null) {
val locationString = ftw.getLocationString(element)
val mkLocationId = { ftw.getLocation(element) }
loggerBase.diagnostic(ftw.getDiagnosticTrapWriter(), Severity.Error, msg, exn?.stackTraceToString(), locationString, mkLocationId)
loggerBase.diagnostic(
ftw.getDiagnosticTrapWriter(),
Severity.Error,
msg,
exn?.stackTraceToString(),
locationString,
mkLocationId
)
}
}

1
java/kotlin-extractor/src/main/kotlin/utils/Psi2IrFacade.kt
View File

@@ -7,5 +7,6 @@ import org.jetbrains.kotlin.psi.KtFile
interface Psi2IrFacade {
fun getKtFile(irFile: IrFile): KtFile?
fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement?
}

42
java/kotlin-extractor/src/main/kotlin/utils/TypeResults.kt
View File

@@ -1,30 +1,40 @@
package com.github.codeql
/**
* A triple of a type's database label, its signature for use in callable signatures, and its short name for use
* in all tables that provide a user-facing type name.
* A triple of a type's database label, its signature for use in callable signatures, and its short
* name for use in all tables that provide a user-facing type name.
*
* `signature` is a Java primitive name (e.g. "int"), a fully-qualified class name ("package.OuterClass.InnerClass"),
* or an array ("componentSignature[]")
* Type variables have the signature of their upper bound.
* Type arguments and anonymous types do not have a signature.
* `signature` is a Java primitive name (e.g. "int"), a fully-qualified class name
* ("package.OuterClass.InnerClass"), or an array ("componentSignature[]") Type variables have the
* signature of their upper bound. Type arguments and anonymous types do not have a signature.
*
* `shortName` is a Java primitive name (e.g. "int"), a class short name with Java-style type arguments ("InnerClass<E>" or
* "OuterClass<ConcreteArgument>" or "OtherClass<? extends Bound>") or an array ("componentShortName[]").
* `shortName` is a Java primitive name (e.g. "int"), a class short name with Java-style type
* arguments ("InnerClass<E>" or "OuterClass<ConcreteArgument>" or "OtherClass<? extends Bound>") or
* an array ("componentShortName[]").
*/
data class TypeResultGeneric<SignatureType,out LabelType: AnyDbType>(val id: Label<out LabelType>, val signature: SignatureType?, val shortName: String) {
fun <U: AnyDbType> cast(): TypeResultGeneric<SignatureType,U> {
@Suppress("UNCHECKED_CAST")
return this as TypeResultGeneric<SignatureType,U>
data class TypeResultGeneric<SignatureType, out LabelType : AnyDbType>(
val id: Label<out LabelType>,
val signature: SignatureType?,
val shortName: String
) {
fun <U : AnyDbType> cast(): TypeResultGeneric<SignatureType, U> {
@Suppress("UNCHECKED_CAST") return this as TypeResultGeneric<SignatureType, U>
}
}
data class TypeResultsGeneric<SignatureType>(val javaResult: TypeResultGeneric<SignatureType,DbType>, val kotlinResult: TypeResultGeneric<SignatureType,DbKt_type>)
typealias TypeResult<T> = TypeResultGeneric<String,T>
typealias TypeResultWithoutSignature<T> = TypeResultGeneric<Unit,T>
data class TypeResultsGeneric<SignatureType>(
val javaResult: TypeResultGeneric<SignatureType, DbType>,
val kotlinResult: TypeResultGeneric<SignatureType, DbKt_type>
)
typealias TypeResult<T> = TypeResultGeneric<String, T>
typealias TypeResultWithoutSignature<T> = TypeResultGeneric<Unit, T>
typealias TypeResults = TypeResultsGeneric<String>
typealias TypeResultsWithoutSignatures = TypeResultsGeneric<Unit>
fun <T: AnyDbType> TypeResult<T>.forgetSignature(): TypeResultWithoutSignature<T> {
fun <T : AnyDbType> TypeResult<T>.forgetSignature(): TypeResultWithoutSignature<T> {
return TypeResultWithoutSignature(this.id, Unit, this.shortName)
}

161
java/kotlin-extractor/src/main/kotlin/utils/TypeSubstitution.kt
View File

@@ -33,15 +33,18 @@ import org.jetbrains.kotlin.types.Variance
import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
fun IrType.substituteTypeArguments(params: List<IrTypeParameter>, arguments: List<IrTypeArgument>) =
when(this) {
when (this) {
is IrSimpleType -> substituteTypeArguments(params.map { it.symbol }.zip(arguments).toMap())
else -> this
}
private fun IrSimpleType.substituteTypeArguments(substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>): IrSimpleType {
private fun IrSimpleType.substituteTypeArguments(
substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>
): IrSimpleType {
if (substitutionMap.isEmpty()) return this
val newArguments = arguments.map {
val newArguments =
arguments.map {
if (it is IrTypeProjection) {
val itType = it.type
if (itType is IrSimpleType) {
@@ -54,30 +57,25 @@ private fun IrSimpleType.substituteTypeArguments(substitutionMap: Map<IrTypePara
}
}
return IrSimpleTypeImpl(
classifier,
isNullable(),
newArguments,
annotations
)
return IrSimpleTypeImpl(classifier, isNullable(), newArguments, annotations)
}
/**
* Returns true if substituting `innerVariance T` into the context `outerVariance []` discards all knowledge about
* what T could be.
* Returns true if substituting `innerVariance T` into the context `outerVariance []` discards all
* knowledge about what T could be.
*
* Note this throws away slightly more information than it could: for example, the projection "in (out List)" can refer to
* any superclass of anything that implements List, which specifically excludes e.g. String, but can't be represented as
* a type projection. The projection "out (in List)" on the other hand really is equivalent to "out Any?", which is to
* say no bound at all.
* Note this throws away slightly more information than it could: for example, the projection "in
* (out List)" can refer to any superclass of anything that implements List, which specifically
* excludes e.g. String, but can't be represented as a type projection. The projection "out (in
* List)" on the other hand really is equivalent to "out Any?", which is to say no bound at all.
*/
private fun conflictingVariance(outerVariance: Variance, innerVariance: Variance) =
(outerVariance == Variance.IN_VARIANCE && innerVariance == Variance.OUT_VARIANCE) ||
(outerVariance == Variance.OUT_VARIANCE && innerVariance == Variance.IN_VARIANCE)
/**
* When substituting `innerVariance T` into the context `outerVariance []`, returns the variance part of the result
* `resultVariance T`. We already know they don't conflict.
* When substituting `innerVariance T` into the context `outerVariance []`, returns the variance
* part of the result `resultVariance T`. We already know they don't conflict.
*/
private fun combineVariance(outerVariance: Variance, innerVariance: Variance) =
when {
@@ -86,13 +84,20 @@ private fun combineVariance(outerVariance: Variance, innerVariance: Variance) =
else -> Variance.INVARIANT
}
private fun subProjectedType(substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>, t: IrSimpleType, outerVariance: Variance): IrTypeArgument =
private fun subProjectedType(
substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>,
t: IrSimpleType,
outerVariance: Variance
): IrTypeArgument =
substitutionMap[t.classifier]?.let { substitutedTypeArg ->
if (substitutedTypeArg is IrTypeProjection) {
if (conflictingVariance(outerVariance, substitutedTypeArg.variance))
IrStarProjectionImpl
else {
val newProjectedType = substitutedTypeArg.type.let { if (t.isNullable()) it.codeQlWithHasQuestionMark(true) else it }
val newProjectedType =
substitutedTypeArg.type.let {
if (t.isNullable()) it.codeQlWithHasQuestionMark(true) else it
}
val newVariance = combineVariance(outerVariance, substitutedTypeArg.variance)
makeTypeProjection(newProjectedType, newVariance)
}
@@ -102,33 +107,43 @@ private fun subProjectedType(substitutionMap: Map<IrTypeParameterSymbol, IrTypeA
} ?: makeTypeProjection(t.substituteTypeArguments(substitutionMap), outerVariance)
private fun IrTypeArgument.upperBound(context: IrPluginContext) =
when(this) {
when (this) {
is IrStarProjection -> context.irBuiltIns.anyNType
is IrTypeProjection -> when(this.variance) {
is IrTypeProjection ->
when (this.variance) {
Variance.INVARIANT -> this.type
Variance.IN_VARIANCE -> if (this.type.isNullable()) context.irBuiltIns.anyNType else context.irBuiltIns.anyType
Variance.IN_VARIANCE ->
if (this.type.isNullable()) context.irBuiltIns.anyNType
else context.irBuiltIns.anyType
Variance.OUT_VARIANCE -> this.type
}
else -> context.irBuiltIns.anyNType
}
private fun IrTypeArgument.lowerBound(context: IrPluginContext) =
when(this) {
when (this) {
is IrStarProjection -> context.irBuiltIns.nothingType
is IrTypeProjection -> when(this.variance) {
is IrTypeProjection ->
when (this.variance) {
Variance.INVARIANT -> this.type
Variance.IN_VARIANCE -> this.type
Variance.OUT_VARIANCE -> if (this.type.isNullable()) context.irBuiltIns.nothingNType else context.irBuiltIns.nothingType
Variance.OUT_VARIANCE ->
if (this.type.isNullable()) context.irBuiltIns.nothingNType
else context.irBuiltIns.nothingType
}
else -> context.irBuiltIns.nothingType
}
fun IrType.substituteTypeAndArguments(substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>?, useContext: KotlinUsesExtractor.TypeContext, pluginContext: IrPluginContext): IrType =
fun IrType.substituteTypeAndArguments(
substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>?,
useContext: KotlinUsesExtractor.TypeContext,
pluginContext: IrPluginContext
): IrType =
substitutionMap?.let { substMap ->
if (this is IrSimpleType) {
val typeClassifier = this.classifier
substMap[typeClassifier]?.let {
when(useContext) {
when (useContext) {
KotlinUsesExtractor.TypeContext.RETURN -> it.upperBound(pluginContext)
else -> it.lowerBound(pluginContext)
}
@@ -139,42 +154,41 @@ fun IrType.substituteTypeAndArguments(substitutionMap: Map<IrTypeParameterSymbol
} ?: this
object RawTypeAnnotation {
// Much of this is taken from JvmGeneratorExtensionsImpl.kt, which is not easily accessible in plugin context.
// The constants "kotlin.internal.ir" and "RawType" could be referred to symbolically, but they move package
// Much of this is taken from JvmGeneratorExtensionsImpl.kt, which is not easily accessible in
// plugin context.
// The constants "kotlin.internal.ir" and "RawType" could be referred to symbolically, but they
// move package
// between different versions of the Kotlin compiler.
val annotationConstructor: IrConstructorCall by lazy {
val irInternalPackage = FqName("kotlin.internal.ir")
val parent = IrExternalPackageFragmentImpl(
val parent =
IrExternalPackageFragmentImpl(
DescriptorlessExternalPackageFragmentSymbol(),
irInternalPackage
)
val annoClass = IrFactoryImpl.buildClass {
val annoClass =
IrFactoryImpl.buildClass {
kind = ClassKind.ANNOTATION_CLASS
name = irInternalPackage.child(Name.identifier("RawType")).shortName()
}.apply {
}
.apply {
createImplicitParameterDeclarationWithWrappedDescriptor()
this.parent = parent
addConstructor {
isPrimary = true
}
addConstructor { isPrimary = true }
}
val constructor = annoClass.constructors.single()
IrConstructorCallImpl.fromSymbolOwner(
constructor.constructedClassType,
constructor.symbol
)
IrConstructorCallImpl.fromSymbolOwner(constructor.constructedClassType, constructor.symbol)
}
}
fun IrType.toRawType(): IrType =
when(this) {
when (this) {
is IrSimpleType -> {
when(val owner = this.classifier.owner) {
when (val owner = this.classifier.owner) {
is IrClass -> {
if (this.arguments.isNotEmpty())
this.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor))
else
this
else this
}
is IrTypeParameter -> owner.superTypes[0].toRawType()
else -> this
@@ -187,29 +201,31 @@ fun IrClass.toRawType(): IrType {
val result = this.typeWith(listOf())
return if (this.typeParameters.isNotEmpty())
result.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor))
else
result
else result
}
fun IrTypeArgument.withQuestionMark(b: Boolean): IrTypeArgument =
when(this) {
when (this) {
is IrStarProjection -> this
is IrTypeProjection ->
this.type.let { when(it) {
is IrSimpleType -> if (it.isNullable() == b) this else makeTypeProjection(it.codeQlWithHasQuestionMark(b), this.variance)
this.type.let {
when (it) {
is IrSimpleType ->
if (it.isNullable() == b) this
else makeTypeProjection(it.codeQlWithHasQuestionMark(b), this.variance)
else -> this
}}
}
}
else -> this
}
typealias TypeSubstitution = (IrType, KotlinUsesExtractor.TypeContext, IrPluginContext) -> IrType
private fun matchingTypeParameters(l: IrTypeParameter?, r: IrTypeParameter): Boolean {
if (l === r)
return true
if (l == null)
return false
// Special case: match List's E and MutableList's E, for example, because in the JVM lowering they will map to the same thing.
if (l === r) return true
if (l == null) return false
// Special case: match List's E and MutableList's E, for example, because in the JVM lowering
// they will map to the same thing.
val lParent = l.parent as? IrClass ?: return false
val rParent = r.parent as? IrClass ?: return false
val lJavaId = getJavaEquivalentClassId(lParent) ?: lParent.classId
@@ -217,26 +233,43 @@ private fun matchingTypeParameters(l: IrTypeParameter?, r: IrTypeParameter): Boo
}
// Returns true if type is C<T1, T2, ...> where C is declared `class C<T1, T2, ...> { ... }`
fun isUnspecialised(paramsContainer: IrTypeParametersContainer, args: List<IrTypeArgument>, logger: Logger): Boolean {
fun isUnspecialised(
paramsContainer: IrTypeParametersContainer,
args: List<IrTypeArgument>,
logger: Logger
): Boolean {
return isUnspecialised(paramsContainer, args, logger, paramsContainer)
}
private fun isUnspecialised(paramsContainer: IrTypeParametersContainer, args: List<IrTypeArgument>, logger: Logger, origParamsContainer: IrTypeParametersContainer): Boolean {
val unspecialisedHere = paramsContainer.typeParameters.zip(args).all { paramAndArg ->
private fun isUnspecialised(
paramsContainer: IrTypeParametersContainer,
args: List<IrTypeArgument>,
logger: Logger,
origParamsContainer: IrTypeParametersContainer
): Boolean {
val unspecialisedHere =
paramsContainer.typeParameters.zip(args).all { paramAndArg ->
(paramAndArg.second as? IrTypeProjection)?.let {
// Type arg refers to the class' own type parameter?
it.variance == Variance.INVARIANT &&
matchingTypeParameters(it.type.classifierOrNull?.owner as? IrTypeParameter, paramAndArg.first)
matchingTypeParameters(
it.type.classifierOrNull?.owner as? IrTypeParameter,
paramAndArg.first
)
} ?: false
}
val remainingArgs = args.drop(paramsContainer.typeParameters.size)
val parentTypeContainer = paramsContainer.parents.firstIsInstanceOrNull<IrTypeParametersContainer>()
val parentTypeContainer =
paramsContainer.parents.firstIsInstanceOrNull<IrTypeParametersContainer>()
val parentUnspecialised = when {
val parentUnspecialised =
when {
remainingArgs.isEmpty() -> true
parentTypeContainer == null -> {
logger.error("Found more type arguments than parameters: ${origParamsContainer.kotlinFqName.asString()}")
logger.error(
"Found more type arguments than parameters: ${origParamsContainer.kotlinFqName.asString()}"
)
false
}
else -> isUnspecialised(parentTypeContainer, remainingArgs, logger, origParamsContainer)
@@ -245,6 +278,6 @@ private fun isUnspecialised(paramsContainer: IrTypeParametersContainer, args: Li
}
// Returns true if type is C<T1, T2, ...> where C is declared `class C<T1, T2, ...> { ... }`
fun isUnspecialised(type: IrSimpleType, logger: Logger) = (type.classifier.owner as? IrClass)?.let {
isUnspecialised(it, type.arguments, logger)
} ?: false
fun isUnspecialised(type: IrSimpleType, logger: Logger) =
(type.classifier.owner as? IrClass)?.let { isUnspecialised(it, type.arguments, logger) }
?: false

6
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/CommentExtractorLighterAST.kt
View File

@@ -3,7 +3,11 @@ package com.github.codeql.comments
import com.github.codeql.*
import org.jetbrains.kotlin.ir.declarations.*
class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFile, fileLabel: Label<out DbFile>): CommentExtractor(fileExtractor, file, fileLabel) {
class CommentExtractorLighterAST(
fileExtractor: KotlinFileExtractor,
file: IrFile,
fileLabel: Label<out DbFile>
) : CommentExtractor(fileExtractor, file, fileLabel) {
// We don't support LighterAST with old Kotlin versions
fun extract(): Boolean {
return false

3
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/IrSymbolInternals.kt
View File

@@ -1,4 +1,3 @@
package org.jetbrains.kotlin.ir.symbols
@RequiresOptIn(level = RequiresOptIn.Level.WARNING)
annotation class IrSymbolInternals
@RequiresOptIn(level = RequiresOptIn.Level.WARNING) annotation class IrSymbolInternals

8
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/IsUnderscoreParameter.kt
View File

@@ -11,9 +11,11 @@ import org.jetbrains.kotlin.utils.addToStdlib.safeAs
fun isUnderscoreParameter(vp: IrValueParameter) =
try {
DescriptorToSourceUtils.getSourceFromDescriptor(vp.descriptor)
?.safeAs<KtParameter>()?.isSingleUnderscore == true
} catch(e: NotImplementedError) {
// Some kinds of descriptor throw in `getSourceFromDescriptor` as that method is not normally expected to
?.safeAs<KtParameter>()
?.isSingleUnderscore == true
} catch (e: NotImplementedError) {
// Some kinds of descriptor throw in `getSourceFromDescriptor` as that method is not
// normally expected to
// be applied to synthetic functions.
false
}

4
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/JavaBinarySourceElement.kt
View File

@@ -7,5 +7,5 @@ import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
We need this class to exist, but the compiler will never give us an
instance of it.
*/
abstract class JavaBinarySourceElement private constructor(val javaClass: BinaryJavaClass): SourceElement {
}
abstract class JavaBinarySourceElement private constructor(val javaClass: BinaryJavaClass) :
SourceElement {}

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/Kotlin2ComponentRegistrar.kt
View File

@@ -3,8 +3,8 @@
package com.github.codeql
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
@OptIn(ExperimentalCompilerApi::class)
abstract class Kotlin2ComponentRegistrar : ComponentRegistrar {

11
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/LinesOfCodeLighterAST.kt
View File

@@ -2,18 +2,17 @@ package com.github.codeql
import org.jetbrains.kotlin.ir.declarations.*
class LinesOfCodeLighterAST(
val logger: FileLogger,
val tw: FileTrapWriter,
val file: IrFile
) {
class LinesOfCodeLighterAST(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
// We don't support LighterAST with old Kotlin versions
fun linesOfCodeInFile(@Suppress("UNUSED_PARAMETER") id: Label<DbFile>): Boolean {
return false
}
// We don't support LighterAST with old Kotlin versions
fun linesOfCodeInDeclaration(@Suppress("UNUSED_PARAMETER") d: IrDeclaration, @Suppress("UNUSED_PARAMETER") id: Label<out DbSourceline>): Boolean {
fun linesOfCodeInDeclaration(
@Suppress("UNUSED_PARAMETER") d: IrDeclaration,
@Suppress("UNUSED_PARAMETER") id: Label<out DbSourceline>
): Boolean {
return false
}
}

12
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/ReferenceEntity.kt
View File

@@ -14,12 +14,20 @@ fun getClassByClassId(pluginContext: IrPluginContext, id: ClassId): IrClassSymbo
return getClassByFqName(pluginContext, id.asSingleFqName())
}
fun getFunctionsByFqName(pluginContext: IrPluginContext, pkgName: FqName, name: Name): Collection<IrSimpleFunctionSymbol> {
fun getFunctionsByFqName(
pluginContext: IrPluginContext,
pkgName: FqName,
name: Name
): Collection<IrSimpleFunctionSymbol> {
val fqName = pkgName.child(name)
return pluginContext.referenceFunctions(fqName)
}
fun getPropertiesByFqName(pluginContext: IrPluginContext, pkgName: FqName, name: Name): Collection<IrPropertySymbol> {
fun getPropertiesByFqName(
pluginContext: IrPluginContext,
pkgName: FqName,
name: Name
): Collection<IrPropertySymbol> {
val fqName = pkgName.child(name)
return pluginContext.referenceProperties(fqName)
}

1
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/SyntheticBodyKind.kt
View File

@@ -3,4 +3,3 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.expressions.IrSyntheticBodyKind
val kind_ENUM_ENTRIES: IrSyntheticBodyKind? = null

1
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/Types.kt
View File

@@ -3,5 +3,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.jvm.codegen.isRawType
import org.jetbrains.kotlin.ir.types.IrSimpleType
fun IrSimpleType.isRawType() = this.isRawType()

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/allOverriddenIncludingSelf.kt
View File

@@ -1,6 +1,6 @@
package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
import org.jetbrains.kotlin.backend.common.ir.allOverridden
import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
fun IrSimpleFunction.allOverriddenIncludingSelf() = this.allOverridden(includeSelf = true)

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/copyTo.kt
View File

@@ -1,7 +1,7 @@
package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.common.ir.copyTo
import org.jetbrains.kotlin.ir.declarations.IrFunction
import org.jetbrains.kotlin.ir.declarations.IrValueParameter
import org.jetbrains.kotlin.backend.common.ir.copyTo
fun copyParameterToFunction(p: IrValueParameter, f: IrFunction) = p.copyTo(f)

5
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/createImplicitParameterDeclarationWithWrappedDescriptor.kt
View File

@@ -1,6 +1,7 @@
package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.backend.common.ir.createImplicitParameterDeclarationWithWrappedDescriptor
import org.jetbrains.kotlin.ir.declarations.IrClass
fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() = this.createImplicitParameterDeclarationWithWrappedDescriptor()
fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() =
this.createImplicitParameterDeclarationWithWrappedDescriptor()

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/getFileClassFqName.kt
View File

@@ -1,7 +1,7 @@
package com.github.codeql.utils.versions
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.name.FqName
fun getFileClassFqName(@Suppress("UNUSED_PARAMETER") d: IrDeclaration): FqName? {
return null

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/parents.kt
View File

@@ -1,4 +1,3 @@
package org.jetbrains.kotlin.ir.util
import org.jetbrains.kotlin.backend.common.lower.parents as kParents
@@ -10,4 +9,3 @@ val IrDeclaration.parents: Sequence<IrDeclarationParent>
val IrDeclaration.parentsWithSelf: Sequence<IrDeclarationParent>
get() = this.kParentsWithSelf

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/withHasQuestionMark.kt
View File

@@ -3,6 +3,6 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.types.IrType
import org.jetbrains.kotlin.ir.types.withHasQuestionMark
fun IrType.codeQlWithHasQuestionMark(b : Boolean): IrType {
fun IrType.codeQlWithHasQuestionMark(b: Boolean): IrType {
return this.withHasQuestionMark(b)
}

4
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_20/Psi2Ir.kt
View File

@@ -1,16 +1,16 @@
package com.github.codeql.utils.versions
import com.github.codeql.utils.Psi2IrFacade
import com.intellij.psi.PsiElement
import org.jetbrains.kotlin.backend.common.psi.PsiSourceManager
import org.jetbrains.kotlin.backend.jvm.ir.getKtFile
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.psi.KtFile
import com.github.codeql.utils.Psi2IrFacade
fun getPsi2Ir(): Psi2IrFacade? = Psi2Ir()
private class Psi2Ir(): Psi2IrFacade {
private class Psi2Ir() : Psi2IrFacade {
override fun getKtFile(irFile: IrFile): KtFile? {
return irFile.getKtFile()
}

3
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_6_0/annotationType.kt
View File

@@ -2,5 +2,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
fun getAnnotationType(context: IrPluginContext) =
context.irBuiltIns.annotationType
fun getAnnotationType(context: IrPluginContext) = context.irBuiltIns.annotationType

3
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_6_20/IsUnderscoreParameter.kt
View File

@@ -3,4 +3,5 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
import org.jetbrains.kotlin.ir.declarations.IrValueParameter
fun isUnderscoreParameter(vp: IrValueParameter) = vp.origin == IrDeclarationOrigin.UNDERSCORE_PARAMETER
fun isUnderscoreParameter(vp: IrValueParameter) =
vp.origin == IrDeclarationOrigin.UNDERSCORE_PARAMETER

1
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_6_20/Types.kt
View File

@@ -3,5 +3,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.jvm.ir.isRawType
import org.jetbrains.kotlin.ir.types.IrSimpleType
fun IrSimpleType.isRawType() = this.isRawType()

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_7_0/getFileClassFqName.kt
View File

@@ -1,10 +1,10 @@
package com.github.codeql.utils.versions
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.IrField
import org.jetbrains.kotlin.ir.declarations.IrMemberWithContainerSource
import org.jetbrains.kotlin.load.kotlin.FacadeClassSource
import org.jetbrains.kotlin.name.FqName
fun getFileClassFqName(d: IrDeclaration): FqName? {
// d is in a file class.

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_7_0/withHasQuestionMark.kt
View File

@@ -4,7 +4,7 @@ import org.jetbrains.kotlin.ir.types.IrType
import org.jetbrains.kotlin.ir.types.makeNotNull
import org.jetbrains.kotlin.ir.types.makeNullable
fun IrType.codeQlWithHasQuestionMark(b : Boolean): IrType {
fun IrType.codeQlWithHasQuestionMark(b: Boolean): IrType {
if (b) {
return this.makeNullable()
} else {

3
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_7_20/createImplicitParameterDeclarationWithWrappedDescriptor.kt
View File

@@ -3,4 +3,5 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.ir.util.createImplicitParameterDeclarationWithWrappedDescriptor
fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() = this.createImplicitParameterDeclarationWithWrappedDescriptor()
fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() =
this.createImplicitParameterDeclarationWithWrappedDescriptor()

15
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_8_0/ReferenceEntity.kt
View File

@@ -1,10 +1,9 @@
package com.github.codeql.utils
import org.jetbrains.kotlin.backend.common.extensions.FirIncompatiblePluginAPI
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.ir.symbols.*
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.CallableId
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
@@ -17,12 +16,20 @@ fun getClassByClassId(pluginContext: IrPluginContext, id: ClassId): IrClassSymbo
return pluginContext.referenceClass(id)
}
fun getFunctionsByFqName(pluginContext: IrPluginContext, pkgName: FqName, name: Name): Collection<IrSimpleFunctionSymbol> {
fun getFunctionsByFqName(
pluginContext: IrPluginContext,
pkgName: FqName,
name: Name
): Collection<IrSimpleFunctionSymbol> {
val id = CallableId(pkgName, name)
return pluginContext.referenceFunctions(id)
}
fun getPropertiesByFqName(pluginContext: IrPluginContext, pkgName: FqName, name: Name): Collection<IrPropertySymbol> {
fun getPropertiesByFqName(
pluginContext: IrPluginContext,
pkgName: FqName,
name: Name
): Collection<IrPropertySymbol> {
val id = CallableId(pkgName, name)
return pluginContext.referenceProperties(id)
}

1
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_8_0/SyntheticBodyKind.kt
View File

@@ -3,4 +3,3 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.expressions.IrSyntheticBodyKind
val kind_ENUM_ENTRIES: IrSyntheticBodyKind? = IrSyntheticBodyKind.ENUM_ENTRIES

38
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_9_0-Beta/CommentExtractorLighterAST.kt
View File

@@ -4,21 +4,26 @@ import com.github.codeql.*
import com.intellij.lang.LighterASTNode
import com.intellij.util.diff.FlyweightCapableTreeStructure
import org.jetbrains.kotlin.fir.backend.FirMetadataSource
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET
import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
import org.jetbrains.kotlin.ir.visitors.acceptVoid
import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
import org.jetbrains.kotlin.kdoc.lexer.KDocTokens
import org.jetbrains.kotlin.lexer.KtTokens
import org.jetbrains.kotlin.util.getChildren
class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFile, fileLabel: Label<out DbFile>): CommentExtractor(fileExtractor, file, fileLabel) {
class CommentExtractorLighterAST(
fileExtractor: KotlinFileExtractor,
file: IrFile,
fileLabel: Label<out DbFile>
) : CommentExtractor(fileExtractor, file, fileLabel) {
// Returns true if it extracted the comments; false otherwise.
fun extract(): Boolean {
val sourceElement = (file.metadata as? FirMetadataSource.File)?.files?.elementAtOrNull(0)?.source
val sourceElement =
(file.metadata as? FirMetadataSource.File)?.files?.elementAtOrNull(0)?.source
val treeStructure = sourceElement?.treeStructure
if (treeStructure == null) {
return false
@@ -33,22 +38,30 @@ class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFil
fun LighterASTNode.isKDocComment() = this.tokenType == KDocTokens.KDOC
val kDocOwners = mutableMapOf<Int, MutableList<IrElement>>()
val visitor = object : IrElementVisitorVoid {
val visitor =
object : IrElementVisitorVoid {
override fun visitElement(element: IrElement) {
val metadata = (element as? IrMetadataSourceOwner)?.metadata
val sourceElement = (metadata as? FirMetadataSource)?.fir?.source
val treeStructure = sourceElement?.treeStructure
if (treeStructure != null) {
sourceElement.lighterASTNode.getChildren(treeStructure).firstOrNull { it.isKDocComment() }
sourceElement.lighterASTNode
.getChildren(treeStructure)
.firstOrNull { it.isKDocComment() }
?.let { kDoc ->
// LighterASTNodes are not stable, so we can't
// use the node itself as the key. But the
// startOffset should uniquely identify them
// anyway.
val startOffset = kDoc.startOffset
if (startOffset != UNDEFINED_OFFSET && startOffset != SYNTHETIC_OFFSET) {
kDocOwners.getOrPut(startOffset, {mutableListOf<IrElement>()}).add(element)
if (
startOffset != UNDEFINED_OFFSET &&
startOffset != SYNTHETIC_OFFSET
) {
kDocOwners
.getOrPut(startOffset, { mutableListOf<IrElement>() })
.add(element)
}
}
}
@@ -60,7 +73,11 @@ class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFil
return kDocOwners
}
private fun extractComments(node: LighterASTNode, treeStructure: FlyweightCapableTreeStructure<LighterASTNode>, owners: Map<Int, List<IrElement>>) {
private fun extractComments(
node: LighterASTNode,
treeStructure: FlyweightCapableTreeStructure<LighterASTNode>,
owners: Map<Int, List<IrElement>>
) {
node.getChildren(treeStructure).forEach {
if (KtTokens.COMMENTS.contains(it.tokenType)) {
extractComment(it, owners)
@@ -71,7 +88,8 @@ class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFil
}
private fun extractComment(comment: LighterASTNode, owners: Map<Int, List<IrElement>>) {
val type: CommentType = when (comment.tokenType) {
val type: CommentType =
when (comment.tokenType) {
KtTokens.EOL_COMMENT -> {
CommentType.SingleLine
}

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_9_0-Beta/Kotlin2ComponentRegistrar.kt
View File

@@ -3,8 +3,8 @@
package com.github.codeql
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
@OptIn(ExperimentalCompilerApi::class)
abstract class Kotlin2ComponentRegistrar : ComponentRegistrar {

42
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_9_0-Beta/LinesOfCodeLighterAST.kt
View File

@@ -2,23 +2,19 @@ package com.github.codeql
import com.intellij.lang.LighterASTNode
import com.intellij.util.diff.FlyweightCapableTreeStructure
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.fir.backend.FirMetadataSource
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.KtSourceElement
import org.jetbrains.kotlin.fir.backend.FirMetadataSource
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.lexer.KtTokens
import org.jetbrains.kotlin.util.getChildren
class LinesOfCodeLighterAST(
val logger: FileLogger,
val tw: FileTrapWriter,
val file: IrFile
) {
class LinesOfCodeLighterAST(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
val fileEntry = file.fileEntry
fun linesOfCodeInFile(id: Label<DbFile>): Boolean {
val sourceElement = (file.metadata as? FirMetadataSource.File)?.files?.elementAtOrNull(0)?.source
val sourceElement =
(file.metadata as? FirMetadataSource.File)?.files?.elementAtOrNull(0)?.source
if (sourceElement == null) {
return false
}
@@ -42,7 +38,11 @@ class LinesOfCodeLighterAST(
return true
}
private fun linesOfCodeInLighterAST(id: Label<out DbSourceline>, e: IrElement, s: KtSourceElement) {
private fun linesOfCodeInLighterAST(
id: Label<out DbSourceline>,
e: IrElement,
s: KtSourceElement
) {
val rootStartOffset = s.startOffset
val rootEndOffset = s.endOffset
if (rootStartOffset < 0 || rootEndOffset < 0) {
@@ -63,14 +63,28 @@ class LinesOfCodeLighterAST(
val treeStructure = s.treeStructure
processSubtree(e, treeStructure, rootFirstLine, rootLastLine, lineContents, s.lighterASTNode)
processSubtree(
e,
treeStructure,
rootFirstLine,
rootLastLine,
lineContents,
s.lighterASTNode
)
val code = lineContents.count { it.containsCode }
val comment = lineContents.count { it.containsComment }
tw.writeNumlines(id, numLines, code, comment)
}
private fun processSubtree(e: IrElement, treeStructure: FlyweightCapableTreeStructure<LighterASTNode>, rootFirstLine: Int, rootLastLine: Int, lineContents: Array<LineContent>, node: LighterASTNode) {
private fun processSubtree(
e: IrElement,
treeStructure: FlyweightCapableTreeStructure<LighterASTNode>,
rootFirstLine: Int,
rootLastLine: Int,
lineContents: Array<LineContent>,
node: LighterASTNode
) {
if (KtTokens.WHITESPACES.contains(node.tokenType)) {
return
}
@@ -120,7 +134,7 @@ class LinesOfCodeLighterAST(
}
}
} else {
for(child in children) {
for (child in children) {
processSubtree(e, treeStructure, rootFirstLine, rootLastLine, lineContents, child)
}
}