Kotlin: Reformat code

Using:
    java -jar ktfmt-0.46-jar-with-dependencies.jar --kotlinlang-style java/kotlin-extractor/**/*.kt
This commit is contained in:
Ian Lynagh
2024-01-09 15:33:41 +00:00
parent 0bc1463ab0
commit bf611feab3
64 changed files with 7420 additions and 3219 deletions

View File

@@ -3,18 +3,26 @@ package com.github.codeql
import com.github.codeql.utils.isExternalFileClassMember import com.github.codeql.utils.isExternalFileClassMember
import com.semmle.extractor.java.OdasaOutput import com.semmle.extractor.java.OdasaOutput
import com.semmle.util.data.StringDigestor import com.semmle.util.data.StringDigestor
import java.io.BufferedWriter
import java.io.File
import java.util.ArrayList
import java.util.HashSet
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.isFileClass import org.jetbrains.kotlin.ir.util.isFileClass
import org.jetbrains.kotlin.ir.util.packageFqName import org.jetbrains.kotlin.ir.util.packageFqName
import java.io.BufferedWriter
import java.io.File
import java.util.ArrayList
import java.util.HashSet
import java.util.zip.GZIPOutputStream
class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression, val invocationTrapFile: String, val sourceFilePath: String, val primitiveTypeMapping: PrimitiveTypeMapping, val pluginContext: IrPluginContext, val globalExtensionState: KotlinExtractorGlobalState, val diagnosticTrapWriter: DiagnosticTrapWriter) { class ExternalDeclExtractor(
val logger: FileLogger,
val compression: Compression,
val invocationTrapFile: String,
val sourceFilePath: String,
val primitiveTypeMapping: PrimitiveTypeMapping,
val pluginContext: IrPluginContext,
val globalExtensionState: KotlinExtractorGlobalState,
val diagnosticTrapWriter: DiagnosticTrapWriter
) {
val declBinaryNames = HashMap<IrDeclaration, String>() val declBinaryNames = HashMap<IrDeclaration, String>()
val externalDeclsDone = HashSet<Pair<String, String>>() val externalDeclsDone = HashSet<Pair<String, String>>()
@@ -23,13 +31,17 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
val propertySignature = ";property" val propertySignature = ";property"
val fieldSignature = ";field" val fieldSignature = ";field"
val output = OdasaOutput(false, compression, logger).also { val output =
it.setCurrentSourceFile(File(sourceFilePath)) OdasaOutput(false, compression, logger).also {
} it.setCurrentSourceFile(File(sourceFilePath))
}
fun extractLater(d: IrDeclarationWithName, signature: String): Boolean { fun extractLater(d: IrDeclarationWithName, signature: String): Boolean {
if (d !is IrClass && !isExternalFileClassMember(d)) { if (d !is IrClass && !isExternalFileClassMember(d)) {
logger.errorElement("External declaration is neither a class, nor a top-level declaration", d) logger.errorElement(
"External declaration is neither a class, nor a top-level declaration",
d
)
return false return false
} }
val declBinaryName = declBinaryNames.getOrPut(d) { getIrElementBinaryName(d) } val declBinaryName = declBinaryNames.getOrPut(d) { getIrElementBinaryName(d) }
@@ -37,34 +49,54 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
if (ret) externalDeclWorkList.add(Pair(d, signature)) if (ret) externalDeclWorkList.add(Pair(d, signature))
return ret return ret
} }
fun extractLater(c: IrClass) = extractLater(c, "") fun extractLater(c: IrClass) = extractLater(c, "")
fun writeStubTrapFile(e: IrElement, signature: String = "") { fun writeStubTrapFile(e: IrElement, signature: String = "") {
extractElement(e, signature, true) { trapFileBW, _, _ -> extractElement(e, signature, true) { trapFileBW, _, _ ->
trapFileBW.write("// Trap file stubbed because this declaration was extracted from source in $sourceFilePath\n") trapFileBW.write(
"// Trap file stubbed because this declaration was extracted from source in $sourceFilePath\n"
)
trapFileBW.write("// Part of invocation $invocationTrapFile\n") trapFileBW.write("// Part of invocation $invocationTrapFile\n")
} }
} }
private fun extractElement(element: IrElement, possiblyLongSignature: String, fromSource: Boolean, extractorFn: (BufferedWriter, String, OdasaOutput.TrapFileManager) -> Unit) { private fun extractElement(
// In order to avoid excessively long signatures which can lead to trap file names longer than the filesystem element: IrElement,
possiblyLongSignature: String,
fromSource: Boolean,
extractorFn: (BufferedWriter, String, OdasaOutput.TrapFileManager) -> Unit
) {
// In order to avoid excessively long signatures which can lead to trap file names longer
// than the filesystem
// limit, we truncate and add a hash to preserve uniqueness if necessary. // limit, we truncate and add a hash to preserve uniqueness if necessary.
val signature = if (possiblyLongSignature.length > 100) { val signature =
possiblyLongSignature.substring(0, 92) + "#" + StringDigestor.digest(possiblyLongSignature).substring(0, 8) if (possiblyLongSignature.length > 100) {
} else { possiblyLongSignature } possiblyLongSignature.substring(0, 92) +
"#" +
StringDigestor.digest(possiblyLongSignature).substring(0, 8)
} else {
possiblyLongSignature
}
output.getTrapLockerForDecl(element, signature, fromSource).useAC { locker -> output.getTrapLockerForDecl(element, signature, fromSource).useAC { locker ->
locker.trapFileManager.useAC { manager -> locker.trapFileManager.useAC { manager ->
val shortName = when(element) { val shortName =
is IrDeclarationWithName -> element.name.asString() when (element) {
is IrFile -> element.name is IrDeclarationWithName -> element.name.asString()
else -> "(unknown name)" is IrFile -> element.name
} else -> "(unknown name)"
}
if (manager == null) { if (manager == null) {
logger.info("Skipping extracting external decl $shortName") logger.info("Skipping extracting external decl $shortName")
} else { } else {
val trapFile = manager.file val trapFile = manager.file
logger.info("Will write TRAP file $trapFile") logger.info("Will write TRAP file $trapFile")
val trapTmpFile = File.createTempFile("${trapFile.nameWithoutExtension}.", ".${trapFile.extension}.tmp", trapFile.parentFile) val trapTmpFile =
File.createTempFile(
"${trapFile.nameWithoutExtension}.",
".${trapFile.extension}.tmp",
trapFile.parentFile
)
logger.debug("Writing temporary TRAP file $trapTmpFile") logger.debug("Writing temporary TRAP file $trapTmpFile")
try { try {
compression.bufferedWriter(trapTmpFile).use { compression.bufferedWriter(trapTmpFile).use {
@@ -77,7 +109,10 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
logger.info("Finished writing TRAP file $trapFile") logger.info("Finished writing TRAP file $trapFile")
} catch (e: Exception) { } catch (e: Exception) {
manager.setHasError() manager.setHasError()
logger.error("Failed to extract '$shortName'. Partial TRAP file location is $trapTmpFile", e) logger.error(
"Failed to extract '$shortName'. Partial TRAP file location is $trapTmpFile",
e
)
} }
} }
} }
@@ -90,37 +125,75 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
externalDeclWorkList.clear() externalDeclWorkList.clear()
nextBatch.forEach { workPair -> nextBatch.forEach { workPair ->
val (irDecl, possiblyLongSignature) = workPair val (irDecl, possiblyLongSignature) = workPair
extractElement(irDecl, possiblyLongSignature, false) { trapFileBW, signature, manager -> extractElement(irDecl, possiblyLongSignature, false) {
trapFileBW,
signature,
manager ->
val binaryPath = getIrDeclarationBinaryPath(irDecl) val binaryPath = getIrDeclarationBinaryPath(irDecl)
if (binaryPath == null) { if (binaryPath == null) {
logger.errorElement("Unable to get binary path", irDecl) logger.errorElement("Unable to get binary path", irDecl)
} else { } else {
// We want our comments to be the first thing in the file, // We want our comments to be the first thing in the file,
// so start off with a PlainTrapWriter // so start off with a PlainTrapWriter
val tw = PlainTrapWriter(logger.loggerBase, TrapLabelManager(), trapFileBW, diagnosticTrapWriter) val tw =
tw.writeComment("Generated by the CodeQL Kotlin extractor for external dependencies") PlainTrapWriter(
logger.loggerBase,
TrapLabelManager(),
trapFileBW,
diagnosticTrapWriter
)
tw.writeComment(
"Generated by the CodeQL Kotlin extractor for external dependencies"
)
tw.writeComment("Part of invocation $invocationTrapFile") tw.writeComment("Part of invocation $invocationTrapFile")
if (signature != possiblyLongSignature) { if (signature != possiblyLongSignature) {
tw.writeComment("Function signature abbreviated; full signature is: $possiblyLongSignature") tw.writeComment(
"Function signature abbreviated; full signature is: $possiblyLongSignature"
)
} }
// Now elevate to a SourceFileTrapWriter, and populate the // Now elevate to a SourceFileTrapWriter, and populate the
// file information if needed: // file information if needed:
val ftw = tw.makeFileTrapWriter(binaryPath, true) val ftw = tw.makeFileTrapWriter(binaryPath, true)
val fileExtractor = KotlinFileExtractor(logger, ftw, null, binaryPath, manager, this, primitiveTypeMapping, pluginContext, KotlinFileExtractor.DeclarationStack(), globalExtensionState) val fileExtractor =
KotlinFileExtractor(
logger,
ftw,
null,
binaryPath,
manager,
this,
primitiveTypeMapping,
pluginContext,
KotlinFileExtractor.DeclarationStack(),
globalExtensionState
)
if (irDecl is IrClass) { if (irDecl is IrClass) {
// Populate a location and compilation-unit package for the file. This is similar to // Populate a location and compilation-unit package for the file. This
// the beginning of `KotlinFileExtractor.extractFileContents` but without an `IrFile` // is similar to
// the beginning of `KotlinFileExtractor.extractFileContents` but
// without an `IrFile`
// to start from. // to start from.
val pkg = irDecl.packageFqName?.asString() ?: "" val pkg = irDecl.packageFqName?.asString() ?: ""
val pkgId = fileExtractor.extractPackage(pkg) val pkgId = fileExtractor.extractPackage(pkg)
ftw.writeHasLocation(ftw.fileId, ftw.getWholeFileLocation()) ftw.writeHasLocation(ftw.fileId, ftw.getWholeFileLocation())
ftw.writeCupackage(ftw.fileId, pkgId) ftw.writeCupackage(ftw.fileId, pkgId)
fileExtractor.extractClassSource(irDecl, extractDeclarations = !irDecl.isFileClass, extractStaticInitializer = false, extractPrivateMembers = false, extractFunctionBodies = false) fileExtractor.extractClassSource(
irDecl,
extractDeclarations = !irDecl.isFileClass,
extractStaticInitializer = false,
extractPrivateMembers = false,
extractFunctionBodies = false
)
} else { } else {
fileExtractor.extractDeclaration(irDecl, extractPrivateMembers = false, extractFunctionBodies = false, extractAnnotations = true) fileExtractor.extractDeclaration(
irDecl,
extractPrivateMembers = false,
extractFunctionBodies = false,
extractAnnotations = true
)
} }
} }
} }
@@ -128,5 +201,4 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
} while (externalDeclWorkList.isNotEmpty()) } while (externalDeclWorkList.isNotEmpty())
output.writeTrapSet() output.writeTrapSet()
} }
} }

View File

@@ -11,66 +11,90 @@ import org.jetbrains.kotlin.config.CompilerConfigurationKey
class KotlinExtractorCommandLineProcessor : CommandLineProcessor { class KotlinExtractorCommandLineProcessor : CommandLineProcessor {
override val pluginId = "kotlin-extractor" override val pluginId = "kotlin-extractor"
override val pluginOptions = listOf( override val pluginOptions =
CliOption( listOf(
optionName = OPTION_INVOCATION_TRAP_FILE, CliOption(
valueDescription = "Invocation TRAP file", optionName = OPTION_INVOCATION_TRAP_FILE,
description = "Extractor will append invocation-related TRAP to this file", valueDescription = "Invocation TRAP file",
required = true, description = "Extractor will append invocation-related TRAP to this file",
allowMultipleOccurrences = false required = true,
), allowMultipleOccurrences = false
CliOption( ),
optionName = OPTION_CHECK_TRAP_IDENTICAL, CliOption(
valueDescription = "Check whether different invocations produce identical TRAP", optionName = OPTION_CHECK_TRAP_IDENTICAL,
description = "Check whether different invocations produce identical TRAP", valueDescription = "Check whether different invocations produce identical TRAP",
required = false, description = "Check whether different invocations produce identical TRAP",
allowMultipleOccurrences = false required = false,
), allowMultipleOccurrences = false
CliOption( ),
optionName = OPTION_COMPILATION_STARTTIME, CliOption(
valueDescription = "The start time of the compilation as a Unix timestamp", optionName = OPTION_COMPILATION_STARTTIME,
description = "The start time of the compilation as a Unix timestamp", valueDescription = "The start time of the compilation as a Unix timestamp",
required = false, description = "The start time of the compilation as a Unix timestamp",
allowMultipleOccurrences = false required = false,
), allowMultipleOccurrences = false
CliOption( ),
optionName = OPTION_EXIT_AFTER_EXTRACTION, CliOption(
valueDescription = "Specify whether to call exitProcess after the extraction has completed", optionName = OPTION_EXIT_AFTER_EXTRACTION,
description = "Specify whether to call exitProcess after the extraction has completed", valueDescription =
required = false, "Specify whether to call exitProcess after the extraction has completed",
allowMultipleOccurrences = false description =
"Specify whether to call exitProcess after the extraction has completed",
required = false,
allowMultipleOccurrences = false
)
) )
)
override fun processOption( override fun processOption(
option: AbstractCliOption, option: AbstractCliOption,
value: String, value: String,
configuration: CompilerConfiguration configuration: CompilerConfiguration
) = when (option.optionName) { ) =
OPTION_INVOCATION_TRAP_FILE -> configuration.put(KEY_INVOCATION_TRAP_FILE, value) when (option.optionName) {
OPTION_CHECK_TRAP_IDENTICAL -> processBooleanOption(value, OPTION_CHECK_TRAP_IDENTICAL, KEY_CHECK_TRAP_IDENTICAL, configuration) OPTION_INVOCATION_TRAP_FILE -> configuration.put(KEY_INVOCATION_TRAP_FILE, value)
OPTION_EXIT_AFTER_EXTRACTION -> processBooleanOption(value, OPTION_EXIT_AFTER_EXTRACTION, KEY_EXIT_AFTER_EXTRACTION, configuration) OPTION_CHECK_TRAP_IDENTICAL ->
OPTION_COMPILATION_STARTTIME -> processBooleanOption(
when (val v = value.toLongOrNull()) { value,
is Long -> configuration.put(KEY_COMPILATION_STARTTIME, v) OPTION_CHECK_TRAP_IDENTICAL,
else -> error("kotlin extractor: Bad argument $value for $OPTION_COMPILATION_STARTTIME") KEY_CHECK_TRAP_IDENTICAL,
} configuration
else -> error("kotlin extractor: Bad option: ${option.optionName}") )
} OPTION_EXIT_AFTER_EXTRACTION ->
processBooleanOption(
value,
OPTION_EXIT_AFTER_EXTRACTION,
KEY_EXIT_AFTER_EXTRACTION,
configuration
)
OPTION_COMPILATION_STARTTIME ->
when (val v = value.toLongOrNull()) {
is Long -> configuration.put(KEY_COMPILATION_STARTTIME, v)
else ->
error(
"kotlin extractor: Bad argument $value for $OPTION_COMPILATION_STARTTIME"
)
}
else -> error("kotlin extractor: Bad option: ${option.optionName}")
}
private fun processBooleanOption(value: String, optionName: String, configKey: CompilerConfigurationKey<Boolean>, configuration: CompilerConfiguration) = private fun processBooleanOption(
value: String,
optionName: String,
configKey: CompilerConfigurationKey<Boolean>,
configuration: CompilerConfiguration
) =
when (value) { when (value) {
"true" -> configuration.put(configKey, true) "true" -> configuration.put(configKey, true)
"false" -> configuration.put(configKey, false) "false" -> configuration.put(configKey, false)
else -> error("kotlin extractor: Bad argument $value for $optionName") else -> error("kotlin extractor: Bad argument $value for $optionName")
} }
} }
private val OPTION_INVOCATION_TRAP_FILE = "invocationTrapFile" private val OPTION_INVOCATION_TRAP_FILE = "invocationTrapFile"
val KEY_INVOCATION_TRAP_FILE = CompilerConfigurationKey<String>(OPTION_INVOCATION_TRAP_FILE) val KEY_INVOCATION_TRAP_FILE = CompilerConfigurationKey<String>(OPTION_INVOCATION_TRAP_FILE)
private val OPTION_CHECK_TRAP_IDENTICAL = "checkTrapIdentical" private val OPTION_CHECK_TRAP_IDENTICAL = "checkTrapIdentical"
val KEY_CHECK_TRAP_IDENTICAL= CompilerConfigurationKey<Boolean>(OPTION_CHECK_TRAP_IDENTICAL) val KEY_CHECK_TRAP_IDENTICAL = CompilerConfigurationKey<Boolean>(OPTION_CHECK_TRAP_IDENTICAL)
private val OPTION_COMPILATION_STARTTIME = "compilationStartTime" private val OPTION_COMPILATION_STARTTIME = "compilationStartTime"
val KEY_COMPILATION_STARTTIME= CompilerConfigurationKey<Long>(OPTION_COMPILATION_STARTTIME) val KEY_COMPILATION_STARTTIME = CompilerConfigurationKey<Long>(OPTION_COMPILATION_STARTTIME)
private val OPTION_EXIT_AFTER_EXTRACTION = "exitAfterExtraction" private val OPTION_EXIT_AFTER_EXTRACTION = "exitAfterExtraction"
val KEY_EXIT_AFTER_EXTRACTION= CompilerConfigurationKey<Boolean>(OPTION_EXIT_AFTER_EXTRACTION) val KEY_EXIT_AFTER_EXTRACTION = CompilerConfigurationKey<Boolean>(OPTION_EXIT_AFTER_EXTRACTION)

View File

@@ -3,12 +3,9 @@
package com.github.codeql package com.github.codeql
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
import com.intellij.mock.MockProject import com.intellij.mock.MockProject
import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.CompilerConfiguration
import com.github.codeql.Kotlin2ComponentRegistrar
class KotlinExtractorComponentRegistrar : Kotlin2ComponentRegistrar() { class KotlinExtractorComponentRegistrar : Kotlin2ComponentRegistrar() {
override fun registerProjectComponents( override fun registerProjectComponents(
@@ -19,10 +16,14 @@ class KotlinExtractorComponentRegistrar : Kotlin2ComponentRegistrar() {
if (invocationTrapFile == null) { if (invocationTrapFile == null) {
throw Exception("Required argument for TRAP invocation file not given") throw Exception("Required argument for TRAP invocation file not given")
} }
IrGenerationExtension.registerExtension(project, KotlinExtractorExtension( IrGenerationExtension.registerExtension(
invocationTrapFile, project,
configuration[KEY_CHECK_TRAP_IDENTICAL] ?: false, KotlinExtractorExtension(
configuration[KEY_COMPILATION_STARTTIME], invocationTrapFile,
configuration[KEY_EXIT_AFTER_EXTRACTION] ?: false)) configuration[KEY_CHECK_TRAP_IDENTICAL] ?: false,
configuration[KEY_COMPILATION_STARTTIME],
configuration[KEY_EXIT_AFTER_EXTRACTION] ?: false
)
)
} }
} }

View File

@@ -1,15 +1,11 @@
package com.github.codeql package com.github.codeql
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension import com.github.codeql.utils.versions.usesK2
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext import com.semmle.util.files.FileUtil
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.*
import org.jetbrains.kotlin.ir.IrElement
import java.io.BufferedReader
import java.io.BufferedWriter
import java.io.BufferedInputStream import java.io.BufferedInputStream
import java.io.BufferedOutputStream import java.io.BufferedOutputStream
import java.io.BufferedReader
import java.io.BufferedWriter
import java.io.File import java.io.File
import java.io.FileInputStream import java.io.FileInputStream
import java.io.FileOutputStream import java.io.FileOutputStream
@@ -20,9 +16,12 @@ import java.nio.file.Files
import java.nio.file.Paths import java.nio.file.Paths
import java.util.zip.GZIPInputStream import java.util.zip.GZIPInputStream
import java.util.zip.GZIPOutputStream import java.util.zip.GZIPOutputStream
import com.github.codeql.utils.versions.usesK2
import com.semmle.util.files.FileUtil
import kotlin.system.exitProcess import kotlin.system.exitProcess
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.*
/* /*
* KotlinExtractorExtension is the main entry point of the CodeQL Kotlin * KotlinExtractorExtension is the main entry point of the CodeQL Kotlin
@@ -55,8 +54,8 @@ class KotlinExtractorExtension(
// can be set to true to make the plugin terminate the kotlinc // can be set to true to make the plugin terminate the kotlinc
// invocation when it has finished. This means that kotlinc will not // invocation when it has finished. This means that kotlinc will not
// write any `.class` files etc. // write any `.class` files etc.
private val exitAfterExtraction: Boolean) private val exitAfterExtraction: Boolean
: IrGenerationExtension { ) : IrGenerationExtension {
// This is the main entry point to the extractor. // This is the main entry point to the extractor.
// It will be called by kotlinc with the IR for the files being // It will be called by kotlinc with the IR for the files being
@@ -65,10 +64,10 @@ class KotlinExtractorExtension(
override fun generate(moduleFragment: IrModuleFragment, pluginContext: IrPluginContext) { override fun generate(moduleFragment: IrModuleFragment, pluginContext: IrPluginContext) {
try { try {
runExtractor(moduleFragment, pluginContext) runExtractor(moduleFragment, pluginContext)
// We catch Throwable rather than Exception, as we want to // We catch Throwable rather than Exception, as we want to
// continue trying to extract everything else even if we get a // continue trying to extract everything else even if we get a
// stack overflow or an assertion failure in one file. // stack overflow or an assertion failure in one file.
} catch(e: Throwable) { } catch (e: Throwable) {
// If we get an exception at the top level, then something's // If we get an exception at the top level, then something's
// gone very wrong. Don't try to be too fancy, but try to // gone very wrong. Don't try to be too fancy, but try to
// log a simple message. // log a simple message.
@@ -80,7 +79,8 @@ class KotlinExtractorExtension(
// We use a slightly different filename pattern compared // We use a slightly different filename pattern compared
// to normal logs. Just the existence of a `-top` log is // to normal logs. Just the existence of a `-top` log is
// a sign that something's gone very wrong. // a sign that something's gone very wrong.
val logFile = File.createTempFile("kotlin-extractor-top.", ".log", File(extractorLogDir)) val logFile =
File.createTempFile("kotlin-extractor-top.", ".log", File(extractorLogDir))
logFile.writeText(msg) logFile.writeText(msg)
// Now we've got that out, let's see if we can append a stack trace too // Now we've got that out, let's see if we can append a stack trace too
logFile.appendText(e.stackTraceToString()) logFile.appendText(e.stackTraceToString())
@@ -99,12 +99,18 @@ class KotlinExtractorExtension(
private fun runExtractor(moduleFragment: IrModuleFragment, pluginContext: IrPluginContext) { private fun runExtractor(moduleFragment: IrModuleFragment, pluginContext: IrPluginContext) {
val startTimeMs = System.currentTimeMillis() val startTimeMs = System.currentTimeMillis()
val usesK2 = usesK2(pluginContext) val usesK2 = usesK2(pluginContext)
// This default should be kept in sync with com.semmle.extractor.java.interceptors.KotlinInterceptor.initializeExtractionContext // This default should be kept in sync with
val trapDir = File(System.getenv("CODEQL_EXTRACTOR_JAVA_TRAP_DIR").takeUnless { it.isNullOrEmpty() } ?: "kotlin-extractor/trap") // com.semmle.extractor.java.interceptors.KotlinInterceptor.initializeExtractionContext
val trapDir =
File(
System.getenv("CODEQL_EXTRACTOR_JAVA_TRAP_DIR").takeUnless { it.isNullOrEmpty() }
?: "kotlin-extractor/trap"
)
// The invocation TRAP file will already have been started // The invocation TRAP file will already have been started
// before the plugin is run, so we always use no compression // before the plugin is run, so we always use no compression
// and we open it in append mode. // and we open it in append mode.
FileOutputStream(File(invocationTrapFile), true).bufferedWriter().use { invocationTrapFileBW -> FileOutputStream(File(invocationTrapFile), true).bufferedWriter().use { invocationTrapFileBW
->
val invocationExtractionProblems = ExtractionProblems() val invocationExtractionProblems = ExtractionProblems()
val lm = TrapLabelManager() val lm = TrapLabelManager()
val logCounter = LogCounter() val logCounter = LogCounter()
@@ -113,12 +119,21 @@ class KotlinExtractorExtension(
// The interceptor has already defined #compilation = * // The interceptor has already defined #compilation = *
val compilation: Label<DbCompilation> = StringLabel("compilation") val compilation: Label<DbCompilation> = StringLabel("compilation")
tw.writeCompilation_started(compilation) tw.writeCompilation_started(compilation)
tw.writeCompilation_info(compilation, "Kotlin Compiler Version", KotlinCompilerVersion.getVersion() ?: "<unknown>") tw.writeCompilation_info(
val extractor_name = this::class.java.getResource("extractor.name")?.readText() ?: "<unknown>" compilation,
"Kotlin Compiler Version",
KotlinCompilerVersion.getVersion() ?: "<unknown>"
)
val extractor_name =
this::class.java.getResource("extractor.name")?.readText() ?: "<unknown>"
tw.writeCompilation_info(compilation, "Kotlin Extractor Name", extractor_name) tw.writeCompilation_info(compilation, "Kotlin Extractor Name", extractor_name)
tw.writeCompilation_info(compilation, "Uses Kotlin 2", usesK2.toString()) tw.writeCompilation_info(compilation, "Uses Kotlin 2", usesK2.toString())
if (compilationStartTime != null) { if (compilationStartTime != null) {
tw.writeCompilation_compiler_times(compilation, -1.0, (System.currentTimeMillis()-compilationStartTime)/1000.0) tw.writeCompilation_compiler_times(
compilation,
-1.0,
(System.currentTimeMillis() - compilationStartTime) / 1000.0
)
} }
tw.flush() tw.flush()
val logger = Logger(loggerBase, tw) val logger = Logger(loggerBase, tw)
@@ -138,23 +153,54 @@ class KotlinExtractorExtension(
// FIXME: FileUtil expects a static global logger // FIXME: FileUtil expects a static global logger
// which should be provided by SLF4J's factory facility. For now we set it here. // which should be provided by SLF4J's factory facility. For now we set it here.
FileUtil.logger = logger FileUtil.logger = logger
val srcDir = File(System.getenv("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR").takeUnless { it.isNullOrEmpty() } ?: "kotlin-extractor/src") val srcDir =
File(
System.getenv("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR").takeUnless {
it.isNullOrEmpty()
} ?: "kotlin-extractor/src"
)
srcDir.mkdirs() srcDir.mkdirs()
val globalExtensionState = KotlinExtractorGlobalState() val globalExtensionState = KotlinExtractorGlobalState()
moduleFragment.files.mapIndexed { index: Int, file: IrFile -> moduleFragment.files.mapIndexed { index: Int, file: IrFile ->
val fileExtractionProblems = FileExtractionProblems(invocationExtractionProblems) val fileExtractionProblems = FileExtractionProblems(invocationExtractionProblems)
val fileTrapWriter = tw.makeSourceFileTrapWriter(file, true) val fileTrapWriter = tw.makeSourceFileTrapWriter(file, true)
loggerBase.setFileNumber(index) loggerBase.setFileNumber(index)
fileTrapWriter.writeCompilation_compiling_files(compilation, index, fileTrapWriter.fileId) fileTrapWriter.writeCompilation_compiling_files(
doFile(compression, fileExtractionProblems, invocationTrapFile, fileTrapWriter, checkTrapIdentical, loggerBase, trapDir, srcDir, file, primitiveTypeMapping, pluginContext, globalExtensionState) compilation,
fileTrapWriter.writeCompilation_compiling_files_completed(compilation, index, fileExtractionProblems.extractionResult()) index,
fileTrapWriter.fileId
)
doFile(
compression,
fileExtractionProblems,
invocationTrapFile,
fileTrapWriter,
checkTrapIdentical,
loggerBase,
trapDir,
srcDir,
file,
primitiveTypeMapping,
pluginContext,
globalExtensionState
)
fileTrapWriter.writeCompilation_compiling_files_completed(
compilation,
index,
fileExtractionProblems.extractionResult()
)
} }
loggerBase.printLimitedDiagnosticCounts(tw) loggerBase.printLimitedDiagnosticCounts(tw)
logPeakMemoryUsage(logger, "after extractor") logPeakMemoryUsage(logger, "after extractor")
logger.info("Extraction completed") logger.info("Extraction completed")
logger.flush() logger.flush()
val compilationTimeMs = System.currentTimeMillis() - startTimeMs val compilationTimeMs = System.currentTimeMillis() - startTimeMs
tw.writeCompilation_finished(compilation, -1.0, compilationTimeMs.toDouble() / 1000, invocationExtractionProblems.extractionResult()) tw.writeCompilation_finished(
compilation,
-1.0,
compilationTimeMs.toDouble() / 1000,
invocationExtractionProblems.extractionResult()
)
tw.flush() tw.flush()
loggerBase.close() loggerBase.close()
} }
@@ -170,13 +216,17 @@ class KotlinExtractorExtension(
try { try {
val compression_option_upper = compression_option.uppercase() val compression_option_upper = compression_option.uppercase()
if (compression_option_upper == "BROTLI") { if (compression_option_upper == "BROTLI") {
logger.warn("Kotlin extractor doesn't support Brotli compression. Using GZip instead.") logger.warn(
"Kotlin extractor doesn't support Brotli compression. Using GZip instead."
)
return Compression.GZIP return Compression.GZIP
} else { } else {
return Compression.valueOf(compression_option_upper) return Compression.valueOf(compression_option_upper)
} }
} catch (e: IllegalArgumentException) { } catch (e: IllegalArgumentException) {
logger.warn("Unsupported compression type (\$$compression_env_var) \"$compression_option\". Supported values are ${Compression.values().joinToString()}.") logger.warn(
"Unsupported compression type (\$$compression_env_var) \"$compression_option\". Supported values are ${Compression.values().joinToString()}."
)
return defaultCompression return defaultCompression
} }
} }
@@ -190,11 +240,18 @@ class KotlinExtractorExtension(
var nonheap: Long = 0 var nonheap: Long = 0
for (bean in beans) { for (bean in beans) {
val peak = bean.getPeakUsage().getUsed() val peak = bean.getPeakUsage().getUsed()
val kind = when (bean.getType()) { val kind =
MemoryType.HEAP -> { heap += peak; "heap" } when (bean.getType()) {
MemoryType.NON_HEAP -> { nonheap += peak; "non-heap" } MemoryType.HEAP -> {
else -> "unknown" heap += peak
} "heap"
}
MemoryType.NON_HEAP -> {
nonheap += peak
"non-heap"
}
else -> "unknown"
}
logger.info("Peak memory: * Peak for $kind bean ${bean.getName()} is $peak") logger.info("Peak memory: * Peak for $kind bean ${bean.getName()} is $peak")
} }
logger.info("Peak memory: * Total heap peak: $heap") logger.info("Peak memory: * Total heap peak: $heap")
@@ -203,9 +260,12 @@ class KotlinExtractorExtension(
} }
class KotlinExtractorGlobalState { class KotlinExtractorGlobalState {
// These three record mappings of classes, functions and fields that should be replaced wherever they are found. // These three record mappings of classes, functions and fields that should be replaced wherever
// As of now these are only used to fix IR generated by the Gradle Android Extensions plugin, hence e.g. IrProperty // they are found.
// doesn't have a map as that plugin doesn't generate them. If and when these are used more widely additional maps // As of now these are only used to fix IR generated by the Gradle Android Extensions plugin,
// hence e.g. IrProperty
// doesn't have a map as that plugin doesn't generate them. If and when these are used more
// widely additional maps
// should be added here. // should be added here.
val syntheticToRealClassMap = HashMap<IrClass, IrClass?>() val syntheticToRealClassMap = HashMap<IrClass, IrClass?>()
val syntheticToRealFunctionMap = HashMap<IrFunction, IrFunction?>() val syntheticToRealFunctionMap = HashMap<IrFunction, IrFunction?>()
@@ -227,13 +287,15 @@ open class ExtractionProblems {
open fun setRecoverableProblem() { open fun setRecoverableProblem() {
recoverableProblem = true recoverableProblem = true
} }
open fun setNonRecoverableProblem() { open fun setNonRecoverableProblem() {
nonRecoverableProblem = true nonRecoverableProblem = true
} }
fun extractionResult(): Int { fun extractionResult(): Int {
if(nonRecoverableProblem) { if (nonRecoverableProblem) {
return 2 return 2
} else if(recoverableProblem) { } else if (recoverableProblem) {
return 1 return 1
} else { } else {
return 0 return 0
@@ -246,11 +308,13 @@ The `FileExtractionProblems` is analogous to `ExtractionProblems`,
except it records whether there were any problems while extracting a except it records whether there were any problems while extracting a
particular source file. particular source file.
*/ */
class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblems): ExtractionProblems() { class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblems) :
ExtractionProblems() {
override fun setRecoverableProblem() { override fun setRecoverableProblem() {
super.setRecoverableProblem() super.setRecoverableProblem()
invocationExtractionProblems.setRecoverableProblem() invocationExtractionProblems.setRecoverableProblem()
} }
override fun setNonRecoverableProblem() { override fun setNonRecoverableProblem() {
super.setNonRecoverableProblem() super.setNonRecoverableProblem()
invocationExtractionProblems.setNonRecoverableProblem() invocationExtractionProblems.setNonRecoverableProblem()
@@ -265,7 +329,7 @@ identical.
private fun equivalentTrap(r1: BufferedReader, r2: BufferedReader): Boolean { private fun equivalentTrap(r1: BufferedReader, r2: BufferedReader): Boolean {
r1.use { br1 -> r1.use { br1 ->
r2.use { br2 -> r2.use { br2 ->
while(true) { while (true) {
val l1 = br1.readLine() val l1 = br1.readLine()
val l2 = br2.readLine() val l2 = br2.readLine()
if (l1 == null && l2 == null) { if (l1 == null && l2 == null) {
@@ -294,7 +358,8 @@ private fun doFile(
srcFile: IrFile, srcFile: IrFile,
primitiveTypeMapping: PrimitiveTypeMapping, primitiveTypeMapping: PrimitiveTypeMapping,
pluginContext: IrPluginContext, pluginContext: IrPluginContext,
globalExtensionState: KotlinExtractorGlobalState) { globalExtensionState: KotlinExtractorGlobalState
) {
val srcFilePath = srcFile.path val srcFilePath = srcFile.path
val logger = FileLogger(loggerBase, fileTrapWriter) val logger = FileLogger(loggerBase, fileTrapWriter)
logger.info("Extracting file $srcFilePath") logger.info("Extracting file $srcFilePath")
@@ -311,10 +376,13 @@ private fun doFile(
val dbSrcFilePath = Paths.get("$dbSrcDir/$srcFileRelativePath") val dbSrcFilePath = Paths.get("$dbSrcDir/$srcFileRelativePath")
val dbSrcDirPath = dbSrcFilePath.parent val dbSrcDirPath = dbSrcFilePath.parent
Files.createDirectories(dbSrcDirPath) Files.createDirectories(dbSrcDirPath)
val srcTmpFile = File.createTempFile(dbSrcFilePath.fileName.toString() + ".", ".src.tmp", dbSrcDirPath.toFile()) val srcTmpFile =
srcTmpFile.outputStream().use { File.createTempFile(
Files.copy(Paths.get(srcFilePath), it) dbSrcFilePath.fileName.toString() + ".",
} ".src.tmp",
dbSrcDirPath.toFile()
)
srcTmpFile.outputStream().use { Files.copy(Paths.get(srcFilePath), it) }
srcTmpFile.renameTo(dbSrcFilePath.toFile()) srcTmpFile.renameTo(dbSrcFilePath.toFile())
val trapFileName = "$dbTrapDir/$srcFileRelativePath.trap" val trapFileName = "$dbTrapDir/$srcFileRelativePath.trap"
@@ -327,22 +395,52 @@ private fun doFile(
trapFileWriter.getTempWriter().use { trapFileBW -> trapFileWriter.getTempWriter().use { trapFileBW ->
// We want our comments to be the first thing in the file, // We want our comments to be the first thing in the file,
// so start off with a mere TrapWriter // so start off with a mere TrapWriter
val tw = PlainTrapWriter(loggerBase, TrapLabelManager(), trapFileBW, fileTrapWriter.getDiagnosticTrapWriter()) val tw =
PlainTrapWriter(
loggerBase,
TrapLabelManager(),
trapFileBW,
fileTrapWriter.getDiagnosticTrapWriter()
)
tw.writeComment("Generated by the CodeQL Kotlin extractor for kotlin source code") tw.writeComment("Generated by the CodeQL Kotlin extractor for kotlin source code")
tw.writeComment("Part of invocation $invocationTrapFile") tw.writeComment("Part of invocation $invocationTrapFile")
// Now elevate to a SourceFileTrapWriter, and populate the // Now elevate to a SourceFileTrapWriter, and populate the
// file information // file information
val sftw = tw.makeSourceFileTrapWriter(srcFile, true) val sftw = tw.makeSourceFileTrapWriter(srcFile, true)
val externalDeclExtractor = ExternalDeclExtractor(logger, compression, invocationTrapFile, srcFilePath, primitiveTypeMapping, pluginContext, globalExtensionState, fileTrapWriter.getDiagnosticTrapWriter()) val externalDeclExtractor =
ExternalDeclExtractor(
logger,
compression,
invocationTrapFile,
srcFilePath,
primitiveTypeMapping,
pluginContext,
globalExtensionState,
fileTrapWriter.getDiagnosticTrapWriter()
)
val linesOfCode = LinesOfCode(logger, sftw, srcFile) val linesOfCode = LinesOfCode(logger, sftw, srcFile)
val fileExtractor = KotlinFileExtractor(logger, sftw, linesOfCode, srcFilePath, null, externalDeclExtractor, primitiveTypeMapping, pluginContext, KotlinFileExtractor.DeclarationStack(), globalExtensionState) val fileExtractor =
KotlinFileExtractor(
logger,
sftw,
linesOfCode,
srcFilePath,
null,
externalDeclExtractor,
primitiveTypeMapping,
pluginContext,
KotlinFileExtractor.DeclarationStack(),
globalExtensionState
)
fileExtractor.extractFileContents(srcFile, sftw.fileId) fileExtractor.extractFileContents(srcFile, sftw.fileId)
externalDeclExtractor.extractExternalClasses() externalDeclExtractor.extractExternalClasses()
} }
if (checkTrapIdentical && trapFileWriter.exists()) { if (checkTrapIdentical && trapFileWriter.exists()) {
if (equivalentTrap(trapFileWriter.getTempReader(), trapFileWriter.getRealReader())) { if (
equivalentTrap(trapFileWriter.getTempReader(), trapFileWriter.getRealReader())
) {
trapFileWriter.deleteTemp() trapFileWriter.deleteTemp()
} else { } else {
trapFileWriter.renameTempToDifferent() trapFileWriter.renameTempToDifferent()
@@ -350,9 +448,9 @@ private fun doFile(
} else { } else {
trapFileWriter.renameTempToReal() trapFileWriter.renameTempToReal()
} }
// We catch Throwable rather than Exception, as we want to // We catch Throwable rather than Exception, as we want to
// continue trying to extract everything else even if we get a // continue trying to extract everything else even if we get a
// stack overflow or an assertion failure in one file. // stack overflow or an assertion failure in one file.
} catch (e: Throwable) { } catch (e: Throwable) {
logger.error("Failed to extract '$srcFilePath'. " + trapFileWriter.debugInfo(), e) logger.error("Failed to extract '$srcFilePath'. " + trapFileWriter.debugInfo(), e)
context.clear() context.clear()
@@ -372,17 +470,26 @@ enum class Compression(val extension: String) {
return GZIPOutputStream(file.outputStream()).bufferedWriter() return GZIPOutputStream(file.outputStream()).bufferedWriter()
} }
}; };
abstract fun bufferedWriter(file: File): BufferedWriter abstract fun bufferedWriter(file: File): BufferedWriter
} }
private fun getTrapFileWriter(compression: Compression, logger: FileLogger, trapFileName: String): TrapFileWriter { private fun getTrapFileWriter(
compression: Compression,
logger: FileLogger,
trapFileName: String
): TrapFileWriter {
return when (compression) { return when (compression) {
Compression.NONE -> NonCompressedTrapFileWriter(logger, trapFileName) Compression.NONE -> NonCompressedTrapFileWriter(logger, trapFileName)
Compression.GZIP -> GZipCompressedTrapFileWriter(logger, trapFileName) Compression.GZIP -> GZipCompressedTrapFileWriter(logger, trapFileName)
} }
} }
private abstract class TrapFileWriter(val logger: FileLogger, trapName: String, val extension: String) { private abstract class TrapFileWriter(
val logger: FileLogger,
trapName: String,
val extension: String
) {
private val realFile = File(trapName + extension) private val realFile = File(trapName + extension)
private val parentDir = realFile.parentFile private val parentDir = realFile.parentFile
lateinit private var tempFile: File lateinit private var tempFile: File
@@ -404,6 +511,7 @@ private abstract class TrapFileWriter(val logger: FileLogger, trapName: String,
} }
abstract protected fun getReader(file: File): BufferedReader abstract protected fun getReader(file: File): BufferedReader
abstract protected fun getWriter(file: File): BufferedWriter abstract protected fun getWriter(file: File): BufferedWriter
fun getRealReader(): BufferedReader { fun getRealReader(): BufferedReader {
@@ -431,7 +539,8 @@ private abstract class TrapFileWriter(val logger: FileLogger, trapName: String,
} }
fun renameTempToDifferent() { fun renameTempToDifferent() {
val trapDifferentFile = File.createTempFile(realFile.getName() + ".", ".trap.different" + extension, parentDir) val trapDifferentFile =
File.createTempFile(realFile.getName() + ".", ".trap.different" + extension, parentDir)
if (tempFile.renameTo(trapDifferentFile)) { if (tempFile.renameTo(trapDifferentFile)) {
logger.warn("TRAP difference: $realFile vs $trapDifferentFile") logger.warn("TRAP difference: $realFile vs $trapDifferentFile")
} else { } else {
@@ -447,7 +556,8 @@ private abstract class TrapFileWriter(val logger: FileLogger, trapName: String,
} }
} }
private class NonCompressedTrapFileWriter(logger: FileLogger, trapName: String): TrapFileWriter(logger, trapName, "") { private class NonCompressedTrapFileWriter(logger: FileLogger, trapName: String) :
TrapFileWriter(logger, trapName, "") {
override protected fun getReader(file: File): BufferedReader { override protected fun getReader(file: File): BufferedReader {
return file.bufferedReader() return file.bufferedReader()
} }
@@ -457,12 +567,17 @@ private class NonCompressedTrapFileWriter(logger: FileLogger, trapName: String):
} }
} }
private class GZipCompressedTrapFileWriter(logger: FileLogger, trapName: String): TrapFileWriter(logger, trapName, ".gz") { private class GZipCompressedTrapFileWriter(logger: FileLogger, trapName: String) :
TrapFileWriter(logger, trapName, ".gz") {
override protected fun getReader(file: File): BufferedReader { override protected fun getReader(file: File): BufferedReader {
return BufferedReader(InputStreamReader(GZIPInputStream(BufferedInputStream(FileInputStream(file))))) return BufferedReader(
InputStreamReader(GZIPInputStream(BufferedInputStream(FileInputStream(file))))
)
} }
override protected fun getWriter(file: File): BufferedWriter { override protected fun getWriter(file: File): BufferedWriter {
return BufferedWriter(OutputStreamWriter(GZIPOutputStream(BufferedOutputStream(FileOutputStream(file))))) return BufferedWriter(
OutputStreamWriter(GZIPOutputStream(BufferedOutputStream(FileOutputStream(file))))
)
} }
} }

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,31 +1,21 @@
package com.github.codeql package com.github.codeql
import java.io.PrintWriter /** This represents a label (`#...`) in a TRAP file. */
import java.io.StringWriter interface Label<T : AnyDbType> {
fun <U : AnyDbType> cast(): Label<U> {
/** @Suppress("UNCHECKED_CAST") return this as Label<U>
* This represents a label (`#...`) in a TRAP file.
*/
interface Label<T: AnyDbType> {
fun <U: AnyDbType> cast(): Label<U> {
@Suppress("UNCHECKED_CAST")
return this as Label<U>
} }
} }
/** /** The label `#i`, e.g. `#123`. Most labels we generate are of this form. */
* The label `#i`, e.g. `#123`. Most labels we generate are of this class IntLabel<T : AnyDbType>(val i: Int) : Label<T> {
* form.
*/
class IntLabel<T: AnyDbType>(val i: Int): Label<T> {
override fun toString(): String = "#$i" override fun toString(): String = "#$i"
} }
/** /**
* The label `#name`, e.g. `#compilation`. This is used when labels are * The label `#name`, e.g. `#compilation`. This is used when labels are shared between different
* shared between different components (e.g. when both the interceptor * components (e.g. when both the interceptor and the extractor need to refer to the same label).
* and the extractor need to refer to the same label).
*/ */
class StringLabel<T: AnyDbType>(val name: String): Label<T> { class StringLabel<T : AnyDbType>(val name: String) : Label<T> {
override fun toString(): String = "#$name" override fun toString(): String = "#$name"
} }

View File

@@ -2,11 +2,7 @@ package com.github.codeql
import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.declarations.*
class LinesOfCode( class LinesOfCode(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
val logger: FileLogger,
val tw: FileTrapWriter,
val file: IrFile
) {
val linesOfCodePSI = LinesOfCodePSI(logger, tw, file) val linesOfCodePSI = LinesOfCodePSI(logger, tw, file)
val linesOfCodeLighterAST = LinesOfCodeLighterAST(logger, tw, file) val linesOfCodeLighterAST = LinesOfCodeLighterAST(logger, tw, file)
@@ -14,7 +10,10 @@ class LinesOfCode(
val psiExtracted = linesOfCodePSI.linesOfCodeInFile(id) val psiExtracted = linesOfCodePSI.linesOfCodeInFile(id)
val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInFile(id) val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInFile(id)
if (psiExtracted && lighterASTExtracted) { if (psiExtracted && lighterASTExtracted) {
logger.warnElement("Both PSI and LighterAST number-of-lines-in-file information for ${file.path}.", file) logger.warnElement(
"Both PSI and LighterAST number-of-lines-in-file information for ${file.path}.",
file
)
} }
} }
@@ -22,7 +21,10 @@ class LinesOfCode(
val psiExtracted = linesOfCodePSI.linesOfCodeInDeclaration(d, id) val psiExtracted = linesOfCodePSI.linesOfCodeInDeclaration(d, id)
val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInDeclaration(d, id) val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInDeclaration(d, id)
if (psiExtracted && lighterASTExtracted) { if (psiExtracted && lighterASTExtracted) {
logger.warnElement("Both PSI and LighterAST number-of-lines-in-file information for declaration.", d) logger.warnElement(
"Both PSI and LighterAST number-of-lines-in-file information for declaration.",
d
)
} }
} }
} }

View File

@@ -7,20 +7,17 @@ import com.intellij.psi.PsiWhiteSpace
import org.jetbrains.kotlin.config.KotlinCompilerVersion import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.kdoc.psi.api.KDocElement
import org.jetbrains.kotlin.psi.KtCodeFragment
import org.jetbrains.kotlin.psi.KtVisitor import org.jetbrains.kotlin.psi.KtVisitor
class LinesOfCodePSI( class LinesOfCodePSI(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
val logger: FileLogger, val psi2Ir =
val tw: FileTrapWriter, getPsi2Ir().also {
val file: IrFile if (it == null) {
) { logger.warn(
val psi2Ir = getPsi2Ir().also { "Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})"
if (it == null) { )
logger.warn("Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})") }
} }
}
fun linesOfCodeInFile(id: Label<DbFile>): Boolean { fun linesOfCodeInFile(id: Label<DbFile>): Boolean {
if (psi2Ir == null) { if (psi2Ir == null) {

View File

@@ -3,6 +3,8 @@ package com.github.codeql
import com.github.codeql.utils.versions.copyParameterToFunction import com.github.codeql.utils.versions.copyParameterToFunction
import com.github.codeql.utils.versions.createImplicitParameterDeclarationWithWrappedDescriptor import com.github.codeql.utils.versions.createImplicitParameterDeclarationWithWrappedDescriptor
import com.github.codeql.utils.versions.getAnnotationType import com.github.codeql.utils.versions.getAnnotationType
import java.lang.annotation.ElementType
import java.util.HashSet
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.builtins.StandardNames import org.jetbrains.kotlin.builtins.StandardNames
import org.jetbrains.kotlin.config.JvmTarget import org.jetbrains.kotlin.config.JvmTarget
@@ -48,10 +50,12 @@ import org.jetbrains.kotlin.load.java.JvmAnnotationNames
import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
import java.lang.annotation.ElementType
import java.util.HashSet
class MetaAnnotationSupport(private val logger: FileLogger, private val pluginContext: IrPluginContext, private val extractor: KotlinFileExtractor) { class MetaAnnotationSupport(
private val logger: FileLogger,
private val pluginContext: IrPluginContext,
private val extractor: KotlinFileExtractor
) {
// Taken from AdditionalIrUtils.kt (not available in Kotlin < 1.6) // Taken from AdditionalIrUtils.kt (not available in Kotlin < 1.6)
private val IrConstructorCall.annotationClass private val IrConstructorCall.annotationClass
@@ -82,8 +86,7 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
wrapAnnotationEntriesInContainer(annotationClass, containerClass, grouped)?.let { wrapAnnotationEntriesInContainer(annotationClass, containerClass, grouped)?.let {
result.add(it) result.add(it)
} }
else else logger.warnElement("Failed to find an annotation container class", annotationClass)
logger.warnElement("Failed to find an annotation container class", annotationClass)
} }
return result return result
} }
@@ -91,9 +94,14 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
// Adapted from RepeatedAnnotationLowering.kt // Adapted from RepeatedAnnotationLowering.kt
private fun getOrCreateContainerClass(annotationClass: IrClass): IrClass? { private fun getOrCreateContainerClass(annotationClass: IrClass): IrClass? {
val metaAnnotations = annotationClass.annotations val metaAnnotations = annotationClass.annotations
val jvmRepeatable = metaAnnotations.find { it.symbol.owner.parentAsClass.fqNameWhenAvailable == JvmAnnotationNames.REPEATABLE_ANNOTATION } val jvmRepeatable =
metaAnnotations.find {
it.symbol.owner.parentAsClass.fqNameWhenAvailable ==
JvmAnnotationNames.REPEATABLE_ANNOTATION
}
return if (jvmRepeatable != null) { return if (jvmRepeatable != null) {
((jvmRepeatable.getValueArgument(0) as? IrClassReference)?.symbol as? IrClassSymbol)?.owner ((jvmRepeatable.getValueArgument(0) as? IrClassReference)?.symbol as? IrClassSymbol)
?.owner
} else { } else {
getOrCreateSyntheticRepeatableAnnotationContainer(annotationClass) getOrCreateSyntheticRepeatableAnnotationContainer(annotationClass)
} }
@@ -108,20 +116,28 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
val annotationType = annotationClass.typeWith() val annotationType = annotationClass.typeWith()
val containerConstructor = containerClass.primaryConstructor val containerConstructor = containerClass.primaryConstructor
if (containerConstructor == null) { if (containerConstructor == null) {
logger.warnElement("Expected container class to have a primary constructor", containerClass) logger.warnElement(
"Expected container class to have a primary constructor",
containerClass
)
return null return null
} else { } else {
return IrConstructorCallImpl.fromSymbolOwner(containerClass.defaultType, containerConstructor.symbol).apply { return IrConstructorCallImpl.fromSymbolOwner(
putValueArgument( containerClass.defaultType,
0, containerConstructor.symbol
IrVarargImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
pluginContext.irBuiltIns.arrayClass.typeWith(annotationType),
annotationType,
entries
)
) )
} .apply {
putValueArgument(
0,
IrVarargImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
pluginContext.irBuiltIns.arrayClass.typeWith(annotationType),
annotationType,
entries
)
)
}
} }
} }
@@ -134,14 +150,19 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
// Taken from AdditionalClassAnnotationLowering.kt // Taken from AdditionalClassAnnotationLowering.kt
private fun loadAnnotationTargets(targetEntry: IrConstructorCall): Set<KotlinTarget>? { private fun loadAnnotationTargets(targetEntry: IrConstructorCall): Set<KotlinTarget>? {
val valueArgument = targetEntry.getValueArgument(0) as? IrVararg ?: return null val valueArgument = targetEntry.getValueArgument(0) as? IrVararg ?: return null
return valueArgument.elements.filterIsInstance<IrGetEnumValue>().mapNotNull { return valueArgument.elements
KotlinTarget.valueOrNull(it.symbol.owner.name.asString()) .filterIsInstance<IrGetEnumValue>()
}.toSet() .mapNotNull { KotlinTarget.valueOrNull(it.symbol.owner.name.asString()) }
.toSet()
} }
private val javaAnnotationTargetElementType by lazy { extractor.referenceExternalClass("java.lang.annotation.ElementType") } private val javaAnnotationTargetElementType by lazy {
extractor.referenceExternalClass("java.lang.annotation.ElementType")
}
private val javaAnnotationTarget by lazy { extractor.referenceExternalClass("java.lang.annotation.Target") } private val javaAnnotationTarget by lazy {
extractor.referenceExternalClass("java.lang.annotation.Target")
}
private fun findEnumEntry(c: IrClass, name: String) = private fun findEnumEntry(c: IrClass, name: String) =
c.declarations.filterIsInstance<IrEnumEntry>().find { it.name.asString() == name } c.declarations.filterIsInstance<IrEnumEntry>().find { it.name.asString() == name }
@@ -168,10 +189,11 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
private val jvm8TargetMap by lazy { private val jvm8TargetMap by lazy {
javaAnnotationTargetElementType?.let { javaAnnotationTargetElementType?.let {
jvm6TargetMap?.let { j6Map -> jvm6TargetMap?.let { j6Map ->
j6Map + mapOf( j6Map +
KotlinTarget.TYPE_PARAMETER to findEnumEntry(it, "TYPE_PARAMETER"), mapOf(
KotlinTarget.TYPE to findEnumEntry(it, "TYPE_USE") KotlinTarget.TYPE_PARAMETER to findEnumEntry(it, "TYPE_PARAMETER"),
) KotlinTarget.TYPE to findEnumEntry(it, "TYPE_USE")
)
} }
} }
} }
@@ -179,45 +201,59 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
private fun getAnnotationTargetMap() = private fun getAnnotationTargetMap() =
if (pluginContext.platform?.any { it.targetPlatformVersion == JvmTarget.JVM_1_6 } == true) if (pluginContext.platform?.any { it.targetPlatformVersion == JvmTarget.JVM_1_6 } == true)
jvm6TargetMap jvm6TargetMap
else else jvm8TargetMap
jvm8TargetMap
// Adapted from AdditionalClassAnnotationLowering.kt // Adapted from AdditionalClassAnnotationLowering.kt
private fun generateTargetAnnotation(c: IrClass): IrConstructorCall? { private fun generateTargetAnnotation(c: IrClass): IrConstructorCall? {
if (c.hasAnnotation(JvmAnnotationNames.TARGET_ANNOTATION)) if (c.hasAnnotation(JvmAnnotationNames.TARGET_ANNOTATION)) return null
return null
val elementType = javaAnnotationTargetElementType ?: return null val elementType = javaAnnotationTargetElementType ?: return null
val targetType = javaAnnotationTarget ?: return null val targetType = javaAnnotationTarget ?: return null
val targetConstructor = targetType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null val targetConstructor =
targetType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null
val targets = getApplicableTargetSet(c) ?: return null val targets = getApplicableTargetSet(c) ?: return null
val annotationTargetMap = getAnnotationTargetMap() ?: return null val annotationTargetMap = getAnnotationTargetMap() ?: return null
val javaTargets = targets.mapNotNullTo(HashSet()) { annotationTargetMap[it] }.sortedBy { val javaTargets =
ElementType.valueOf(it.symbol.owner.name.asString()) targets
} .mapNotNullTo(HashSet()) { annotationTargetMap[it] }
val vararg = IrVarargImpl( .sortedBy { ElementType.valueOf(it.symbol.owner.name.asString()) }
UNDEFINED_OFFSET, UNDEFINED_OFFSET, val vararg =
type = pluginContext.irBuiltIns.arrayClass.typeWith(elementType.defaultType), IrVarargImpl(
varargElementType = elementType.defaultType UNDEFINED_OFFSET,
) UNDEFINED_OFFSET,
type = pluginContext.irBuiltIns.arrayClass.typeWith(elementType.defaultType),
varargElementType = elementType.defaultType
)
for (target in javaTargets) { for (target in javaTargets) {
vararg.elements.add( vararg.elements.add(
IrGetEnumValueImpl( IrGetEnumValueImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, elementType.defaultType, target.symbol UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
elementType.defaultType,
target.symbol
) )
) )
} }
return IrConstructorCallImpl.fromSymbolOwner( return IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, targetConstructor.returnType, targetConstructor.symbol, 0 UNDEFINED_OFFSET,
).apply { UNDEFINED_OFFSET,
putValueArgument(0, vararg) targetConstructor.returnType,
} targetConstructor.symbol,
0
)
.apply { putValueArgument(0, vararg) }
} }
private val javaAnnotationRetention by lazy { extractor.referenceExternalClass("java.lang.annotation.Retention") } private val javaAnnotationRetention by lazy {
private val javaAnnotationRetentionPolicy by lazy { extractor.referenceExternalClass("java.lang.annotation.RetentionPolicy") } extractor.referenceExternalClass("java.lang.annotation.Retention")
private val javaAnnotationRetentionPolicyRuntime by lazy { javaAnnotationRetentionPolicy?.let { findEnumEntry(it, "RUNTIME") } } }
private val javaAnnotationRetentionPolicy by lazy {
extractor.referenceExternalClass("java.lang.annotation.RetentionPolicy")
}
private val javaAnnotationRetentionPolicyRuntime by lazy {
javaAnnotationRetentionPolicy?.let { findEnumEntry(it, "RUNTIME") }
}
private val annotationRetentionMap by lazy { private val annotationRetentionMap by lazy {
javaAnnotationRetentionPolicy?.let { javaAnnotationRetentionPolicy?.let {
@@ -232,118 +268,164 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
// Taken from AnnotationCodegen.kt (not available in Kotlin < 1.6.20) // Taken from AnnotationCodegen.kt (not available in Kotlin < 1.6.20)
private fun IrClass.getAnnotationRetention(): KotlinRetention? { private fun IrClass.getAnnotationRetention(): KotlinRetention? {
val retentionArgument = val retentionArgument =
getAnnotation(StandardNames.FqNames.retention)?.getValueArgument(0) getAnnotation(StandardNames.FqNames.retention)?.getValueArgument(0) as? IrGetEnumValue
as? IrGetEnumValue ?: return null ?: return null
val retentionArgumentValue = retentionArgument.symbol.owner val retentionArgumentValue = retentionArgument.symbol.owner
return KotlinRetention.valueOf(retentionArgumentValue.name.asString()) return KotlinRetention.valueOf(retentionArgumentValue.name.asString())
} }
// Taken from AdditionalClassAnnotationLowering.kt // Taken from AdditionalClassAnnotationLowering.kt
private fun generateRetentionAnnotation(irClass: IrClass): IrConstructorCall? { private fun generateRetentionAnnotation(irClass: IrClass): IrConstructorCall? {
if (irClass.hasAnnotation(JvmAnnotationNames.RETENTION_ANNOTATION)) if (irClass.hasAnnotation(JvmAnnotationNames.RETENTION_ANNOTATION)) return null
return null
val retentionMap = annotationRetentionMap ?: return null val retentionMap = annotationRetentionMap ?: return null
val kotlinRetentionPolicy = irClass.getAnnotationRetention() val kotlinRetentionPolicy = irClass.getAnnotationRetention()
val javaRetentionPolicy = kotlinRetentionPolicy?.let { retentionMap[it] } ?: javaAnnotationRetentionPolicyRuntime ?: return null val javaRetentionPolicy =
kotlinRetentionPolicy?.let { retentionMap[it] }
?: javaAnnotationRetentionPolicyRuntime
?: return null
val retentionPolicyType = javaAnnotationRetentionPolicy ?: return null val retentionPolicyType = javaAnnotationRetentionPolicy ?: return null
val retentionType = javaAnnotationRetention ?: return null val retentionType = javaAnnotationRetention ?: return null
val targetConstructor = retentionType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null val targetConstructor =
retentionType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null
return IrConstructorCallImpl.fromSymbolOwner( return IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, targetConstructor.returnType, targetConstructor.symbol, 0 UNDEFINED_OFFSET,
).apply { UNDEFINED_OFFSET,
putValueArgument( targetConstructor.returnType,
0, targetConstructor.symbol,
IrGetEnumValueImpl( 0
UNDEFINED_OFFSET, UNDEFINED_OFFSET, retentionPolicyType.defaultType, javaRetentionPolicy.symbol
)
) )
} .apply {
putValueArgument(
0,
IrGetEnumValueImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
retentionPolicyType.defaultType,
javaRetentionPolicy.symbol
)
)
}
} }
private val javaAnnotationRepeatable by lazy { extractor.referenceExternalClass("java.lang.annotation.Repeatable") } private val javaAnnotationRepeatable by lazy {
private val kotlinAnnotationRepeatableContainer by lazy { extractor.referenceExternalClass("kotlin.jvm.internal.RepeatableContainer") } extractor.referenceExternalClass("java.lang.annotation.Repeatable")
}
private val kotlinAnnotationRepeatableContainer by lazy {
extractor.referenceExternalClass("kotlin.jvm.internal.RepeatableContainer")
}
// Taken from declarationBuilders.kt (not available in Kotlin < 1.6): // Taken from declarationBuilders.kt (not available in Kotlin < 1.6):
private fun addDefaultGetter(p: IrProperty, parentClass: IrClass) { private fun addDefaultGetter(p: IrProperty, parentClass: IrClass) {
val field = p.backingField ?: val field =
run { logger.warnElement("Expected property to have a backing field", p); return } p.backingField
?: run {
logger.warnElement("Expected property to have a backing field", p)
return
}
p.addGetter { p.addGetter {
origin = IrDeclarationOrigin.DEFAULT_PROPERTY_ACCESSOR origin = IrDeclarationOrigin.DEFAULT_PROPERTY_ACCESSOR
returnType = field.type returnType = field.type
}.apply { }
val thisReceiever = parentClass.thisReceiver ?: .apply {
run { logger.warnElement("Expected property's parent class to have a receiver parameter", parentClass); return } val thisReceiever =
val newParam = copyParameterToFunction(thisReceiever, this) parentClass.thisReceiver
dispatchReceiverParameter = newParam ?: run {
body = factory.createBlockBody(UNDEFINED_OFFSET, UNDEFINED_OFFSET).apply({ logger.warnElement(
this.statements.add( "Expected property's parent class to have a receiver parameter",
IrReturnImpl( parentClass
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
pluginContext.irBuiltIns.nothingType,
symbol,
IrGetFieldImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
field.symbol,
field.type,
IrGetValueImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
newParam.type,
newParam.symbol
) )
) return
) }
) val newParam = copyParameterToFunction(thisReceiever, this)
}) dispatchReceiverParameter = newParam
} body =
factory
.createBlockBody(UNDEFINED_OFFSET, UNDEFINED_OFFSET)
.apply({
this.statements.add(
IrReturnImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
pluginContext.irBuiltIns.nothingType,
symbol,
IrGetFieldImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
field.symbol,
field.type,
IrGetValueImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
newParam.type,
newParam.symbol
)
)
)
)
})
}
} }
// Taken from JvmCachedDeclarations.kt // Taken from JvmCachedDeclarations.kt
private fun getOrCreateSyntheticRepeatableAnnotationContainer(annotationClass: IrClass) = private fun getOrCreateSyntheticRepeatableAnnotationContainer(annotationClass: IrClass) =
extractor.globalExtensionState.syntheticRepeatableAnnotationContainers.getOrPut(annotationClass) { extractor.globalExtensionState.syntheticRepeatableAnnotationContainers.getOrPut(
val containerClass = pluginContext.irFactory.buildClass { annotationClass
kind = ClassKind.ANNOTATION_CLASS ) {
name = Name.identifier("Container") val containerClass =
}.apply { pluginContext.irFactory
createImplicitParameterDeclarationWithWrappedDescriptor() .buildClass {
parent = annotationClass kind = ClassKind.ANNOTATION_CLASS
superTypes = listOf(getAnnotationType(pluginContext)) name = Name.identifier("Container")
} }
.apply {
createImplicitParameterDeclarationWithWrappedDescriptor()
parent = annotationClass
superTypes = listOf(getAnnotationType(pluginContext))
}
val propertyName = Name.identifier("value") val propertyName = Name.identifier("value")
val propertyType = pluginContext.irBuiltIns.arrayClass.typeWith(annotationClass.typeWith()) val propertyType =
pluginContext.irBuiltIns.arrayClass.typeWith(annotationClass.typeWith())
containerClass.addConstructor { containerClass
isPrimary = true .addConstructor { isPrimary = true }
}.apply { .apply { addValueParameter(propertyName.identifier, propertyType) }
addValueParameter(propertyName.identifier, propertyType)
}
containerClass.addProperty { containerClass
name = propertyName .addProperty { name = propertyName }
}.apply property@{ .apply property@{
backingField = pluginContext.irFactory.buildField { backingField =
name = propertyName pluginContext.irFactory
type = propertyType .buildField {
}.apply { name = propertyName
parent = containerClass type = propertyType
correspondingPropertySymbol = this@property.symbol }
.apply {
parent = containerClass
correspondingPropertySymbol = this@property.symbol
}
addDefaultGetter(this, containerClass)
} }
addDefaultGetter(this, containerClass)
}
val repeatableContainerAnnotation = kotlinAnnotationRepeatableContainer?.constructors?.single() val repeatableContainerAnnotation =
kotlinAnnotationRepeatableContainer?.constructors?.single()
containerClass.annotations = annotationClass.annotations containerClass.annotations =
.filter { annotationClass.annotations
it.isAnnotationWithEqualFqName(StandardNames.FqNames.retention) || .filter {
it.isAnnotationWithEqualFqName(StandardNames.FqNames.retention) ||
it.isAnnotationWithEqualFqName(StandardNames.FqNames.target) it.isAnnotationWithEqualFqName(StandardNames.FqNames.target)
} }
.map { it.deepCopyWithSymbols(containerClass) } + .map { it.deepCopyWithSymbols(containerClass) } +
listOfNotNull( listOfNotNull(
repeatableContainerAnnotation?.let { repeatableContainerAnnotation?.let {
IrConstructorCallImpl.fromSymbolOwner( IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, it.returnType, it.symbol, 0 UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
it.returnType,
it.symbol,
0
) )
} }
) )
@@ -352,45 +434,80 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
} }
// Adapted from AdditionalClassAnnotationLowering.kt // Adapted from AdditionalClassAnnotationLowering.kt
private fun generateRepeatableAnnotation(irClass: IrClass, extractAnnotationTypeAccesses: Boolean): IrConstructorCall? { private fun generateRepeatableAnnotation(
if (!irClass.hasAnnotation(StandardNames.FqNames.repeatable) || irClass: IrClass,
irClass.hasAnnotation(JvmAnnotationNames.REPEATABLE_ANNOTATION) extractAnnotationTypeAccesses: Boolean
) return null ): IrConstructorCall? {
if (
!irClass.hasAnnotation(StandardNames.FqNames.repeatable) ||
irClass.hasAnnotation(JvmAnnotationNames.REPEATABLE_ANNOTATION)
)
return null
val repeatableConstructor = javaAnnotationRepeatable?.declarations?.firstIsInstanceOrNull<IrConstructor>() ?: return null val repeatableConstructor =
javaAnnotationRepeatable?.declarations?.firstIsInstanceOrNull<IrConstructor>()
?: return null
val containerClass = getOrCreateSyntheticRepeatableAnnotationContainer(irClass) val containerClass = getOrCreateSyntheticRepeatableAnnotationContainer(irClass)
// Whenever a repeatable annotation with a Kotlin-synthesised container is extracted, extract the synthetic container to the same trap file. // Whenever a repeatable annotation with a Kotlin-synthesised container is extracted,
extractor.extractClassSource(containerClass, extractDeclarations = true, extractStaticInitializer = true, extractPrivateMembers = true, extractFunctionBodies = extractAnnotationTypeAccesses) // extract the synthetic container to the same trap file.
extractor.extractClassSource(
val containerReference = IrClassReferenceImpl( containerClass,
UNDEFINED_OFFSET, UNDEFINED_OFFSET, pluginContext.irBuiltIns.kClassClass.typeWith(containerClass.defaultType), extractDeclarations = true,
containerClass.symbol, containerClass.defaultType extractStaticInitializer = true,
extractPrivateMembers = true,
extractFunctionBodies = extractAnnotationTypeAccesses
) )
val containerReference =
IrClassReferenceImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
pluginContext.irBuiltIns.kClassClass.typeWith(containerClass.defaultType),
containerClass.symbol,
containerClass.defaultType
)
return IrConstructorCallImpl.fromSymbolOwner( return IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, repeatableConstructor.returnType, repeatableConstructor.symbol, 0 UNDEFINED_OFFSET,
).apply { UNDEFINED_OFFSET,
putValueArgument(0, containerReference) repeatableConstructor.returnType,
} repeatableConstructor.symbol,
0
)
.apply { putValueArgument(0, containerReference) }
} }
private val javaAnnotationDocumented by lazy { extractor.referenceExternalClass("java.lang.annotation.Documented") } private val javaAnnotationDocumented by lazy {
extractor.referenceExternalClass("java.lang.annotation.Documented")
}
// Taken from AdditionalClassAnnotationLowering.kt // Taken from AdditionalClassAnnotationLowering.kt
private fun generateDocumentedAnnotation(irClass: IrClass): IrConstructorCall? { private fun generateDocumentedAnnotation(irClass: IrClass): IrConstructorCall? {
if (!irClass.hasAnnotation(StandardNames.FqNames.mustBeDocumented) || if (
irClass.hasAnnotation(JvmAnnotationNames.DOCUMENTED_ANNOTATION) !irClass.hasAnnotation(StandardNames.FqNames.mustBeDocumented) ||
) return null irClass.hasAnnotation(JvmAnnotationNames.DOCUMENTED_ANNOTATION)
)
return null
val documentedConstructor = javaAnnotationDocumented?.declarations?.firstIsInstanceOrNull<IrConstructor>() ?: return null val documentedConstructor =
javaAnnotationDocumented?.declarations?.firstIsInstanceOrNull<IrConstructor>()
?: return null
return IrConstructorCallImpl.fromSymbolOwner( return IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, documentedConstructor.returnType, documentedConstructor.symbol, 0 UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
documentedConstructor.returnType,
documentedConstructor.symbol,
0
) )
} }
fun generateJavaMetaAnnotations(c: IrClass, extractAnnotationTypeAccesses: Boolean) = fun generateJavaMetaAnnotations(c: IrClass, extractAnnotationTypeAccesses: Boolean) =
// This is essentially AdditionalClassAnnotationLowering adapted to run outside the backend. // This is essentially AdditionalClassAnnotationLowering adapted to run outside the backend.
listOfNotNull(generateTargetAnnotation(c), generateRetentionAnnotation(c), generateRepeatableAnnotation(c, extractAnnotationTypeAccesses), generateDocumentedAnnotation(c)) listOfNotNull(
generateTargetAnnotation(c),
generateRetentionAnnotation(c),
generateRepeatableAnnotation(c, extractAnnotationTypeAccesses),
generateDocumentedAnnotation(c)
)
} }

View File

@@ -1,93 +1,105 @@
package com.github.codeql package com.github.codeql
import com.github.codeql.utils.*
import com.github.codeql.utils.versions.*
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.builtins.StandardNames import org.jetbrains.kotlin.builtins.StandardNames
import org.jetbrains.kotlin.ir.declarations.IrClass import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.ir.declarations.IrPackageFragment import org.jetbrains.kotlin.ir.declarations.IrPackageFragment
import org.jetbrains.kotlin.ir.types.IrSimpleType import org.jetbrains.kotlin.ir.types.IrSimpleType
import org.jetbrains.kotlin.ir.types.classOrNull import org.jetbrains.kotlin.ir.types.classOrNull
import org.jetbrains.kotlin.name.FqName
import com.github.codeql.utils.*
import com.github.codeql.utils.versions.*
class PrimitiveTypeMapping(val logger: Logger, val pluginContext: IrPluginContext) { class PrimitiveTypeMapping(val logger: Logger, val pluginContext: IrPluginContext) {
fun getPrimitiveInfo(s: IrSimpleType) = fun getPrimitiveInfo(s: IrSimpleType) =
s.classOrNull?.let { s.classOrNull?.let {
if ((it.owner.parent as? IrPackageFragment)?.packageFqName == StandardNames.BUILT_INS_PACKAGE_FQ_NAME) if (
(it.owner.parent as? IrPackageFragment)?.packageFqName ==
StandardNames.BUILT_INS_PACKAGE_FQ_NAME
)
mapping[it.owner.name] mapping[it.owner.name]
else else null
null
} }
data class PrimitiveTypeInfo( data class PrimitiveTypeInfo(
val primitiveName: String?, val primitiveName: String?,
val otherIsPrimitive: Boolean, val otherIsPrimitive: Boolean,
val javaClass: IrClass, val javaClass: IrClass,
val kotlinPackageName: String, val kotlinClassName: String val kotlinPackageName: String,
val kotlinClassName: String
) )
private fun findClass(fqName: String, fallback: IrClass): IrClass { private fun findClass(fqName: String, fallback: IrClass): IrClass {
val symbol = getClassByFqName(pluginContext, fqName) val symbol = getClassByFqName(pluginContext, fqName)
if(symbol == null) { if (symbol == null) {
logger.warn("Can't find $fqName") logger.warn("Can't find $fqName")
// Do the best we can // Do the best we can
return fallback return fallback
} else { } else {
return symbol.owner return symbol.owner
} }
} }
private val mapping = { private val mapping =
val kotlinByte = pluginContext.irBuiltIns.byteClass.owner {
val javaLangByte = findClass("java.lang.Byte", kotlinByte) val kotlinByte = pluginContext.irBuiltIns.byteClass.owner
val kotlinShort = pluginContext.irBuiltIns.shortClass.owner val javaLangByte = findClass("java.lang.Byte", kotlinByte)
val javaLangShort = findClass("java.lang.Short", kotlinShort) val kotlinShort = pluginContext.irBuiltIns.shortClass.owner
val kotlinInt = pluginContext.irBuiltIns.intClass.owner val javaLangShort = findClass("java.lang.Short", kotlinShort)
val javaLangInteger = findClass("java.lang.Integer", kotlinInt) val kotlinInt = pluginContext.irBuiltIns.intClass.owner
val kotlinLong = pluginContext.irBuiltIns.longClass.owner val javaLangInteger = findClass("java.lang.Integer", kotlinInt)
val javaLangLong = findClass("java.lang.Long", kotlinLong) val kotlinLong = pluginContext.irBuiltIns.longClass.owner
val javaLangLong = findClass("java.lang.Long", kotlinLong)
val kotlinUByte = findClass("kotlin.UByte", kotlinByte) val kotlinUByte = findClass("kotlin.UByte", kotlinByte)
val kotlinUShort = findClass("kotlin.UShort", kotlinShort) val kotlinUShort = findClass("kotlin.UShort", kotlinShort)
val kotlinUInt = findClass("kotlin.UInt", kotlinInt) val kotlinUInt = findClass("kotlin.UInt", kotlinInt)
val kotlinULong = findClass("kotlin.ULong", kotlinLong) val kotlinULong = findClass("kotlin.ULong", kotlinLong)
val kotlinDouble = pluginContext.irBuiltIns.doubleClass.owner val kotlinDouble = pluginContext.irBuiltIns.doubleClass.owner
val javaLangDouble = findClass("java.lang.Double", kotlinDouble) val javaLangDouble = findClass("java.lang.Double", kotlinDouble)
val kotlinFloat = pluginContext.irBuiltIns.floatClass.owner val kotlinFloat = pluginContext.irBuiltIns.floatClass.owner
val javaLangFloat = findClass("java.lang.Float", kotlinFloat) val javaLangFloat = findClass("java.lang.Float", kotlinFloat)
val kotlinBoolean = pluginContext.irBuiltIns.booleanClass.owner val kotlinBoolean = pluginContext.irBuiltIns.booleanClass.owner
val javaLangBoolean = findClass("java.lang.Boolean", kotlinBoolean) val javaLangBoolean = findClass("java.lang.Boolean", kotlinBoolean)
val kotlinChar = pluginContext.irBuiltIns.charClass.owner val kotlinChar = pluginContext.irBuiltIns.charClass.owner
val javaLangCharacter = findClass("java.lang.Character", kotlinChar) val javaLangCharacter = findClass("java.lang.Character", kotlinChar)
val kotlinUnit = pluginContext.irBuiltIns.unitClass.owner val kotlinUnit = pluginContext.irBuiltIns.unitClass.owner
val kotlinNothing = pluginContext.irBuiltIns.nothingClass.owner val kotlinNothing = pluginContext.irBuiltIns.nothingClass.owner
val javaLangVoid = findClass("java.lang.Void", kotlinNothing) val javaLangVoid = findClass("java.lang.Void", kotlinNothing)
mapOf( mapOf(
StandardNames.FqNames._byte.shortName() to PrimitiveTypeInfo("byte", true, javaLangByte, "kotlin", "Byte"), StandardNames.FqNames._byte.shortName() to
StandardNames.FqNames._short.shortName() to PrimitiveTypeInfo("short", true, javaLangShort, "kotlin", "Short"), PrimitiveTypeInfo("byte", true, javaLangByte, "kotlin", "Byte"),
StandardNames.FqNames._int.shortName() to PrimitiveTypeInfo("int", true, javaLangInteger, "kotlin", "Int"), StandardNames.FqNames._short.shortName() to
StandardNames.FqNames._long.shortName() to PrimitiveTypeInfo("long", true, javaLangLong, "kotlin", "Long"), PrimitiveTypeInfo("short", true, javaLangShort, "kotlin", "Short"),
StandardNames.FqNames._int.shortName() to
StandardNames.FqNames.uByteFqName.shortName() to PrimitiveTypeInfo("byte", true, kotlinUByte, "kotlin", "UByte"), PrimitiveTypeInfo("int", true, javaLangInteger, "kotlin", "Int"),
StandardNames.FqNames.uShortFqName.shortName() to PrimitiveTypeInfo("short", true, kotlinUShort, "kotlin", "UShort"), StandardNames.FqNames._long.shortName() to
StandardNames.FqNames.uIntFqName.shortName() to PrimitiveTypeInfo("int", true, kotlinUInt, "kotlin", "UInt"), PrimitiveTypeInfo("long", true, javaLangLong, "kotlin", "Long"),
StandardNames.FqNames.uLongFqName.shortName() to PrimitiveTypeInfo("long", true, kotlinULong, "kotlin", "ULong"), StandardNames.FqNames.uByteFqName.shortName() to
PrimitiveTypeInfo("byte", true, kotlinUByte, "kotlin", "UByte"),
StandardNames.FqNames._double.shortName() to PrimitiveTypeInfo("double", true, javaLangDouble, "kotlin", "Double"), StandardNames.FqNames.uShortFqName.shortName() to
StandardNames.FqNames._float.shortName() to PrimitiveTypeInfo("float", true, javaLangFloat, "kotlin", "Float"), PrimitiveTypeInfo("short", true, kotlinUShort, "kotlin", "UShort"),
StandardNames.FqNames.uIntFqName.shortName() to
StandardNames.FqNames._boolean.shortName() to PrimitiveTypeInfo("boolean", true, javaLangBoolean, "kotlin", "Boolean"), PrimitiveTypeInfo("int", true, kotlinUInt, "kotlin", "UInt"),
StandardNames.FqNames.uLongFqName.shortName() to
StandardNames.FqNames._char.shortName() to PrimitiveTypeInfo("char", true, javaLangCharacter, "kotlin", "Char"), PrimitiveTypeInfo("long", true, kotlinULong, "kotlin", "ULong"),
StandardNames.FqNames._double.shortName() to
StandardNames.FqNames.unit.shortName() to PrimitiveTypeInfo("void", false, kotlinUnit, "kotlin", "Unit"), PrimitiveTypeInfo("double", true, javaLangDouble, "kotlin", "Double"),
StandardNames.FqNames.nothing.shortName() to PrimitiveTypeInfo(null, true, javaLangVoid, "kotlin", "Nothing"), StandardNames.FqNames._float.shortName() to
) PrimitiveTypeInfo("float", true, javaLangFloat, "kotlin", "Float"),
}() StandardNames.FqNames._boolean.shortName() to
PrimitiveTypeInfo("boolean", true, javaLangBoolean, "kotlin", "Boolean"),
StandardNames.FqNames._char.shortName() to
PrimitiveTypeInfo("char", true, javaLangCharacter, "kotlin", "Char"),
StandardNames.FqNames.unit.shortName() to
PrimitiveTypeInfo("void", false, kotlinUnit, "kotlin", "Unit"),
StandardNames.FqNames.nothing.shortName() to
PrimitiveTypeInfo(null, true, javaLangVoid, "kotlin", "Nothing"),
)
}()
} }

View File

@@ -1,92 +1,87 @@
package com.github.codeql package com.github.codeql
import com.github.codeql.KotlinUsesExtractor.LocallyVisibleFunctionLabels import com.github.codeql.KotlinUsesExtractor.LocallyVisibleFunctionLabels
import com.semmle.extractor.java.PopulateFile
import com.semmle.util.unicode.UTF8Util
import java.io.BufferedWriter import java.io.BufferedWriter
import java.io.File import java.io.File
import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.path import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
import org.jetbrains.kotlin.ir.declarations.IrClass import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.ir.declarations.IrFile import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.ir.declarations.IrFunction import org.jetbrains.kotlin.ir.declarations.IrFunction
import org.jetbrains.kotlin.ir.declarations.IrVariable import org.jetbrains.kotlin.ir.declarations.IrVariable
import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET import org.jetbrains.kotlin.ir.declarations.path
import org.jetbrains.kotlin.ir.expressions.IrCall
import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET
import com.semmle.extractor.java.PopulateFile
import com.semmle.util.unicode.UTF8Util
import org.jetbrains.kotlin.ir.expressions.IrCall
/** /**
* Each `.trap` file has a `TrapLabelManager` while we are writing it. * Each `.trap` file has a `TrapLabelManager` while we are writing it. It provides fresh TRAP label
* It provides fresh TRAP label names, and maintains a mapping from keys * names, and maintains a mapping from keys (`@"..."`) to labels.
* (`@"..."`) to labels.
*/ */
class TrapLabelManager { class TrapLabelManager {
/** The next integer to use as a label name. */ /** The next integer to use as a label name. */
private var nextInt: Int = 100 private var nextInt: Int = 100
/** Returns a fresh label. */ /** Returns a fresh label. */
fun <T: AnyDbType> getFreshLabel(): Label<T> { fun <T : AnyDbType> getFreshLabel(): Label<T> {
return IntLabel(nextInt++) return IntLabel(nextInt++)
} }
/** /** A mapping from a key (`@"..."`) to the label defined to be that key, if any. */
* A mapping from a key (`@"..."`) to the label defined to be that
* key, if any.
*/
val labelMapping: MutableMap<String, Label<*>> = mutableMapOf<String, Label<*>>() val labelMapping: MutableMap<String, Label<*>> = mutableMapOf<String, Label<*>>()
val anonymousTypeMapping: MutableMap<IrClass, TypeResults> = mutableMapOf() val anonymousTypeMapping: MutableMap<IrClass, TypeResults> = mutableMapOf()
val locallyVisibleFunctionLabelMapping: MutableMap<IrFunction, LocallyVisibleFunctionLabels> = mutableMapOf() val locallyVisibleFunctionLabelMapping: MutableMap<IrFunction, LocallyVisibleFunctionLabels> =
mutableMapOf()
/** /**
* The set of labels of generic specialisations that we have extracted * The set of labels of generic specialisations that we have extracted in this TRAP file. We
* in this TRAP file. * can't easily avoid duplication between TRAP files, as the labels contain references to other
* We can't easily avoid duplication between TRAP files, as the labels * labels, so we just accept this duplication.
* contain references to other labels, so we just accept this
* duplication.
*/ */
val genericSpecialisationsExtracted = HashSet<String>() val genericSpecialisationsExtracted = HashSet<String>()
/** /**
* Sometimes, when we extract a file class we don't have the IrFile * Sometimes, when we extract a file class we don't have the IrFile for it, so we are not able
* for it, so we are not able to give it a location. This means that * to give it a location. This means that the location is written outside of the label creation.
* the location is written outside of the label creation. * This allows us to keep track of whether we've written the location already in this TRAP file,
* This allows us to keep track of whether we've written the location * to avoid duplication.
* already in this TRAP file, to avoid duplication.
*/ */
val fileClassLocationsExtracted = HashSet<IrFile>() val fileClassLocationsExtracted = HashSet<IrFile>()
} }
/** /**
* A `TrapWriter` is used to write TRAP to a particular TRAP file. * A `TrapWriter` is used to write TRAP to a particular TRAP file. There may be multiple
* There may be multiple `TrapWriter`s for the same file, as different * `TrapWriter`s for the same file, as different instances will have different additional state, but
* instances will have different additional state, but they must all * they must all share the same `TrapLabelManager` and `BufferedWriter`.
* share the same `TrapLabelManager` and `BufferedWriter`.
*/ */
// TODO lm was `protected` before anonymousTypeMapping and locallyVisibleFunctionLabelMapping moved into it. Should we re-protect it and provide accessors? // TODO lm was `protected` before anonymousTypeMapping and locallyVisibleFunctionLabelMapping moved
abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLabelManager, private val bw: BufferedWriter) { // into it. Should we re-protect it and provide accessors?
abstract class TrapWriter(
protected val loggerBase: LoggerBase,
val lm: TrapLabelManager,
private val bw: BufferedWriter
) {
abstract fun getDiagnosticTrapWriter(): DiagnosticTrapWriter abstract fun getDiagnosticTrapWriter(): DiagnosticTrapWriter
/** /**
* Returns the label that is defined to be the given key, if such * Returns the label that is defined to be the given key, if such a label exists, and `null`
* a label exists, and `null` otherwise. Most users will want to use * otherwise. Most users will want to use `getLabelFor` instead, which allows non-existent
* `getLabelFor` instead, which allows non-existent labels to be * labels to be initialised.
* initialised.
*/ */
fun <T: AnyDbType> getExistingLabelFor(key: String): Label<T>? { fun <T : AnyDbType> getExistingLabelFor(key: String): Label<T>? {
return lm.labelMapping.get(key)?.cast<T>() return lm.labelMapping.get(key)?.cast<T>()
} }
/** /**
* Returns the label for the given key, if one exists. * Returns the label for the given key, if one exists. Otherwise, a fresh label is bound to that
* Otherwise, a fresh label is bound to that key, `initialise` * key, `initialise` is run on it, and it is returned.
* is run on it, and it is returned.
*/ */
@JvmOverloads // Needed so Java can call a method with an optional argument @JvmOverloads // Needed so Java can call a method with an optional argument
fun <T: AnyDbType> getLabelFor(key: String, initialise: (Label<T>) -> Unit = {}): Label<T> { fun <T : AnyDbType> getLabelFor(key: String, initialise: (Label<T>) -> Unit = {}): Label<T> {
val maybeLabel: Label<T>? = getExistingLabelFor(key) val maybeLabel: Label<T>? = getExistingLabelFor(key)
if(maybeLabel == null) { if (maybeLabel == null) {
val label: Label<T> = lm.getFreshLabel() val label: Label<T> = lm.getFreshLabel()
lm.labelMapping.put(key, label) lm.labelMapping.put(key, label)
writeTrap("$label = $key\n") writeTrap("$label = $key\n")
@@ -97,30 +92,24 @@ abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLab
} }
} }
/** /** Returns a label for a fresh ID (i.e. a new label bound to `*`). */
* Returns a label for a fresh ID (i.e. a new label bound to `*`). fun <T : AnyDbType> getFreshIdLabel(): Label<T> {
*/
fun <T: AnyDbType> getFreshIdLabel(): Label<T> {
val label: Label<T> = lm.getFreshLabel() val label: Label<T> = lm.getFreshLabel()
writeTrap("$label = *\n") writeTrap("$label = *\n")
return label return label
} }
/** /**
* It is not easy to assign keys to local variables, so they get * It is not easy to assign keys to local variables, so they get given `*` IDs. However, the
* given `*` IDs. However, the same variable may be referred to * same variable may be referred to from distant places in the IR, so we need a way to find out
* from distant places in the IR, so we need a way to find out * which label is used for a given local variable. This information is stored in this mapping.
* which label is used for a given local variable. This information
* is stored in this mapping.
*/
private val variableLabelMapping: MutableMap<IrVariable, Label<out DbLocalvar>> = mutableMapOf<IrVariable, Label<out DbLocalvar>>()
/**
* This returns the label used for a local variable, creating one
* if none currently exists.
*/ */
private val variableLabelMapping: MutableMap<IrVariable, Label<out DbLocalvar>> =
mutableMapOf<IrVariable, Label<out DbLocalvar>>()
/** This returns the label used for a local variable, creating one if none currently exists. */
fun <T> getVariableLabelFor(v: IrVariable): Label<out DbLocalvar> { fun <T> getVariableLabelFor(v: IrVariable): Label<out DbLocalvar> {
val maybeLabel = variableLabelMapping.get(v) val maybeLabel = variableLabelMapping.get(v)
if(maybeLabel == null) { if (maybeLabel == null) {
val label = getFreshIdLabel<DbLocalvar>() val label = getFreshIdLabel<DbLocalvar>()
variableLabelMapping.put(v, label) variableLabelMapping.put(v, label)
return label return label
@@ -134,43 +123,41 @@ abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLab
} }
/** /**
* This returns a label for the location described by its arguments. * This returns a label for the location described by its arguments. Typically users will not
* Typically users will not want to call this directly, but instead * want to call this directly, but instead use `unknownLocation`, or overloads of this defined
* use `unknownLocation`, or overloads of this defined by subclasses. * by subclasses.
*/ */
fun getLocation(fileId: Label<DbFile>, startLine: Int, startColumn: Int, endLine: Int, endColumn: Int): Label<DbLocation> { fun getLocation(
fileId: Label<DbFile>,
startLine: Int,
startColumn: Int,
endLine: Int,
endColumn: Int
): Label<DbLocation> {
return getLabelFor("@\"loc,{$fileId},$startLine,$startColumn,$endLine,$endColumn\"") { return getLabelFor("@\"loc,{$fileId},$startLine,$startColumn,$endLine,$endColumn\"") {
writeLocations_default(it, fileId, startLine, startColumn, endLine, endColumn) writeLocations_default(it, fileId, startLine, startColumn, endLine, endColumn)
} }
} }
/** /**
* The label for the 'unknown' file ID. * The label for the 'unknown' file ID. Users will want to use `unknownLocation` instead. This
* Users will want to use `unknownLocation` instead. * is lazy, as we don't want to define it in a TRAP file unless the TRAP file actually contains
* This is lazy, as we don't want to define it in a TRAP file unless * something in the 'unknown' file.
* the TRAP file actually contains something in the 'unknown' file.
*/ */
protected val unknownFileId: Label<DbFile> by lazy { protected val unknownFileId: Label<DbFile> by lazy {
val unknownFileLabel = "@\";sourcefile\"" val unknownFileLabel = "@\";sourcefile\""
getLabelFor(unknownFileLabel, { getLabelFor(unknownFileLabel, { writeFiles(it, "") })
writeFiles(it, "")
})
} }
/** /**
* The label for the 'unknown' location. * The label for the 'unknown' location. This is lazy, as we don't want to define it in a TRAP
* This is lazy, as we don't want to define it in a TRAP file unless * file unless the TRAP file actually contains something with an 'unknown' location.
* the TRAP file actually contains something with an 'unknown'
* location.
*/ */
val unknownLocation: Label<DbLocation> by lazy { val unknownLocation: Label<DbLocation> by lazy { getWholeFileLocation(unknownFileId) }
getWholeFileLocation(unknownFileId)
}
/** /**
* Returns the label for the file `filePath`. * Returns the label for the file `filePath`. If `populateFileTables` is true, then this also
* If `populateFileTables` is true, then this also adds rows to the * adds rows to the `files` and `folders` tables for this file.
* `files` and `folders` tables for this file.
*/ */
fun mkFileId(filePath: String, populateFileTables: Boolean): Label<DbFile> { fun mkFileId(filePath: String, populateFileTables: Boolean): Label<DbFile> {
// If a file is in a jar, then the Kotlin compiler gives // If a file is in a jar, then the Kotlin compiler gives
@@ -178,65 +165,65 @@ abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLab
// it as appropriate, to make the right file ID. // it as appropriate, to make the right file ID.
val populateFile = PopulateFile(this) val populateFile = PopulateFile(this)
val splitFilePath = filePath.split("!/") val splitFilePath = filePath.split("!/")
if(splitFilePath.size == 1) { if (splitFilePath.size == 1) {
return populateFile.getFileLabel(File(filePath), populateFileTables) return populateFile.getFileLabel(File(filePath), populateFileTables)
} else { } else {
return populateFile.getFileInJarLabel(File(splitFilePath.get(0)), splitFilePath.get(1), populateFileTables) return populateFile.getFileInJarLabel(
File(splitFilePath.get(0)),
splitFilePath.get(1),
populateFileTables
)
} }
} }
/** /**
* If you have an ID for a file, then this gets a label for the * If you have an ID for a file, then this gets a label for the location representing the whole
* location representing the whole of that file. * of that file.
*/ */
fun getWholeFileLocation(fileId: Label<DbFile>): Label<DbLocation> { fun getWholeFileLocation(fileId: Label<DbFile>): Label<DbLocation> {
return getLocation(fileId, 0, 0, 0, 0) return getLocation(fileId, 0, 0, 0, 0)
} }
/** /**
* Write a raw string into the TRAP file. Users should call one of * Write a raw string into the TRAP file. Users should call one of the wrapper functions
* the wrapper functions instead. * instead.
*/ */
fun writeTrap(trap: String) { fun writeTrap(trap: String) {
bw.write(trap) bw.write(trap)
} }
/** /** Write a comment into the TRAP file. */
* Write a comment into the TRAP file.
*/
fun writeComment(comment: String) { fun writeComment(comment: String) {
writeTrap("// ${comment.replace("\n", "\n// ")}\n") writeTrap("// ${comment.replace("\n", "\n// ")}\n")
} }
/** /** Flush the TRAP file. */
* Flush the TRAP file.
*/
fun flush() { fun flush() {
bw.flush() bw.flush()
} }
/** /**
* Escape a string so that it can be used in a TRAP string literal, * Escape a string so that it can be used in a TRAP string literal, i.e. with `"` escaped as
* i.e. with `"` escaped as `""`. * `""`.
*/ */
fun escapeTrapString(str: String) = str.replace("\"", "\"\"") fun escapeTrapString(str: String) = str.replace("\"", "\"\"")
/** /** TRAP string literals are limited to 1 megabyte. */
* TRAP string literals are limited to 1 megabyte.
*/
private val MAX_STRLEN = 1.shl(20) private val MAX_STRLEN = 1.shl(20)
/** /**
* Truncate a string, if necessary, so that it can be used as a TRAP * Truncate a string, if necessary, so that it can be used as a TRAP string literal. TRAP string
* string literal. TRAP string literals are limited to 1 megabyte. * literals are limited to 1 megabyte.
*/ */
fun truncateString(str: String): String { fun truncateString(str: String): String {
val len = str.length val len = str.length
val newLen = UTF8Util.encodablePrefixLength(str, MAX_STRLEN) val newLen = UTF8Util.encodablePrefixLength(str, MAX_STRLEN)
if (newLen < len) { if (newLen < len) {
loggerBase.warn(this.getDiagnosticTrapWriter(), loggerBase.warn(
this.getDiagnosticTrapWriter(),
"Truncated string of length $len", "Truncated string of length $len",
"Truncated string of length $len, starting '${str.take(100)}', ending '${str.takeLast(100)}'") "Truncated string of length $len, starting '${str.take(100)}', ending '${str.takeLast(100)}'"
)
return str.take(newLen) return str.take(newLen)
} else { } else {
return str return str
@@ -244,69 +231,75 @@ abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLab
} }
/** /**
* Gets a FileTrapWriter like this one (using the same label manager, * Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the
* writer etc), but using the given `filePath` for locations. * given `filePath` for locations.
*/ */
fun makeFileTrapWriter(filePath: String, populateFileTables: Boolean) = fun makeFileTrapWriter(filePath: String, populateFileTables: Boolean) =
FileTrapWriter(loggerBase, lm, bw, this.getDiagnosticTrapWriter(), filePath, populateFileTables) FileTrapWriter(
loggerBase,
lm,
bw,
this.getDiagnosticTrapWriter(),
filePath,
populateFileTables
)
/** /**
* Gets a FileTrapWriter like this one (using the same label manager, * Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the
* writer etc), but using the given `IrFile` for locations. * given `IrFile` for locations.
*/ */
fun makeSourceFileTrapWriter(file: IrFile, populateFileTables: Boolean) = fun makeSourceFileTrapWriter(file: IrFile, populateFileTables: Boolean) =
SourceFileTrapWriter(loggerBase, lm, bw, this.getDiagnosticTrapWriter(), file, populateFileTables) SourceFileTrapWriter(
loggerBase,
lm,
bw,
this.getDiagnosticTrapWriter(),
file,
populateFileTables
)
} }
/** /** A `PlainTrapWriter` has no additional context of its own. */
* A `PlainTrapWriter` has no additional context of its own. class PlainTrapWriter(
*/
class PlainTrapWriter (
loggerBase: LoggerBase, loggerBase: LoggerBase,
lm: TrapLabelManager, lm: TrapLabelManager,
bw: BufferedWriter, bw: BufferedWriter,
val dtw: DiagnosticTrapWriter val dtw: DiagnosticTrapWriter
): TrapWriter (loggerBase, lm, bw) { ) : TrapWriter(loggerBase, lm, bw) {
override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter { override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter {
return dtw return dtw
} }
} }
/** /**
* A `DiagnosticTrapWriter` is a TrapWriter that diagnostics can be * A `DiagnosticTrapWriter` is a TrapWriter that diagnostics can be written to; i.e. it has
* written to; i.e. it has the #compilation label defined. In practice, * the #compilation label defined. In practice, this means that it is a TrapWriter for the
* this means that it is a TrapWriter for the invocation TRAP file. * invocation TRAP file.
*/ */
class DiagnosticTrapWriter ( class DiagnosticTrapWriter(loggerBase: LoggerBase, lm: TrapLabelManager, bw: BufferedWriter) :
loggerBase: LoggerBase, TrapWriter(loggerBase, lm, bw) {
lm: TrapLabelManager,
bw: BufferedWriter
): TrapWriter (loggerBase, lm, bw) {
override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter { override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter {
return this return this
} }
} }
/** /**
* A `FileTrapWriter` is used when we know which file we are extracting * A `FileTrapWriter` is used when we know which file we are extracting entities from, so we can at
* entities from, so we can at least give the right file as a location. * least give the right file as a location.
* *
* An ID for the file will be created, and if `populateFileTables` is * An ID for the file will be created, and if `populateFileTables` is true then we will also add
* true then we will also add rows to the `files` and `folders` tables * rows to the `files` and `folders` tables for it.
* for it.
*/ */
open class FileTrapWriter ( open class FileTrapWriter(
loggerBase: LoggerBase, loggerBase: LoggerBase,
lm: TrapLabelManager, lm: TrapLabelManager,
bw: BufferedWriter, bw: BufferedWriter,
val dtw: DiagnosticTrapWriter, val dtw: DiagnosticTrapWriter,
val filePath: String, val filePath: String,
populateFileTables: Boolean populateFileTables: Boolean
): TrapWriter (loggerBase, lm, bw) { ) : TrapWriter(loggerBase, lm, bw) {
/** /** The ID for the file that we are extracting from. */
* The ID for the file that we are extracting from.
*/
val fileId = mkFileId(filePath, populateFileTables) val fileId = mkFileId(filePath, populateFileTables)
override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter { override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter {
@@ -320,7 +313,12 @@ open class FileTrapWriter (
var currentMin = default var currentMin = default
for (option in options) { for (option in options) {
if (option != null && option != UNDEFINED_OFFSET && option != SYNTHETIC_OFFSET && option < currentMin) { if (
option != null &&
option != UNDEFINED_OFFSET &&
option != SYNTHETIC_OFFSET &&
option < currentMin
) {
currentMin = option currentMin = option
} }
} }
@@ -344,15 +342,13 @@ open class FileTrapWriter (
return e.endOffset return e.endOffset
} }
/** /** Gets a label for the location of `e`. */
* Gets a label for the location of `e`.
*/
fun getLocation(e: IrElement): Label<DbLocation> { fun getLocation(e: IrElement): Label<DbLocation> {
return getLocation(getStartOffset(e), getEndOffset(e)) return getLocation(getStartOffset(e), getEndOffset(e))
} }
/** /**
* Gets a label for the location corresponding to `startOffset` and * Gets a label for the location corresponding to `startOffset` and `endOffset` within this
* `endOffset` within this file. * file.
*/ */
open fun getLocation(startOffset: Int, endOffset: Int): Label<DbLocation> { open fun getLocation(startOffset: Int, endOffset: Int): Label<DbLocation> {
// We don't have a FileEntry to look up the offsets in, so all // We don't have a FileEntry to look up the offsets in, so all
@@ -360,8 +356,8 @@ open class FileTrapWriter (
return getWholeFileLocation() return getWholeFileLocation()
} }
/** /**
* Gets the location of `e` as a human-readable string. Only used in * Gets the location of `e` as a human-readable string. Only used in log messages and exception
* log messages and exception messages. * messages.
*/ */
open fun getLocationString(e: IrElement): String { open fun getLocationString(e: IrElement): String {
// We don't have a FileEntry to look up the offsets in, so all // We don't have a FileEntry to look up the offsets in, so all
@@ -371,50 +367,54 @@ open class FileTrapWriter (
// to be 0. // to be 0.
return "file://$filePath" return "file://$filePath"
} }
/** /** Gets a label for the location representing the whole of this file. */
* Gets a label for the location representing the whole of this file.
*/
fun getWholeFileLocation(): Label<DbLocation> { fun getWholeFileLocation(): Label<DbLocation> {
return getWholeFileLocation(fileId) return getWholeFileLocation(fileId)
} }
} }
/** /**
* A `SourceFileTrapWriter` is used when not only do we know which file * A `SourceFileTrapWriter` is used when not only do we know which file we are extracting entities
* we are extracting entities from, but we also have an `IrFileEntry` * from, but we also have an `IrFileEntry` (from an `IrFile`) which allows us to map byte offsets to
* (from an `IrFile`) which allows us to map byte offsets to line * line and column numbers.
* and column numbers.
* *
* An ID for the file will be created, and if `populateFileTables` is * An ID for the file will be created, and if `populateFileTables` is true then we will also add
* true then we will also add rows to the `files` and `folders` tables * rows to the `files` and `folders` tables for it.
* for it.
*/ */
class SourceFileTrapWriter ( class SourceFileTrapWriter(
loggerBase: LoggerBase, loggerBase: LoggerBase,
lm: TrapLabelManager, lm: TrapLabelManager,
bw: BufferedWriter, bw: BufferedWriter,
dtw: DiagnosticTrapWriter, dtw: DiagnosticTrapWriter,
val irFile: IrFile, val irFile: IrFile,
populateFileTables: Boolean) : populateFileTables: Boolean
FileTrapWriter(loggerBase, lm, bw, dtw, irFile.path, populateFileTables) { ) : FileTrapWriter(loggerBase, lm, bw, dtw, irFile.path, populateFileTables) {
/** /**
* The file entry for the file that we are extracting from. * The file entry for the file that we are extracting from. Used to map offsets to line/column
* Used to map offsets to line/column numbers. * numbers.
*/ */
private val fileEntry = irFile.fileEntry private val fileEntry = irFile.fileEntry
override fun getLocation(startOffset: Int, endOffset: Int): Label<DbLocation> { override fun getLocation(startOffset: Int, endOffset: Int): Label<DbLocation> {
if (startOffset == UNDEFINED_OFFSET || endOffset == UNDEFINED_OFFSET) { if (startOffset == UNDEFINED_OFFSET || endOffset == UNDEFINED_OFFSET) {
if (startOffset != endOffset) { if (startOffset != endOffset) {
loggerBase.warn(dtw, "Location with inconsistent offsets (start $startOffset, end $endOffset)", null) loggerBase.warn(
dtw,
"Location with inconsistent offsets (start $startOffset, end $endOffset)",
null
)
} }
return getWholeFileLocation() return getWholeFileLocation()
} }
if (startOffset == SYNTHETIC_OFFSET || endOffset == SYNTHETIC_OFFSET) { if (startOffset == SYNTHETIC_OFFSET || endOffset == SYNTHETIC_OFFSET) {
if (startOffset != endOffset) { if (startOffset != endOffset) {
loggerBase.warn(dtw, "Location with inconsistent offsets (start $startOffset, end $endOffset)", null) loggerBase.warn(
dtw,
"Location with inconsistent offsets (start $startOffset, end $endOffset)",
null
)
} }
return getWholeFileLocation() return getWholeFileLocation()
} }
@@ -428,28 +428,37 @@ class SourceFileTrapWriter (
fileEntry.getLineNumber(startOffset) + 1, fileEntry.getLineNumber(startOffset) + 1,
fileEntry.getColumnNumber(startOffset) + 1, fileEntry.getColumnNumber(startOffset) + 1,
fileEntry.getLineNumber(endOffset) + 1, fileEntry.getLineNumber(endOffset) + 1,
fileEntry.getColumnNumber(endOffset) + endColumnOffset) fileEntry.getColumnNumber(endOffset) + endColumnOffset
)
} }
override fun getLocationString(e: IrElement): String { override fun getLocationString(e: IrElement): String {
if (e.startOffset == UNDEFINED_OFFSET || e.endOffset == UNDEFINED_OFFSET) { if (e.startOffset == UNDEFINED_OFFSET || e.endOffset == UNDEFINED_OFFSET) {
if (e.startOffset != e.endOffset) { if (e.startOffset != e.endOffset) {
loggerBase.warn(dtw, "Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})", null) loggerBase.warn(
dtw,
"Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})",
null
)
} }
return "<unknown location while processing $filePath>" return "<unknown location while processing $filePath>"
} }
if (e.startOffset == SYNTHETIC_OFFSET || e.endOffset == SYNTHETIC_OFFSET) { if (e.startOffset == SYNTHETIC_OFFSET || e.endOffset == SYNTHETIC_OFFSET) {
if (e.startOffset != e.endOffset) { if (e.startOffset != e.endOffset) {
loggerBase.warn(dtw, "Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})", null) loggerBase.warn(
dtw,
"Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})",
null
)
} }
return "<synthetic location while processing $filePath>" return "<synthetic location while processing $filePath>"
} }
val startLine = fileEntry.getLineNumber(e.startOffset) + 1 val startLine = fileEntry.getLineNumber(e.startOffset) + 1
val startColumn = fileEntry.getColumnNumber(e.startOffset) + 1 val startColumn = fileEntry.getColumnNumber(e.startOffset) + 1
val endLine = fileEntry.getLineNumber(e.endOffset) + 1 val endLine = fileEntry.getLineNumber(e.endOffset) + 1
val endColumn = fileEntry.getColumnNumber(e.endOffset) val endColumn = fileEntry.getColumnNumber(e.endOffset)
return "file://$filePath:$startLine:$startColumn:$endLine:$endColumn" return "file://$filePath:$startLine:$startColumn:$endLine:$endColumn"
} }
} }

View File

@@ -8,13 +8,16 @@ import org.jetbrains.kotlin.ir.expressions.IrBody
import org.jetbrains.kotlin.ir.expressions.IrExpression import org.jetbrains.kotlin.ir.expressions.IrExpression
import org.jetbrains.kotlin.ir.util.parentClassOrNull import org.jetbrains.kotlin.ir.util.parentClassOrNull
open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, protected val file: IrFile, protected val fileLabel: Label<out DbFile>) { open class CommentExtractor(
protected val fileExtractor: KotlinFileExtractor,
protected val file: IrFile,
protected val fileLabel: Label<out DbFile>
) {
protected val tw = fileExtractor.tw protected val tw = fileExtractor.tw
protected val logger = fileExtractor.logger protected val logger = fileExtractor.logger
protected fun getLabel(element: IrElement): Label<out DbTop>? { protected fun getLabel(element: IrElement): Label<out DbTop>? {
if (element == file) if (element == file) return fileLabel
return fileLabel
if (element is IrValueParameter && element.index == -1) { if (element is IrValueParameter && element.index == -1) {
// Don't attribute comments to the implicit `this` parameter of a function. // Don't attribute comments to the implicit `this` parameter of a function.
@@ -22,18 +25,21 @@ open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, pr
} }
val label: String val label: String
val existingLabel = if (element is IrVariable) { val existingLabel =
// local variables are not named globally, so we need to get them from the variable label cache if (element is IrVariable) {
label = "variable ${element.name.asString()}" // local variables are not named globally, so we need to get them from the variable
tw.getExistingVariableLabelFor(element) // label cache
} else if (element is IrFunction && element.isLocalFunction()) { label = "variable ${element.name.asString()}"
// local functions are not named globally, so we need to get them from the local function label cache tw.getExistingVariableLabelFor(element)
label = "local function ${element.name.asString()}" } else if (element is IrFunction && element.isLocalFunction()) {
fileExtractor.getExistingLocallyVisibleFunctionLabel(element) // local functions are not named globally, so we need to get them from the local
} else { // function label cache
label = getLabelForNamedElement(element) ?: return null label = "local function ${element.name.asString()}"
tw.getExistingLabelFor<DbTop>(label) fileExtractor.getExistingLocallyVisibleFunctionLabel(element)
} } else {
label = getLabelForNamedElement(element) ?: return null
tw.getExistingLabelFor<DbTop>(label)
}
if (existingLabel == null) { if (existingLabel == null) {
logger.warn("Couldn't get existing label for $label") logger.warn("Couldn't get existing label for $label")
return null return null
@@ -41,7 +47,7 @@ open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, pr
return existingLabel return existingLabel
} }
private fun getLabelForNamedElement(element: IrElement) : String? { private fun getLabelForNamedElement(element: IrElement): String? {
when (element) { when (element) {
is IrClass -> return fileExtractor.getClassLabel(element, listOf()).classLabel is IrClass -> return fileExtractor.getClassLabel(element, listOf()).classLabel
is IrTypeParameter -> return fileExtractor.getTypeParameterLabel(element) is IrTypeParameter -> return fileExtractor.getTypeParameterLabel(element)
@@ -57,14 +63,14 @@ open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, pr
is IrField -> return fileExtractor.getFieldLabel(element) is IrField -> return fileExtractor.getFieldLabel(element)
is IrEnumEntry -> return fileExtractor.getEnumEntryLabel(element) is IrEnumEntry -> return fileExtractor.getEnumEntryLabel(element)
is IrTypeAlias -> return fileExtractor.getTypeAliasLabel(element) is IrTypeAlias -> return fileExtractor.getTypeAliasLabel(element)
is IrAnonymousInitializer -> { is IrAnonymousInitializer -> {
val parentClass = element.parentClassOrNull val parentClass = element.parentClassOrNull
if (parentClass == null) { if (parentClass == null) {
logger.warnElement("Parent of anonymous initializer is not a class", element) logger.warnElement("Parent of anonymous initializer is not a class", element)
return null return null
} }
// Assign the comment to the class. The content of the `init` blocks might be extracted in multiple constructors. // Assign the comment to the class. The content of the `init` blocks might be
// extracted in multiple constructors.
return getLabelForNamedElement(parentClass) return getLabelForNamedElement(parentClass)
} }
@@ -74,7 +80,10 @@ open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, pr
// todo add others: // todo add others:
else -> { else -> {
logger.warnElement("Unhandled element type found during comment extraction: ${element::class}", element) logger.warnElement(
"Unhandled element type found during comment extraction: ${element::class}",
element
)
return null return null
} }
} }

View File

@@ -15,12 +15,18 @@ import org.jetbrains.kotlin.psi.KtVisitor
import org.jetbrains.kotlin.psi.psiUtil.endOffset import org.jetbrains.kotlin.psi.psiUtil.endOffset
import org.jetbrains.kotlin.psi.psiUtil.startOffset import org.jetbrains.kotlin.psi.psiUtil.startOffset
class CommentExtractorPSI(fileExtractor: KotlinFileExtractor, file: IrFile, fileLabel: Label<out DbFile>): CommentExtractor(fileExtractor, file, fileLabel) { class CommentExtractorPSI(
fileExtractor: KotlinFileExtractor,
file: IrFile,
fileLabel: Label<out DbFile>
) : CommentExtractor(fileExtractor, file, fileLabel) {
// Returns true if it extracted the comments; false otherwise. // Returns true if it extracted the comments; false otherwise.
fun extract(): Boolean { fun extract(): Boolean {
val psi2Ir = getPsi2Ir() val psi2Ir = getPsi2Ir()
if (psi2Ir == null) { if (psi2Ir == null) {
logger.warn("Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})") logger.warn(
"Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})"
)
return false return false
} }
val ktFile = psi2Ir.getKtFile(file) val ktFile = psi2Ir.getKtFile(file)
@@ -37,28 +43,30 @@ class CommentExtractorPSI(fileExtractor: KotlinFileExtractor, file: IrFile, file
override fun visitElement(element: PsiElement) { override fun visitElement(element: PsiElement) {
element.acceptChildren(this) element.acceptChildren(this)
// Slightly hacky, but `visitComment` doesn't seem to visit comments with `tokenType` `KtTokens.DOC_COMMENT` // Slightly hacky, but `visitComment` doesn't seem to visit comments with
if (element is PsiComment){ // `tokenType` `KtTokens.DOC_COMMENT`
if (element is PsiComment) {
visitCommentElement(element) visitCommentElement(element)
} }
} }
private fun visitCommentElement(comment: PsiComment) { private fun visitCommentElement(comment: PsiComment) {
val type: CommentType = when (comment.tokenType) { val type: CommentType =
KtTokens.EOL_COMMENT -> { when (comment.tokenType) {
CommentType.SingleLine KtTokens.EOL_COMMENT -> {
CommentType.SingleLine
}
KtTokens.BLOCK_COMMENT -> {
CommentType.Block
}
KtTokens.DOC_COMMENT -> {
CommentType.Doc
}
else -> {
logger.warn("Unhandled comment token type: ${comment.tokenType}")
return
}
} }
KtTokens.BLOCK_COMMENT -> {
CommentType.Block
}
KtTokens.DOC_COMMENT -> {
CommentType.Doc
}
else -> {
logger.warn("Unhandled comment token type: ${comment.tokenType}")
return
}
}
val commentLabel = tw.getFreshIdLabel<DbKtcomment>() val commentLabel = tw.getFreshIdLabel<DbKtcomment>()
tw.writeKtComments(commentLabel, type.value, comment.text) tw.writeKtComments(commentLabel, type.value, comment.text)
@@ -101,10 +109,12 @@ class CommentExtractorPSI(fileExtractor: KotlinFileExtractor, file: IrFile, file
} }
} }
private fun getKDocOwner(comment: KDoc) : PsiElement? { private fun getKDocOwner(comment: KDoc): PsiElement? {
val owner = comment.owner val owner = comment.owner
if (owner == null) { if (owner == null) {
logger.warn("Couldn't get owner of KDoc. The comment is extracted without an owner.") logger.warn(
"Couldn't get owner of KDoc. The comment is extracted without an owner."
)
} }
return owner return owner
} }

View File

@@ -1,5 +1,7 @@
package com.github.codeql.comments package com.github.codeql.comments
enum class CommentType(val value: Int) { enum class CommentType(val value: Int) {
SingleLine(1), Block(2), Doc(3) SingleLine(1),
} Block(2),
Doc(3)
}

View File

@@ -1,43 +1,47 @@
package com.github.codeql package com.github.codeql
// Functions copied from stdlib/jdk7/src/kotlin/AutoCloseable.kt, which is not available within kotlinc, // Functions copied from stdlib/jdk7/src/kotlin/AutoCloseable.kt, which is not available within
// kotlinc,
// but allows the `.use` pattern to be applied to JDK7 AutoCloseables: // but allows the `.use` pattern to be applied to JDK7 AutoCloseables:
/** /**
* Executes the given [block] function on this resource and then closes it down correctly whether an exception * Executes the given [block] function on this resource and then closes it down correctly whether an
* is thrown or not. * exception is thrown or not.
* *
* In case if the resource is being closed due to an exception occurred in [block], and the closing also fails with an exception, * In case if the resource is being closed due to an exception occurred in [block], and the closing
* the latter is added to the [suppressed][java.lang.Throwable.addSuppressed] exceptions of the former. * also fails with an exception, the latter is added to the
* [suppressed][java.lang.Throwable.addSuppressed] exceptions of the former.
* *
* @param block a function to process this [AutoCloseable] resource. * @param block a function to process this [AutoCloseable] resource.
* @return the result of [block] function invoked on this resource. * @return the result of [block] function invoked on this resource.
*/ */
public inline fun <T : AutoCloseable?, R> T.useAC(block: (T) -> R): R { public inline fun <T : AutoCloseable?, R> T.useAC(block: (T) -> R): R {
var exception: Throwable? = null var exception: Throwable? = null
try { try {
return block(this) return block(this)
} catch (e: Throwable) { } catch (e: Throwable) {
exception = e exception = e
throw e throw e
} finally { } finally {
this.closeFinallyAC(exception) this.closeFinallyAC(exception)
} }
} }
/** /**
* Closes this [AutoCloseable], suppressing possible exception or error thrown by [AutoCloseable.close] function when * Closes this [AutoCloseable], suppressing possible exception or error thrown by
* it's being closed due to some other [cause] exception occurred. * [AutoCloseable.close] function when it's being closed due to some other [cause] exception
* * occurred.
* The suppressed exception is added to the list of suppressed exceptions of [cause] exception. *
*/ * The suppressed exception is added to the list of suppressed exceptions of [cause] exception.
fun AutoCloseable?.closeFinallyAC(cause: Throwable?) = when { */
this == null -> {} fun AutoCloseable?.closeFinallyAC(cause: Throwable?) =
cause == null -> close() when {
else -> this == null -> {}
try { cause == null -> close()
close() else ->
} catch (closeException: Throwable) { try {
cause.addSuppressed(closeException) close()
} } catch (closeException: Throwable) {
} cause.addSuppressed(closeException)
}
}

View File

@@ -3,32 +3,33 @@ package com.github.codeql
import com.github.codeql.utils.getJvmName import com.github.codeql.utils.getJvmName
import com.github.codeql.utils.versions.* import com.github.codeql.utils.versions.*
import com.intellij.openapi.vfs.StandardFileSystems import com.intellij.openapi.vfs.StandardFileSystems
import org.jetbrains.kotlin.fir.java.JavaBinarySourceElement
import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClass
import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement
import com.intellij.openapi.vfs.VirtualFile import com.intellij.openapi.vfs.VirtualFile
import org.jetbrains.kotlin.builtins.jvm.JavaToKotlinClassMap import org.jetbrains.kotlin.builtins.jvm.JavaToKotlinClassMap
import org.jetbrains.kotlin.fir.java.JavaBinarySourceElement
import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable
import org.jetbrains.kotlin.ir.util.parentClassOrNull import org.jetbrains.kotlin.ir.util.parentClassOrNull
import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
import org.jetbrains.kotlin.load.kotlin.JvmPackagePartSource import org.jetbrains.kotlin.load.kotlin.JvmPackagePartSource
import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement
import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClass
// Adapted from Kotlin's interpreter/Utils.kt function 'internalName' // Adapted from Kotlin's interpreter/Utils.kt function 'internalName'
// Translates class names into their JLS section 13.1 binary name, // Translates class names into their JLS section 13.1 binary name,
// and declarations within them into the parent class' JLS 13.1 name as // and declarations within them into the parent class' JLS 13.1 name as
// specified above, followed by a `$` separator and then the short name // specified above, followed by a `$` separator and then the short name
// for `that`. // for `that`.
private fun getName(d: IrDeclarationWithName) = (d as? IrAnnotationContainer)?.let { getJvmName(it) } ?: d.name.asString() private fun getName(d: IrDeclarationWithName) =
(d as? IrAnnotationContainer)?.let { getJvmName(it) } ?: d.name.asString()
fun getFileClassName(f: IrFile) = fun getFileClassName(f: IrFile) =
getJvmName(f) ?: getJvmName(f)
((f.fileEntry.name.replaceFirst(Regex(""".*[/\\]"""), "") ?: ((f.fileEntry.name
.replaceFirst(Regex("""\.kt$"""), "") .replaceFirst(Regex(""".*[/\\]"""), "")
.replaceFirstChar { it.uppercase() }) + "Kt") .replaceFirst(Regex("""\.kt$"""), "")
.replaceFirstChar { it.uppercase() }) + "Kt")
fun getIrElementBinaryName(that: IrElement): String { fun getIrElementBinaryName(that: IrElement): String {
if (that is IrFile) { if (that is IrFile) {
@@ -38,19 +39,21 @@ fun getIrElementBinaryName(that: IrElement): String {
} }
if (that !is IrDeclaration) { if (that !is IrDeclaration) {
return "(unknown-name)" return "(unknown-name)"
} }
val shortName = when(that) { val shortName =
is IrDeclarationWithName -> getName(that) when (that) {
else -> "(unknown-name)" is IrDeclarationWithName -> getName(that)
} else -> "(unknown-name)"
}
val internalName = StringBuilder(shortName) val internalName = StringBuilder(shortName)
if (that !is IrClass) { if (that !is IrClass) {
val parent = that.parent val parent = that.parent
if (parent is IrFile) { if (parent is IrFile) {
// Note we'll fall through and do the IrPackageFragment case as well, since IrFile <: IrPackageFragment // Note we'll fall through and do the IrPackageFragment case as well, since IrFile <:
// IrPackageFragment
internalName.insert(0, getFileClassName(parent) + "$") internalName.insert(0, getFileClassName(parent) + "$")
} }
} }
@@ -59,7 +62,11 @@ fun getIrElementBinaryName(that: IrElement): String {
.forEach { .forEach {
when (it) { when (it) {
is IrClass -> internalName.insert(0, getName(it) + "$") is IrClass -> internalName.insert(0, getName(it) + "$")
is IrPackageFragment -> it.packageFqName.asString().takeIf { fqName -> fqName.isNotEmpty() }?.let { fqName -> internalName.insert(0, "$fqName.") } is IrPackageFragment ->
it.packageFqName
.asString()
.takeIf { fqName -> fqName.isNotEmpty() }
?.let { fqName -> internalName.insert(0, "$fqName.") }
} }
} }
return internalName.toString() return internalName.toString()
@@ -67,16 +74,18 @@ fun getIrElementBinaryName(that: IrElement): String {
fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? { fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
val cSource = irClass.source val cSource = irClass.source
// Don't emit a location for multi-file classes until we're sure we can cope with different declarations // Don't emit a location for multi-file classes until we're sure we can cope with different
// inside a class disagreeing about their source file. In particular this currently causes problems when // declarations
// a source-location for a declarations tries to refer to a file-id which is assumed to be declared in // inside a class disagreeing about their source file. In particular this currently causes
// problems when
// a source-location for a declarations tries to refer to a file-id which is assumed to be
// declared in
// the class trap file. // the class trap file.
if (irClass.origin == IrDeclarationOrigin.JVM_MULTIFILE_CLASS) if (irClass.origin == IrDeclarationOrigin.JVM_MULTIFILE_CLASS) return null
return null when (cSource) {
when(cSource) {
is JavaSourceElement -> { is JavaSourceElement -> {
val element = cSource.javaElement val element = cSource.javaElement
when(element) { when (element) {
is BinaryJavaClass -> return element.virtualFile is BinaryJavaClass -> return element.virtualFile
} }
} }
@@ -85,7 +94,7 @@ fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
} }
is KotlinJvmBinarySourceElement -> { is KotlinJvmBinarySourceElement -> {
val binaryClass = cSource.binaryClass val binaryClass = cSource.binaryClass
when(binaryClass) { when (binaryClass) {
is VirtualFileKotlinClass -> return binaryClass.file is VirtualFileKotlinClass -> return binaryClass.file
} }
} }
@@ -102,17 +111,17 @@ fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
private fun getRawIrClassBinaryPath(irClass: IrClass) = private fun getRawIrClassBinaryPath(irClass: IrClass) =
getIrClassVirtualFile(irClass)?.let { getIrClassVirtualFile(irClass)?.let {
val path = it.path val path = it.path
if(it.fileSystem.protocol == StandardFileSystems.JRT_PROTOCOL) if (it.fileSystem.protocol == StandardFileSystems.JRT_PROTOCOL)
// For JRT files, which we assume to be the JDK, hide the containing JAR path to match the Java extractor's behaviour. // For JRT files, which we assume to be the JDK, hide the containing JAR path to match the
"/${path.split("!/", limit = 2)[1]}" // Java extractor's behaviour.
else "/${path.split("!/", limit = 2)[1]}"
path else path
} }
fun getIrClassBinaryPath(irClass: IrClass): String { fun getIrClassBinaryPath(irClass: IrClass): String {
return getRawIrClassBinaryPath(irClass) return getRawIrClassBinaryPath(irClass)
// Otherwise, make up a fake location: // Otherwise, make up a fake location:
?: getUnknownBinaryLocation(getIrElementBinaryName(irClass)) ?: getUnknownBinaryLocation(getIrElementBinaryName(irClass))
} }
fun getIrDeclarationBinaryPath(d: IrDeclaration): String? { fun getIrDeclarationBinaryPath(d: IrDeclaration): String? {

View File

@@ -5,7 +5,6 @@ import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
import org.jetbrains.kotlin.ir.declarations.IrExternalPackageFragment import org.jetbrains.kotlin.ir.declarations.IrExternalPackageFragment
import org.jetbrains.kotlin.ir.util.isFileClass import org.jetbrains.kotlin.ir.util.isFileClass
import org.jetbrains.kotlin.ir.util.parentClassOrNull
fun isExternalDeclaration(d: IrDeclaration): Boolean { fun isExternalDeclaration(d: IrDeclaration): Boolean {
/* /*
@@ -15,9 +14,11 @@ fun isExternalDeclaration(d: IrDeclaration): Boolean {
EXPRESSION_BODY EXPRESSION_BODY
CONST Int type=kotlin.Int value=-2147483648 CONST Int type=kotlin.Int value=-2147483648
*/ */
if (d.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB || if (
d.origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB || d.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB ||
d.origin.toString() == "FUNCTION_INTERFACE_CLASS") { // Treat kotlin.coroutines.* like ordinary library classes d.origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB ||
d.origin.toString() == "FUNCTION_INTERFACE_CLASS"
) { // Treat kotlin.coroutines.* like ordinary library classes
return true return true
} }
/* /*
@@ -39,11 +40,11 @@ fun isExternalDeclaration(d: IrDeclaration): Boolean {
return false return false
} }
/** /** Returns true if `d` is not itself a class, but is a member of an external file class. */
* Returns true if `d` is not itself a class, but is a member of an external file class.
*/
fun isExternalFileClassMember(d: IrDeclaration): Boolean { fun isExternalFileClassMember(d: IrDeclaration): Boolean {
if (d is IrClass) { return false } if (d is IrClass) {
return false
}
val p = d.parent val p = d.parent
when (p) { when (p) {
is IrClass -> return p.isFileClass is IrClass -> return p.isFileClass
@@ -53,4 +54,3 @@ fun isExternalFileClassMember(d: IrDeclaration): Boolean {
} }
return false return false
} }

View File

@@ -9,10 +9,18 @@ fun getClassByFqName(pluginContext: IrPluginContext, fqName: String): IrClassSym
return getClassByFqName(pluginContext, FqName(fqName)) return getClassByFqName(pluginContext, FqName(fqName))
} }
fun getFunctionsByFqName(pluginContext: IrPluginContext, pkgName: String, name: String): Collection<IrSimpleFunctionSymbol> { fun getFunctionsByFqName(
pluginContext: IrPluginContext,
pkgName: String,
name: String
): Collection<IrSimpleFunctionSymbol> {
return getFunctionsByFqName(pluginContext, FqName(pkgName), Name.identifier(name)) return getFunctionsByFqName(pluginContext, FqName(pkgName), Name.identifier(name))
} }
fun getPropertiesByFqName(pluginContext: IrPluginContext, pkgName: String, name: String): Collection<IrPropertySymbol> { fun getPropertiesByFqName(
pluginContext: IrPluginContext,
pkgName: String,
name: String
): Collection<IrPropertySymbol> {
return getPropertiesByFqName(pluginContext, FqName(pkgName), Name.identifier(name)) return getPropertiesByFqName(pluginContext, FqName(pkgName), Name.identifier(name))
} }

View File

@@ -9,4 +9,5 @@ fun IrFunction.isLocalFunction(): Boolean {
return this.visibility == DescriptorVisibilities.LOCAL return this.visibility == DescriptorVisibilities.LOCAL
} }
val IrClass.isInterfaceLike get() = kind == ClassKind.INTERFACE || kind == ClassKind.ANNOTATION_CLASS val IrClass.isInterfaceLike
get() = kind == ClassKind.INTERFACE || kind == ClassKind.ANNOTATION_CLASS

View File

@@ -1,6 +1,5 @@
package com.github.codeql.utils package com.github.codeql.utils
import com.github.codeql.utils.Psi2IrFacade
import com.intellij.psi.PsiElement import com.intellij.psi.PsiElement
import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.IrDeclaration import org.jetbrains.kotlin.ir.declarations.IrDeclaration
@@ -8,8 +7,11 @@ import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.ir.util.isFakeOverride import org.jetbrains.kotlin.ir.util.isFakeOverride
import org.jetbrains.kotlin.ir.visitors.IrElementVisitor import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
class IrVisitorLookup(private val psi2Ir: Psi2IrFacade, private val psi: PsiElement, private val file: IrFile) : class IrVisitorLookup(
IrElementVisitor<Unit, MutableCollection<IrElement>> { private val psi2Ir: Psi2IrFacade,
private val psi: PsiElement,
private val file: IrFile
) : IrElementVisitor<Unit, MutableCollection<IrElement>> {
private val location = psi.getLocation() private val location = psi.getLocation()
override fun visitElement(element: IrElement, data: MutableCollection<IrElement>): Unit { override fun visitElement(element: IrElement, data: MutableCollection<IrElement>): Unit {

View File

@@ -3,11 +3,10 @@ package com.github.codeql
import org.jetbrains.kotlin.ir.declarations.IrDeclaration import org.jetbrains.kotlin.ir.declarations.IrDeclaration
/** /**
* This behaves the same as Iterable<IrDeclaration>.find, but requires * This behaves the same as Iterable<IrDeclaration>.find, but requires that the value found is of
* that the value found is of the subtype S, and it casts * the subtype S, and it casts the result for you appropriately.
* the result for you appropriately.
*/ */
inline fun <reified S: IrDeclaration> Iterable<IrDeclaration>.findSubType( inline fun <reified S : IrDeclaration> Iterable<IrDeclaration>.findSubType(
predicate: (S) -> Boolean predicate: (S) -> Boolean
): S? { ): S? {
return this.find { it is S && predicate(it) } as S? return this.find { it is S && predicate(it) } as S?

View File

@@ -17,43 +17,46 @@ import org.jetbrains.kotlin.name.Name
private data class MethodKey(val className: FqName, val functionName: Name) private data class MethodKey(val className: FqName, val functionName: Name)
private fun makeDescription(className: FqName, functionName: String) = MethodKey(className, Name.guessByFirstCharacter(functionName)) private fun makeDescription(className: FqName, functionName: String) =
MethodKey(className, Name.guessByFirstCharacter(functionName))
// This essentially mirrors SpecialBridgeMethods.kt, a backend pass which isn't easily available to our extractor. // This essentially mirrors SpecialBridgeMethods.kt, a backend pass which isn't easily available to
private val specialFunctions = mapOf( // our extractor.
makeDescription(StandardNames.FqNames.collection, "<get-size>") to "size", private val specialFunctions =
makeDescription(FqName("java.util.Collection"), "<get-size>") to "size", mapOf(
makeDescription(StandardNames.FqNames.map, "<get-size>") to "size", makeDescription(StandardNames.FqNames.collection, "<get-size>") to "size",
makeDescription(FqName("java.util.Map"), "<get-size>") to "size", makeDescription(FqName("java.util.Collection"), "<get-size>") to "size",
makeDescription(StandardNames.FqNames.charSequence.toSafe(), "<get-length>") to "length", makeDescription(StandardNames.FqNames.map, "<get-size>") to "size",
makeDescription(FqName("java.lang.CharSequence"), "<get-length>") to "length", makeDescription(FqName("java.util.Map"), "<get-size>") to "size",
makeDescription(StandardNames.FqNames.map, "<get-keys>") to "keySet", makeDescription(StandardNames.FqNames.charSequence.toSafe(), "<get-length>") to "length",
makeDescription(FqName("java.util.Map"), "<get-keys>") to "keySet", makeDescription(FqName("java.lang.CharSequence"), "<get-length>") to "length",
makeDescription(StandardNames.FqNames.map, "<get-values>") to "values", makeDescription(StandardNames.FqNames.map, "<get-keys>") to "keySet",
makeDescription(FqName("java.util.Map"), "<get-values>") to "values", makeDescription(FqName("java.util.Map"), "<get-keys>") to "keySet",
makeDescription(StandardNames.FqNames.map, "<get-entries>") to "entrySet", makeDescription(StandardNames.FqNames.map, "<get-values>") to "values",
makeDescription(FqName("java.util.Map"), "<get-entries>") to "entrySet", makeDescription(FqName("java.util.Map"), "<get-values>") to "values",
makeDescription(StandardNames.FqNames.mutableList, "removeAt") to "remove", makeDescription(StandardNames.FqNames.map, "<get-entries>") to "entrySet",
makeDescription(FqName("java.util.List"), "removeAt") to "remove", makeDescription(FqName("java.util.Map"), "<get-entries>") to "entrySet",
makeDescription(StandardNames.FqNames._enum.toSafe(), "<get-ordinal>") to "ordinal", makeDescription(StandardNames.FqNames.mutableList, "removeAt") to "remove",
makeDescription(FqName("java.lang.Enum"), "<get-ordinal>") to "ordinal", makeDescription(FqName("java.util.List"), "removeAt") to "remove",
makeDescription(StandardNames.FqNames._enum.toSafe(), "<get-name>") to "name", makeDescription(StandardNames.FqNames._enum.toSafe(), "<get-ordinal>") to "ordinal",
makeDescription(FqName("java.lang.Enum"), "<get-name>") to "name", makeDescription(FqName("java.lang.Enum"), "<get-ordinal>") to "ordinal",
makeDescription(StandardNames.FqNames.number.toSafe(), "toByte") to "byteValue", makeDescription(StandardNames.FqNames._enum.toSafe(), "<get-name>") to "name",
makeDescription(FqName("java.lang.Number"), "toByte") to "byteValue", makeDescription(FqName("java.lang.Enum"), "<get-name>") to "name",
makeDescription(StandardNames.FqNames.number.toSafe(), "toShort") to "shortValue", makeDescription(StandardNames.FqNames.number.toSafe(), "toByte") to "byteValue",
makeDescription(FqName("java.lang.Number"), "toShort") to "shortValue", makeDescription(FqName("java.lang.Number"), "toByte") to "byteValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toInt") to "intValue", makeDescription(StandardNames.FqNames.number.toSafe(), "toShort") to "shortValue",
makeDescription(FqName("java.lang.Number"), "toInt") to "intValue", makeDescription(FqName("java.lang.Number"), "toShort") to "shortValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toLong") to "longValue", makeDescription(StandardNames.FqNames.number.toSafe(), "toInt") to "intValue",
makeDescription(FqName("java.lang.Number"), "toLong") to "longValue", makeDescription(FqName("java.lang.Number"), "toInt") to "intValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toFloat") to "floatValue", makeDescription(StandardNames.FqNames.number.toSafe(), "toLong") to "longValue",
makeDescription(FqName("java.lang.Number"), "toFloat") to "floatValue", makeDescription(FqName("java.lang.Number"), "toLong") to "longValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toDouble") to "doubleValue", makeDescription(StandardNames.FqNames.number.toSafe(), "toFloat") to "floatValue",
makeDescription(FqName("java.lang.Number"), "toDouble") to "doubleValue", makeDescription(FqName("java.lang.Number"), "toFloat") to "floatValue",
makeDescription(StandardNames.FqNames.string.toSafe(), "get") to "charAt", makeDescription(StandardNames.FqNames.number.toSafe(), "toDouble") to "doubleValue",
makeDescription(FqName("java.lang.String"), "get") to "charAt", makeDescription(FqName("java.lang.Number"), "toDouble") to "doubleValue",
) makeDescription(StandardNames.FqNames.string.toSafe(), "get") to "charAt",
makeDescription(FqName("java.lang.String"), "get") to "charAt",
)
private val specialFunctionShortNames = specialFunctions.keys.map { it.functionName }.toSet() private val specialFunctionShortNames = specialFunctions.keys.map { it.functionName }.toSet()
@@ -71,7 +74,7 @@ private fun getSpecialJvmName(f: IrFunction): String? {
} }
fun getJvmName(container: IrAnnotationContainer): String? { fun getJvmName(container: IrAnnotationContainer): String? {
for(a: IrConstructorCall in container.annotations) { for (a: IrConstructorCall in container.annotations) {
val t = a.type val t = a.type
if (t is IrSimpleType && a.valueArgumentsCount == 1) { if (t is IrSimpleType && a.valueArgumentsCount == 1) {
val owner = t.classifier.owner val owner = t.classifier.owner
@@ -79,7 +82,7 @@ fun getJvmName(container: IrAnnotationContainer): String? {
if (owner is IrClass) { if (owner is IrClass) {
val aPkg = owner.packageFqName?.asString() val aPkg = owner.packageFqName?.asString()
val name = owner.name.asString() val name = owner.name.asString()
if(aPkg == "kotlin.jvm" && name == "JvmName" && v is IrConst<*>) { if (aPkg == "kotlin.jvm" && name == "JvmName" && v is IrConst<*>) {
val value = v.value val value = v.value
if (value is String) { if (value is String) {
return value return value

View File

@@ -1,13 +1,13 @@
package com.github.codeql package com.github.codeql
/** /**
* Turns this list of nullable elements into a list of non-nullable * Turns this list of nullable elements into a list of non-nullable elements if they are all
* elements if they are all non-null, or returns null otherwise. * non-null, or returns null otherwise.
*/ */
public fun <T : Any> List<T?>.requireNoNullsOrNull(): List<T>? { public fun <T : Any> List<T?>.requireNoNullsOrNull(): List<T>? {
try { try {
return this.requireNoNulls() return this.requireNoNulls()
} catch (e: IllegalArgumentException) { } catch (e: IllegalArgumentException) {
return null; return null
} }
} }

View File

@@ -5,8 +5,8 @@ import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.psi.psiUtil.endOffset import org.jetbrains.kotlin.psi.psiUtil.endOffset
import org.jetbrains.kotlin.psi.psiUtil.startOffset import org.jetbrains.kotlin.psi.psiUtil.startOffset
data class Location(val startOffset: Int, val endOffset: Int){ data class Location(val startOffset: Int, val endOffset: Int) {
fun contains(location: Location) : Boolean { fun contains(location: Location): Boolean {
return this.startOffset <= location.startOffset && this.endOffset >= location.endOffset return this.startOffset <= location.startOffset && this.endOffset >= location.endOffset
} }
@@ -15,10 +15,10 @@ data class Location(val startOffset: Int, val endOffset: Int){
} }
} }
fun IrElement.getLocation() : Location { fun IrElement.getLocation(): Location {
return Location(this.startOffset, this.endOffset) return Location(this.startOffset, this.endOffset)
} }
fun PsiElement.getLocation() : Location { fun PsiElement.getLocation(): Location {
return Location(this.startOffset, this.endOffset) return Location(this.startOffset, this.endOffset)
} }

View File

@@ -12,8 +12,10 @@ import org.jetbrains.kotlin.ir.IrElement
class LogCounter() { class LogCounter() {
public val diagnosticInfo = mutableMapOf<String, Pair<Severity, Int>>() public val diagnosticInfo = mutableMapOf<String, Pair<Severity, Int>>()
public val diagnosticLimit: Int public val diagnosticLimit: Int
init { init {
diagnosticLimit = System.getenv("CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT")?.toIntOrNull() ?: 100 diagnosticLimit =
System.getenv("CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT")?.toIntOrNull() ?: 100
} }
} }
@@ -35,6 +37,7 @@ enum class Severity(val sev: Int) {
class LogMessage(private val kind: String, private val message: String) { class LogMessage(private val kind: String, private val message: String) {
val timestamp: String val timestamp: String
init { init {
timestamp = "${SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Date())}" timestamp = "${SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Date())}"
} }
@@ -45,45 +48,57 @@ class LogMessage(private val kind: String, private val message: String) {
private fun escape(str: String): String { private fun escape(str: String): String {
return str.replace("\\", "\\\\") return str.replace("\\", "\\\\")
.replace("\"", "\\\"") .replace("\"", "\\\"")
.replace("\u0000", "\\u0000") .replace("\u0000", "\\u0000")
.replace("\u0001", "\\u0001") .replace("\u0001", "\\u0001")
.replace("\u0002", "\\u0002") .replace("\u0002", "\\u0002")
.replace("\u0003", "\\u0003") .replace("\u0003", "\\u0003")
.replace("\u0004", "\\u0004") .replace("\u0004", "\\u0004")
.replace("\u0005", "\\u0005") .replace("\u0005", "\\u0005")
.replace("\u0006", "\\u0006") .replace("\u0006", "\\u0006")
.replace("\u0007", "\\u0007") .replace("\u0007", "\\u0007")
.replace("\u0008", "\\b") .replace("\u0008", "\\b")
.replace("\u0009", "\\t") .replace("\u0009", "\\t")
.replace("\u000A", "\\n") .replace("\u000A", "\\n")
.replace("\u000B", "\\u000B") .replace("\u000B", "\\u000B")
.replace("\u000C", "\\f") .replace("\u000C", "\\f")
.replace("\u000D", "\\r") .replace("\u000D", "\\r")
.replace("\u000E", "\\u000E") .replace("\u000E", "\\u000E")
.replace("\u000F", "\\u000F") .replace("\u000F", "\\u000F")
} }
fun toJsonLine(): String { fun toJsonLine(): String {
val kvs = listOf(Pair("origin", "CodeQL Kotlin extractor"), val kvs =
Pair("timestamp", timestamp), listOf(
Pair("kind", kind), Pair("origin", "CodeQL Kotlin extractor"),
Pair("message", message)) Pair("timestamp", timestamp),
return "{ " + kvs.map { p -> "\"${p.first}\": \"${escape(p.second)}\""}.joinToString(", ") + " }\n" Pair("kind", kind),
Pair("message", message)
)
return "{ " +
kvs.map { p -> "\"${p.first}\": \"${escape(p.second)}\"" }.joinToString(", ") +
" }\n"
} }
} }
data class ExtractorContext(val kind: String, val element: IrElement, val name: String, val loc: String) data class ExtractorContext(
val kind: String,
val element: IrElement,
val name: String,
val loc: String
)
open class LoggerBase(val logCounter: LogCounter) { open class LoggerBase(val logCounter: LogCounter) {
val extractorContextStack = Stack<ExtractorContext>() val extractorContextStack = Stack<ExtractorContext>()
private val verbosity: Int private val verbosity: Int
init { init {
verbosity = System.getenv("CODEQL_EXTRACTOR_KOTLIN_VERBOSITY")?.toIntOrNull() ?: 3 verbosity = System.getenv("CODEQL_EXTRACTOR_KOTLIN_VERBOSITY")?.toIntOrNull() ?: 3
} }
private val logStream: Writer private val logStream: Writer
init { init {
val extractorLogDir = System.getenv("CODEQL_EXTRACTOR_JAVA_LOG_DIR") val extractorLogDir = System.getenv("CODEQL_EXTRACTOR_JAVA_LOG_DIR")
if (extractorLogDir == null || extractorLogDir == "") { if (extractorLogDir == null || extractorLogDir == "") {
@@ -96,8 +111,8 @@ open class LoggerBase(val logCounter: LogCounter) {
private fun getDiagnosticLocation(): String? { private fun getDiagnosticLocation(): String? {
val st = Exception().stackTrace val st = Exception().stackTrace
for(x in st) { for (x in st) {
when(x.className) { when (x.className) {
"com.github.codeql.LoggerBase", "com.github.codeql.LoggerBase",
"com.github.codeql.Logger", "com.github.codeql.Logger",
"com.github.codeql.FileLogger" -> {} "com.github.codeql.FileLogger" -> {}
@@ -117,20 +132,29 @@ open class LoggerBase(val logCounter: LogCounter) {
file_number_diagnostic_number = 0 file_number_diagnostic_number = 0
} }
fun diagnostic(dtw: DiagnosticTrapWriter, severity: Severity, msg: String, extraInfo: String?, locationString: String? = null, mkLocationId: () -> Label<DbLocation> = { dtw.unknownLocation }) { fun diagnostic(
dtw: DiagnosticTrapWriter,
severity: Severity,
msg: String,
extraInfo: String?,
locationString: String? = null,
mkLocationId: () -> Label<DbLocation> = { dtw.unknownLocation }
) {
val diagnosticLoc = getDiagnosticLocation() val diagnosticLoc = getDiagnosticLocation()
val diagnosticLocStr = if(diagnosticLoc == null) "<unknown location>" else diagnosticLoc val diagnosticLocStr = if (diagnosticLoc == null) "<unknown location>" else diagnosticLoc
val suffix = val suffix =
if(diagnosticLoc == null) { if (diagnosticLoc == null) {
" Missing caller information.\n" " Missing caller information.\n"
} else { } else {
val oldInfo = logCounter.diagnosticInfo.getOrDefault(diagnosticLoc, Pair(severity, 0)) val oldInfo =
if(severity != oldInfo.first) { logCounter.diagnosticInfo.getOrDefault(diagnosticLoc, Pair(severity, 0))
if (severity != oldInfo.first) {
// We don't want to get in a loop, so just emit this // We don't want to get in a loop, so just emit this
// directly without going through the diagnostic // directly without going through the diagnostic
// counting machinery // counting machinery
if (verbosity >= 1) { if (verbosity >= 1) {
val message = "Severity mismatch ($severity vs ${oldInfo.first}) at $diagnosticLoc" val message =
"Severity mismatch ($severity vs ${oldInfo.first}) at $diagnosticLoc"
emitDiagnostic(dtw, Severity.Error, "Inconsistency", message, message) emitDiagnostic(dtw, Severity.Error, "Inconsistency", message, message)
} }
} }
@@ -139,7 +163,8 @@ open class LoggerBase(val logCounter: LogCounter) {
logCounter.diagnosticInfo[diagnosticLoc] = newInfo logCounter.diagnosticInfo[diagnosticLoc] = newInfo
when { when {
logCounter.diagnosticLimit <= 0 -> "" logCounter.diagnosticLimit <= 0 -> ""
newCount == logCounter.diagnosticLimit -> " Limit reached for diagnostics from $diagnosticLoc.\n" newCount == logCounter.diagnosticLimit ->
" Limit reached for diagnostics from $diagnosticLoc.\n"
newCount > logCounter.diagnosticLimit -> return newCount > logCounter.diagnosticLimit -> return
else -> "" else -> ""
} }
@@ -162,15 +187,36 @@ open class LoggerBase(val logCounter: LogCounter) {
emitDiagnostic(dtw, severity, diagnosticLocStr, msg, fullMsg, locationString, mkLocationId) emitDiagnostic(dtw, severity, diagnosticLocStr, msg, fullMsg, locationString, mkLocationId)
} }
private fun emitDiagnostic(dtw: DiagnosticTrapWriter, severity: Severity, diagnosticLocStr: String, msg: String, fullMsg: String, locationString: String? = null, mkLocationId: () -> Label<DbLocation> = { dtw.unknownLocation }) { private fun emitDiagnostic(
dtw: DiagnosticTrapWriter,
severity: Severity,
diagnosticLocStr: String,
msg: String,
fullMsg: String,
locationString: String? = null,
mkLocationId: () -> Label<DbLocation> = { dtw.unknownLocation }
) {
val locStr = if (locationString == null) "" else "At " + locationString + ": " val locStr = if (locationString == null) "" else "At " + locationString + ": "
val kind = if (severity <= Severity.WarnHigh) "WARN" else "ERROR" val kind = if (severity <= Severity.WarnHigh) "WARN" else "ERROR"
val logMessage = LogMessage(kind, "Diagnostic($diagnosticLocStr): $locStr$fullMsg") val logMessage = LogMessage(kind, "Diagnostic($diagnosticLocStr): $locStr$fullMsg")
// We don't actually make the location until after the `return` above // We don't actually make the location until after the `return` above
val locationId = mkLocationId() val locationId = mkLocationId()
val diagLabel = dtw.getFreshIdLabel<DbDiagnostic>() val diagLabel = dtw.getFreshIdLabel<DbDiagnostic>()
dtw.writeDiagnostics(diagLabel, "CodeQL Kotlin extractor", severity.sev, "", msg, "${logMessage.timestamp} $fullMsg", locationId) dtw.writeDiagnostics(
dtw.writeDiagnostic_for(diagLabel, StringLabel("compilation"), file_number, file_number_diagnostic_number++) diagLabel,
"CodeQL Kotlin extractor",
severity.sev,
"",
msg,
"${logMessage.timestamp} $fullMsg",
locationId
)
dtw.writeDiagnostic_for(
diagLabel,
StringLabel("compilation"),
file_number,
file_number_diagnostic_number++
)
logStream.write(logMessage.toJsonLine()) logStream.write(logMessage.toJsonLine())
} }
@@ -203,6 +249,7 @@ open class LoggerBase(val logCounter: LogCounter) {
diagnostic(dtw, Severity.Warn, msg, extraInfo) diagnostic(dtw, Severity.Warn, msg, extraInfo)
} }
} }
fun error(dtw: DiagnosticTrapWriter, msg: String, extraInfo: String?) { fun error(dtw: DiagnosticTrapWriter, msg: String, extraInfo: String?) {
if (verbosity >= 1) { if (verbosity >= 1) {
diagnostic(dtw, Severity.Error, msg, extraInfo) diagnostic(dtw, Severity.Error, msg, extraInfo)
@@ -210,11 +257,12 @@ open class LoggerBase(val logCounter: LogCounter) {
} }
fun printLimitedDiagnosticCounts(dtw: DiagnosticTrapWriter) { fun printLimitedDiagnosticCounts(dtw: DiagnosticTrapWriter) {
for((caller, info) in logCounter.diagnosticInfo) { for ((caller, info) in logCounter.diagnosticInfo) {
val severity = info.first val severity = info.first
val count = info.second val count = info.second
if(count >= logCounter.diagnosticLimit) { if (count >= logCounter.diagnosticLimit) {
val message = "Total of $count diagnostics (reached limit of ${logCounter.diagnosticLimit}) from $caller." val message =
"Total of $count diagnostics (reached limit of ${logCounter.diagnosticLimit}) from $caller."
if (verbosity >= 1) { if (verbosity >= 1) {
emitDiagnostic(dtw, severity, "Limit", message, message) emitDiagnostic(dtw, severity, "Limit", message, message)
} }
@@ -240,9 +288,11 @@ open class Logger(val loggerBase: LoggerBase, open val dtw: DiagnosticTrapWriter
fun trace(msg: String) { fun trace(msg: String) {
loggerBase.trace(dtw, msg) loggerBase.trace(dtw, msg)
} }
fun trace(msg: String, exn: Throwable) { fun trace(msg: String, exn: Throwable) {
trace(msg + "\n" + exn.stackTraceToString()) trace(msg + "\n" + exn.stackTraceToString())
} }
fun debug(msg: String) { fun debug(msg: String) {
loggerBase.debug(dtw, msg) loggerBase.debug(dtw, msg)
} }
@@ -254,9 +304,11 @@ open class Logger(val loggerBase: LoggerBase, open val dtw: DiagnosticTrapWriter
private fun warn(msg: String, extraInfo: String?) { private fun warn(msg: String, extraInfo: String?) {
loggerBase.warn(dtw, msg, extraInfo) loggerBase.warn(dtw, msg, extraInfo)
} }
fun warn(msg: String, exn: Throwable) { fun warn(msg: String, exn: Throwable) {
warn(msg, exn.stackTraceToString()) warn(msg, exn.stackTraceToString())
} }
fun warn(msg: String) { fun warn(msg: String) {
warn(msg, null) warn(msg, null)
} }
@@ -264,24 +316,41 @@ open class Logger(val loggerBase: LoggerBase, open val dtw: DiagnosticTrapWriter
private fun error(msg: String, extraInfo: String?) { private fun error(msg: String, extraInfo: String?) {
loggerBase.error(dtw, msg, extraInfo) loggerBase.error(dtw, msg, extraInfo)
} }
fun error(msg: String) { fun error(msg: String) {
error(msg, null) error(msg, null)
} }
fun error(msg: String, exn: Throwable) { fun error(msg: String, exn: Throwable) {
error(msg, exn.stackTraceToString()) error(msg, exn.stackTraceToString())
} }
} }
class FileLogger(loggerBase: LoggerBase, val ftw: FileTrapWriter): Logger(loggerBase, ftw.getDiagnosticTrapWriter()) { class FileLogger(loggerBase: LoggerBase, val ftw: FileTrapWriter) :
Logger(loggerBase, ftw.getDiagnosticTrapWriter()) {
fun warnElement(msg: String, element: IrElement, exn: Throwable? = null) { fun warnElement(msg: String, element: IrElement, exn: Throwable? = null) {
val locationString = ftw.getLocationString(element) val locationString = ftw.getLocationString(element)
val mkLocationId = { ftw.getLocation(element) } val mkLocationId = { ftw.getLocation(element) }
loggerBase.diagnostic(ftw.getDiagnosticTrapWriter(), Severity.Warn, msg, exn?.stackTraceToString(), locationString, mkLocationId) loggerBase.diagnostic(
ftw.getDiagnosticTrapWriter(),
Severity.Warn,
msg,
exn?.stackTraceToString(),
locationString,
mkLocationId
)
} }
fun errorElement(msg: String, element: IrElement, exn: Throwable? = null) { fun errorElement(msg: String, element: IrElement, exn: Throwable? = null) {
val locationString = ftw.getLocationString(element) val locationString = ftw.getLocationString(element)
val mkLocationId = { ftw.getLocation(element) } val mkLocationId = { ftw.getLocation(element) }
loggerBase.diagnostic(ftw.getDiagnosticTrapWriter(), Severity.Error, msg, exn?.stackTraceToString(), locationString, mkLocationId) loggerBase.diagnostic(
ftw.getDiagnosticTrapWriter(),
Severity.Error,
msg,
exn?.stackTraceToString(),
locationString,
mkLocationId
)
} }
} }

View File

@@ -7,5 +7,6 @@ import org.jetbrains.kotlin.psi.KtFile
interface Psi2IrFacade { interface Psi2IrFacade {
fun getKtFile(irFile: IrFile): KtFile? fun getKtFile(irFile: IrFile): KtFile?
fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement? fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement?
} }

View File

@@ -1,30 +1,40 @@
package com.github.codeql package com.github.codeql
/** /**
* A triple of a type's database label, its signature for use in callable signatures, and its short name for use * A triple of a type's database label, its signature for use in callable signatures, and its short
* in all tables that provide a user-facing type name. * name for use in all tables that provide a user-facing type name.
* *
* `signature` is a Java primitive name (e.g. "int"), a fully-qualified class name ("package.OuterClass.InnerClass"), * `signature` is a Java primitive name (e.g. "int"), a fully-qualified class name
* or an array ("componentSignature[]") * ("package.OuterClass.InnerClass"), or an array ("componentSignature[]") Type variables have the
* Type variables have the signature of their upper bound. * signature of their upper bound. Type arguments and anonymous types do not have a signature.
* Type arguments and anonymous types do not have a signature.
* *
* `shortName` is a Java primitive name (e.g. "int"), a class short name with Java-style type arguments ("InnerClass<E>" or * `shortName` is a Java primitive name (e.g. "int"), a class short name with Java-style type
* "OuterClass<ConcreteArgument>" or "OtherClass<? extends Bound>") or an array ("componentShortName[]"). * arguments ("InnerClass<E>" or "OuterClass<ConcreteArgument>" or "OtherClass<? extends Bound>") or
* an array ("componentShortName[]").
*/ */
data class TypeResultGeneric<SignatureType,out LabelType: AnyDbType>(val id: Label<out LabelType>, val signature: SignatureType?, val shortName: String) { data class TypeResultGeneric<SignatureType, out LabelType : AnyDbType>(
fun <U: AnyDbType> cast(): TypeResultGeneric<SignatureType,U> { val id: Label<out LabelType>,
@Suppress("UNCHECKED_CAST") val signature: SignatureType?,
return this as TypeResultGeneric<SignatureType,U> val shortName: String
) {
fun <U : AnyDbType> cast(): TypeResultGeneric<SignatureType, U> {
@Suppress("UNCHECKED_CAST") return this as TypeResultGeneric<SignatureType, U>
} }
} }
data class TypeResultsGeneric<SignatureType>(val javaResult: TypeResultGeneric<SignatureType,DbType>, val kotlinResult: TypeResultGeneric<SignatureType,DbKt_type>)
typealias TypeResult<T> = TypeResultGeneric<String,T> data class TypeResultsGeneric<SignatureType>(
typealias TypeResultWithoutSignature<T> = TypeResultGeneric<Unit,T> val javaResult: TypeResultGeneric<SignatureType, DbType>,
val kotlinResult: TypeResultGeneric<SignatureType, DbKt_type>
)
typealias TypeResult<T> = TypeResultGeneric<String, T>
typealias TypeResultWithoutSignature<T> = TypeResultGeneric<Unit, T>
typealias TypeResults = TypeResultsGeneric<String> typealias TypeResults = TypeResultsGeneric<String>
typealias TypeResultsWithoutSignatures = TypeResultsGeneric<Unit> typealias TypeResultsWithoutSignatures = TypeResultsGeneric<Unit>
fun <T: AnyDbType> TypeResult<T>.forgetSignature(): TypeResultWithoutSignature<T> { fun <T : AnyDbType> TypeResult<T>.forgetSignature(): TypeResultWithoutSignature<T> {
return TypeResultWithoutSignature(this.id, Unit, this.shortName) return TypeResultWithoutSignature(this.id, Unit, this.shortName)
} }

View File

@@ -33,51 +33,49 @@ import org.jetbrains.kotlin.types.Variance
import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
fun IrType.substituteTypeArguments(params: List<IrTypeParameter>, arguments: List<IrTypeArgument>) = fun IrType.substituteTypeArguments(params: List<IrTypeParameter>, arguments: List<IrTypeArgument>) =
when(this) { when (this) {
is IrSimpleType -> substituteTypeArguments(params.map { it.symbol }.zip(arguments).toMap()) is IrSimpleType -> substituteTypeArguments(params.map { it.symbol }.zip(arguments).toMap())
else -> this else -> this
} }
private fun IrSimpleType.substituteTypeArguments(substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>): IrSimpleType { private fun IrSimpleType.substituteTypeArguments(
substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>
): IrSimpleType {
if (substitutionMap.isEmpty()) return this if (substitutionMap.isEmpty()) return this
val newArguments = arguments.map { val newArguments =
if (it is IrTypeProjection) { arguments.map {
val itType = it.type if (it is IrTypeProjection) {
if (itType is IrSimpleType) { val itType = it.type
subProjectedType(substitutionMap, itType, it.variance) if (itType is IrSimpleType) {
subProjectedType(substitutionMap, itType, it.variance)
} else {
it
}
} else { } else {
it it
} }
} else {
it
} }
}
return IrSimpleTypeImpl( return IrSimpleTypeImpl(classifier, isNullable(), newArguments, annotations)
classifier,
isNullable(),
newArguments,
annotations
)
} }
/** /**
* Returns true if substituting `innerVariance T` into the context `outerVariance []` discards all knowledge about * Returns true if substituting `innerVariance T` into the context `outerVariance []` discards all
* what T could be. * knowledge about what T could be.
* *
* Note this throws away slightly more information than it could: for example, the projection "in (out List)" can refer to * Note this throws away slightly more information than it could: for example, the projection "in
* any superclass of anything that implements List, which specifically excludes e.g. String, but can't be represented as * (out List)" can refer to any superclass of anything that implements List, which specifically
* a type projection. The projection "out (in List)" on the other hand really is equivalent to "out Any?", which is to * excludes e.g. String, but can't be represented as a type projection. The projection "out (in
* say no bound at all. * List)" on the other hand really is equivalent to "out Any?", which is to say no bound at all.
*/ */
private fun conflictingVariance(outerVariance: Variance, innerVariance: Variance) = private fun conflictingVariance(outerVariance: Variance, innerVariance: Variance) =
(outerVariance == Variance.IN_VARIANCE && innerVariance == Variance.OUT_VARIANCE) || (outerVariance == Variance.IN_VARIANCE && innerVariance == Variance.OUT_VARIANCE) ||
(outerVariance == Variance.OUT_VARIANCE && innerVariance == Variance.IN_VARIANCE) (outerVariance == Variance.OUT_VARIANCE && innerVariance == Variance.IN_VARIANCE)
/** /**
* When substituting `innerVariance T` into the context `outerVariance []`, returns the variance part of the result * When substituting `innerVariance T` into the context `outerVariance []`, returns the variance
* `resultVariance T`. We already know they don't conflict. * part of the result `resultVariance T`. We already know they don't conflict.
*/ */
private fun combineVariance(outerVariance: Variance, innerVariance: Variance) = private fun combineVariance(outerVariance: Variance, innerVariance: Variance) =
when { when {
@@ -86,13 +84,20 @@ private fun combineVariance(outerVariance: Variance, innerVariance: Variance) =
else -> Variance.INVARIANT else -> Variance.INVARIANT
} }
private fun subProjectedType(substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>, t: IrSimpleType, outerVariance: Variance): IrTypeArgument = private fun subProjectedType(
substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>,
t: IrSimpleType,
outerVariance: Variance
): IrTypeArgument =
substitutionMap[t.classifier]?.let { substitutedTypeArg -> substitutionMap[t.classifier]?.let { substitutedTypeArg ->
if (substitutedTypeArg is IrTypeProjection) { if (substitutedTypeArg is IrTypeProjection) {
if (conflictingVariance(outerVariance, substitutedTypeArg.variance)) if (conflictingVariance(outerVariance, substitutedTypeArg.variance))
IrStarProjectionImpl IrStarProjectionImpl
else { else {
val newProjectedType = substitutedTypeArg.type.let { if (t.isNullable()) it.codeQlWithHasQuestionMark(true) else it } val newProjectedType =
substitutedTypeArg.type.let {
if (t.isNullable()) it.codeQlWithHasQuestionMark(true) else it
}
val newVariance = combineVariance(outerVariance, substitutedTypeArg.variance) val newVariance = combineVariance(outerVariance, substitutedTypeArg.variance)
makeTypeProjection(newProjectedType, newVariance) makeTypeProjection(newProjectedType, newVariance)
} }
@@ -102,33 +107,43 @@ private fun subProjectedType(substitutionMap: Map<IrTypeParameterSymbol, IrTypeA
} ?: makeTypeProjection(t.substituteTypeArguments(substitutionMap), outerVariance) } ?: makeTypeProjection(t.substituteTypeArguments(substitutionMap), outerVariance)
private fun IrTypeArgument.upperBound(context: IrPluginContext) = private fun IrTypeArgument.upperBound(context: IrPluginContext) =
when(this) { when (this) {
is IrStarProjection -> context.irBuiltIns.anyNType is IrStarProjection -> context.irBuiltIns.anyNType
is IrTypeProjection -> when(this.variance) { is IrTypeProjection ->
Variance.INVARIANT -> this.type when (this.variance) {
Variance.IN_VARIANCE -> if (this.type.isNullable()) context.irBuiltIns.anyNType else context.irBuiltIns.anyType Variance.INVARIANT -> this.type
Variance.OUT_VARIANCE -> this.type Variance.IN_VARIANCE ->
} if (this.type.isNullable()) context.irBuiltIns.anyNType
else context.irBuiltIns.anyType
Variance.OUT_VARIANCE -> this.type
}
else -> context.irBuiltIns.anyNType else -> context.irBuiltIns.anyNType
} }
private fun IrTypeArgument.lowerBound(context: IrPluginContext) = private fun IrTypeArgument.lowerBound(context: IrPluginContext) =
when(this) { when (this) {
is IrStarProjection -> context.irBuiltIns.nothingType is IrStarProjection -> context.irBuiltIns.nothingType
is IrTypeProjection -> when(this.variance) { is IrTypeProjection ->
Variance.INVARIANT -> this.type when (this.variance) {
Variance.IN_VARIANCE -> this.type Variance.INVARIANT -> this.type
Variance.OUT_VARIANCE -> if (this.type.isNullable()) context.irBuiltIns.nothingNType else context.irBuiltIns.nothingType Variance.IN_VARIANCE -> this.type
} Variance.OUT_VARIANCE ->
if (this.type.isNullable()) context.irBuiltIns.nothingNType
else context.irBuiltIns.nothingType
}
else -> context.irBuiltIns.nothingType else -> context.irBuiltIns.nothingType
} }
fun IrType.substituteTypeAndArguments(substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>?, useContext: KotlinUsesExtractor.TypeContext, pluginContext: IrPluginContext): IrType = fun IrType.substituteTypeAndArguments(
substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>?,
useContext: KotlinUsesExtractor.TypeContext,
pluginContext: IrPluginContext
): IrType =
substitutionMap?.let { substMap -> substitutionMap?.let { substMap ->
if (this is IrSimpleType) { if (this is IrSimpleType) {
val typeClassifier = this.classifier val typeClassifier = this.classifier
substMap[typeClassifier]?.let { substMap[typeClassifier]?.let {
when(useContext) { when (useContext) {
KotlinUsesExtractor.TypeContext.RETURN -> it.upperBound(pluginContext) KotlinUsesExtractor.TypeContext.RETURN -> it.upperBound(pluginContext)
else -> it.lowerBound(pluginContext) else -> it.lowerBound(pluginContext)
} }
@@ -139,42 +154,41 @@ fun IrType.substituteTypeAndArguments(substitutionMap: Map<IrTypeParameterSymbol
} ?: this } ?: this
object RawTypeAnnotation { object RawTypeAnnotation {
// Much of this is taken from JvmGeneratorExtensionsImpl.kt, which is not easily accessible in plugin context. // Much of this is taken from JvmGeneratorExtensionsImpl.kt, which is not easily accessible in
// The constants "kotlin.internal.ir" and "RawType" could be referred to symbolically, but they move package // plugin context.
// The constants "kotlin.internal.ir" and "RawType" could be referred to symbolically, but they
// move package
// between different versions of the Kotlin compiler. // between different versions of the Kotlin compiler.
val annotationConstructor: IrConstructorCall by lazy { val annotationConstructor: IrConstructorCall by lazy {
val irInternalPackage = FqName("kotlin.internal.ir") val irInternalPackage = FqName("kotlin.internal.ir")
val parent = IrExternalPackageFragmentImpl( val parent =
DescriptorlessExternalPackageFragmentSymbol(), IrExternalPackageFragmentImpl(
irInternalPackage DescriptorlessExternalPackageFragmentSymbol(),
) irInternalPackage
val annoClass = IrFactoryImpl.buildClass { )
kind = ClassKind.ANNOTATION_CLASS val annoClass =
name = irInternalPackage.child(Name.identifier("RawType")).shortName() IrFactoryImpl.buildClass {
}.apply { kind = ClassKind.ANNOTATION_CLASS
createImplicitParameterDeclarationWithWrappedDescriptor() name = irInternalPackage.child(Name.identifier("RawType")).shortName()
this.parent = parent }
addConstructor { .apply {
isPrimary = true createImplicitParameterDeclarationWithWrappedDescriptor()
} this.parent = parent
} addConstructor { isPrimary = true }
}
val constructor = annoClass.constructors.single() val constructor = annoClass.constructors.single()
IrConstructorCallImpl.fromSymbolOwner( IrConstructorCallImpl.fromSymbolOwner(constructor.constructedClassType, constructor.symbol)
constructor.constructedClassType,
constructor.symbol
)
} }
} }
fun IrType.toRawType(): IrType = fun IrType.toRawType(): IrType =
when(this) { when (this) {
is IrSimpleType -> { is IrSimpleType -> {
when(val owner = this.classifier.owner) { when (val owner = this.classifier.owner) {
is IrClass -> { is IrClass -> {
if (this.arguments.isNotEmpty()) if (this.arguments.isNotEmpty())
this.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor)) this.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor))
else else this
this
} }
is IrTypeParameter -> owner.superTypes[0].toRawType() is IrTypeParameter -> owner.superTypes[0].toRawType()
else -> this else -> this
@@ -187,29 +201,31 @@ fun IrClass.toRawType(): IrType {
val result = this.typeWith(listOf()) val result = this.typeWith(listOf())
return if (this.typeParameters.isNotEmpty()) return if (this.typeParameters.isNotEmpty())
result.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor)) result.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor))
else else result
result
} }
fun IrTypeArgument.withQuestionMark(b: Boolean): IrTypeArgument = fun IrTypeArgument.withQuestionMark(b: Boolean): IrTypeArgument =
when(this) { when (this) {
is IrStarProjection -> this is IrStarProjection -> this
is IrTypeProjection -> is IrTypeProjection ->
this.type.let { when(it) { this.type.let {
is IrSimpleType -> if (it.isNullable() == b) this else makeTypeProjection(it.codeQlWithHasQuestionMark(b), this.variance) when (it) {
else -> this is IrSimpleType ->
}} if (it.isNullable() == b) this
else makeTypeProjection(it.codeQlWithHasQuestionMark(b), this.variance)
else -> this
}
}
else -> this else -> this
} }
typealias TypeSubstitution = (IrType, KotlinUsesExtractor.TypeContext, IrPluginContext) -> IrType typealias TypeSubstitution = (IrType, KotlinUsesExtractor.TypeContext, IrPluginContext) -> IrType
private fun matchingTypeParameters(l: IrTypeParameter?, r: IrTypeParameter): Boolean { private fun matchingTypeParameters(l: IrTypeParameter?, r: IrTypeParameter): Boolean {
if (l === r) if (l === r) return true
return true if (l == null) return false
if (l == null) // Special case: match List's E and MutableList's E, for example, because in the JVM lowering
return false // they will map to the same thing.
// Special case: match List's E and MutableList's E, for example, because in the JVM lowering they will map to the same thing.
val lParent = l.parent as? IrClass ?: return false val lParent = l.parent as? IrClass ?: return false
val rParent = r.parent as? IrClass ?: return false val rParent = r.parent as? IrClass ?: return false
val lJavaId = getJavaEquivalentClassId(lParent) ?: lParent.classId val lJavaId = getJavaEquivalentClassId(lParent) ?: lParent.classId
@@ -217,34 +233,51 @@ private fun matchingTypeParameters(l: IrTypeParameter?, r: IrTypeParameter): Boo
} }
// Returns true if type is C<T1, T2, ...> where C is declared `class C<T1, T2, ...> { ... }` // Returns true if type is C<T1, T2, ...> where C is declared `class C<T1, T2, ...> { ... }`
fun isUnspecialised(paramsContainer: IrTypeParametersContainer, args: List<IrTypeArgument>, logger: Logger): Boolean { fun isUnspecialised(
paramsContainer: IrTypeParametersContainer,
args: List<IrTypeArgument>,
logger: Logger
): Boolean {
return isUnspecialised(paramsContainer, args, logger, paramsContainer) return isUnspecialised(paramsContainer, args, logger, paramsContainer)
} }
private fun isUnspecialised(paramsContainer: IrTypeParametersContainer, args: List<IrTypeArgument>, logger: Logger, origParamsContainer: IrTypeParametersContainer): Boolean { private fun isUnspecialised(
val unspecialisedHere = paramsContainer.typeParameters.zip(args).all { paramAndArg -> paramsContainer: IrTypeParametersContainer,
(paramAndArg.second as? IrTypeProjection)?.let { args: List<IrTypeArgument>,
// Type arg refers to the class' own type parameter? logger: Logger,
it.variance == Variance.INVARIANT && origParamsContainer: IrTypeParametersContainer
matchingTypeParameters(it.type.classifierOrNull?.owner as? IrTypeParameter, paramAndArg.first) ): Boolean {
} ?: false val unspecialisedHere =
} paramsContainer.typeParameters.zip(args).all { paramAndArg ->
(paramAndArg.second as? IrTypeProjection)?.let {
// Type arg refers to the class' own type parameter?
it.variance == Variance.INVARIANT &&
matchingTypeParameters(
it.type.classifierOrNull?.owner as? IrTypeParameter,
paramAndArg.first
)
} ?: false
}
val remainingArgs = args.drop(paramsContainer.typeParameters.size) val remainingArgs = args.drop(paramsContainer.typeParameters.size)
val parentTypeContainer = paramsContainer.parents.firstIsInstanceOrNull<IrTypeParametersContainer>() val parentTypeContainer =
paramsContainer.parents.firstIsInstanceOrNull<IrTypeParametersContainer>()
val parentUnspecialised = when { val parentUnspecialised =
remainingArgs.isEmpty() -> true when {
parentTypeContainer == null -> { remainingArgs.isEmpty() -> true
logger.error("Found more type arguments than parameters: ${origParamsContainer.kotlinFqName.asString()}") parentTypeContainer == null -> {
false logger.error(
"Found more type arguments than parameters: ${origParamsContainer.kotlinFqName.asString()}"
)
false
}
else -> isUnspecialised(parentTypeContainer, remainingArgs, logger, origParamsContainer)
} }
else -> isUnspecialised(parentTypeContainer, remainingArgs, logger, origParamsContainer)
}
return unspecialisedHere && parentUnspecialised return unspecialisedHere && parentUnspecialised
} }
// Returns true if type is C<T1, T2, ...> where C is declared `class C<T1, T2, ...> { ... }` // Returns true if type is C<T1, T2, ...> where C is declared `class C<T1, T2, ...> { ... }`
fun isUnspecialised(type: IrSimpleType, logger: Logger) = (type.classifier.owner as? IrClass)?.let { fun isUnspecialised(type: IrSimpleType, logger: Logger) =
isUnspecialised(it, type.arguments, logger) (type.classifier.owner as? IrClass)?.let { isUnspecialised(it, type.arguments, logger) }
} ?: false ?: false

View File

@@ -3,7 +3,11 @@ package com.github.codeql.comments
import com.github.codeql.* import com.github.codeql.*
import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.declarations.*
class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFile, fileLabel: Label<out DbFile>): CommentExtractor(fileExtractor, file, fileLabel) { class CommentExtractorLighterAST(
fileExtractor: KotlinFileExtractor,
file: IrFile,
fileLabel: Label<out DbFile>
) : CommentExtractor(fileExtractor, file, fileLabel) {
// We don't support LighterAST with old Kotlin versions // We don't support LighterAST with old Kotlin versions
fun extract(): Boolean { fun extract(): Boolean {
return false return false

View File

@@ -2,4 +2,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.SourceManager import org.jetbrains.kotlin.ir.SourceManager
typealias FileEntry = SourceManager.FileEntry typealias FileEntry = SourceManager.FileEntry

View File

@@ -1,4 +1,3 @@
package org.jetbrains.kotlin.ir.symbols package org.jetbrains.kotlin.ir.symbols
@RequiresOptIn(level = RequiresOptIn.Level.WARNING) @RequiresOptIn(level = RequiresOptIn.Level.WARNING) annotation class IrSymbolInternals
annotation class IrSymbolInternals

View File

@@ -11,9 +11,11 @@ import org.jetbrains.kotlin.utils.addToStdlib.safeAs
fun isUnderscoreParameter(vp: IrValueParameter) = fun isUnderscoreParameter(vp: IrValueParameter) =
try { try {
DescriptorToSourceUtils.getSourceFromDescriptor(vp.descriptor) DescriptorToSourceUtils.getSourceFromDescriptor(vp.descriptor)
?.safeAs<KtParameter>()?.isSingleUnderscore == true ?.safeAs<KtParameter>()
} catch(e: NotImplementedError) { ?.isSingleUnderscore == true
// Some kinds of descriptor throw in `getSourceFromDescriptor` as that method is not normally expected to } catch (e: NotImplementedError) {
// Some kinds of descriptor throw in `getSourceFromDescriptor` as that method is not
// normally expected to
// be applied to synthetic functions. // be applied to synthetic functions.
false false
} }

View File

@@ -7,5 +7,5 @@ import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
We need this class to exist, but the compiler will never give us an We need this class to exist, but the compiler will never give us an
instance of it. instance of it.
*/ */
abstract class JavaBinarySourceElement private constructor(val javaClass: BinaryJavaClass): SourceElement { abstract class JavaBinarySourceElement private constructor(val javaClass: BinaryJavaClass) :
} SourceElement {}

View File

@@ -3,8 +3,8 @@
package com.github.codeql package com.github.codeql
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
@OptIn(ExperimentalCompilerApi::class) @OptIn(ExperimentalCompilerApi::class)
abstract class Kotlin2ComponentRegistrar : ComponentRegistrar { abstract class Kotlin2ComponentRegistrar : ComponentRegistrar {

View File

@@ -2,18 +2,17 @@ package com.github.codeql
import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.declarations.*
class LinesOfCodeLighterAST( class LinesOfCodeLighterAST(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
val logger: FileLogger,
val tw: FileTrapWriter,
val file: IrFile
) {
// We don't support LighterAST with old Kotlin versions // We don't support LighterAST with old Kotlin versions
fun linesOfCodeInFile(@Suppress("UNUSED_PARAMETER") id: Label<DbFile>): Boolean { fun linesOfCodeInFile(@Suppress("UNUSED_PARAMETER") id: Label<DbFile>): Boolean {
return false return false
} }
// We don't support LighterAST with old Kotlin versions // We don't support LighterAST with old Kotlin versions
fun linesOfCodeInDeclaration(@Suppress("UNUSED_PARAMETER") d: IrDeclaration, @Suppress("UNUSED_PARAMETER") id: Label<out DbSourceline>): Boolean { fun linesOfCodeInDeclaration(
@Suppress("UNUSED_PARAMETER") d: IrDeclaration,
@Suppress("UNUSED_PARAMETER") id: Label<out DbSourceline>
): Boolean {
return false return false
} }
} }

View File

@@ -14,12 +14,20 @@ fun getClassByClassId(pluginContext: IrPluginContext, id: ClassId): IrClassSymbo
return getClassByFqName(pluginContext, id.asSingleFqName()) return getClassByFqName(pluginContext, id.asSingleFqName())
} }
fun getFunctionsByFqName(pluginContext: IrPluginContext, pkgName: FqName, name: Name): Collection<IrSimpleFunctionSymbol> { fun getFunctionsByFqName(
pluginContext: IrPluginContext,
pkgName: FqName,
name: Name
): Collection<IrSimpleFunctionSymbol> {
val fqName = pkgName.child(name) val fqName = pkgName.child(name)
return pluginContext.referenceFunctions(fqName) return pluginContext.referenceFunctions(fqName)
} }
fun getPropertiesByFqName(pluginContext: IrPluginContext, pkgName: FqName, name: Name): Collection<IrPropertySymbol> { fun getPropertiesByFqName(
pluginContext: IrPluginContext,
pkgName: FqName,
name: Name
): Collection<IrPropertySymbol> {
val fqName = pkgName.child(name) val fqName = pkgName.child(name)
return pluginContext.referenceProperties(fqName) return pluginContext.referenceProperties(fqName)
} }

View File

@@ -3,4 +3,3 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.expressions.IrSyntheticBodyKind import org.jetbrains.kotlin.ir.expressions.IrSyntheticBodyKind
val kind_ENUM_ENTRIES: IrSyntheticBodyKind? = null val kind_ENUM_ENTRIES: IrSyntheticBodyKind? = null

View File

@@ -3,5 +3,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.jvm.codegen.isRawType import org.jetbrains.kotlin.backend.jvm.codegen.isRawType
import org.jetbrains.kotlin.ir.types.IrSimpleType import org.jetbrains.kotlin.ir.types.IrSimpleType
fun IrSimpleType.isRawType() = this.isRawType()
fun IrSimpleType.isRawType() = this.isRawType()

View File

@@ -1,6 +1,6 @@
package com.github.codeql.utils.versions package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
import org.jetbrains.kotlin.backend.common.ir.allOverridden import org.jetbrains.kotlin.backend.common.ir.allOverridden
import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
fun IrSimpleFunction.allOverriddenIncludingSelf() = this.allOverridden(includeSelf = true) fun IrSimpleFunction.allOverriddenIncludingSelf() = this.allOverridden(includeSelf = true)

View File

@@ -5,4 +5,4 @@ import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
@OptIn(ObsoleteDescriptorBasedAPI::class) @OptIn(ObsoleteDescriptorBasedAPI::class)
fun getAnnotationType(context: IrPluginContext) = fun getAnnotationType(context: IrPluginContext) =
context.typeTranslator.translateType(context.builtIns.annotationType) context.typeTranslator.translateType(context.builtIns.annotationType)

View File

@@ -1,7 +1,7 @@
package com.github.codeql.utils.versions package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.common.ir.copyTo
import org.jetbrains.kotlin.ir.declarations.IrFunction import org.jetbrains.kotlin.ir.declarations.IrFunction
import org.jetbrains.kotlin.ir.declarations.IrValueParameter import org.jetbrains.kotlin.ir.declarations.IrValueParameter
import org.jetbrains.kotlin.backend.common.ir.copyTo
fun copyParameterToFunction(p: IrValueParameter, f: IrFunction) = p.copyTo(f) fun copyParameterToFunction(p: IrValueParameter, f: IrFunction) = p.copyTo(f)

View File

@@ -1,6 +1,7 @@
package com.github.codeql.utils.versions package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.backend.common.ir.createImplicitParameterDeclarationWithWrappedDescriptor import org.jetbrains.kotlin.backend.common.ir.createImplicitParameterDeclarationWithWrappedDescriptor
import org.jetbrains.kotlin.ir.declarations.IrClass
fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() = this.createImplicitParameterDeclarationWithWrappedDescriptor() fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() =
this.createImplicitParameterDeclarationWithWrappedDescriptor()

View File

@@ -1,7 +1,7 @@
package com.github.codeql.utils.versions package com.github.codeql.utils.versions
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.ir.declarations.IrDeclaration import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.name.FqName
fun getFileClassFqName(@Suppress("UNUSED_PARAMETER") d: IrDeclaration): FqName? { fun getFileClassFqName(@Suppress("UNUSED_PARAMETER") d: IrDeclaration): FqName? {
return null return null

View File

@@ -3,4 +3,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.types.IrSimpleType import org.jetbrains.kotlin.ir.types.IrSimpleType
import org.jetbrains.kotlin.ir.types.impl.IrTypeBase import org.jetbrains.kotlin.ir.types.impl.IrTypeBase
fun getKotlinType(s: IrSimpleType) = (s as? IrTypeBase)?.kotlinType fun getKotlinType(s: IrSimpleType) = (s as? IrTypeBase)?.kotlinType

View File

@@ -1,4 +1,3 @@
package org.jetbrains.kotlin.ir.util package org.jetbrains.kotlin.ir.util
import org.jetbrains.kotlin.backend.common.lower.parents as kParents import org.jetbrains.kotlin.backend.common.lower.parents as kParents
@@ -10,4 +9,3 @@ val IrDeclaration.parents: Sequence<IrDeclarationParent>
val IrDeclaration.parentsWithSelf: Sequence<IrDeclarationParent> val IrDeclaration.parentsWithSelf: Sequence<IrDeclarationParent>
get() = this.kParentsWithSelf get() = this.kParentsWithSelf

View File

@@ -3,6 +3,6 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.types.IrType import org.jetbrains.kotlin.ir.types.IrType
import org.jetbrains.kotlin.ir.types.withHasQuestionMark import org.jetbrains.kotlin.ir.types.withHasQuestionMark
fun IrType.codeQlWithHasQuestionMark(b : Boolean): IrType { fun IrType.codeQlWithHasQuestionMark(b: Boolean): IrType {
return this.withHasQuestionMark(b) return this.withHasQuestionMark(b)
} }

View File

@@ -2,4 +2,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.IrFileEntry import org.jetbrains.kotlin.ir.IrFileEntry
typealias FileEntry = IrFileEntry typealias FileEntry = IrFileEntry

View File

@@ -1,16 +1,16 @@
package com.github.codeql.utils.versions package com.github.codeql.utils.versions
import com.github.codeql.utils.Psi2IrFacade
import com.intellij.psi.PsiElement import com.intellij.psi.PsiElement
import org.jetbrains.kotlin.backend.common.psi.PsiSourceManager import org.jetbrains.kotlin.backend.common.psi.PsiSourceManager
import org.jetbrains.kotlin.backend.jvm.ir.getKtFile import org.jetbrains.kotlin.backend.jvm.ir.getKtFile
import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.IrFile import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.psi.KtFile
import com.github.codeql.utils.Psi2IrFacade
fun getPsi2Ir(): Psi2IrFacade? = Psi2Ir() fun getPsi2Ir(): Psi2IrFacade? = Psi2Ir()
private class Psi2Ir(): Psi2IrFacade { private class Psi2Ir() : Psi2IrFacade {
override fun getKtFile(irFile: IrFile): KtFile? { override fun getKtFile(irFile: IrFile): KtFile? {
return irFile.getKtFile() return irFile.getKtFile()
} }

View File

@@ -2,4 +2,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
fun functionN(pluginContext: IrPluginContext) = pluginContext.irBuiltIns::functionN fun functionN(pluginContext: IrPluginContext) = pluginContext.irBuiltIns::functionN

View File

@@ -2,5 +2,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
fun getAnnotationType(context: IrPluginContext) = fun getAnnotationType(context: IrPluginContext) = context.irBuiltIns.annotationType
context.irBuiltIns.annotationType

View File

@@ -3,4 +3,5 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
import org.jetbrains.kotlin.ir.declarations.IrValueParameter import org.jetbrains.kotlin.ir.declarations.IrValueParameter
fun isUnderscoreParameter(vp: IrValueParameter) = vp.origin == IrDeclarationOrigin.UNDERSCORE_PARAMETER fun isUnderscoreParameter(vp: IrValueParameter) =
vp.origin == IrDeclarationOrigin.UNDERSCORE_PARAMETER

View File

@@ -3,5 +3,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.jvm.ir.isRawType import org.jetbrains.kotlin.backend.jvm.ir.isRawType
import org.jetbrains.kotlin.ir.types.IrSimpleType import org.jetbrains.kotlin.ir.types.IrSimpleType
fun IrSimpleType.isRawType() = this.isRawType()
fun IrSimpleType.isRawType() = this.isRawType()

View File

@@ -1,10 +1,10 @@
package com.github.codeql.utils.versions package com.github.codeql.utils.versions
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.ir.declarations.IrDeclaration import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.IrField import org.jetbrains.kotlin.ir.declarations.IrField
import org.jetbrains.kotlin.ir.declarations.IrMemberWithContainerSource import org.jetbrains.kotlin.ir.declarations.IrMemberWithContainerSource
import org.jetbrains.kotlin.load.kotlin.FacadeClassSource import org.jetbrains.kotlin.load.kotlin.FacadeClassSource
import org.jetbrains.kotlin.name.FqName
fun getFileClassFqName(d: IrDeclaration): FqName? { fun getFileClassFqName(d: IrDeclaration): FqName? {
// d is in a file class. // d is in a file class.

View File

@@ -2,4 +2,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.types.IrSimpleType import org.jetbrains.kotlin.ir.types.IrSimpleType
fun getKotlinType(s: IrSimpleType) = s.kotlinType fun getKotlinType(s: IrSimpleType) = s.kotlinType

View File

@@ -4,7 +4,7 @@ import org.jetbrains.kotlin.ir.types.IrType
import org.jetbrains.kotlin.ir.types.makeNotNull import org.jetbrains.kotlin.ir.types.makeNotNull
import org.jetbrains.kotlin.ir.types.makeNullable import org.jetbrains.kotlin.ir.types.makeNullable
fun IrType.codeQlWithHasQuestionMark(b : Boolean): IrType { fun IrType.codeQlWithHasQuestionMark(b: Boolean): IrType {
if (b) { if (b) {
return this.makeNullable() return this.makeNullable()
} else { } else {

View File

@@ -3,4 +3,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
import org.jetbrains.kotlin.ir.util.allOverridden import org.jetbrains.kotlin.ir.util.allOverridden
fun IrSimpleFunction.allOverriddenIncludingSelf() = this.allOverridden(includeSelf = true) fun IrSimpleFunction.allOverriddenIncludingSelf() = this.allOverridden(includeSelf = true)

View File

@@ -4,4 +4,4 @@ import org.jetbrains.kotlin.ir.declarations.IrFunction
import org.jetbrains.kotlin.ir.declarations.IrValueParameter import org.jetbrains.kotlin.ir.declarations.IrValueParameter
import org.jetbrains.kotlin.ir.util.copyTo import org.jetbrains.kotlin.ir.util.copyTo
fun copyParameterToFunction(p: IrValueParameter, f: IrFunction) = p.copyTo(f) fun copyParameterToFunction(p: IrValueParameter, f: IrFunction) = p.copyTo(f)

View File

@@ -3,4 +3,5 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrClass import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.ir.util.createImplicitParameterDeclarationWithWrappedDescriptor import org.jetbrains.kotlin.ir.util.createImplicitParameterDeclarationWithWrappedDescriptor
fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() = this.createImplicitParameterDeclarationWithWrappedDescriptor() fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() =
this.createImplicitParameterDeclarationWithWrappedDescriptor()

View File

@@ -1,10 +1,9 @@
package com.github.codeql.utils package com.github.codeql.utils
import org.jetbrains.kotlin.backend.common.extensions.FirIncompatiblePluginAPI
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.ir.symbols.* import org.jetbrains.kotlin.ir.symbols.*
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.CallableId import org.jetbrains.kotlin.name.CallableId
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.name.Name
@@ -17,12 +16,20 @@ fun getClassByClassId(pluginContext: IrPluginContext, id: ClassId): IrClassSymbo
return pluginContext.referenceClass(id) return pluginContext.referenceClass(id)
} }
fun getFunctionsByFqName(pluginContext: IrPluginContext, pkgName: FqName, name: Name): Collection<IrSimpleFunctionSymbol> { fun getFunctionsByFqName(
pluginContext: IrPluginContext,
pkgName: FqName,
name: Name
): Collection<IrSimpleFunctionSymbol> {
val id = CallableId(pkgName, name) val id = CallableId(pkgName, name)
return pluginContext.referenceFunctions(id) return pluginContext.referenceFunctions(id)
} }
fun getPropertiesByFqName(pluginContext: IrPluginContext, pkgName: FqName, name: Name): Collection<IrPropertySymbol> { fun getPropertiesByFqName(
pluginContext: IrPluginContext,
pkgName: FqName,
name: Name
): Collection<IrPropertySymbol> {
val id = CallableId(pkgName, name) val id = CallableId(pkgName, name)
return pluginContext.referenceProperties(id) return pluginContext.referenceProperties(id)
} }

View File

@@ -3,4 +3,3 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.expressions.IrSyntheticBodyKind import org.jetbrains.kotlin.ir.expressions.IrSyntheticBodyKind
val kind_ENUM_ENTRIES: IrSyntheticBodyKind? = IrSyntheticBodyKind.ENUM_ENTRIES val kind_ENUM_ENTRIES: IrSyntheticBodyKind? = IrSyntheticBodyKind.ENUM_ENTRIES

View File

@@ -4,21 +4,26 @@ import com.github.codeql.*
import com.intellij.lang.LighterASTNode import com.intellij.lang.LighterASTNode
import com.intellij.util.diff.FlyweightCapableTreeStructure import com.intellij.util.diff.FlyweightCapableTreeStructure
import org.jetbrains.kotlin.fir.backend.FirMetadataSource import org.jetbrains.kotlin.fir.backend.FirMetadataSource
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET
import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
import org.jetbrains.kotlin.ir.visitors.acceptVoid import org.jetbrains.kotlin.ir.visitors.acceptVoid
import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
import org.jetbrains.kotlin.kdoc.lexer.KDocTokens import org.jetbrains.kotlin.kdoc.lexer.KDocTokens
import org.jetbrains.kotlin.lexer.KtTokens import org.jetbrains.kotlin.lexer.KtTokens
import org.jetbrains.kotlin.util.getChildren import org.jetbrains.kotlin.util.getChildren
class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFile, fileLabel: Label<out DbFile>): CommentExtractor(fileExtractor, file, fileLabel) { class CommentExtractorLighterAST(
fileExtractor: KotlinFileExtractor,
file: IrFile,
fileLabel: Label<out DbFile>
) : CommentExtractor(fileExtractor, file, fileLabel) {
// Returns true if it extracted the comments; false otherwise. // Returns true if it extracted the comments; false otherwise.
fun extract(): Boolean { fun extract(): Boolean {
val sourceElement = (file.metadata as? FirMetadataSource.File)?.files?.elementAtOrNull(0)?.source val sourceElement =
(file.metadata as? FirMetadataSource.File)?.files?.elementAtOrNull(0)?.source
val treeStructure = sourceElement?.treeStructure val treeStructure = sourceElement?.treeStructure
if (treeStructure == null) { if (treeStructure == null) {
return false return false
@@ -33,34 +38,46 @@ class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFil
fun LighterASTNode.isKDocComment() = this.tokenType == KDocTokens.KDOC fun LighterASTNode.isKDocComment() = this.tokenType == KDocTokens.KDOC
val kDocOwners = mutableMapOf<Int, MutableList<IrElement>>() val kDocOwners = mutableMapOf<Int, MutableList<IrElement>>()
val visitor = object : IrElementVisitorVoid { val visitor =
override fun visitElement(element: IrElement) { object : IrElementVisitorVoid {
val metadata = (element as? IrMetadataSourceOwner)?.metadata override fun visitElement(element: IrElement) {
val sourceElement = (metadata as? FirMetadataSource)?.fir?.source val metadata = (element as? IrMetadataSourceOwner)?.metadata
val treeStructure = sourceElement?.treeStructure val sourceElement = (metadata as? FirMetadataSource)?.fir?.source
val treeStructure = sourceElement?.treeStructure
if (treeStructure != null) { if (treeStructure != null) {
sourceElement.lighterASTNode.getChildren(treeStructure).firstOrNull { it.isKDocComment() } sourceElement.lighterASTNode
?.let { kDoc -> .getChildren(treeStructure)
// LighterASTNodes are not stable, so we can't .firstOrNull { it.isKDocComment() }
// use the node itself as the key. But the ?.let { kDoc ->
// startOffset should uniquely identify them // LighterASTNodes are not stable, so we can't
// anyway. // use the node itself as the key. But the
val startOffset = kDoc.startOffset // startOffset should uniquely identify them
if (startOffset != UNDEFINED_OFFSET && startOffset != SYNTHETIC_OFFSET) { // anyway.
kDocOwners.getOrPut(startOffset, {mutableListOf<IrElement>()}).add(element) val startOffset = kDoc.startOffset
if (
startOffset != UNDEFINED_OFFSET &&
startOffset != SYNTHETIC_OFFSET
) {
kDocOwners
.getOrPut(startOffset, { mutableListOf<IrElement>() })
.add(element)
}
} }
} }
}
element.acceptChildrenVoid(this) element.acceptChildrenVoid(this)
}
} }
}
file.acceptVoid(visitor) file.acceptVoid(visitor)
return kDocOwners return kDocOwners
} }
private fun extractComments(node: LighterASTNode, treeStructure: FlyweightCapableTreeStructure<LighterASTNode>, owners: Map<Int, List<IrElement>>) { private fun extractComments(
node: LighterASTNode,
treeStructure: FlyweightCapableTreeStructure<LighterASTNode>,
owners: Map<Int, List<IrElement>>
) {
node.getChildren(treeStructure).forEach { node.getChildren(treeStructure).forEach {
if (KtTokens.COMMENTS.contains(it.tokenType)) { if (KtTokens.COMMENTS.contains(it.tokenType)) {
extractComment(it, owners) extractComment(it, owners)
@@ -71,21 +88,22 @@ class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFil
} }
private fun extractComment(comment: LighterASTNode, owners: Map<Int, List<IrElement>>) { private fun extractComment(comment: LighterASTNode, owners: Map<Int, List<IrElement>>) {
val type: CommentType = when (comment.tokenType) { val type: CommentType =
KtTokens.EOL_COMMENT -> { when (comment.tokenType) {
CommentType.SingleLine KtTokens.EOL_COMMENT -> {
CommentType.SingleLine
}
KtTokens.BLOCK_COMMENT -> {
CommentType.Block
}
KtTokens.DOC_COMMENT -> {
CommentType.Doc
}
else -> {
logger.warn("Unhandled comment token type: ${comment.tokenType}")
return
}
} }
KtTokens.BLOCK_COMMENT -> {
CommentType.Block
}
KtTokens.DOC_COMMENT -> {
CommentType.Doc
}
else -> {
logger.warn("Unhandled comment token type: ${comment.tokenType}")
return
}
}
val commentLabel = tw.getFreshIdLabel<DbKtcomment>() val commentLabel = tw.getFreshIdLabel<DbKtcomment>()
tw.writeKtComments(commentLabel, type.value, comment.toString()) tw.writeKtComments(commentLabel, type.value, comment.toString())

View File

@@ -3,8 +3,8 @@
package com.github.codeql package com.github.codeql
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
@OptIn(ExperimentalCompilerApi::class) @OptIn(ExperimentalCompilerApi::class)
abstract class Kotlin2ComponentRegistrar : ComponentRegistrar { abstract class Kotlin2ComponentRegistrar : ComponentRegistrar {

View File

@@ -2,23 +2,19 @@ package com.github.codeql
import com.intellij.lang.LighterASTNode import com.intellij.lang.LighterASTNode
import com.intellij.util.diff.FlyweightCapableTreeStructure import com.intellij.util.diff.FlyweightCapableTreeStructure
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.fir.backend.FirMetadataSource
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.KtSourceElement import org.jetbrains.kotlin.KtSourceElement
import org.jetbrains.kotlin.fir.backend.FirMetadataSource
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.lexer.KtTokens import org.jetbrains.kotlin.lexer.KtTokens
import org.jetbrains.kotlin.util.getChildren import org.jetbrains.kotlin.util.getChildren
class LinesOfCodeLighterAST( class LinesOfCodeLighterAST(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
val logger: FileLogger,
val tw: FileTrapWriter,
val file: IrFile
) {
val fileEntry = file.fileEntry val fileEntry = file.fileEntry
fun linesOfCodeInFile(id: Label<DbFile>): Boolean { fun linesOfCodeInFile(id: Label<DbFile>): Boolean {
val sourceElement = (file.metadata as? FirMetadataSource.File)?.files?.elementAtOrNull(0)?.source val sourceElement =
(file.metadata as? FirMetadataSource.File)?.files?.elementAtOrNull(0)?.source
if (sourceElement == null) { if (sourceElement == null) {
return false return false
} }
@@ -42,7 +38,11 @@ class LinesOfCodeLighterAST(
return true return true
} }
private fun linesOfCodeInLighterAST(id: Label<out DbSourceline>, e: IrElement, s: KtSourceElement) { private fun linesOfCodeInLighterAST(
id: Label<out DbSourceline>,
e: IrElement,
s: KtSourceElement
) {
val rootStartOffset = s.startOffset val rootStartOffset = s.startOffset
val rootEndOffset = s.endOffset val rootEndOffset = s.endOffset
if (rootStartOffset < 0 || rootEndOffset < 0) { if (rootStartOffset < 0 || rootEndOffset < 0) {
@@ -63,14 +63,28 @@ class LinesOfCodeLighterAST(
val treeStructure = s.treeStructure val treeStructure = s.treeStructure
processSubtree(e, treeStructure, rootFirstLine, rootLastLine, lineContents, s.lighterASTNode) processSubtree(
e,
treeStructure,
rootFirstLine,
rootLastLine,
lineContents,
s.lighterASTNode
)
val code = lineContents.count { it.containsCode } val code = lineContents.count { it.containsCode }
val comment = lineContents.count { it.containsComment } val comment = lineContents.count { it.containsComment }
tw.writeNumlines(id, numLines, code, comment) tw.writeNumlines(id, numLines, code, comment)
} }
private fun processSubtree(e: IrElement, treeStructure: FlyweightCapableTreeStructure<LighterASTNode>, rootFirstLine: Int, rootLastLine: Int, lineContents: Array<LineContent>, node: LighterASTNode) { private fun processSubtree(
e: IrElement,
treeStructure: FlyweightCapableTreeStructure<LighterASTNode>,
rootFirstLine: Int,
rootLastLine: Int,
lineContents: Array<LineContent>,
node: LighterASTNode
) {
if (KtTokens.WHITESPACES.contains(node.tokenType)) { if (KtTokens.WHITESPACES.contains(node.tokenType)) {
return return
} }
@@ -120,7 +134,7 @@ class LinesOfCodeLighterAST(
} }
} }
} else { } else {
for(child in children) { for (child in children) {
processSubtree(e, treeStructure, rootFirstLine, rootLastLine, lineContents, child) processSubtree(e, treeStructure, rootFirstLine, rootLastLine, lineContents, child)
} }
} }