hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-17 01:03:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

Kotlin: Reformat code

Browse Source 
Using:
    java -jar ktfmt-0.46-jar-with-dependencies.jar --kotlinlang-style java/kotlin-extractor/**/*.kt
This commit is contained in:
Ian Lynagh
2024-01-09 15:33:41 +00:00
parent 0bc1463ab0
commit bf611feab3
64 changed files with 7420 additions and 3219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
java/kotlin-extractor/src/main/kotlin/ExternalDeclExtractor.kt
View File

@@ -3,18 +3,26 @@ package com.github.codeql
import com.github.codeql.utils.isExternalFileClassMember
import com.semmle.extractor.java.OdasaOutput
import com.semmle.util.data.StringDigestor
import java.io.BufferedWriter
import java.io.File
import java.util.ArrayList
import java.util.HashSet
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.isFileClass
import org.jetbrains.kotlin.ir.util.packageFqName
import java.io.BufferedWriter
import java.io.File
import java.util.ArrayList
import java.util.HashSet
import java.util.zip.GZIPOutputStream
class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression, val invocationTrapFile: String, val sourceFilePath: String, val primitiveTypeMapping: PrimitiveTypeMapping, val pluginContext: IrPluginContext, val globalExtensionState: KotlinExtractorGlobalState, val diagnosticTrapWriter: DiagnosticTrapWriter) {
class ExternalDeclExtractor(
val logger: FileLogger,
val compression: Compression,
val invocationTrapFile: String,
val sourceFilePath: String,
val primitiveTypeMapping: PrimitiveTypeMapping,
val pluginContext: IrPluginContext,
val globalExtensionState: KotlinExtractorGlobalState,
val diagnosticTrapWriter: DiagnosticTrapWriter
) {
val declBinaryNames = HashMap<IrDeclaration, String>()
val externalDeclsDone = HashSet<Pair<String, String>>()
@@ -23,13 +31,17 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
val propertySignature = ";property"
val fieldSignature = ";field"
val output = OdasaOutput(false, compression, logger).also {
it.setCurrentSourceFile(File(sourceFilePath))
}
val output =
OdasaOutput(false, compression, logger).also {
it.setCurrentSourceFile(File(sourceFilePath))
}
fun extractLater(d: IrDeclarationWithName, signature: String): Boolean {
if (d !is IrClass && !isExternalFileClassMember(d)) {
logger.errorElement("External declaration is neither a class, nor a top-level declaration", d)
logger.errorElement(
"External declaration is neither a class, nor a top-level declaration",
d
)
return false
}
val declBinaryName = declBinaryNames.getOrPut(d) { getIrElementBinaryName(d) }
@@ -37,34 +49,54 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
if (ret) externalDeclWorkList.add(Pair(d, signature))
return ret
}
fun extractLater(c: IrClass) = extractLater(c, "")
fun writeStubTrapFile(e: IrElement, signature: String = "") {
extractElement(e, signature, true) { trapFileBW, _, _ ->
trapFileBW.write("// Trap file stubbed because this declaration was extracted from source in $sourceFilePath\n")
trapFileBW.write(
"// Trap file stubbed because this declaration was extracted from source in $sourceFilePath\n"
)
trapFileBW.write("// Part of invocation $invocationTrapFile\n")
}
}
private fun extractElement(element: IrElement, possiblyLongSignature: String, fromSource: Boolean, extractorFn: (BufferedWriter, String, OdasaOutput.TrapFileManager) -> Unit) {
// In order to avoid excessively long signatures which can lead to trap file names longer than the filesystem
private fun extractElement(
element: IrElement,
possiblyLongSignature: String,
fromSource: Boolean,
extractorFn: (BufferedWriter, String, OdasaOutput.TrapFileManager) -> Unit
) {
// In order to avoid excessively long signatures which can lead to trap file names longer
// than the filesystem
// limit, we truncate and add a hash to preserve uniqueness if necessary.
val signature = if (possiblyLongSignature.length > 100) {
possiblyLongSignature.substring(0, 92) + "#" + StringDigestor.digest(possiblyLongSignature).substring(0, 8)
} else { possiblyLongSignature }
val signature =
if (possiblyLongSignature.length > 100) {
possiblyLongSignature.substring(0, 92) +
"#" +
StringDigestor.digest(possiblyLongSignature).substring(0, 8)
} else {
possiblyLongSignature
}
output.getTrapLockerForDecl(element, signature, fromSource).useAC { locker ->
locker.trapFileManager.useAC { manager ->
val shortName = when(element) {
is IrDeclarationWithName -> element.name.asString()
is IrFile -> element.name
else -> "(unknown name)"
}
val shortName =
when (element) {
is IrDeclarationWithName -> element.name.asString()
is IrFile -> element.name
else -> "(unknown name)"
}
if (manager == null) {
logger.info("Skipping extracting external decl $shortName")
} else {
val trapFile = manager.file
logger.info("Will write TRAP file $trapFile")
val trapTmpFile = File.createTempFile("${trapFile.nameWithoutExtension}.", ".${trapFile.extension}.tmp", trapFile.parentFile)
val trapTmpFile =
File.createTempFile(
"${trapFile.nameWithoutExtension}.",
".${trapFile.extension}.tmp",
trapFile.parentFile
)
logger.debug("Writing temporary TRAP file $trapTmpFile")
try {
compression.bufferedWriter(trapTmpFile).use {
@@ -77,7 +109,10 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
logger.info("Finished writing TRAP file $trapFile")
} catch (e: Exception) {
manager.setHasError()
logger.error("Failed to extract '$shortName'. Partial TRAP file location is $trapTmpFile", e)
logger.error(
"Failed to extract '$shortName'. Partial TRAP file location is $trapTmpFile",
e
)
}
}
}
@@ -90,37 +125,75 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
externalDeclWorkList.clear()
nextBatch.forEach { workPair ->
val (irDecl, possiblyLongSignature) = workPair
extractElement(irDecl, possiblyLongSignature, false) { trapFileBW, signature, manager ->
extractElement(irDecl, possiblyLongSignature, false) {
trapFileBW,
signature,
manager ->
val binaryPath = getIrDeclarationBinaryPath(irDecl)
if (binaryPath == null) {
logger.errorElement("Unable to get binary path", irDecl)
} else {
// We want our comments to be the first thing in the file,
// so start off with a PlainTrapWriter
val tw = PlainTrapWriter(logger.loggerBase, TrapLabelManager(), trapFileBW, diagnosticTrapWriter)
tw.writeComment("Generated by the CodeQL Kotlin extractor for external dependencies")
val tw =
PlainTrapWriter(
logger.loggerBase,
TrapLabelManager(),
trapFileBW,
diagnosticTrapWriter
)
tw.writeComment(
"Generated by the CodeQL Kotlin extractor for external dependencies"
)
tw.writeComment("Part of invocation $invocationTrapFile")
if (signature != possiblyLongSignature) {
tw.writeComment("Function signature abbreviated; full signature is: $possiblyLongSignature")
tw.writeComment(
"Function signature abbreviated; full signature is: $possiblyLongSignature"
)
}
// Now elevate to a SourceFileTrapWriter, and populate the
// file information if needed:
val ftw = tw.makeFileTrapWriter(binaryPath, true)
val fileExtractor = KotlinFileExtractor(logger, ftw, null, binaryPath, manager, this, primitiveTypeMapping, pluginContext, KotlinFileExtractor.DeclarationStack(), globalExtensionState)
val fileExtractor =
KotlinFileExtractor(
logger,
ftw,
null,
binaryPath,
manager,
this,
primitiveTypeMapping,
pluginContext,
KotlinFileExtractor.DeclarationStack(),
globalExtensionState
)
if (irDecl is IrClass) {
// Populate a location and compilation-unit package for the file. This is similar to
// the beginning of `KotlinFileExtractor.extractFileContents` but without an `IrFile`
// Populate a location and compilation-unit package for the file. This
// is similar to
// the beginning of `KotlinFileExtractor.extractFileContents` but
// without an `IrFile`
// to start from.
val pkg = irDecl.packageFqName?.asString() ?: ""
val pkgId = fileExtractor.extractPackage(pkg)
ftw.writeHasLocation(ftw.fileId, ftw.getWholeFileLocation())
ftw.writeCupackage(ftw.fileId, pkgId)
fileExtractor.extractClassSource(irDecl, extractDeclarations = !irDecl.isFileClass, extractStaticInitializer = false, extractPrivateMembers = false, extractFunctionBodies = false)
fileExtractor.extractClassSource(
irDecl,
extractDeclarations = !irDecl.isFileClass,
extractStaticInitializer = false,
extractPrivateMembers = false,
extractFunctionBodies = false
)
} else {
fileExtractor.extractDeclaration(irDecl, extractPrivateMembers = false, extractFunctionBodies = false, extractAnnotations = true)
fileExtractor.extractDeclaration(
irDecl,
extractPrivateMembers = false,
extractFunctionBodies = false,
extractAnnotations = true
)
}
}
}
@@ -128,5 +201,4 @@ class ExternalDeclExtractor(val logger: FileLogger, val compression: Compression
} while (externalDeclWorkList.isNotEmpty())
output.writeTrapSet()
}
}

114
java/kotlin-extractor/src/main/kotlin/KotlinExtractorCommandLineProcessor.kt
View File

@@ -11,66 +11,90 @@ import org.jetbrains.kotlin.config.CompilerConfigurationKey
class KotlinExtractorCommandLineProcessor : CommandLineProcessor {
override val pluginId = "kotlin-extractor"
override val pluginOptions = listOf(
CliOption(
optionName = OPTION_INVOCATION_TRAP_FILE,
valueDescription = "Invocation TRAP file",
description = "Extractor will append invocation-related TRAP to this file",
required = true,
allowMultipleOccurrences = false
),
CliOption(
optionName = OPTION_CHECK_TRAP_IDENTICAL,
valueDescription = "Check whether different invocations produce identical TRAP",
description = "Check whether different invocations produce identical TRAP",
required = false,
allowMultipleOccurrences = false
),
CliOption(
optionName = OPTION_COMPILATION_STARTTIME,
valueDescription = "The start time of the compilation as a Unix timestamp",
description = "The start time of the compilation as a Unix timestamp",
required = false,
allowMultipleOccurrences = false
),
CliOption(
optionName = OPTION_EXIT_AFTER_EXTRACTION,
valueDescription = "Specify whether to call exitProcess after the extraction has completed",
description = "Specify whether to call exitProcess after the extraction has completed",
required = false,
allowMultipleOccurrences = false
override val pluginOptions =
listOf(
CliOption(
optionName = OPTION_INVOCATION_TRAP_FILE,
valueDescription = "Invocation TRAP file",
description = "Extractor will append invocation-related TRAP to this file",
required = true,
allowMultipleOccurrences = false
),
CliOption(
optionName = OPTION_CHECK_TRAP_IDENTICAL,
valueDescription = "Check whether different invocations produce identical TRAP",
description = "Check whether different invocations produce identical TRAP",
required = false,
allowMultipleOccurrences = false
),
CliOption(
optionName = OPTION_COMPILATION_STARTTIME,
valueDescription = "The start time of the compilation as a Unix timestamp",
description = "The start time of the compilation as a Unix timestamp",
required = false,
allowMultipleOccurrences = false
),
CliOption(
optionName = OPTION_EXIT_AFTER_EXTRACTION,
valueDescription =
"Specify whether to call exitProcess after the extraction has completed",
description =
"Specify whether to call exitProcess after the extraction has completed",
required = false,
allowMultipleOccurrences = false
)
)
)
override fun processOption(
option: AbstractCliOption,
value: String,
configuration: CompilerConfiguration
) = when (option.optionName) {
OPTION_INVOCATION_TRAP_FILE -> configuration.put(KEY_INVOCATION_TRAP_FILE, value)
OPTION_CHECK_TRAP_IDENTICAL -> processBooleanOption(value, OPTION_CHECK_TRAP_IDENTICAL, KEY_CHECK_TRAP_IDENTICAL, configuration)
OPTION_EXIT_AFTER_EXTRACTION -> processBooleanOption(value, OPTION_EXIT_AFTER_EXTRACTION, KEY_EXIT_AFTER_EXTRACTION, configuration)
OPTION_COMPILATION_STARTTIME ->
when (val v = value.toLongOrNull()) {
is Long -> configuration.put(KEY_COMPILATION_STARTTIME, v)
else -> error("kotlin extractor: Bad argument $value for $OPTION_COMPILATION_STARTTIME")
}
else -> error("kotlin extractor: Bad option: ${option.optionName}")
}
) =
when (option.optionName) {
OPTION_INVOCATION_TRAP_FILE -> configuration.put(KEY_INVOCATION_TRAP_FILE, value)
OPTION_CHECK_TRAP_IDENTICAL ->
processBooleanOption(
value,
OPTION_CHECK_TRAP_IDENTICAL,
KEY_CHECK_TRAP_IDENTICAL,
configuration
)
OPTION_EXIT_AFTER_EXTRACTION ->
processBooleanOption(
value,
OPTION_EXIT_AFTER_EXTRACTION,
KEY_EXIT_AFTER_EXTRACTION,
configuration
)
OPTION_COMPILATION_STARTTIME ->
when (val v = value.toLongOrNull()) {
is Long -> configuration.put(KEY_COMPILATION_STARTTIME, v)
else ->
error(
"kotlin extractor: Bad argument $value for $OPTION_COMPILATION_STARTTIME"
)
}
else -> error("kotlin extractor: Bad option: ${option.optionName}")
}
private fun processBooleanOption(value: String, optionName: String, configKey: CompilerConfigurationKey<Boolean>, configuration: CompilerConfiguration) =
private fun processBooleanOption(
value: String,
optionName: String,
configKey: CompilerConfigurationKey<Boolean>,
configuration: CompilerConfiguration
) =
when (value) {
"true" -> configuration.put(configKey, true)
"false" -> configuration.put(configKey, false)
else -> error("kotlin extractor: Bad argument $value for $optionName")
}
}
}
private val OPTION_INVOCATION_TRAP_FILE = "invocationTrapFile"
val KEY_INVOCATION_TRAP_FILE = CompilerConfigurationKey<String>(OPTION_INVOCATION_TRAP_FILE)
private val OPTION_CHECK_TRAP_IDENTICAL = "checkTrapIdentical"
val KEY_CHECK_TRAP_IDENTICAL= CompilerConfigurationKey<Boolean>(OPTION_CHECK_TRAP_IDENTICAL)
val KEY_CHECK_TRAP_IDENTICAL = CompilerConfigurationKey<Boolean>(OPTION_CHECK_TRAP_IDENTICAL)
private val OPTION_COMPILATION_STARTTIME = "compilationStartTime"
val KEY_COMPILATION_STARTTIME= CompilerConfigurationKey<Long>(OPTION_COMPILATION_STARTTIME)
val KEY_COMPILATION_STARTTIME = CompilerConfigurationKey<Long>(OPTION_COMPILATION_STARTTIME)
private val OPTION_EXIT_AFTER_EXTRACTION = "exitAfterExtraction"
val KEY_EXIT_AFTER_EXTRACTION= CompilerConfigurationKey<Boolean>(OPTION_EXIT_AFTER_EXTRACTION)
val KEY_EXIT_AFTER_EXTRACTION = CompilerConfigurationKey<Boolean>(OPTION_EXIT_AFTER_EXTRACTION)

19
java/kotlin-extractor/src/main/kotlin/KotlinExtractorComponentRegistrar.kt
View File

@@ -3,12 +3,9 @@
package com.github.codeql
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
import com.intellij.mock.MockProject
import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.config.CompilerConfiguration
import com.github.codeql.Kotlin2ComponentRegistrar
class KotlinExtractorComponentRegistrar : Kotlin2ComponentRegistrar() {
override fun registerProjectComponents(
@@ -19,10 +16,14 @@ class KotlinExtractorComponentRegistrar : Kotlin2ComponentRegistrar() {
if (invocationTrapFile == null) {
throw Exception("Required argument for TRAP invocation file not given")
}
IrGenerationExtension.registerExtension(project, KotlinExtractorExtension(
invocationTrapFile,
configuration[KEY_CHECK_TRAP_IDENTICAL] ?: false,
configuration[KEY_COMPILATION_STARTTIME],
configuration[KEY_EXIT_AFTER_EXTRACTION] ?: false))
IrGenerationExtension.registerExtension(
project,
KotlinExtractorExtension(
invocationTrapFile,
configuration[KEY_CHECK_TRAP_IDENTICAL] ?: false,
configuration[KEY_COMPILATION_STARTTIME],
configuration[KEY_EXIT_AFTER_EXTRACTION] ?: false
)
)
}
}

237
java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt
View File

@@ -1,15 +1,11 @@
package com.github.codeql
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.*
import org.jetbrains.kotlin.ir.IrElement
import java.io.BufferedReader
import java.io.BufferedWriter
import com.github.codeql.utils.versions.usesK2
import com.semmle.util.files.FileUtil
import java.io.BufferedInputStream
import java.io.BufferedOutputStream
import java.io.BufferedReader
import java.io.BufferedWriter
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
@@ -20,9 +16,12 @@ import java.nio.file.Files
import java.nio.file.Paths
import java.util.zip.GZIPInputStream
import java.util.zip.GZIPOutputStream
import com.github.codeql.utils.versions.usesK2
import com.semmle.util.files.FileUtil
import kotlin.system.exitProcess
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.*
/*
* KotlinExtractorExtension is the main entry point of the CodeQL Kotlin
@@ -55,8 +54,8 @@ class KotlinExtractorExtension(
// can be set to true to make the plugin terminate the kotlinc
// invocation when it has finished. This means that kotlinc will not
// write any `.class` files etc.
private val exitAfterExtraction: Boolean)
: IrGenerationExtension {
private val exitAfterExtraction: Boolean
) : IrGenerationExtension {
// This is the main entry point to the extractor.
// It will be called by kotlinc with the IR for the files being
@@ -65,10 +64,10 @@ class KotlinExtractorExtension(
override fun generate(moduleFragment: IrModuleFragment, pluginContext: IrPluginContext) {
try {
runExtractor(moduleFragment, pluginContext)
// We catch Throwable rather than Exception, as we want to
// continue trying to extract everything else even if we get a
// stack overflow or an assertion failure in one file.
} catch(e: Throwable) {
// We catch Throwable rather than Exception, as we want to
// continue trying to extract everything else even if we get a
// stack overflow or an assertion failure in one file.
} catch (e: Throwable) {
// If we get an exception at the top level, then something's
// gone very wrong. Don't try to be too fancy, but try to
// log a simple message.
@@ -80,7 +79,8 @@ class KotlinExtractorExtension(
// We use a slightly different filename pattern compared
// to normal logs. Just the existence of a `-top` log is
// a sign that something's gone very wrong.
val logFile = File.createTempFile("kotlin-extractor-top.", ".log", File(extractorLogDir))
val logFile =
File.createTempFile("kotlin-extractor-top.", ".log", File(extractorLogDir))
logFile.writeText(msg)
// Now we've got that out, let's see if we can append a stack trace too
logFile.appendText(e.stackTraceToString())
@@ -99,12 +99,18 @@ class KotlinExtractorExtension(
private fun runExtractor(moduleFragment: IrModuleFragment, pluginContext: IrPluginContext) {
val startTimeMs = System.currentTimeMillis()
val usesK2 = usesK2(pluginContext)
// This default should be kept in sync with com.semmle.extractor.java.interceptors.KotlinInterceptor.initializeExtractionContext
val trapDir = File(System.getenv("CODEQL_EXTRACTOR_JAVA_TRAP_DIR").takeUnless { it.isNullOrEmpty() } ?: "kotlin-extractor/trap")
// This default should be kept in sync with
// com.semmle.extractor.java.interceptors.KotlinInterceptor.initializeExtractionContext
val trapDir =
File(
System.getenv("CODEQL_EXTRACTOR_JAVA_TRAP_DIR").takeUnless { it.isNullOrEmpty() }
?: "kotlin-extractor/trap"
)
// The invocation TRAP file will already have been started
// before the plugin is run, so we always use no compression
// and we open it in append mode.
FileOutputStream(File(invocationTrapFile), true).bufferedWriter().use { invocationTrapFileBW ->
FileOutputStream(File(invocationTrapFile), true).bufferedWriter().use { invocationTrapFileBW
->
val invocationExtractionProblems = ExtractionProblems()
val lm = TrapLabelManager()
val logCounter = LogCounter()
@@ -113,12 +119,21 @@ class KotlinExtractorExtension(
// The interceptor has already defined #compilation = *
val compilation: Label<DbCompilation> = StringLabel("compilation")
tw.writeCompilation_started(compilation)
tw.writeCompilation_info(compilation, "Kotlin Compiler Version", KotlinCompilerVersion.getVersion() ?: "<unknown>")
val extractor_name = this::class.java.getResource("extractor.name")?.readText() ?: "<unknown>"
tw.writeCompilation_info(
compilation,
"Kotlin Compiler Version",
KotlinCompilerVersion.getVersion() ?: "<unknown>"
)
val extractor_name =
this::class.java.getResource("extractor.name")?.readText() ?: "<unknown>"
tw.writeCompilation_info(compilation, "Kotlin Extractor Name", extractor_name)
tw.writeCompilation_info(compilation, "Uses Kotlin 2", usesK2.toString())
if (compilationStartTime != null) {
tw.writeCompilation_compiler_times(compilation, -1.0, (System.currentTimeMillis()-compilationStartTime)/1000.0)
tw.writeCompilation_compiler_times(
compilation,
-1.0,
(System.currentTimeMillis() - compilationStartTime) / 1000.0
)
}
tw.flush()
val logger = Logger(loggerBase, tw)
@@ -138,23 +153,54 @@ class KotlinExtractorExtension(
// FIXME: FileUtil expects a static global logger
// which should be provided by SLF4J's factory facility. For now we set it here.
FileUtil.logger = logger
val srcDir = File(System.getenv("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR").takeUnless { it.isNullOrEmpty() } ?: "kotlin-extractor/src")
val srcDir =
File(
System.getenv("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR").takeUnless {
it.isNullOrEmpty()
} ?: "kotlin-extractor/src"
)
srcDir.mkdirs()
val globalExtensionState = KotlinExtractorGlobalState()
moduleFragment.files.mapIndexed { index: Int, file: IrFile ->
val fileExtractionProblems = FileExtractionProblems(invocationExtractionProblems)
val fileTrapWriter = tw.makeSourceFileTrapWriter(file, true)
loggerBase.setFileNumber(index)
fileTrapWriter.writeCompilation_compiling_files(compilation, index, fileTrapWriter.fileId)
doFile(compression, fileExtractionProblems, invocationTrapFile, fileTrapWriter, checkTrapIdentical, loggerBase, trapDir, srcDir, file, primitiveTypeMapping, pluginContext, globalExtensionState)
fileTrapWriter.writeCompilation_compiling_files_completed(compilation, index, fileExtractionProblems.extractionResult())
fileTrapWriter.writeCompilation_compiling_files(
compilation,
index,
fileTrapWriter.fileId
)
doFile(
compression,
fileExtractionProblems,
invocationTrapFile,
fileTrapWriter,
checkTrapIdentical,
loggerBase,
trapDir,
srcDir,
file,
primitiveTypeMapping,
pluginContext,
globalExtensionState
)
fileTrapWriter.writeCompilation_compiling_files_completed(
compilation,
index,
fileExtractionProblems.extractionResult()
)
}
loggerBase.printLimitedDiagnosticCounts(tw)
logPeakMemoryUsage(logger, "after extractor")
logger.info("Extraction completed")
logger.flush()
val compilationTimeMs = System.currentTimeMillis() - startTimeMs
tw.writeCompilation_finished(compilation, -1.0, compilationTimeMs.toDouble() / 1000, invocationExtractionProblems.extractionResult())
tw.writeCompilation_finished(
compilation,
-1.0,
compilationTimeMs.toDouble() / 1000,
invocationExtractionProblems.extractionResult()
)
tw.flush()
loggerBase.close()
}
@@ -170,13 +216,17 @@ class KotlinExtractorExtension(
try {
val compression_option_upper = compression_option.uppercase()
if (compression_option_upper == "BROTLI") {
logger.warn("Kotlin extractor doesn't support Brotli compression. Using GZip instead.")
logger.warn(
"Kotlin extractor doesn't support Brotli compression. Using GZip instead."
)
return Compression.GZIP
} else {
return Compression.valueOf(compression_option_upper)
}
} catch (e: IllegalArgumentException) {
logger.warn("Unsupported compression type (\$$compression_env_var) \"$compression_option\". Supported values are ${Compression.values().joinToString()}.")
logger.warn(
"Unsupported compression type (\$$compression_env_var) \"$compression_option\". Supported values are ${Compression.values().joinToString()}."
)
return defaultCompression
}
}
@@ -190,11 +240,18 @@ class KotlinExtractorExtension(
var nonheap: Long = 0
for (bean in beans) {
val peak = bean.getPeakUsage().getUsed()
val kind = when (bean.getType()) {
MemoryType.HEAP -> { heap += peak; "heap" }
MemoryType.NON_HEAP -> { nonheap += peak; "non-heap" }
else -> "unknown"
}
val kind =
when (bean.getType()) {
MemoryType.HEAP -> {
heap += peak
"heap"
}
MemoryType.NON_HEAP -> {
nonheap += peak
"non-heap"
}
else -> "unknown"
}
logger.info("Peak memory: * Peak for $kind bean ${bean.getName()} is $peak")
}
logger.info("Peak memory: * Total heap peak: $heap")
@@ -203,9 +260,12 @@ class KotlinExtractorExtension(
}
class KotlinExtractorGlobalState {
// These three record mappings of classes, functions and fields that should be replaced wherever they are found.
// As of now these are only used to fix IR generated by the Gradle Android Extensions plugin, hence e.g. IrProperty
// doesn't have a map as that plugin doesn't generate them. If and when these are used more widely additional maps
// These three record mappings of classes, functions and fields that should be replaced wherever
// they are found.
// As of now these are only used to fix IR generated by the Gradle Android Extensions plugin,
// hence e.g. IrProperty
// doesn't have a map as that plugin doesn't generate them. If and when these are used more
// widely additional maps
// should be added here.
val syntheticToRealClassMap = HashMap<IrClass, IrClass?>()
val syntheticToRealFunctionMap = HashMap<IrFunction, IrFunction?>()
@@ -227,13 +287,15 @@ open class ExtractionProblems {
open fun setRecoverableProblem() {
recoverableProblem = true
}
open fun setNonRecoverableProblem() {
nonRecoverableProblem = true
}
fun extractionResult(): Int {
if(nonRecoverableProblem) {
if (nonRecoverableProblem) {
return 2
} else if(recoverableProblem) {
} else if (recoverableProblem) {
return 1
} else {
return 0
@@ -246,11 +308,13 @@ The `FileExtractionProblems` is analogous to `ExtractionProblems`,
except it records whether there were any problems while extracting a
particular source file.
*/
class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblems): ExtractionProblems() {
class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblems) :
ExtractionProblems() {
override fun setRecoverableProblem() {
super.setRecoverableProblem()
invocationExtractionProblems.setRecoverableProblem()
}
override fun setNonRecoverableProblem() {
super.setNonRecoverableProblem()
invocationExtractionProblems.setNonRecoverableProblem()
@@ -265,7 +329,7 @@ identical.
private fun equivalentTrap(r1: BufferedReader, r2: BufferedReader): Boolean {
r1.use { br1 ->
r2.use { br2 ->
while(true) {
while (true) {
val l1 = br1.readLine()
val l2 = br2.readLine()
if (l1 == null && l2 == null) {
@@ -294,7 +358,8 @@ private fun doFile(
srcFile: IrFile,
primitiveTypeMapping: PrimitiveTypeMapping,
pluginContext: IrPluginContext,
globalExtensionState: KotlinExtractorGlobalState) {
globalExtensionState: KotlinExtractorGlobalState
) {
val srcFilePath = srcFile.path
val logger = FileLogger(loggerBase, fileTrapWriter)
logger.info("Extracting file $srcFilePath")
@@ -311,10 +376,13 @@ private fun doFile(
val dbSrcFilePath = Paths.get("$dbSrcDir/$srcFileRelativePath")
val dbSrcDirPath = dbSrcFilePath.parent
Files.createDirectories(dbSrcDirPath)
val srcTmpFile = File.createTempFile(dbSrcFilePath.fileName.toString() + ".", ".src.tmp", dbSrcDirPath.toFile())
srcTmpFile.outputStream().use {
Files.copy(Paths.get(srcFilePath), it)
}
val srcTmpFile =
File.createTempFile(
dbSrcFilePath.fileName.toString() + ".",
".src.tmp",
dbSrcDirPath.toFile()
)
srcTmpFile.outputStream().use { Files.copy(Paths.get(srcFilePath), it) }
srcTmpFile.renameTo(dbSrcFilePath.toFile())
val trapFileName = "$dbTrapDir/$srcFileRelativePath.trap"
@@ -327,22 +395,52 @@ private fun doFile(
trapFileWriter.getTempWriter().use { trapFileBW ->
// We want our comments to be the first thing in the file,
// so start off with a mere TrapWriter
val tw = PlainTrapWriter(loggerBase, TrapLabelManager(), trapFileBW, fileTrapWriter.getDiagnosticTrapWriter())
val tw =
PlainTrapWriter(
loggerBase,
TrapLabelManager(),
trapFileBW,
fileTrapWriter.getDiagnosticTrapWriter()
)
tw.writeComment("Generated by the CodeQL Kotlin extractor for kotlin source code")
tw.writeComment("Part of invocation $invocationTrapFile")
// Now elevate to a SourceFileTrapWriter, and populate the
// file information
val sftw = tw.makeSourceFileTrapWriter(srcFile, true)
val externalDeclExtractor = ExternalDeclExtractor(logger, compression, invocationTrapFile, srcFilePath, primitiveTypeMapping, pluginContext, globalExtensionState, fileTrapWriter.getDiagnosticTrapWriter())
val externalDeclExtractor =
ExternalDeclExtractor(
logger,
compression,
invocationTrapFile,
srcFilePath,
primitiveTypeMapping,
pluginContext,
globalExtensionState,
fileTrapWriter.getDiagnosticTrapWriter()
)
val linesOfCode = LinesOfCode(logger, sftw, srcFile)
val fileExtractor = KotlinFileExtractor(logger, sftw, linesOfCode, srcFilePath, null, externalDeclExtractor, primitiveTypeMapping, pluginContext, KotlinFileExtractor.DeclarationStack(), globalExtensionState)
val fileExtractor =
KotlinFileExtractor(
logger,
sftw,
linesOfCode,
srcFilePath,
null,
externalDeclExtractor,
primitiveTypeMapping,
pluginContext,
KotlinFileExtractor.DeclarationStack(),
globalExtensionState
)
fileExtractor.extractFileContents(srcFile, sftw.fileId)
externalDeclExtractor.extractExternalClasses()
}
if (checkTrapIdentical && trapFileWriter.exists()) {
if (equivalentTrap(trapFileWriter.getTempReader(), trapFileWriter.getRealReader())) {
if (
equivalentTrap(trapFileWriter.getTempReader(), trapFileWriter.getRealReader())
) {
trapFileWriter.deleteTemp()
} else {
trapFileWriter.renameTempToDifferent()
@@ -350,9 +448,9 @@ private fun doFile(
} else {
trapFileWriter.renameTempToReal()
}
// We catch Throwable rather than Exception, as we want to
// continue trying to extract everything else even if we get a
// stack overflow or an assertion failure in one file.
// We catch Throwable rather than Exception, as we want to
// continue trying to extract everything else even if we get a
// stack overflow or an assertion failure in one file.
} catch (e: Throwable) {
logger.error("Failed to extract '$srcFilePath'. " + trapFileWriter.debugInfo(), e)
context.clear()
@@ -372,17 +470,26 @@ enum class Compression(val extension: String) {
return GZIPOutputStream(file.outputStream()).bufferedWriter()
}
};
abstract fun bufferedWriter(file: File): BufferedWriter
}
private fun getTrapFileWriter(compression: Compression, logger: FileLogger, trapFileName: String): TrapFileWriter {
private fun getTrapFileWriter(
compression: Compression,
logger: FileLogger,
trapFileName: String
): TrapFileWriter {
return when (compression) {
Compression.NONE -> NonCompressedTrapFileWriter(logger, trapFileName)
Compression.GZIP -> GZipCompressedTrapFileWriter(logger, trapFileName)
}
}
private abstract class TrapFileWriter(val logger: FileLogger, trapName: String, val extension: String) {
private abstract class TrapFileWriter(
val logger: FileLogger,
trapName: String,
val extension: String
) {
private val realFile = File(trapName + extension)
private val parentDir = realFile.parentFile
lateinit private var tempFile: File
@@ -404,6 +511,7 @@ private abstract class TrapFileWriter(val logger: FileLogger, trapName: String,
}
abstract protected fun getReader(file: File): BufferedReader
abstract protected fun getWriter(file: File): BufferedWriter
fun getRealReader(): BufferedReader {
@@ -431,7 +539,8 @@ private abstract class TrapFileWriter(val logger: FileLogger, trapName: String,
}
fun renameTempToDifferent() {
val trapDifferentFile = File.createTempFile(realFile.getName() + ".", ".trap.different" + extension, parentDir)
val trapDifferentFile =
File.createTempFile(realFile.getName() + ".", ".trap.different" + extension, parentDir)
if (tempFile.renameTo(trapDifferentFile)) {
logger.warn("TRAP difference: $realFile vs $trapDifferentFile")
} else {
@@ -447,7 +556,8 @@ private abstract class TrapFileWriter(val logger: FileLogger, trapName: String,
}
}
private class NonCompressedTrapFileWriter(logger: FileLogger, trapName: String): TrapFileWriter(logger, trapName, "") {
private class NonCompressedTrapFileWriter(logger: FileLogger, trapName: String) :
TrapFileWriter(logger, trapName, "") {
override protected fun getReader(file: File): BufferedReader {
return file.bufferedReader()
}
@@ -457,12 +567,17 @@ private class NonCompressedTrapFileWriter(logger: FileLogger, trapName: String):
}
}
private class GZipCompressedTrapFileWriter(logger: FileLogger, trapName: String): TrapFileWriter(logger, trapName, ".gz") {
private class GZipCompressedTrapFileWriter(logger: FileLogger, trapName: String) :
TrapFileWriter(logger, trapName, ".gz") {
override protected fun getReader(file: File): BufferedReader {
return BufferedReader(InputStreamReader(GZIPInputStream(BufferedInputStream(FileInputStream(file)))))
return BufferedReader(
InputStreamReader(GZIPInputStream(BufferedInputStream(FileInputStream(file))))
)
}
override protected fun getWriter(file: File): BufferedWriter {
return BufferedWriter(OutputStreamWriter(GZIPOutputStream(BufferedOutputStream(FileOutputStream(file)))))
return BufferedWriter(
OutputStreamWriter(GZIPOutputStream(BufferedOutputStream(FileOutputStream(file))))
)
}
}

6532
java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt
View File

File diff suppressed because it is too large Load Diff

1599
java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt
View File

File diff suppressed because it is too large Load Diff

28
java/kotlin-extractor/src/main/kotlin/Label.kt
View File

@@ -1,31 +1,21 @@
package com.github.codeql
import java.io.PrintWriter
import java.io.StringWriter
/**
* This represents a label (`#...`) in a TRAP file.
*/
interface Label<T: AnyDbType> {
fun <U: AnyDbType> cast(): Label<U> {
@Suppress("UNCHECKED_CAST")
return this as Label<U>
/** This represents a label (`#...`) in a TRAP file. */
interface Label<T : AnyDbType> {
fun <U : AnyDbType> cast(): Label<U> {
@Suppress("UNCHECKED_CAST") return this as Label<U>
}
}
/**
* The label `#i`, e.g. `#123`. Most labels we generate are of this
* form.
*/
class IntLabel<T: AnyDbType>(val i: Int): Label<T> {
/** The label `#i`, e.g. `#123`. Most labels we generate are of this form. */
class IntLabel<T : AnyDbType>(val i: Int) : Label<T> {
override fun toString(): String = "#$i"
}
/**
* The label `#name`, e.g. `#compilation`. This is used when labels are
* shared between different components (e.g. when both the interceptor
* and the extractor need to refer to the same label).
* The label `#name`, e.g. `#compilation`. This is used when labels are shared between different
* components (e.g. when both the interceptor and the extractor need to refer to the same label).
*/
class StringLabel<T: AnyDbType>(val name: String): Label<T> {
class StringLabel<T : AnyDbType>(val name: String) : Label<T> {
override fun toString(): String = "#$name"
}

16
java/kotlin-extractor/src/main/kotlin/LinesOfCode.kt
View File

@@ -2,11 +2,7 @@ package com.github.codeql
import org.jetbrains.kotlin.ir.declarations.*
class LinesOfCode(
val logger: FileLogger,
val tw: FileTrapWriter,
val file: IrFile
) {
class LinesOfCode(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
val linesOfCodePSI = LinesOfCodePSI(logger, tw, file)
val linesOfCodeLighterAST = LinesOfCodeLighterAST(logger, tw, file)
@@ -14,7 +10,10 @@ class LinesOfCode(
val psiExtracted = linesOfCodePSI.linesOfCodeInFile(id)
val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInFile(id)
if (psiExtracted && lighterASTExtracted) {
logger.warnElement("Both PSI and LighterAST number-of-lines-in-file information for ${file.path}.", file)
logger.warnElement(
"Both PSI and LighterAST number-of-lines-in-file information for ${file.path}.",
file
)
}
}
@@ -22,7 +21,10 @@ class LinesOfCode(
val psiExtracted = linesOfCodePSI.linesOfCodeInDeclaration(d, id)
val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInDeclaration(d, id)
if (psiExtracted && lighterASTExtracted) {
logger.warnElement("Both PSI and LighterAST number-of-lines-in-file information for declaration.", d)
logger.warnElement(
"Both PSI and LighterAST number-of-lines-in-file information for declaration.",
d
)
}
}
}

19
java/kotlin-extractor/src/main/kotlin/LinesOfCodePSI.kt
View File

@@ -7,20 +7,17 @@ import com.intellij.psi.PsiWhiteSpace
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.kdoc.psi.api.KDocElement
import org.jetbrains.kotlin.psi.KtCodeFragment
import org.jetbrains.kotlin.psi.KtVisitor
class LinesOfCodePSI(
val logger: FileLogger,
val tw: FileTrapWriter,
val file: IrFile
) {
val psi2Ir = getPsi2Ir().also {
if (it == null) {
logger.warn("Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
class LinesOfCodePSI(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
val psi2Ir =
getPsi2Ir().also {
if (it == null) {
logger.warn(
"Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})"
)
}
}
}
fun linesOfCodeInFile(id: Label<DbFile>): Boolean {
if (psi2Ir == null) {

413
java/kotlin-extractor/src/main/kotlin/MetaAnnotationSupport.kt
View File

@@ -3,6 +3,8 @@ package com.github.codeql
import com.github.codeql.utils.versions.copyParameterToFunction
import com.github.codeql.utils.versions.createImplicitParameterDeclarationWithWrappedDescriptor
import com.github.codeql.utils.versions.getAnnotationType
import java.lang.annotation.ElementType
import java.util.HashSet
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.builtins.StandardNames
import org.jetbrains.kotlin.config.JvmTarget
@@ -48,10 +50,12 @@ import org.jetbrains.kotlin.load.java.JvmAnnotationNames
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
import java.lang.annotation.ElementType
import java.util.HashSet
class MetaAnnotationSupport(private val logger: FileLogger, private val pluginContext: IrPluginContext, private val extractor: KotlinFileExtractor) {
class MetaAnnotationSupport(
private val logger: FileLogger,
private val pluginContext: IrPluginContext,
private val extractor: KotlinFileExtractor
) {
// Taken from AdditionalIrUtils.kt (not available in Kotlin < 1.6)
private val IrConstructorCall.annotationClass
@@ -82,8 +86,7 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
wrapAnnotationEntriesInContainer(annotationClass, containerClass, grouped)?.let {
result.add(it)
}
else
logger.warnElement("Failed to find an annotation container class", annotationClass)
else logger.warnElement("Failed to find an annotation container class", annotationClass)
}
return result
}
@@ -91,9 +94,14 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
// Adapted from RepeatedAnnotationLowering.kt
private fun getOrCreateContainerClass(annotationClass: IrClass): IrClass? {
val metaAnnotations = annotationClass.annotations
val jvmRepeatable = metaAnnotations.find { it.symbol.owner.parentAsClass.fqNameWhenAvailable == JvmAnnotationNames.REPEATABLE_ANNOTATION }
val jvmRepeatable =
metaAnnotations.find {
it.symbol.owner.parentAsClass.fqNameWhenAvailable ==
JvmAnnotationNames.REPEATABLE_ANNOTATION
}
return if (jvmRepeatable != null) {
((jvmRepeatable.getValueArgument(0) as? IrClassReference)?.symbol as? IrClassSymbol)?.owner
((jvmRepeatable.getValueArgument(0) as? IrClassReference)?.symbol as? IrClassSymbol)
?.owner
} else {
getOrCreateSyntheticRepeatableAnnotationContainer(annotationClass)
}
@@ -108,20 +116,28 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
val annotationType = annotationClass.typeWith()
val containerConstructor = containerClass.primaryConstructor
if (containerConstructor == null) {
logger.warnElement("Expected container class to have a primary constructor", containerClass)
logger.warnElement(
"Expected container class to have a primary constructor",
containerClass
)
return null
} else {
return IrConstructorCallImpl.fromSymbolOwner(containerClass.defaultType, containerConstructor.symbol).apply {
putValueArgument(
0,
IrVarargImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
pluginContext.irBuiltIns.arrayClass.typeWith(annotationType),
annotationType,
entries
)
return IrConstructorCallImpl.fromSymbolOwner(
containerClass.defaultType,
containerConstructor.symbol
)
}
.apply {
putValueArgument(
0,
IrVarargImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
pluginContext.irBuiltIns.arrayClass.typeWith(annotationType),
annotationType,
entries
)
)
}
}
}
@@ -134,14 +150,19 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
// Taken from AdditionalClassAnnotationLowering.kt
private fun loadAnnotationTargets(targetEntry: IrConstructorCall): Set<KotlinTarget>? {
val valueArgument = targetEntry.getValueArgument(0) as? IrVararg ?: return null
return valueArgument.elements.filterIsInstance<IrGetEnumValue>().mapNotNull {
KotlinTarget.valueOrNull(it.symbol.owner.name.asString())
}.toSet()
return valueArgument.elements
.filterIsInstance<IrGetEnumValue>()
.mapNotNull { KotlinTarget.valueOrNull(it.symbol.owner.name.asString()) }
.toSet()
}
private val javaAnnotationTargetElementType by lazy { extractor.referenceExternalClass("java.lang.annotation.ElementType") }
private val javaAnnotationTargetElementType by lazy {
extractor.referenceExternalClass("java.lang.annotation.ElementType")
}
private val javaAnnotationTarget by lazy { extractor.referenceExternalClass("java.lang.annotation.Target") }
private val javaAnnotationTarget by lazy {
extractor.referenceExternalClass("java.lang.annotation.Target")
}
private fun findEnumEntry(c: IrClass, name: String) =
c.declarations.filterIsInstance<IrEnumEntry>().find { it.name.asString() == name }
@@ -168,10 +189,11 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
private val jvm8TargetMap by lazy {
javaAnnotationTargetElementType?.let {
jvm6TargetMap?.let { j6Map ->
j6Map + mapOf(
KotlinTarget.TYPE_PARAMETER to findEnumEntry(it, "TYPE_PARAMETER"),
KotlinTarget.TYPE to findEnumEntry(it, "TYPE_USE")
)
j6Map +
mapOf(
KotlinTarget.TYPE_PARAMETER to findEnumEntry(it, "TYPE_PARAMETER"),
KotlinTarget.TYPE to findEnumEntry(it, "TYPE_USE")
)
}
}
}
@@ -179,45 +201,59 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
private fun getAnnotationTargetMap() =
if (pluginContext.platform?.any { it.targetPlatformVersion == JvmTarget.JVM_1_6 } == true)
jvm6TargetMap
else
jvm8TargetMap
else jvm8TargetMap
// Adapted from AdditionalClassAnnotationLowering.kt
private fun generateTargetAnnotation(c: IrClass): IrConstructorCall? {
if (c.hasAnnotation(JvmAnnotationNames.TARGET_ANNOTATION))
return null
if (c.hasAnnotation(JvmAnnotationNames.TARGET_ANNOTATION)) return null
val elementType = javaAnnotationTargetElementType ?: return null
val targetType = javaAnnotationTarget ?: return null
val targetConstructor = targetType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null
val targetConstructor =
targetType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null
val targets = getApplicableTargetSet(c) ?: return null
val annotationTargetMap = getAnnotationTargetMap() ?: return null
val javaTargets = targets.mapNotNullTo(HashSet()) { annotationTargetMap[it] }.sortedBy {
ElementType.valueOf(it.symbol.owner.name.asString())
}
val vararg = IrVarargImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
type = pluginContext.irBuiltIns.arrayClass.typeWith(elementType.defaultType),
varargElementType = elementType.defaultType
)
val javaTargets =
targets
.mapNotNullTo(HashSet()) { annotationTargetMap[it] }
.sortedBy { ElementType.valueOf(it.symbol.owner.name.asString()) }
val vararg =
IrVarargImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
type = pluginContext.irBuiltIns.arrayClass.typeWith(elementType.defaultType),
varargElementType = elementType.defaultType
)
for (target in javaTargets) {
vararg.elements.add(
IrGetEnumValueImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, elementType.defaultType, target.symbol
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
elementType.defaultType,
target.symbol
)
)
}
return IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, targetConstructor.returnType, targetConstructor.symbol, 0
).apply {
putValueArgument(0, vararg)
}
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
targetConstructor.returnType,
targetConstructor.symbol,
0
)
.apply { putValueArgument(0, vararg) }
}
private val javaAnnotationRetention by lazy { extractor.referenceExternalClass("java.lang.annotation.Retention") }
private val javaAnnotationRetentionPolicy by lazy { extractor.referenceExternalClass("java.lang.annotation.RetentionPolicy") }
private val javaAnnotationRetentionPolicyRuntime by lazy { javaAnnotationRetentionPolicy?.let { findEnumEntry(it, "RUNTIME") } }
private val javaAnnotationRetention by lazy {
extractor.referenceExternalClass("java.lang.annotation.Retention")
}
private val javaAnnotationRetentionPolicy by lazy {
extractor.referenceExternalClass("java.lang.annotation.RetentionPolicy")
}
private val javaAnnotationRetentionPolicyRuntime by lazy {
javaAnnotationRetentionPolicy?.let { findEnumEntry(it, "RUNTIME") }
}
private val annotationRetentionMap by lazy {
javaAnnotationRetentionPolicy?.let {
@@ -232,118 +268,164 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
// Taken from AnnotationCodegen.kt (not available in Kotlin < 1.6.20)
private fun IrClass.getAnnotationRetention(): KotlinRetention? {
val retentionArgument =
getAnnotation(StandardNames.FqNames.retention)?.getValueArgument(0)
as? IrGetEnumValue ?: return null
getAnnotation(StandardNames.FqNames.retention)?.getValueArgument(0) as? IrGetEnumValue
?: return null
val retentionArgumentValue = retentionArgument.symbol.owner
return KotlinRetention.valueOf(retentionArgumentValue.name.asString())
}
// Taken from AdditionalClassAnnotationLowering.kt
private fun generateRetentionAnnotation(irClass: IrClass): IrConstructorCall? {
if (irClass.hasAnnotation(JvmAnnotationNames.RETENTION_ANNOTATION))
return null
if (irClass.hasAnnotation(JvmAnnotationNames.RETENTION_ANNOTATION)) return null
val retentionMap = annotationRetentionMap ?: return null
val kotlinRetentionPolicy = irClass.getAnnotationRetention()
val javaRetentionPolicy = kotlinRetentionPolicy?.let { retentionMap[it] } ?: javaAnnotationRetentionPolicyRuntime ?: return null
val javaRetentionPolicy =
kotlinRetentionPolicy?.let { retentionMap[it] }
?: javaAnnotationRetentionPolicyRuntime
?: return null
val retentionPolicyType = javaAnnotationRetentionPolicy ?: return null
val retentionType = javaAnnotationRetention ?: return null
val targetConstructor = retentionType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null
val targetConstructor =
retentionType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null
return IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, targetConstructor.returnType, targetConstructor.symbol, 0
).apply {
putValueArgument(
0,
IrGetEnumValueImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, retentionPolicyType.defaultType, javaRetentionPolicy.symbol
)
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
targetConstructor.returnType,
targetConstructor.symbol,
0
)
}
.apply {
putValueArgument(
0,
IrGetEnumValueImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
retentionPolicyType.defaultType,
javaRetentionPolicy.symbol
)
)
}
}
private val javaAnnotationRepeatable by lazy { extractor.referenceExternalClass("java.lang.annotation.Repeatable") }
private val kotlinAnnotationRepeatableContainer by lazy { extractor.referenceExternalClass("kotlin.jvm.internal.RepeatableContainer") }
private val javaAnnotationRepeatable by lazy {
extractor.referenceExternalClass("java.lang.annotation.Repeatable")
}
private val kotlinAnnotationRepeatableContainer by lazy {
extractor.referenceExternalClass("kotlin.jvm.internal.RepeatableContainer")
}
// Taken from declarationBuilders.kt (not available in Kotlin < 1.6):
private fun addDefaultGetter(p: IrProperty, parentClass: IrClass) {
val field = p.backingField ?:
run { logger.warnElement("Expected property to have a backing field", p); return }
val field =
p.backingField
?: run {
logger.warnElement("Expected property to have a backing field", p)
return
}
p.addGetter {
origin = IrDeclarationOrigin.DEFAULT_PROPERTY_ACCESSOR
returnType = field.type
}.apply {
val thisReceiever = parentClass.thisReceiver ?:
run { logger.warnElement("Expected property's parent class to have a receiver parameter", parentClass); return }
val newParam = copyParameterToFunction(thisReceiever, this)
dispatchReceiverParameter = newParam
body = factory.createBlockBody(UNDEFINED_OFFSET, UNDEFINED_OFFSET).apply({
this.statements.add(
IrReturnImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
pluginContext.irBuiltIns.nothingType,
symbol,
IrGetFieldImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
field.symbol,
field.type,
IrGetValueImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET,
newParam.type,
newParam.symbol
origin = IrDeclarationOrigin.DEFAULT_PROPERTY_ACCESSOR
returnType = field.type
}
.apply {
val thisReceiever =
parentClass.thisReceiver
?: run {
logger.warnElement(
"Expected property's parent class to have a receiver parameter",
parentClass
)
)
)
)
})
}
return
}
val newParam = copyParameterToFunction(thisReceiever, this)
dispatchReceiverParameter = newParam
body =
factory
.createBlockBody(UNDEFINED_OFFSET, UNDEFINED_OFFSET)
.apply({
this.statements.add(
IrReturnImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
pluginContext.irBuiltIns.nothingType,
symbol,
IrGetFieldImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
field.symbol,
field.type,
IrGetValueImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
newParam.type,
newParam.symbol
)
)
)
)
})
}
}
// Taken from JvmCachedDeclarations.kt
private fun getOrCreateSyntheticRepeatableAnnotationContainer(annotationClass: IrClass) =
extractor.globalExtensionState.syntheticRepeatableAnnotationContainers.getOrPut(annotationClass) {
val containerClass = pluginContext.irFactory.buildClass {
kind = ClassKind.ANNOTATION_CLASS
name = Name.identifier("Container")
}.apply {
createImplicitParameterDeclarationWithWrappedDescriptor()
parent = annotationClass
superTypes = listOf(getAnnotationType(pluginContext))
}
extractor.globalExtensionState.syntheticRepeatableAnnotationContainers.getOrPut(
annotationClass
) {
val containerClass =
pluginContext.irFactory
.buildClass {
kind = ClassKind.ANNOTATION_CLASS
name = Name.identifier("Container")
}
.apply {
createImplicitParameterDeclarationWithWrappedDescriptor()
parent = annotationClass
superTypes = listOf(getAnnotationType(pluginContext))
}
val propertyName = Name.identifier("value")
val propertyType = pluginContext.irBuiltIns.arrayClass.typeWith(annotationClass.typeWith())
val propertyType =
pluginContext.irBuiltIns.arrayClass.typeWith(annotationClass.typeWith())
containerClass.addConstructor {
isPrimary = true
}.apply {
addValueParameter(propertyName.identifier, propertyType)
}
containerClass
.addConstructor { isPrimary = true }
.apply { addValueParameter(propertyName.identifier, propertyType) }
containerClass.addProperty {
name = propertyName
}.apply property@{
backingField = pluginContext.irFactory.buildField {
name = propertyName
type = propertyType
}.apply {
parent = containerClass
correspondingPropertySymbol = this@property.symbol
containerClass
.addProperty { name = propertyName }
.apply property@{
backingField =
pluginContext.irFactory
.buildField {
name = propertyName
type = propertyType
}
.apply {
parent = containerClass
correspondingPropertySymbol = this@property.symbol
}
addDefaultGetter(this, containerClass)
}
addDefaultGetter(this, containerClass)
}
val repeatableContainerAnnotation = kotlinAnnotationRepeatableContainer?.constructors?.single()
val repeatableContainerAnnotation =
kotlinAnnotationRepeatableContainer?.constructors?.single()
containerClass.annotations = annotationClass.annotations
.filter {
it.isAnnotationWithEqualFqName(StandardNames.FqNames.retention) ||
containerClass.annotations =
annotationClass.annotations
.filter {
it.isAnnotationWithEqualFqName(StandardNames.FqNames.retention) ||
it.isAnnotationWithEqualFqName(StandardNames.FqNames.target)
}
.map { it.deepCopyWithSymbols(containerClass) } +
}
.map { it.deepCopyWithSymbols(containerClass) } +
listOfNotNull(
repeatableContainerAnnotation?.let {
IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, it.returnType, it.symbol, 0
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
it.returnType,
it.symbol,
0
)
}
)
@@ -352,45 +434,80 @@ class MetaAnnotationSupport(private val logger: FileLogger, private val pluginCo
}
// Adapted from AdditionalClassAnnotationLowering.kt
private fun generateRepeatableAnnotation(irClass: IrClass, extractAnnotationTypeAccesses: Boolean): IrConstructorCall? {
if (!irClass.hasAnnotation(StandardNames.FqNames.repeatable) ||
irClass.hasAnnotation(JvmAnnotationNames.REPEATABLE_ANNOTATION)
) return null
private fun generateRepeatableAnnotation(
irClass: IrClass,
extractAnnotationTypeAccesses: Boolean
): IrConstructorCall? {
if (
!irClass.hasAnnotation(StandardNames.FqNames.repeatable) ||
irClass.hasAnnotation(JvmAnnotationNames.REPEATABLE_ANNOTATION)
)
return null
val repeatableConstructor = javaAnnotationRepeatable?.declarations?.firstIsInstanceOrNull<IrConstructor>() ?: return null
val repeatableConstructor =
javaAnnotationRepeatable?.declarations?.firstIsInstanceOrNull<IrConstructor>()
?: return null
val containerClass = getOrCreateSyntheticRepeatableAnnotationContainer(irClass)
// Whenever a repeatable annotation with a Kotlin-synthesised container is extracted, extract the synthetic container to the same trap file.
extractor.extractClassSource(containerClass, extractDeclarations = true, extractStaticInitializer = true, extractPrivateMembers = true, extractFunctionBodies = extractAnnotationTypeAccesses)
val containerReference = IrClassReferenceImpl(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, pluginContext.irBuiltIns.kClassClass.typeWith(containerClass.defaultType),
containerClass.symbol, containerClass.defaultType
// Whenever a repeatable annotation with a Kotlin-synthesised container is extracted,
// extract the synthetic container to the same trap file.
extractor.extractClassSource(
containerClass,
extractDeclarations = true,
extractStaticInitializer = true,
extractPrivateMembers = true,
extractFunctionBodies = extractAnnotationTypeAccesses
)
val containerReference =
IrClassReferenceImpl(
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
pluginContext.irBuiltIns.kClassClass.typeWith(containerClass.defaultType),
containerClass.symbol,
containerClass.defaultType
)
return IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, repeatableConstructor.returnType, repeatableConstructor.symbol, 0
).apply {
putValueArgument(0, containerReference)
}
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
repeatableConstructor.returnType,
repeatableConstructor.symbol,
0
)
.apply { putValueArgument(0, containerReference) }
}
private val javaAnnotationDocumented by lazy { extractor.referenceExternalClass("java.lang.annotation.Documented") }
private val javaAnnotationDocumented by lazy {
extractor.referenceExternalClass("java.lang.annotation.Documented")
}
// Taken from AdditionalClassAnnotationLowering.kt
private fun generateDocumentedAnnotation(irClass: IrClass): IrConstructorCall? {
if (!irClass.hasAnnotation(StandardNames.FqNames.mustBeDocumented) ||
irClass.hasAnnotation(JvmAnnotationNames.DOCUMENTED_ANNOTATION)
) return null
if (
!irClass.hasAnnotation(StandardNames.FqNames.mustBeDocumented) ||
irClass.hasAnnotation(JvmAnnotationNames.DOCUMENTED_ANNOTATION)
)
return null
val documentedConstructor = javaAnnotationDocumented?.declarations?.firstIsInstanceOrNull<IrConstructor>() ?: return null
val documentedConstructor =
javaAnnotationDocumented?.declarations?.firstIsInstanceOrNull<IrConstructor>()
?: return null
return IrConstructorCallImpl.fromSymbolOwner(
UNDEFINED_OFFSET, UNDEFINED_OFFSET, documentedConstructor.returnType, documentedConstructor.symbol, 0
UNDEFINED_OFFSET,
UNDEFINED_OFFSET,
documentedConstructor.returnType,
documentedConstructor.symbol,
0
)
}
fun generateJavaMetaAnnotations(c: IrClass, extractAnnotationTypeAccesses: Boolean) =
// This is essentially AdditionalClassAnnotationLowering adapted to run outside the backend.
listOfNotNull(generateTargetAnnotation(c), generateRetentionAnnotation(c), generateRepeatableAnnotation(c, extractAnnotationTypeAccesses), generateDocumentedAnnotation(c))
listOfNotNull(
generateTargetAnnotation(c),
generateRetentionAnnotation(c),
generateRepeatableAnnotation(c, extractAnnotationTypeAccesses),
generateDocumentedAnnotation(c)
)
}

122
java/kotlin-extractor/src/main/kotlin/PrimitiveTypeInfo.kt
View File

@@ -1,93 +1,105 @@
package com.github.codeql
import com.github.codeql.utils.*
import com.github.codeql.utils.versions.*
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.builtins.StandardNames
import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.ir.declarations.IrPackageFragment
import org.jetbrains.kotlin.ir.types.IrSimpleType
import org.jetbrains.kotlin.ir.types.classOrNull
import org.jetbrains.kotlin.name.FqName
import com.github.codeql.utils.*
import com.github.codeql.utils.versions.*
class PrimitiveTypeMapping(val logger: Logger, val pluginContext: IrPluginContext) {
fun getPrimitiveInfo(s: IrSimpleType) =
s.classOrNull?.let {
if ((it.owner.parent as? IrPackageFragment)?.packageFqName == StandardNames.BUILT_INS_PACKAGE_FQ_NAME)
if (
(it.owner.parent as? IrPackageFragment)?.packageFqName ==
StandardNames.BUILT_INS_PACKAGE_FQ_NAME
)
mapping[it.owner.name]
else
null
else null
}
data class PrimitiveTypeInfo(
val primitiveName: String?,
val otherIsPrimitive: Boolean,
val javaClass: IrClass,
val kotlinPackageName: String, val kotlinClassName: String
val kotlinPackageName: String,
val kotlinClassName: String
)
private fun findClass(fqName: String, fallback: IrClass): IrClass {
val symbol = getClassByFqName(pluginContext, fqName)
if(symbol == null) {
if (symbol == null) {
logger.warn("Can't find $fqName")
// Do the best we can
return fallback
} else {
return symbol.owner
return symbol.owner
}
}
private val mapping = {
val kotlinByte = pluginContext.irBuiltIns.byteClass.owner
val javaLangByte = findClass("java.lang.Byte", kotlinByte)
val kotlinShort = pluginContext.irBuiltIns.shortClass.owner
val javaLangShort = findClass("java.lang.Short", kotlinShort)
val kotlinInt = pluginContext.irBuiltIns.intClass.owner
val javaLangInteger = findClass("java.lang.Integer", kotlinInt)
val kotlinLong = pluginContext.irBuiltIns.longClass.owner
val javaLangLong = findClass("java.lang.Long", kotlinLong)
private val mapping =
{
val kotlinByte = pluginContext.irBuiltIns.byteClass.owner
val javaLangByte = findClass("java.lang.Byte", kotlinByte)
val kotlinShort = pluginContext.irBuiltIns.shortClass.owner
val javaLangShort = findClass("java.lang.Short", kotlinShort)
val kotlinInt = pluginContext.irBuiltIns.intClass.owner
val javaLangInteger = findClass("java.lang.Integer", kotlinInt)
val kotlinLong = pluginContext.irBuiltIns.longClass.owner
val javaLangLong = findClass("java.lang.Long", kotlinLong)
val kotlinUByte = findClass("kotlin.UByte", kotlinByte)
val kotlinUShort = findClass("kotlin.UShort", kotlinShort)
val kotlinUInt = findClass("kotlin.UInt", kotlinInt)
val kotlinULong = findClass("kotlin.ULong", kotlinLong)
val kotlinUByte = findClass("kotlin.UByte", kotlinByte)
val kotlinUShort = findClass("kotlin.UShort", kotlinShort)
val kotlinUInt = findClass("kotlin.UInt", kotlinInt)
val kotlinULong = findClass("kotlin.ULong", kotlinLong)
val kotlinDouble = pluginContext.irBuiltIns.doubleClass.owner
val javaLangDouble = findClass("java.lang.Double", kotlinDouble)
val kotlinFloat = pluginContext.irBuiltIns.floatClass.owner
val javaLangFloat = findClass("java.lang.Float", kotlinFloat)
val kotlinDouble = pluginContext.irBuiltIns.doubleClass.owner
val javaLangDouble = findClass("java.lang.Double", kotlinDouble)
val kotlinFloat = pluginContext.irBuiltIns.floatClass.owner
val javaLangFloat = findClass("java.lang.Float", kotlinFloat)
val kotlinBoolean = pluginContext.irBuiltIns.booleanClass.owner
val javaLangBoolean = findClass("java.lang.Boolean", kotlinBoolean)
val kotlinBoolean = pluginContext.irBuiltIns.booleanClass.owner
val javaLangBoolean = findClass("java.lang.Boolean", kotlinBoolean)
val kotlinChar = pluginContext.irBuiltIns.charClass.owner
val javaLangCharacter = findClass("java.lang.Character", kotlinChar)
val kotlinChar = pluginContext.irBuiltIns.charClass.owner
val javaLangCharacter = findClass("java.lang.Character", kotlinChar)
val kotlinUnit = pluginContext.irBuiltIns.unitClass.owner
val kotlinUnit = pluginContext.irBuiltIns.unitClass.owner
val kotlinNothing = pluginContext.irBuiltIns.nothingClass.owner
val javaLangVoid = findClass("java.lang.Void", kotlinNothing)
val kotlinNothing = pluginContext.irBuiltIns.nothingClass.owner
val javaLangVoid = findClass("java.lang.Void", kotlinNothing)
mapOf(
StandardNames.FqNames._byte.shortName() to PrimitiveTypeInfo("byte", true, javaLangByte, "kotlin", "Byte"),
StandardNames.FqNames._short.shortName() to PrimitiveTypeInfo("short", true, javaLangShort, "kotlin", "Short"),
StandardNames.FqNames._int.shortName() to PrimitiveTypeInfo("int", true, javaLangInteger, "kotlin", "Int"),
StandardNames.FqNames._long.shortName() to PrimitiveTypeInfo("long", true, javaLangLong, "kotlin", "Long"),
StandardNames.FqNames.uByteFqName.shortName() to PrimitiveTypeInfo("byte", true, kotlinUByte, "kotlin", "UByte"),
StandardNames.FqNames.uShortFqName.shortName() to PrimitiveTypeInfo("short", true, kotlinUShort, "kotlin", "UShort"),
StandardNames.FqNames.uIntFqName.shortName() to PrimitiveTypeInfo("int", true, kotlinUInt, "kotlin", "UInt"),
StandardNames.FqNames.uLongFqName.shortName() to PrimitiveTypeInfo("long", true, kotlinULong, "kotlin", "ULong"),
StandardNames.FqNames._double.shortName() to PrimitiveTypeInfo("double", true, javaLangDouble, "kotlin", "Double"),
StandardNames.FqNames._float.shortName() to PrimitiveTypeInfo("float", true, javaLangFloat, "kotlin", "Float"),
StandardNames.FqNames._boolean.shortName() to PrimitiveTypeInfo("boolean", true, javaLangBoolean, "kotlin", "Boolean"),
StandardNames.FqNames._char.shortName() to PrimitiveTypeInfo("char", true, javaLangCharacter, "kotlin", "Char"),
StandardNames.FqNames.unit.shortName() to PrimitiveTypeInfo("void", false, kotlinUnit, "kotlin", "Unit"),
StandardNames.FqNames.nothing.shortName() to PrimitiveTypeInfo(null, true, javaLangVoid, "kotlin", "Nothing"),
)
}()
mapOf(
StandardNames.FqNames._byte.shortName() to
PrimitiveTypeInfo("byte", true, javaLangByte, "kotlin", "Byte"),
StandardNames.FqNames._short.shortName() to
PrimitiveTypeInfo("short", true, javaLangShort, "kotlin", "Short"),
StandardNames.FqNames._int.shortName() to
PrimitiveTypeInfo("int", true, javaLangInteger, "kotlin", "Int"),
StandardNames.FqNames._long.shortName() to
PrimitiveTypeInfo("long", true, javaLangLong, "kotlin", "Long"),
StandardNames.FqNames.uByteFqName.shortName() to
PrimitiveTypeInfo("byte", true, kotlinUByte, "kotlin", "UByte"),
StandardNames.FqNames.uShortFqName.shortName() to
PrimitiveTypeInfo("short", true, kotlinUShort, "kotlin", "UShort"),
StandardNames.FqNames.uIntFqName.shortName() to
PrimitiveTypeInfo("int", true, kotlinUInt, "kotlin", "UInt"),
StandardNames.FqNames.uLongFqName.shortName() to
PrimitiveTypeInfo("long", true, kotlinULong, "kotlin", "ULong"),
StandardNames.FqNames._double.shortName() to
PrimitiveTypeInfo("double", true, javaLangDouble, "kotlin", "Double"),
StandardNames.FqNames._float.shortName() to
PrimitiveTypeInfo("float", true, javaLangFloat, "kotlin", "Float"),
StandardNames.FqNames._boolean.shortName() to
PrimitiveTypeInfo("boolean", true, javaLangBoolean, "kotlin", "Boolean"),
StandardNames.FqNames._char.shortName() to
PrimitiveTypeInfo("char", true, javaLangCharacter, "kotlin", "Char"),
StandardNames.FqNames.unit.shortName() to
PrimitiveTypeInfo("void", false, kotlinUnit, "kotlin", "Unit"),
StandardNames.FqNames.nothing.shortName() to
PrimitiveTypeInfo(null, true, javaLangVoid, "kotlin", "Nothing"),
)
}()
}

327
java/kotlin-extractor/src/main/kotlin/TrapWriter.kt
View File

@@ -1,92 +1,87 @@
package com.github.codeql
import com.github.codeql.KotlinUsesExtractor.LocallyVisibleFunctionLabels
import com.semmle.extractor.java.PopulateFile
import com.semmle.util.unicode.UTF8Util
import java.io.BufferedWriter
import java.io.File
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.path
import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.ir.declarations.IrFunction
import org.jetbrains.kotlin.ir.declarations.IrVariable
import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
import org.jetbrains.kotlin.ir.declarations.path
import org.jetbrains.kotlin.ir.expressions.IrCall
import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET
import com.semmle.extractor.java.PopulateFile
import com.semmle.util.unicode.UTF8Util
import org.jetbrains.kotlin.ir.expressions.IrCall
/**
* Each `.trap` file has a `TrapLabelManager` while we are writing it.
* It provides fresh TRAP label names, and maintains a mapping from keys
* (`@"..."`) to labels.
* Each `.trap` file has a `TrapLabelManager` while we are writing it. It provides fresh TRAP label
* names, and maintains a mapping from keys (`@"..."`) to labels.
*/
class TrapLabelManager {
/** The next integer to use as a label name. */
private var nextInt: Int = 100
/** Returns a fresh label. */
fun <T: AnyDbType> getFreshLabel(): Label<T> {
fun <T : AnyDbType> getFreshLabel(): Label<T> {
return IntLabel(nextInt++)
}
/**
* A mapping from a key (`@"..."`) to the label defined to be that
* key, if any.
*/
/** A mapping from a key (`@"..."`) to the label defined to be that key, if any. */
val labelMapping: MutableMap<String, Label<*>> = mutableMapOf<String, Label<*>>()
val anonymousTypeMapping: MutableMap<IrClass, TypeResults> = mutableMapOf()
val locallyVisibleFunctionLabelMapping: MutableMap<IrFunction, LocallyVisibleFunctionLabels> = mutableMapOf()
val locallyVisibleFunctionLabelMapping: MutableMap<IrFunction, LocallyVisibleFunctionLabels> =
mutableMapOf()
/**
* The set of labels of generic specialisations that we have extracted
* in this TRAP file.
* We can't easily avoid duplication between TRAP files, as the labels
* contain references to other labels, so we just accept this
* duplication.
* The set of labels of generic specialisations that we have extracted in this TRAP file. We
* can't easily avoid duplication between TRAP files, as the labels contain references to other
* labels, so we just accept this duplication.
*/
val genericSpecialisationsExtracted = HashSet<String>()
/**
* Sometimes, when we extract a file class we don't have the IrFile
* for it, so we are not able to give it a location. This means that
* the location is written outside of the label creation.
* This allows us to keep track of whether we've written the location
* already in this TRAP file, to avoid duplication.
* Sometimes, when we extract a file class we don't have the IrFile for it, so we are not able
* to give it a location. This means that the location is written outside of the label creation.
* This allows us to keep track of whether we've written the location already in this TRAP file,
* to avoid duplication.
*/
val fileClassLocationsExtracted = HashSet<IrFile>()
}
/**
* A `TrapWriter` is used to write TRAP to a particular TRAP file.
* There may be multiple `TrapWriter`s for the same file, as different
* instances will have different additional state, but they must all
* share the same `TrapLabelManager` and `BufferedWriter`.
* A `TrapWriter` is used to write TRAP to a particular TRAP file. There may be multiple
* `TrapWriter`s for the same file, as different instances will have different additional state, but
* they must all share the same `TrapLabelManager` and `BufferedWriter`.
*/
// TODO lm was `protected` before anonymousTypeMapping and locallyVisibleFunctionLabelMapping moved into it. Should we re-protect it and provide accessors?
abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLabelManager, private val bw: BufferedWriter) {
// TODO lm was `protected` before anonymousTypeMapping and locallyVisibleFunctionLabelMapping moved
// into it. Should we re-protect it and provide accessors?
abstract class TrapWriter(
protected val loggerBase: LoggerBase,
val lm: TrapLabelManager,
private val bw: BufferedWriter
) {
abstract fun getDiagnosticTrapWriter(): DiagnosticTrapWriter
/**
* Returns the label that is defined to be the given key, if such
* a label exists, and `null` otherwise. Most users will want to use
* `getLabelFor` instead, which allows non-existent labels to be
* initialised.
* Returns the label that is defined to be the given key, if such a label exists, and `null`
* otherwise. Most users will want to use `getLabelFor` instead, which allows non-existent
* labels to be initialised.
*/
fun <T: AnyDbType> getExistingLabelFor(key: String): Label<T>? {
fun <T : AnyDbType> getExistingLabelFor(key: String): Label<T>? {
return lm.labelMapping.get(key)?.cast<T>()
}
/**
* Returns the label for the given key, if one exists.
* Otherwise, a fresh label is bound to that key, `initialise`
* is run on it, and it is returned.
* Returns the label for the given key, if one exists. Otherwise, a fresh label is bound to that
* key, `initialise` is run on it, and it is returned.
*/
@JvmOverloads // Needed so Java can call a method with an optional argument
fun <T: AnyDbType> getLabelFor(key: String, initialise: (Label<T>) -> Unit = {}): Label<T> {
fun <T : AnyDbType> getLabelFor(key: String, initialise: (Label<T>) -> Unit = {}): Label<T> {
val maybeLabel: Label<T>? = getExistingLabelFor(key)
if(maybeLabel == null) {
if (maybeLabel == null) {
val label: Label<T> = lm.getFreshLabel()
lm.labelMapping.put(key, label)
writeTrap("$label = $key\n")
@@ -97,30 +92,24 @@ abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLab
}
}
/**
* Returns a label for a fresh ID (i.e. a new label bound to `*`).
*/
fun <T: AnyDbType> getFreshIdLabel(): Label<T> {
/** Returns a label for a fresh ID (i.e. a new label bound to `*`). */
fun <T : AnyDbType> getFreshIdLabel(): Label<T> {
val label: Label<T> = lm.getFreshLabel()
writeTrap("$label = *\n")
return label
}
/**
* It is not easy to assign keys to local variables, so they get
* given `*` IDs. However, the same variable may be referred to
* from distant places in the IR, so we need a way to find out
* which label is used for a given local variable. This information
* is stored in this mapping.
*/
private val variableLabelMapping: MutableMap<IrVariable, Label<out DbLocalvar>> = mutableMapOf<IrVariable, Label<out DbLocalvar>>()
/**
* This returns the label used for a local variable, creating one
* if none currently exists.
* It is not easy to assign keys to local variables, so they get given `*` IDs. However, the
* same variable may be referred to from distant places in the IR, so we need a way to find out
* which label is used for a given local variable. This information is stored in this mapping.
*/
private val variableLabelMapping: MutableMap<IrVariable, Label<out DbLocalvar>> =
mutableMapOf<IrVariable, Label<out DbLocalvar>>()
/** This returns the label used for a local variable, creating one if none currently exists. */
fun <T> getVariableLabelFor(v: IrVariable): Label<out DbLocalvar> {
val maybeLabel = variableLabelMapping.get(v)
if(maybeLabel == null) {
if (maybeLabel == null) {
val label = getFreshIdLabel<DbLocalvar>()
variableLabelMapping.put(v, label)
return label
@@ -134,43 +123,41 @@ abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLab
}
/**
* This returns a label for the location described by its arguments.
* Typically users will not want to call this directly, but instead
* use `unknownLocation`, or overloads of this defined by subclasses.
* This returns a label for the location described by its arguments. Typically users will not
* want to call this directly, but instead use `unknownLocation`, or overloads of this defined
* by subclasses.
*/
fun getLocation(fileId: Label<DbFile>, startLine: Int, startColumn: Int, endLine: Int, endColumn: Int): Label<DbLocation> {
fun getLocation(
fileId: Label<DbFile>,
startLine: Int,
startColumn: Int,
endLine: Int,
endColumn: Int
): Label<DbLocation> {
return getLabelFor("@\"loc,{$fileId},$startLine,$startColumn,$endLine,$endColumn\"") {
writeLocations_default(it, fileId, startLine, startColumn, endLine, endColumn)
}
}
/**
* The label for the 'unknown' file ID.
* Users will want to use `unknownLocation` instead.
* This is lazy, as we don't want to define it in a TRAP file unless
* the TRAP file actually contains something in the 'unknown' file.
* The label for the 'unknown' file ID. Users will want to use `unknownLocation` instead. This
* is lazy, as we don't want to define it in a TRAP file unless the TRAP file actually contains
* something in the 'unknown' file.
*/
protected val unknownFileId: Label<DbFile> by lazy {
val unknownFileLabel = "@\";sourcefile\""
getLabelFor(unknownFileLabel, {
writeFiles(it, "")
})
getLabelFor(unknownFileLabel, { writeFiles(it, "") })
}
/**
* The label for the 'unknown' location.
* This is lazy, as we don't want to define it in a TRAP file unless
* the TRAP file actually contains something with an 'unknown'
* location.
* The label for the 'unknown' location. This is lazy, as we don't want to define it in a TRAP
* file unless the TRAP file actually contains something with an 'unknown' location.
*/
val unknownLocation: Label<DbLocation> by lazy {
getWholeFileLocation(unknownFileId)
}
val unknownLocation: Label<DbLocation> by lazy { getWholeFileLocation(unknownFileId) }
/**
* Returns the label for the file `filePath`.
* If `populateFileTables` is true, then this also adds rows to the
* `files` and `folders` tables for this file.
* Returns the label for the file `filePath`. If `populateFileTables` is true, then this also
* adds rows to the `files` and `folders` tables for this file.
*/
fun mkFileId(filePath: String, populateFileTables: Boolean): Label<DbFile> {
// If a file is in a jar, then the Kotlin compiler gives
@@ -178,65 +165,65 @@ abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLab
// it as appropriate, to make the right file ID.
val populateFile = PopulateFile(this)
val splitFilePath = filePath.split("!/")
if(splitFilePath.size == 1) {
if (splitFilePath.size == 1) {
return populateFile.getFileLabel(File(filePath), populateFileTables)
} else {
return populateFile.getFileInJarLabel(File(splitFilePath.get(0)), splitFilePath.get(1), populateFileTables)
return populateFile.getFileInJarLabel(
File(splitFilePath.get(0)),
splitFilePath.get(1),
populateFileTables
)
}
}
/**
* If you have an ID for a file, then this gets a label for the
* location representing the whole of that file.
* If you have an ID for a file, then this gets a label for the location representing the whole
* of that file.
*/
fun getWholeFileLocation(fileId: Label<DbFile>): Label<DbLocation> {
return getLocation(fileId, 0, 0, 0, 0)
}
/**
* Write a raw string into the TRAP file. Users should call one of
* the wrapper functions instead.
* Write a raw string into the TRAP file. Users should call one of the wrapper functions
* instead.
*/
fun writeTrap(trap: String) {
bw.write(trap)
}
/**
* Write a comment into the TRAP file.
*/
/** Write a comment into the TRAP file. */
fun writeComment(comment: String) {
writeTrap("// ${comment.replace("\n", "\n// ")}\n")
}
/**
* Flush the TRAP file.
*/
/** Flush the TRAP file. */
fun flush() {
bw.flush()
}
/**
* Escape a string so that it can be used in a TRAP string literal,
* i.e. with `"` escaped as `""`.
*/
* Escape a string so that it can be used in a TRAP string literal, i.e. with `"` escaped as
* `""`.
*/
fun escapeTrapString(str: String) = str.replace("\"", "\"\"")
/**
* TRAP string literals are limited to 1 megabyte.
*/
/** TRAP string literals are limited to 1 megabyte. */
private val MAX_STRLEN = 1.shl(20)
/**
* Truncate a string, if necessary, so that it can be used as a TRAP
* string literal. TRAP string literals are limited to 1 megabyte.
* Truncate a string, if necessary, so that it can be used as a TRAP string literal. TRAP string
* literals are limited to 1 megabyte.
*/
fun truncateString(str: String): String {
val len = str.length
val newLen = UTF8Util.encodablePrefixLength(str, MAX_STRLEN)
if (newLen < len) {
loggerBase.warn(this.getDiagnosticTrapWriter(),
loggerBase.warn(
this.getDiagnosticTrapWriter(),
"Truncated string of length $len",
"Truncated string of length $len, starting '${str.take(100)}', ending '${str.takeLast(100)}'")
"Truncated string of length $len, starting '${str.take(100)}', ending '${str.takeLast(100)}'"
)
return str.take(newLen)
} else {
return str
@@ -244,69 +231,75 @@ abstract class TrapWriter (protected val loggerBase: LoggerBase, val lm: TrapLab
}
/**
* Gets a FileTrapWriter like this one (using the same label manager,
* writer etc), but using the given `filePath` for locations.
* Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the
* given `filePath` for locations.
*/
fun makeFileTrapWriter(filePath: String, populateFileTables: Boolean) =
FileTrapWriter(loggerBase, lm, bw, this.getDiagnosticTrapWriter(), filePath, populateFileTables)
FileTrapWriter(
loggerBase,
lm,
bw,
this.getDiagnosticTrapWriter(),
filePath,
populateFileTables
)
/**
* Gets a FileTrapWriter like this one (using the same label manager,
* writer etc), but using the given `IrFile` for locations.
* Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the
* given `IrFile` for locations.
*/
fun makeSourceFileTrapWriter(file: IrFile, populateFileTables: Boolean) =
SourceFileTrapWriter(loggerBase, lm, bw, this.getDiagnosticTrapWriter(), file, populateFileTables)
SourceFileTrapWriter(
loggerBase,
lm,
bw,
this.getDiagnosticTrapWriter(),
file,
populateFileTables
)
}
/**
* A `PlainTrapWriter` has no additional context of its own.
*/
class PlainTrapWriter (
/** A `PlainTrapWriter` has no additional context of its own. */
class PlainTrapWriter(
loggerBase: LoggerBase,
lm: TrapLabelManager,
bw: BufferedWriter,
val dtw: DiagnosticTrapWriter
): TrapWriter (loggerBase, lm, bw) {
) : TrapWriter(loggerBase, lm, bw) {
override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter {
return dtw
}
}
/**
* A `DiagnosticTrapWriter` is a TrapWriter that diagnostics can be
* written to; i.e. it has the #compilation label defined. In practice,
* this means that it is a TrapWriter for the invocation TRAP file.
* A `DiagnosticTrapWriter` is a TrapWriter that diagnostics can be written to; i.e. it has
* the #compilation label defined. In practice, this means that it is a TrapWriter for the
* invocation TRAP file.
*/
class DiagnosticTrapWriter (
loggerBase: LoggerBase,
lm: TrapLabelManager,
bw: BufferedWriter
): TrapWriter (loggerBase, lm, bw) {
class DiagnosticTrapWriter(loggerBase: LoggerBase, lm: TrapLabelManager, bw: BufferedWriter) :
TrapWriter(loggerBase, lm, bw) {
override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter {
return this
}
}
/**
* A `FileTrapWriter` is used when we know which file we are extracting
* entities from, so we can at least give the right file as a location.
* A `FileTrapWriter` is used when we know which file we are extracting entities from, so we can at
* least give the right file as a location.
*
* An ID for the file will be created, and if `populateFileTables` is
* true then we will also add rows to the `files` and `folders` tables
* for it.
* An ID for the file will be created, and if `populateFileTables` is true then we will also add
* rows to the `files` and `folders` tables for it.
*/
open class FileTrapWriter (
open class FileTrapWriter(
loggerBase: LoggerBase,
lm: TrapLabelManager,
bw: BufferedWriter,
val dtw: DiagnosticTrapWriter,
val filePath: String,
populateFileTables: Boolean
): TrapWriter (loggerBase, lm, bw) {
) : TrapWriter(loggerBase, lm, bw) {
/**
* The ID for the file that we are extracting from.
*/
/** The ID for the file that we are extracting from. */
val fileId = mkFileId(filePath, populateFileTables)
override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter {
@@ -320,7 +313,12 @@ open class FileTrapWriter (
var currentMin = default
for (option in options) {
if (option != null && option != UNDEFINED_OFFSET && option != SYNTHETIC_OFFSET && option < currentMin) {
if (
option != null &&
option != UNDEFINED_OFFSET &&
option != SYNTHETIC_OFFSET &&
option < currentMin
) {
currentMin = option
}
}
@@ -344,15 +342,13 @@ open class FileTrapWriter (
return e.endOffset
}
/**
* Gets a label for the location of `e`.
*/
/** Gets a label for the location of `e`. */
fun getLocation(e: IrElement): Label<DbLocation> {
return getLocation(getStartOffset(e), getEndOffset(e))
}
/**
* Gets a label for the location corresponding to `startOffset` and
* `endOffset` within this file.
* Gets a label for the location corresponding to `startOffset` and `endOffset` within this
* file.
*/
open fun getLocation(startOffset: Int, endOffset: Int): Label<DbLocation> {
// We don't have a FileEntry to look up the offsets in, so all
@@ -360,8 +356,8 @@ open class FileTrapWriter (
return getWholeFileLocation()
}
/**
* Gets the location of `e` as a human-readable string. Only used in
* log messages and exception messages.
* Gets the location of `e` as a human-readable string. Only used in log messages and exception
* messages.
*/
open fun getLocationString(e: IrElement): String {
// We don't have a FileEntry to look up the offsets in, so all
@@ -371,50 +367,54 @@ open class FileTrapWriter (
// to be 0.
return "file://$filePath"
}
/**
* Gets a label for the location representing the whole of this file.
*/
/** Gets a label for the location representing the whole of this file. */
fun getWholeFileLocation(): Label<DbLocation> {
return getWholeFileLocation(fileId)
}
}
/**
* A `SourceFileTrapWriter` is used when not only do we know which file
* we are extracting entities from, but we also have an `IrFileEntry`
* (from an `IrFile`) which allows us to map byte offsets to line
* and column numbers.
* A `SourceFileTrapWriter` is used when not only do we know which file we are extracting entities
* from, but we also have an `IrFileEntry` (from an `IrFile`) which allows us to map byte offsets to
* line and column numbers.
*
* An ID for the file will be created, and if `populateFileTables` is
* true then we will also add rows to the `files` and `folders` tables
* for it.
* An ID for the file will be created, and if `populateFileTables` is true then we will also add
* rows to the `files` and `folders` tables for it.
*/
class SourceFileTrapWriter (
class SourceFileTrapWriter(
loggerBase: LoggerBase,
lm: TrapLabelManager,
bw: BufferedWriter,
dtw: DiagnosticTrapWriter,
val irFile: IrFile,
populateFileTables: Boolean) :
FileTrapWriter(loggerBase, lm, bw, dtw, irFile.path, populateFileTables) {
populateFileTables: Boolean
) : FileTrapWriter(loggerBase, lm, bw, dtw, irFile.path, populateFileTables) {
/**
* The file entry for the file that we are extracting from.
* Used to map offsets to line/column numbers.
* The file entry for the file that we are extracting from. Used to map offsets to line/column
* numbers.
*/
private val fileEntry = irFile.fileEntry
override fun getLocation(startOffset: Int, endOffset: Int): Label<DbLocation> {
if (startOffset == UNDEFINED_OFFSET || endOffset == UNDEFINED_OFFSET) {
if (startOffset != endOffset) {
loggerBase.warn(dtw, "Location with inconsistent offsets (start $startOffset, end $endOffset)", null)
loggerBase.warn(
dtw,
"Location with inconsistent offsets (start $startOffset, end $endOffset)",
null
)
}
return getWholeFileLocation()
}
if (startOffset == SYNTHETIC_OFFSET || endOffset == SYNTHETIC_OFFSET) {
if (startOffset != endOffset) {
loggerBase.warn(dtw, "Location with inconsistent offsets (start $startOffset, end $endOffset)", null)
loggerBase.warn(
dtw,
"Location with inconsistent offsets (start $startOffset, end $endOffset)",
null
)
}
return getWholeFileLocation()
}
@@ -428,28 +428,37 @@ class SourceFileTrapWriter (
fileEntry.getLineNumber(startOffset) + 1,
fileEntry.getColumnNumber(startOffset) + 1,
fileEntry.getLineNumber(endOffset) + 1,
fileEntry.getColumnNumber(endOffset) + endColumnOffset)
fileEntry.getColumnNumber(endOffset) + endColumnOffset
)
}
override fun getLocationString(e: IrElement): String {
if (e.startOffset == UNDEFINED_OFFSET || e.endOffset == UNDEFINED_OFFSET) {
if (e.startOffset != e.endOffset) {
loggerBase.warn(dtw, "Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})", null)
loggerBase.warn(
dtw,
"Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})",
null
)
}
return "<unknown location while processing $filePath>"
}
if (e.startOffset == SYNTHETIC_OFFSET || e.endOffset == SYNTHETIC_OFFSET) {
if (e.startOffset != e.endOffset) {
loggerBase.warn(dtw, "Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})", null)
loggerBase.warn(
dtw,
"Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})",
null
)
}
return "<synthetic location while processing $filePath>"
}
val startLine = fileEntry.getLineNumber(e.startOffset) + 1
val startLine = fileEntry.getLineNumber(e.startOffset) + 1
val startColumn = fileEntry.getColumnNumber(e.startOffset) + 1
val endLine = fileEntry.getLineNumber(e.endOffset) + 1
val endColumn = fileEntry.getColumnNumber(e.endOffset)
val endLine = fileEntry.getLineNumber(e.endOffset) + 1
val endColumn = fileEntry.getColumnNumber(e.endOffset)
return "file://$filePath:$startLine:$startColumn:$endLine:$endColumn"
}
}

47
java/kotlin-extractor/src/main/kotlin/comments/CommentExtractor.kt
View File

@@ -8,13 +8,16 @@ import org.jetbrains.kotlin.ir.expressions.IrBody
import org.jetbrains.kotlin.ir.expressions.IrExpression
import org.jetbrains.kotlin.ir.util.parentClassOrNull
open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, protected val file: IrFile, protected val fileLabel: Label<out DbFile>) {
open class CommentExtractor(
protected val fileExtractor: KotlinFileExtractor,
protected val file: IrFile,
protected val fileLabel: Label<out DbFile>
) {
protected val tw = fileExtractor.tw
protected val logger = fileExtractor.logger
protected fun getLabel(element: IrElement): Label<out DbTop>? {
if (element == file)
return fileLabel
if (element == file) return fileLabel
if (element is IrValueParameter && element.index == -1) {
// Don't attribute comments to the implicit `this` parameter of a function.
@@ -22,18 +25,21 @@ open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, pr
}
val label: String
val existingLabel = if (element is IrVariable) {
// local variables are not named globally, so we need to get them from the variable label cache
label = "variable ${element.name.asString()}"
tw.getExistingVariableLabelFor(element)
} else if (element is IrFunction && element.isLocalFunction()) {
// local functions are not named globally, so we need to get them from the local function label cache
label = "local function ${element.name.asString()}"
fileExtractor.getExistingLocallyVisibleFunctionLabel(element)
} else {
label = getLabelForNamedElement(element) ?: return null
tw.getExistingLabelFor<DbTop>(label)
}
val existingLabel =
if (element is IrVariable) {
// local variables are not named globally, so we need to get them from the variable
// label cache
label = "variable ${element.name.asString()}"
tw.getExistingVariableLabelFor(element)
} else if (element is IrFunction && element.isLocalFunction()) {
// local functions are not named globally, so we need to get them from the local
// function label cache
label = "local function ${element.name.asString()}"
fileExtractor.getExistingLocallyVisibleFunctionLabel(element)
} else {
label = getLabelForNamedElement(element) ?: return null
tw.getExistingLabelFor<DbTop>(label)
}
if (existingLabel == null) {
logger.warn("Couldn't get existing label for $label")
return null
@@ -41,7 +47,7 @@ open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, pr
return existingLabel
}
private fun getLabelForNamedElement(element: IrElement) : String? {
private fun getLabelForNamedElement(element: IrElement): String? {
when (element) {
is IrClass -> return fileExtractor.getClassLabel(element, listOf()).classLabel
is IrTypeParameter -> return fileExtractor.getTypeParameterLabel(element)
@@ -57,14 +63,14 @@ open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, pr
is IrField -> return fileExtractor.getFieldLabel(element)
is IrEnumEntry -> return fileExtractor.getEnumEntryLabel(element)
is IrTypeAlias -> return fileExtractor.getTypeAliasLabel(element)
is IrAnonymousInitializer -> {
val parentClass = element.parentClassOrNull
if (parentClass == null) {
logger.warnElement("Parent of anonymous initializer is not a class", element)
return null
}
// Assign the comment to the class. The content of the `init` blocks might be extracted in multiple constructors.
// Assign the comment to the class. The content of the `init` blocks might be
// extracted in multiple constructors.
return getLabelForNamedElement(parentClass)
}
@@ -74,7 +80,10 @@ open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, pr
// todo add others:
else -> {
logger.warnElement("Unhandled element type found during comment extraction: ${element::class}", element)
logger.warnElement(
"Unhandled element type found during comment extraction: ${element::class}",
element
)
return null
}
}

50
java/kotlin-extractor/src/main/kotlin/comments/CommentExtractorPSI.kt
View File

@@ -15,12 +15,18 @@ import org.jetbrains.kotlin.psi.KtVisitor
import org.jetbrains.kotlin.psi.psiUtil.endOffset
import org.jetbrains.kotlin.psi.psiUtil.startOffset
class CommentExtractorPSI(fileExtractor: KotlinFileExtractor, file: IrFile, fileLabel: Label<out DbFile>): CommentExtractor(fileExtractor, file, fileLabel) {
class CommentExtractorPSI(
fileExtractor: KotlinFileExtractor,
file: IrFile,
fileLabel: Label<out DbFile>
) : CommentExtractor(fileExtractor, file, fileLabel) {
// Returns true if it extracted the comments; false otherwise.
fun extract(): Boolean {
val psi2Ir = getPsi2Ir()
if (psi2Ir == null) {
logger.warn("Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
logger.warn(
"Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})"
)
return false
}
val ktFile = psi2Ir.getKtFile(file)
@@ -37,28 +43,30 @@ class CommentExtractorPSI(fileExtractor: KotlinFileExtractor, file: IrFile, file
override fun visitElement(element: PsiElement) {
element.acceptChildren(this)
// Slightly hacky, but `visitComment` doesn't seem to visit comments with `tokenType` `KtTokens.DOC_COMMENT`
if (element is PsiComment){
// Slightly hacky, but `visitComment` doesn't seem to visit comments with
// `tokenType` `KtTokens.DOC_COMMENT`
if (element is PsiComment) {
visitCommentElement(element)
}
}
private fun visitCommentElement(comment: PsiComment) {
val type: CommentType = when (comment.tokenType) {
KtTokens.EOL_COMMENT -> {
CommentType.SingleLine
val type: CommentType =
when (comment.tokenType) {
KtTokens.EOL_COMMENT -> {
CommentType.SingleLine
}
KtTokens.BLOCK_COMMENT -> {
CommentType.Block
}
KtTokens.DOC_COMMENT -> {
CommentType.Doc
}
else -> {
logger.warn("Unhandled comment token type: ${comment.tokenType}")
return
}
}
KtTokens.BLOCK_COMMENT -> {
CommentType.Block
}
KtTokens.DOC_COMMENT -> {
CommentType.Doc
}
else -> {
logger.warn("Unhandled comment token type: ${comment.tokenType}")
return
}
}
val commentLabel = tw.getFreshIdLabel<DbKtcomment>()
tw.writeKtComments(commentLabel, type.value, comment.text)
@@ -101,10 +109,12 @@ class CommentExtractorPSI(fileExtractor: KotlinFileExtractor, file: IrFile, file
}
}
private fun getKDocOwner(comment: KDoc) : PsiElement? {
private fun getKDocOwner(comment: KDoc): PsiElement? {
val owner = comment.owner
if (owner == null) {
logger.warn("Couldn't get owner of KDoc. The comment is extracted without an owner.")
logger.warn(
"Couldn't get owner of KDoc. The comment is extracted without an owner."
)
}
return owner
}

6
java/kotlin-extractor/src/main/kotlin/comments/CommentType.kt
View File

@@ -1,5 +1,7 @@
package com.github.codeql.comments
enum class CommentType(val value: Int) {
SingleLine(1), Block(2), Doc(3)
}
SingleLine(1),
Block(2),
Doc(3)
}

62
java/kotlin-extractor/src/main/kotlin/utils/AutoCloseableUse.kt
View File

@@ -1,43 +1,47 @@
package com.github.codeql
// Functions copied from stdlib/jdk7/src/kotlin/AutoCloseable.kt, which is not available within kotlinc,
// Functions copied from stdlib/jdk7/src/kotlin/AutoCloseable.kt, which is not available within
// kotlinc,
// but allows the `.use` pattern to be applied to JDK7 AutoCloseables:
/**
* Executes the given [block] function on this resource and then closes it down correctly whether an exception
* is thrown or not.
* Executes the given [block] function on this resource and then closes it down correctly whether an
* exception is thrown or not.
*
* In case if the resource is being closed due to an exception occurred in [block], and the closing also fails with an exception,
* the latter is added to the [suppressed][java.lang.Throwable.addSuppressed] exceptions of the former.
* In case if the resource is being closed due to an exception occurred in [block], and the closing
* also fails with an exception, the latter is added to the
* [suppressed][java.lang.Throwable.addSuppressed] exceptions of the former.
*
* @param block a function to process this [AutoCloseable] resource.
* @return the result of [block] function invoked on this resource.
*/
public inline fun <T : AutoCloseable?, R> T.useAC(block: (T) -> R): R {
var exception: Throwable? = null
try {
return block(this)
} catch (e: Throwable) {
exception = e
throw e
} finally {
this.closeFinallyAC(exception)
}
var exception: Throwable? = null
try {
return block(this)
} catch (e: Throwable) {
exception = e
throw e
} finally {
this.closeFinallyAC(exception)
}
}
/**
* Closes this [AutoCloseable], suppressing possible exception or error thrown by [AutoCloseable.close] function when
* it's being closed due to some other [cause] exception occurred.
*
* The suppressed exception is added to the list of suppressed exceptions of [cause] exception.
*/
fun AutoCloseable?.closeFinallyAC(cause: Throwable?) = when {
this == null -> {}
cause == null -> close()
else ->
try {
close()
} catch (closeException: Throwable) {
cause.addSuppressed(closeException)
}
}
* Closes this [AutoCloseable], suppressing possible exception or error thrown by
* [AutoCloseable.close] function when it's being closed due to some other [cause] exception
* occurred.
*
* The suppressed exception is added to the list of suppressed exceptions of [cause] exception.
*/
fun AutoCloseable?.closeFinallyAC(cause: Throwable?) =
when {
this == null -> {}
cause == null -> close()
else ->
try {
close()
} catch (closeException: Throwable) {
cause.addSuppressed(closeException)
}
}

77
java/kotlin-extractor/src/main/kotlin/utils/ClassNames.kt
View File

@@ -3,32 +3,33 @@ package com.github.codeql
import com.github.codeql.utils.getJvmName
import com.github.codeql.utils.versions.*
import com.intellij.openapi.vfs.StandardFileSystems
import org.jetbrains.kotlin.fir.java.JavaBinarySourceElement
import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClass
import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement
import com.intellij.openapi.vfs.VirtualFile
import org.jetbrains.kotlin.builtins.jvm.JavaToKotlinClassMap
import org.jetbrains.kotlin.fir.java.JavaBinarySourceElement
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable
import org.jetbrains.kotlin.ir.util.parentClassOrNull
import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
import org.jetbrains.kotlin.load.kotlin.JvmPackagePartSource
import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement
import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClass
// Adapted from Kotlin's interpreter/Utils.kt function 'internalName'
// Translates class names into their JLS section 13.1 binary name,
// and declarations within them into the parent class' JLS 13.1 name as
// specified above, followed by a `$` separator and then the short name
// for `that`.
private fun getName(d: IrDeclarationWithName) = (d as? IrAnnotationContainer)?.let { getJvmName(it) } ?: d.name.asString()
private fun getName(d: IrDeclarationWithName) =
(d as? IrAnnotationContainer)?.let { getJvmName(it) } ?: d.name.asString()
fun getFileClassName(f: IrFile) =
getJvmName(f) ?:
((f.fileEntry.name.replaceFirst(Regex(""".*[/\\]"""), "")
.replaceFirst(Regex("""\.kt$"""), "")
.replaceFirstChar { it.uppercase() }) + "Kt")
getJvmName(f)
?: ((f.fileEntry.name
.replaceFirst(Regex(""".*[/\\]"""), "")
.replaceFirst(Regex("""\.kt$"""), "")
.replaceFirstChar { it.uppercase() }) + "Kt")
fun getIrElementBinaryName(that: IrElement): String {
if (that is IrFile) {
@@ -38,19 +39,21 @@ fun getIrElementBinaryName(that: IrElement): String {
}
if (that !is IrDeclaration) {
return "(unknown-name)"
return "(unknown-name)"
}
val shortName = when(that) {
is IrDeclarationWithName -> getName(that)
else -> "(unknown-name)"
}
val shortName =
when (that) {
is IrDeclarationWithName -> getName(that)
else -> "(unknown-name)"
}
val internalName = StringBuilder(shortName)
if (that !is IrClass) {
val parent = that.parent
if (parent is IrFile) {
// Note we'll fall through and do the IrPackageFragment case as well, since IrFile <: IrPackageFragment
// Note we'll fall through and do the IrPackageFragment case as well, since IrFile <:
// IrPackageFragment
internalName.insert(0, getFileClassName(parent) + "$")
}
}
@@ -59,7 +62,11 @@ fun getIrElementBinaryName(that: IrElement): String {
.forEach {
when (it) {
is IrClass -> internalName.insert(0, getName(it) + "$")
is IrPackageFragment -> it.packageFqName.asString().takeIf { fqName -> fqName.isNotEmpty() }?.let { fqName -> internalName.insert(0, "$fqName.") }
is IrPackageFragment ->
it.packageFqName
.asString()
.takeIf { fqName -> fqName.isNotEmpty() }
?.let { fqName -> internalName.insert(0, "$fqName.") }
}
}
return internalName.toString()
@@ -67,16 +74,18 @@ fun getIrElementBinaryName(that: IrElement): String {
fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
val cSource = irClass.source
// Don't emit a location for multi-file classes until we're sure we can cope with different declarations
// inside a class disagreeing about their source file. In particular this currently causes problems when
// a source-location for a declarations tries to refer to a file-id which is assumed to be declared in
// Don't emit a location for multi-file classes until we're sure we can cope with different
// declarations
// inside a class disagreeing about their source file. In particular this currently causes
// problems when
// a source-location for a declarations tries to refer to a file-id which is assumed to be
// declared in
// the class trap file.
if (irClass.origin == IrDeclarationOrigin.JVM_MULTIFILE_CLASS)
return null
when(cSource) {
if (irClass.origin == IrDeclarationOrigin.JVM_MULTIFILE_CLASS) return null
when (cSource) {
is JavaSourceElement -> {
val element = cSource.javaElement
when(element) {
when (element) {
is BinaryJavaClass -> return element.virtualFile
}
}
@@ -85,7 +94,7 @@ fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
}
is KotlinJvmBinarySourceElement -> {
val binaryClass = cSource.binaryClass
when(binaryClass) {
when (binaryClass) {
is VirtualFileKotlinClass -> return binaryClass.file
}
}
@@ -102,17 +111,17 @@ fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
private fun getRawIrClassBinaryPath(irClass: IrClass) =
getIrClassVirtualFile(irClass)?.let {
val path = it.path
if(it.fileSystem.protocol == StandardFileSystems.JRT_PROTOCOL)
// For JRT files, which we assume to be the JDK, hide the containing JAR path to match the Java extractor's behaviour.
"/${path.split("!/", limit = 2)[1]}"
else
path
if (it.fileSystem.protocol == StandardFileSystems.JRT_PROTOCOL)
// For JRT files, which we assume to be the JDK, hide the containing JAR path to match the
// Java extractor's behaviour.
"/${path.split("!/", limit = 2)[1]}"
else path
}
fun getIrClassBinaryPath(irClass: IrClass): String {
return getRawIrClassBinaryPath(irClass)
// Otherwise, make up a fake location:
?: getUnknownBinaryLocation(getIrElementBinaryName(irClass))
return getRawIrClassBinaryPath(irClass)
// Otherwise, make up a fake location:
?: getUnknownBinaryLocation(getIrElementBinaryName(irClass))
}
fun getIrDeclarationBinaryPath(d: IrDeclaration): String? {

18
java/kotlin-extractor/src/main/kotlin/utils/ExternalDecls.kt
View File

@@ -5,7 +5,6 @@ import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
import org.jetbrains.kotlin.ir.declarations.IrExternalPackageFragment
import org.jetbrains.kotlin.ir.util.isFileClass
import org.jetbrains.kotlin.ir.util.parentClassOrNull
fun isExternalDeclaration(d: IrDeclaration): Boolean {
/*
@@ -15,9 +14,11 @@ fun isExternalDeclaration(d: IrDeclaration): Boolean {
EXPRESSION_BODY
CONST Int type=kotlin.Int value=-2147483648
*/
if (d.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB ||
d.origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB ||
d.origin.toString() == "FUNCTION_INTERFACE_CLASS") { // Treat kotlin.coroutines.* like ordinary library classes
if (
d.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB ||
d.origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB ||
d.origin.toString() == "FUNCTION_INTERFACE_CLASS"
) { // Treat kotlin.coroutines.* like ordinary library classes
return true
}
/*
@@ -39,11 +40,11 @@ fun isExternalDeclaration(d: IrDeclaration): Boolean {
return false
}
/**
* Returns true if `d` is not itself a class, but is a member of an external file class.
*/
/** Returns true if `d` is not itself a class, but is a member of an external file class. */
fun isExternalFileClassMember(d: IrDeclaration): Boolean {
if (d is IrClass) { return false }
if (d is IrClass) {
return false
}
val p = d.parent
when (p) {
is IrClass -> return p.isFileClass
@@ -53,4 +54,3 @@ fun isExternalFileClassMember(d: IrDeclaration): Boolean {
}
return false
}

12
java/kotlin-extractor/src/main/kotlin/utils/GetByFqName.kt
View File

@@ -9,10 +9,18 @@ fun getClassByFqName(pluginContext: IrPluginContext, fqName: String): IrClassSym
return getClassByFqName(pluginContext, FqName(fqName))
}
fun getFunctionsByFqName(pluginContext: IrPluginContext, pkgName: String, name: String): Collection<IrSimpleFunctionSymbol> {
fun getFunctionsByFqName(
pluginContext: IrPluginContext,
pkgName: String,
name: String
): Collection<IrSimpleFunctionSymbol> {
return getFunctionsByFqName(pluginContext, FqName(pkgName), Name.identifier(name))
}
fun getPropertiesByFqName(pluginContext: IrPluginContext, pkgName: String, name: String): Collection<IrPropertySymbol> {
fun getPropertiesByFqName(
pluginContext: IrPluginContext,
pkgName: String,
name: String
): Collection<IrPropertySymbol> {
return getPropertiesByFqName(pluginContext, FqName(pkgName), Name.identifier(name))
}

3
java/kotlin-extractor/src/main/kotlin/utils/Helpers.kt
View File

@@ -9,4 +9,5 @@ fun IrFunction.isLocalFunction(): Boolean {
return this.visibility == DescriptorVisibilities.LOCAL
}
val IrClass.isInterfaceLike get() = kind == ClassKind.INTERFACE || kind == ClassKind.ANNOTATION_CLASS
val IrClass.isInterfaceLike
get() = kind == ClassKind.INTERFACE || kind == ClassKind.ANNOTATION_CLASS

8
java/kotlin-extractor/src/main/kotlin/utils/IrVisitorLookup.kt
View File

@@ -1,6 +1,5 @@
package com.github.codeql.utils
import com.github.codeql.utils.Psi2IrFacade
import com.intellij.psi.PsiElement
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
@@ -8,8 +7,11 @@ import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.ir.util.isFakeOverride
import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
class IrVisitorLookup(private val psi2Ir: Psi2IrFacade, private val psi: PsiElement, private val file: IrFile) :
IrElementVisitor<Unit, MutableCollection<IrElement>> {
class IrVisitorLookup(
private val psi2Ir: Psi2IrFacade,
private val psi: PsiElement,
private val file: IrFile
) : IrElementVisitor<Unit, MutableCollection<IrElement>> {
private val location = psi.getLocation()
override fun visitElement(element: IrElement, data: MutableCollection<IrElement>): Unit {

7
java/kotlin-extractor/src/main/kotlin/utils/Iterable.kt
View File

@@ -3,11 +3,10 @@ package com.github.codeql
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
/**
* This behaves the same as Iterable<IrDeclaration>.find, but requires
* that the value found is of the subtype S, and it casts
* the result for you appropriately.
* This behaves the same as Iterable<IrDeclaration>.find, but requires that the value found is of
* the subtype S, and it casts the result for you appropriately.
*/
inline fun <reified S: IrDeclaration> Iterable<IrDeclaration>.findSubType(
inline fun <reified S : IrDeclaration> Iterable<IrDeclaration>.findSubType(
predicate: (S) -> Boolean
): S? {
return this.find { it is S && predicate(it) } as S?

79
java/kotlin-extractor/src/main/kotlin/utils/JvmNames.kt
View File

@@ -17,43 +17,46 @@ import org.jetbrains.kotlin.name.Name
private data class MethodKey(val className: FqName, val functionName: Name)
private fun makeDescription(className: FqName, functionName: String) = MethodKey(className, Name.guessByFirstCharacter(functionName))
private fun makeDescription(className: FqName, functionName: String) =
MethodKey(className, Name.guessByFirstCharacter(functionName))
// This essentially mirrors SpecialBridgeMethods.kt, a backend pass which isn't easily available to our extractor.
private val specialFunctions = mapOf(
makeDescription(StandardNames.FqNames.collection, "<get-size>") to "size",
makeDescription(FqName("java.util.Collection"), "<get-size>") to "size",
makeDescription(StandardNames.FqNames.map, "<get-size>") to "size",
makeDescription(FqName("java.util.Map"), "<get-size>") to "size",
makeDescription(StandardNames.FqNames.charSequence.toSafe(), "<get-length>") to "length",
makeDescription(FqName("java.lang.CharSequence"), "<get-length>") to "length",
makeDescription(StandardNames.FqNames.map, "<get-keys>") to "keySet",
makeDescription(FqName("java.util.Map"), "<get-keys>") to "keySet",
makeDescription(StandardNames.FqNames.map, "<get-values>") to "values",
makeDescription(FqName("java.util.Map"), "<get-values>") to "values",
makeDescription(StandardNames.FqNames.map, "<get-entries>") to "entrySet",
makeDescription(FqName("java.util.Map"), "<get-entries>") to "entrySet",
makeDescription(StandardNames.FqNames.mutableList, "removeAt") to "remove",
makeDescription(FqName("java.util.List"), "removeAt") to "remove",
makeDescription(StandardNames.FqNames._enum.toSafe(), "<get-ordinal>") to "ordinal",
makeDescription(FqName("java.lang.Enum"), "<get-ordinal>") to "ordinal",
makeDescription(StandardNames.FqNames._enum.toSafe(), "<get-name>") to "name",
makeDescription(FqName("java.lang.Enum"), "<get-name>") to "name",
makeDescription(StandardNames.FqNames.number.toSafe(), "toByte") to "byteValue",
makeDescription(FqName("java.lang.Number"), "toByte") to "byteValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toShort") to "shortValue",
makeDescription(FqName("java.lang.Number"), "toShort") to "shortValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toInt") to "intValue",
makeDescription(FqName("java.lang.Number"), "toInt") to "intValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toLong") to "longValue",
makeDescription(FqName("java.lang.Number"), "toLong") to "longValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toFloat") to "floatValue",
makeDescription(FqName("java.lang.Number"), "toFloat") to "floatValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toDouble") to "doubleValue",
makeDescription(FqName("java.lang.Number"), "toDouble") to "doubleValue",
makeDescription(StandardNames.FqNames.string.toSafe(), "get") to "charAt",
makeDescription(FqName("java.lang.String"), "get") to "charAt",
)
// This essentially mirrors SpecialBridgeMethods.kt, a backend pass which isn't easily available to
// our extractor.
private val specialFunctions =
mapOf(
makeDescription(StandardNames.FqNames.collection, "<get-size>") to "size",
makeDescription(FqName("java.util.Collection"), "<get-size>") to "size",
makeDescription(StandardNames.FqNames.map, "<get-size>") to "size",
makeDescription(FqName("java.util.Map"), "<get-size>") to "size",
makeDescription(StandardNames.FqNames.charSequence.toSafe(), "<get-length>") to "length",
makeDescription(FqName("java.lang.CharSequence"), "<get-length>") to "length",
makeDescription(StandardNames.FqNames.map, "<get-keys>") to "keySet",
makeDescription(FqName("java.util.Map"), "<get-keys>") to "keySet",
makeDescription(StandardNames.FqNames.map, "<get-values>") to "values",
makeDescription(FqName("java.util.Map"), "<get-values>") to "values",
makeDescription(StandardNames.FqNames.map, "<get-entries>") to "entrySet",
makeDescription(FqName("java.util.Map"), "<get-entries>") to "entrySet",
makeDescription(StandardNames.FqNames.mutableList, "removeAt") to "remove",
makeDescription(FqName("java.util.List"), "removeAt") to "remove",
makeDescription(StandardNames.FqNames._enum.toSafe(), "<get-ordinal>") to "ordinal",
makeDescription(FqName("java.lang.Enum"), "<get-ordinal>") to "ordinal",
makeDescription(StandardNames.FqNames._enum.toSafe(), "<get-name>") to "name",
makeDescription(FqName("java.lang.Enum"), "<get-name>") to "name",
makeDescription(StandardNames.FqNames.number.toSafe(), "toByte") to "byteValue",
makeDescription(FqName("java.lang.Number"), "toByte") to "byteValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toShort") to "shortValue",
makeDescription(FqName("java.lang.Number"), "toShort") to "shortValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toInt") to "intValue",
makeDescription(FqName("java.lang.Number"), "toInt") to "intValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toLong") to "longValue",
makeDescription(FqName("java.lang.Number"), "toLong") to "longValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toFloat") to "floatValue",
makeDescription(FqName("java.lang.Number"), "toFloat") to "floatValue",
makeDescription(StandardNames.FqNames.number.toSafe(), "toDouble") to "doubleValue",
makeDescription(FqName("java.lang.Number"), "toDouble") to "doubleValue",
makeDescription(StandardNames.FqNames.string.toSafe(), "get") to "charAt",
makeDescription(FqName("java.lang.String"), "get") to "charAt",
)
private val specialFunctionShortNames = specialFunctions.keys.map { it.functionName }.toSet()
@@ -71,7 +74,7 @@ private fun getSpecialJvmName(f: IrFunction): String? {
}
fun getJvmName(container: IrAnnotationContainer): String? {
for(a: IrConstructorCall in container.annotations) {
for (a: IrConstructorCall in container.annotations) {
val t = a.type
if (t is IrSimpleType && a.valueArgumentsCount == 1) {
val owner = t.classifier.owner
@@ -79,7 +82,7 @@ fun getJvmName(container: IrAnnotationContainer): String? {
if (owner is IrClass) {
val aPkg = owner.packageFqName?.asString()
val name = owner.name.asString()
if(aPkg == "kotlin.jvm" && name == "JvmName" && v is IrConst<*>) {
if (aPkg == "kotlin.jvm" && name == "JvmName" && v is IrConst<*>) {
val value = v.value
if (value is String) {
return value

14
java/kotlin-extractor/src/main/kotlin/utils/List.kt
View File

@@ -1,13 +1,13 @@
package com.github.codeql
/**
* Turns this list of nullable elements into a list of non-nullable
* elements if they are all non-null, or returns null otherwise.
* Turns this list of nullable elements into a list of non-nullable elements if they are all
* non-null, or returns null otherwise.
*/
public fun <T : Any> List<T?>.requireNoNullsOrNull(): List<T>? {
try {
return this.requireNoNulls()
} catch (e: IllegalArgumentException) {
return null;
}
try {
return this.requireNoNulls()
} catch (e: IllegalArgumentException) {
return null
}
}

10
java/kotlin-extractor/src/main/kotlin/utils/Location.kt
View File

@@ -5,8 +5,8 @@ import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.psi.psiUtil.endOffset
import org.jetbrains.kotlin.psi.psiUtil.startOffset
data class Location(val startOffset: Int, val endOffset: Int){
fun contains(location: Location) : Boolean {
data class Location(val startOffset: Int, val endOffset: Int) {
fun contains(location: Location): Boolean {
return this.startOffset <= location.startOffset && this.endOffset >= location.endOffset
}
@@ -15,10 +15,10 @@ data class Location(val startOffset: Int, val endOffset: Int){
}
}
fun IrElement.getLocation() : Location {
fun IrElement.getLocation(): Location {
return Location(this.startOffset, this.endOffset)
}
fun PsiElement.getLocation() : Location {
fun PsiElement.getLocation(): Location {
return Location(this.startOffset, this.endOffset)
}
}

153
java/kotlin-extractor/src/main/kotlin/utils/Logger.kt
View File

@@ -12,8 +12,10 @@ import org.jetbrains.kotlin.ir.IrElement
class LogCounter() {
public val diagnosticInfo = mutableMapOf<String, Pair<Severity, Int>>()
public val diagnosticLimit: Int
init {
diagnosticLimit = System.getenv("CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT")?.toIntOrNull() ?: 100
diagnosticLimit =
System.getenv("CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT")?.toIntOrNull() ?: 100
}
}
@@ -35,6 +37,7 @@ enum class Severity(val sev: Int) {
class LogMessage(private val kind: String, private val message: String) {
val timestamp: String
init {
timestamp = "${SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Date())}"
}
@@ -45,45 +48,57 @@ class LogMessage(private val kind: String, private val message: String) {
private fun escape(str: String): String {
return str.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\u0000", "\\u0000")
.replace("\u0001", "\\u0001")
.replace("\u0002", "\\u0002")
.replace("\u0003", "\\u0003")
.replace("\u0004", "\\u0004")
.replace("\u0005", "\\u0005")
.replace("\u0006", "\\u0006")
.replace("\u0007", "\\u0007")
.replace("\u0008", "\\b")
.replace("\u0009", "\\t")
.replace("\u000A", "\\n")
.replace("\u000B", "\\u000B")
.replace("\u000C", "\\f")
.replace("\u000D", "\\r")
.replace("\u000E", "\\u000E")
.replace("\u000F", "\\u000F")
.replace("\"", "\\\"")
.replace("\u0000", "\\u0000")
.replace("\u0001", "\\u0001")
.replace("\u0002", "\\u0002")
.replace("\u0003", "\\u0003")
.replace("\u0004", "\\u0004")
.replace("\u0005", "\\u0005")
.replace("\u0006", "\\u0006")
.replace("\u0007", "\\u0007")
.replace("\u0008", "\\b")
.replace("\u0009", "\\t")
.replace("\u000A", "\\n")
.replace("\u000B", "\\u000B")
.replace("\u000C", "\\f")
.replace("\u000D", "\\r")
.replace("\u000E", "\\u000E")
.replace("\u000F", "\\u000F")
}
fun toJsonLine(): String {
val kvs = listOf(Pair("origin", "CodeQL Kotlin extractor"),
Pair("timestamp", timestamp),
Pair("kind", kind),
Pair("message", message))
return "{ " + kvs.map { p -> "\"${p.first}\": \"${escape(p.second)}\""}.joinToString(", ") + " }\n"
val kvs =
listOf(
Pair("origin", "CodeQL Kotlin extractor"),
Pair("timestamp", timestamp),
Pair("kind", kind),
Pair("message", message)
)
return "{ " +
kvs.map { p -> "\"${p.first}\": \"${escape(p.second)}\"" }.joinToString(", ") +
" }\n"
}
}
data class ExtractorContext(val kind: String, val element: IrElement, val name: String, val loc: String)
data class ExtractorContext(
val kind: String,
val element: IrElement,
val name: String,
val loc: String
)
open class LoggerBase(val logCounter: LogCounter) {
val extractorContextStack = Stack<ExtractorContext>()
private val verbosity: Int
init {
verbosity = System.getenv("CODEQL_EXTRACTOR_KOTLIN_VERBOSITY")?.toIntOrNull() ?: 3
}
private val logStream: Writer
init {
val extractorLogDir = System.getenv("CODEQL_EXTRACTOR_JAVA_LOG_DIR")
if (extractorLogDir == null || extractorLogDir == "") {
@@ -96,8 +111,8 @@ open class LoggerBase(val logCounter: LogCounter) {
private fun getDiagnosticLocation(): String? {
val st = Exception().stackTrace
for(x in st) {
when(x.className) {
for (x in st) {
when (x.className) {
"com.github.codeql.LoggerBase",
"com.github.codeql.Logger",
"com.github.codeql.FileLogger" -> {}
@@ -117,20 +132,29 @@ open class LoggerBase(val logCounter: LogCounter) {
file_number_diagnostic_number = 0
}
fun diagnostic(dtw: DiagnosticTrapWriter, severity: Severity, msg: String, extraInfo: String?, locationString: String? = null, mkLocationId: () -> Label<DbLocation> = { dtw.unknownLocation }) {
fun diagnostic(
dtw: DiagnosticTrapWriter,
severity: Severity,
msg: String,
extraInfo: String?,
locationString: String? = null,
mkLocationId: () -> Label<DbLocation> = { dtw.unknownLocation }
) {
val diagnosticLoc = getDiagnosticLocation()
val diagnosticLocStr = if(diagnosticLoc == null) "<unknown location>" else diagnosticLoc
val diagnosticLocStr = if (diagnosticLoc == null) "<unknown location>" else diagnosticLoc
val suffix =
if(diagnosticLoc == null) {
if (diagnosticLoc == null) {
" Missing caller information.\n"
} else {
val oldInfo = logCounter.diagnosticInfo.getOrDefault(diagnosticLoc, Pair(severity, 0))
if(severity != oldInfo.first) {
val oldInfo =
logCounter.diagnosticInfo.getOrDefault(diagnosticLoc, Pair(severity, 0))
if (severity != oldInfo.first) {
// We don't want to get in a loop, so just emit this
// directly without going through the diagnostic
// counting machinery
if (verbosity >= 1) {
val message = "Severity mismatch ($severity vs ${oldInfo.first}) at $diagnosticLoc"
val message =
"Severity mismatch ($severity vs ${oldInfo.first}) at $diagnosticLoc"
emitDiagnostic(dtw, Severity.Error, "Inconsistency", message, message)
}
}
@@ -139,7 +163,8 @@ open class LoggerBase(val logCounter: LogCounter) {
logCounter.diagnosticInfo[diagnosticLoc] = newInfo
when {
logCounter.diagnosticLimit <= 0 -> ""
newCount == logCounter.diagnosticLimit -> " Limit reached for diagnostics from $diagnosticLoc.\n"
newCount == logCounter.diagnosticLimit ->
" Limit reached for diagnostics from $diagnosticLoc.\n"
newCount > logCounter.diagnosticLimit -> return
else -> ""
}
@@ -162,15 +187,36 @@ open class LoggerBase(val logCounter: LogCounter) {
emitDiagnostic(dtw, severity, diagnosticLocStr, msg, fullMsg, locationString, mkLocationId)
}
private fun emitDiagnostic(dtw: DiagnosticTrapWriter, severity: Severity, diagnosticLocStr: String, msg: String, fullMsg: String, locationString: String? = null, mkLocationId: () -> Label<DbLocation> = { dtw.unknownLocation }) {
private fun emitDiagnostic(
dtw: DiagnosticTrapWriter,
severity: Severity,
diagnosticLocStr: String,
msg: String,
fullMsg: String,
locationString: String? = null,
mkLocationId: () -> Label<DbLocation> = { dtw.unknownLocation }
) {
val locStr = if (locationString == null) "" else "At " + locationString + ": "
val kind = if (severity <= Severity.WarnHigh) "WARN" else "ERROR"
val logMessage = LogMessage(kind, "Diagnostic($diagnosticLocStr): $locStr$fullMsg")
// We don't actually make the location until after the `return` above
val locationId = mkLocationId()
val diagLabel = dtw.getFreshIdLabel<DbDiagnostic>()
dtw.writeDiagnostics(diagLabel, "CodeQL Kotlin extractor", severity.sev, "", msg, "${logMessage.timestamp} $fullMsg", locationId)
dtw.writeDiagnostic_for(diagLabel, StringLabel("compilation"), file_number, file_number_diagnostic_number++)
dtw.writeDiagnostics(
diagLabel,
"CodeQL Kotlin extractor",
severity.sev,
"",
msg,
"${logMessage.timestamp} $fullMsg",
locationId
)
dtw.writeDiagnostic_for(
diagLabel,
StringLabel("compilation"),
file_number,
file_number_diagnostic_number++
)
logStream.write(logMessage.toJsonLine())
}
@@ -203,6 +249,7 @@ open class LoggerBase(val logCounter: LogCounter) {
diagnostic(dtw, Severity.Warn, msg, extraInfo)
}
}
fun error(dtw: DiagnosticTrapWriter, msg: String, extraInfo: String?) {
if (verbosity >= 1) {
diagnostic(dtw, Severity.Error, msg, extraInfo)
@@ -210,11 +257,12 @@ open class LoggerBase(val logCounter: LogCounter) {
}
fun printLimitedDiagnosticCounts(dtw: DiagnosticTrapWriter) {
for((caller, info) in logCounter.diagnosticInfo) {
for ((caller, info) in logCounter.diagnosticInfo) {
val severity = info.first
val count = info.second
if(count >= logCounter.diagnosticLimit) {
val message = "Total of $count diagnostics (reached limit of ${logCounter.diagnosticLimit}) from $caller."
if (count >= logCounter.diagnosticLimit) {
val message =
"Total of $count diagnostics (reached limit of ${logCounter.diagnosticLimit}) from $caller."
if (verbosity >= 1) {
emitDiagnostic(dtw, severity, "Limit", message, message)
}
@@ -240,9 +288,11 @@ open class Logger(val loggerBase: LoggerBase, open val dtw: DiagnosticTrapWriter
fun trace(msg: String) {
loggerBase.trace(dtw, msg)
}
fun trace(msg: String, exn: Throwable) {
trace(msg + "\n" + exn.stackTraceToString())
}
fun debug(msg: String) {
loggerBase.debug(dtw, msg)
}
@@ -254,9 +304,11 @@ open class Logger(val loggerBase: LoggerBase, open val dtw: DiagnosticTrapWriter
private fun warn(msg: String, extraInfo: String?) {
loggerBase.warn(dtw, msg, extraInfo)
}
fun warn(msg: String, exn: Throwable) {
warn(msg, exn.stackTraceToString())
}
fun warn(msg: String) {
warn(msg, null)
}
@@ -264,24 +316,41 @@ open class Logger(val loggerBase: LoggerBase, open val dtw: DiagnosticTrapWriter
private fun error(msg: String, extraInfo: String?) {
loggerBase.error(dtw, msg, extraInfo)
}
fun error(msg: String) {
error(msg, null)
}
fun error(msg: String, exn: Throwable) {
error(msg, exn.stackTraceToString())
}
}
class FileLogger(loggerBase: LoggerBase, val ftw: FileTrapWriter): Logger(loggerBase, ftw.getDiagnosticTrapWriter()) {
class FileLogger(loggerBase: LoggerBase, val ftw: FileTrapWriter) :
Logger(loggerBase, ftw.getDiagnosticTrapWriter()) {
fun warnElement(msg: String, element: IrElement, exn: Throwable? = null) {
val locationString = ftw.getLocationString(element)
val mkLocationId = { ftw.getLocation(element) }
loggerBase.diagnostic(ftw.getDiagnosticTrapWriter(), Severity.Warn, msg, exn?.stackTraceToString(), locationString, mkLocationId)
loggerBase.diagnostic(
ftw.getDiagnosticTrapWriter(),
Severity.Warn,
msg,
exn?.stackTraceToString(),
locationString,
mkLocationId
)
}
fun errorElement(msg: String, element: IrElement, exn: Throwable? = null) {
val locationString = ftw.getLocationString(element)
val mkLocationId = { ftw.getLocation(element) }
loggerBase.diagnostic(ftw.getDiagnosticTrapWriter(), Severity.Error, msg, exn?.stackTraceToString(), locationString, mkLocationId)
loggerBase.diagnostic(
ftw.getDiagnosticTrapWriter(),
Severity.Error,
msg,
exn?.stackTraceToString(),
locationString,
mkLocationId
)
}
}

1
java/kotlin-extractor/src/main/kotlin/utils/Psi2IrFacade.kt
View File

@@ -7,5 +7,6 @@ import org.jetbrains.kotlin.psi.KtFile
interface Psi2IrFacade {
fun getKtFile(irFile: IrFile): KtFile?
fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement?
}

42
java/kotlin-extractor/src/main/kotlin/utils/TypeResults.kt
View File

@@ -1,30 +1,40 @@
package com.github.codeql
/**
* A triple of a type's database label, its signature for use in callable signatures, and its short name for use
* in all tables that provide a user-facing type name.
* A triple of a type's database label, its signature for use in callable signatures, and its short
* name for use in all tables that provide a user-facing type name.
*
* `signature` is a Java primitive name (e.g. "int"), a fully-qualified class name ("package.OuterClass.InnerClass"),
* or an array ("componentSignature[]")
* Type variables have the signature of their upper bound.
* Type arguments and anonymous types do not have a signature.
* `signature` is a Java primitive name (e.g. "int"), a fully-qualified class name
* ("package.OuterClass.InnerClass"), or an array ("componentSignature[]") Type variables have the
* signature of their upper bound. Type arguments and anonymous types do not have a signature.
*
* `shortName` is a Java primitive name (e.g. "int"), a class short name with Java-style type arguments ("InnerClass<E>" or
* "OuterClass<ConcreteArgument>" or "OtherClass<? extends Bound>") or an array ("componentShortName[]").
* `shortName` is a Java primitive name (e.g. "int"), a class short name with Java-style type
* arguments ("InnerClass<E>" or "OuterClass<ConcreteArgument>" or "OtherClass<? extends Bound>") or
* an array ("componentShortName[]").
*/
data class TypeResultGeneric<SignatureType,out LabelType: AnyDbType>(val id: Label<out LabelType>, val signature: SignatureType?, val shortName: String) {
fun <U: AnyDbType> cast(): TypeResultGeneric<SignatureType,U> {
@Suppress("UNCHECKED_CAST")
return this as TypeResultGeneric<SignatureType,U>
data class TypeResultGeneric<SignatureType, out LabelType : AnyDbType>(
val id: Label<out LabelType>,
val signature: SignatureType?,
val shortName: String
) {
fun <U : AnyDbType> cast(): TypeResultGeneric<SignatureType, U> {
@Suppress("UNCHECKED_CAST") return this as TypeResultGeneric<SignatureType, U>
}
}
data class TypeResultsGeneric<SignatureType>(val javaResult: TypeResultGeneric<SignatureType,DbType>, val kotlinResult: TypeResultGeneric<SignatureType,DbKt_type>)
typealias TypeResult<T> = TypeResultGeneric<String,T>
typealias TypeResultWithoutSignature<T> = TypeResultGeneric<Unit,T>
data class TypeResultsGeneric<SignatureType>(
val javaResult: TypeResultGeneric<SignatureType, DbType>,
val kotlinResult: TypeResultGeneric<SignatureType, DbKt_type>
)
typealias TypeResult<T> = TypeResultGeneric<String, T>
typealias TypeResultWithoutSignature<T> = TypeResultGeneric<Unit, T>
typealias TypeResults = TypeResultsGeneric<String>
typealias TypeResultsWithoutSignatures = TypeResultsGeneric<Unit>
fun <T: AnyDbType> TypeResult<T>.forgetSignature(): TypeResultWithoutSignature<T> {
fun <T : AnyDbType> TypeResult<T>.forgetSignature(): TypeResultWithoutSignature<T> {
return TypeResultWithoutSignature(this.id, Unit, this.shortName)
}

227
java/kotlin-extractor/src/main/kotlin/utils/TypeSubstitution.kt
View File

@@ -33,51 +33,49 @@ import org.jetbrains.kotlin.types.Variance
import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
fun IrType.substituteTypeArguments(params: List<IrTypeParameter>, arguments: List<IrTypeArgument>) =
when(this) {
when (this) {
is IrSimpleType -> substituteTypeArguments(params.map { it.symbol }.zip(arguments).toMap())
else -> this
}
private fun IrSimpleType.substituteTypeArguments(substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>): IrSimpleType {
private fun IrSimpleType.substituteTypeArguments(
substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>
): IrSimpleType {
if (substitutionMap.isEmpty()) return this
val newArguments = arguments.map {
if (it is IrTypeProjection) {
val itType = it.type
if (itType is IrSimpleType) {
subProjectedType(substitutionMap, itType, it.variance)
val newArguments =
arguments.map {
if (it is IrTypeProjection) {
val itType = it.type
if (itType is IrSimpleType) {
subProjectedType(substitutionMap, itType, it.variance)
} else {
it
}
} else {
it
}
} else {
it
}
}
return IrSimpleTypeImpl(
classifier,
isNullable(),
newArguments,
annotations
)
return IrSimpleTypeImpl(classifier, isNullable(), newArguments, annotations)
}
/**
* Returns true if substituting `innerVariance T` into the context `outerVariance []` discards all knowledge about
* what T could be.
* Returns true if substituting `innerVariance T` into the context `outerVariance []` discards all
* knowledge about what T could be.
*
* Note this throws away slightly more information than it could: for example, the projection "in (out List)" can refer to
* any superclass of anything that implements List, which specifically excludes e.g. String, but can't be represented as
* a type projection. The projection "out (in List)" on the other hand really is equivalent to "out Any?", which is to
* say no bound at all.
* Note this throws away slightly more information than it could: for example, the projection "in
* (out List)" can refer to any superclass of anything that implements List, which specifically
* excludes e.g. String, but can't be represented as a type projection. The projection "out (in
* List)" on the other hand really is equivalent to "out Any?", which is to say no bound at all.
*/
private fun conflictingVariance(outerVariance: Variance, innerVariance: Variance) =
(outerVariance == Variance.IN_VARIANCE && innerVariance == Variance.OUT_VARIANCE) ||
(outerVariance == Variance.OUT_VARIANCE && innerVariance == Variance.IN_VARIANCE)
(outerVariance == Variance.OUT_VARIANCE && innerVariance == Variance.IN_VARIANCE)
/**
* When substituting `innerVariance T` into the context `outerVariance []`, returns the variance part of the result
* `resultVariance T`. We already know they don't conflict.
* When substituting `innerVariance T` into the context `outerVariance []`, returns the variance
* part of the result `resultVariance T`. We already know they don't conflict.
*/
private fun combineVariance(outerVariance: Variance, innerVariance: Variance) =
when {
@@ -86,13 +84,20 @@ private fun combineVariance(outerVariance: Variance, innerVariance: Variance) =
else -> Variance.INVARIANT
}
private fun subProjectedType(substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>, t: IrSimpleType, outerVariance: Variance): IrTypeArgument =
private fun subProjectedType(
substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>,
t: IrSimpleType,
outerVariance: Variance
): IrTypeArgument =
substitutionMap[t.classifier]?.let { substitutedTypeArg ->
if (substitutedTypeArg is IrTypeProjection) {
if (conflictingVariance(outerVariance, substitutedTypeArg.variance))
IrStarProjectionImpl
else {
val newProjectedType = substitutedTypeArg.type.let { if (t.isNullable()) it.codeQlWithHasQuestionMark(true) else it }
val newProjectedType =
substitutedTypeArg.type.let {
if (t.isNullable()) it.codeQlWithHasQuestionMark(true) else it
}
val newVariance = combineVariance(outerVariance, substitutedTypeArg.variance)
makeTypeProjection(newProjectedType, newVariance)
}
@@ -102,33 +107,43 @@ private fun subProjectedType(substitutionMap: Map<IrTypeParameterSymbol, IrTypeA
} ?: makeTypeProjection(t.substituteTypeArguments(substitutionMap), outerVariance)
private fun IrTypeArgument.upperBound(context: IrPluginContext) =
when(this) {
when (this) {
is IrStarProjection -> context.irBuiltIns.anyNType
is IrTypeProjection -> when(this.variance) {
Variance.INVARIANT -> this.type
Variance.IN_VARIANCE -> if (this.type.isNullable()) context.irBuiltIns.anyNType else context.irBuiltIns.anyType
Variance.OUT_VARIANCE -> this.type
}
is IrTypeProjection ->
when (this.variance) {
Variance.INVARIANT -> this.type
Variance.IN_VARIANCE ->
if (this.type.isNullable()) context.irBuiltIns.anyNType
else context.irBuiltIns.anyType
Variance.OUT_VARIANCE -> this.type
}
else -> context.irBuiltIns.anyNType
}
private fun IrTypeArgument.lowerBound(context: IrPluginContext) =
when(this) {
when (this) {
is IrStarProjection -> context.irBuiltIns.nothingType
is IrTypeProjection -> when(this.variance) {
Variance.INVARIANT -> this.type
Variance.IN_VARIANCE -> this.type
Variance.OUT_VARIANCE -> if (this.type.isNullable()) context.irBuiltIns.nothingNType else context.irBuiltIns.nothingType
}
is IrTypeProjection ->
when (this.variance) {
Variance.INVARIANT -> this.type
Variance.IN_VARIANCE -> this.type
Variance.OUT_VARIANCE ->
if (this.type.isNullable()) context.irBuiltIns.nothingNType
else context.irBuiltIns.nothingType
}
else -> context.irBuiltIns.nothingType
}
fun IrType.substituteTypeAndArguments(substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>?, useContext: KotlinUsesExtractor.TypeContext, pluginContext: IrPluginContext): IrType =
fun IrType.substituteTypeAndArguments(
substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>?,
useContext: KotlinUsesExtractor.TypeContext,
pluginContext: IrPluginContext
): IrType =
substitutionMap?.let { substMap ->
if (this is IrSimpleType) {
val typeClassifier = this.classifier
substMap[typeClassifier]?.let {
when(useContext) {
when (useContext) {
KotlinUsesExtractor.TypeContext.RETURN -> it.upperBound(pluginContext)
else -> it.lowerBound(pluginContext)
}
@@ -139,42 +154,41 @@ fun IrType.substituteTypeAndArguments(substitutionMap: Map<IrTypeParameterSymbol
} ?: this
object RawTypeAnnotation {
// Much of this is taken from JvmGeneratorExtensionsImpl.kt, which is not easily accessible in plugin context.
// The constants "kotlin.internal.ir" and "RawType" could be referred to symbolically, but they move package
// Much of this is taken from JvmGeneratorExtensionsImpl.kt, which is not easily accessible in
// plugin context.
// The constants "kotlin.internal.ir" and "RawType" could be referred to symbolically, but they
// move package
// between different versions of the Kotlin compiler.
val annotationConstructor: IrConstructorCall by lazy {
val irInternalPackage = FqName("kotlin.internal.ir")
val parent = IrExternalPackageFragmentImpl(
DescriptorlessExternalPackageFragmentSymbol(),
irInternalPackage
)
val annoClass = IrFactoryImpl.buildClass {
kind = ClassKind.ANNOTATION_CLASS
name = irInternalPackage.child(Name.identifier("RawType")).shortName()
}.apply {
createImplicitParameterDeclarationWithWrappedDescriptor()
this.parent = parent
addConstructor {
isPrimary = true
}
}
val parent =
IrExternalPackageFragmentImpl(
DescriptorlessExternalPackageFragmentSymbol(),
irInternalPackage
)
val annoClass =
IrFactoryImpl.buildClass {
kind = ClassKind.ANNOTATION_CLASS
name = irInternalPackage.child(Name.identifier("RawType")).shortName()
}
.apply {
createImplicitParameterDeclarationWithWrappedDescriptor()
this.parent = parent
addConstructor { isPrimary = true }
}
val constructor = annoClass.constructors.single()
IrConstructorCallImpl.fromSymbolOwner(
constructor.constructedClassType,
constructor.symbol
)
IrConstructorCallImpl.fromSymbolOwner(constructor.constructedClassType, constructor.symbol)
}
}
fun IrType.toRawType(): IrType =
when(this) {
when (this) {
is IrSimpleType -> {
when(val owner = this.classifier.owner) {
when (val owner = this.classifier.owner) {
is IrClass -> {
if (this.arguments.isNotEmpty())
this.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor))
else
this
else this
}
is IrTypeParameter -> owner.superTypes[0].toRawType()
else -> this
@@ -187,29 +201,31 @@ fun IrClass.toRawType(): IrType {
val result = this.typeWith(listOf())
return if (this.typeParameters.isNotEmpty())
result.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor))
else
result
else result
}
fun IrTypeArgument.withQuestionMark(b: Boolean): IrTypeArgument =
when(this) {
when (this) {
is IrStarProjection -> this
is IrTypeProjection ->
this.type.let { when(it) {
is IrSimpleType -> if (it.isNullable() == b) this else makeTypeProjection(it.codeQlWithHasQuestionMark(b), this.variance)
else -> this
}}
this.type.let {
when (it) {
is IrSimpleType ->
if (it.isNullable() == b) this
else makeTypeProjection(it.codeQlWithHasQuestionMark(b), this.variance)
else -> this
}
}
else -> this
}
typealias TypeSubstitution = (IrType, KotlinUsesExtractor.TypeContext, IrPluginContext) -> IrType
private fun matchingTypeParameters(l: IrTypeParameter?, r: IrTypeParameter): Boolean {
if (l === r)
return true
if (l == null)
return false
// Special case: match List's E and MutableList's E, for example, because in the JVM lowering they will map to the same thing.
if (l === r) return true
if (l == null) return false
// Special case: match List's E and MutableList's E, for example, because in the JVM lowering
// they will map to the same thing.
val lParent = l.parent as? IrClass ?: return false
val rParent = r.parent as? IrClass ?: return false
val lJavaId = getJavaEquivalentClassId(lParent) ?: lParent.classId
@@ -217,34 +233,51 @@ private fun matchingTypeParameters(l: IrTypeParameter?, r: IrTypeParameter): Boo
}
// Returns true if type is C<T1, T2, ...> where C is declared `class C<T1, T2, ...> { ... }`
fun isUnspecialised(paramsContainer: IrTypeParametersContainer, args: List<IrTypeArgument>, logger: Logger): Boolean {
fun isUnspecialised(
paramsContainer: IrTypeParametersContainer,
args: List<IrTypeArgument>,
logger: Logger
): Boolean {
return isUnspecialised(paramsContainer, args, logger, paramsContainer)
}
private fun isUnspecialised(paramsContainer: IrTypeParametersContainer, args: List<IrTypeArgument>, logger: Logger, origParamsContainer: IrTypeParametersContainer): Boolean {
val unspecialisedHere = paramsContainer.typeParameters.zip(args).all { paramAndArg ->
(paramAndArg.second as? IrTypeProjection)?.let {
// Type arg refers to the class' own type parameter?
it.variance == Variance.INVARIANT &&
matchingTypeParameters(it.type.classifierOrNull?.owner as? IrTypeParameter, paramAndArg.first)
} ?: false
}
private fun isUnspecialised(
paramsContainer: IrTypeParametersContainer,
args: List<IrTypeArgument>,
logger: Logger,
origParamsContainer: IrTypeParametersContainer
): Boolean {
val unspecialisedHere =
paramsContainer.typeParameters.zip(args).all { paramAndArg ->
(paramAndArg.second as? IrTypeProjection)?.let {
// Type arg refers to the class' own type parameter?
it.variance == Variance.INVARIANT &&
matchingTypeParameters(
it.type.classifierOrNull?.owner as? IrTypeParameter,
paramAndArg.first
)
} ?: false
}
val remainingArgs = args.drop(paramsContainer.typeParameters.size)
val parentTypeContainer = paramsContainer.parents.firstIsInstanceOrNull<IrTypeParametersContainer>()
val parentTypeContainer =
paramsContainer.parents.firstIsInstanceOrNull<IrTypeParametersContainer>()
val parentUnspecialised = when {
remainingArgs.isEmpty() -> true
parentTypeContainer == null -> {
logger.error("Found more type arguments than parameters: ${origParamsContainer.kotlinFqName.asString()}")
false
val parentUnspecialised =
when {
remainingArgs.isEmpty() -> true
parentTypeContainer == null -> {
logger.error(
"Found more type arguments than parameters: ${origParamsContainer.kotlinFqName.asString()}"
)
false
}
else -> isUnspecialised(parentTypeContainer, remainingArgs, logger, origParamsContainer)
}
else -> isUnspecialised(parentTypeContainer, remainingArgs, logger, origParamsContainer)
}
return unspecialisedHere && parentUnspecialised
}
// Returns true if type is C<T1, T2, ...> where C is declared `class C<T1, T2, ...> { ... }`
fun isUnspecialised(type: IrSimpleType, logger: Logger) = (type.classifier.owner as? IrClass)?.let {
isUnspecialised(it, type.arguments, logger)
} ?: false
fun isUnspecialised(type: IrSimpleType, logger: Logger) =
(type.classifier.owner as? IrClass)?.let { isUnspecialised(it, type.arguments, logger) }
?: false

6
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/CommentExtractorLighterAST.kt
View File

@@ -3,7 +3,11 @@ package com.github.codeql.comments
import com.github.codeql.*
import org.jetbrains.kotlin.ir.declarations.*
class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFile, fileLabel: Label<out DbFile>): CommentExtractor(fileExtractor, file, fileLabel) {
class CommentExtractorLighterAST(
fileExtractor: KotlinFileExtractor,
file: IrFile,
fileLabel: Label<out DbFile>
) : CommentExtractor(fileExtractor, file, fileLabel) {
// We don't support LighterAST with old Kotlin versions
fun extract(): Boolean {
return false

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/FileEntry.kt
View File

@@ -2,4 +2,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.SourceManager
typealias FileEntry = SourceManager.FileEntry
typealias FileEntry = SourceManager.FileEntry

3
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/IrSymbolInternals.kt
View File

@@ -1,4 +1,3 @@
package org.jetbrains.kotlin.ir.symbols
@RequiresOptIn(level = RequiresOptIn.Level.WARNING)
annotation class IrSymbolInternals
@RequiresOptIn(level = RequiresOptIn.Level.WARNING) annotation class IrSymbolInternals

10
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/IsUnderscoreParameter.kt
View File

@@ -11,9 +11,11 @@ import org.jetbrains.kotlin.utils.addToStdlib.safeAs
fun isUnderscoreParameter(vp: IrValueParameter) =
try {
DescriptorToSourceUtils.getSourceFromDescriptor(vp.descriptor)
?.safeAs<KtParameter>()?.isSingleUnderscore == true
} catch(e: NotImplementedError) {
// Some kinds of descriptor throw in `getSourceFromDescriptor` as that method is not normally expected to
?.safeAs<KtParameter>()
?.isSingleUnderscore == true
} catch (e: NotImplementedError) {
// Some kinds of descriptor throw in `getSourceFromDescriptor` as that method is not
// normally expected to
// be applied to synthetic functions.
false
}
}

4
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/JavaBinarySourceElement.kt
View File

@@ -7,5 +7,5 @@ import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
We need this class to exist, but the compiler will never give us an
instance of it.
*/
abstract class JavaBinarySourceElement private constructor(val javaClass: BinaryJavaClass): SourceElement {
}
abstract class JavaBinarySourceElement private constructor(val javaClass: BinaryJavaClass) :
SourceElement {}

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/Kotlin2ComponentRegistrar.kt
View File

@@ -3,8 +3,8 @@
package com.github.codeql
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
@OptIn(ExperimentalCompilerApi::class)
abstract class Kotlin2ComponentRegistrar : ComponentRegistrar {

11
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/LinesOfCodeLighterAST.kt
View File

@@ -2,18 +2,17 @@ package com.github.codeql
import org.jetbrains.kotlin.ir.declarations.*
class LinesOfCodeLighterAST(
val logger: FileLogger,
val tw: FileTrapWriter,
val file: IrFile
) {
class LinesOfCodeLighterAST(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
// We don't support LighterAST with old Kotlin versions
fun linesOfCodeInFile(@Suppress("UNUSED_PARAMETER") id: Label<DbFile>): Boolean {
return false
}
// We don't support LighterAST with old Kotlin versions
fun linesOfCodeInDeclaration(@Suppress("UNUSED_PARAMETER") d: IrDeclaration, @Suppress("UNUSED_PARAMETER") id: Label<out DbSourceline>): Boolean {
fun linesOfCodeInDeclaration(
@Suppress("UNUSED_PARAMETER") d: IrDeclaration,
@Suppress("UNUSED_PARAMETER") id: Label<out DbSourceline>
): Boolean {
return false
}
}

12
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/ReferenceEntity.kt
View File

@@ -14,12 +14,20 @@ fun getClassByClassId(pluginContext: IrPluginContext, id: ClassId): IrClassSymbo
return getClassByFqName(pluginContext, id.asSingleFqName())
}
fun getFunctionsByFqName(pluginContext: IrPluginContext, pkgName: FqName, name: Name): Collection<IrSimpleFunctionSymbol> {
fun getFunctionsByFqName(
pluginContext: IrPluginContext,
pkgName: FqName,
name: Name
): Collection<IrSimpleFunctionSymbol> {
val fqName = pkgName.child(name)
return pluginContext.referenceFunctions(fqName)
}
fun getPropertiesByFqName(pluginContext: IrPluginContext, pkgName: FqName, name: Name): Collection<IrPropertySymbol> {
fun getPropertiesByFqName(
pluginContext: IrPluginContext,
pkgName: FqName,
name: Name
): Collection<IrPropertySymbol> {
val fqName = pkgName.child(name)
return pluginContext.referenceProperties(fqName)
}

1
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/SyntheticBodyKind.kt
View File

@@ -3,4 +3,3 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.expressions.IrSyntheticBodyKind
val kind_ENUM_ENTRIES: IrSyntheticBodyKind? = null

3
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/Types.kt
View File

@@ -3,5 +3,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.jvm.codegen.isRawType
import org.jetbrains.kotlin.ir.types.IrSimpleType
fun IrSimpleType.isRawType() = this.isRawType()
fun IrSimpleType.isRawType() = this.isRawType()

4
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/allOverriddenIncludingSelf.kt
View File

@@ -1,6 +1,6 @@
package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
import org.jetbrains.kotlin.backend.common.ir.allOverridden
import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
fun IrSimpleFunction.allOverriddenIncludingSelf() = this.allOverridden(includeSelf = true)
fun IrSimpleFunction.allOverriddenIncludingSelf() = this.allOverridden(includeSelf = true)

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/annotationType.kt
View File

@@ -5,4 +5,4 @@ import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
@OptIn(ObsoleteDescriptorBasedAPI::class)
fun getAnnotationType(context: IrPluginContext) =
context.typeTranslator.translateType(context.builtIns.annotationType)
context.typeTranslator.translateType(context.builtIns.annotationType)

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/copyTo.kt
View File

@@ -1,7 +1,7 @@
package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.common.ir.copyTo
import org.jetbrains.kotlin.ir.declarations.IrFunction
import org.jetbrains.kotlin.ir.declarations.IrValueParameter
import org.jetbrains.kotlin.backend.common.ir.copyTo
fun copyParameterToFunction(p: IrValueParameter, f: IrFunction) = p.copyTo(f)

5
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/createImplicitParameterDeclarationWithWrappedDescriptor.kt
View File

@@ -1,6 +1,7 @@
package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.backend.common.ir.createImplicitParameterDeclarationWithWrappedDescriptor
import org.jetbrains.kotlin.ir.declarations.IrClass
fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() = this.createImplicitParameterDeclarationWithWrappedDescriptor()
fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() =
this.createImplicitParameterDeclarationWithWrappedDescriptor()

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/getFileClassFqName.kt
View File

@@ -1,7 +1,7 @@
package com.github.codeql.utils.versions
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.name.FqName
fun getFileClassFqName(@Suppress("UNUSED_PARAMETER") d: IrDeclaration): FqName? {
return null

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/getKotlinType.kt
View File

@@ -3,4 +3,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.types.IrSimpleType
import org.jetbrains.kotlin.ir.types.impl.IrTypeBase
fun getKotlinType(s: IrSimpleType) = (s as? IrTypeBase)?.kotlinType
fun getKotlinType(s: IrSimpleType) = (s as? IrTypeBase)?.kotlinType

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/parents.kt
View File

@@ -1,4 +1,3 @@
package org.jetbrains.kotlin.ir.util
import org.jetbrains.kotlin.backend.common.lower.parents as kParents
@@ -10,4 +9,3 @@ val IrDeclaration.parents: Sequence<IrDeclarationParent>
val IrDeclaration.parentsWithSelf: Sequence<IrDeclarationParent>
get() = this.kParentsWithSelf

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/withHasQuestionMark.kt
View File

@@ -3,6 +3,6 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.types.IrType
import org.jetbrains.kotlin.ir.types.withHasQuestionMark
fun IrType.codeQlWithHasQuestionMark(b : Boolean): IrType {
fun IrType.codeQlWithHasQuestionMark(b: Boolean): IrType {
return this.withHasQuestionMark(b)
}

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_20/FileEntry.kt
View File

@@ -2,4 +2,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.IrFileEntry
typealias FileEntry = IrFileEntry
typealias FileEntry = IrFileEntry

4
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_20/Psi2Ir.kt
View File

@@ -1,16 +1,16 @@
package com.github.codeql.utils.versions
import com.github.codeql.utils.Psi2IrFacade
import com.intellij.psi.PsiElement
import org.jetbrains.kotlin.backend.common.psi.PsiSourceManager
import org.jetbrains.kotlin.backend.jvm.ir.getKtFile
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.psi.KtFile
import com.github.codeql.utils.Psi2IrFacade
fun getPsi2Ir(): Psi2IrFacade? = Psi2Ir()
private class Psi2Ir(): Psi2IrFacade {
private class Psi2Ir() : Psi2IrFacade {
override fun getKtFile(irFile: IrFile): KtFile? {
return irFile.getKtFile()
}

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_6_0/Functions.kt
View File

@@ -2,4 +2,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
fun functionN(pluginContext: IrPluginContext) = pluginContext.irBuiltIns::functionN
fun functionN(pluginContext: IrPluginContext) = pluginContext.irBuiltIns::functionN

3
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_6_0/annotationType.kt
View File

@@ -2,5 +2,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
fun getAnnotationType(context: IrPluginContext) =
context.irBuiltIns.annotationType
fun getAnnotationType(context: IrPluginContext) = context.irBuiltIns.annotationType

3
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_6_20/IsUnderscoreParameter.kt
View File

@@ -3,4 +3,5 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
import org.jetbrains.kotlin.ir.declarations.IrValueParameter
fun isUnderscoreParameter(vp: IrValueParameter) = vp.origin == IrDeclarationOrigin.UNDERSCORE_PARAMETER
fun isUnderscoreParameter(vp: IrValueParameter) =
vp.origin == IrDeclarationOrigin.UNDERSCORE_PARAMETER

3
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_6_20/Types.kt
View File

@@ -3,5 +3,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.backend.jvm.ir.isRawType
import org.jetbrains.kotlin.ir.types.IrSimpleType
fun IrSimpleType.isRawType() = this.isRawType()
fun IrSimpleType.isRawType() = this.isRawType()

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_7_0/getFileClassFqName.kt
View File

@@ -1,10 +1,10 @@
package com.github.codeql.utils.versions
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.IrField
import org.jetbrains.kotlin.ir.declarations.IrMemberWithContainerSource
import org.jetbrains.kotlin.load.kotlin.FacadeClassSource
import org.jetbrains.kotlin.name.FqName
fun getFileClassFqName(d: IrDeclaration): FqName? {
// d is in a file class.

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_7_0/getKotlinType.kt
View File

@@ -2,4 +2,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.types.IrSimpleType
fun getKotlinType(s: IrSimpleType) = s.kotlinType
fun getKotlinType(s: IrSimpleType) = s.kotlinType

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_7_0/withHasQuestionMark.kt
View File

@@ -4,7 +4,7 @@ import org.jetbrains.kotlin.ir.types.IrType
import org.jetbrains.kotlin.ir.types.makeNotNull
import org.jetbrains.kotlin.ir.types.makeNullable
fun IrType.codeQlWithHasQuestionMark(b : Boolean): IrType {
fun IrType.codeQlWithHasQuestionMark(b: Boolean): IrType {
if (b) {
return this.makeNullable()
} else {

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_7_20/allOverriddenIncludingSelf.kt
View File

@@ -3,4 +3,4 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
import org.jetbrains.kotlin.ir.util.allOverridden
fun IrSimpleFunction.allOverriddenIncludingSelf() = this.allOverridden(includeSelf = true)
fun IrSimpleFunction.allOverriddenIncludingSelf() = this.allOverridden(includeSelf = true)

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_7_20/copyTo.kt
View File

@@ -4,4 +4,4 @@ import org.jetbrains.kotlin.ir.declarations.IrFunction
import org.jetbrains.kotlin.ir.declarations.IrValueParameter
import org.jetbrains.kotlin.ir.util.copyTo
fun copyParameterToFunction(p: IrValueParameter, f: IrFunction) = p.copyTo(f)
fun copyParameterToFunction(p: IrValueParameter, f: IrFunction) = p.copyTo(f)

3
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_7_20/createImplicitParameterDeclarationWithWrappedDescriptor.kt
View File

@@ -3,4 +3,5 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.ir.util.createImplicitParameterDeclarationWithWrappedDescriptor
fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() = this.createImplicitParameterDeclarationWithWrappedDescriptor()
fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() =
this.createImplicitParameterDeclarationWithWrappedDescriptor()

15
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_8_0/ReferenceEntity.kt
View File

@@ -1,10 +1,9 @@
package com.github.codeql.utils
import org.jetbrains.kotlin.backend.common.extensions.FirIncompatiblePluginAPI
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.ir.symbols.*
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.CallableId
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
@@ -17,12 +16,20 @@ fun getClassByClassId(pluginContext: IrPluginContext, id: ClassId): IrClassSymbo
return pluginContext.referenceClass(id)
}
fun getFunctionsByFqName(pluginContext: IrPluginContext, pkgName: FqName, name: Name): Collection<IrSimpleFunctionSymbol> {
fun getFunctionsByFqName(
pluginContext: IrPluginContext,
pkgName: FqName,
name: Name
): Collection<IrSimpleFunctionSymbol> {
val id = CallableId(pkgName, name)
return pluginContext.referenceFunctions(id)
}
fun getPropertiesByFqName(pluginContext: IrPluginContext, pkgName: FqName, name: Name): Collection<IrPropertySymbol> {
fun getPropertiesByFqName(
pluginContext: IrPluginContext,
pkgName: FqName,
name: Name
): Collection<IrPropertySymbol> {
val id = CallableId(pkgName, name)
return pluginContext.referenceProperties(id)
}

1
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_8_0/SyntheticBodyKind.kt
View File

@@ -3,4 +3,3 @@ package com.github.codeql.utils.versions
import org.jetbrains.kotlin.ir.expressions.IrSyntheticBodyKind
val kind_ENUM_ENTRIES: IrSyntheticBodyKind? = IrSyntheticBodyKind.ENUM_ENTRIES

94
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_9_0-Beta/CommentExtractorLighterAST.kt
View File

@@ -4,21 +4,26 @@ import com.github.codeql.*
import com.intellij.lang.LighterASTNode
import com.intellij.util.diff.FlyweightCapableTreeStructure
import org.jetbrains.kotlin.fir.backend.FirMetadataSource
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET
import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
import org.jetbrains.kotlin.ir.visitors.acceptVoid
import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
import org.jetbrains.kotlin.kdoc.lexer.KDocTokens
import org.jetbrains.kotlin.lexer.KtTokens
import org.jetbrains.kotlin.util.getChildren
class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFile, fileLabel: Label<out DbFile>): CommentExtractor(fileExtractor, file, fileLabel) {
class CommentExtractorLighterAST(
fileExtractor: KotlinFileExtractor,
file: IrFile,
fileLabel: Label<out DbFile>
) : CommentExtractor(fileExtractor, file, fileLabel) {
// Returns true if it extracted the comments; false otherwise.
fun extract(): Boolean {
val sourceElement = (file.metadata as? FirMetadataSource.File)?.files?.elementAtOrNull(0)?.source
val sourceElement =
(file.metadata as? FirMetadataSource.File)?.files?.elementAtOrNull(0)?.source
val treeStructure = sourceElement?.treeStructure
if (treeStructure == null) {
return false
@@ -33,34 +38,46 @@ class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFil
fun LighterASTNode.isKDocComment() = this.tokenType == KDocTokens.KDOC
val kDocOwners = mutableMapOf<Int, MutableList<IrElement>>()
val visitor = object : IrElementVisitorVoid {
override fun visitElement(element: IrElement) {
val metadata = (element as? IrMetadataSourceOwner)?.metadata
val sourceElement = (metadata as? FirMetadataSource)?.fir?.source
val treeStructure = sourceElement?.treeStructure
val visitor =
object : IrElementVisitorVoid {
override fun visitElement(element: IrElement) {
val metadata = (element as? IrMetadataSourceOwner)?.metadata
val sourceElement = (metadata as? FirMetadataSource)?.fir?.source
val treeStructure = sourceElement?.treeStructure
if (treeStructure != null) {
sourceElement.lighterASTNode.getChildren(treeStructure).firstOrNull { it.isKDocComment() }
?.let { kDoc ->
// LighterASTNodes are not stable, so we can't
// use the node itself as the key. But the
// startOffset should uniquely identify them
// anyway.
val startOffset = kDoc.startOffset
if (startOffset != UNDEFINED_OFFSET && startOffset != SYNTHETIC_OFFSET) {
kDocOwners.getOrPut(startOffset, {mutableListOf<IrElement>()}).add(element)
if (treeStructure != null) {
sourceElement.lighterASTNode
.getChildren(treeStructure)
.firstOrNull { it.isKDocComment() }
?.let { kDoc ->
// LighterASTNodes are not stable, so we can't
// use the node itself as the key. But the
// startOffset should uniquely identify them
// anyway.
val startOffset = kDoc.startOffset
if (
startOffset != UNDEFINED_OFFSET &&
startOffset != SYNTHETIC_OFFSET
) {
kDocOwners
.getOrPut(startOffset, { mutableListOf<IrElement>() })
.add(element)
}
}
}
}
}
element.acceptChildrenVoid(this)
element.acceptChildrenVoid(this)
}
}
}
file.acceptVoid(visitor)
return kDocOwners
}
private fun extractComments(node: LighterASTNode, treeStructure: FlyweightCapableTreeStructure<LighterASTNode>, owners: Map<Int, List<IrElement>>) {
private fun extractComments(
node: LighterASTNode,
treeStructure: FlyweightCapableTreeStructure<LighterASTNode>,
owners: Map<Int, List<IrElement>>
) {
node.getChildren(treeStructure).forEach {
if (KtTokens.COMMENTS.contains(it.tokenType)) {
extractComment(it, owners)
@@ -71,21 +88,22 @@ class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFil
}
private fun extractComment(comment: LighterASTNode, owners: Map<Int, List<IrElement>>) {
val type: CommentType = when (comment.tokenType) {
KtTokens.EOL_COMMENT -> {
CommentType.SingleLine
val type: CommentType =
when (comment.tokenType) {
KtTokens.EOL_COMMENT -> {
CommentType.SingleLine
}
KtTokens.BLOCK_COMMENT -> {
CommentType.Block
}
KtTokens.DOC_COMMENT -> {
CommentType.Doc
}
else -> {
logger.warn("Unhandled comment token type: ${comment.tokenType}")
return
}
}
KtTokens.BLOCK_COMMENT -> {
CommentType.Block
}
KtTokens.DOC_COMMENT -> {
CommentType.Doc
}
else -> {
logger.warn("Unhandled comment token type: ${comment.tokenType}")
return
}
}
val commentLabel = tw.getFreshIdLabel<DbKtcomment>()
tw.writeKtComments(commentLabel, type.value, comment.toString())

2
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_9_0-Beta/Kotlin2ComponentRegistrar.kt
View File

@@ -3,8 +3,8 @@
package com.github.codeql
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
@OptIn(ExperimentalCompilerApi::class)
abstract class Kotlin2ComponentRegistrar : ComponentRegistrar {

42
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_9_0-Beta/LinesOfCodeLighterAST.kt
View File

@@ -2,23 +2,19 @@ package com.github.codeql
import com.intellij.lang.LighterASTNode
import com.intellij.util.diff.FlyweightCapableTreeStructure
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.fir.backend.FirMetadataSource
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.KtSourceElement
import org.jetbrains.kotlin.fir.backend.FirMetadataSource
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.lexer.KtTokens
import org.jetbrains.kotlin.util.getChildren
class LinesOfCodeLighterAST(
val logger: FileLogger,
val tw: FileTrapWriter,
val file: IrFile
) {
class LinesOfCodeLighterAST(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
val fileEntry = file.fileEntry
fun linesOfCodeInFile(id: Label<DbFile>): Boolean {
val sourceElement = (file.metadata as? FirMetadataSource.File)?.files?.elementAtOrNull(0)?.source
val sourceElement =
(file.metadata as? FirMetadataSource.File)?.files?.elementAtOrNull(0)?.source
if (sourceElement == null) {
return false
}
@@ -42,7 +38,11 @@ class LinesOfCodeLighterAST(
return true
}
private fun linesOfCodeInLighterAST(id: Label<out DbSourceline>, e: IrElement, s: KtSourceElement) {
private fun linesOfCodeInLighterAST(
id: Label<out DbSourceline>,
e: IrElement,
s: KtSourceElement
) {
val rootStartOffset = s.startOffset
val rootEndOffset = s.endOffset
if (rootStartOffset < 0 || rootEndOffset < 0) {
@@ -63,14 +63,28 @@ class LinesOfCodeLighterAST(
val treeStructure = s.treeStructure
processSubtree(e, treeStructure, rootFirstLine, rootLastLine, lineContents, s.lighterASTNode)
processSubtree(
e,
treeStructure,
rootFirstLine,
rootLastLine,
lineContents,
s.lighterASTNode
)
val code = lineContents.count { it.containsCode }
val comment = lineContents.count { it.containsComment }
tw.writeNumlines(id, numLines, code, comment)
}
private fun processSubtree(e: IrElement, treeStructure: FlyweightCapableTreeStructure<LighterASTNode>, rootFirstLine: Int, rootLastLine: Int, lineContents: Array<LineContent>, node: LighterASTNode) {
private fun processSubtree(
e: IrElement,
treeStructure: FlyweightCapableTreeStructure<LighterASTNode>,
rootFirstLine: Int,
rootLastLine: Int,
lineContents: Array<LineContent>,
node: LighterASTNode
) {
if (KtTokens.WHITESPACES.contains(node.tokenType)) {
return
}
@@ -120,7 +134,7 @@ class LinesOfCodeLighterAST(
}
}
} else {
for(child in children) {
for (child in children) {
processSubtree(e, treeStructure, rootFirstLine, rootLastLine, lineContents, child)
}
}