hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-18 01:33:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

External class extraction prototype

Browse Source
This commit is contained in:
Chris Smowton
2021-09-30 12:27:48 +01:00
committed by Ian Lynagh
parent e9b249855b
commit 2cc003ff0e
51 changed files with 11119 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt
View File

@@ -22,6 +22,10 @@ import java.io.StringWriter
import java.nio.file.Files
import java.nio.file.Paths
import java.util.*
import com.intellij.openapi.vfs.StandardFileSystems
import com.semmle.extractor.java.OdasaOutput
import com.semmle.extractor.java.OdasaOutput.TrapFileManager
import com.semmle.util.files.FileUtil
import kotlin.system.exitProcess
class KotlinExtractorExtension(private val invocationTrapFile: String, private val checkTrapIdentical: Boolean) : IrGenerationExtension {
@@ -40,10 +44,13 @@ class KotlinExtractorExtension(private val invocationTrapFile: String, private v
val logger = Logger(logCounter, tw)
logger.info("Extraction started")
logger.flush()
// FIXME: FileUtil expects a static global logger
// which should be provided by SLF4J's factory facility. For now we set it here.
FileUtil.logger = logger
val srcDir = File(System.getenv("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR").takeUnless { it.isNullOrEmpty() } ?: "kotlin-extractor/src")
srcDir.mkdirs()
moduleFragment.files.mapIndexed { index: Int, file: IrFile ->
val fileTrapWriter = FileTrapWriter(lm, invocationTrapFileBW, file)
val fileTrapWriter = SourceFileTrapWriter(lm, invocationTrapFileBW, file)
fileTrapWriter.writeCompilation_compiling_files(compilation, index, fileTrapWriter.fileId)
doFile(invocationTrapFile, fileTrapWriter, checkTrapIdentical, logCounter, trapDir, srcDir, file, pluginContext)
}
@@ -124,9 +131,11 @@ fun doFile(invocationTrapFile: String,
val trapTmpFile = File.createTempFile("$filePath.", ".trap.tmp", trapFileDir)
trapTmpFile.bufferedWriter().use { trapFileBW ->
trapFileBW.write("// Generated by invocation ${invocationTrapFile.replace("\n", "\n// ")}\n")
val tw = FileTrapWriter(TrapLabelManager(), trapFileBW, file)
val fileExtractor = KotlinFileExtractor(logger, tw, file, pluginContext)
val tw = SourceFileTrapWriter(TrapLabelManager(), trapFileBW, file)
val externalClassExtractor = ExternalClassExtractor(logger, file.path, pluginContext)
val fileExtractor = KotlinSourceFileExtractor(logger, tw, file, externalClassExtractor, pluginContext)
fileExtractor.extractFileContents(tw.fileId)
externalClassExtractor.extractExternalClasses()
}
if (checkTrapIdentical && trapFile.exists()) {
if(equivalentTrap(trapTmpFile, trapFile)) {
@@ -160,7 +169,51 @@ fun <T> fakeLabel(): Label<T> {
return IntLabel(0)
}
class KotlinFileExtractor(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile, val pluginContext: IrPluginContext) {
class ExternalClassExtractor(val logger: FileLogger, val sourceFilePath: String, val pluginContext: IrPluginContext) {
val externalClassesDone = HashSet<IrClass>()
val externalClassWorkList = ArrayList<IrClass>()
fun extractLater(c: IrClass): Boolean {
val ret = externalClassesDone.add(c)
if(ret) externalClassWorkList.add(c)
return ret
}
fun extractExternalClasses() {
val output = OdasaOutput(false, logger)
output.setCurrentSourceFile(File(sourceFilePath))
do {
val nextBatch = ArrayList<IrClass>(externalClassWorkList)
externalClassWorkList.clear()
nextBatch.forEach { irClass ->
output.getTrapLockerForClassFile(irClass).useAC { locker ->
locker.getTrapFileManager().useAC { manager ->
if(manager == null) {
logger.info("Skipping extracting class ${irClass.name}")
return
}
manager.getFile().bufferedWriter().use { trapFileBW ->
val tw = ClassFileTrapWriter(TrapLabelManager(), trapFileBW, getIrClassBinaryPath(irClass))
val fileExtractor = KotlinFileExtractor(logger, tw, manager, this, pluginContext)
fileExtractor.extractClassSource(irClass)
}
}
}
}
} while (!externalClassWorkList.isEmpty());
}
}
class KotlinSourceFileExtractor(
logger: FileLogger,
tw: FileTrapWriter,
val file: IrFile,
externalClassExtractor: ExternalClassExtractor,
pluginContext: IrPluginContext) :
KotlinFileExtractor(logger, tw, null, externalClassExtractor, pluginContext) {
val fileClass by lazy {
extractFileClass(file)
}
@@ -171,7 +224,7 @@ class KotlinFileExtractor(val logger: FileLogger, val tw: FileTrapWriter, val fi
val pkgId = extractPackage(pkg)
tw.writeHasLocation(id, locId)
tw.writeCupackage(id, pkgId)
file.declarations.map { extractDeclaration(it) }
file.declarations.map { extractDeclaration(it, fileClass) }
CommentExtractor(this).extract()
}
@@ -207,6 +260,15 @@ class KotlinFileExtractor(val logger: FileLogger, val tw: FileTrapWriter, val fi
return id
}
}
open class KotlinFileExtractor(
val logger: FileLogger,
val tw: FileTrapWriter,
val dependencyCollector: TrapFileManager?,
val externalClassExtractor: ExternalClassExtractor,
val pluginContext: IrPluginContext) {
fun usePackage(pkg: String): Label<out DbPackage> {
return extractPackage(pkg)
}
@@ -219,14 +281,14 @@ class KotlinFileExtractor(val logger: FileLogger, val tw: FileTrapWriter, val fi
return id
}
fun extractDeclaration(declaration: IrDeclaration) {
fun extractDeclaration(declaration: IrDeclaration, parentId: Label<out DbReftype>) {
when (declaration) {
is IrClass -> extractClassSource(declaration)
is IrFunction -> extractFunction(declaration)
is IrFunction -> extractFunction(declaration, parentId)
is IrAnonymousInitializer -> {
// Leaving this intentionally empty. init blocks are extracted during class extraction.
}
is IrProperty -> extractProperty(declaration)
is IrProperty -> extractProperty(declaration, parentId)
else -> logger.warnElement(Severity.ErrorSevere, "Unrecognised IrDeclaration: " + declaration.javaClass, declaration)
}
}
@@ -476,6 +538,11 @@ class X {
return id
}
fun extractExternalClassLater(c: IrClass) {
dependencyCollector?.addDependency(c)
externalClassExtractor.extractLater(c)
}
private fun getClassLabel(c: IrClass, typeArgs: List<IrTypeArgument>): String {
val pkg = c.packageFqName?.asString() ?: ""
val cls = c.name.asString()
@@ -548,7 +615,7 @@ class X {
// so for now we extract the source class for those too
if (c.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB ||
c.origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB) {
extractClassSource(c)
extractExternalClassLater(c)
}
})
}
@@ -598,7 +665,7 @@ class X {
extractClassCommon(c, id)
c.typeParameters.map { extractTypeParameter(it) }
c.declarations.map { extractDeclaration(it) }
c.declarations.map { extractDeclaration(it, id) }
extractObjectInitializerFunction(c, id)
return id
@@ -821,7 +888,7 @@ class X {
}
}
fun extractFunction(f: IrFunction) {
fun extractFunction(f: IrFunction, parentId: Label<out DbReftype>) {
currentFunction = f
f.typeParameters.map { extractTypeParameter(it) }
@@ -829,16 +896,6 @@ class X {
val locId = tw.getLocation(f)
val signature = "TODO"
val parent = f.parent
val parentId = when (parent) {
is IrClass -> useClassSource(parent)
is IrFile -> fileClass
else -> {
logger.warnElement(Severity.ErrorSevere, "Unrecognised function parent: " + parent.javaClass, parent)
fakeLabel()
}
}
val id: Label<out DbCallable>
if (f.symbol is IrConstructorSymbol) {
val returnTypeId = useTypeOld(erase(f.returnType))
@@ -880,7 +937,7 @@ class X {
return id
}
fun extractProperty(p: IrProperty) {
fun extractProperty(p: IrProperty, parentId: Label<out DbReftype>) {
val bf = p.backingField
if(bf == null) {
logger.warnElement(Severity.ErrorSevere, "IrProperty without backing field", p)
@@ -888,7 +945,6 @@ class X {
val id = useProperty(p)
val locId = tw.getLocation(p)
val typeId = useTypeOld(bf.type)
val parentId = if (p.parent is IrClass) useClassSource(p.parent as IrClass) else fileClass
tw.writeFields(id, p.name.asString(), typeId, parentId, id)
tw.writeHasLocation(id, locId)
}
@@ -1407,5 +1463,5 @@ class X {
tw.writeKtBreakContinueTargets(id, loopId)
}
}
}

58
java/kotlin-extractor/src/main/kotlin/TrapWriter.kt
View File

@@ -3,6 +3,7 @@ package com.github.codeql
import java.io.BufferedWriter
import java.io.File
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.IrFileEntry
import org.jetbrains.kotlin.ir.declarations.path
import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.ir.declarations.IrVariable
@@ -60,14 +61,42 @@ open class TrapWriter (val lm: TrapLabelManager, val bw: BufferedWriter) {
}
}
class FileTrapWriter (
abstract class SourceOffsetResolver {
abstract fun getLineNumber(offset: Int): Int
abstract fun getColumnNumber(offset: Int): Int
}
class FileSourceOffsetResolver(val fileEntry: IrFileEntry) : SourceOffsetResolver() {
override fun getLineNumber(offset: Int) = fileEntry.getLineNumber(offset)
override fun getColumnNumber(offset: Int) = fileEntry.getLineNumber(offset)
}
object NullSourceOffsetResolver : SourceOffsetResolver() {
override fun getLineNumber(offset: Int) = 0
override fun getColumnNumber(offset: Int) = 0
}
class SourceFileTrapWriter (
lm: TrapLabelManager,
bw: BufferedWriter,
val irFile: IrFile
irFile: IrFile) :
FileTrapWriter(lm, bw, irFile.path, FileSourceOffsetResolver(irFile.fileEntry)) {
}
class ClassFileTrapWriter (
lm: TrapLabelManager,
bw: BufferedWriter,
filePath: String) :
FileTrapWriter(lm, bw, filePath, NullSourceOffsetResolver) {
}
open class FileTrapWriter (
lm: TrapLabelManager,
bw: BufferedWriter,
val filePath: String,
val sourceOffsetResolver: SourceOffsetResolver
): TrapWriter (lm, bw) {
private val fileEntry = irFile.fileEntry
val fileId = {
val filePath = irFile.path
val fileLabel = "@\"$filePath;sourcefile\""
val id: Label<DbFile> = getLabelFor(fileLabel)
writeFiles(id, filePath)
@@ -87,24 +116,23 @@ class FileTrapWriter (
// be a zero-width location. QL doesn't support these, so we translate it
// into a one-width location.
val zeroWidthLoc = !unknownLoc && startOffset == endOffset
val startLine = if(unknownLoc) 0 else fileEntry.getLineNumber(startOffset) + 1
val startColumn = if(unknownLoc) 0 else fileEntry.getColumnNumber(startOffset) + 1
val endLine = if(unknownLoc) 0 else fileEntry.getLineNumber(endOffset) + 1
val endColumn = if(unknownLoc) 0 else fileEntry.getColumnNumber(endOffset)
val startLine = if(unknownLoc) 0 else sourceOffsetResolver.getLineNumber(startOffset) + 1
val startColumn = if(unknownLoc) 0 else sourceOffsetResolver.getColumnNumber(startOffset) + 1
val endLine = if(unknownLoc) 0 else sourceOffsetResolver.getLineNumber(endOffset) + 1
val endColumn = if(unknownLoc) 0 else sourceOffsetResolver.getColumnNumber(endOffset)
val endColumn2 = if(zeroWidthLoc) endColumn + 1 else endColumn
val locFileId: Label<DbFile> = if (unknownLoc) unknownFileId else fileId
return getLocation(locFileId, startLine, startColumn, endLine, endColumn2)
}
fun getLocationString(e: IrElement): String {
val path = irFile.path
if (e.startOffset == -1 && e.endOffset == -1) {
return "unknown location, while processing $path"
return "unknown location, while processing $filePath"
} else {
val startLine = fileEntry.getLineNumber(e.startOffset) + 1
val startColumn = fileEntry.getColumnNumber(e.startOffset) + 1
val endLine = fileEntry.getLineNumber(e.endOffset) + 1
val endColumn = fileEntry.getColumnNumber(e.endOffset)
return "file://$path:$startLine:$startColumn:$endLine:$endColumn"
val startLine = sourceOffsetResolver.getLineNumber(e.startOffset) + 1
val startColumn = sourceOffsetResolver.getColumnNumber(e.startOffset) + 1
val endLine = sourceOffsetResolver.getLineNumber(e.endOffset) + 1
val endColumn = sourceOffsetResolver.getColumnNumber(e.endOffset)
return "file://$filePath:$startLine:$startColumn:$endLine:$endColumn"
}
}
val variableLabelMapping: MutableMap<IrVariable, Label<out DbLocalvar>> = mutableMapOf<IrVariable, Label<out DbLocalvar>>()

3
java/kotlin-extractor/src/main/kotlin/comments/CommentExtractor.kt
View File

@@ -13,7 +13,7 @@ import org.jetbrains.kotlin.psi.KtVisitor
import org.jetbrains.kotlin.psi.psiUtil.endOffset
import org.jetbrains.kotlin.psi.psiUtil.startOffset
class CommentExtractor(private val fileExtractor: KotlinFileExtractor) {
class CommentExtractor(private val fileExtractor: KotlinSourceFileExtractor) {
private val file = fileExtractor.file
private val tw = fileExtractor.tw
private val logger = fileExtractor.logger
@@ -109,4 +109,3 @@ class CommentExtractor(private val fileExtractor: KotlinFileExtractor) {
})
}
}

43
java/kotlin-extractor/src/main/kotlin/utils/AutoCloseableUse.kt Normal file
View File

@@ -0,0 +1,43 @@
package com.github.codeql
// Functions copied from stdlib/jdk7/src/kotlin/AutoCloseable.kt, which is not available within kotlinc,
// but allows the `.use` pattern to be applied to JDK7 AutoCloseables:
/**
* Executes the given [block] function on this resource and then closes it down correctly whether an exception
* is thrown or not.
*
* In case if the resource is being closed due to an exception occurred in [block], and the closing also fails with an exception,
* the latter is added to the [suppressed][java.lang.Throwable.addSuppressed] exceptions of the former.
*
* @param block a function to process this [AutoCloseable] resource.
* @return the result of [block] function invoked on this resource.
*/
public inline fun <T : AutoCloseable?, R> T.useAC(block: (T) -> R): R {
var exception: Throwable? = null
try {
return block(this)
} catch (e: Throwable) {
exception = e
throw e
} finally {
this.closeFinallyAC(exception)
}
}
/**
* Closes this [AutoCloseable], suppressing possible exception or error thrown by [AutoCloseable.close] function when
* it's being closed due to some other [cause] exception occurred.
*
* The suppressed exception is added to the list of suppressed exceptions of [cause] exception.
*/
fun AutoCloseable?.closeFinallyAC(cause: Throwable?) = when {
this == null -> {}
cause == null -> close()
else ->
try {
close()
} catch (closeException: Throwable) {
cause.addSuppressed(closeException)
}
}

47
java/kotlin-extractor/src/main/kotlin/utils/ClassNames.kt Normal file
View File

@@ -0,0 +1,47 @@
package com.github.codeql
import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.IrDeclarationParent
import org.jetbrains.kotlin.ir.declarations.IrPackageFragment
import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement
// Taken from Kotlin's interpreter/Utils.kt function 'internalName'
// Translates class names into their JLS section 13.1 binary name
fun getClassBinaryName(that: IrClass): String {
val internalName = StringBuilder(that.name.asString())
generateSequence(that as? IrDeclarationParent) { (it as? IrDeclaration)?.parent }
.drop(1)
.forEach {
when (it) {
is IrClass -> internalName.insert(0, it.name.asString() + "$")
is IrPackageFragment -> it.fqName.asString().takeIf { it.isNotEmpty() }?.let { internalName.insert(0, "$it.") }
}
}
return internalName.toString()
}
fun getRawIrClassBinaryPath(irClass: IrClass): String? {
val cSource = irClass.source
when(cSource) {
is JavaSourceElement -> {
val element = cSource.javaElement
when(element) {
is BinaryJavaClass -> return element.virtualFile.getPath()
}
}
is KotlinJvmBinarySourceElement -> {
return cSource.binaryClass.location
}
}
return null
}
fun getIrClassBinaryPath(irClass: IrClass): String {
// If a class location is known, replace the JAR delimiter !/:
return getRawIrClassBinaryPath(irClass)?.replaceFirst("!/", "/")
// Otherwise, make up a fake location:
?: "/!unknown-binary-location/${getClassBinaryName(irClass).replace(".", "/")}.class"
}

21
java/kotlin-extractor/src/main/kotlin/utils/Logger.kt
View File

@@ -42,6 +42,15 @@ open class Logger(val logCounter: LogCounter, open val tw: TrapWriter) {
tw.writeTrap("// " + fullMsg.replace("\n", "\n//") + "\n")
println(fullMsg)
}
fun trace(msg: String) {
info(msg)
}
fun debug(msg: String) {
info(msg)
}
fun trace(msg: String, exn: Exception) {
info(msg + " // " + exn)
}
fun warn(severity: Severity, msg: String, locationString: String? = null, locationId: Label<DbLocation> = tw.unknownLocation, stackIndex: Int = 2) {
val st = Exception().stackTrace
val suffix =
@@ -63,6 +72,18 @@ open class Logger(val logCounter: LogCounter, open val tw: TrapWriter) {
val locStr = if (locationString == null) "" else "At " + locationString + ": "
print("$ts Warning: $locStr$msg\n$suffix")
}
fun warn(msg: String, exn: Exception) {
warn(Severity.Warn, msg + " // " + exn)
}
fun warn(msg: String) {
warn(Severity.Warn, msg)
}
fun error(msg: String) {
warn(Severity.Error, msg)
}
fun error(msg: String, exn: Exception) {
error(msg + " // " + exn)
}
fun printLimitedWarningCounts() {
for((caller, count) in logCounter.warningCounts) {
if(count >= logCounter.warningLimit) {