hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-05-02 12:15:17 +02:00
Code Issues Packages Projects Releases Wiki Activity

Handle exceptions on file level

Browse Source
This commit is contained in:
Tamas Vajk
2022-01-20 11:00:35 +01:00
committed by Ian Lynagh
parent 468a911f83
commit 29f4eb96e1
3 changed files with 86 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java
View File

@@ -310,6 +310,7 @@ public class OdasaOutput {
private TrapDependencies trapDependenciesForClass;
private File trapFile;
private IrClass sym;
private boolean hasError = false;
private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrClass sym) {
trapDependenciesForClass = new TrapDependencies(relative);
@@ -326,6 +327,10 @@ public class OdasaOutput {
}
public void close() {
if (hasError) {
return;
}
writeTrapDependencies(trapDependenciesForClass);
// Record major/minor version information for extracted class files.
// This is subsequently used to determine whether to re-extract (a newer version of) the same class.
@@ -346,6 +351,10 @@ public class OdasaOutput {
trapDependencies.save(
currentSpecFileEntry.getTrapFolder().toPath().resolve(dep));
}
public void closeWithoutAdditionalFiles() {
hasError = true;
}
}
/*

47
java/kotlin-extractor/src/main/kotlin/ExternalClassExtractor.kt
View File

@@ -28,32 +28,39 @@ class ExternalClassExtractor(val logger: FileLogger, val invocationTrapFile: Str
externalClassWorkList.clear()
nextBatch.forEach { irClass ->
output.getTrapLockerForClassFile(irClass).useAC { locker ->
locker.getTrapFileManager().useAC { manager ->
locker.trapFileManager.useAC { manager ->
if(manager == null) {
logger.info("Skipping extracting class ${irClass.name}")
} else {
GZIPOutputStream(manager.getFile().outputStream()).bufferedWriter().use { trapFileBW ->
// We want our comments to be the first thing in the file,
// so start off with a mere TrapWriter
val tw = TrapWriter(TrapLabelManager(), trapFileBW)
tw.writeComment("Generated by the CodeQL Kotlin extractor for external dependencies")
tw.writeComment("Part of invocation $invocationTrapFile")
// Now elevate to a SourceFileTrapWriter, and populate the
// file information
val binaryPath = getIrClassBinaryPath(irClass)
val ftw = tw.makeFileTrapWriter(binaryPath, true)
val trapFile = manager.file
val binaryPath = getIrClassBinaryPath(irClass)
try {
GZIPOutputStream(trapFile.outputStream()).bufferedWriter().use { trapFileBW ->
// We want our comments to be the first thing in the file,
// so start off with a mere TrapWriter
val tw = TrapWriter(TrapLabelManager(), trapFileBW)
tw.writeComment("Generated by the CodeQL Kotlin extractor for external dependencies")
tw.writeComment("Part of invocation $invocationTrapFile")
// Now elevate to a SourceFileTrapWriter, and populate the
// file information
val ftw = tw.makeFileTrapWriter(binaryPath, true)
val fileExtractor = KotlinFileExtractor(logger, ftw, binaryPath, manager, this, primitiveTypeMapping, pluginContext, genericSpecialisationsExtracted)
val fileExtractor = KotlinFileExtractor(logger, ftw, binaryPath, manager, this, primitiveTypeMapping, pluginContext, genericSpecialisationsExtracted)
// Populate a location and compilation-unit package for the file. This is similar to
// the beginning of `KotlinFileExtractor.extractFileContents` but without an `IrFile`
// to start from.
val pkg = irClass.packageFqName?.asString() ?: ""
val pkgId = fileExtractor.extractPackage(pkg)
ftw.writeHasLocation(ftw.fileId, ftw.getWholeFileLocation())
ftw.writeCupackage(ftw.fileId, pkgId)
// Populate a location and compilation-unit package for the file. This is similar to
// the beginning of `KotlinFileExtractor.extractFileContents` but without an `IrFile`
// to start from.
val pkg = irClass.packageFqName?.asString() ?: ""
val pkgId = fileExtractor.extractPackage(pkg)
ftw.writeHasLocation(ftw.fileId, ftw.getWholeFileLocation())
ftw.writeCupackage(ftw.fileId, pkgId)
fileExtractor.extractClassSource(irClass)
fileExtractor.extractClassSource(irClass)
}
} catch (e: Exception) {
manager.closeWithoutAdditionalFiles()
trapFile.delete()
logger.error("Failed to extract '$binaryPath'", e)
}
}
}

86
java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt
View File

@@ -3,7 +3,6 @@ package com.github.codeql
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.*
import java.io.File
import java.io.FileOutputStream
import java.nio.file.Files
@@ -89,30 +88,33 @@ fun doFile(invocationTrapFile: String,
fileTrapWriter: FileTrapWriter,
checkTrapIdentical: Boolean,
logCounter: LogCounter,
trapDir: File,
srcDir: File,
file: IrFile,
dbTrapDir: File,
dbSrcDir: File,
srcFile: IrFile,
primitiveTypeMapping: PrimitiveTypeMapping,
pluginContext: IrPluginContext,
genericSpecialisationsExtracted: MutableSet<String>) {
val filePath = file.path
val srcFilePath = srcFile.path
val logger = FileLogger(logCounter, fileTrapWriter)
logger.info("Extracting file $filePath")
logger.info("Extracting file $srcFilePath")
logger.flush()
val dest = Paths.get("$srcDir/${file.path}")
val destDir = dest.getParent()
Files.createDirectories(destDir)
val srcTmpFile = File.createTempFile(dest.getFileName().toString() + ".", ".src.tmp", destDir.toFile())
val srcTmpOS = FileOutputStream(srcTmpFile)
Files.copy(Paths.get(file.path), srcTmpOS)
srcTmpOS.close()
srcTmpFile.renameTo(dest.toFile())
val trapFile = File("$trapDir/$filePath.trap")
val trapFileDir = trapFile.getParentFile()
val dbSrcFilePath = Paths.get("$dbSrcDir/$srcFilePath")
val dbSrcDirPath = dbSrcFilePath.parent
Files.createDirectories(dbSrcDirPath)
val srcTmpFile = File.createTempFile(dbSrcFilePath.fileName.toString() + ".", ".src.tmp", dbSrcDirPath.toFile())
srcTmpFile.outputStream().use {
Files.copy(Paths.get(srcFilePath), it)
}
srcTmpFile.renameTo(dbSrcFilePath.toFile())
val trapFile = File("$dbTrapDir/$srcFilePath.trap")
val trapFileDir = trapFile.parentFile
trapFileDir.mkdirs()
if (checkTrapIdentical || !trapFile.exists()) {
val trapTmpFile = File.createTempFile("$filePath.", ".trap.tmp", trapFileDir)
val trapTmpFile = File.createTempFile("$srcFilePath.", ".trap.tmp", trapFileDir)
var hasError = false
trapTmpFile.bufferedWriter().use { trapFileBW ->
// We want our comments to be the first thing in the file,
// so start off with a mere TrapWriter
@@ -121,28 +123,40 @@ fun doFile(invocationTrapFile: String,
tw.writeComment("Part of invocation $invocationTrapFile")
// Now elevate to a SourceFileTrapWriter, and populate the
// file information
val sftw = tw.makeSourceFileTrapWriter(file, true)
val externalClassExtractor = ExternalClassExtractor(logger, invocationTrapFile, file.path, primitiveTypeMapping, pluginContext, genericSpecialisationsExtracted)
val fileExtractor = KotlinFileExtractor(logger, sftw, file.path, null, externalClassExtractor, primitiveTypeMapping, pluginContext, genericSpecialisationsExtracted)
fileExtractor.extractFileContents(file, sftw.fileId)
externalClassExtractor.extractExternalClasses()
val sftw = tw.makeSourceFileTrapWriter(srcFile, true)
val externalClassExtractor = ExternalClassExtractor(logger, invocationTrapFile, srcFilePath, primitiveTypeMapping, pluginContext, genericSpecialisationsExtracted)
val fileExtractor = KotlinFileExtractor(logger, sftw, srcFilePath, null, externalClassExtractor, primitiveTypeMapping, pluginContext, genericSpecialisationsExtracted)
try {
fileExtractor.extractFileContents(srcFile, sftw.fileId)
externalClassExtractor.extractExternalClasses()
} catch (e: Exception) {
hasError = true
logger.error("Failed to extract '$srcFilePath'", e)
}
}
if (checkTrapIdentical && trapFile.exists()) {
if(equivalentTrap(trapTmpFile, trapFile)) {
if(!trapTmpFile.delete()) {
logger.warn(Severity.WarnLow, "Failed to delete $trapTmpFile")
}
} else {
val trapDifferentFile = File.createTempFile("$filePath.", ".trap.different", trapDir)
if(trapTmpFile.renameTo(trapDifferentFile)) {
logger.warn(Severity.Warn, "TRAP difference: $trapFile vs $trapDifferentFile")
} else {
logger.warn(Severity.WarnLow, "Failed to rename $trapTmpFile to $trapFile")
}
if (hasError) {
if (!trapTmpFile.delete()) {
logger.warn(Severity.WarnLow, "Failed to delete $trapTmpFile")
}
} else {
if(!trapTmpFile.renameTo(trapFile)) {
logger.warn(Severity.WarnLow, "Failed to rename $trapTmpFile to $trapFile")
if (checkTrapIdentical && trapFile.exists()) {
if (equivalentTrap(trapTmpFile, trapFile)) {
if (!trapTmpFile.delete()) {
logger.warn(Severity.WarnLow, "Failed to delete $trapTmpFile")
}
} else {
val trapDifferentFile = File.createTempFile("$srcFilePath.", ".trap.different", dbTrapDir)
if (trapTmpFile.renameTo(trapDifferentFile)) {
logger.warn(Severity.Warn, "TRAP difference: $trapFile vs $trapDifferentFile")
} else {
logger.warn(Severity.WarnLow, "Failed to rename $trapTmpFile to $trapFile")
}
}
} else {
if (!trapTmpFile.renameTo(trapFile)) {
logger.warn(Severity.WarnLow, "Failed to rename $trapTmpFile to $trapFile")
}
}
}
}