From 8ebd07e655ffcf46728a165e949028a2a2cb1d87 Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Thu, 29 Aug 2024 10:55:04 +0100 Subject: [PATCH] KE2: Get TrapFileWriter working --- .../src/main/kotlin/KotlinExtractor.kt | 71 ++-------- .../src/main/kotlin/TrapFileWriter.kt | 131 ++++++++++-------- 2 files changed, 88 insertions(+), 114 deletions(-) diff --git a/java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt b/java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt index e3b17b90bd0..696691bebe8 100644 --- a/java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt +++ b/java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt @@ -199,6 +199,8 @@ fun doAnalysis( } } + val checkTrapIdentical = false // TODO + val psiFiles = session.modulesWithFiles.getValue(sourceModule) /* OLD: KE1 @@ -228,10 +230,7 @@ OLD: KE1 */ fileDiagnosticTrapWriter, loggerBase, -/* -OLD: KE1 checkTrapIdentical, -*/ trapDir, srcDir, psiFile, @@ -431,34 +430,6 @@ class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblem } } -/* -OLD: KE1 -/* -This function determines whether 2 TRAP files should be considered to be -equivalent. It returns `true` iff all of their non-comment lines are -identical. -*/ -private fun equivalentTrap(r1: BufferedReader, r2: BufferedReader): Boolean { - r1.use { br1 -> - r2.use { br2 -> - while (true) { - val l1 = br1.readLine() - val l2 = br2.readLine() - if (l1 == null && l2 == null) { - return true - } else if (l1 == null || l2 == null) { - return false - } else if (l1 != l2) { - if (!l1.startsWith("//") || !l2.startsWith("//")) { - return false - } - } - } - } - } -} -*/ - context (KaSession) private fun doFile( compression: Compression, @@ -469,10 +440,7 @@ OLD: KE1 */ fileDiagnosticTrapWriter: FileTrapWriter, loggerBase: LoggerBase, -/* -OLD: KE1 checkTrapIdentical: Boolean, -*/ dbTrapDir: File, dbSrcDir: File, srcFile: KtFile, @@ -515,15 +483,13 @@ OLD: KE1 srcTmpFile.renameTo(dbSrcFilePath.toFile()) val trapFileName = FileUtil.appendAbsolutePath(dbTrapDir, "$srcFileRelativePath.trap").getAbsolutePath() - val trapFileWriter = getTrapFileWriter(compression, logger, trapFileName) - -/* -OLD: KE1 - if (checkTrapIdentical || !trapFileWriter.exists()) { - trapFileWriter.makeParentDirectory() + val trapFileWriter = getTrapFileWriter(compression, logger, checkTrapIdentical, trapFileName) try { - trapFileWriter.getTempWriter().use { trapFileBW -> + trapFileWriter.run { trapFileBW -> + TODO() +/* +OLD: KE1 // We want our comments to be the first thing in the file, // so start off with a mere TrapWriter val tw = @@ -566,27 +532,18 @@ OLD: KE1 fileExtractor.extractFileContents(srcFile, sftw.fileId) externalDeclExtractor.extractExternalClasses() +*/ } - if (checkTrapIdentical && trapFileWriter.exists()) { - if ( - equivalentTrap(trapFileWriter.getTempReader(), trapFileWriter.getRealReader()) - ) { - trapFileWriter.deleteTemp() - } else { - trapFileWriter.renameTempToDifferent() - } - } else { - trapFileWriter.renameTempToReal() - } - // We catch Throwable rather than Exception, as we want to - // continue trying to extract everything else even if we get a - // stack overflow or an assertion failure in one file. + // We catch Throwable rather than Exception, as we want to + // continue trying to extract everything else even if we get a + // stack overflow or an assertion failure in one file. } catch (e: Throwable) { +/* +OLD: KE1 logger.error("Failed to extract '$srcFilePath'. " + trapFileWriter.debugInfo(), e) context.clear() fileExtractionProblems.setNonRecoverableProblem() - } - } */ + } } diff --git a/java/kotlin-extractor2/src/main/kotlin/TrapFileWriter.kt b/java/kotlin-extractor2/src/main/kotlin/TrapFileWriter.kt index 65e8193ad2c..34265a159af 100644 --- a/java/kotlin-extractor2/src/main/kotlin/TrapFileWriter.kt +++ b/java/kotlin-extractor2/src/main/kotlin/TrapFileWriter.kt @@ -7,10 +7,19 @@ import com.intellij.psi.PsiFile import com.semmle.util.files.FileUtil import com.semmle.util.trap.pathtransformers.PathTransformer */ +import java.io.BufferedInputStream +import java.io.BufferedOutputStream +import java.io.BufferedReader +import java.io.BufferedWriter import java.io.File +import java.io.FileInputStream +import java.io.FileOutputStream +import java.io.InputStreamReader +import java.io.OutputStreamWriter +import java.util.zip.GZIPInputStream +import java.util.zip.GZIPOutputStream /* OLD: KE1 -import java.io.FileOutputStream import java.nio.file.Files import java.nio.file.Paths import org.jetbrains.kotlin.analysis.api.KaAnalysisApiInternals @@ -29,21 +38,11 @@ import org.jetbrains.kotlin.platform.jvm.JvmPlatforms import org.jetbrains.kotlin.psi.* import com.github.codeql.utils.versions.usesK2 -import java.io.BufferedInputStream -import java.io.BufferedOutputStream -import java.io.BufferedReader */ -import java.io.BufferedWriter /* OLD: KE1 -import java.io.File -import java.io.FileInputStream -import java.io.InputStreamReader -import java.io.OutputStreamWriter import java.lang.management.* -import java.util.zip.GZIPInputStream */ -import java.util.zip.GZIPOutputStream /* OLD: KE1 import kotlin.system.exitProcess @@ -98,25 +97,50 @@ fun getCompression(logger: Logger): Compression { fun getTrapFileWriter( compression: Compression, logger: FileLogger, + checkTrapIdentical: Boolean, trapFileName: String ): TrapFileWriter { return when (compression) { - Compression.NONE -> NonCompressedTrapFileWriter(logger, trapFileName) - Compression.GZIP -> GZipCompressedTrapFileWriter(logger, trapFileName) + Compression.NONE -> NonCompressedTrapFileWriter(logger, checkTrapIdentical, trapFileName) + Compression.GZIP -> GZipCompressedTrapFileWriter(logger, checkTrapIdentical, trapFileName) } } abstract class TrapFileWriter( val logger: FileLogger, + val checkTrapIdentical: Boolean, trapName: String, val extension: String ) { -/* -OLD: KE1 + abstract protected fun getReader(file: File): BufferedReader + abstract protected fun getWriter(file: File): BufferedWriter + private val realFile = File(trapName + extension) private val parentDir = realFile.parentFile - lateinit private var tempFile: File + fun run(block: (bw: BufferedWriter) -> Unit) { + if (checkTrapIdentical || !exists()) { + parentDir.mkdirs() + + logger.info("Will write TRAP file $realFile") + val tempFile = File.createTempFile(realFile.getName() + ".", ".trap.tmp" + extension, parentDir) + logger.debug("Writing temporary TRAP file $tempFile") + getWriter(tempFile).use { bw -> block(bw) } + + if (checkTrapIdentical && exists()) { + if (equivalentTrap(getReader(tempFile), getReader(realFile))) { + deleteTemp(tempFile) + } else { + renameTempToDifferent(tempFile) + } + } else { + renameTempToReal(tempFile) + } + } + } + +/* +OLD: KE1 fun debugInfo(): String { if (this::tempFile.isInitialized) { return "Partial TRAP file location is $tempFile" @@ -124,44 +148,19 @@ OLD: KE1 return "Temporary file not yet created." } } +*/ - fun makeParentDirectory() { - parentDir.mkdirs() - } - - fun exists(): Boolean { + private fun exists(): Boolean { return realFile.exists() } - abstract protected fun getReader(file: File): BufferedReader - - abstract protected fun getWriter(file: File): BufferedWriter - - fun getRealReader(): BufferedReader { - return getReader(realFile) - } - - fun getTempReader(): BufferedReader { - return getReader(tempFile) - } - - fun getTempWriter(): BufferedWriter { - logger.info("Will write TRAP file $realFile") - if (this::tempFile.isInitialized) { - logger.error("Temp writer reinitialized for $realFile") - } - tempFile = File.createTempFile(realFile.getName() + ".", ".trap.tmp" + extension, parentDir) - logger.debug("Writing temporary TRAP file $tempFile") - return getWriter(tempFile) - } - - fun deleteTemp() { + private fun deleteTemp(tempFile: File) { if (!tempFile.delete()) { logger.warn("Failed to delete $tempFile") } } - fun renameTempToDifferent() { + private fun renameTempToDifferent(tempFile: File) { val trapDifferentFile = File.createTempFile(realFile.getName() + ".", ".trap.different" + extension, parentDir) if (tempFile.renameTo(trapDifferentFile)) { @@ -171,19 +170,16 @@ OLD: KE1 } } - fun renameTempToReal() { + private fun renameTempToReal(tempFile: File) { if (!tempFile.renameTo(realFile)) { logger.warn("Failed to rename $tempFile to $realFile") } logger.info("Finished writing TRAP file $realFile") } -*/ } -private class NonCompressedTrapFileWriter(logger: FileLogger, trapName: String) : - TrapFileWriter(logger, trapName, "") { -/* -OLD: KE1 +private class NonCompressedTrapFileWriter(logger: FileLogger, checkTrapIdentical: Boolean, trapName: String) : + TrapFileWriter(logger, checkTrapIdentical, trapName, "") { override protected fun getReader(file: File): BufferedReader { return file.bufferedReader() } @@ -191,13 +187,10 @@ OLD: KE1 override protected fun getWriter(file: File): BufferedWriter { return file.bufferedWriter() } -*/ } -private class GZipCompressedTrapFileWriter(logger: FileLogger, trapName: String) : - TrapFileWriter(logger, trapName, ".gz") { -/* -OLD: KE1 +private class GZipCompressedTrapFileWriter(logger: FileLogger, checkTrapIdentical: Boolean, trapName: String) : + TrapFileWriter(logger, checkTrapIdentical, trapName, ".gz") { override protected fun getReader(file: File): BufferedReader { return BufferedReader( InputStreamReader(GZIPInputStream(BufferedInputStream(FileInputStream(file)))) @@ -209,5 +202,29 @@ OLD: KE1 OutputStreamWriter(GZIPOutputStream(BufferedOutputStream(FileOutputStream(file)))) ) } -*/ +} + +/* +This function determines whether 2 TRAP files should be considered to be +equivalent. It returns `true` iff all of their non-comment lines are +identical. +*/ +private fun equivalentTrap(r1: BufferedReader, r2: BufferedReader): Boolean { + r1.use { br1 -> + r2.use { br2 -> + while (true) { + val l1 = br1.readLine() + val l2 = br2.readLine() + if (l1 == null && l2 == null) { + return true + } else if (l1 == null || l2 == null) { + return false + } else if (l1 != l2) { + if (!l1.startsWith("//") || !l2.startsWith("//")) { + return false + } + } + } + } + } }