hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-05-22 23:27:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

KE2: Make concurrent extraction possible

Browse Source
This commit is contained in:
Ian Lynagh
2024-10-02 16:40:16 +01:00
parent 5be65ffead
commit 4701bc7aef
1 changed files with 72 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt
View File

@@ -9,6 +9,11 @@ import java.io.File
import java.io.FileOutputStream
import java.nio.file.Files
import java.nio.file.Paths
import java.util.concurrent.Executors
import kotlinx.coroutines.asCoroutineDispatcher
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import org.jetbrains.kotlin.analysis.api.KaAnalysisApiInternals
import org.jetbrains.kotlin.analysis.api.KaSession
import org.jetbrains.kotlin.analysis.api.analyze
@@ -215,65 +220,76 @@ fun doAnalysis(
val checkTrapIdentical = false // TODO
analyze(sourceModule) {
val psiFiles = session.modulesWithFiles.getValue(sourceModule)
var fileNumber = 0
val dump_psi = System.getenv("CODEQL_EXTRACTOR_JAVA_KOTLIN_DUMP") == "true"
for (psiFile in psiFiles) {
if (psiFile is KtFile) {
if (dump_psi) {
val showWhitespaces = false
val showRanges = true
loggerBase.info(dtw, DebugUtil.psiToString(psiFile, showWhitespaces, showRanges))
val maxThreads = 1 // TODO: Default to 8 temporarily to ensure concurrency,
// and $CODEQL_THREADS or Runtime.getRuntime().availableProcessors() later
Executors.newFixedThreadPool(maxThreads).asCoroutineDispatcher().use { dispatcher ->
runBlocking {
withContext(dispatcher) {
val psiFiles = session.modulesWithFiles.getValue(sourceModule)
var fileNumber = 0
val dump_psi = System.getenv("CODEQL_EXTRACTOR_JAVA_KOTLIN_DUMP") == "true"
for (psiFile in psiFiles) {
launch {
if (psiFile is KtFile) {
if (dump_psi) {
val showWhitespaces = false
val showRanges = true
loggerBase.info(dtw, DebugUtil.psiToString(psiFile, showWhitespaces, showRanges))
}
val fileExtractionProblems = FileExtractionProblems(invocationExtractionProblems)
try {
val fileDiagnosticTrapWriter = dtw.makeSourceFileTrapWriter(psiFile, true)
fileDiagnosticTrapWriter.writeCompilation_compiling_files(
compilation,
fileNumber,
fileDiagnosticTrapWriter.fileId
)
doFile(
fileNumber,
compression,
/*
OLD: KE1
fileExtractionProblems,
invocationTrapFile,
*/
fileDiagnosticTrapWriter,
loggerBase,
checkTrapIdentical,
trapDir,
srcDir,
psiFile,
/*
OLD: KE1
primitiveTypeMapping,
pluginContext,
globalExtensionState
*/
)
fileDiagnosticTrapWriter.writeCompilation_compiling_files_completed(
compilation,
fileNumber,
fileExtractionProblems.extractionResult()
)
// We catch Throwable rather than Exception, as we want to
// continue trying to extract everything else even if we get a
// stack overflow or an assertion failure in one file.
} catch (e: Throwable) {
/*
OLD: KE1
logger.error("Extraction failed while extracting '${psiFile.virtualFilePath}'.", e)
fileExtractionProblems.setNonRecoverableProblem()
*/
}
} else {
System.out.println("Warning: Not a KtFile")
}
}
fileNumber += 1
}
}
val fileExtractionProblems = FileExtractionProblems(invocationExtractionProblems)
try {
val fileDiagnosticTrapWriter = dtw.makeSourceFileTrapWriter(psiFile, true)
fileDiagnosticTrapWriter.writeCompilation_compiling_files(
compilation,
fileNumber,
fileDiagnosticTrapWriter.fileId
)
doFile(
fileNumber,
compression,
/*
OLD: KE1
fileExtractionProblems,
invocationTrapFile,
*/
fileDiagnosticTrapWriter,
loggerBase,
checkTrapIdentical,
trapDir,
srcDir,
psiFile,
/*
OLD: KE1
primitiveTypeMapping,
pluginContext,
globalExtensionState
*/
)
fileDiagnosticTrapWriter.writeCompilation_compiling_files_completed(
compilation,
fileNumber,
fileExtractionProblems.extractionResult()
)
// We catch Throwable rather than Exception, as we want to
// continue trying to extract everything else even if we get a
// stack overflow or an assertion failure in one file.
} catch (e: Throwable) {
/*
OLD: KE1
logger.error("Extraction failed while extracting '${psiFile.virtualFilePath}'.", e)
fileExtractionProblems.setNonRecoverableProblem()
*/
}
} else {
System.out.println("Warning: Not a KtFile")
}
}
fileNumber += 1
}
}