hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-21 03:06:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

Kotlin: Refactor TrapWriter/Logger

Browse Source 
It's now Tpossible for TrapWriter to log warnings. This required a
little juggling to break the dependency loop between the two classes.
This commit is contained in:
Ian Lynagh
2022-02-28 11:31:04 +00:00
parent 1d824a4e2f
commit dc7f8a6a5a
6 changed files with 70 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
java/kotlin-extractor/src/main/kotlin/ExternalClassExtractor.kt
View File

@@ -40,7 +40,7 @@ class ExternalClassExtractor(val logger: FileLogger, val invocationTrapFile: Str
GZIPOutputStream(trapTmpFile.outputStream()).bufferedWriter().use { trapFileBW ->
// We want our comments to be the first thing in the file,
// so start off with a mere TrapWriter
val tw = TrapWriter(TrapLabelManager(), trapFileBW)
val tw = TrapWriter(logger, TrapLabelManager(), trapFileBW)
tw.writeComment("Generated by the CodeQL Kotlin extractor for external dependencies")
tw.writeComment("Part of invocation $invocationTrapFile")
// Now elevate to a SourceFileTrapWriter, and populate the

14
java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt
View File

@@ -8,7 +8,6 @@ import java.io.FileOutputStream
import java.nio.file.Files
import java.nio.file.Paths
import com.semmle.util.files.FileUtil
import com.semmle.util.unicode.UTF8Util
import kotlin.system.exitProcess
class KotlinExtractorExtension(
@@ -40,7 +39,9 @@ class KotlinExtractorExtension(
val trapDir = File(System.getenv("CODEQL_EXTRACTOR_JAVA_TRAP_DIR").takeUnless { it.isNullOrEmpty() } ?: "kotlin-extractor/trap")
FileOutputStream(File(invocationTrapFile), true).bufferedWriter().use { invocationTrapFileBW ->
val lm = TrapLabelManager()
val tw = TrapWriter(lm, invocationTrapFileBW)
val logCounter = LogCounter()
val loggerBase = LoggerBase(logCounter)
val tw = TrapWriter(loggerBase, lm, invocationTrapFileBW)
// The interceptor has already defined #compilation = *
val compilation: Label<DbCompilation> = StringLabel("compilation")
tw.writeCompilation_started(compilation)
@@ -48,7 +49,6 @@ class KotlinExtractorExtension(
tw.writeCompilation_compiler_times(compilation, -1.0, (System.currentTimeMillis()-compilationStartTime)/1000.0)
}
tw.flush()
val logCounter = LogCounter()
val logger = Logger(logCounter, tw)
logger.info("Extraction started")
logger.flush()
@@ -116,12 +116,6 @@ class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblem
}
}
fun escapeTrapString(str: String) = str.replace("\"", "\"\"")
const val MAX_STRLEN = 1.shl(20) // 1 megabyte
fun truncateString(str: String) = str.substring(0, UTF8Util.encodablePrefixLength(str, MAX_STRLEN))
private fun equivalentTrap(f1: File, f2: File): Boolean {
f1.bufferedReader().use { bw1 ->
f2.bufferedReader().use { bw2 ->
@@ -178,7 +172,7 @@ fun doFile(fileExtractionProblems: FileExtractionProblems,
trapTmpFile.bufferedWriter().use { trapFileBW ->
// We want our comments to be the first thing in the file,
// so start off with a mere TrapWriter
val tw = TrapWriter(TrapLabelManager(), trapFileBW)
val tw = TrapWriter(logger, TrapLabelManager(), trapFileBW)
tw.writeComment("Generated by the CodeQL Kotlin extractor for kotlin source code")
tw.writeComment("Part of invocation $invocationTrapFile")
// Now elevate to a SourceFileTrapWriter, and populate the

30
java/kotlin-extractor/src/main/kotlin/TrapWriter.kt
View File

@@ -11,6 +11,7 @@ import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET
import com.semmle.extractor.java.PopulateFile
import com.semmle.util.unicode.UTF8Util
/**
* Each `.trap` file has a `TrapLabelManager` while we are writing it.
@@ -39,7 +40,7 @@ class TrapLabelManager {
* instances will have different additional state, but they must all
* share the same `TrapLabelManager` and `BufferedWriter`.
*/
open class TrapWriter (protected val lm: TrapLabelManager, private val bw: BufferedWriter) {
open class TrapWriter (protected val loggerBase: LoggerBase, protected val lm: TrapLabelManager, private val bw: BufferedWriter) {
/**
* Returns the label that is defined to be the given key, if such
* a label exists, and `null` otherwise. Most users will want to use
@@ -176,19 +177,36 @@ open class TrapWriter (protected val lm: TrapLabelManager, private val bw: Buffe
bw.flush()
}
fun escapeTrapString(str: String) = str.replace("\"", "\"\"")
val MAX_STRLEN = 1.shl(20) // 1 megabyte
fun truncateString(str: String): String {
val len = str.length
val newLen = UTF8Util.encodablePrefixLength(str, MAX_STRLEN)
if (newLen < len) {
loggerBase.warn(this,
"Truncated string of length $len",
"Truncated string of length $len, starting '${str.take(100)}', ending '${str.takeLast(100)}'")
return str.take(newLen)
} else {
return str
}
}
/**
* Gets a FileTrapWriter like this one (using the same label manager,
* writer etc), but using the given `filePath` for locations.
*/
fun makeFileTrapWriter(filePath: String, populateFileTables: Boolean) =
FileTrapWriter(lm, bw, filePath, populateFileTables)
FileTrapWriter(loggerBase, lm, bw, filePath, populateFileTables)
/**
* Gets a FileTrapWriter like this one (using the same label manager,
* writer etc), but using the given `IrFile` for locations.
*/
fun makeSourceFileTrapWriter(file: IrFile, populateFileTables: Boolean) =
SourceFileTrapWriter(lm, bw, file, populateFileTables)
SourceFileTrapWriter(loggerBase, lm, bw, file, populateFileTables)
}
/**
@@ -196,11 +214,12 @@ open class TrapWriter (protected val lm: TrapLabelManager, private val bw: Buffe
* entities from, so we can at least give the right file as a location.
*/
open class FileTrapWriter (
loggerBase: LoggerBase,
lm: TrapLabelManager,
bw: BufferedWriter,
val filePath: String,
populateFileTables: Boolean
): TrapWriter (lm, bw) {
): TrapWriter (loggerBase, lm, bw) {
val fileId = mkFileId(filePath, populateFileTables)
/**
@@ -244,11 +263,12 @@ open class FileTrapWriter (
* and column numbers.
*/
class SourceFileTrapWriter (
loggerBase: LoggerBase,
lm: TrapLabelManager,
bw: BufferedWriter,
irFile: IrFile,
populateFileTables: Boolean) :
FileTrapWriter(lm, bw, irFile.path, populateFileTables) {
FileTrapWriter(loggerBase, lm, bw, irFile.path, populateFileTables) {
private val fileEntry = irFile.fileEntry

8
java/kotlin-extractor/src/main/kotlin/comments/CommentExtractor.kt
View File

@@ -56,7 +56,7 @@ class CommentExtractor(private val fileExtractor: KotlinFileExtractor, private v
}
val commentLabel = tw.getFreshIdLabel<DbKtcomment>()
tw.writeKtComments(commentLabel, type.value, escapeTrapString(comment.text))
tw.writeKtComments(commentLabel, type.value, tw.escapeTrapString(comment.text))
val locId = tw.getLocation(comment.startOffset, comment.endOffset)
tw.writeHasLocation(commentLabel, locId)
@@ -71,14 +71,14 @@ class CommentExtractor(private val fileExtractor: KotlinFileExtractor, private v
for (sec in comment.getAllSections()) {
val commentSectionLabel = tw.getFreshIdLabel<DbKtcommentsection>()
tw.writeKtCommentSections(commentSectionLabel, commentLabel, escapeTrapString(sec.getContent()))
tw.writeKtCommentSections(commentSectionLabel, commentLabel, tw.escapeTrapString(sec.getContent()))
val name = sec.name
if (name != null) {
tw.writeKtCommentSectionNames(commentSectionLabel, escapeTrapString(name))
tw.writeKtCommentSectionNames(commentSectionLabel, tw.escapeTrapString(name))
}
val subjectName = sec.getSubjectName()
if (subjectName != null) {
tw.writeKtCommentSectionSubjectNames(commentSectionLabel, escapeTrapString(subjectName))
tw.writeKtCommentSectionSubjectNames(commentSectionLabel, tw.escapeTrapString(subjectName))
}
}

51
java/kotlin-extractor/src/main/kotlin/utils/Logger.kt
View File

@@ -28,8 +28,8 @@ enum class Severity(val sev: Int) {
ErrorGlobal(8)
}
open class Logger(val logCounter: LogCounter, open val tw: TrapWriter) {
private fun timestamp(): String {
open class LoggerBase(val logCounter: LogCounter) {
protected fun timestamp(): String {
return "[${SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Date())} K]"
}
@@ -47,12 +47,7 @@ open class Logger(val logCounter: LogCounter, open val tw: TrapWriter) {
return null
}
fun flush() {
tw.flush()
System.out.flush()
}
fun diagnostic(severity: Severity, msg: String, extraInfo: String?, locationString: String? = null, mkLocationId: () -> Label<DbLocation> = { tw.unknownLocation }) {
fun diagnostic(tw: TrapWriter, severity: Severity, msg: String, extraInfo: String?, locationString: String? = null, mkLocationId: () -> Label<DbLocation> = { tw.unknownLocation }) {
val diagnosticLoc = getDiagnosticLocation()
val diagnosticLocStr = if(diagnosticLoc == null) "<unknown location>" else diagnosticLoc
val extraInfoStr = if (extraInfo == null) "" else (extraInfo + "\n")
@@ -78,6 +73,34 @@ open class Logger(val logCounter: LogCounter, open val tw: TrapWriter) {
print("$ts Diagnostic($diagnosticLocStr): $locStr$fullMsg")
}
fun warn(tw: TrapWriter, msg: String, extraInfo: String?) {
diagnostic(tw, Severity.Warn, msg, extraInfo)
}
fun error(tw: TrapWriter, msg: String, extraInfo: String?) {
diagnostic(tw, Severity.Error, msg, extraInfo)
}
}
open class Logger(logCounter: LogCounter, open val tw: TrapWriter): LoggerBase(logCounter) {
private fun getDiagnosticLocation(): String? {
val st = Exception().stackTrace
for(x in st) {
when(x.className) {
"com.github.codeql.Logger",
"com.github.codeql.FileLogger" -> {}
else -> {
return x.toString()
}
}
}
return null
}
fun flush() {
tw.flush()
System.out.flush()
}
fun info(msg: String) {
val fullMsg = "${timestamp()} $msg"
tw.writeComment(fullMsg)
@@ -98,13 +121,13 @@ open class Logger(val logCounter: LogCounter, open val tw: TrapWriter) {
warn(msg, exn.stackTraceToString())
}
fun warn(msg: String, extraInfo: String?) {
diagnostic(Severity.Warn, msg, extraInfo)
warn(tw, msg, extraInfo)
}
fun warn(msg: String) {
warn(msg, null)
}
fun error(msg: String, extraInfo: String?) {
diagnostic(Severity.Error, msg, extraInfo)
error(tw, msg, extraInfo)
}
fun error(msg: String) {
error(msg, null)
@@ -124,19 +147,15 @@ open class Logger(val logCounter: LogCounter, open val tw: TrapWriter) {
}
class FileLogger(logCounter: LogCounter, override val tw: FileTrapWriter): Logger(logCounter, tw) {
private fun timestamp(): String {
return "[${SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Date())} K]"
}
fun warnElement(msg: String, element: IrElement) {
val locationString = tw.getLocationString(element)
val mkLocationId = { tw.getLocation(element) }
diagnostic(Severity.Warn, msg, null, locationString, mkLocationId)
diagnostic(tw, Severity.Warn, msg, null, locationString, mkLocationId)
}
fun errorElement(msg: String, element: IrElement) {
val locationString = tw.getLocationString(element)
val mkLocationId = { tw.getLocation(element) }
diagnostic(Severity.Error, msg, null, locationString, mkLocationId)
diagnostic(tw, Severity.Error, msg, null, locationString, mkLocationId)
}
}