hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-24 04:36:35 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10894 from igfoo/igfoo/psi

Browse Source 
Kotlin: Refactor PSI handling
This commit is contained in:
Ian Lynagh
2022-10-21 11:43:49 +01:00
committed by GitHub
parent a56ed88db2 9bc0c98b8e
commit 2e6d6e1538
8 changed files with 39 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt
View File

@@ -2,6 +2,7 @@ package com.github.codeql
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.*
import org.jetbrains.kotlin.ir.IrElement
@@ -139,6 +140,8 @@ class KotlinExtractorExtension(
logger.flush()
logger.info("Extraction for invocation TRAP file $invocationTrapFile")
logger.flush()
logger.info("Kotlin version ${KotlinCompilerVersion.getVersion()}")
logger.flush()
logPeakMemoryUsage(logger, "before extractor")
if (System.getenv("CODEQL_EXTRACTOR_JAVA_KOTLIN_DUMP") == "true") {
logger.info("moduleFragment:\n" + moduleFragment.dump())

15
java/kotlin-extractor/src/main/kotlin/LinesOfCode.kt
View File

@@ -1,9 +1,10 @@
package com.github.codeql
import com.github.codeql.utils.versions.Psi2Ir
import com.github.codeql.utils.versions.getPsi2Ir
import com.intellij.psi.PsiComment
import com.intellij.psi.PsiElement
import com.intellij.psi.PsiWhiteSpace
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.kdoc.psi.api.KDocElement
@@ -15,9 +16,16 @@ class LinesOfCode(
val tw: FileTrapWriter,
val file: IrFile
) {
val psi2Ir = Psi2Ir(logger)
val psi2Ir = getPsi2Ir(logger).also {
if (it == null) {
logger.warn("Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
}
}
fun linesOfCodeInFile(id: Label<DbFile>) {
if (psi2Ir == null) {
return
}
val ktFile = psi2Ir.getKtFile(file)
if (ktFile == null) {
return
@@ -26,6 +34,9 @@ class LinesOfCode(
}
fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label<out DbSourceline>) {
if (psi2Ir == null) {
return
}
val p = psi2Ir.findPsiElement(d, file)
if (p == null) {
return

19
java/kotlin-extractor/src/main/kotlin/comments/CommentExtractor.kt
View File

@@ -3,9 +3,11 @@ package com.github.codeql.comments
import com.github.codeql.*
import com.github.codeql.utils.IrVisitorLookup
import com.github.codeql.utils.isLocalFunction
import com.github.codeql.utils.versions.Psi2Ir
import com.github.codeql.utils.versions.getPsi2Ir
import com.github.codeql.utils.versions.Psi2IrFacade
import com.intellij.psi.PsiComment
import com.intellij.psi.PsiElement
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.expressions.IrBody
@@ -21,18 +23,23 @@ import org.jetbrains.kotlin.psi.psiUtil.startOffset
class CommentExtractor(private val fileExtractor: KotlinFileExtractor, private val file: IrFile, private val fileLabel: Label<out DbFile>) {
private val tw = fileExtractor.tw
private val logger = fileExtractor.logger
private val psi2Ir = Psi2Ir(logger)
private val ktFile = psi2Ir.getKtFile(file)
fun extract() {
val psi2Ir = getPsi2Ir(logger)
if (psi2Ir == null) {
logger.warn("Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
return
}
val ktFile = psi2Ir.getKtFile(file)
if (ktFile == null) {
logger.warn("Comments are not being processed in ${file.path}.")
} else {
return
}
val commentVisitor = mkCommentVisitor(psi2Ir)
ktFile.accept(commentVisitor)
}
}
private val commentVisitor =
private fun mkCommentVisitor(psi2Ir: Psi2IrFacade): KtVisitor<Unit, Unit> =
object : KtVisitor<Unit, Unit>() {
override fun visitElement(element: PsiElement) {
element.acceptChildren(this)

4
java/kotlin-extractor/src/main/kotlin/utils/IrVisitorLookup.kt
View File

@@ -1,6 +1,6 @@
package com.github.codeql.utils
import com.github.codeql.utils.versions.Psi2Ir
import com.github.codeql.utils.versions.Psi2IrFacade
import com.intellij.psi.PsiElement
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
@@ -8,7 +8,7 @@ import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.ir.util.isFakeOverride
import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
class IrVisitorLookup(private val psi2Ir: Psi2Ir, private val psi: PsiElement, private val file: IrFile) :
class IrVisitorLookup(private val psi2Ir: Psi2IrFacade, private val psi: PsiElement, private val file: IrFile) :
IrElementVisitor<Unit, MutableCollection<IrElement>> {
private val location = psi.getLocation()

16
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_4_32/Psi2Ir.kt
View File

@@ -1,19 +1,5 @@
package com.github.codeql.utils.versions
import com.github.codeql.FileLogger
import com.intellij.psi.PsiElement
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.psi.KtFile
class Psi2Ir(private val logger: FileLogger) : Psi2IrFacade {
override fun getKtFile(irFile: IrFile): KtFile? {
logger.warn("Comment extraction is not supported for Kotlin < 1.5.20")
return null
}
override fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement? {
logger.error("Attempted comment extraction for Kotlin < 1.5.20")
return null
}
}
fun getPsi2Ir(@Suppress("UNUSED_PARAMETER") logger: FileLogger): Psi2IrFacade? = null

4
java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_20/Psi2Ir.kt
View File

@@ -8,7 +8,9 @@ import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.psi.KtFile
class Psi2Ir(private val logger: FileLogger): Psi2IrFacade {
fun getPsi2Ir(logger: FileLogger): Psi2IrFacade? = Psi2Ir(logger)
private class Psi2Ir(private val logger: FileLogger): Psi2IrFacade {
override fun getKtFile(irFile: IrFile): KtFile? {
return irFile.getKtFile()
}

1
java/ql/integration-tests/posix-only/kotlin/logs/index_logs.py
View File

@@ -32,6 +32,7 @@ with open('logs.csv', 'w', newline='') as f_out:
j = json.loads(line)
msg = j['message']
msg = re.sub('(?<=Extraction for invocation TRAP file ).*/kt-db/trap/java/invocations/kotlin\..*\.trap', '<FILENAME>', msg)
msg = re.sub('(?<=Kotlin version )[0-9.]+', '<VERSION>', msg)
if msg.startswith('Peak memory: '):
# Peak memory information varies from run to run, so just ignore it
continue

5
java/ql/integration-tests/posix-only/kotlin/logs/logs.expected
View File

@@ -1,5 +1,6 @@
| 1 | 1 | Test script | Log file | 1 |
| 1 | 2 | CodeQL Kotlin extractor | INFO | Extraction started |
| 1 | 3 | CodeQL Kotlin extractor | INFO | Extraction for invocation TRAP file <FILENAME> |
| 1 | 4 | CodeQL Kotlin extractor | INFO | Extracting file test.kt |
| 1 | 5 | CodeQL Kotlin extractor | INFO | Extraction completed |
| 1 | 4 | CodeQL Kotlin extractor | INFO | Kotlin version <VERSION> |
| 1 | 5 | CodeQL Kotlin extractor | INFO | Extracting file test.kt |
| 1 | 6 | CodeQL Kotlin extractor | INFO | Extraction completed |