diff --git a/java/kotlin-extractor2/src/main/kotlin/LinesOfCode.kt b/java/kotlin-extractor2/src/main/kotlin/LinesOfCode.kt index c97014a23ad..4bf67278497 100644 --- a/java/kotlin-extractor2/src/main/kotlin/LinesOfCode.kt +++ b/java/kotlin-extractor2/src/main/kotlin/LinesOfCode.kt @@ -2,32 +2,152 @@ package com.github.codeql /* OLD: KE1 +import com.github.codeql.utils.versions.getPsi2Ir +import com.intellij.psi.PsiComment +import com.intellij.psi.PsiElement +import com.intellij.psi.PsiWhiteSpace +import org.jetbrains.kotlin.config.KotlinCompilerVersion +import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.psi.KtVisitor class LinesOfCode(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) { - val linesOfCodePSI = LinesOfCodePSI(logger, tw, file) - val linesOfCodeLighterAST = LinesOfCodeLighterAST(logger, tw, file) - - fun linesOfCodeInFile(id: Label) { - val psiExtracted = linesOfCodePSI.linesOfCodeInFile(id) - val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInFile(id) - if (psiExtracted && lighterASTExtracted) { - logger.warnElement( - "Both PSI and LighterAST number-of-lines-in-file information for ${file.path}.", - file - ) + val psi2Ir = + getPsi2Ir().also { + if (it == null) { + logger.warn( + "Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})" + ) + } } + + fun linesOfCodeInFile(id: Label): Boolean { + if (psi2Ir == null) { + return false + } + val ktFile = psi2Ir.getKtFile(file) + if (ktFile == null) { + return false + } + linesOfCodeInPsi(id, ktFile, file) + // Even if linesOfCodeInPsi didn't manage to extract any + // information, if we got as far as calling it then we have + // PSI info for the file + return true } - fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label) { - val psiExtracted = linesOfCodePSI.linesOfCodeInDeclaration(d, id) - val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInDeclaration(d, id) - if (psiExtracted && lighterASTExtracted) { - logger.warnElement( - "Both PSI and LighterAST number-of-lines-in-file information for declaration.", - d - ) + fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label): Boolean { + if (psi2Ir == null) { + return false } + val p = psi2Ir.findPsiElement(d, file) + if (p == null) { + return false + } + linesOfCodeInPsi(id, p, d) + // Even if linesOfCodeInPsi didn't manage to extract any + // information, if we got as far as calling it then we have + // PSI info for the declaration + return true + } + + private fun linesOfCodeInPsi(id: Label, root: PsiElement, e: IrElement) { + val document = root.getContainingFile().getViewProvider().getDocument() + if (document == null) { + logger.errorElement("Cannot find document for PSI", e) + tw.writeNumlines(id, 0, 0, 0) + return + } + + val rootRange = root.getTextRange() + val rootStartOffset = rootRange.getStartOffset() + val rootEndOffset = rootRange.getEndOffset() + if (rootStartOffset < 0 || rootEndOffset < 0) { + // This is synthetic, or has an invalid location + tw.writeNumlines(id, 0, 0, 0) + return + } + val rootFirstLine = document.getLineNumber(rootStartOffset) + val rootLastLine = document.getLineNumber(rootEndOffset) + if (rootLastLine < rootFirstLine) { + logger.errorElement("PSI ends before it starts", e) + tw.writeNumlines(id, 0, 0, 0) + return + } + val numLines = 1 + rootLastLine - rootFirstLine + val lineContents = Array(numLines) { LineContent() } + + val visitor = + object : KtVisitor() { + override fun visitElement(element: PsiElement) { + val isComment = element is PsiComment + // Comments may include nodes that aren't PsiComments, + // so we don't want to visit them or we'll think they + // are code. + if (!isComment) { + element.acceptChildren(this) + } + + if (element is PsiWhiteSpace) { + return + } + // Leaf nodes are assumed to be tokens, and + // therefore we count any lines that they are on. + // For comments, we actually need to look at the + // outermost node, as the leaves of KDocs don't + // necessarily cover all lines. + if (isComment || element.getChildren().size == 0) { + val range = element.getTextRange() + val startOffset = range.getStartOffset() + val endOffset = range.getEndOffset() + // The PSI doesn't seem to have anything like + // the IR's UNDEFINED_OFFSET and SYNTHETIC_OFFSET, + // but < 0 still seem to represent bad/unknown + // locations. + if (startOffset < 0 || endOffset < 0) { + logger.errorElement("PSI element has negative offset", e) + return + } + if (startOffset > endOffset) { + logger.errorElement("PSI element has negative size", e) + return + } + // We might get e.g. an import list for a file + // with no imports, which claims to have start + // and end offsets of 0. Anything of 0 width + // we therefore just skip. + if (startOffset == endOffset) { + return + } + val firstLine = document.getLineNumber(startOffset) + val lastLine = document.getLineNumber(endOffset) + if (firstLine < rootFirstLine) { + logger.errorElement("PSI element starts before root", e) + return + } else if (lastLine > rootLastLine) { + logger.errorElement("PSI element ends after root", e) + return + } + for (line in firstLine..lastLine) { + val lineContent = lineContents[line - rootFirstLine] + if (isComment) { + lineContent.containsComment = true + } else { + lineContent.containsCode = true + } + } + } + } + } + root.accept(visitor) + val code = lineContents.count { it.containsCode } + val comment = lineContents.count { it.containsComment } + tw.writeNumlines(id, numLines, code, comment) + } + + private class LineContent { + var containsComment = false + var containsCode = false } } */ diff --git a/java/kotlin-extractor2/src/main/kotlin/LinesOfCodePSI.kt b/java/kotlin-extractor2/src/main/kotlin/LinesOfCodePSI.kt deleted file mode 100644 index 19238346880..00000000000 --- a/java/kotlin-extractor2/src/main/kotlin/LinesOfCodePSI.kt +++ /dev/null @@ -1,153 +0,0 @@ -package com.github.codeql - -/* -OLD: KE1 -import com.github.codeql.utils.versions.getPsi2Ir -import com.intellij.psi.PsiComment -import com.intellij.psi.PsiElement -import com.intellij.psi.PsiWhiteSpace -import org.jetbrains.kotlin.config.KotlinCompilerVersion -import org.jetbrains.kotlin.ir.IrElement -import org.jetbrains.kotlin.ir.declarations.* -import org.jetbrains.kotlin.psi.KtVisitor - -class LinesOfCodePSI(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) { - val psi2Ir = - getPsi2Ir().also { - if (it == null) { - logger.warn( - "Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})" - ) - } - } - - fun linesOfCodeInFile(id: Label): Boolean { - if (psi2Ir == null) { - return false - } - val ktFile = psi2Ir.getKtFile(file) - if (ktFile == null) { - return false - } - linesOfCodeInPsi(id, ktFile, file) - // Even if linesOfCodeInPsi didn't manage to extract any - // information, if we got as far as calling it then we have - // PSI info for the file - return true - } - - fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label): Boolean { - if (psi2Ir == null) { - return false - } - val p = psi2Ir.findPsiElement(d, file) - if (p == null) { - return false - } - linesOfCodeInPsi(id, p, d) - // Even if linesOfCodeInPsi didn't manage to extract any - // information, if we got as far as calling it then we have - // PSI info for the declaration - return true - } - - private fun linesOfCodeInPsi(id: Label, root: PsiElement, e: IrElement) { - val document = root.getContainingFile().getViewProvider().getDocument() - if (document == null) { - logger.errorElement("Cannot find document for PSI", e) - tw.writeNumlines(id, 0, 0, 0) - return - } - - val rootRange = root.getTextRange() - val rootStartOffset = rootRange.getStartOffset() - val rootEndOffset = rootRange.getEndOffset() - if (rootStartOffset < 0 || rootEndOffset < 0) { - // This is synthetic, or has an invalid location - tw.writeNumlines(id, 0, 0, 0) - return - } - val rootFirstLine = document.getLineNumber(rootStartOffset) - val rootLastLine = document.getLineNumber(rootEndOffset) - if (rootLastLine < rootFirstLine) { - logger.errorElement("PSI ends before it starts", e) - tw.writeNumlines(id, 0, 0, 0) - return - } - val numLines = 1 + rootLastLine - rootFirstLine - val lineContents = Array(numLines) { LineContent() } - - val visitor = - object : KtVisitor() { - override fun visitElement(element: PsiElement) { - val isComment = element is PsiComment - // Comments may include nodes that aren't PsiComments, - // so we don't want to visit them or we'll think they - // are code. - if (!isComment) { - element.acceptChildren(this) - } - - if (element is PsiWhiteSpace) { - return - } - // Leaf nodes are assumed to be tokens, and - // therefore we count any lines that they are on. - // For comments, we actually need to look at the - // outermost node, as the leaves of KDocs don't - // necessarily cover all lines. - if (isComment || element.getChildren().size == 0) { - val range = element.getTextRange() - val startOffset = range.getStartOffset() - val endOffset = range.getEndOffset() - // The PSI doesn't seem to have anything like - // the IR's UNDEFINED_OFFSET and SYNTHETIC_OFFSET, - // but < 0 still seem to represent bad/unknown - // locations. - if (startOffset < 0 || endOffset < 0) { - logger.errorElement("PSI element has negative offset", e) - return - } - if (startOffset > endOffset) { - logger.errorElement("PSI element has negative size", e) - return - } - // We might get e.g. an import list for a file - // with no imports, which claims to have start - // and end offsets of 0. Anything of 0 width - // we therefore just skip. - if (startOffset == endOffset) { - return - } - val firstLine = document.getLineNumber(startOffset) - val lastLine = document.getLineNumber(endOffset) - if (firstLine < rootFirstLine) { - logger.errorElement("PSI element starts before root", e) - return - } else if (lastLine > rootLastLine) { - logger.errorElement("PSI element ends after root", e) - return - } - for (line in firstLine..lastLine) { - val lineContent = lineContents[line - rootFirstLine] - if (isComment) { - lineContent.containsComment = true - } else { - lineContent.containsCode = true - } - } - } - } - } - root.accept(visitor) - val code = lineContents.count { it.containsCode } - val comment = lineContents.count { it.containsComment } - tw.writeNumlines(id, numLines, code, comment) - } - - private class LineContent { - var containsComment = false - var containsCode = false - } -} -*/ diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/LinesOfCodeLighterAST.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/LinesOfCodeLighterAST.kt deleted file mode 100644 index 564568ec06a..00000000000 --- a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/LinesOfCodeLighterAST.kt +++ /dev/null @@ -1,21 +0,0 @@ -package com.github.codeql - -/* -OLD: KE1 -import org.jetbrains.kotlin.ir.declarations.* - -class LinesOfCodeLighterAST(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) { - // We don't support LighterAST with old Kotlin versions - fun linesOfCodeInFile(@Suppress("UNUSED_PARAMETER") id: Label): Boolean { - return false - } - - // We don't support LighterAST with old Kotlin versions - fun linesOfCodeInDeclaration( - @Suppress("UNUSED_PARAMETER") d: IrDeclaration, - @Suppress("UNUSED_PARAMETER") id: Label - ): Boolean { - return false - } -} -*/ diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/LinesOfCodeLighterAST.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/LinesOfCodeLighterAST.kt deleted file mode 100644 index 24d9574a16f..00000000000 --- a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/LinesOfCodeLighterAST.kt +++ /dev/null @@ -1,151 +0,0 @@ -package com.github.codeql - -/* -OLD: KE1 -import com.github.codeql.utils.versions.* -import com.intellij.lang.LighterASTNode -import com.intellij.util.diff.FlyweightCapableTreeStructure -import org.jetbrains.kotlin.KtSourceElement -import org.jetbrains.kotlin.fir.backend.FirMetadataSource -import org.jetbrains.kotlin.ir.IrElement -import org.jetbrains.kotlin.ir.declarations.* -import org.jetbrains.kotlin.lexer.KtTokens -import org.jetbrains.kotlin.util.getChildren - -class LinesOfCodeLighterAST(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) { - val fileEntry = file.fileEntry - - fun linesOfCodeInFile(id: Label): Boolean { - val sourceElement = - (file.metadata as? FirMetadataSource.File)?.firFile?.source - if (sourceElement == null) { - return false - } - linesOfCodeInLighterAST(id, file, sourceElement) - // Even if linesOfCodeInLighterAST didn't manage to extract any - // information, if we got as far as calling it then we have - // LighterAST info for the file - return true - } - - fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label): Boolean { - val metadata = (d as? IrMetadataSourceOwner)?.metadata - val sourceElement = (metadata as? FirMetadataSource)?.fir?.source - if (sourceElement == null) { - return false - } - linesOfCodeInLighterAST(id, d, sourceElement) - // Even if linesOfCodeInLighterAST didn't manage to extract any - // information, if we got as far as calling it then we have - // LighterAST info for the declaration - return true - } - - private fun linesOfCodeInLighterAST( - id: Label, - e: IrElement, - s: KtSourceElement - ) { - val rootStartOffset = s.startOffset - val rootEndOffset = s.endOffset - if (rootStartOffset < 0 || rootEndOffset < 0) { - // This is synthetic, or has an invalid location - tw.writeNumlines(id, 0, 0, 0) - return - } - val rootFirstLine = fileEntry.getLineNumber(rootStartOffset) - val rootLastLine = fileEntry.getLineNumber(rootEndOffset) - if (rootLastLine < rootFirstLine) { - logger.errorElement("Source element ends before it starts", e) - tw.writeNumlines(id, 0, 0, 0) - return - } - - val numLines = 1 + rootLastLine - rootFirstLine - val lineContents = Array(numLines) { LineContent() } - - val treeStructure = s.treeStructure - - processSubtree( - e, - treeStructure, - rootFirstLine, - rootLastLine, - lineContents, - s.lighterASTNode - ) - - val code = lineContents.count { it.containsCode } - val comment = lineContents.count { it.containsComment } - tw.writeNumlines(id, numLines, code, comment) - } - - private fun processSubtree( - e: IrElement, - treeStructure: FlyweightCapableTreeStructure, - rootFirstLine: Int, - rootLastLine: Int, - lineContents: Array, - node: LighterASTNode - ) { - if (KtTokens.WHITESPACES.contains(node.tokenType)) { - return - } - - val isComment = KtTokens.COMMENTS.contains(node.tokenType) - val children = node.getChildren(treeStructure) - - // Leaf nodes are assumed to be tokens, and - // therefore we count any lines that they are on. - // For comments, we actually need to look at the - // outermost node, as the leaves of KDocs don't - // necessarily cover all lines. - if (isComment || children.isEmpty()) { - val startOffset = node.getStartOffset() - val endOffset = node.getEndOffset() - if (startOffset < 0 || endOffset < 0) { - logger.errorElement("LighterAST node has negative offset", e) - return - } - if (startOffset > endOffset) { - logger.errorElement("LighterAST node has negative size", e) - return - } - // This may not be possible with LighterAST, but: - // We might get e.g. an import list for a file - // with no imports, which claims to have start - // and end offsets of 0. Anything of 0 width - // we therefore just skip. - if (startOffset == endOffset) { - return - } - val firstLine = fileEntry.getLineNumber(startOffset) - val lastLine = fileEntry.getLineNumber(endOffset) - if (firstLine < rootFirstLine) { - logger.errorElement("LighterAST element starts before root", e) - return - } else if (lastLine > rootLastLine) { - logger.errorElement("LighterAST element ends after root", e) - return - } - for (line in firstLine..lastLine) { - val lineContent = lineContents[line - rootFirstLine] - if (isComment) { - lineContent.containsComment = true - } else { - lineContent.containsCode = true - } - } - } else { - for (child in children) { - processSubtree(e, treeStructure, rootFirstLine, rootLastLine, lineContents, child) - } - } - } - - private class LineContent { - var containsComment = false - var containsCode = false - } -} -*/