From 40c28f76f22ef7aa0e74f3f743e95fbcfa971390 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Mon, 23 Sep 2024 15:32:18 +0200 Subject: [PATCH] KE2 WIP: reintroduce source class extraction --- .../src/main/kotlin/KotlinExtractor.kt | 1 + .../src/main/kotlin/KotlinFileExtractor.kt | 426 ++++++++++-------- .../src/main/kotlin/KotlinUsesExtractor.kt | 43 +- .../src/main/kotlin/utils/Helpers.kt | 15 +- .../src/main/kotlin/utils/Logger.kt | 28 +- 5 files changed, 275 insertions(+), 238 deletions(-) diff --git a/java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt b/java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt index c87f5d4d9c3..a9da8c214af 100644 --- a/java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt +++ b/java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt @@ -554,6 +554,7 @@ private fun doFile( KotlinFileExtractor( logger, sftw, + KotlinFileExtractor.DeclarationStack() /* OLD: KE1 linesOfCode, diff --git a/java/kotlin-extractor2/src/main/kotlin/KotlinFileExtractor.kt b/java/kotlin-extractor2/src/main/kotlin/KotlinFileExtractor.kt index c525cd25036..f21e066ca28 100644 --- a/java/kotlin-extractor2/src/main/kotlin/KotlinFileExtractor.kt +++ b/java/kotlin-extractor2/src/main/kotlin/KotlinFileExtractor.kt @@ -1,13 +1,17 @@ package com.github.codeql +import com.github.codeql.utils.isInterfaceLike import com.intellij.openapi.util.TextRange import org.jetbrains.kotlin.analysis.api.components.KaDiagnosticCheckerFilter import org.jetbrains.kotlin.analysis.api.KaSession import org.jetbrains.kotlin.analysis.api.types.KaType import org.jetbrains.kotlin.KtNodeTypes +import org.jetbrains.kotlin.analysis.api.KaExperimentalApi import org.jetbrains.kotlin.analysis.api.symbols.* import org.jetbrains.kotlin.psi.* import org.jetbrains.kotlin.parsing.parseNumericLiteral +import java.io.Closeable +import java.util.* /* OLD: KE1 @@ -83,6 +87,7 @@ context (KaSession) open class KotlinFileExtractor( override val logger: FileLogger, override val tw: FileTrapWriter, + val declarationStack: DeclarationStack, /* OLD: KE1 val linesOfCode: LinesOfCode?, @@ -313,17 +318,24 @@ OLD: KE1 if (isExternalDeclaration(declaration)) { extractExternalClassLater(declaration) } else { - extractClassSource( - declaration, - extractDeclarations = true, - extractStaticInitializer = true, - extractPrivateMembers = extractPrivateMembers, - extractFunctionBodies = extractFunctionBodies - ) - } + */ + extractClassSource( + declaration, + extractDeclarations = true, + /* + OLD: KE1 + extractStaticInitializer = true, + extractPrivateMembers = extractPrivateMembers, + extractFunctionBodies = extractFunctionBodies + */ + ) + /* + OLD: KE1 + } */ } + // TODO: the below should be KaFunctionSymbol is KaNamedFunctionSymbol -> { val parentId = useDeclarationParentOf(declaration, false)?.cast() if (parentId != null) { @@ -339,7 +351,8 @@ OLD: KE1 listOf() */ ) - } + } else + Unit /* OLD: KE1 Unit @@ -396,7 +409,8 @@ OLD: KE1 is IrTypeAlias -> extractTypeAlias(declaration) */ else -> - TODO() + // TODO + Unit /* OLD: KE1 logger.errorElement( @@ -1014,158 +1028,184 @@ OLD: KE1 } } - fun extractClassSource( - c: IrClass, - extractDeclarations: Boolean, - extractStaticInitializer: Boolean, - extractPrivateMembers: Boolean, - extractFunctionBodies: Boolean - ): Label { - with("class source", c) { - DeclarationStackAdjuster(c).use { - val id = useClassSource(c) - val pkg = c.packageFqName?.asString() ?: "" - val cls = if (c.isAnonymousObject) "" else c.name.asString() - val pkgId = extractPackage(pkg) - tw.writeClasses_or_interfaces(id, cls, pkgId, id) - if (c.isInterfaceLike) { - tw.writeIsInterface(id) - if (c.kind == ClassKind.ANNOTATION_CLASS) { - tw.writeIsAnnotType(id) + */ + + @OptIn(KaExperimentalApi::class) + fun extractClassSource( + c: KaClassSymbol, + extractDeclarations: Boolean, + /* + OLD: KE1 + extractStaticInitializer: Boolean, + extractPrivateMembers: Boolean, + extractFunctionBodies: Boolean + */ + ): Label { + with("class source", c.psiSafe() ?: TODO()) { + DeclarationStackAdjuster(c).use { + val id = useClassSource(c) + val pkg = c.classId?.packageFqName?.asString() ?: "" + val cls = + if (c.classKind == KaClassKind.ANONYMOUS_OBJECT) "" else c.name!!.asString() // TODO: Remove !! + val pkgId = extractPackage(pkg) + tw.writeClasses_or_interfaces(id, cls, pkgId, id) + if (c.isInterfaceLike) { + tw.writeIsInterface(id) + if (c.classKind == KaClassKind.ANNOTATION_CLASS) { + tw.writeIsAnnotType(id) + } + } else { + val kind = c.classKind + if (kind == KaClassKind.ENUM_CLASS) { + tw.writeIsEnumType(id) + } else if ( + kind != KaClassKind.CLASS && + kind != KaClassKind.OBJECT //&& + //OLD KE1: kind != ClassKind.ENUM_ENTRY + ) { + logger.warnElement("Unrecognised class kind $kind", c.psiSafe() ?: TODO()) + } + + /* + OLD: KE1 + if (c.origin == IrDeclarationOrigin.FILE_CLASS) { + tw.writeFile_class(id) + } + */ + + if ((c as? KaNamedClassSymbol)?.isData == true) { + tw.writeKtDataClasses(id) + } + } + + val locId = tw.getLocation(c.psiSafe() ?: TODO()) + tw.writeHasLocation(id, locId) + + // OLD: KE1 + //extractEnclosingClass(c.parent, id, c, locId, listOf()) + //val javaClass = (c.source as? JavaSourceElement)?.javaElement as? JavaClass + + c.typeParameters.mapIndexed { idx, param -> + //extractTypeParameter(param, idx, javaClass?.typeParameters?.getOrNull(idx)) + } + if (extractDeclarations) { + if (c.classKind == KaClassKind.ANNOTATION_CLASS) { + c.declaredMemberScope.declarations.filterIsInstance().forEach { + val getter = it.getter + if (getter == null) { + logger.warnElement( + "Expected an annotation property to have a getter", + it.psiSafe() ?: TODO() + ) + } else { + extractFunction( + getter, + id, + /* OLD: KE1 + extractBody = false, + extractMethodAndParameterTypeAccesses = + extractFunctionBodies, + extractAnnotations = true, + null, + listOf() + */ + ) + ?.also { functionLabel -> + tw.writeIsAnnotElem(functionLabel.cast()) + } + } } } else { - val kind = c.kind - if (kind == ClassKind.ENUM_CLASS) { - tw.writeIsEnumType(id) - } else if ( - kind != ClassKind.CLASS && - kind != ClassKind.OBJECT && - kind != ClassKind.ENUM_ENTRY - ) { - logger.warnElement("Unrecognised class kind $kind", c) - } - - if (c.origin == IrDeclarationOrigin.FILE_CLASS) { - tw.writeFile_class(id) - } - - if (c.isData) { - tw.writeKtDataClasses(id) - } - } - - val locId = tw.getLocation(c) - tw.writeHasLocation(id, locId) - - extractEnclosingClass(c.parent, id, c, locId, listOf()) - - val javaClass = (c.source as? JavaSourceElement)?.javaElement as? JavaClass - - c.typeParameters.mapIndexed { idx, param -> - extractTypeParameter(param, idx, javaClass?.typeParameters?.getOrNull(idx)) - } - if (extractDeclarations) { - if (c.kind == ClassKind.ANNOTATION_CLASS) { - c.declarations.filterIsInstance().forEach { - val getter = it.getter - if (getter == null) { - logger.warnElement( - "Expected an annotation property to have a getter", - it - ) - } else { - extractFunction( - getter, - id, - extractBody = false, - extractMethodAndParameterTypeAccesses = - extractFunctionBodies, - extractAnnotations = true, - null, - listOf() - ) - ?.also { functionLabel -> - tw.writeIsAnnotElem(functionLabel.cast()) - } - } - } - } else { - try { - c.declarations.forEach { - extractDeclaration( - it, - extractPrivateMembers = extractPrivateMembers, - extractFunctionBodies = extractFunctionBodies, - extractAnnotations = true - ) - } - if (extractStaticInitializer) extractStaticInitializer(c, { id }) - extractJvmStaticProxyMethods( - c, - id, - extractPrivateMembers, - extractFunctionBodies + try { + val decl = c.declaredMemberScope.declarations.toList() + c.declaredMemberScope.declarations.forEach { + extractDeclaration( + it, + /* + OLD: KE1 + extractPrivateMembers = extractPrivateMembers, + extractFunctionBodies = extractFunctionBodies, + extractAnnotations = true + */ ) - } catch (e: IllegalArgumentException) { - // A Kotlin bug causes this to throw: https://youtrack.jetbrains.com/issue/KT-63847/K2-IllegalStateException-IrFieldPublicSymbolImpl-for-java.time-Clock.OffsetClock.offset0-is-already-bound - // TODO: This should either be removed or log something, once the bug is fixed } + /* + OLD: KE1 + if (extractStaticInitializer) extractStaticInitializer(c, { id }) + extractJvmStaticProxyMethods( + c, + id, + extractPrivateMembers, + extractFunctionBodies + ) + */ + } catch (e: IllegalArgumentException) { + // A Kotlin bug causes this to throw: https://youtrack.jetbrains.com/issue/KT-63847/K2-IllegalStateException-IrFieldPublicSymbolImpl-for-java.time-Clock.OffsetClock.offset0-is-already-bound + // TODO: This should either be removed or log something, once the bug is fixed } } - if (c.isNonCompanionObject) { - // For `object MyObject { ... }`, the .class has an - // automatically-generated `public static final MyObject INSTANCE` - // field that may be referenced from Java code, and is used in our - // IrGetObjectValue support. We therefore need to fabricate it - // here. - val instance = useObjectClassInstance(c) - val type = useSimpleTypeClass(c, emptyList(), false) - tw.writeFields(instance.id, instance.name, type.javaResult.id, id, instance.id) - tw.writeFieldsKotlinType(instance.id, type.kotlinResult.id) - tw.writeHasLocation(instance.id, locId) - addModifiers(instance.id, "public", "static", "final") - tw.writeClass_object(id, instance.id) - } - if (c.isObject) { - addModifiers(id, "static") - } - if (extractFunctionBodies && needsObinitFunction(c)) { - extractObinitFunction(c, id) - } - - extractClassModifiers(c, id) - extractClassSupertypes( - c, - id, - inReceiverContext = true - ) // inReceiverContext = true is specified to force extraction of member prototypes - // of base types - - linesOfCode?.linesOfCodeInDeclaration(c, id) - - val additionalAnnotations = - if ( - c.kind == ClassKind.ANNOTATION_CLASS && - c.origin != IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB - ) - metaAnnotationSupport.generateJavaMetaAnnotations(c, extractFunctionBodies) - else listOf() - - extractAnnotations( - c, - c.annotations + additionalAnnotations, - id, - extractFunctionBodies - ) - - if (extractFunctionBodies && !c.isAnonymousObject && !c.isLocal) - externalClassExtractor.writeStubTrapFile(c) - - return id } + /* + OLD: KE1 + if (c.isNonCompanionObject) { + // For `object MyObject { ... }`, the .class has an + // automatically-generated `public static final MyObject INSTANCE` + // field that may be referenced from Java code, and is used in our + // IrGetObjectValue support. We therefore need to fabricate it + // here. + val instance = useObjectClassInstance(c) + val type = useSimpleTypeClass(c, emptyList(), false) + tw.writeFields(instance.id, instance.name, type.javaResult.id, id, instance.id) + tw.writeFieldsKotlinType(instance.id, type.kotlinResult.id) + tw.writeHasLocation(instance.id, locId) + addModifiers(instance.id, "public", "static", "final") + tw.writeClass_object(id, instance.id) + } + */ + if (c.classKind == KaClassKind.OBJECT) { + addModifiers(id, "static") + } + /* + OLD: KE1 + if (extractFunctionBodies && needsObinitFunction(c)) { + extractObinitFunction(c, id) + } + + extractClassModifiers(c, id) + extractClassSupertypes( + c, + id, + inReceiverContext = true + ) // inReceiverContext = true is specified to force extraction of member prototypes + // of base types + + linesOfCode?.linesOfCodeInDeclaration(c, id) + + val additionalAnnotations = + if ( + c.kind == ClassKind.ANNOTATION_CLASS && + c.origin != IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB + ) + metaAnnotationSupport.generateJavaMetaAnnotations(c, extractFunctionBodies) + else listOf() + + extractAnnotations( + c, + c.annotations + additionalAnnotations, + id, + extractFunctionBodies + ) + + if (extractFunctionBodies && !c.isAnonymousObject && !c.isLocal) + externalClassExtractor.writeStubTrapFile(c) + */ + + return id } } - + } + /* + OLD: KE1 val jvmStaticFqName = FqName("kotlin.jvm.JvmStatic") private fun extractJvmStaticProxyMethods( @@ -1832,7 +1872,7 @@ OLD: KE1 */ private fun extractFunction( - f: KaNamedFunctionSymbol, + f: KaFunctionSymbol, parentId: Label, /* OLD: KE1 @@ -2474,7 +2514,7 @@ OLD: KE1 // TODO: Can this be inlined? private fun forceExtractFunction( - f: KaNamedFunctionSymbol, + f: KaFunctionSymbol, parentId: Label, /* OLD: KE1 @@ -2620,7 +2660,7 @@ OLD: KE1 OLD: KE1 locId, */ - f.name.asString(), // TODO: Remove !!, // OLD: KE1: shortNames.nameInDB, + f.name!!.asString(), // TODO: Remove !!, // OLD: KE1: shortNames.nameInDB, f.returnType, // OLD: KE1: substReturnType, paramsSignature, parentId, @@ -9417,47 +9457,49 @@ OLD: KE1 } } - private inner class DeclarationStackAdjuster( - val declaration: IrDeclaration, - val overriddenAttributes: OverriddenFunctionAttributes? = null - ) : Closeable { - init { - declarationStack.push(declaration, overriddenAttributes) - } - - override fun close() { - declarationStack.pop() - } + */ + private inner class DeclarationStackAdjuster( + val declaration: KaDeclarationSymbol, + val overriddenAttributes: OverriddenFunctionAttributes? = null + ) : Closeable { + init { + declarationStack.push(declaration, overriddenAttributes) } - class DeclarationStack { - private val stack: Stack> = Stack() - - fun push(item: IrDeclaration, overriddenAttributes: OverriddenFunctionAttributes?) = - stack.push(Pair(item, overriddenAttributes)) - - fun pop() = stack.pop() - - fun isEmpty() = stack.isEmpty() - - fun peek() = stack.peek() - - fun tryPeek() = if (stack.isEmpty()) null else stack.peek() - - fun findOverriddenAttributes(f: IrFunction) = stack.lastOrNull { it.first == f }?.second + override fun close() { + declarationStack.pop() } + } - data class OverriddenFunctionAttributes( - val id: Label? = null, - val sourceDeclarationId: Label? = null, - val sourceLoc: Label? = null, - val valueParameters: List? = null, - val typeParameters: List? = null, - val isStatic: Boolean? = null, - val visibility: DescriptorVisibility? = null, - val modality: Modality? = null, - ) + class DeclarationStack { + private val stack: Stack> = Stack() + fun push(item: KaDeclarationSymbol, overriddenAttributes: OverriddenFunctionAttributes?) = + stack.push(Pair(item, overriddenAttributes)) + + fun pop() = stack.pop() + + fun isEmpty() = stack.isEmpty() + + fun peek() = stack.peek() + + fun tryPeek() = if (stack.isEmpty()) null else stack.peek() + + fun findOverriddenAttributes(f: KaFunctionSymbol) = stack.lastOrNull { it.first == f }?.second + } + + data class OverriddenFunctionAttributes( + val id: Label? = null, + val sourceDeclarationId: Label? = null, + val sourceLoc: Label? = null, + val valueParameters: List? = null, + val typeParameters: List? = null, + val isStatic: Boolean? = null, + val visibility: KaSymbolVisibility? = null, + val modality: KaSymbolModality? = null, + ) + /* + OLD: KE1 private fun peekDeclStackAsDeclarationParent( elementToReportOn: IrElement ): IrDeclarationParent? { diff --git a/java/kotlin-extractor2/src/main/kotlin/KotlinUsesExtractor.kt b/java/kotlin-extractor2/src/main/kotlin/KotlinUsesExtractor.kt index a2d689806d6..8cecb8ef377 100644 --- a/java/kotlin-extractor2/src/main/kotlin/KotlinUsesExtractor.kt +++ b/java/kotlin-extractor2/src/main/kotlin/KotlinUsesExtractor.kt @@ -1088,6 +1088,9 @@ OLD: KE1 extractFileClass(it) } } + + is KaClassSymbol -> + useClassSource(dp) /* OLD: KE1 is IrClass -> @@ -1438,7 +1441,7 @@ OLD: KE1 @OptIn(ObsoleteDescriptorBasedAPI::class) */ fun getFunctionLabel( - f: KaNamedFunctionSymbol, + f: KaFunctionSymbol, parentId: Label, /* OLD: KE1 @@ -1452,7 +1455,7 @@ OLD: KE1 f.parent, */ parentId, - f.name.asString(), // TODO: Remove the !! // OLD KE1: getFunctionShortName(f).nameInDB, + f.name!!.asString(), // TODO: Remove the !! // OLD KE1: getFunctionShortName(f).nameInDB, /* OLD: KE1 (maybeParameterList ?: f.valueParameters).map { it.type }, @@ -1851,7 +1854,7 @@ OLD: KE1 */ fun useFunction( - f: KaNamedFunctionSymbol, + f: KaFunctionSymbol, parentId: Label, /* OLD: KE1 @@ -1871,8 +1874,8 @@ OLD: KE1 } private fun useFunction( - f: KaNamedFunctionSymbol, - javaFun: KaNamedFunctionSymbol, + f: KaFunctionSymbol, + javaFun: KaFunctionSymbol, parentId: Label, /* OLD: KE1 @@ -2076,14 +2079,14 @@ OLD: KE1 return ClassLabelResults("@\"class;${unquotedLabel.classLabel}\"" /* TODO , unquotedLabel.shortName */) } - /* - OLD: KE1 - fun useClassSource(c: IrClass): Label { - // For source classes, the label doesn't include any type arguments - val classTypeResult = addClassLabel(c, listOf()) - return classTypeResult.id - } + fun useClassSource(c: KaClassSymbol): Label { + // For source classes, the label doesn't include any type arguments + val classTypeResult = addClassLabel(buildClassType(c) as KaClassType) + return classTypeResult.id + } + /* + OLD: KE1 fun getTypeParameterParentLabel(param: IrTypeParameter) = param.parent.let { when (it) { @@ -2132,15 +2135,17 @@ OLD: KE1 param.name.asString() ) - private fun extractModifier(m: String): Label { - val modifierLabel = "@\"modifier;$m\"" - val id: Label = tw.getLabelFor(modifierLabel, { tw.writeModifiers(it, m) }) - return id - } + */ + private fun extractModifier(m: String): Label { + val modifierLabel = "@\"modifier;$m\"" + val id: Label = tw.getLabelFor(modifierLabel, { tw.writeModifiers(it, m) }) + return id + } - fun addModifiers(modifiable: Label, vararg modifiers: String) = - modifiers.forEach { tw.writeHasModifier(modifiable, extractModifier(it)) } + fun addModifiers(modifiable: Label, vararg modifiers: String) = + modifiers.forEach { tw.writeHasModifier(modifiable, extractModifier(it)) } + /* sealed class ExtractSupertypesMode { object Unbound : ExtractSupertypesMode() diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/Helpers.kt b/java/kotlin-extractor2/src/main/kotlin/utils/Helpers.kt index 1f08a14d617..c60badc360d 100644 --- a/java/kotlin-extractor2/src/main/kotlin/utils/Helpers.kt +++ b/java/kotlin-extractor2/src/main/kotlin/utils/Helpers.kt @@ -1,16 +1,7 @@ package com.github.codeql.utils -/* -OLD: KE1 -import org.jetbrains.kotlin.descriptors.ClassKind -import org.jetbrains.kotlin.descriptors.DescriptorVisibilities -import org.jetbrains.kotlin.ir.declarations.IrClass -import org.jetbrains.kotlin.ir.declarations.IrFunction +import org.jetbrains.kotlin.analysis.api.symbols.* -fun IrFunction.isLocalFunction(): Boolean { - return this.visibility == DescriptorVisibilities.LOCAL -} +val KaClassSymbol.isInterfaceLike + get() = classKind == KaClassKind.INTERFACE || classKind == KaClassKind.ANNOTATION_CLASS -val IrClass.isInterfaceLike - get() = kind == ClassKind.INTERFACE || kind == ClassKind.ANNOTATION_CLASS -*/ diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt b/java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt index 293384e989d..e91ca9e6d9d 100644 --- a/java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt +++ b/java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt @@ -394,21 +394,19 @@ class FileLogger(loggerBase: LoggerBase, val ftw: FileTrapWriter, fileNumber: In loggerBase.warn(dtw, msg, extraInfo, loggerState) } - /* - OLD: KE1 - fun warnElement(msg: String, element: IrElement, exn: Throwable? = null) { - val locationString = ftw.getLocationString(element) - val mkLocationId = { ftw.getLocation(element) } - loggerBase.diagnostic( - ftw.getDiagnosticTrapWriter(), - Severity.Warn, - msg, - exn?.stackTraceToString(), - locationString, - mkLocationId - ) - } - */ + fun warnElement(msg: String, element: PsiElement/* TODO , exn: Throwable? = null */) { + val locationString = ftw.getLocationString(element) + val mkLocationId = { ftw.getLocation(element) } + loggerBase.diagnostic( + ftw.getDiagnosticTrapWriter(), + Severity.Warn, + msg, + null, // OLD: KE1: exn?.stackTraceToString(), + loggerState, + locationString, + mkLocationId + ) + } override fun error(dtw: DiagnosticTrapWriter, msg: String, extraInfo: String?) { loggerBase.error(dtw, msg, extraInfo, loggerState)