Implement raw type extraction

This commit is contained in:
Chris Smowton
2021-12-15 17:54:05 +00:00
committed by Ian Lynagh
parent 6455c988f2
commit acad36cab4
4 changed files with 181 additions and 73 deletions

View File

@@ -1,9 +1,9 @@
package com.github.codeql
import com.github.codeql.utils.TypeSubstitution
import com.github.codeql.utils.versions.functionN
import com.github.codeql.utils.lowerBound
import com.github.codeql.utils.substituteTypeAndArguments
import com.github.codeql.utils.upperBound
import com.github.codeql.utils.toRawType
import com.semmle.extractor.java.OdasaOutput
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.builtins.functions.BuiltInFunctionArity
@@ -16,7 +16,6 @@ import org.jetbrains.kotlin.ir.IrStatement
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.expressions.*
import org.jetbrains.kotlin.ir.symbols.IrConstructorSymbol
import org.jetbrains.kotlin.ir.symbols.IrTypeParameterSymbol
import org.jetbrains.kotlin.ir.types.*
import org.jetbrains.kotlin.ir.util.*
import org.jetbrains.kotlin.name.FqName
@@ -115,8 +114,10 @@ open class KotlinFileExtractor(
extractVisibility(c, id, c.visibility)
}
fun extractClassInstance(c: IrClass, typeArgs: List<IrTypeArgument>): Label<out DbClassorinterface> {
if (typeArgs.isEmpty()) {
// `typeArgs` can be null to describe a raw generic type.
// For non-generic types it will be zero-length list.
fun extractClassInstance(c: IrClass, typeArgs: List<IrTypeArgument>?): Label<out DbClassorinterface> {
if (typeArgs?.isEmpty() == true) {
logger.warn(Severity.ErrorSevere, "Instance without type arguments: " + c.name.asString())
}
@@ -143,15 +144,20 @@ open class KotlinFileExtractor(
}
}
for ((idx, arg) in typeArgs.withIndex()) {
val argId = getTypeArgumentLabel(arg).id
tw.writeTypeArgs(argId, idx, id)
if (typeArgs != null) {
for ((idx, arg) in typeArgs.withIndex()) {
val argId = getTypeArgumentLabel(arg).id
tw.writeTypeArgs(argId, idx, id)
}
tw.writeIsParameterized(id)
} else {
tw.writeIsRaw(id)
}
tw.writeIsParameterized(id)
val unbound = useClassSource(c)
tw.writeErasure(id, unbound)
extractClassModifiers(c, id)
extractClassSupertypes(c, id, typeArgs)
extractClassSupertypes(c, id, if (typeArgs == null) ExtractSupertypesMode.Raw else ExtractSupertypesMode.Specialised(typeArgs))
val locId = tw.getLocation(c)
tw.writeHasLocation(id, locId)
@@ -159,8 +165,24 @@ open class KotlinFileExtractor(
return id
}
fun extractMemberPrototypes(c: IrClass, typeArgs: List<IrTypeArgument>, id: Label<out DbClassorinterface>) {
val typeParamSubstitution = c.typeParameters.map({ it.symbol }).zip(typeArgs).toMap()
// `typeArgs` can be null to describe a raw generic type.
// For non-generic types it will be zero-length list.
fun extractMemberPrototypes(c: IrClass, typeArgs: List<IrTypeArgument>?, id: Label<out DbClassorinterface>) {
val typeParamSubstitution =
when (typeArgs) {
null -> { x: IrType, _: TypeContext, _: IrPluginContext -> x.toRawType() }
else -> {
c.typeParameters.map({ it.symbol }).zip(typeArgs).toMap().let {
{ x: IrType, useContext: TypeContext, pluginContext: IrPluginContext ->
x.substituteTypeAndArguments(
it,
useContext,
pluginContext
)
}
}
}
}
c.declarations.map {
when(it) {
@@ -299,14 +321,12 @@ open class KotlinFileExtractor(
return FieldResult(instanceId, instanceName)
}
private fun extractValueParameter(vp: IrValueParameter, parent: Label<out DbCallable>, idx: Int, typeSubstitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>?): TypeResults {
return extractValueParameter(useValueParameter(vp, parent), vp.type, vp.name.asString(), tw.getLocation(vp), parent, idx, typeSubstitutionMap)
private fun extractValueParameter(vp: IrValueParameter, parent: Label<out DbCallable>, idx: Int, typeSubstitution: TypeSubstitution?): TypeResults {
return extractValueParameter(useValueParameter(vp, parent), vp.type, vp.name.asString(), tw.getLocation(vp), parent, idx, typeSubstitution)
}
private fun extractValueParameter(id: Label<out DbParam>, t: IrType, name: String, locId: Label<DbLocation>, parent: Label<out DbCallable>, idx: Int, typeSubstitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>?): TypeResults {
val substitutedType = t.substituteTypeAndArguments(typeSubstitutionMap) {
it.lowerBound(pluginContext)
}
private fun extractValueParameter(id: Label<out DbParam>, t: IrType, name: String, locId: Label<DbLocation>, parent: Label<out DbCallable>, idx: Int, typeSubstitution: TypeSubstitution?): TypeResults {
val substitutedType = typeSubstitution?.let { it(t, TypeContext.OTHER, pluginContext) } ?: t
val type = useType(substitutedType)
tw.writeParams(id, type.javaResult.id, type.kotlinResult.id, idx, parent, id)
tw.writeHasLocation(id, locId)
@@ -381,13 +401,13 @@ open class KotlinFileExtractor(
}
}
fun extractFunctionIfReal(f: IrFunction, parentId: Label<out DbReftype>, extractBody: Boolean = true, typeSubstitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>? = null) {
fun extractFunctionIfReal(f: IrFunction, parentId: Label<out DbReftype>, extractBody: Boolean = true, typeSubstitution: TypeSubstitution? = null) {
if (f.origin == IrDeclarationOrigin.FAKE_OVERRIDE)
return
extractFunction(f, parentId, extractBody, typeSubstitutionMap)
extractFunction(f, parentId, extractBody, typeSubstitution)
}
fun extractFunction(f: IrFunction, parentId: Label<out DbReftype>, extractBody: Boolean = true, typeSubstitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>? = null): Label<out DbCallable> {
fun extractFunction(f: IrFunction, parentId: Label<out DbReftype>, extractBody: Boolean = true, typeSubstitution: TypeSubstitution? = null): Label<out DbCallable> {
currentFunction = f
f.typeParameters.map { extractTypeParameter(it) }
@@ -405,7 +425,7 @@ open class KotlinFileExtractor(
val extReceiver = f.extensionReceiverParameter
val idxOffset = if (extReceiver != null) 1 else 0
val paramTypes = f.valueParameters.mapIndexed { i, vp ->
extractValueParameter(vp, id, i + idxOffset, typeSubstitutionMap)
extractValueParameter(vp, id, i + idxOffset, typeSubstitution)
}
val allParamTypes = if (extReceiver != null) {
val extendedType = useType(extReceiver.type)
@@ -422,12 +442,10 @@ open class KotlinFileExtractor(
val paramsSignature = allParamTypes.joinToString(separator = ",", prefix = "(", postfix = ")") { it.javaResult.signature!! }
val substReturnType = f.returnType.substituteTypeAndArguments(typeSubstitutionMap) {
it.upperBound(pluginContext)
}
val substReturnType = typeSubstitution?.let { it(f.returnType, TypeContext.RETURN, pluginContext) } ?: f.returnType
val sourceDeclaration =
if (typeSubstitutionMap != null)
if (typeSubstitution != null)
useFunction(f)
else
id
@@ -436,7 +454,7 @@ open class KotlinFileExtractor(
val unitType = useType(pluginContext.irBuiltIns.unitType, TypeContext.RETURN)
val shortName = when {
f.returnType.isAnonymous -> ""
typeSubstitutionMap != null -> useType(substReturnType).javaResult.shortName
typeSubstitution != null -> useType(substReturnType).javaResult.shortName
else -> f.returnType.classFqName?.shortName()?.asString() ?: f.name.asString()
}
@Suppress("UNCHECKED_CAST")
@@ -452,7 +470,7 @@ open class KotlinFileExtractor(
tw.writeHasLocation(id, locId)
val body = f.body
if(body != null && extractBody) {
if(typeSubstitutionMap != null)
if(typeSubstitution != null)
logger.warnElement(Severity.ErrorSevere, "Type substitution should only be used to extract a function prototype, not the body", f)
extractBody(body, id)
}
@@ -475,7 +493,7 @@ open class KotlinFileExtractor(
return id
}
fun extractProperty(p: IrProperty, parentId: Label<out DbReftype>, extractBackingField: Boolean = true, typeSubstitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>? = null) {
fun extractProperty(p: IrProperty, parentId: Label<out DbReftype>, extractBackingField: Boolean = true, typeSubstitution: TypeSubstitution? = null) {
val id = useProperty(p, parentId)
val locId = tw.getLocation(p)
tw.writeKtProperties(id, p.name.asString())
@@ -487,7 +505,7 @@ open class KotlinFileExtractor(
if(getter != null) {
@Suppress("UNCHECKED_CAST")
val getterId = extractFunction(getter, parentId, extractBackingField, typeSubstitutionMap) as Label<out DbMethod>
val getterId = extractFunction(getter, parentId, extractBackingField, typeSubstitution) as Label<out DbMethod>
tw.writeKtPropertyGetters(id, getterId)
} else {
if (p.modality != Modality.FINAL || !isExternalDeclaration(p)) {
@@ -500,7 +518,7 @@ open class KotlinFileExtractor(
logger.warnElement(Severity.ErrorSevere, "!isVar property with a setter", p)
}
@Suppress("UNCHECKED_CAST")
val setterId = extractFunction(setter, parentId, extractBackingField, typeSubstitutionMap) as Label<out DbMethod>
val setterId = extractFunction(setter, parentId, extractBackingField, typeSubstitution) as Label<out DbMethod>
tw.writeKtPropertySetters(id, setterId)
} else {
if (p.isVar && !isExternalDeclaration(p)) {

View File

@@ -3,10 +3,9 @@ package com.github.codeql
import com.github.codeql.utils.substituteTypeArguments
import com.semmle.extractor.java.OdasaOutput
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.backend.jvm.ir.propertyIfAccessor
import org.jetbrains.kotlin.backend.jvm.codegen.isRawType
import org.jetbrains.kotlin.builtins.jvm.JavaToKotlinClassMap
import org.jetbrains.kotlin.descriptors.DescriptorVisibilities
import org.jetbrains.kotlin.descriptors.Modality
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.symbols.IrClassSymbol
import org.jetbrains.kotlin.ir.symbols.IrClassifierSymbol
@@ -92,7 +91,9 @@ open class KotlinUsesExtractor(
return KotlinSourceFileExtractor(newLogger, newTrapWriter, clsFile, externalClassExtractor, primitiveTypeMapping, pluginContext, genericSpecialisationsExtracted)
}
fun useClassInstance(c: IrClass, typeArgs: List<IrTypeArgument>, inReceiverContext: Boolean = false): UseClassInstanceResult {
// `typeArgs` can be null to describe a raw generic type.
// For non-generic types it will be zero-length list.
fun useClassInstance(c: IrClass, typeArgs: List<IrTypeArgument>?, inReceiverContext: Boolean = false): UseClassInstanceResult {
if (c.isAnonymousObject) {
logger.warn(Severity.ErrorSevere, "Unexpected access to anonymous class instance")
}
@@ -143,7 +144,9 @@ open class KotlinUsesExtractor(
externalClassExtractor.extractLater(c)
}
fun addClassLabel(c: IrClass, typeArgs: List<IrTypeArgument>, inReceiverContext: Boolean = false): TypeResult<DbClassorinterface> {
// `typeArgs` can be null to describe a raw generic type.
// For non-generic types it will be zero-length list.
fun addClassLabel(c: IrClass, typeArgs: List<IrTypeArgument>?, inReceiverContext: Boolean = false): TypeResult<DbClassorinterface> {
val classLabelResult = getClassLabel(c, typeArgs)
var instanceSeenBefore = true
@@ -154,8 +157,8 @@ open class KotlinUsesExtractor(
extractClassLaterIfExternal(c)
})
if (typeArgs.isNotEmpty()) {
// If this is a generic type instantiation then it has no
if (typeArgs == null || typeArgs.isNotEmpty()) {
// If this is a generic type instantiation or a raw type then it has no
// source entity, so we need to extract it here
val extractorWithCSource by lazy { this.withSourceFileOfClass(c) }
@@ -202,9 +205,11 @@ open class KotlinUsesExtractor(
return fakeKotlinTypeId
}
fun useSimpleTypeClass(c: IrClass, args: List<IrTypeArgument>, hasQuestionMark: Boolean): TypeResults {
// `args` can be null to describe a raw generic type.
// For non-generic types it will be zero-length list.
fun useSimpleTypeClass(c: IrClass, args: List<IrTypeArgument>?, hasQuestionMark: Boolean): TypeResults {
if (c.isAnonymousObject) {
if (args.isNotEmpty()) {
if (args?.isNotEmpty() == true) {
logger.warn(Severity.ErrorHigh, "Anonymous class with unexpected type arguments")
}
if (hasQuestionMark) {
@@ -287,7 +292,7 @@ open class KotlinUsesExtractor(
dimensions,
componentTypeResults.javaResult.id)
extractClassSupertypes(arrayType.classifier.owner as IrClass, it, arrayType.arguments)
extractClassSupertypes(arrayType.classifier.owner as IrClass, it, ExtractSupertypesMode.Specialised(arrayType.arguments))
// array.length
val length = tw.getLabelFor<DbField>("@\"field;{$it};length\"")
@@ -431,7 +436,9 @@ class X {
val classifier: IrClassifierSymbol = s.classifier
val cls: IrClass = classifier.owner as IrClass
return useSimpleTypeClass(cls, s.arguments, s.hasQuestionMark)
val args = if (s.isRawType()) null else s.arguments
return useSimpleTypeClass(cls, args, s.hasQuestionMark)
}
s.classifier.owner is IrTypeParameter -> {
val javaResult = useTypeParameter(s.classifier.owner as IrTypeParameter)
@@ -629,10 +636,13 @@ class X {
val classLabel: String, val shortName: String
)
/*
This returns the `X` in c's label `@"class;X"`.
*/
private fun getUnquotedClassLabel(c: IrClass, typeArgs: List<IrTypeArgument>): ClassLabelResults {
/**
* This returns the `X` in c's label `@"class;X"`.
*
* `typeArgs` can be null to describe a raw generic type.
* For non-generic types it will be zero-length list.
*/
private fun getUnquotedClassLabel(c: IrClass, typeArgs: List<IrTypeArgument>?): ClassLabelResults {
val pkg = c.packageFqName?.asString() ?: ""
val cls = c.name.asString()
val parent = c.parent
@@ -644,20 +654,24 @@ class X {
if (pkg.isEmpty()) cls else "$pkg.$cls"
}
val typeArgLabels = typeArgs.map { getTypeArgumentLabel(it) }
val typeArgLabels = typeArgs?.map { getTypeArgumentLabel(it) }
val typeArgsShortName =
if(typeArgs.isEmpty())
if (typeArgLabels == null)
"<>"
else if(typeArgLabels.isEmpty())
""
else
typeArgLabels.joinToString(prefix = "<", postfix = ">", separator = ",") { it.shortName }
return ClassLabelResults(
label + typeArgLabels.joinToString(separator = "") { ";{${it.id}}" },
label + (typeArgLabels?.joinToString(separator = "") { ";{${it.id}}" } ?: "<>"),
cls + typeArgsShortName
)
}
fun getClassLabel(c: IrClass, typeArgs: List<IrTypeArgument>): ClassLabelResults {
// `args` can be null to describe a raw generic type.
// For non-generic types it will be zero-length list.
fun getClassLabel(c: IrClass, typeArgs: List<IrTypeArgument>?): ClassLabelResults {
if (c.isAnonymousObject) {
logger.warn(Severity.ErrorSevere, "Label generation should not be requested for an anonymous class")
}
@@ -706,26 +720,36 @@ class X {
fun addModifiers(modifiable: Label<out DbModifiable>, vararg modifiers: String) =
modifiers.forEach { tw.writeHasModifier(modifiable, extractModifier(it)) }
/**
* Extracts the supertypes of class `c` with arguments `typeArgsQ`, or if `typeArgsQ` is null, the non-paramterised
* version of `c`. `id` is the label of this class or class instantiation.
*
* For example, for type `List` if `typeArgsQ` is non-null list `[String]` then we will extract the supertypes
* of `List<String>`, i.e. `Appendable<String>` etc, or if `typeArgsQ` is null we will extract `Appendable<E>`
* where `E` is the type variable declared as `List<E>`.
*/
fun extractClassSupertypes(c: IrClass, id: Label<out DbReftype>, typeArgsQ: List<IrTypeArgument>? = null) {
extractClassSupertypes(c.superTypes, c.typeParameters, id, typeArgsQ)
sealed class ExtractSupertypesMode {
object Unbound : ExtractSupertypesMode()
object Raw : ExtractSupertypesMode()
data class Specialised(val typeArgs : List<IrTypeArgument>) : ExtractSupertypesMode()
}
fun extractClassSupertypes(superTypes: List<IrType>, typeParameters: List<IrTypeParameter>, id: Label<out DbReftype>, typeArgsQ: List<IrTypeArgument>? = null) {
/**
* Extracts the supertypes of class `c`, either the unbound version, raw version or a specialisation to particular
* type arguments, depending on the value of `mode`. `id` is the label of this class or class instantiation.
*
* For example, for type `List` if `mode` `Specialised([String])` then we will extract the supertypes
* of `List<String>`, i.e. `Appendable<String>` etc, or if `mode` is `Unbound` we will extract `Appendable<E>`
* where `E` is the type variable declared as `List<E>`. Finally if `mode` is `Raw` we will extract the raw type
* `Appendable`, represented in QL as `Appendable<>`.
*/
fun extractClassSupertypes(c: IrClass, id: Label<out DbReftype>, mode: ExtractSupertypesMode = ExtractSupertypesMode.Unbound) {
extractClassSupertypes(c.superTypes, c.typeParameters, id, mode)
}
fun extractClassSupertypes(superTypes: List<IrType>, typeParameters: List<IrTypeParameter>, id: Label<out DbReftype>, mode: ExtractSupertypesMode = ExtractSupertypesMode.Unbound) {
// Note we only need to substitute type args here because it is illegal to directly extend a type variable.
// (For example, we can't have `class A<E> : E`, but can have `class A<E> : Comparable<E>`)
val subbedSupertypes = typeArgsQ?.let { typeArgs ->
superTypes.map {
it.substituteTypeArguments(typeParameters, typeArgs)
val subbedSupertypes = when(mode) {
is ExtractSupertypesMode.Specialised -> {
superTypes.map {
it.substituteTypeArguments(typeParameters, mode.typeArgs)
}
}
} ?: superTypes
else -> superTypes
}
for(t in subbedSupertypes) {
when(t) {
@@ -734,7 +758,7 @@ class X {
is IrClass -> {
val classifier: IrClassifierSymbol = t.classifier
val tcls: IrClass = classifier.owner as IrClass
val l = useClassInstance(tcls, t.arguments).typeResult.id
val l = useClassInstance(tcls, if (t.arguments.isNotEmpty() && mode is ExtractSupertypesMode.Raw) null else t.arguments).typeResult.id
tw.writeExtendsReftype(id, l)
}
else -> {

View File

@@ -1,12 +1,27 @@
package com.github.codeql.utils
import com.github.codeql.KotlinUsesExtractor
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.backend.common.ir.createImplicitParameterDeclarationWithWrappedDescriptor
import org.jetbrains.kotlin.descriptors.ClassKind
import org.jetbrains.kotlin.ir.builders.declarations.addConstructor
import org.jetbrains.kotlin.ir.builders.declarations.buildClass
import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.ir.declarations.IrTypeParameter
import org.jetbrains.kotlin.ir.declarations.impl.IrExternalPackageFragmentImpl
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl
import org.jetbrains.kotlin.ir.expressions.IrConstructorCall
import org.jetbrains.kotlin.ir.expressions.impl.IrConstructorCallImpl
import org.jetbrains.kotlin.ir.symbols.IrTypeParameterSymbol
import org.jetbrains.kotlin.ir.symbols.impl.DescriptorlessExternalPackageFragmentSymbol
import org.jetbrains.kotlin.ir.types.*
import org.jetbrains.kotlin.ir.types.impl.IrSimpleTypeImpl
import org.jetbrains.kotlin.ir.types.impl.IrStarProjectionImpl
import org.jetbrains.kotlin.ir.types.impl.makeTypeProjection
import org.jetbrains.kotlin.ir.util.constructedClassType
import org.jetbrains.kotlin.ir.util.constructors
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.types.Variance
fun IrType.substituteTypeArguments(params: List<IrTypeParameter>, arguments: List<IrTypeArgument>) =
@@ -100,11 +115,61 @@ fun IrTypeArgument.lowerBound(context: IrPluginContext) =
else -> context.irBuiltIns.nothingType
}
fun IrType.substituteTypeAndArguments(substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>?, topLevelMatchHandler: (IrTypeArgument) -> IrType): IrType =
fun IrType.substituteTypeAndArguments(substitutionMap: Map<IrTypeParameterSymbol, IrTypeArgument>?, useContext: KotlinUsesExtractor.TypeContext, pluginContext: IrPluginContext): IrType =
substitutionMap?.let { substMap ->
this.classifierOrNull?.let { typeClassifier ->
substMap[typeClassifier]?.let {
topLevelMatchHandler(it)
when(useContext) {
KotlinUsesExtractor.TypeContext.RETURN -> it.upperBound(pluginContext)
else -> it.lowerBound(pluginContext)
}
} ?: (this as IrSimpleType).substituteTypeArguments(substMap)
} ?: this
} ?: this
} ?: this
private object RawTypeAnnotation {
// Much of this is taken from JvmGeneratorExtensionsImpl.kt, which is not easily accessible in plugin context.
// The constants "kotlin.internal.ir" and "RawType" could be referred to symbolically, but they move package
// between different versions of the Kotlin compiler.
val annotationConstructor: IrConstructorCall by lazy {
val irInternalPackage = FqName("kotlin.internal.ir")
val parent = IrExternalPackageFragmentImpl(
DescriptorlessExternalPackageFragmentSymbol(),
irInternalPackage
)
val annoClass = IrFactoryImpl.buildClass {
kind = ClassKind.ANNOTATION_CLASS
name = irInternalPackage.child(Name.identifier("RawType")).shortName()
}.apply {
createImplicitParameterDeclarationWithWrappedDescriptor()
this.parent = parent
addConstructor {
isPrimary = true
}
}
val constructor = annoClass.constructors.single()
IrConstructorCallImpl.fromSymbolOwner(
constructor.constructedClassType,
constructor.symbol
)
}
}
fun IrType.toRawType(): IrType =
when(this) {
is IrSimpleType -> {
when(val owner = this.classifier.owner) {
is IrClass -> {
if (this.arguments.isNotEmpty())
this.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor))
else
this
}
is IrTypeParameter -> owner.superTypes[0].toRawType()
else -> this
}
}
else -> this
}
typealias TypeSubstitution = (IrType, KotlinUsesExtractor.TypeContext, IrPluginContext) -> IrType