Merge branch 'main' into starcke/automodel-pack

java/documentation/library-coverage/coverage.csv

@@ -59,9 +59,9 @@ java.beans,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
 java.io,50,,46,,,22,,,,,,,,,,,,,,28,,,,,,,,,,,,,,,,,,,44,2
 java.lang,31,,93,,13,,,,,,,,,,,,8,,,5,,,4,,,1,,,,,,,,,,,,,57,36
 java.net,13,3,23,,,,,,,,,,,,,,,,,,,,,,,,,,13,,,,,,,,,3,23,
-java.nio,53,,36,,,5,,,,,,,,,,,,,,47,,,,,,,,,1,,,,,,,,,,36,
+java.nio,49,,36,,,5,,,,,,,,,,,,,,43,,,,,,,,,1,,,,,,,,,,36,
 java.sql,13,,2,,,,,,,,,,,,,,,,,,,,,,,,,,4,,9,,,,,,,,2,
-java.util,45,,485,,,1,,,,,,,,,,,34,,,,,,,5,2,,1,2,,,,,,,,,,,45,440
+java.util,45,,487,,,1,,,,,,,,,,,34,,,,,,,5,2,,1,2,,,,,,,,,,,45,442
 javafx.scene.web,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,
 javax.faces.context,2,7,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,7,,
 javax.imageio.stream,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,

java/documentation/library-coverage/coverage.rst

@@ -18,10 +18,10 @@ Java framework & library support
    `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,730,41,7,,,,,
    JBoss Logging,``org.jboss.logging``,,,324,,,,,,
    `JSON-java <https://github.com/stleary/JSON-java>`_,``org.json``,,236,,,,,,,
-   Java Standard Library,``java.*``,3,689,205,80,,9,,,18
+   Java Standard Library,``java.*``,3,691,201,76,,9,,,18
    Java extensions,"``javax.*``, ``jakarta.*``",63,672,34,2,4,,1,1,2
    Kotlin Standard Library,``kotlin*``,,1849,16,14,,,,,2
    `Spring <https://spring.io/>`_,``org.springframework.*``,29,483,115,4,,28,14,,35
    Others,"``actions.osgi``, ``antlr``, ``cn.hutool.core.codec``, ``com.alibaba.druid.sql``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.google.gson``, ``com.hubspot.jinjava``, ``com.jcraft.jsch``, ``com.mitchellbosecke.pebble``, ``com.opensymphony.xwork2``, ``com.rabbitmq.client``, ``com.thoughtworks.xstream``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``freemarker.cache``, ``freemarker.template``, ``groovy.lang``, ``groovy.text``, ``groovy.util``, ``hudson``, ``io.jsonwebtoken``, ``io.netty.bootstrap``, ``io.netty.buffer``, ``io.netty.channel``, ``io.netty.handler.codec``, ``io.netty.handler.ssl``, ``io.netty.handler.stream``, ``io.netty.resolver``, ``io.netty.util``, ``javafx.scene.web``, ``jenkins``, ``jodd.json``, ``net.sf.json``, ``net.sf.saxon.s9api``, ``ognl``, ``okhttp3``, ``org.acegisecurity``, ``org.antlr.runtime``, ``org.apache.commons.codec``, ``org.apache.commons.compress.archivers.tar``, ``org.apache.commons.exec``, ``org.apache.commons.httpclient.util``, ``org.apache.commons.jelly``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.lang``, ``org.apache.commons.logging``, ``org.apache.commons.net``, ``org.apache.commons.ognl``, ``org.apache.directory.ldap.client.api``, ``org.apache.hadoop.fs``, ``org.apache.hadoop.hive.metastore``, ``org.apache.hc.client5.http.async.methods``, ``org.apache.hc.client5.http.classic.methods``, ``org.apache.hc.client5.http.fluent``, ``org.apache.hive.hcatalog.templeton``, ``org.apache.ibatis.jdbc``, ``org.apache.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.apache.struts.beanvalidation.validation.interceptor``, ``org.apache.struts2``, ``org.apache.tools.ant``, ``org.apache.tools.zip``, ``org.apache.velocity.app``, ``org.apache.velocity.runtime``, ``org.codehaus.cargo.container.installer``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.eclipse.jetty.client``, ``org.fusesource.leveldbjni``, ``org.geogebra.web.full.main``, ``org.gradle.api.file``, ``org.hibernate``, ``org.influxdb``, ``org.jdbi.v3.core``, ``org.jenkins.ui.icon``, ``org.jenkins.ui.symbol``, ``org.jooq``, ``org.kohsuke.stapler``, ``org.mvel2``, ``org.openjdk.jmh.runner.options``, ``org.scijava.log``, ``org.slf4j``, ``org.thymeleaf``, ``org.xml.sax``, ``org.xmlpull.v1``, ``org.yaml.snakeyaml``, ``play.libs.ws``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``, ``retrofit2``",126,10081,652,89,6,18,18,,200
-   Totals,,283,18451,2142,290,16,122,33,1,391
+   Totals,,283,18453,2138,286,16,122,33,1,391
 

java/kotlin-extractor/src/main/kotlin/ExternalDeclExtractor.kt

@@ -88,12 +88,10 @@ class ExternalDeclExtractor(val logger: FileLogger, val invocationTrapFile: Stri
             nextBatch.forEach { workPair ->
                 val (irDecl, possiblyLongSignature) = workPair
                 extractElement(irDecl, possiblyLongSignature, false) { trapFileBW, signature, manager ->
-                    val containingClass = getContainingClassOrSelf(irDecl)
-                    if (containingClass == null) {
-                        logger.errorElement("Unable to get containing class", irDecl)
+                    val binaryPath = getIrDeclarationBinaryPath(irDecl)
+                    if (binaryPath == null) {
+                        logger.errorElement("Unable to get binary path", irDecl)
                     } else {
-                        val binaryPath = getIrClassBinaryPath(containingClass)
-
                         // We want our comments to be the first thing in the file,
                         // so start off with a PlainTrapWriter
                         val tw = PlainTrapWriter(logger.loggerBase, TrapLabelManager(), trapFileBW, diagnosticTrapWriter)

java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt

@@ -191,7 +191,7 @@ open class KotlinFileExtractor(
                     }
                 }
                 is IrFunction -> {
-                    val parentId = useDeclarationParent(declaration.parent, false)?.cast<DbReftype>()
+                    val parentId = useDeclarationParentOf(declaration, false)?.cast<DbReftype>()
                     if (parentId != null) {
                         extractFunction(declaration, parentId, extractBody = extractFunctionBodies, extractMethodAndParameterTypeAccesses = extractFunctionBodies, extractAnnotations = extractAnnotations, null, listOf())
                     }
@@ -201,21 +201,21 @@ open class KotlinFileExtractor(
                     // Leaving this intentionally empty. init blocks are extracted during class extraction.
                 }
                 is IrProperty -> {
-                    val parentId = useDeclarationParent(declaration.parent, false)?.cast<DbReftype>()
+                    val parentId = useDeclarationParentOf(declaration, false)?.cast<DbReftype>()
                     if (parentId != null) {
                         extractProperty(declaration, parentId, extractBackingField = true, extractFunctionBodies = extractFunctionBodies, extractPrivateMembers = extractPrivateMembers, extractAnnotations = extractAnnotations, null, listOf())
                     }
                     Unit
                 }
                 is IrEnumEntry -> {
-                    val parentId = useDeclarationParent(declaration.parent, false)?.cast<DbReftype>()
+                    val parentId = useDeclarationParentOf(declaration, false)?.cast<DbReftype>()
                     if (parentId != null) {
                         extractEnumEntry(declaration, parentId, extractPrivateMembers, extractFunctionBodies)
                     }
                     Unit
                 }
                 is IrField -> {
-                    val parentId = useDeclarationParent(getFieldParent(declaration), false)?.cast<DbReftype>()
+                    val parentId = useDeclarationParentOf(declaration, false)?.cast<DbReftype>()
                     if (parentId != null) {
                         // For consistency with the Java extractor, enum entries get type accesses only if we're extracting from .kt source (i.e., when `extractFunctionBodies` is set)
                         extractField(declaration, parentId, extractAnnotationEnumTypeAccesses = extractFunctionBodies)
@@ -408,11 +408,10 @@ open class KotlinFileExtractor(
 
     private fun getLocation(decl: IrDeclaration, typeArgs: List<IrTypeArgument>?): Label<DbLocation> {
         return if (typeArgs != null && typeArgs.isNotEmpty()) {
-            val c = getContainingClassOrSelf(decl)
-            if (c == null) {
+            val binaryPath = getIrDeclarationBinaryPath(decl)
+            if (binaryPath == null) {
                 tw.getLocation(decl)
             } else {
-                val binaryPath = getIrClassBinaryPath(c)
                 val newTrapWriter = tw.makeFileTrapWriter(binaryPath, true)
                 newTrapWriter.getWholeFileLocation()
             }
@@ -472,7 +471,7 @@ open class KotlinFileExtractor(
 
     private fun extractObinitFunction(c: IrClass, parentId: Label<out DbClassorinterface>) {
         // add method:
-        val obinitLabel = getObinitLabel(c)
+        val obinitLabel = getObinitLabel(c, parentId)
         val obinitId = tw.getLabelFor<DbMethod>(obinitLabel)
         val returnType = useType(pluginContext.irBuiltIns.unitType, TypeContext.RETURN)
         tw.writeMethods(obinitId, "<obinit>", "<obinit>()", returnType.javaResult.id, parentId, obinitId)
@@ -552,9 +551,13 @@ open class KotlinFileExtractor(
                 logger.warnElement("Expected annotation property to define a getter", prop)
             } else {
                 val getterId = useFunction<DbMethod>(getter)
-                val exprId = extractAnnotationValueExpression(v, id, i, "{$getterId}", getter.returnType, extractEnumTypeAccesses)
-                if (exprId != null) {
-                    tw.writeAnnotValue(id, getterId, exprId)
+                if (getterId == null) {
+                    logger.errorElement("Couldn't get ID for getter", getter)
+                } else {
+                    val exprId = extractAnnotationValueExpression(v, id, i, "{$getterId}", getter.returnType, extractEnumTypeAccesses)
+                    if (exprId != null) {
+                        tw.writeAnnotValue(id, getterId, exprId)
+                    }
                 }
             }
         }
@@ -979,6 +982,10 @@ open class KotlinFileExtractor(
     private fun extractInstanceInitializerBlock(parent: StmtParent, enclosingConstructor: IrConstructor) {
         with("object initializer block", enclosingConstructor) {
             val constructorId = useFunction<DbConstructor>(enclosingConstructor)
+            if (constructorId == null) {
+                logger.errorElement("Cannot get ID for constructor", enclosingConstructor)
+                return
+            }
             val enclosingClass = enclosingConstructor.parentClassOrNull
             if (enclosingClass == null) {
                 logger.errorElement("Constructor's parent is not a class", enclosingConstructor)
@@ -1152,6 +1159,10 @@ open class KotlinFileExtractor(
             return
 
         val id = getDefaultsMethodLabel(f)
+        if (id == null) {
+            logger.errorElement("Cannot get defaults method label for function", f)
+            return
+        }
         val locId = getLocation(f, null)
         val extReceiver = f.extensionReceiverParameter
         val dispatchReceiver = if (f.shouldExtractAsStatic) null else f.dispatchReceiverParameter
@@ -1273,9 +1284,13 @@ open class KotlinFileExtractor(
             val sourceParentId =
                 maybeSourceParentId ?:
                     if (typeSubstitution != null)
-                        useDeclarationParent(f.parent, false)
+                        useDeclarationParentOf(f, false)
                     else
                         parentId
+            if (sourceParentId == null) {
+                logger.errorElement("Cannot get source parent ID for function", f)
+                return
+            }
             val sourceDeclId = tw.getLabelFor<DbCallable>(getFunctionLabel(f, sourceParentId, listOf(), overloadParameters))
             val overriddenAttributes = OverriddenFunctionAttributes(id = overloadId, sourceDeclarationId = sourceDeclId, valueParameters = overloadParameters)
             forceExtractFunction(f, parentId, extractBody = false, extractMethodAndParameterTypeAccesses, extractAnnotations = false, typeSubstitution, classTypeArgsIncludingOuterClasses, overriddenAttributes = overriddenAttributes)
@@ -1293,7 +1308,7 @@ open class KotlinFileExtractor(
                         val constructorCallId = tw.getFreshIdLabel<DbConstructorinvocationstmt>()
                         tw.writeStmts_constructorinvocationstmt(constructorCallId, blockId, 0, overloadId)
                         tw.writeHasLocation(constructorCallId, realFunctionLocId)
-                        tw.writeCallableBinding(constructorCallId, getDefaultsMethodLabel(f))
+                        tw.writeCallableBinding(constructorCallId, getDefaultsMethodLabel(f, parentId))
 
                         extractDefaultsCallArguments(constructorCallId, f, overloadId, constructorCallId, regularArgs, null, null)
                     } else {
@@ -1410,10 +1425,17 @@ open class KotlinFileExtractor(
 
                 val sourceDeclaration =
                     overriddenAttributes?.sourceDeclarationId ?:
-                        if (typeSubstitution != null && overriddenAttributes?.id == null)
-                            useFunction(f)
-                        else
+                        if (typeSubstitution != null && overriddenAttributes?.id == null) {
+                            val sourceFunId = useFunction<DbCallable>(f)
+                            if (sourceFunId == null) {
+                                logger.errorElement("Cannot get source ID for function", f)
+                                id // TODO: This is wrong; we ought to just fail in this case
+                            } else {
+                                sourceFunId
+                            }
+                        } else {
                             id
+                        }
 
                 val extReceiver = f.extensionReceiverParameter
                 // The following parameter order is correct, because member $default methods (where the order would be [dispatchParam], [extensionParam], normalParams) are not extracted here
@@ -1594,7 +1616,7 @@ open class KotlinFileExtractor(
                 }
 
                 if (bf != null && extractBackingField) {
-                    val fieldParentId = useDeclarationParent(getFieldParent(bf), false)
+                    val fieldParentId = useDeclarationParentOf(bf, false)
                     if (fieldParentId != null) {
                         val fieldId = extractField(bf, fieldParentId.cast(), extractFunctionBodies)
                         tw.writeKtPropertyBackingFields(id, fieldId)
@@ -2066,13 +2088,23 @@ open class KotlinFileExtractor(
             getFunctionShortName(f).nameInDB + "\$default"
         }
 
-    private fun getDefaultsMethodLabel(f: IrFunction): Label<out DbCallable> {
+    private fun getDefaultsMethodLabel(f: IrFunction): Label<out DbCallable>? {
+        val classTypeArgsIncludingOuterClasses = null
+        val parentId = useDeclarationParentOf(f, false, classTypeArgsIncludingOuterClasses, true)
+        if (parentId == null) {
+            logger.errorElement("Couldn't get parent ID for defaults method", f)
+            return null
+        }
+        return getDefaultsMethodLabel(f, parentId)
+    }
+
+    private fun getDefaultsMethodLabel(f: IrFunction, parentId: Label<out DbElement>): Label<out DbCallable> {
         val defaultsMethodName = if (f is IrConstructor) "<init>" else getDefaultsMethodName(f)
         val argTypes = getDefaultsMethodArgTypes(f)
 
         val defaultMethodLabelStr = getFunctionLabel(
             f.parent,
-            maybeParentId = null,
+            parentId,
             defaultsMethodName,
             argTypes,
             erase(f.returnType),
@@ -2121,7 +2153,7 @@ open class KotlinFileExtractor(
         if (overriddenCallTarget.isLocalFunction()) {
             extractTypeAccess(getLocallyVisibleFunctionLabels(overriddenCallTarget).type, locId, id, -1, enclosingCallable, enclosingStmt)
         } else {
-            extractStaticTypeAccessQualifierUnchecked(overriddenCallTarget.parent, id, locId, enclosingCallable, enclosingStmt)
+            extractStaticTypeAccessQualifierUnchecked(overriddenCallTarget, id, locId, enclosingCallable, enclosingStmt)
         }
 
         extractDefaultsCallArguments(id, overriddenCallTarget, enclosingCallable, enclosingStmt, valueArguments, dispatchReceiver, extensionReceiver)
@@ -2347,8 +2379,17 @@ open class KotlinFileExtractor(
         extractValueArguments(argParent, idxOffset)
     }
 
-    private fun extractStaticTypeAccessQualifierUnchecked(parent: IrDeclarationParent, parentExpr: Label<out DbExprparent>, locId: Label<DbLocation>, enclosingCallable: Label<out DbCallable>?, enclosingStmt: Label<out DbStmt>?) {
-        if (parent is IrClass) {
+    private fun extractStaticTypeAccessQualifierUnchecked(target: IrDeclaration, parentExpr: Label<out DbExprparent>, locId: Label<DbLocation>, enclosingCallable: Label<out DbCallable>?, enclosingStmt: Label<out DbStmt>?) {
+        val parent = target.parent
+        if (parent is IrExternalPackageFragment) {
+            // This is in a file class.
+            val fqName = getFileClassFqName(target)
+            if (fqName == null) {
+                logger.error("Can't get FqName for element in external package fragment ${target.javaClass}")
+            } else {
+                extractTypeAccess(useFileClassType(fqName), locId, parentExpr, -1, enclosingCallable, enclosingStmt)
+            }
+        } else if (parent is IrClass) {
             extractTypeAccessRecursive(parent.toRawType(), locId, parentExpr, -1, enclosingCallable, enclosingStmt)
         } else if (parent is IrFile) {
             extractTypeAccess(useFileClassType(parent), locId, parentExpr, -1, enclosingCallable, enclosingStmt)
@@ -2359,7 +2400,7 @@ open class KotlinFileExtractor(
 
     private fun extractStaticTypeAccessQualifier(target: IrDeclaration, parentExpr: Label<out DbExprparent>, locId: Label<DbLocation>, enclosingCallable: Label<out DbCallable>?, enclosingStmt: Label<out DbStmt>?) {
         if (target.shouldExtractAsStatic) {
-            extractStaticTypeAccessQualifierUnchecked(target.parent, parentExpr, locId, enclosingCallable, enclosingStmt)
+            extractStaticTypeAccessQualifierUnchecked(target, parentExpr, locId, enclosingCallable, enclosingStmt)
         }
     }
 
@@ -2926,7 +2967,11 @@ open class KotlinFileExtractor(
                         tw.writeStmts_throwstmt(throwId, stmtParent.parent, stmtParent.idx, callable)
                         tw.writeHasLocation(throwId, locId)
                         val newExprId = extractNewExpr(it, null, thrownType, locId, throwId, 0, callable, throwId)
-                        extractTypeAccess(thrownType, locId, newExprId, -3, callable, throwId)
+                        if (newExprId == null) {
+                            logger.errorElement("No ID for newExpr in noWhenBranchMatchedException", c)
+                        } else {
+                            extractTypeAccess(thrownType, locId, newExprId, -3, callable, throwId)
+                        }
                     }
                 }
                 isBuiltinCallInternal(c, "illegalArgumentException") -> {
@@ -3270,13 +3315,20 @@ open class KotlinFileExtractor(
         idx: Int,
         callable: Label<out DbCallable>,
         enclosingStmt: Label<out DbStmt>
-    ): Label<DbNewexpr> = extractNewExpr(useFunction<DbConstructor>(calledConstructor, constructorTypeArgs), constructedType, locId, parent, idx, callable, enclosingStmt)
+    ): Label<DbNewexpr>? {
+        val funId = useFunction<DbConstructor>(calledConstructor, constructorTypeArgs)
+        if (funId == null) {
+            logger.error("Cannot get ID for newExpr function")
+            return null
+        }
+        return extractNewExpr(funId, constructedType, locId, parent, idx, callable, enclosingStmt)
+    }
 
     private fun needsObinitFunction(c: IrClass) = c.primaryConstructor == null && c.constructors.count() > 1
 
-    private fun getObinitLabel(c: IrClass) = getFunctionLabel(
+    private fun getObinitLabel(c: IrClass, parentId: Label<out DbElement>): String = getFunctionLabel(
         c,
-        null,
+        parentId,
         "<obinit>",
         listOf(),
         pluginContext.irBuiltIns.unitType,
@@ -3306,30 +3358,40 @@ open class KotlinFileExtractor(
         val valueArgs = (0 until e.valueArgumentsCount).map { e.getValueArgument(it) }
 
         val id = if (e !is IrEnumConstructorCall && callUsesDefaultArguments(e.symbol.owner, valueArgs)) {
-            extractNewExpr(getDefaultsMethodLabel(e.symbol.owner).cast(), type, locId, parent, idx, callable, enclosingStmt).also {
+            val defaultsMethodId = getDefaultsMethodLabel(e.symbol.owner)
+            if (defaultsMethodId == null) {
+                logger.errorElement("Cannot get defaults method ID", e)
+                return
+            }
+            extractNewExpr(defaultsMethodId.cast(), type, locId, parent, idx, callable, enclosingStmt).also {
                 extractDefaultsCallArguments(it, e.symbol.owner, callable, enclosingStmt, valueArgs, null, null)
             }
         } else {
-            extractNewExpr(e.symbol.owner, eType.arguments, type, locId, parent, idx, callable, enclosingStmt).also {
-
-                val realCallTarget = e.symbol.owner.realOverrideTarget
-                // Generated constructor calls to kotlin.Enum have no arguments in IR, but the constructor takes two parameters.
-                if (e is IrEnumConstructorCall &&
-                    realCallTarget is IrConstructor &&
-                    realCallTarget.parentClassOrNull?.fqNameWhenAvailable?.asString() == "kotlin.Enum" &&
-                    realCallTarget.valueParameters.size == 2 &&
-                    realCallTarget.valueParameters[0].type == pluginContext.irBuiltIns.stringType &&
-                    realCallTarget.valueParameters[1].type == pluginContext.irBuiltIns.intType) {
-
-                    val id0 = extractNull(pluginContext.irBuiltIns.stringType, locId, it, 0, callable, enclosingStmt)
-                    tw.writeCompiler_generated(id0, CompilerGeneratedKinds.ENUM_CONSTRUCTOR_ARGUMENT.kind)
-
-                    val id1 = extractConstantInteger(0, locId, it, 1, callable, enclosingStmt)
-                    tw.writeCompiler_generated(id1, CompilerGeneratedKinds.ENUM_CONSTRUCTOR_ARGUMENT.kind)
-                } else {
-                    extractCallValueArguments(it, e, enclosingStmt, callable, 0)
-                }
+            val newExprId = extractNewExpr(e.symbol.owner, eType.arguments, type, locId, parent, idx, callable, enclosingStmt)
+            if (newExprId == null) {
+                logger.errorElement("Cannot get newExpr ID", e)
+                return
             }
+
+            val realCallTarget = e.symbol.owner.realOverrideTarget
+            // Generated constructor calls to kotlin.Enum have no arguments in IR, but the constructor takes two parameters.
+            if (e is IrEnumConstructorCall &&
+                realCallTarget is IrConstructor &&
+                realCallTarget.parentClassOrNull?.fqNameWhenAvailable?.asString() == "kotlin.Enum" &&
+                realCallTarget.valueParameters.size == 2 &&
+                realCallTarget.valueParameters[0].type == pluginContext.irBuiltIns.stringType &&
+                realCallTarget.valueParameters[1].type == pluginContext.irBuiltIns.intType) {
+
+                val id0 = extractNull(pluginContext.irBuiltIns.stringType, locId, newExprId, 0, callable, enclosingStmt)
+                tw.writeCompiler_generated(id0, CompilerGeneratedKinds.ENUM_CONSTRUCTOR_ARGUMENT.kind)
+
+                val id1 = extractConstantInteger(0, locId, newExprId, 1, callable, enclosingStmt)
+                tw.writeCompiler_generated(id1, CompilerGeneratedKinds.ENUM_CONSTRUCTOR_ARGUMENT.kind)
+            } else {
+                extractCallValueArguments(newExprId, e, enclosingStmt, callable, 0)
+            }
+
+            newExprId
         }
 
         if (isAnonymous) {
@@ -3698,9 +3760,13 @@ open class KotlinFileExtractor(
 
                     val locId = tw.getLocation(e)
                     val methodId = useFunction<DbConstructor>(e.symbol.owner)
+                    if (methodId == null) {
+                        logger.errorElement("Cannot get ID for delegating constructor", e)
+                    } else {
+                        tw.writeCallableBinding(id.cast<DbCaller>(), methodId)
+                    }
 
                     tw.writeHasLocation(id, locId)
-                    tw.writeCallableBinding(id.cast<DbCaller>(), methodId)
                     extractCallValueArguments(id, e, id, callable, 0)
                     val dr = e.dispatchReceiver
                     if (dr != null) {
@@ -3782,7 +3848,13 @@ open class KotlinFileExtractor(
                         val id = tw.getFreshIdLabel<DbMethodaccess>()
                         val type = useType(pluginContext.irBuiltIns.unitType)
                         val locId = tw.getLocation(e)
-                        val methodLabel = getObinitLabel(irConstructor.parentAsClass)
+                        val parentClass = irConstructor.parentAsClass
+                        val parentId = useDeclarationParentOf(irConstructor, false, null, true)
+                        if (parentId == null) {
+                            logger.errorElement("Cannot get parent ID for obinit", e)
+                            return
+                        }
+                        val methodLabel = getObinitLabel(parentClass, parentId)
                         val methodId = tw.getLabelFor<DbMethod>(methodLabel)
                         tw.writeExprs_methodaccess(id, type.javaResult.id, exprParent.parent, exprParent.idx)
                         tw.writeExprsKotlinType(id, type.kotlinResult.id)
@@ -4636,7 +4708,11 @@ open class KotlinFileExtractor(
             extractExprContext(callId, locId, labels.methodId, retId)
 
             val callableId = useFunction<DbCallable>(target.owner.realOverrideTarget, classTypeArgsIncludingOuterClasses)
-            tw.writeCallableBinding(callId.cast<DbCaller>(), callableId)
+            if (callableId == null) {
+                logger.error("Cannot get ID for reflection target")
+            } else {
+                tw.writeCallableBinding(callId.cast<DbCaller>(), callableId)
+            }
 
             val useFirstArgAsDispatch: Boolean
             if (dispatchReceiverInfo != null) {
@@ -4818,20 +4894,24 @@ open class KotlinFileExtractor(
             val getterReturnType = parameterTypes.last()
 
             if (getter != null) {
-                val getLabels = addFunctionManual(tw.getFreshIdLabel(), OperatorNameConventions.GET.asString(), getterParameterTypes, getterReturnType, classId, locId)
                 val getterCallableId = useFunction<DbCallable>(getter.owner.realOverrideTarget, classTypeArguments)
+                if (getterCallableId == null) {
+                    logger.errorElement("Cannot get ID for getter", propertyReferenceExpr)
+                } else {
+                    val getLabels = addFunctionManual(tw.getFreshIdLabel(), OperatorNameConventions.GET.asString(), getterParameterTypes, getterReturnType, classId, locId)
 
-                helper.extractCallToReflectionTarget(
-                    getLabels,
-                    getter,
-                    getterReturnType,
-                    expressionTypeArguments,
-                    classTypeArguments
-                )
+                    helper.extractCallToReflectionTarget(
+                        getLabels,
+                        getter,
+                        getterReturnType,
+                        expressionTypeArguments,
+                        classTypeArguments
+                    )
 
-                tw.writePropertyRefGetBinding(idPropertyRef, getterCallableId)
+                    tw.writePropertyRefGetBinding(idPropertyRef, getterCallableId)
 
-                helper.extractPropertyReferenceInvoke(getLabels.methodId, getterParameterTypes, getterReturnType)
+                    helper.extractPropertyReferenceInvoke(getLabels.methodId, getterParameterTypes, getterReturnType)
+                }
             } else {
                 // Property without a getter.
                 if (backingField == null) {
@@ -4852,19 +4932,22 @@ open class KotlinFileExtractor(
             }
 
             if (setter != null) {
-                val setLabels = addFunctionManual(tw.getFreshIdLabel(), OperatorNameConventions.SET.asString(), parameterTypes, pluginContext.irBuiltIns.unitType, classId, locId)
-
                 val setterCallableId = useFunction<DbCallable>(setter.owner.realOverrideTarget, classTypeArguments)
+                if (setterCallableId == null) {
+                    logger.errorElement("Cannot get ID for setter", propertyReferenceExpr)
+                } else {
+                    val setLabels = addFunctionManual(tw.getFreshIdLabel(), OperatorNameConventions.SET.asString(), parameterTypes, pluginContext.irBuiltIns.unitType, classId, locId)
 
-                helper.extractCallToReflectionTarget(
-                    setLabels,
-                    setter,
-                    pluginContext.irBuiltIns.unitType,
-                    expressionTypeArguments,
-                    classTypeArguments
-                )
+                    helper.extractCallToReflectionTarget(
+                        setLabels,
+                        setter,
+                        pluginContext.irBuiltIns.unitType,
+                        expressionTypeArguments,
+                        classTypeArguments
+                    )
 
-                tw.writePropertyRefSetBinding(idPropertyRef, setterCallableId)
+                    tw.writePropertyRefSetBinding(idPropertyRef, setterCallableId)
+                }
             } else {
                 if (backingField != null && !backingField.owner.isFinal) {
                     val setLabels = addFunctionManual(tw.getFreshIdLabel(), OperatorNameConventions.SET.asString(), parameterTypes, pluginContext.irBuiltIns.unitType, classId, locId)
@@ -4999,7 +5082,11 @@ open class KotlinFileExtractor(
             tw.writeCallableBinding(idMemberRef, ids.constructor)
 
             val targetCallableId = useFunction<DbCallable>(target.owner.realOverrideTarget, classTypeArguments)
-            tw.writeMemberRefBinding(idMemberRef, targetCallableId)
+            if (targetCallableId == null) {
+                logger.errorElement("Cannot get ID for function reference callable", functionReferenceExpr)
+            } else {
+                tw.writeMemberRefBinding(idMemberRef, targetCallableId)
+            }
 
             val helper = CallableReferenceHelper(functionReferenceExpr, locId, ids)
 
@@ -5145,7 +5232,11 @@ open class KotlinFileExtractor(
         tw.writeExprsKotlinType(callId, callType.kotlinResult.id)
         extractExprContext(callId, locId, funLabels.methodId, retId)
         val calledMethodId = useFunction<DbMethod>(lambda)
-        tw.writeCallableBinding(callId, calledMethodId)
+        if (calledMethodId == null) {
+            logger.errorElement("Cannot get ID for called lambda", lambda)
+        } else {
+            tw.writeCallableBinding(callId, calledMethodId)
+        }
 
         // this access
         extractThisAccess(ids.type, funLabels.methodId, callId, -1, retId, locId)
@@ -5614,7 +5705,11 @@ open class KotlinFileExtractor(
                     tw.writeExprsKotlinType(callId, callType.kotlinResult.id)
                     extractExprContext(callId, locId, ids.function, returnId)
                     val calledMethodId = useFunction<DbMethod>(invokeMethod, functionType.arguments)
-                    tw.writeCallableBinding(callId, calledMethodId)
+                    if (calledMethodId == null) {
+                        logger.errorElement("Cannot get ID for called method", invokeMethod)
+                    } else {
+                        tw.writeCallableBinding(callId, calledMethodId)
+                    }
 
                     // <fn> access
                     val lhsId = tw.getFreshIdLabel<DbVaraccess>()
@@ -5737,14 +5832,17 @@ open class KotlinFileExtractor(
             if (baseConstructor == null) {
                 logger.warnElement("Cannot find base constructor", elementToReportOn)
             } else {
-                val superCallId = tw.getFreshIdLabel<DbSuperconstructorinvocationstmt>()
-                tw.writeStmts_superconstructorinvocationstmt(superCallId, constructorBlockId, 0, ids.constructor)
-
                 val baseConstructorId = useFunction<DbConstructor>(baseConstructor)
+                if (baseConstructorId == null) {
+                    logger.errorElement("Cannot find base constructor ID", elementToReportOn)
+                } else {
+                    val superCallId = tw.getFreshIdLabel<DbSuperconstructorinvocationstmt>()
+                    tw.writeStmts_superconstructorinvocationstmt(superCallId, constructorBlockId, 0, ids.constructor)
 
-                tw.writeHasLocation(superCallId, locId)
-                tw.writeCallableBinding(superCallId.cast<DbCaller>(), baseConstructorId)
-                extractSuperconstructorArgs(superCallId)
+                    tw.writeHasLocation(superCallId, locId)
+                    tw.writeCallableBinding(superCallId.cast<DbCaller>(), baseConstructorId)
+                    extractSuperconstructorArgs(superCallId)
+                }
             }
         }
 

java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt

@@ -2,6 +2,7 @@ package com.github.codeql
 
 import com.github.codeql.utils.*
 import com.github.codeql.utils.versions.codeQlWithHasQuestionMark
+import com.github.codeql.utils.versions.getFileClassFqName
 import com.github.codeql.utils.versions.getKotlinType
 import com.github.codeql.utils.versions.isRawType
 import com.semmle.extractor.java.OdasaOutput
@@ -71,18 +72,41 @@ open class KotlinUsesExtractor(
         TypeResult(fakeKotlinType(), "", "")
     )
 
+    fun useFileClassType(fqName: FqName) = TypeResults(
+        TypeResult(extractFileClass(fqName), "", ""),
+        TypeResult(fakeKotlinType(), "", "")
+    )
+
+    private fun useFileClassType(pkg: String, jvmName: String) = TypeResults(
+        TypeResult(extractFileClass(pkg, jvmName), "", ""),
+        TypeResult(fakeKotlinType(), "", "")
+    )
+
     fun extractFileClass(f: IrFile): Label<out DbClassorinterface> {
         val pkg = f.fqName.asString()
         val jvmName = getFileClassName(f)
+        val id = extractFileClass(pkg, jvmName)
+        if (tw.lm.fileClassLocationsExtracted.add(f)) {
+            val fileId = tw.mkFileId(f.path, false)
+            val locId = tw.getWholeFileLocation(fileId)
+            tw.writeHasLocation(id, locId)
+        }
+        return id
+    }
+
+    private fun extractFileClass(fqName: FqName): Label<out DbClassorinterface> {
+        val pkg = if (fqName.isRoot()) "" else fqName.parent().asString()
+        val jvmName = fqName.shortName().asString()
+        return extractFileClass(pkg, jvmName)
+    }
+
+    private fun extractFileClass(pkg: String, jvmName: String): Label<out DbClassorinterface> {
         val qualClassName = if (pkg.isEmpty()) jvmName else "$pkg.$jvmName"
         val label = "@\"class;$qualClassName\""
         val id: Label<DbClassorinterface> = tw.getLabelFor(label) {
-            val fileId = tw.mkFileId(f.path, false)
-            val locId = tw.getWholeFileLocation(fileId)
             val pkgId = extractPackage(pkg)
             tw.writeClasses_or_interfaces(it, jvmName, pkgId, it)
             tw.writeFile_class(it)
-            tw.writeHasLocation(it, locId)
 
             addModifiers(it, "public", "final")
         }
@@ -250,7 +274,11 @@ open class KotlinUsesExtractor(
             is IrClass -> extractExternalClassLater(parent)
             is IrFunction -> extractExternalEnclosingClassLater(parent)
             is IrFile -> logger.error("extractExternalEnclosingClassLater but no enclosing class.")
-            else -> logger.error("Unrecognised extractExternalEnclosingClassLater: " + d.javaClass)
+            is IrExternalPackageFragment -> {
+                // The parent is a (multi)file class. We don't need
+                // extract it separately.
+            }
+            else -> logger.error("Unrecognised extractExternalEnclosingClassLater ${parent.javaClass} for ${d.javaClass}")
         }
     }
 
@@ -293,7 +321,17 @@ open class KotlinUsesExtractor(
 
     private fun extractFunctionLaterIfExternalFileMember(f: IrFunction) {
         if (isExternalFileClassMember(f)) {
-            extractExternalClassLater(f.parentAsClass)
+            val p = f.parent
+            when (p) {
+                is IrClass -> extractExternalClassLater(p)
+                is IrExternalPackageFragment -> {
+                    // The parent is a (multi)file class. We don't need
+                    // extract it separately.
+                }
+                else -> {
+                    logger.warn("Unexpected parent type ${p.javaClass} for external file class member")
+                }
+            }
             (f as? IrSimpleFunction)?.correspondingPropertySymbol?.let {
                 extractPropertyLaterIfExternalFileMember(it.owner)
                 // No need to extract the function specifically, as the property's
@@ -761,6 +799,41 @@ open class KotlinUsesExtractor(
         }
     }
 
+    private fun parentOf(d: IrDeclaration): IrDeclarationParent {
+        if (d is IrField) {
+            return getFieldParent(d)
+        }
+        return d.parent
+    }
+
+    fun useDeclarationParentOf(
+        // The declaration
+        d: IrDeclaration,
+        // Whether the type of entity whose parent this is can be a
+        // top-level entity in the JVM's eyes. If so, then its parent may
+        // be a file; otherwise, if dp is a file foo.kt, then the parent
+        // is really the JVM class FooKt.
+        canBeTopLevel: Boolean,
+        classTypeArguments: List<IrTypeArgument>? = null,
+        inReceiverContext: Boolean = false):
+        Label<out DbElement>? {
+
+        val parent = parentOf(d)
+        if (parent is IrExternalPackageFragment) {
+            // This is in a file class.
+            val fqName = getFileClassFqName(d)
+            if (fqName == null) {
+                logger.error("Can't get FqName for element in external package fragment ${d.javaClass}")
+                return null
+            }
+            return extractFileClass(fqName)
+        }
+        return useDeclarationParent(parent, canBeTopLevel, classTypeArguments, inReceiverContext)
+    }
+
+    // Generally, useDeclarationParentOf should be used instead of
+    // calling this directly, as this cannot handle
+    // IrExternalPackageFragment
     fun useDeclarationParent(
         // The declaration parent according to Kotlin
         dp: IrDeclarationParent,
@@ -792,8 +865,7 @@ open class KotlinUsesExtractor(
                 }
             is IrFunction -> useFunction(dp)
             is IrExternalPackageFragment -> {
-                // TODO
-                logger.error("Unhandled IrExternalPackageFragment")
+                logger.error("Unable to handle IrExternalPackageFragment as an IrDeclarationParent")
                 null
             }
             else -> {
@@ -1034,8 +1106,13 @@ open class KotlinUsesExtractor(
      * enclosing classes to get the instantiation that this function is
      * in.
      */
-    fun getFunctionLabel(f: IrFunction, classTypeArgsIncludingOuterClasses: List<IrTypeArgument>?) : String {
-        return getFunctionLabel(f, null, classTypeArgsIncludingOuterClasses)
+    fun getFunctionLabel(f: IrFunction, classTypeArgsIncludingOuterClasses: List<IrTypeArgument>?): String? {
+        val parentId = useDeclarationParentOf(f, false, classTypeArgsIncludingOuterClasses, true)
+        if (parentId == null) {
+            logger.error("Couldn't get parent ID for function label")
+            return null
+        }
+        return getFunctionLabel(f, parentId, classTypeArgsIncludingOuterClasses)
     }
 
     /*
@@ -1052,10 +1129,10 @@ open class KotlinUsesExtractor(
      * that omit one or more parameters that has a default value specified.
      */
     @OptIn(ObsoleteDescriptorBasedAPI::class)
-    fun getFunctionLabel(f: IrFunction, maybeParentId: Label<out DbElement>?, classTypeArgsIncludingOuterClasses: List<IrTypeArgument>?, maybeParameterList: List<IrValueParameter>? = null) =
+    fun getFunctionLabel(f: IrFunction, parentId: Label<out DbElement>, classTypeArgsIncludingOuterClasses: List<IrTypeArgument>?, maybeParameterList: List<IrValueParameter>? = null): String =
         getFunctionLabel(
             f.parent,
-            maybeParentId,
+            parentId,
             getFunctionShortName(f).nameInDB,
             (maybeParameterList ?: f.valueParameters).map { it.type },
             getAdjustedReturnType(f),
@@ -1078,7 +1155,7 @@ open class KotlinUsesExtractor(
         // The parent of the function; normally f.parent.
         parent: IrDeclarationParent,
         // The ID of the function's parent, or null if we should work it out ourselves.
-        maybeParentId: Label<out DbElement>?,
+        parentId: Label<out DbElement>,
         // The name of the function; normally f.name.asString().
         name: String,
         // The types of the value parameters that the functions takes; normally f.valueParameters.map { it.type }.
@@ -1102,7 +1179,6 @@ open class KotlinUsesExtractor(
         // The prefix used in the label. "callable", unless a property label is created, then it's "property".
         prefix: String = "callable"
     ): String {
-        val parentId = maybeParentId ?: useDeclarationParent(parent, false, classTypeArgsIncludingOuterClasses, true)
         val allParamTypes = if (extensionParamType == null) parameterTypes else listOf(extensionParamType) + parameterTypes
 
         val substitutionMap = classTypeArgsIncludingOuterClasses?.let { notNullArgs ->
@@ -1322,16 +1398,30 @@ open class KotlinUsesExtractor(
             else -> false
         }
 
-    fun <T: DbCallable> useFunction(f: IrFunction, classTypeArgsIncludingOuterClasses: List<IrTypeArgument>? = null, noReplace: Boolean = false): Label<out T> {
-        return useFunction(f, null, classTypeArgsIncludingOuterClasses, noReplace)
-    }
-
-    fun <T: DbCallable> useFunction(f: IrFunction, parentId: Label<out DbElement>?, classTypeArgsIncludingOuterClasses: List<IrTypeArgument>?, noReplace: Boolean = false): Label<out T> {
+    fun <T: DbCallable> useFunction(f: IrFunction, classTypeArgsIncludingOuterClasses: List<IrTypeArgument>? = null, noReplace: Boolean = false): Label<out T>? {
         if (f.isLocalFunction()) {
             val ids = getLocallyVisibleFunctionLabels(f)
             return ids.function.cast<T>()
         }
         val javaFun = kotlinFunctionToJavaEquivalent(f, noReplace)
+        val parentId = useDeclarationParentOf(javaFun, false, classTypeArgsIncludingOuterClasses, true)
+        if (parentId == null) {
+            logger.error("Couldn't find parent ID for function ${f.name.asString()}")
+            return null
+        }
+        return useFunction(f, javaFun, parentId, classTypeArgsIncludingOuterClasses)
+    }
+
+    fun <T: DbCallable> useFunction(f: IrFunction, parentId: Label<out DbElement>, classTypeArgsIncludingOuterClasses: List<IrTypeArgument>?, noReplace: Boolean = false): Label<out T> {
+        if (f.isLocalFunction()) {
+            val ids = getLocallyVisibleFunctionLabels(f)
+            return ids.function.cast<T>()
+        }
+        val javaFun = kotlinFunctionToJavaEquivalent(f, noReplace)
+        return useFunction(f, javaFun, parentId, classTypeArgsIncludingOuterClasses)
+    }
+
+    private fun <T: DbCallable> useFunction(f: IrFunction, javaFun: IrFunction, parentId: Label<out DbElement>, classTypeArgsIncludingOuterClasses: List<IrTypeArgument>?): Label<out T> {
         val label = getFunctionLabel(javaFun, parentId, classTypeArgsIncludingOuterClasses)
         val id: Label<T> = tw.getLabelFor(label) {
             extractPrivateSpecialisedDeclaration(f, classTypeArgsIncludingOuterClasses)
@@ -1621,7 +1711,7 @@ open class KotlinUsesExtractor(
         val overriddenParentAttributes = (declarationParent as? IrFunction)?.let {
             (this as? KotlinFileExtractor)?.declarationStack?.findOverriddenAttributes(it)
         }
-        val parentId = parent ?: overriddenParentAttributes?.id ?: useDeclarationParent(declarationParent, false)
+        val parentId = parent ?: overriddenParentAttributes?.id ?: useDeclarationParentOf(vp, false)
 
         val idxBase = overriddenParentAttributes?.valueParameters?.indexOf(vp) ?: vp.index
         val idxOffset = if (declarationParent is IrFunction && declarationParent.extensionReceiverParameter != null)
@@ -1649,7 +1739,7 @@ open class KotlinUsesExtractor(
             it.isConst || it.isLateinit
         } ?: false
 
-    fun getFieldParent(f: IrField) =
+    private fun getFieldParent(f: IrField) =
         f.parentClassOrNull?.let {
             if (it.isCompanion && isDirectlyExposableCompanionObjectField(f))
                 it.parent
@@ -1666,7 +1756,7 @@ open class KotlinUsesExtractor(
         }
 
     fun getFieldLabel(f: IrField): String {
-        val parentId = useDeclarationParent(getFieldParent(f), false)
+        val parentId = useDeclarationParentOf(f, false)
         // Distinguish backing fields of properties based on their extension receiver type;
         // otherwise two extension properties declared in the same enclosing context will get
         // clashing trap labels. These are always private, so we can just make up a label without
@@ -1679,7 +1769,7 @@ open class KotlinUsesExtractor(
         tw.getLabelFor<DbField>(getFieldLabel(f)).also { extractFieldLaterIfExternalFileMember(f) }
 
     fun getPropertyLabel(p: IrProperty): String? {
-        val parentId = useDeclarationParent(p.parent, false)
+        val parentId = useDeclarationParentOf(p, false)
         if (parentId == null) {
             return null
         } else {
@@ -1711,7 +1801,7 @@ open class KotlinUsesExtractor(
         }
 
     fun getEnumEntryLabel(ee: IrEnumEntry): String {
-        val parentId = useDeclarationParent(ee.parent, false)
+        val parentId = useDeclarationParentOf(ee, false)
         return "@\"field;{$parentId};${ee.name.asString()}\""
     }
 
@@ -1719,7 +1809,7 @@ open class KotlinUsesExtractor(
         tw.getLabelFor(getEnumEntryLabel(ee))
 
     fun getTypeAliasLabel(ta: IrTypeAlias): String {
-        val parentId = useDeclarationParent(ta.parent, true)
+        val parentId = useDeclarationParentOf(ta, true)
         return "@\"type_alias;{$parentId};${ta.name.asString()}\""
     }
 

java/kotlin-extractor/src/main/kotlin/TrapWriter.kt

@@ -48,6 +48,15 @@ class TrapLabelManager {
      * duplication.
      */
     val genericSpecialisationsExtracted = HashSet<String>()
+
+    /**
+     * Sometimes, when we extract a file class we don't have the IrFile
+     * for it, so we are not able to give it a location. This means that
+     * the location is written outside of the label creation.
+     * This allows us to keep track of whether we've written the location
+     * already in this TRAP file, to avoid duplication.
+     */
+    val fileClassLocationsExtracted = HashSet<IrFile>()
 }
 
 /**

java/kotlin-extractor/src/main/kotlin/utils/ClassNames.kt

@@ -1,6 +1,7 @@
 package com.github.codeql
 
 import com.github.codeql.utils.getJvmName
+import com.github.codeql.utils.versions.getFileClassFqName
 import com.intellij.openapi.vfs.StandardFileSystems
 import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
 import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
@@ -108,14 +109,29 @@ private fun getRawIrClassBinaryPath(irClass: IrClass) =
 fun getIrClassBinaryPath(irClass: IrClass): String {
   return getRawIrClassBinaryPath(irClass)
   // Otherwise, make up a fake location:
-    ?: "/!unknown-binary-location/${getIrElementBinaryName(irClass).replace(".", "/")}.class"
+    ?: getUnknownBinaryLocation(getIrElementBinaryName(irClass))
 }
 
-fun getContainingClassOrSelf(decl: IrDeclaration): IrClass? {
-    return when(decl) {
-        is IrClass -> decl
-        else -> decl.parentClassOrNull
+fun getIrDeclarationBinaryPath(d: IrDeclaration): String? {
+    if (d is IrClass) {
+        return getIrClassBinaryPath(d)
     }
+    val parentClass = d.parentClassOrNull
+    if (parentClass != null) {
+        return getIrClassBinaryPath(parentClass)
+    }
+    if (d.parent is IrExternalPackageFragment) {
+        // This is in a file class.
+        val fqName = getFileClassFqName(d)
+        if (fqName != null) {
+            return getUnknownBinaryLocation(fqName.asString())
+        }
+    }
+    return null
+}
+
+private fun getUnknownBinaryLocation(s: String): String {
+    return "/!unknown-binary-location/${s.replace(".", "/")}.class"
 }
 
 fun getJavaEquivalentClassId(c: IrClass) =

java/kotlin-extractor/src/main/kotlin/utils/ExternalDecls.kt

@@ -3,17 +3,54 @@ package com.github.codeql.utils
 import org.jetbrains.kotlin.ir.declarations.IrClass
 import org.jetbrains.kotlin.ir.declarations.IrDeclaration
 import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
+import org.jetbrains.kotlin.ir.declarations.IrExternalPackageFragment
 import org.jetbrains.kotlin.ir.util.isFileClass
 import org.jetbrains.kotlin.ir.util.parentClassOrNull
 
 fun isExternalDeclaration(d: IrDeclaration): Boolean {
-    return d.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB ||
-           d.origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB ||
-           d.origin.toString() == "FUNCTION_INTERFACE_CLASS" // Treat kotlin.coroutines.* like ordinary library classes
+    /*
+    With Kotlin 1 we get things like (from .dump()):
+        PROPERTY IR_EXTERNAL_JAVA_DECLARATION_STUB name:MIN_VALUE visibility:public modality:FINAL [const,val]
+          FIELD IR_EXTERNAL_JAVA_DECLARATION_STUB name:MIN_VALUE type:kotlin.Int visibility:public [final,static]
+            EXPRESSION_BODY
+              CONST Int type=kotlin.Int value=-2147483648
+    */
+    if (d.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB ||
+        d.origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB ||
+        d.origin.toString() == "FUNCTION_INTERFACE_CLASS") { // Treat kotlin.coroutines.* like ordinary library classes
+        return true
+    }
+    /*
+    With Kotlin 2, the property itself is not marked as an external stub, but it parent is:
+        CLASS IR_EXTERNAL_DECLARATION_STUB OBJECT name:Companion modality:OPEN visibility:public [companion] superTypes:[]
+          PROPERTY name:MIN_VALUE visibility:public modality:FINAL [const,val]
+            FIELD PROPERTY_BACKING_FIELD name:MIN_VALUE type:kotlin.Int visibility:public [final]
+              EXPRESSION_BODY
+                CONST Int type=kotlin.Int value=-2147483648
+    */
+    val p = d.parent
+    if (p is IrExternalPackageFragment) {
+        // This is an external declaration in a (multi)file class
+        return true
+    }
+    if (p is IrDeclaration) {
+        return isExternalDeclaration(p)
+    }
+    return false
 }
 
 /**
  * Returns true if `d` is not itself a class, but is a member of an external file class.
  */
-fun isExternalFileClassMember(d: IrDeclaration) = d !is IrClass && (d.parentClassOrNull?.let { it.isFileClass } ?: false)
+fun isExternalFileClassMember(d: IrDeclaration): Boolean {
+    if (d is IrClass) { return false }
+    val p = d.parent
+    when (p) {
+        is IrClass -> return p.isFileClass
+        is IrExternalPackageFragment ->
+            // This is an external declaration in a (multi)file class
+            return true
+    }
+    return false
+}
 

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_4_32/getFileClassFqName.kt

@@ -0,0 +1,8 @@
+package com.github.codeql.utils.versions
+
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.ir.declarations.IrDeclaration
+
+fun getFileClassFqName(@Suppress("UNUSED_PARAMETER") d: IrDeclaration): FqName? {
+    return null
+}

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_7_0/getFileClassFqName.kt

@@ -0,0 +1,29 @@
+package com.github.codeql.utils.versions
+
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.ir.declarations.IrDeclaration
+import org.jetbrains.kotlin.ir.declarations.IrMemberWithContainerSource
+import org.jetbrains.kotlin.load.kotlin.FacadeClassSource
+
+fun getFileClassFqName(d: IrDeclaration): FqName? {
+    // d is in a file class.
+    // Get the name in a similar way to the compiler's ExternalPackageParentPatcherLowering
+    // visitMemberAccess/generateOrGetFacadeClass.
+    if (d is IrMemberWithContainerSource) {
+        val containerSource = d.containerSource
+        if (containerSource is FacadeClassSource) {
+            val facadeClassName = containerSource.facadeClassName
+            if (facadeClassName != null) {
+                // TODO: This is really a multifile-class rather than a file-class,
+                // but for now we treat them the same.
+                return facadeClassName.fqNameForTopLevelClassMaybeWithDollars
+            } else {
+                return containerSource.className.fqNameForTopLevelClassMaybeWithDollars
+            }
+        } else {
+            return null
+        }
+    } else {
+        return null
+    }
+}

java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll

@@ -21,21 +21,97 @@ import AutomodelEndpointTypes as AutomodelEndpointTypes
 
 newtype JavaRelatedLocationType = CallContext()
 
+newtype TApplicationModeEndpoint =
+  TExplicitArgument(Call call, DataFlow::Node arg) {
+    exists(Argument argExpr |
+      arg.asExpr() = argExpr and call = argExpr.getCall() and not argExpr.isVararg()
+    )
+  } or
+  TInstanceArgument(Call call, DataFlow::Node arg) { arg = DataFlow::getInstanceArgument(call) } or
+  TImplicitVarargsArray(Call call, DataFlow::Node arg, int idx) {
+    exists(Argument argExpr |
+      arg.asExpr() = argExpr and
+      call.getArgument(idx) = argExpr and
+      argExpr.isVararg() and
+      not exists(int i | i < idx and call.getArgument(i).(Argument).isVararg())
+    )
+  }
+
+/**
+ * An endpoint is a node that is a candidate for modeling.
+ */
+abstract private class ApplicationModeEndpoint extends TApplicationModeEndpoint {
+  abstract predicate isArgOf(Call c, int idx);
+
+  Call getCall() { this.isArgOf(result, _) }
+
+  int getArgIndex() { this.isArgOf(_, result) }
+
+  abstract Top asTop();
+
+  abstract DataFlow::Node asNode();
+
+  abstract string toString();
+}
+
 /**
  * A class representing nodes that are arguments to calls.
  */
-private class ArgumentNode extends DataFlow::Node {
-  Call c;
+class ExplicitArgument extends ApplicationModeEndpoint, TExplicitArgument {
+  Call call;
+  DataFlow::Node arg;
 
-  ArgumentNode() {
-    exists(Argument arg | this.asExpr() = arg and not arg.isVararg() and c = arg.getCall())
-    or
-    this.(DataFlow::ImplicitVarargsArray).getCall() = c
-    or
-    this = DataFlow::getInstanceArgument(c)
+  ExplicitArgument() { this = TExplicitArgument(call, arg) }
+
+  override predicate isArgOf(Call c, int idx) { c = call and this.asTop() = c.getArgument(idx) }
+
+  override Top asTop() { result = arg.asExpr() }
+
+  override DataFlow::Node asNode() { result = arg }
+
+  override string toString() { result = arg.toString() }
+}
+
+class InstanceArgument extends ApplicationModeEndpoint, TInstanceArgument {
+  Call call;
+  DataFlow::Node arg;
+
+  InstanceArgument() { this = TInstanceArgument(call, arg) }
+
+  override predicate isArgOf(Call c, int idx) {
+    c = call and this.asTop() = c.getQualifier() and idx = -1
   }
 
-  Call getCall() { result = c }
+  override Top asTop() { if exists(arg.asExpr()) then result = arg.asExpr() else result = call }
+
+  override DataFlow::Node asNode() { result = arg }
+
+  override string toString() { result = arg.toString() }
+}
+
+/**
+ * An endpoint that represents an implicit varargs array.
+ * We choose to represent the varargs array as a single endpoint, rather than as multiple endpoints.
+ *
+ * This avoids the problem of having to deal with redundant endpoints downstream.
+ *
+ * In order to be able to distinguish between varargs endpoints and regular endpoints, we export the `isVarargsArray`
+ * meta data field in the extraction queries.
+ */
+class ImplicitVarargsArray extends ApplicationModeEndpoint, TImplicitVarargsArray {
+  Call call;
+  DataFlow::Node vararg;
+  int idx;
+
+  ImplicitVarargsArray() { this = TImplicitVarargsArray(call, vararg, idx) }
+
+  override predicate isArgOf(Call c, int i) { c = call and i = idx }
+
+  override Top asTop() { result = this.getCall() }
+
+  override DataFlow::Node asNode() { result = vararg }
+
+  override string toString() { result = vararg.toString() }
 }
 
 /**
@@ -47,7 +123,7 @@ private class ArgumentNode extends DataFlow::Node {
  */
 module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig {
   // for documentation of the implementations here, see the QLDoc in the CandidateSig signature module.
-  class Endpoint = ArgumentNode;
+  class Endpoint = ApplicationModeEndpoint;
 
   class EndpointType = AutomodelEndpointTypes::EndpointType;
 
@@ -61,18 +137,18 @@ module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig
   predicate isSanitizer(Endpoint e, EndpointType t) {
     exists(t) and
     (
-      e.getType() instanceof BoxedType
+      e.asNode().getType() instanceof BoxedType
       or
-      e.getType() instanceof PrimitiveType
+      e.asNode().getType() instanceof PrimitiveType
       or
-      e.getType() instanceof NumberType
+      e.asNode().getType() instanceof NumberType
     )
     or
     t instanceof AutomodelEndpointTypes::PathInjectionSinkType and
-    e instanceof PathSanitizer::PathInjectionSanitizer
+    e.asNode() instanceof PathSanitizer::PathInjectionSanitizer
   }
 
-  RelatedLocation asLocation(Endpoint e) { result = e.asExpr() }
+  RelatedLocation asLocation(Endpoint e) { result = e.asTop() }
 
   predicate isKnownKind = AutomodelJavaUtil::isKnownKind/2;
 
@@ -98,16 +174,7 @@ module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig
     ApplicationModeGetCallable::getCallable(e).hasQualifiedName(package, type, name) and
     signature = ExternalFlow::paramsString(ApplicationModeGetCallable::getCallable(e)) and
     ext = "" and
-    (
-      exists(Call c, int argIdx |
-        e.asExpr() = c.getArgument(argIdx) and
-        input = AutomodelJavaUtil::getArgumentForIndex(argIdx)
-      )
-      or
-      exists(Call c |
-        e.asExpr() = c.getQualifier() and input = AutomodelJavaUtil::getArgumentForIndex(-1)
-      )
-    )
+    input = AutomodelJavaUtil::getArgumentForIndex(e.getArgIndex())
   }
 
   /**
@@ -118,7 +185,7 @@ module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig
    */
   RelatedLocation getRelatedLocation(Endpoint e, RelatedLocationType type) {
     type = CallContext() and
-    result = any(Call c | e.asExpr() = [c.getAnArgument(), c.getQualifier()])
+    result = e.getCall()
   }
 }
 
@@ -132,12 +199,7 @@ private module ApplicationModeGetCallable implements AutomodelSharedGetCallable:
   /**
    * Returns the API callable being modeled.
    */
-  Callable getCallable(Endpoint e) {
-    exists(Call c |
-      e.asExpr() = [c.getAnArgument(), c.getQualifier()] and
-      result = c.getCallee()
-    )
-  }
+  Callable getCallable(Endpoint e) { result = e.getCall().getCallee() }
 }
 
 /**
@@ -145,7 +207,7 @@ private module ApplicationModeGetCallable implements AutomodelSharedGetCallable:
  * should be empty.
  */
 private predicate isCustomSink(Endpoint e, string kind) {
-  e instanceof QueryInjectionSink and kind = "sql"
+  e.asNode() instanceof QueryInjectionSink and kind = "sql"
 }
 
 module CharacteristicsImpl =
@@ -167,23 +229,21 @@ class ApplicationModeMetadataExtractor extends string {
 
   predicate hasMetadata(
     Endpoint e, string package, string type, string subtypes, string name, string signature,
-    string input
+    string input, string isVarargsArray
   ) {
-    exists(Call call, Callable callable, int argIdx |
-      call.getCallee() = callable and
-      (
-        e.asExpr() = call.getArgument(argIdx)
-        or
-        e.asExpr() = call.getQualifier() and argIdx = -1
-      ) and
-      input = AutomodelJavaUtil::getArgumentForIndex(argIdx) and
+    exists(Callable callable |
+      e.getCall().getCallee() = callable and
+      input = AutomodelJavaUtil::getArgumentForIndex(e.getArgIndex()) and
       package = callable.getDeclaringType().getPackage().getName() and
       // we're using the erased types because the MaD convention is to not specify type parameters.
       // Whether something is or isn't a sink doesn't usually depend on the type parameters.
       type = callable.getDeclaringType().getErasure().(RefType).nestedName() and
       subtypes = AutomodelJavaUtil::considerSubtypes(callable).toString() and
       name = callable.getName() and
-      signature = ExternalFlow::paramsString(callable)
+      signature = ExternalFlow::paramsString(callable) and
+      if e instanceof ImplicitVarargsArray
+      then isVarargsArray = "true"
+      else isVarargsArray = "false"
     )
   }
 }
@@ -253,28 +313,10 @@ private class IsMaDTaintStepCharacteristic extends CharacteristicsImpl::NotASink
   IsMaDTaintStepCharacteristic() { this = "taint step" }
 
   override predicate appliesToEndpoint(Endpoint e) {
-    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(e, _, _) or
-    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(e, _, _) or
-    FlowSummaryImpl::Private::Steps::summaryGetterStep(e, _, _, _) or
-    FlowSummaryImpl::Private::Steps::summarySetterStep(e, _, _, _)
-  }
-}
-
-/**
- * A negative characteristic that filters out qualifiers that are classes (i.e. static calls). These
- * are unlikely to have any non-trivial flow going into them.
- *
- * Technically, an accessed type _could_ come from outside of the source code, but there's not
- * much likelihood of that being user-controlled.
- */
-private class ClassQualifierCharacteristic extends CharacteristicsImpl::NotASinkCharacteristic {
-  ClassQualifierCharacteristic() { this = "class qualifier" }
-
-  override predicate appliesToEndpoint(Endpoint e) {
-    exists(Call c |
-      e.asExpr() = c.getQualifier() and
-      c.getQualifier() instanceof TypeAccess
-    )
+    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(e.asNode(), _, _) or
+    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(e.asNode(), _, _) or
+    FlowSummaryImpl::Private::Steps::summaryGetterStep(e.asNode(), _, _, _) or
+    FlowSummaryImpl::Private::Steps::summarySetterStep(e.asNode(), _, _, _)
   }
 }
 
@@ -351,7 +393,7 @@ private class OtherArgumentToModeledMethodCharacteristic extends Characteristics
 private class FunctionValueCharacteristic extends CharacteristicsImpl::LikelyNotASinkCharacteristic {
   FunctionValueCharacteristic() { this = "function value" }
 
-  override predicate appliesToEndpoint(Endpoint e) { e.asExpr() instanceof FunctionalExpr }
+  override predicate appliesToEndpoint(Endpoint e) { e.asNode().asExpr() instanceof FunctionalExpr }
 }
 
 /**
@@ -371,12 +413,12 @@ private class CannotBeTaintedCharacteristic extends CharacteristicsImpl::LikelyN
    * Holds if the node `n` is known as the predecessor in a modeled flow step.
    */
   private predicate isKnownOutNodeForStep(Endpoint e) {
-    e.asExpr() instanceof Call or // we just assume flow in that case
-    TaintTracking::localTaintStep(_, e) or
-    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(_, e, _) or
-    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(_, e, _) or
-    FlowSummaryImpl::Private::Steps::summaryGetterStep(_, _, e, _) or
-    FlowSummaryImpl::Private::Steps::summarySetterStep(_, _, e, _)
+    e.asNode().asExpr() instanceof Call or // we just assume flow in that case
+    TaintTracking::localTaintStep(_, e.asNode()) or
+    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(_, e.asNode(), _) or
+    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(_, e.asNode(), _) or
+    FlowSummaryImpl::Private::Steps::summaryGetterStep(_, _, e.asNode(), _) or
+    FlowSummaryImpl::Private::Steps::summarySetterStep(_, _, e.asNode(), _)
   }
 }
 

java/ql/automodel/src/AutomodelApplicationModeExtractCandidates.ql

@@ -25,16 +25,18 @@ private import AutomodelJavaUtil
 bindingset[limit]
 private Endpoint getSampleForSignature(
   int limit, string package, string type, string subtypes, string name, string signature,
-  string input
+  string input, string isVarargs
 ) {
   exists(int n, int num_endpoints, ApplicationModeMetadataExtractor meta |
     num_endpoints =
-      count(Endpoint e | meta.hasMetadata(e, package, type, subtypes, name, signature, input))
+      count(Endpoint e |
+        meta.hasMetadata(e, package, type, subtypes, name, signature, input, isVarargs)
+      )
   |
     result =
       rank[n](Endpoint e, Location loc |
-        loc = e.getLocation() and
-        meta.hasMetadata(e, package, type, subtypes, name, signature, input)
+        loc = e.asTop().getLocation() and
+        meta.hasMetadata(e, package, type, subtypes, name, signature, input, isVarargs)
       |
         e
         order by
@@ -53,19 +55,20 @@ private Endpoint getSampleForSignature(
 from
   Endpoint endpoint, string message, ApplicationModeMetadataExtractor meta, DollarAtString package,
   DollarAtString type, DollarAtString subtypes, DollarAtString name, DollarAtString signature,
-  DollarAtString input
+  DollarAtString input, DollarAtString isVarargsArray
 where
   not exists(CharacteristicsImpl::UninterestingToModelCharacteristic u |
     u.appliesToEndpoint(endpoint)
   ) and
-  endpoint = getSampleForSignature(9, package, type, subtypes, name, signature, input) and
+  endpoint =
+    getSampleForSignature(9, package, type, subtypes, name, signature, input, isVarargsArray) and
   // If a node is already a known sink for any of our existing ATM queries and is already modeled as a MaD sink, we
   // don't include it as a candidate. Otherwise, we might include it as a candidate for query A, but the model will
   // label it as a sink for one of the sink types of query B, for which it's already a known sink. This would result in
   // overlap between our detected sinks and the pre-existing modeling. We assume that, if a sink has already been
   // modeled in a MaD model, then it doesn't belong to any additional sink types, and we don't need to reexamine it.
   not CharacteristicsImpl::isSink(endpoint, _, _) and
-  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input) and
+  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input, isVarargsArray) and
   includeAutomodelCandidate(package, type, name, signature) and
   // The message is the concatenation of all sink types for which this endpoint is known neither to be a sink nor to be
   // a non-sink, and we surface only endpoints that have at least one such sink type.
@@ -76,11 +79,13 @@ where
     |
       sinkType, ", "
     )
-select endpoint, message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
+select endpoint.asNode(),
+  message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@.", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()), "CallContext", //
   package, "package", //
   type, "type", //
   subtypes, "subtypes", //
   name, "name", // method name
   signature, "signature", //
-  input, "input" //
+  input, "input", //
+  isVarargsArray, "isVarargsArray"

java/ql/automodel/src/AutomodelApplicationModeExtractNegativeExamples.ql

@@ -24,7 +24,7 @@ Endpoint getSampleForCharacteristic(EndpointCharacteristic c, int limit) {
   exists(int n, int num_endpoints | num_endpoints = count(Endpoint e | c.appliesToEndpoint(e)) |
     result =
       rank[n](Endpoint e, Location loc |
-        loc = e.getLocation() and c.appliesToEndpoint(e)
+        loc = e.asTop().getLocation() and c.appliesToEndpoint(e)
       |
         e
         order by
@@ -43,7 +43,8 @@ Endpoint getSampleForCharacteristic(EndpointCharacteristic c, int limit) {
 from
   Endpoint endpoint, EndpointCharacteristic characteristic, float confidence, string message,
   ApplicationModeMetadataExtractor meta, DollarAtString package, DollarAtString type,
-  DollarAtString subtypes, DollarAtString name, DollarAtString signature, DollarAtString input
+  DollarAtString subtypes, DollarAtString name, DollarAtString signature, DollarAtString input,
+  DollarAtString isVarargsArray
 where
   endpoint = getSampleForCharacteristic(characteristic, 100) and
   confidence >= SharedCharacteristics::highConfidence() and
@@ -51,7 +52,7 @@ where
   // Exclude endpoints that have contradictory endpoint characteristics, because we only want examples we're highly
   // certain about in the prompt.
   not erroneousEndpoints(endpoint, _, _, _, _, false) and
-  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input) and
+  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input, isVarargsArray) and
   // It's valid for a node to satisfy the logic for both `isSink` and `isSanitizer`, but in that case it will be
   // treated by the actual query as a sanitizer, since the final logic is something like
   // `isSink(n) and not isSanitizer(n)`. We don't want to include such nodes as negative examples in the prompt, because
@@ -63,11 +64,13 @@ where
     characteristic2.hasImplications(positiveType, true, confidence2)
   ) and
   message = characteristic
-select endpoint, message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
+select endpoint.asNode(),
+  message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@.", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()), "CallContext", //
   package, "package", //
   type, "type", //
   subtypes, "subtypes", //
   name, "name", //
   signature, "signature", //
-  input, "input" //
+  input, "input", //
+  isVarargsArray, "isVarargsArray" //

java/ql/automodel/src/AutomodelApplicationModeExtractPositiveExamples.ql

@@ -15,19 +15,22 @@ private import AutomodelJavaUtil
 from
   Endpoint endpoint, SinkType sinkType, ApplicationModeMetadataExtractor meta,
   DollarAtString package, DollarAtString type, DollarAtString subtypes, DollarAtString name,
-  DollarAtString signature, DollarAtString input
+  DollarAtString signature, DollarAtString input, DollarAtString isVarargsArray
 where
   // Exclude endpoints that have contradictory endpoint characteristics, because we only want examples we're highly
   // certain about in the prompt.
   not erroneousEndpoints(endpoint, _, _, _, _, false) and
-  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input) and
+  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input, isVarargsArray) and
   // Extract positive examples of sinks belonging to the existing ATM query configurations.
-  CharacteristicsImpl::isKnownSink(endpoint, sinkType)
-select endpoint, sinkType + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
+  CharacteristicsImpl::isKnownSink(endpoint, sinkType) and
+  exists(CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()))
+select endpoint.asNode(),
+  sinkType + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@.", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()), "CallContext", //
   package, "package", //
   type, "type", //
   subtypes, "subtypes", //
   name, "name", //
   signature, "signature", //
-  input, "input" //
+  input, "input", //
+  isVarargsArray, "isVarargsArray"

java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected

@@ -1,2 +1,3 @@
-| Test.java:16:3:16:11 | reference | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:16:3:16:24 | set(...) | CallContext | file://java.util.concurrent.atomic:1:1:1:1 | java.util.concurrent.atomic | package | file://AtomicReference:1:1:1:1 | AtomicReference | type | file://false:1:1:1:1 | false | subtypes | file://set:1:1:1:1 | set | name | file://(String):1:1:1:1 | (String) | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input |
-| Test.java:21:3:21:10 | supplier | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:21:3:21:16 | get(...) | CallContext | file://java.util.function:1:1:1:1 | java.util.function | package | file://Supplier:1:1:1:1 | Supplier | type | file://true:1:1:1:1 | true | subtypes | file://get:1:1:1:1 | get | name | file://():1:1:1:1 | () | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input |
+| Test.java:16:3:16:11 | reference | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@. | Test.java:16:3:16:24 | set(...) | CallContext | file://java.util.concurrent.atomic:1:1:1:1 | java.util.concurrent.atomic | package | file://AtomicReference:1:1:1:1 | AtomicReference | type | file://false:1:1:1:1 | false | subtypes | file://set:1:1:1:1 | set | name | file://(String):1:1:1:1 | (String) | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input | file://false:1:1:1:1 | false | isVarargsArray |
+| Test.java:21:3:21:10 | supplier | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@. | Test.java:21:3:21:16 | get(...) | CallContext | file://java.util.function:1:1:1:1 | java.util.function | package | file://Supplier:1:1:1:1 | Supplier | type | file://true:1:1:1:1 | true | subtypes | file://get:1:1:1:1 | get | name | file://():1:1:1:1 | () | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input | file://false:1:1:1:1 | false | isVarargsArray |
+| Test.java:53:4:53:4 | o | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@. | Test.java:51:3:56:3 | walk(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://walk:1:1:1:1 | walk | name | file://(Path,FileVisitOption[]):1:1:1:1 | (Path,FileVisitOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input | file://true:1:1:1:1 | true | isVarargsArray |

java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.expected

@@ -1,2 +1,2 @@
-| Test.java:40:14:40:21 | openPath | taint step\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:40:4:40:22 | get(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Paths:1:1:1:1 | Paths | type | file://false:1:1:1:1 | false | subtypes | file://get:1:1:1:1 | get | name | file://(String,String[]):1:1:1:1 | (String,String[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input |
-| Test.java:46:4:46:5 | f2 | known non-sink\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:45:10:47:3 | compareTo(...) | CallContext | file://java.io:1:1:1:1 | java.io | package | file://File:1:1:1:1 | File | type | file://true:1:1:1:1 | true | subtypes | file://compareTo:1:1:1:1 | compareTo | name | file://(File):1:1:1:1 | (File) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input |
+| Test.java:46:4:46:5 | f2 | known non-sink\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@. | Test.java:45:10:47:3 | compareTo(...) | CallContext | file://java.io:1:1:1:1 | java.io | package | file://File:1:1:1:1 | File | type | file://true:1:1:1:1 | true | subtypes | file://compareTo:1:1:1:1 | compareTo | name | file://(File):1:1:1:1 | (File) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://false:1:1:1:1 | false | isVarargsArray |
+| Test.java:52:4:52:4 | p | taint step\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@. | Test.java:51:3:56:3 | walk(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://walk:1:1:1:1 | walk | name | file://(Path,FileVisitOption[]):1:1:1:1 | (Path,FileVisitOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://false:1:1:1:1 | false | isVarargsArray |

java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.expected

@@ -1,3 +1,3 @@
-| Test.java:26:4:26:9 | source | path-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:25:3:29:3 | copy(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input |
-| Test.java:27:4:27:9 | target | path-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:25:3:29:3 | copy(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input |
-| Test.java:34:4:34:11 | openPath | path-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@. | Test.java:33:10:35:3 | newInputStream(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://newInputStream:1:1:1:1 | newInputStream | name | file://(Path,OpenOption[]):1:1:1:1 | (Path,OpenOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input |
+| Test.java:26:4:26:9 | source | path-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@. | Test.java:25:3:29:3 | copy(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://false:1:1:1:1 | false | isVarargsArray |
+| Test.java:27:4:27:9 | target | path-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@. | Test.java:25:3:29:3 | copy(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input | file://false:1:1:1:1 | false | isVarargsArray |
+| Test.java:34:4:34:11 | openPath | path-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@. | Test.java:33:10:35:3 | newInputStream(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://newInputStream:1:1:1:1 | newInputStream | name | file://(Path,OpenOption[]):1:1:1:1 | (Path,OpenOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://false:1:1:1:1 | false | isVarargsArray |

java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java

@@ -8,7 +8,7 @@ import java.nio.file.Paths;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 import java.io.File;
-
+import java.nio.file.FileVisitOption;
 
 class Test {
 	public static void main(String[] args) throws Exception {
@@ -46,5 +46,14 @@ class Test {
 			f2 // negative example (modeled as not a sink)
 		);
 	}
+		
+	public static void FilesWalkExample(Path p, FileVisitOption o) throws Exception {
+		Files.walk(
+			p, // negative example (modeled as a taint step)
+			o, // the implicit varargs array is a candidate
+			o // not a candidate (only the first arg corresponding to a varargs array
+			  // is extracted)
+		);
+	}
 }
 

java/ql/lib/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 0.7.2
+
+### New Features
+
+* A `Diagnostic.getCompilationInfo()` predicate has been added.
+
+### Minor Analysis Improvements
+
+* Fixed a typo in the `StdlibRandomSource` class in `RandomDataSource.qll`, which caused the class to improperly model calls to the `nextBytes` method. Queries relying on `StdlibRandomSource` may see an increase in results.
+* Improved the precision of virtual dispatch of `java.io.InputStream` methods. Now, calls to these methods will not dispatch to arbitrary implementations of `InputStream` if there is a high-confidence alternative (like a models-as-data summary).
+* Added more dataflow steps for `java.io.InputStream`s that wrap other `java.io.InputStream`s.
+* Added models for the Struts 2 framework.
+* Improved the modeling of Struts 2 sources of untrusted data by tainting the whole object graph of the objects unmarshaled from an HTTP request.
+
 ## 0.7.1
 
 ### New Features

java/ql/lib/change-notes/2023-06-16-initial-version.md

@@ -0,0 +1,4 @@
+---
+category: majorAnalysis
+---
+* Improved support for flow through captured variables that properly adheres to inter-procedural control flow.

java/ql/lib/change-notes/2023-07-12-add-models-for-struts2-framework.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Added models for the Struts 2 framework.
-

java/ql/lib/change-notes/2023-07-12-improve-sources-for-the-struts2-framework.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Improved the modeling of Struts 2 sources of untrusted data by tainting the whole object graph of the objects unmarshaled from an HTTP request.
-

java/ql/lib/change-notes/2023-07-14-getCompilationInfo.md

@@ -1,4 +0,0 @@
----
-category: feature
----
-* A `Diagnostic.getCompilationInfo()` predicate has been added.

java/ql/lib/change-notes/2023-07-19-inputstream-dispatch.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Improved the precision of virtual dispatch of `java.io.InputStream` methods. Now, calls to these methods will not dispatch to arbitrary implementations of `InputStream` if there is a high-confidence alternative (like a models-as-data summary).
-  

java/ql/lib/change-notes/2023-07-19-inputstream-wrapper-steps.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Added more dataflow steps for `java.io.InputStream`s that wrap other `java.io.InputStream`s.

java/ql/lib/change-notes/2023-08-04-mad-withoutelement.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Add support for `WithElement` and `WithoutElement` for MaD access paths.

java/ql/lib/change-notes/2023-08-07-randomdatasource-typo-fix.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Fixed a typo in the `StdlibRandomSource` class in `RandomDataSource.qll`, which caused the class to improperly model calls to the `nextBytes` method. Queries relying on `StdlibRandomSource` may see an increase in results.

java/ql/lib/change-notes/released/0.7.2.md

@@ -0,0 +1,13 @@
+## 0.7.2
+
+### New Features
+
+* A `Diagnostic.getCompilationInfo()` predicate has been added.
+
+### Minor Analysis Improvements
+
+* Fixed a typo in the `StdlibRandomSource` class in `RandomDataSource.qll`, which caused the class to improperly model calls to the `nextBytes` method. Queries relying on `StdlibRandomSource` may see an increase in results.
+* Improved the precision of virtual dispatch of `java.io.InputStream` methods. Now, calls to these methods will not dispatch to arbitrary implementations of `InputStream` if there is a high-confidence alternative (like a models-as-data summary).
+* Added more dataflow steps for `java.io.InputStream`s that wrap other `java.io.InputStream`s.
+* Added models for the Struts 2 framework.
+* Improved the modeling of Struts 2 sources of untrusted data by tainting the whole object graph of the objects unmarshaled from an HTTP request.

java/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.7.1
+lastReleaseVersion: 0.7.2

java/ql/lib/ext/java.lang.model.yml

@@ -176,7 +176,6 @@ extensions:
       - ["java.lang", "Object", "getClass", "()", "summary", "manual"]
       - ["java.lang", "Object", "hashCode", "()", "summary", "manual"]
       - ["java.lang", "Object", "toString", "()", "summary", "manual"]
-      - ["java.lang", "Runnable", "run", "()", "summary", "manual"]
       - ["java.lang", "Runtime", "getRuntime", "()", "summary", "manual"]
       - ["java.lang", "String", "compareTo", "(String)", "summary", "manual"]
       - ["java.lang", "String", "contains", "(CharSequence)", "summary", "manual"]

java/ql/lib/ext/java.util.model.yml

@@ -140,7 +140,8 @@ extensions:
       - ["java.util", "LinkedHashSet", False, "LinkedHashSet", "(Collection)", "", "Argument[0].Element", "Argument[this].Element", "value", "manual"]
       - ["java.util", "LinkedList", False, "LinkedList", "(Collection)", "", "Argument[0].Element", "Argument[this].Element", "value", "manual"]
       - ["java.util", "List", True, "add", "(int,Object)", "", "Argument[1]", "Argument[this].Element", "value", "manual"]
-      - ["java.util", "List", True, "addAll", "(int,Collection)", "", "Argument[1].Element", "Argument[this].Element", "value", "manual"]
+      - ["java.util", "List", True, "addAll", "(int,Collection)", "", "Argument[1].WithElement", "Argument[this]", "value", "manual"]
+      - ["java.util", "List", True, "clear", "()", "", "Argument[this].WithoutElement", "Argument[this]", "value", "manual"]
       - ["java.util", "List", False, "copyOf", "(Collection)", "", "Argument[0].Element", "ReturnValue.Element", "value", "manual"]
       - ["java.util", "List", True, "get", "(int)", "", "Argument[this].Element", "ReturnValue", "value", "manual"]
       - ["java.util", "List", True, "listIterator", "", "", "Argument[this].Element", "ReturnValue.Element", "value", "manual"]
@@ -313,6 +314,7 @@ extensions:
       - ["java.util", "Scanner", True, "useLocale", "", "", "Argument[this]", "ReturnValue", "value", "manual"]
       - ["java.util", "Scanner", True, "useRadix", "", "", "Argument[this]", "ReturnValue", "value", "manual"]
       - ["java.util", "Set", False, "copyOf", "(Collection)", "", "Argument[0].Element", "ReturnValue.Element", "value", "manual"]
+      - ["java.util", "Set", False, "clear", "()", "", "Argument[this].WithoutElement", "Argument[this]", "value", "manual"]
       - ["java.util", "Set", False, "of", "(Object)", "", "Argument[0]", "ReturnValue.Element", "value", "manual"]
       - ["java.util", "Set", False, "of", "(Object,Object)", "", "Argument[0..1]", "ReturnValue.Element", "value", "manual"]
       - ["java.util", "Set", False, "of", "(Object,Object,Object)", "", "Argument[0..2]", "ReturnValue.Element", "value", "manual"]
@@ -424,10 +426,8 @@ extensions:
       # When `WithoutElement` is implemented, these should be changed to summary models of the form `Argument[this].WithoutElement -> Argument[this]`.
       - ["java.util", "Collection", "removeIf", "(Predicate)", "summary", "manual"]
       - ["java.util", "Iterator", "remove", "()", "summary", "manual"]
-      - ["java.util", "List", "clear", "()", "summary", "manual"]
       - ["java.util", "List", "remove", "(Object)", "summary", "manual"]
       - ["java.util", "Map", "clear", "()", "summary", "manual"]
-      - ["java.util", "Set", "clear", "()", "summary", "manual"]
       - ["java.util", "Set", "remove", "(Object)", "summary", "manual"]
       - ["java.util", "Set", "removeAll", "(Collection)", "summary", "manual"]
 

java/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-all
-version: 0.7.2-dev
+version: 0.7.3-dev
 groups: java
 dbscheme: config/semmlecode.dbscheme
 extractor: java

java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll

@@ -35,8 +35,9 @@
  *    or method, or a parameter.
  * 7. The `input` column specifies how data enters the element selected by the
  *    first 6 columns, and the `output` column specifies how data leaves the
- *    element selected by the first 6 columns. An `input` can be either "",
- *    "Argument[n]", "Argument[n1..n2]", "ReturnValue":
+ *    element selected by the first 6 columns. An `input` can be a dot separated
+ *    path consisting of either "", "Argument[n]", "Argument[n1..n2]",
+ *    "ReturnValue", "Element", "WithoutElement", or "WithElement":
  *    - "": Selects a write to the selected element in case this is a field.
  *    - "Argument[n]": Selects an argument in a call to the selected element.
  *      The arguments are zero-indexed, and `this` specifies the qualifier.
@@ -44,9 +45,15 @@
  *      the given range. The range is inclusive at both ends.
  *    - "ReturnValue": Selects a value being returned by the selected element.
  *      This requires that the selected element is a method with a body.
+ *    - "Element": Selects the collection elements of the selected element.
+ *    - "WithoutElement": Selects the selected element but without
+ *       its collection elements.
+ *    - "WithElement": Selects the collection elements of the selected element, but
+ *       points to the selected element.
  *
- *    An `output` can be either "", "Argument[n]", "Argument[n1..n2]", "Parameter",
- *    "Parameter[n]", "Parameter[n1..n2]", or "ReturnValue":
+ *    An `output` can be can be a dot separated path consisting of either "",
+ *    "Argument[n]", "Argument[n1..n2]", "Parameter", "Parameter[n]",
+ *    "Parameter[n1..n2]", "ReturnValue", or "Element":
  *    - "": Selects a read of a selected field, or a selected parameter.
  *    - "Argument[n]": Selects the post-update value of an argument in a call to the
  *      selected element. That is, the value of the argument after the call returns.
@@ -61,6 +68,7 @@
  *    - "Parameter[n1..n2]": Similar to "Parameter[n]" but selects any parameter
  *      in the given range. The range is inclusive at both ends.
  *    - "ReturnValue": Selects the return value of a call to the selected element.
+ *    - "Element": Selects the collection elements of the selected element.
  * 8. The `kind` column is a tag that can be referenced from QL to determine to
  *    which classes the interpreted elements should be added. For example, for
  *    sources "remote" indicates a default remote flow source, and for summaries

java/ql/lib/semmle/code/java/dataflow/FlowSummary.qll

@@ -101,6 +101,7 @@ abstract class SyntheticCallable extends string {
  * A module for importing frameworks that define synthetic callables.
  */
 private module SyntheticCallables {
+  private import semmle.code.java.dispatch.WrappedInvocation
   private import semmle.code.java.frameworks.android.Intent
   private import semmle.code.java.frameworks.Stream
 }
@@ -170,6 +171,8 @@ class SummarizedCallableBase extends TSummarizedCallableBase {
   }
 }
 
+class Provenance = Impl::Public::Provenance;
+
 class SummarizedCallable = Impl::Public::SummarizedCallable;
 
 class NeutralCallable = Impl::Public::NeutralCallable;

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowNodes.qll

@@ -55,7 +55,8 @@ private module Cached {
       )
     } or
     TFlowSummaryNode(FlowSummaryImpl::Private::SummaryNode sn) or
-    TFieldValueNode(Field f)
+    TFieldValueNode(Field f) or
+    TCaptureNode(CaptureFlow::SynthesizedCaptureNode cn)
 
   cached
   newtype TContent =
@@ -64,6 +65,7 @@ private module Cached {
     TCollectionContent() or
     TMapKeyContent() or
     TMapValueContent() or
+    TCapturedVariableContent(CapturedVariable v) or
     TSyntheticFieldContent(SyntheticField s)
 
   cached
@@ -73,6 +75,7 @@ private module Cached {
     TCollectionContentApprox() or
     TMapKeyContentApprox() or
     TMapValueContentApprox() or
+    TCapturedVariableContentApprox(CapturedVariable v) or
     TSyntheticFieldApproxContent()
 }
 
@@ -127,6 +130,8 @@ module Public {
       or
       result = this.(ImplicitPostUpdateNode).getPreUpdateNode().getType()
       or
+      result = this.(CaptureNode).getTypeImpl()
+      or
       result = this.(FieldValueNode).getField().getType()
     }
 
@@ -372,6 +377,7 @@ module Private {
     result.asCallable() = n.(MallocNode).getClassInstanceExpr().getEnclosingCallable() or
     result = nodeGetEnclosingCallable(n.(ImplicitPostUpdateNode).getPreUpdateNode()) or
     result.asSummarizedCallable() = n.(FlowSummaryNode).getSummarizedCallable() or
+    result.asCallable() = n.(CaptureNode).getSynthesizedCaptureNode().getEnclosingCallable() or
     result.asFieldScope() = n.(FieldValueNode).getField()
   }
 
@@ -491,6 +497,28 @@ module Private {
       c.asSummarizedCallable() = this.getSummarizedCallable() and pos = this.getPosition()
     }
   }
+
+  /**
+   * A synthesized data flow node representing a closure object that tracks
+   * captured variables.
+   */
+  class CaptureNode extends Node, TCaptureNode {
+    private CaptureFlow::SynthesizedCaptureNode cn;
+
+    CaptureNode() { this = TCaptureNode(cn) }
+
+    CaptureFlow::SynthesizedCaptureNode getSynthesizedCaptureNode() { result = cn }
+
+    override Location getLocation() { result = cn.getLocation() }
+
+    override string toString() { result = cn.toString() }
+
+    Type getTypeImpl() {
+      exists(Variable v | cn.isVariableAccess(v) and result = v.getType())
+      or
+      cn.isInstanceAccess() and result = cn.getEnclosingCallable().getDeclaringType()
+    }
+  }
 }
 
 private import Private
@@ -520,3 +548,14 @@ private class SummaryPostUpdateNode extends FlowSummaryNode, PostUpdateNode {
 
   override Node getPreUpdateNode() { result = pre }
 }
+
+private class CapturePostUpdateNode extends PostUpdateNode, CaptureNode {
+  private CaptureNode pre;
+
+  CapturePostUpdateNode() {
+    CaptureFlow::capturePostUpdateNode(this.getSynthesizedCaptureNode(),
+      pre.getSynthesizedCaptureNode())
+  }
+
+  override Node getPreUpdateNode() { result = pre }
+}

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowPrivate.qll

@@ -10,6 +10,7 @@ private import semmle.code.java.dataflow.FlowSummary
 private import FlowSummaryImpl as FlowSummaryImpl
 private import DataFlowImplConsistency
 private import DataFlowNodes
+private import codeql.dataflow.VariableCapture as VariableCapture
 import DataFlowNodes::Private
 
 private newtype TReturnKind = TNormalReturnKind()
@@ -51,26 +52,131 @@ private predicate fieldStep(Node node1, Node node2) {
   )
 }
 
-/**
- * Holds if data can flow from `node1` to `node2` through variable capture.
- */
-private predicate variableCaptureStep(Node node1, ExprNode node2) {
-  exists(SsaImplicitInit closure, SsaVariable captured |
-    closure.captures(captured) and
-    node2.getExpr() = closure.getAFirstUse()
-  |
-    node1.asExpr() = captured.getAUse()
-    or
-    not exists(captured.getAUse()) and
-    exists(SsaVariable capturedDef | capturedDef = captured.getAnUltimateDefinition() |
-      capturedDef.(SsaImplicitInit).isParameterDefinition(node1.asParameter()) or
-      capturedDef.(SsaExplicitUpdate).getDefiningExpr().(VariableAssign).getSource() =
-        node1.asExpr() or
-      capturedDef.(SsaExplicitUpdate).getDefiningExpr().(AssignOp) = node1.asExpr()
-    )
+private predicate closureFlowStep(Expr e1, Expr e2) {
+  simpleAstFlowStep(e1, e2)
+  or
+  exists(SsaVariable v |
+    v.getAUse() = e2 and
+    v.getAnUltimateDefinition().(SsaExplicitUpdate).getDefiningExpr().(VariableAssign).getSource() =
+      e1
   )
 }
 
+private module CaptureInput implements VariableCapture::InputSig {
+  private import java as J
+
+  class Location = J::Location;
+
+  class BasicBlock instanceof J::BasicBlock {
+    string toString() { result = super.toString() }
+
+    Callable getEnclosingCallable() { result = super.getEnclosingCallable() }
+
+    Location getLocation() { result = super.getLocation() }
+  }
+
+  BasicBlock getImmediateBasicBlockDominator(BasicBlock bb) { bbIDominates(result, bb) }
+
+  BasicBlock getABasicBlockSuccessor(BasicBlock bb) {
+    result = bb.(J::BasicBlock).getABBSuccessor()
+  }
+
+  //TODO: support capture of `this` in lambdas
+  class CapturedVariable instanceof LocalScopeVariable {
+    CapturedVariable() {
+      2 <=
+        strictcount(J::Callable c |
+          c = this.getCallable() or c = this.getAnAccess().getEnclosingCallable()
+        )
+    }
+
+    string toString() { result = super.toString() }
+
+    Callable getCallable() { result = super.getCallable() }
+
+    Location getLocation() { result = super.getLocation() }
+  }
+
+  class CapturedParameter extends CapturedVariable instanceof Parameter { }
+
+  class Expr instanceof J::Expr {
+    string toString() { result = super.toString() }
+
+    Location getLocation() { result = super.getLocation() }
+
+    predicate hasCfgNode(BasicBlock bb, int i) { this = bb.(J::BasicBlock).getNode(i) }
+  }
+
+  class VariableWrite extends Expr instanceof VariableUpdate {
+    CapturedVariable v;
+
+    VariableWrite() { super.getDestVar() = v }
+
+    CapturedVariable getVariable() { result = v }
+
+    Expr getSource() {
+      result = this.(VariableAssign).getSource() or
+      result = this.(AssignOp)
+    }
+  }
+
+  class VariableRead extends Expr instanceof RValue {
+    CapturedVariable v;
+
+    VariableRead() { super.getVariable() = v }
+
+    CapturedVariable getVariable() { result = v }
+  }
+
+  class ClosureExpr extends Expr instanceof ClassInstanceExpr {
+    NestedClass nc;
+
+    ClosureExpr() {
+      nc.(AnonymousClass).getClassInstanceExpr() = this
+      or
+      nc instanceof LocalClass and
+      super.getConstructedType().getASourceSupertype*().getSourceDeclaration() = nc
+    }
+
+    predicate hasBody(Callable body) { nc.getACallable() = body }
+
+    predicate hasAliasedAccess(Expr f) { closureFlowStep+(this, f) and not closureFlowStep(f, _) }
+  }
+
+  class Callable extends J::Callable {
+    predicate isConstructor() { this instanceof Constructor }
+  }
+}
+
+class CapturedVariable = CaptureInput::CapturedVariable;
+
+class CapturedParameter = CaptureInput::CapturedParameter;
+
+module CaptureFlow = VariableCapture::Flow<CaptureInput>;
+
+private CaptureFlow::ClosureNode asClosureNode(Node n) {
+  result = n.(CaptureNode).getSynthesizedCaptureNode() or
+  result.(CaptureFlow::ExprNode).getExpr() = n.asExpr() or
+  result.(CaptureFlow::ExprPostUpdateNode).getExpr() =
+    n.(PostUpdateNode).getPreUpdateNode().asExpr() or
+  result.(CaptureFlow::ParameterNode).getParameter() = n.asParameter() or
+  result.(CaptureFlow::ThisParameterNode).getCallable() = n.(InstanceParameterNode).getCallable() or
+  exprNode(result.(CaptureFlow::MallocNode).getClosureExpr()).(PostUpdateNode).getPreUpdateNode() =
+    n
+}
+
+private predicate captureStoreStep(Node node1, CapturedVariableContent c, Node node2) {
+  CaptureFlow::storeStep(asClosureNode(node1), c.getVariable(), asClosureNode(node2))
+}
+
+private predicate captureReadStep(Node node1, CapturedVariableContent c, Node node2) {
+  CaptureFlow::readStep(asClosureNode(node1), c.getVariable(), asClosureNode(node2))
+}
+
+predicate captureValueStep(Node node1, Node node2) {
+  CaptureFlow::localFlowStep(asClosureNode(node1), asClosureNode(node2))
+}
+
 /**
  * Holds if data can flow from `node1` to `node2` through a field or
  * variable capture.
@@ -78,10 +184,6 @@ private predicate variableCaptureStep(Node node1, ExprNode node2) {
 predicate jumpStep(Node node1, Node node2) {
   fieldStep(node1, node2)
   or
-  variableCaptureStep(node1, node2)
-  or
-  variableCaptureStep(node1.(PostUpdateNode).getPreUpdateNode(), node2)
-  or
   any(AdditionalValueStep a).step(node1, node2) and
   node1.getEnclosingCallable() != node2.getEnclosingCallable()
   or
@@ -117,6 +219,8 @@ predicate storeStep(Node node1, ContentSet f, Node node2) {
   or
   FlowSummaryImpl::Private::Steps::summaryStoreStep(node1.(FlowSummaryNode).getSummaryNode(), f,
     node2.(FlowSummaryNode).getSummaryNode())
+  or
+  captureStoreStep(node1, f, node2)
 }
 
 /**
@@ -149,6 +253,8 @@ predicate readStep(Node node1, ContentSet f, Node node2) {
   or
   FlowSummaryImpl::Private::Steps::summaryReadStep(node1.(FlowSummaryNode).getSummaryNode(), f,
     node2.(FlowSummaryNode).getSummaryNode())
+  or
+  captureReadStep(node1, f, node2)
 }
 
 /**
@@ -231,19 +337,29 @@ private newtype TDataFlowCallable =
   TSummarizedCallable(SummarizedCallable c) or
   TFieldScope(Field f)
 
+/**
+ * A callable or scope enclosing some number of data flow nodes. This can either
+ * be a source callable, a synthesized callable for which we have a summary
+ * model, or a synthetic scope for a field value node.
+ */
 class DataFlowCallable extends TDataFlowCallable {
+  /** Gets the source callable corresponding to this callable, if any. */
   Callable asCallable() { this = TSrcCallable(result) }
 
+  /** Gets the summary model callable corresponding to this callable, if any. */
   SummarizedCallable asSummarizedCallable() { this = TSummarizedCallable(result) }
 
+  /** Gets the field corresponding to this callable, if it is a field value scope. */
   Field asFieldScope() { this = TFieldScope(result) }
 
+  /** Gets a textual representation of this callable. */
   string toString() {
     result = this.asCallable().toString() or
     result = "Synthetic: " + this.asSummarizedCallable().toString() or
     result = "Field scope: " + this.asFieldScope().toString()
   }
 
+  /** Gets the location of this callable. */
   Location getLocation() {
     result = this.asCallable().getLocation() or
     result = this.asSummarizedCallable().getLocation() or
@@ -406,6 +522,8 @@ predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preserves
  */
 predicate allowParameterReturnInSelf(ParameterNode p) {
   FlowSummaryImpl::Private::summaryAllowParameterReturnInSelf(p)
+  or
+  CaptureFlow::heuristicAllowInstanceParameterReturnInSelf(p.(InstanceParameterNode).getCallable())
 }
 
 /** An approximated `Content`. */
@@ -447,6 +565,10 @@ ContentApprox getContentApprox(Content c) {
   or
   c instanceof MapValueContent and result = TMapValueContentApprox()
   or
+  exists(CapturedVariable v |
+    c = TCapturedVariableContent(v) and result = TCapturedVariableContentApprox(v)
+  )
+  or
   c instanceof SyntheticFieldContent and result = TSyntheticFieldApproxContent()
 }
 

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowUtil.qll

@@ -135,6 +135,30 @@ private module Cached {
 
 import Cached
 
+private predicate capturedVariableRead(Node n) {
+  n.asExpr().(RValue).getVariable() instanceof CapturedVariable
+}
+
+/**
+ * Holds if there is a data flow step from `e1` to `e2` that only steps from
+ * child to parent in the AST.
+ */
+predicate simpleAstFlowStep(Expr e1, Expr e2) {
+  e2.(CastingExpr).getExpr() = e1
+  or
+  e2.(ChooseExpr).getAResultExpr() = e1
+  or
+  e2.(AssignExpr).getSource() = e1
+  or
+  e2.(ArrayCreationExpr).getInit() = e1
+  or
+  e2 = any(StmtExpr stmtExpr | e1 = stmtExpr.getResultExpr())
+  or
+  e2 = any(NotNullExpr nne | e1 = nne.getExpr())
+  or
+  e2.(WhenExpr).getBranch(_).getAResult() = e1
+}
+
 private predicate simpleLocalFlowStep0(Node node1, Node node2) {
   TaintTrackingUtil::forceCachingInSameStage() and
   // Variable flow steps through adjacent def-use and use-use pairs.
@@ -142,39 +166,31 @@ private predicate simpleLocalFlowStep0(Node node1, Node node2) {
     upd.getDefiningExpr().(VariableAssign).getSource() = node1.asExpr() or
     upd.getDefiningExpr().(AssignOp) = node1.asExpr()
   |
-    node2.asExpr() = upd.getAFirstUse()
+    node2.asExpr() = upd.getAFirstUse() and
+    not capturedVariableRead(node2)
   )
   or
   exists(SsaImplicitInit init |
     init.isParameterDefinition(node1.asParameter()) and
-    node2.asExpr() = init.getAFirstUse()
+    node2.asExpr() = init.getAFirstUse() and
+    not capturedVariableRead(node2)
   )
   or
   adjacentUseUse(node1.asExpr(), node2.asExpr()) and
   not exists(FieldRead fr |
     hasNonlocalValue(fr) and fr.getField().isStatic() and fr = node1.asExpr()
   ) and
-  not FlowSummaryImpl::Private::Steps::prohibitsUseUseFlow(node1, _)
+  not FlowSummaryImpl::Private::Steps::prohibitsUseUseFlow(node1, _) and
+  not capturedVariableRead(node2)
   or
   ThisFlow::adjacentThisRefs(node1, node2)
   or
-  adjacentUseUse(node1.(PostUpdateNode).getPreUpdateNode().asExpr(), node2.asExpr())
+  adjacentUseUse(node1.(PostUpdateNode).getPreUpdateNode().asExpr(), node2.asExpr()) and
+  not capturedVariableRead(node2)
   or
   ThisFlow::adjacentThisRefs(node1.(PostUpdateNode).getPreUpdateNode(), node2)
   or
-  node2.asExpr().(CastingExpr).getExpr() = node1.asExpr()
-  or
-  node2.asExpr().(ChooseExpr).getAResultExpr() = node1.asExpr()
-  or
-  node2.asExpr().(AssignExpr).getSource() = node1.asExpr()
-  or
-  node2.asExpr().(ArrayCreationExpr).getInit() = node1.asExpr()
-  or
-  node2.asExpr() = any(StmtExpr stmtExpr | node1.asExpr() = stmtExpr.getResultExpr())
-  or
-  node2.asExpr() = any(NotNullExpr nne | node1.asExpr() = nne.getExpr())
-  or
-  node2.asExpr().(WhenExpr).getBranch(_).getAResult() = node1.asExpr()
+  simpleAstFlowStep(node1.asExpr(), node2.asExpr())
   or
   exists(MethodAccess ma, ValuePreservingMethod m, int argNo |
     ma.getCallee().getSourceDeclaration() = m and m.returnsValue(argNo)
@@ -185,6 +201,8 @@ private predicate simpleLocalFlowStep0(Node node1, Node node2) {
   or
   FlowSummaryImpl::Private::Steps::summaryLocalStep(node1.(FlowSummaryNode).getSummaryNode(),
     node2.(FlowSummaryNode).getSummaryNode(), true)
+  or
+  captureValueStep(node1, node2)
 }
 
 /**
@@ -256,6 +274,19 @@ class MapValueContent extends Content, TMapValueContent {
   override string toString() { result = "<map.value>" }
 }
 
+/** A captured variable. */
+class CapturedVariableContent extends Content, TCapturedVariableContent {
+  CapturedVariable v;
+
+  CapturedVariableContent() { this = TCapturedVariableContent(v) }
+
+  CapturedVariable getVariable() { result = v }
+
+  override DataFlowType getType() { result = getErasedRepr(v.(Variable).getType()) }
+
+  override string toString() { result = v.toString() }
+}
+
 /** A reference through a synthetic instance field. */
 class SyntheticFieldContent extends Content, TSyntheticFieldContent {
   SyntheticField s;

java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImplSpecific.qll

@@ -170,6 +170,10 @@ predicate neutralSummaryElement(SummarizedCallableBase c, string provenance) {
 bindingset[c]
 SummaryComponent interpretComponentSpecific(AccessPathToken c) {
   exists(Content content | parseContent(c, content) and result = SummaryComponent::content(content))
+  or
+  c = "WithoutElement" and result = SummaryComponent::withoutContent(any(CollectionContent cc))
+  or
+  c = "WithElement" and result = SummaryComponent::withContent(any(CollectionContent cc))
 }
 
 /** Gets the summary component for specification component `c`, if any. */
@@ -196,6 +200,10 @@ private string getContentSpecific(Content c) {
 /** Gets the textual representation of the content in the format used for MaD models. */
 string getMadRepresentationSpecific(SummaryComponent sc) {
   exists(Content c | sc = TContentSummaryComponent(c) and result = getContentSpecific(c))
+  or
+  sc = TWithoutContentSummaryComponent(_) and result = "WithoutElement"
+  or
+  sc = TWithContentSummaryComponent(_) and result = "WithElement"
 }
 
 bindingset[pos]

java/ql/lib/semmle/code/java/dispatch/WrappedInvocation.qll

@@ -58,3 +58,37 @@ Method getRunnerTarget(MethodAccess ma) {
     result.overridesOrInstantiates*(runmethod)
   )
 }
+
+import semmle.code.java.dataflow.FlowSummary
+import semmle.code.java.dataflow.internal.FlowSummaryImplSpecific as ImplSpecific
+
+private predicate mayInvokeCallback(SrcMethod m, int n) {
+  m.getParameterType(n).(RefType).getSourceDeclaration() instanceof FunctionalInterface and
+  (not m.fromSource() or m.isNative() or m.getFile().getAbsolutePath().matches("%/test/stubs/%"))
+}
+
+private class SummarizedCallableWithCallback extends SummarizedCallable {
+  private int pos;
+
+  SummarizedCallableWithCallback() { mayInvokeCallback(this.asCallable(), pos) }
+
+  override predicate propagatesFlow(
+    SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
+  ) {
+    input = SummaryComponentStack::argument(pos) and
+    output = SummaryComponentStack::push(SummaryComponent::parameter(-1), input) and
+    preservesValue = true
+  }
+
+  override predicate hasProvenance(Provenance provenance) { provenance = "hq-generated" }
+}
+
+private class RequiredComponentStackForCallback extends RequiredSummaryComponentStack {
+  override predicate required(SummaryComponent head, SummaryComponentStack tail) {
+    exists(int pos |
+      mayInvokeCallback(_, pos) and
+      head = SummaryComponent::parameter(-1) and
+      tail = SummaryComponentStack::argument(pos)
+    )
+  }
+}

java/ql/src/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.7.2
+
+### Minor Analysis Improvements
+
+* The sanitizer in `java/potentially-weak-cryptographic-algorithm` has been improved, so the query may yield additional results.
+
 ## 0.7.1
 
 ### Minor Analysis Improvements

java/ql/src/Metrics/Summaries/TopJdkApis.qll

@@ -307,6 +307,7 @@ class TopJdkApi extends SummarizedCallableBase {
   predicate hasManualMadModel() { this.hasManualSummary() or this.hasManualNeutral() }
   /*
    * Note: the following top JDK APIs are not modeled with MaD:
+   * `java.lang.Runnable#run()`: specialised lambda flow
    * `java.lang.String#valueOf(Object)`: a complex case; an alias for `Object.toString`, except the dispatch is hidden
    * `java.lang.System#getProperty(String)`: needs to be modeled by regular CodeQL matching the get and set keys to reduce FPs
    * `java.lang.System#setProperty(String,String)`: needs to be modeled by regular CodeQL matching the get and set keys to reduce FPs

java/ql/src/change-notes/released/0.7.2.md

@@ -1,4 +1,5 @@
----
-category: minorAnalysis
----
+## 0.7.2
+
+### Minor Analysis Improvements
+
 * The sanitizer in `java/potentially-weak-cryptographic-algorithm` has been improved, so the query may yield additional results.

java/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.7.1
+lastReleaseVersion: 0.7.2

java/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-queries
-version: 0.7.2-dev
+version: 0.7.3-dev
 groups: 
   - java
   - queries

java/ql/test/ext/TopJdkApis/TopJdkApisTest.expected

@@ -1,3 +1,4 @@
+| java.lang.Runnable#run() | no manual model |
 | java.lang.String#valueOf(Object) | no manual model |
 | java.lang.System#getProperty(String) | no manual model |
 | java.lang.System#setProperty(String,String) | no manual model |

java/ql/test/library-tests/dataflow/capture/B.java

@@ -0,0 +1,251 @@
+import java.util.*;
+import java.util.function.*;
+
+public class B {
+  static String source(String label) { return null; }
+
+  static void sink(String s) { }
+
+  static void test1() {
+    List<String> l1 = new ArrayList<>();
+    l1.add(source("L"));
+    List<String> l2 = new ArrayList<>();
+    l1.forEach(e -> l2.add(e));
+    sink(l2.get(0)); // $ hasValueFlow=L
+  }
+
+  String bf1;
+  String bf2;
+
+  void test2() {
+    B other = new B();
+    Consumer<String> f = x -> { this.bf1 = x; bf2 = x; other.bf1 = x; };
+
+    // no flow
+    sink(bf1);
+    sink(this.bf2);
+    sink(other.bf1);
+    sink(other.bf2);
+
+    f.accept(source("T"));
+
+    sink(bf1); // $ MISSING: hasValueFlow=T
+    sink(this.bf2); // $ MISSING: hasValueFlow=T
+    sink(other.bf1); // $ hasValueFlow=T
+    sink(other.bf2);
+  }
+
+  static void convert(Map<String, String> inp, Map<String, String> out) {
+    inp.forEach((key, value) -> { out.put(key, value); });
+  }
+
+  void test3() {
+    HashMap<String,String> m1 = new HashMap<>();
+    HashMap<String,String> m2 = new HashMap<>();
+    m1.put(source("Key"), source("Value"));
+    convert(m1, m2);
+    m2.forEach((k, v) -> {
+      sink(k); // $ hasValueFlow=Key
+      sink(v); // $ hasValueFlow=Value
+    });
+  }
+
+  String elem;
+
+  void testParamIn1() {
+    elem = source("pin.This.elem");
+    testParamIn2(source("pin.Arg"));
+  }
+
+  void testParamIn2(String param) {
+    Runnable r = () -> {
+      sink(elem); // $ MISSING: hasValueFlow=pin.This.elem
+      sink(this.elem); // $ MISSING: hasValueFlow=pin.This.elem
+      sink(param); // $ hasValueFlow=pin.Arg
+    };
+    r.run();
+  }
+
+  void testParamOut1() {
+    B other = new B();
+    testParamOut2(other);
+    sink(elem); // $ MISSING: hasValueFlow=pout.This.elem
+    sink(this.elem); // $ MISSING: hasValueFlow=pout.This.elem
+    sink(other.elem); // $ hasValueFlow=pout.param
+  }
+
+  void testParamOut2(B param) {
+    Runnable r = () -> {
+      this.elem = source("pout.This.elem");
+      param.elem = source("pout.param");
+    };
+    r.run();
+  }
+
+  void testCrossLambda() {
+    B b = new B();
+    Runnable sink1 = () -> { sink(b.elem); };
+    Runnable sink2 = () -> { sink(b.elem); }; // $ hasValueFlow=src
+    Runnable src = () -> { b.elem = source("src"); };
+    doRun(sink1);
+    doRun(src);
+    doRun(sink2);
+  }
+
+  void doRun(Runnable r) {
+    r.run();
+  }
+
+  void testNested() {
+    List<String> l1 = new ArrayList<>();
+    List<List<String>> l2 = new ArrayList<>();
+    l1.add(source("nest.out"));
+    l2.add(l1);
+    String s = source("nest.in");
+    List<String> out1 = new ArrayList<>();
+    List<String> out2 = new ArrayList<>();
+    l2.forEach(l -> l.forEach(x -> {
+      sink(s); // $ hasValueFlow=nest.in
+      out1.add(x);
+      out2.add(s);
+    }));
+    sink(out1.get(0)); // $ hasValueFlow=nest.out
+    sink(out2.get(0)); // $ hasValueFlow=nest.in
+  }
+
+  static interface TwoRuns {
+    void run1();
+    void run2();
+  }
+
+  void testAnonymousClass() {
+    List<String> l1 = new ArrayList<>();
+    List<String> l2 = new ArrayList<>();
+    TwoRuns r = new TwoRuns() {
+      @Override
+      public void run1() {
+        l1.add(source("run1"));
+      }
+      @Override
+      public void run2() {
+        l2.add(l1.get(0));
+      }
+    };
+    r.run2();
+    sink(l2.get(0));
+    r.run1();
+    r.run2();
+    sink(l2.get(0)); // $ hasValueFlow=run1
+  }
+
+  void testLocalClass1() {
+    String s = source("local1");
+    class MyLocal {
+      String f;
+      MyLocal() { this.f = s; }
+      String getF() { return this.f; }
+    }
+    MyLocal m = new MyLocal();
+    sink(m.getF()); // $ hasValueFlow=local1
+  }
+
+  void testLocalClass2() {
+    String s1 = source("s1");
+    String s2 = source("s2");
+    List<String> l = new ArrayList<>();
+    class MyLocal {
+      String f;
+      MyLocal() {
+        this.f = s1;
+        sink(s2); // $ hasValueFlow=s2
+      }
+      void test() {
+        sink(f); // $ hasValueFlow=s1
+        sink(s2); // $ hasValueFlow=s2
+      }
+      void add(String s) {
+        l.add(s);
+      }
+      String get() {
+        return l.get(0);
+      }
+    }
+    MyLocal m1 = new MyLocal();
+    MyLocal m2 = new MyLocal();
+    m1.test();
+    sink(m1.get());
+    m1.add(source("m1.add"));
+    sink(m2.get()); // $ hasValueFlow=m1.add
+  }
+
+  void testComplex() {
+    String s = source("complex");
+    class LocalComplex {
+      Supplier<StringBox> getBoxSupplier() {
+        return new Supplier<StringBox>() {
+          StringBox b = new StringBox();
+          @Override
+          public StringBox get() { return b; }
+        };
+      }
+      class StringBox {
+        String get() {
+          // capture through regular nested class inside local nested class
+          return s;
+        }
+      }
+    }
+    LocalComplex lc = new LocalComplex();
+    sink(lc.getBoxSupplier().get().get()); // $ MISSING: hasValueFlow=complex
+  }
+
+  void testCapturedLambda() {
+    String s = source("double.capture.in");
+    List<String> out = new ArrayList<>();
+    Runnable r1 = () -> {
+      sink(s); // $ hasValueFlow=double.capture.in
+      out.add(source("double.capture.out"));
+    };
+    Runnable r2 = () -> {
+      r1.run();
+    };
+    r2.run();
+    sink(out.get(0)); // $ MISSING: hasValueFlow=double.capture.out
+  }
+
+  void testEnhancedForStmtCapture() {
+    List<String> l = new ArrayList<>();
+    l.add(source("list"));
+    String[] a = new String[] { source("array") };
+    for (String x : l) {
+      Runnable r = () -> sink(x); // $ MISSING: hasValueFlow=list
+      r.run();
+    }
+    for (String x : a) {
+      Runnable r = () -> sink(x); // $ MISSING: hasValueFlow=array
+      r.run();
+    }
+  }
+
+  void testDoubleCall() {
+    String s = source("src");
+    List<String> l = new ArrayList<>();
+    List<String> l2 = new ArrayList<>();
+    class MyLocal2 {
+      MyLocal2() {
+        sink(l.get(0)); // no flow
+        sink(l2.get(0)); // no flow
+        l.add(s);
+      }
+      void run() {
+        l2.add(l.get(0));
+      }
+    }
+    // The ClassInstanceExpr has two calls in the same cfg node:
+    // First the constructor call for which it is the postupdate,
+    // and then as instance argument to the run call.
+    new MyLocal2().run();
+    sink(l.get(0)); // $ hasValueFlow=src
+    sink(l2.get(0)); // $ hasValueFlow=src
+  }
+}

java/ql/test/library-tests/dataflow/capture/inlinetest.expected

@@ -0,0 +1,2 @@
+failures
+testFailures

java/ql/test/library-tests/dataflow/capture/inlinetest.ql

@@ -0,0 +1,2 @@
+import TestUtilities.InlineFlowTest
+import DefaultFlowTest

java/ql/test/library-tests/dataflow/capture/test.expected

@@ -1,26 +1,93 @@
-| A.java:14:14:14:16 | "A" | A.java:14:14:14:16 | "A" |
-| A.java:14:14:14:16 | "A" | A.java:15:16:15:22 | get(...) |
-| A.java:14:14:14:16 | "A" | A.java:18:8:18:15 | p |
-| A.java:14:14:14:16 | "A" | A.java:32:26:32:26 | p |
-| A.java:21:11:21:13 | "B" | A.java:15:16:15:22 | get(...) |
-| A.java:21:11:21:13 | "B" | A.java:21:7:21:13 | ...=... |
-| A.java:21:11:21:13 | "B" | A.java:21:11:21:13 | "B" |
-| A.java:21:11:21:13 | "B" | A.java:33:26:33:26 | s |
-| A.java:23:11:23:13 | "C" | A.java:15:16:15:22 | get(...) |
-| A.java:23:11:23:13 | "C" | A.java:23:7:23:13 | ...=... |
-| A.java:23:11:23:13 | "C" | A.java:23:11:23:13 | "C" |
-| A.java:23:11:23:13 | "C" | A.java:33:26:33:26 | s |
-| A.java:25:22:25:24 | "D" | A.java:4:9:4:16 | e |
-| A.java:25:22:25:24 | "D" | A.java:4:21:4:28 | ...=... |
-| A.java:25:22:25:24 | "D" | A.java:4:28:4:28 | e |
-| A.java:25:22:25:24 | "D" | A.java:6:31:6:34 | elem |
-| A.java:25:22:25:24 | "D" | A.java:15:16:15:22 | get(...) |
-| A.java:25:22:25:24 | "D" | A.java:25:22:25:24 | "D" |
-| A.java:25:22:25:24 | "D" | A.java:34:26:34:37 | getElem(...) |
-| A.java:27:16:27:18 | "E" | A.java:5:18:5:25 | e |
-| A.java:27:16:27:18 | "E" | A.java:5:30:5:37 | ...=... |
-| A.java:27:16:27:18 | "E" | A.java:5:37:5:37 | e |
-| A.java:27:16:27:18 | "E" | A.java:6:31:6:34 | elem |
-| A.java:27:16:27:18 | "E" | A.java:15:16:15:22 | get(...) |
-| A.java:27:16:27:18 | "E" | A.java:27:16:27:18 | "E" |
-| A.java:27:16:27:18 | "E" | A.java:35:26:35:37 | getElem(...) |
+| A.java:14:14:14:16 | "A" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:15:16:15:16 | a : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:15:16:15:22 | get(...) : String |
+| A.java:14:14:14:16 | "A" : String | A.java:18:8:18:15 | p : String |
+| A.java:14:14:14:16 | "A" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:28:11:38:5 | p : String |
+| A.java:14:14:14:16 | "A" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:31:17:31:17 | this : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:32:26:32:26 | p : String |
+| A.java:14:14:14:16 | "A" : String | A.java:32:26:32:26 | this : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:33:26:33:26 | this : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:34:26:34:27 | this : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:35:26:35:27 | this : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:39:12:39:12 | a : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:39:12:39:12 | p : String |
+| A.java:21:11:21:13 | "B" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:15:16:15:16 | a : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:15:16:15:22 | get(...) : String |
+| A.java:21:11:21:13 | "B" : String | A.java:21:7:21:13 | ...=... : String |
+| A.java:21:11:21:13 | "B" : String | A.java:25:5:25:26 | phi(String s) : String |
+| A.java:21:11:21:13 | "B" : String | A.java:28:11:38:5 | String s : String |
+| A.java:21:11:21:13 | "B" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:31:17:31:17 | this : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:32:26:32:26 | this : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:33:26:33:26 | s : String |
+| A.java:21:11:21:13 | "B" : String | A.java:33:26:33:26 | this : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:34:26:34:27 | this : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:35:26:35:27 | this : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:39:12:39:12 | String s : String |
+| A.java:21:11:21:13 | "B" : String | A.java:39:12:39:12 | a : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:15:16:15:16 | a : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:15:16:15:22 | get(...) : String |
+| A.java:23:11:23:13 | "C" : String | A.java:23:7:23:13 | ...=... : String |
+| A.java:23:11:23:13 | "C" : String | A.java:25:5:25:26 | phi(String s) : String |
+| A.java:23:11:23:13 | "C" : String | A.java:28:11:38:5 | String s : String |
+| A.java:23:11:23:13 | "C" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:31:17:31:17 | this : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:32:26:32:26 | this : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:33:26:33:26 | s : String |
+| A.java:23:11:23:13 | "C" : String | A.java:33:26:33:26 | this : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:34:26:34:27 | this : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:35:26:35:27 | this : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:39:12:39:12 | String s : String |
+| A.java:23:11:23:13 | "C" : String | A.java:39:12:39:12 | a : new A(...) { ... } [String s] |
+| A.java:25:22:25:24 | "D" : String | A.java:4:9:4:16 | e : String |
+| A.java:25:22:25:24 | "D" : String | A.java:4:21:4:24 | this <.field> [post update] : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:4:21:4:28 | ...=... : String |
+| A.java:25:22:25:24 | "D" : String | A.java:4:28:4:28 | e : String |
+| A.java:25:22:25:24 | "D" : String | A.java:6:12:6:18 | parameter this : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:6:31:6:34 | elem : String |
+| A.java:25:22:25:24 | "D" : String | A.java:6:31:6:34 | this <.field> : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:15:16:15:16 | a : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:15:16:15:22 | get(...) : String |
+| A.java:25:22:25:24 | "D" : String | A.java:25:14:25:25 | new Box(...) : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:28:11:38:5 | Box b1 : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:31:17:31:17 | this : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:32:26:32:26 | this : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:33:26:33:26 | this : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:34:26:34:27 | b1 : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:34:26:34:27 | this : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:34:26:34:37 | getElem(...) : String |
+| A.java:25:22:25:24 | "D" : String | A.java:35:26:35:27 | this : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:39:12:39:12 | Box b1 : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:39:12:39:12 | a : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:5:18:5:25 | e : String |
+| A.java:27:16:27:18 | "E" : String | A.java:5:30:5:33 | this <.field> [post update] : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:5:30:5:37 | ...=... : String |
+| A.java:27:16:27:18 | "E" : String | A.java:5:37:5:37 | e : String |
+| A.java:27:16:27:18 | "E" : String | A.java:6:12:6:18 | parameter this : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:6:31:6:34 | elem : String |
+| A.java:27:16:27:18 | "E" : String | A.java:6:31:6:34 | this <.field> : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:15:16:15:16 | a : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:15:16:15:22 | get(...) : String |
+| A.java:27:16:27:18 | "E" : String | A.java:27:5:27:6 | b2 [post update] : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:28:11:38:5 | Box b2 : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:31:17:31:17 | this : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:32:26:32:26 | this : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:33:26:33:26 | this : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:34:26:34:27 | this : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:35:26:35:27 | b2 : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:35:26:35:27 | this : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:35:26:35:37 | getElem(...) : String |
+| A.java:27:16:27:18 | "E" : String | A.java:39:12:39:12 | Box b2 : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:39:12:39:12 | a : new A(...) { ... } [Box b2, ... (2)] |

java/ql/test/library-tests/dataflow/capture/test.ql

@@ -1,16 +1,23 @@
 import java
 import semmle.code.java.dataflow.DataFlow
 
-StringLiteral src() { result.getCompilationUnit().fromSource() }
+StringLiteral src() {
+  result.getCompilationUnit().fromSource() and
+  result.getFile().toString() = "A"
+}
 
 module Config implements DataFlow::ConfigSig {
   predicate isSource(DataFlow::Node n) { n.asExpr() = src() }
 
-  predicate isSink(DataFlow::Node n) { any() }
+  predicate isSink(DataFlow::Node n) { none() }
 }
 
 module Flow = DataFlow::Global<Config>;
 
-from DataFlow::Node src, DataFlow::Node sink
-where Flow::flow(src, sink)
+int explorationLimit() { result = 100 }
+
+module PartialFlow = Flow::FlowExploration<explorationLimit/0>;
+
+from PartialFlow::PartialPathNode src, PartialFlow::PartialPathNode sink
+where PartialFlow::partialFlow(src, sink, _)
 select src, sink

java/ql/test/library-tests/dataflow/stream-read/A.java

@@ -99,7 +99,7 @@ public class A {
     }
 
     public static void testWrapCall() {
-        sink(wrapStream(null)); // $ SPURIOUS: hasTaintFlow
+        sink(wrapStream(null)); // no flow
         sink(wrapStream(source())); // $ hasTaintFlow
     }
 

java/ql/test/library-tests/frameworks/ratpack/resources/IntegrationTest.java

@@ -107,13 +107,13 @@ class IntegrationTest {
                 filterAndMerge_2(pojoForm, mergedParams, name -> false);
                 return mergedParams;
             }).then(pojoMap -> {
-                sink(pojoMap.keySet().iterator().next()); //TODO:$hasTaintFlow
-                sink(pojoMap.get("value")); //TODO:$hasTaintFlow
+                sink(pojoMap.keySet().iterator().next()); //$hasTaintFlow
+                sink(pojoMap.get("value")); //$hasTaintFlow
                 pojoMap.forEach((key, value) -> {
-                    sink(key); //TODO:$hasTaintFlow
-                    sink(value); //TODO:$hasTaintFlow
+                    sink(key); //$hasTaintFlow
+                    sink(value); //$hasTaintFlow
                     List<Object> values = (List<Object>) value;
-                    sink(values.get(0)); //TODO:$hasTaintFlow
+                    sink(values.get(0)); //$hasTaintFlow
                 });
             });
     }

java/ql/test/query-tests/Metrics/GeneratedVsManualCoverage/TopJdkApisTest/TopJdkApisTest.expected

@@ -13,7 +13,7 @@
 | java.time | 0 | 0 | 0 | 17 | 17 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
 | java.time.chrono | 0 | 0 | 0 | 1 | 1 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
 | java.time.format | 0 | 0 | 0 | 2 | 2 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
-| java.util | 0 | 0 | 84 | 68 | 152 | 0.5526315789473685 | 0.0 | 0.5526315789473685 | 0.0 | NaN | 0.4473684210526316 |
+| java.util | 0 | 0 | 86 | 66 | 152 | 0.5657894736842105 | 0.0 | 0.5657894736842105 | 0.0 | NaN | 0.4342105263157895 |
 | java.util.concurrent | 0 | 0 | 9 | 9 | 18 | 0.5 | 0.0 | 0.5 | 0.0 | NaN | 0.5 |
 | java.util.concurrent.atomic | 0 | 0 | 2 | 11 | 13 | 0.15384615384615385 | 0.0 | 0.15384615384615385 | 0.0 | NaN | 0.8461538461538461 |
 | java.util.concurrent.locks | 0 | 0 | 0 | 2 | 2 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |