Extract external file declarations to individual trap files

java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java

@@ -3,23 +3,22 @@ package com.semmle.extractor.java;
 import java.lang.reflect.*;
 import java.io.File;
 import java.io.IOException;
-import java.net.URI;
 import java.util.Arrays;
 import java.util.LinkedHashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.regex.Pattern;
 
 import com.github.codeql.Logger;
-import com.github.codeql.Severity;
-import static com.github.codeql.ClassNamesKt.getIrClassBinaryName;
+import static com.github.codeql.ClassNamesKt.getIrDeclBinaryName;
 import static com.github.codeql.ClassNamesKt.getIrClassVirtualFile;
 
 import org.jetbrains.kotlin.ir.declarations.IrClass;
 
 import com.intellij.openapi.vfs.VirtualFile;
 
+import org.jetbrains.kotlin.ir.declarations.IrDeclaration;
+import org.jetbrains.kotlin.ir.declarations.IrDeclarationWithName;
 import org.jetbrains.org.objectweb.asm.ClassVisitor;
 import org.jetbrains.org.objectweb.asm.ClassReader;
 import org.jetbrains.org.objectweb.asm.Opcodes;
@@ -134,8 +133,8 @@ public class OdasaOutput {
 			currentSpecFileEntry.getTrapFolder(), PathTransformer.std().fileAsDatabaseString(file) + ".set");
 	}
 
-	public void addDependency(IrClass sym) {
-		String path = trapFilePathForClass(sym);
+	public void addDependency(IrDeclaration sym, String signature) {
+		String path = trapFilePathForDecl(sym, signature);
 		trapDependenciesForSource.addDependency(path);
 	}
 
@@ -200,31 +199,31 @@ public class OdasaOutput {
 				PathTransformer.std().fileAsDatabaseString(file) + ".trap.gz");
 	}
 
-	private File getTrapFileForClassFile(IrClass sym) {
+	private File getTrapFileForDecl(IrDeclaration sym, String signature) {
 		if (currentSpecFileEntry == null)
 			return null;
-		return trapFileForClass(sym);
+		return trapFileForDecl(sym, signature);
 	}
 
-	private File trapFileForClass(IrClass sym) {
+	private File trapFileForDecl(IrDeclaration sym, String signature) {
 		return FileUtil.fileRelativeTo(currentSpecFileEntry.getTrapFolder(),
-				trapFilePathForClass(sym));
+				trapFilePathForDecl(sym, signature));
 	}
 
 	private final Map<String, String> memberTrapPaths = new LinkedHashMap<String, String>();
 	private static final Pattern dots = Pattern.compile(".", Pattern.LITERAL);
-	private String trapFilePathForClass(IrClass sym) {
-		String classId = getIrClassBinaryName(sym);
+	private String trapFilePathForDecl(IrDeclaration sym, String signature) {
+		String binaryName = getIrDeclBinaryName(sym, signature);
 		// TODO: Reinstate this?
 		//if (getTrackClassOrigins())
 		//  classId += "-" + StringDigestor.digest(sym.getSourceFileId());
-		String result = memberTrapPaths.get(classId);
+		String result = memberTrapPaths.get(binaryName);
 		if (result == null) {
 			result = CLASSES_DIR + "/" +
-					dots.matcher(classId).replaceAll("/") +
+					dots.matcher(binaryName).replaceAll("/") +
 					".members" +
 					".trap.gz";
-			memberTrapPaths.put(classId, result);
+			memberTrapPaths.put(binaryName, result);
 		}
 		return result;
 	}
@@ -233,8 +232,8 @@ public class OdasaOutput {
 	 * Deletion of existing trap files.
 	 */
 
-	private void deleteTrapFileAndDependencies(IrClass sym) {
-		File trap = trapFileForClass(sym);
+	private void deleteTrapFileAndDependencies(IrDeclaration sym, String signature) {
+		File trap = trapFileForDecl(sym, signature);
 		if (trap.exists()) {
 			trap.delete();
 			File depFile = new File(trap.getParentFile(), trap.getName().replace(".trap.gz", ".dep"));
@@ -258,73 +257,83 @@ public class OdasaOutput {
 		File trapFile = getTrapFileForCurrentSourceFile();
 		if (trapFile==null)
 			return null;
-		return trapWriter(trapFile, null);
+		return trapWriter(trapFile, null, null);
 	}
 
 	/**
 	 * Get a {@link TrapFileManager} to write members
-	 * about a class, or <code>null</code> if the class shouldn't be populated.
+	 * about a declaration, or <code>null</code> if the declaration shouldn't be populated.
 	 *
 	 * @param sym
-	 * 		The class's symbol, including, in particular, its fully qualified
+	 * 		The declaration's symbol, including, in particular, its fully qualified
 	 * 		binary class name.
+	 * @param signature
+	 * 		Any unique suffix needed to distinguish `sym` from other declarations with the same name.
+	 * 		For functions for example, this means its parameter signature.
 	 */
-	private TrapFileManager getMembersWriterForClass(IrClass sym) {
-		File trap = getTrapFileForClassFile(sym);
+	private TrapFileManager getMembersWriterForDecl(IrDeclaration sym, String signature) {
+		File trap = getTrapFileForDecl(sym, signature);
 		if (trap==null)
 			return null;
 		TrapClassVersion currVersion = TrapClassVersion.fromSymbol(sym, log);
+		String shortName = sym instanceof IrDeclarationWithName ? ((IrDeclarationWithName)sym).getName().asString() : "(name unknown)";
 		if (trap.exists()) {
 			// Only re-write an existing trap file if we encountered a newer version of the same class.
 			TrapClassVersion trapVersion = readVersionInfo(trap);
 			if (!currVersion.isValid()) {
-				log.warn("Not rewriting trap file for: " + sym.getName() + " " + trapVersion + " " + currVersion + " " + trap);
+				log.warn("Not rewriting trap file for: " + shortName + " " + trapVersion + " " + currVersion + " " + trap);
 			} else if (currVersion.newerThan(trapVersion)) {
-				log.trace("Rewriting trap file for: " + sym.getName() + " " + trapVersion + " " + currVersion + " " + trap);
-				deleteTrapFileAndDependencies(sym);
+				log.trace("Rewriting trap file for: " + shortName + " " + trapVersion + " " + currVersion + " " + trap);
+				deleteTrapFileAndDependencies(sym, signature);
 			} else {
 				return null;
 			}
 		} else {
-			log.trace("Writing trap file for: " + sym.getName() + " " + currVersion + " " + trap);
+			log.trace("Writing trap file for: " + shortName + " " + currVersion + " " + trap);
 		}
-		return trapWriter(trap, sym);
+		return trapWriter(trap, sym, signature);
 	}
 
-	private TrapFileManager trapWriter(File trapFile, IrClass sym) {
+	private TrapFileManager trapWriter(File trapFile, IrDeclaration sym, String signature) {
 		if (!trapFile.getName().endsWith(".trap.gz"))
 			throw new CatastrophicError("OdasaOutput only supports writing to compressed trap files");
 		String relative = FileUtil.relativePath(trapFile, currentSpecFileEntry.getTrapFolder());
 		trapFile.getParentFile().mkdirs();
 		trapsCreated.addTrap(relative);
-		return concurrentWriter(trapFile, relative, log, sym);
+		return concurrentWriter(trapFile, relative, log, sym, signature);
 	}
 
-	private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, IrClass sym) {
+	private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, IrDeclaration sym, String signature) {
 		if (trapFile.exists())
 			return null;
-		return new TrapFileManager(trapFile, relative, true, log, sym);
+		return new TrapFileManager(trapFile, relative, true, log, sym, signature);
 	}
 
 	public class TrapFileManager implements AutoCloseable {
 
 		private TrapDependencies trapDependenciesForClass;
 		private File trapFile;
-		private IrClass sym;
+		private IrDeclaration sym;
+		private String signature;
 		private boolean hasError = false;
 
-		private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrClass sym) {
+		private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrDeclaration sym, String signature) {
 			trapDependenciesForClass = new TrapDependencies(relative);
 			this.trapFile = trapFile;
 			this.sym = sym;
+			this.signature = signature;
 		}
 
 		public File getFile() {
 			return trapFile;
 		}
 
-		public void addDependency(IrClass dep) {
-			trapDependenciesForClass.addDependency(trapFilePathForClass(dep));
+		public void addDependency(IrDeclaration dep, String signature) {
+			trapDependenciesForClass.addDependency(trapFilePathForDecl(dep, signature));
+		}
+
+		public void addDependency(IrClass c) {
+			addDependency(c, "");
 		}
 
 		public void close() {
@@ -374,7 +383,7 @@ public class OdasaOutput {
 	 * previously set by a call to {@link OdasaOutput#setCurrentSourceFile(File)}.
 	 */
 	public TrapLocker getTrapLockerForCurrentSourceFile() {
-		return new TrapLocker((IrClass)null);
+		return new TrapLocker((IrClass)null, null);
 	}
 
 	/**
@@ -412,37 +421,41 @@ public class OdasaOutput {
 	 *
 	 * @return  a {@link TrapLocker} for the trap file corresponding to the given class symbol.
 	 */
-	public TrapLocker getTrapLockerForClassFile(IrClass sym) {
-		return new TrapLocker(sym);
+	public TrapLocker getTrapLockerForDecl(IrDeclaration sym, String signature) {
+		return new TrapLocker(sym, signature);
 	}
 
 	public class TrapLocker implements AutoCloseable {
-		private final IrClass sym;
+		private final IrDeclaration sym;
 		private final File trapFile;
+		private final String signature;
 		private final boolean isNonSourceTrapFile;
-		private TrapLocker(IrClass sym) {
-			this.sym = sym;
+		private TrapLocker(IrDeclaration decl, String signature) {
+			this.sym = decl;
+			this.signature = signature;
 			if (sym==null) {
 				trapFile = getTrapFileForCurrentSourceFile();
 			} else {
-				trapFile = getTrapFileForClassFile(sym);
+				trapFile = getTrapFileForDecl(sym, signature);
 			}
 			isNonSourceTrapFile = false;
 		}
 		private TrapLocker(File jarFile) {
 			sym = null;
+			signature = null;
 			trapFile = getTrapFileForJarFile(jarFile);
 			isNonSourceTrapFile = true;
 		}
 		private TrapLocker(String moduleName) {
 			sym = null;
+			signature = null;
 			trapFile = getTrapFileForModule(moduleName);
 			isNonSourceTrapFile = true;
 		}
 		public TrapFileManager getTrapFileManager() {
 			if (trapFile!=null) {
 				lockTrapFile(trapFile);
-				return getMembersWriterForClass(sym);
+				return getMembersWriterForDecl(sym, signature);
 			} else {
 				return null;
 			}
@@ -532,8 +545,10 @@ public class OdasaOutput {
 					(tcv.majorVersion == majorVersion && tcv.minorVersion == minorVersion &&
 					tcv.lastModified < lastModified);
 		}
-		private static TrapClassVersion fromSymbol(IrClass sym, Logger log) {
-			VirtualFile vf = getIrClassVirtualFile(sym);
+		private static TrapClassVersion fromSymbol(IrDeclaration sym, Logger log) {
+			VirtualFile vf = sym instanceof IrClass ? getIrClassVirtualFile((IrClass)sym) :
+					sym.getParent() instanceof IrClass ? getIrClassVirtualFile((IrClass)sym.getParent()) :
+					null;
 			if(vf == null)
 				return new TrapClassVersion(0, 0, 0, null);
 

java/kotlin-extractor/src/main/kotlin/ExternalClassExtractor.kt

@@ -1,79 +0,0 @@
-package com.github.codeql
-
-import com.semmle.extractor.java.OdasaOutput
-import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
-import org.jetbrains.kotlin.ir.declarations.IrClass
-import org.jetbrains.kotlin.ir.util.packageFqName
-import java.io.File
-import java.util.ArrayList
-import java.util.HashSet
-import java.util.zip.GZIPOutputStream
-
-class ExternalClassExtractor(val logger: FileLogger, val invocationTrapFile: String, val sourceFilePath: String, val primitiveTypeMapping: PrimitiveTypeMapping, val pluginContext: IrPluginContext, val globalExtensionState: KotlinExtractorGlobalState) {
-
-    val externalClassesDone = HashSet<IrClass>()
-    val externalClassWorkList = ArrayList<IrClass>()
-
-    fun extractLater(c: IrClass): Boolean {
-        val ret = externalClassesDone.add(c)
-        if(ret) externalClassWorkList.add(c)
-        return ret
-    }
-
-    fun extractExternalClasses() {
-        val output = OdasaOutput(false, logger)
-        output.setCurrentSourceFile(File(sourceFilePath))
-        do {
-            val nextBatch = ArrayList(externalClassWorkList)
-            externalClassWorkList.clear()
-            nextBatch.forEach { irClass ->
-                output.getTrapLockerForClassFile(irClass).useAC { locker ->
-                    locker.trapFileManager.useAC { manager ->
-                        if(manager == null) {
-                            logger.info("Skipping extracting class ${irClass.name}")
-                        } else {
-                            val trapFile = manager.file
-                            val trapTmpFile = File.createTempFile("${trapFile.nameWithoutExtension}.", ".${trapFile.extension}.tmp", trapFile.parentFile)
-
-                            val binaryPath = getIrClassBinaryPath(irClass)
-                            try {
-                                GZIPOutputStream(trapTmpFile.outputStream()).bufferedWriter().use { trapFileBW ->
-                                    // We want our comments to be the first thing in the file,
-                                    // so start off with a mere TrapWriter
-                                    val tw = TrapWriter(logger.loggerBase, TrapLabelManager(), trapFileBW)
-                                    tw.writeComment("Generated by the CodeQL Kotlin extractor for external dependencies")
-                                    tw.writeComment("Part of invocation $invocationTrapFile")
-                                    // Now elevate to a SourceFileTrapWriter, and populate the
-                                    // file information
-                                    val ftw = tw.makeFileTrapWriter(binaryPath, true)
-
-                                    val fileExtractor = KotlinFileExtractor(logger, ftw, binaryPath, manager, this, primitiveTypeMapping, pluginContext, globalExtensionState)
-
-                                    // Populate a location and compilation-unit package for the file. This is similar to
-                                    // the beginning of `KotlinFileExtractor.extractFileContents` but without an `IrFile`
-                                    // to start from.
-                                    val pkg = irClass.packageFqName?.asString() ?: ""
-                                    val pkgId = fileExtractor.extractPackage(pkg)
-                                    ftw.writeHasLocation(ftw.fileId, ftw.getWholeFileLocation())
-                                    ftw.writeCupackage(ftw.fileId, pkgId)
-
-                                    fileExtractor.extractClassSource(irClass)
-                                }
-
-                                if (!trapTmpFile.renameTo(trapFile)) {
-                                    logger.error("Failed to rename $trapTmpFile to $trapFile")
-                                }
-
-                            } catch (e: Exception) {
-                                manager.setHasError()
-                                logger.error("Failed to extract '$binaryPath'. Partial TRAP file location is $trapTmpFile", e)
-                            }
-                        }
-                    }
-                }
-            }
-        } while (externalClassWorkList.isNotEmpty())
-        output.writeTrapSet()
-    }
-
-}

java/kotlin-extractor/src/main/kotlin/ExternalDeclExtractor.kt

@@ -0,0 +1,112 @@
+package com.github.codeql
+
+import com.github.codeql.utils.isExternalDeclaration
+import com.github.codeql.utils.isExternalFileClassMember
+import com.semmle.extractor.java.OdasaOutput
+import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.ir.util.isFileClass
+import org.jetbrains.kotlin.ir.util.packageFqName
+import org.jetbrains.kotlin.ir.util.parentAsClass
+import org.jetbrains.kotlin.ir.util.parentClassOrNull
+import java.io.File
+import java.util.ArrayList
+import java.util.HashSet
+import java.util.zip.GZIPOutputStream
+
+class ExternalDeclExtractor(val logger: FileLogger, val invocationTrapFile: String, val sourceFilePath: String, val primitiveTypeMapping: PrimitiveTypeMapping, val pluginContext: IrPluginContext, val globalExtensionState: KotlinExtractorGlobalState) {
+
+    val externalDeclsDone = HashSet<IrDeclaration>()
+    val externalDeclWorkList = ArrayList<Pair<IrDeclaration, String>>()
+
+    fun extractLater(d: IrDeclaration, signature: String): Boolean {
+        if (d !is IrClass && !isExternalFileClassMember(d)) {
+            logger.warnElement("External declaration is neither a class, nor a top-level declaration", d)
+            return false
+        }
+        val ret = externalDeclsDone.add(d)
+        if (ret) externalDeclWorkList.add(Pair(d, signature))
+        return ret
+    }
+
+    val propertySignature = ";property"
+    val fieldSignature = ";field"
+
+    fun extractLater(p: IrProperty) = extractLater(p, propertySignature)
+    fun extractLater(f: IrField) = extractLater(f, fieldSignature)
+
+    fun extractLater(c: IrClass) = extractLater(c, "")
+
+    fun extractExternalClasses() {
+        val output = OdasaOutput(false, logger)
+        output.setCurrentSourceFile(File(sourceFilePath))
+        do {
+            val nextBatch = ArrayList(externalDeclWorkList)
+            externalDeclWorkList.clear()
+            nextBatch.forEach { workPair ->
+                val (irDecl, signature) = workPair
+                output.getTrapLockerForDecl(irDecl, signature).useAC { locker ->
+                    locker.trapFileManager.useAC { manager ->
+                        val shortName = when(irDecl) {
+                            is IrDeclarationWithName -> irDecl.name.asString()
+                            else -> "(unknown name)"
+                        }
+                        if(manager == null) {
+                            logger.info("Skipping extracting external decl $shortName")
+                        } else {
+                            val trapFile = manager.file
+                            val trapTmpFile = File.createTempFile("${trapFile.nameWithoutExtension}.", ".${trapFile.extension}.tmp", trapFile.parentFile)
+
+                            val containingClass = when(irDecl) {
+                                is IrClass -> irDecl
+                                else -> irDecl.parentClassOrNull
+                            }
+                            if (containingClass == null) {
+                                logger.warnElement("Unable to get containing class", irDecl)
+                                return
+                            }
+                            val binaryPath = getIrClassBinaryPath(containingClass)
+                            try {
+                                GZIPOutputStream(trapTmpFile.outputStream()).bufferedWriter().use { trapFileBW ->
+                                    // We want our comments to be the first thing in the file,
+                                    // so start off with a mere TrapWriter
+                                    val tw = TrapWriter(logger.loggerBase, TrapLabelManager(), trapFileBW)
+                                    tw.writeComment("Generated by the CodeQL Kotlin extractor for external dependencies")
+                                    tw.writeComment("Part of invocation $invocationTrapFile")
+                                    // Now elevate to a SourceFileTrapWriter, and populate the
+                                    // file information if needed:
+                                    val ftw = tw.makeFileTrapWriter(binaryPath, irDecl is IrClass)
+
+                                    val fileExtractor = KotlinFileExtractor(logger, ftw, binaryPath, manager, this, primitiveTypeMapping, pluginContext, globalExtensionState)
+
+                                    if (irDecl is IrClass) {
+                                        // Populate a location and compilation-unit package for the file. This is similar to
+                                        // the beginning of `KotlinFileExtractor.extractFileContents` but without an `IrFile`
+                                        // to start from.
+                                        val pkg = irDecl.packageFqName?.asString() ?: ""
+                                        val pkgId = fileExtractor.extractPackage(pkg)
+                                        ftw.writeHasLocation(ftw.fileId, ftw.getWholeFileLocation())
+                                        ftw.writeCupackage(ftw.fileId, pkgId)
+
+                                        fileExtractor.extractClassSource(irDecl, !irDecl.isFileClass)
+                                    } else {
+                                        fileExtractor.extractDeclaration(irDecl)
+                                    }
+                                }
+
+                                if (!trapTmpFile.renameTo(trapFile)) {
+                                    logger.error("Failed to rename $trapTmpFile to $trapFile")
+                                }
+                            } catch (e: Exception) {
+                                manager.setHasError()
+                                logger.error("Failed to extract '$shortName'. Partial TRAP file location is $trapTmpFile", e)
+                            }
+                        }
+                    }
+                }
+            }
+        } while (externalDeclWorkList.isNotEmpty())
+        output.writeTrapSet()
+    }
+
+}

java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt

@@ -244,11 +244,11 @@ private fun doFile(
                 // Now elevate to a SourceFileTrapWriter, and populate the
                 // file information
                 val sftw = tw.makeSourceFileTrapWriter(srcFile, true)
-                val externalClassExtractor = ExternalClassExtractor(logger, invocationTrapFile, srcFilePath, primitiveTypeMapping, pluginContext, globalExtensionState)
-                val fileExtractor = KotlinFileExtractor(logger, sftw, srcFilePath, null, externalClassExtractor, primitiveTypeMapping, pluginContext, globalExtensionState)
+                val externalDeclExtractor = ExternalDeclExtractor(logger, invocationTrapFile, srcFilePath, primitiveTypeMapping, pluginContext, globalExtensionState)
+                val fileExtractor = KotlinFileExtractor(logger, sftw, srcFilePath, null, externalDeclExtractor, primitiveTypeMapping, pluginContext, globalExtensionState)
 
                 fileExtractor.extractFileContents(srcFile, sftw.fileId)
-                externalClassExtractor.extractExternalClasses()
+                externalDeclExtractor.extractExternalClasses()
             }
 
             if (checkTrapIdentical && trapFile.exists()) {

java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt

@@ -1,11 +1,8 @@
 package com.github.codeql
 
 import com.github.codeql.comments.CommentExtractor
-import com.github.codeql.utils.TypeSubstitution
+import com.github.codeql.utils.*
 import com.github.codeql.utils.versions.functionN
-import com.github.codeql.utils.substituteTypeAndArguments
-import com.github.codeql.utils.substituteTypeArguments
-import com.github.codeql.utils.toRawType
 import com.github.codeql.utils.versions.getIrStubFromDescriptor
 import com.semmle.extractor.java.OdasaOutput
 import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
@@ -35,7 +32,7 @@ open class KotlinFileExtractor(
     override val tw: FileTrapWriter,
     val filePath: String,
     dependencyCollector: OdasaOutput.TrapFileManager?,
-    externalClassExtractor: ExternalClassExtractor,
+    externalClassExtractor: ExternalDeclExtractor,
     primitiveTypeMapping: PrimitiveTypeMapping,
     pluginContext: IrPluginContext,
     globalExtensionState: KotlinExtractorGlobalState
@@ -87,7 +84,7 @@ open class KotlinFileExtractor(
                     if (isExternalDeclaration(declaration)) {
                         extractExternalClassLater(declaration)
                     } else {
-                        extractClassSource(declaration)
+                        extractClassSource(declaration, true)
                     }
                 }
                 is IrFunction -> {
@@ -311,7 +308,7 @@ open class KotlinFileExtractor(
 
     private fun extractLocalTypeDeclStmt(c: IrClass, callable: Label<out DbCallable>, parent: Label<out DbStmtparent>, idx: Int) {
         @Suppress("UNCHECKED_CAST")
-        val id = extractClassSource(c) as Label<out DbClass>
+        val id = extractClassSource(c, true) as Label<out DbClass>
         extractLocalTypeDeclStmt(id, c, callable, parent, idx)
     }
 
@@ -323,7 +320,7 @@ open class KotlinFileExtractor(
         tw.writeHasLocation(stmtId, locId)
     }
 
-    fun extractClassSource(c: IrClass): Label<out DbClassorinterface> {
+    fun extractClassSource(c: IrClass, extractDeclarations: Boolean): Label<out DbClassorinterface> {
         with("class source", c) {
             DeclarationStackAdjuster(c).use {
 
@@ -356,7 +353,8 @@ open class KotlinFileExtractor(
                 extractEnclosingClass(c, id, locId, listOf())
 
                 c.typeParameters.mapIndexed { idx, it -> extractTypeParameter(it, idx) }
-                c.declarations.map { extractDeclaration(it) }
+                if (extractDeclarations)
+                    c.declarations.map { extractDeclaration(it) }
                 if (c.isNonCompanionObject) {
                     // For `object MyObject { ... }`, the .class has an
                     // automatically-generated `public static final MyObject INSTANCE`

java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt

@@ -1,11 +1,8 @@
 package com.github.codeql
 
-import com.github.codeql.utils.substituteTypeAndArguments
-import com.github.codeql.utils.substituteTypeArguments
-import com.github.codeql.utils.withQuestionMark
+import com.github.codeql.utils.*
 import com.semmle.extractor.java.OdasaOutput
 import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
-import org.jetbrains.kotlin.backend.common.lower.parents
 import org.jetbrains.kotlin.backend.common.lower.parentsWithSelf
 import org.jetbrains.kotlin.backend.jvm.codegen.isRawType
 import org.jetbrains.kotlin.builtins.jvm.JavaToKotlinClassMap
@@ -29,7 +26,7 @@ open class KotlinUsesExtractor(
     open val logger: Logger,
     open val tw: TrapWriter,
     val dependencyCollector: OdasaOutput.TrapFileManager?,
-    val externalClassExtractor: ExternalClassExtractor,
+    val externalClassExtractor: ExternalDeclExtractor,
     val primitiveTypeMapping: PrimitiveTypeMapping,
     val pluginContext: IrPluginContext,
     val globalExtensionState: KotlinExtractorGlobalState
@@ -207,11 +204,6 @@ open class KotlinUsesExtractor(
         return UseClassInstanceResult(classTypeResult, extractClass)
     }
 
-    fun isExternalDeclaration(d: IrDeclaration): Boolean {
-        return d.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB ||
-               d.origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB
-    }
-
     fun isArray(t: IrSimpleType) = t.isBoxedArray || t.isPrimitiveArray()
 
     fun extractClassLaterIfExternal(c: IrClass) {
@@ -229,6 +221,38 @@ open class KotlinUsesExtractor(
         }
     }
 
+    fun extractPropertyLaterIfExternalFileMember(p: IrProperty) {
+        if (isExternalFileClassMember(p)) {
+            extractExternalClassLater(p.parentAsClass)
+            dependencyCollector?.addDependency(p, externalClassExtractor.propertySignature)
+            externalClassExtractor.extractLater(p)
+        }
+    }
+
+    fun extractFieldLaterIfExternalFileMember(f: IrField) {
+        if (isExternalFileClassMember(f)) {
+            extractExternalClassLater(f.parentAsClass)
+            dependencyCollector?.addDependency(f, externalClassExtractor.fieldSignature)
+            externalClassExtractor.extractLater(f)
+        }
+    }
+
+    fun extractFunctionLaterIfExternalFileMember(f: IrFunction) {
+        if (isExternalFileClassMember(f)) {
+            extractExternalClassLater(f.parentAsClass)
+            (f as? IrSimpleFunction)?.correspondingPropertySymbol?.let {
+                extractPropertyLaterIfExternalFileMember(it.owner)
+                // No need to extract the function specifically, as the property's
+                // getters and setters are extracted alongside it
+                return
+            }
+            val paramTypes = f.valueParameters.map { useType(it.type) }
+            val signature = paramTypes.joinToString(separator = ",", prefix = "(", postfix = ")") { it.javaResult.signature!! }
+            dependencyCollector?.addDependency(f, signature)
+            externalClassExtractor.extractLater(f, signature)
+        }
+    }
+
     fun extractExternalClassLater(c: IrClass) {
         dependencyCollector?.addDependency(c)
         externalClassExtractor.extractLater(c)
@@ -791,6 +815,7 @@ open class KotlinUsesExtractor(
     fun <T: DbCallable> useFunctionCommon(f: IrFunction, label: String): Label<out T> {
         val id: Label<T> = tw.getLabelFor(label)
         if (isExternalDeclaration(f)) {
+            extractFunctionLaterIfExternalFileMember(f)
             extractExternalEnclosingClassLater(f)
         }
         return id
@@ -1064,7 +1089,7 @@ open class KotlinUsesExtractor(
     }
 
     fun useField(f: IrField): Label<out DbField> =
-        tw.getLabelFor(getFieldLabel(f))
+        tw.getLabelFor<DbField>(getFieldLabel(f)).also { extractFieldLaterIfExternalFileMember(f) }
 
     fun getPropertyLabel(p: IrProperty) =
         getPropertyLabel(p, useDeclarationParent(p.parent, false))
@@ -1073,10 +1098,10 @@ open class KotlinUsesExtractor(
         "@\"property;{$parentId};${p.name.asString()}\""
 
     fun useProperty(p: IrProperty): Label<out DbKt_property> =
-        tw.getLabelFor(getPropertyLabel(p))
+        tw.getLabelFor<DbKt_property>(getPropertyLabel(p)).also { extractPropertyLaterIfExternalFileMember(p) }
 
     fun useProperty(p: IrProperty, parentId: Label<out DbElement>): Label<out DbKt_property> =
-        tw.getLabelFor(getPropertyLabel(p, parentId))
+        tw.getLabelFor<DbKt_property>(getPropertyLabel(p, parentId)).also { extractPropertyLaterIfExternalFileMember(p) }
 
     fun getEnumEntryLabel(ee: IrEnumEntry): String {
         val parentId = useDeclarationParent(ee.parent, false)

java/kotlin-extractor/src/main/kotlin/utils/ClassNames.kt

@@ -1,23 +1,27 @@
 package com.github.codeql
 
 import com.intellij.openapi.vfs.StandardFileSystems
-import org.jetbrains.kotlin.ir.declarations.IrClass
-import org.jetbrains.kotlin.ir.declarations.IrDeclaration
-import org.jetbrains.kotlin.ir.declarations.IrDeclarationParent
-import org.jetbrains.kotlin.ir.declarations.IrPackageFragment
 import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
 import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
 import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClass
 import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement
 
 import com.intellij.openapi.vfs.VirtualFile
+import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.load.kotlin.JvmPackagePartSource
 
-// Taken from Kotlin's interpreter/Utils.kt function 'internalName'
-// Translates class names into their JLS section 13.1 binary name
-fun getIrClassBinaryName(that: IrClass): String {
-  val internalName = StringBuilder(that.name.asString())
-  generateSequence(that as? IrDeclarationParent) { (it as? IrDeclaration)?.parent }
-      .drop(1)
+// Adapted from Kotlin's interpreter/Utils.kt function 'internalName'
+// Translates class names into their JLS section 13.1 binary name,
+// and declarations within them into the parent class' JLS 13.1 name as
+// specified above, followed by a `$` separator and then a unique identifier
+// for `that`, consisting of its short name followed by any supplied signature.
+fun getIrDeclBinaryName(that: IrDeclaration, signature: String): String {
+  val shortName = when(that) {
+      is IrDeclarationWithName -> that.name.asString()
+      else -> "(unknown-name)"
+  }
+  val internalName = StringBuilder(shortName + signature);
+  generateSequence(that.parent) { (it as? IrDeclaration)?.parent }
       .forEach {
           when (it) {
               is IrClass -> internalName.insert(0, it.name.asString() + "$")
@@ -29,6 +33,12 @@ fun getIrClassBinaryName(that: IrClass): String {
 
 fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
     val cSource = irClass.source
+    // Don't emit a location for multi-file classes until we're sure we can cope with different declarations
+    // inside a class disagreeing about their source file. In particular this currently causes problems when
+    // a source-location for a declarations tries to refer to a file-id which is assumed to be declared in
+    // the class trap file.
+    if (irClass.origin == IrDeclarationOrigin.JVM_MULTIFILE_CLASS)
+        return null
     when(cSource) {
         is JavaSourceElement -> {
             val element = cSource.javaElement
@@ -42,6 +52,12 @@ fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
                 is VirtualFileKotlinClass -> return binaryClass.file
             }
         }
+        is JvmPackagePartSource -> {
+            val binaryClass = cSource.knownJvmBinaryClass
+            if (binaryClass != null && binaryClass is VirtualFileKotlinClass) {
+                return binaryClass.file
+            }
+        }
     }
     return null
 }
@@ -59,5 +75,5 @@ fun getRawIrClassBinaryPath(irClass: IrClass) =
 fun getIrClassBinaryPath(irClass: IrClass): String {
   return getRawIrClassBinaryPath(irClass)
   // Otherwise, make up a fake location:
-    ?: "/!unknown-binary-location/${getIrClassBinaryName(irClass).replace(".", "/")}.class"
+    ?: "/!unknown-binary-location/${getIrDeclBinaryName(irClass, "").replace(".", "/")}.class"
 }

java/kotlin-extractor/src/main/kotlin/utils/ExternalDecls.kt

@@ -0,0 +1,18 @@
+package com.github.codeql.utils
+
+import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrDeclaration
+import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
+import org.jetbrains.kotlin.ir.util.isFileClass
+import org.jetbrains.kotlin.ir.util.parentClassOrNull
+
+fun isExternalDeclaration(d: IrDeclaration): Boolean {
+    return d.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB ||
+           d.origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB
+}
+
+/**
+ * Returns true if `d` is not itself a class, but is a member of an external file class.
+ */
+fun isExternalFileClassMember(d: IrDeclaration) = d !is IrClass && (d.parentClassOrNull?.let { it.isFileClass } ?: false)
+