Merge pull request #10956 from github/redsun82/swift-linkage-awareness

Swift: disambuigate entities using linkage awareness on modules

swift/extractor/SwiftExtractor.cpp

@@ -8,9 +8,8 @@
 #include <swift/AST/SourceFile.h>
 #include <swift/AST/Builtins.h>
 
-#include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/translators/SwiftVisitor.h"
-#include "swift/extractor/TargetTrapFile.h"
+#include "swift/extractor/infra/TargetDomains.h"
 #include "swift/extractor/SwiftBuiltinSymbols.h"
 #include "swift/extractor/infra/file/Path.h"
 
@@ -109,21 +108,24 @@ static llvm::SmallVector<swift::Decl*> getTopLevelDecls(swift::ModuleDecl& modul
 }
 
 static std::unordered_set<swift::ModuleDecl*> extractDeclarations(
-    const SwiftExtractorConfiguration& config,
+    SwiftExtractorState& state,
     swift::CompilerInstance& compiler,
     swift::ModuleDecl& module,
     swift::SourceFile* primaryFile = nullptr) {
   auto filename = getFilename(module, primaryFile);
+  if (primaryFile) {
+    state.sourceFiles.push_back(filename);
+  }
 
   // The extractor can be called several times from different processes with
   // the same input file(s). Using `TargetFile` the first process will win, and the following
   // will just skip the work
-  auto trapTarget = createTargetTrapFile(config, filename);
-  if (!trapTarget) {
+  const auto trapType = primaryFile ? TrapType::source : TrapType::module;
+  auto trap = createTargetTrapDomain(state, filename, trapType);
+  if (!trap) {
     // another process arrived first, nothing to do for us
     return {};
   }
-  TrapDomain trap{*trapTarget};
 
   std::vector<swift::Token> comments;
   if (primaryFile && primaryFile->getBufferID().hasValue()) {
@@ -137,7 +139,7 @@ static std::unordered_set<swift::ModuleDecl*> extractDeclarations(
     }
   }
 
-  SwiftVisitor visitor(compiler.getSourceMgr(), trap, module, primaryFile);
+  SwiftVisitor visitor(compiler.getSourceMgr(), *trap, module, primaryFile);
   auto topLevelDecls = getTopLevelDecls(module, primaryFile);
   for (auto decl : topLevelDecls) {
     visitor.extract(decl);
@@ -171,11 +173,10 @@ static std::vector<swift::ModuleDecl*> collectLoadedModules(swift::CompilerInsta
   return ret;
 }
 
-void codeql::extractSwiftFiles(const SwiftExtractorConfiguration& config,
-                               swift::CompilerInstance& compiler) {
+void codeql::extractSwiftFiles(SwiftExtractorState& state, swift::CompilerInstance& compiler) {
   auto inputFiles = collectInputFilenames(compiler);
   std::vector<swift::ModuleDecl*> todo = collectLoadedModules(compiler);
-  std::unordered_set<swift::ModuleDecl*> seen{todo.begin(), todo.end()};
+  state.encounteredModules.insert(todo.begin(), todo.end());
 
   while (!todo.empty()) {
     auto module = todo.back();
@@ -191,16 +192,16 @@ void codeql::extractSwiftFiles(const SwiftExtractorConfiguration& config,
       if (inputFiles.count(sourceFile->getFilename().str()) == 0) {
         continue;
       }
-      archiveFile(config, *sourceFile);
-      encounteredModules = extractDeclarations(config, compiler, *module, sourceFile);
+      archiveFile(state.configuration, *sourceFile);
+      encounteredModules = extractDeclarations(state, compiler, *module, sourceFile);
     }
     if (!isFromSourceFile) {
-      encounteredModules = extractDeclarations(config, compiler, *module);
+      encounteredModules = extractDeclarations(state, compiler, *module);
     }
     for (auto encountered : encounteredModules) {
-      if (seen.count(encountered) == 0) {
+      if (state.encounteredModules.count(encountered) == 0) {
         todo.push_back(encountered);
-        seen.insert(encountered);
+        state.encounteredModules.insert(encountered);
       }
     }
   }

swift/extractor/SwiftExtractor.h

@@ -1,11 +1,10 @@
 #pragma once
 
-#include "swift/extractor/config/SwiftExtractorConfiguration.h"
+#include "swift/extractor/config/SwiftExtractorState.h"
 #include <swift/AST/SourceFile.h>
 #include <swift/Frontend/Frontend.h>
 #include <memory>
 
 namespace codeql {
-void extractSwiftFiles(const SwiftExtractorConfiguration& config,
-                       swift::CompilerInstance& compiler);
+void extractSwiftFiles(SwiftExtractorState& state, swift::CompilerInstance& compiler);
 }  // namespace codeql

swift/extractor/TargetTrapFile.cpp

@@ -1,18 +0,0 @@
-#include "swift/extractor/TargetTrapFile.h"
-#include <iomanip>
-namespace codeql {
-std::optional<TargetFile> createTargetTrapFile(const SwiftExtractorConfiguration& configuration,
-                                               const std::filesystem::path& target) {
-  auto trap = target;
-  trap += ".trap";
-  auto ret = TargetFile::create(trap, configuration.trapDir, configuration.getTempTrapDir());
-  if (ret) {
-    *ret << "/* extractor-args:\n";
-    for (const auto& opt : configuration.frontendOptions) {
-      *ret << "  " << std::quoted(opt) << " \\\n";
-    }
-    *ret << "\n*/\n";
-  }
-  return ret;
-}
-}  // namespace codeql

swift/extractor/TargetTrapFile.h

@@ -1,11 +0,0 @@
-#pragma once
-
-#include "swift/extractor/infra/file/TargetFile.h"
-#include "swift/extractor/config/SwiftExtractorConfiguration.h"
-
-namespace codeql {
-
-std::optional<TargetFile> createTargetTrapFile(const SwiftExtractorConfiguration& configuration,
-                                               const std::filesystem::path& target);
-
-}  // namespace codeql

swift/extractor/config/SwiftExtractorState.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <vector>
+#include <unordered_set>
+#include <filesystem>
+
+#include <swift/AST/Decl.h>
+
+#include "swift/extractor/config/SwiftExtractorConfiguration.h"
+
+namespace codeql {
+struct SwiftExtractorState {
+  const SwiftExtractorConfiguration configuration;
+
+  // All the trap files related to this extraction. This may also include trap files generated in a
+  // previous run but that this run requested as well. Paths are relative to `configuration.trapDir`
+  std::vector<std::filesystem::path> traps;
+
+  // All modules encountered during this extractor run, which therefore are dependencies of the
+  // outcomes of this run
+  std::unordered_set<const swift::ModuleDecl*> encounteredModules;
+
+  std::vector<std::filesystem::path> sourceFiles;
+
+  // The path for the modules outputted by the underlying frontend run, ignoring path redirection
+  std::vector<std::filesystem::path> originalOutputModules;
+};
+
+}  // namespace codeql

swift/extractor/infra/BUILD.bazel

@@ -6,6 +6,7 @@ swift_cc_library(
     hdrs = glob(["*.h"]),
     visibility = ["//swift:__subpackages__"],
     deps = [
+        "//swift/extractor/config",
         "//swift/extractor/infra/file",
         "//swift/extractor/trap",
         "//swift/third_party/swift-llvm-support",

swift/extractor/infra/TargetDomains.cpp

@@ -0,0 +1,85 @@
+#include "swift/extractor/infra/TargetDomains.h"
+#include <iomanip>
+
+namespace codeql {
+
+static const char* typeToStr(TrapType type) {
+  switch (type) {
+    case TrapType::source:
+      return "sources";
+    case TrapType::module:
+      return "modules";
+    case TrapType::invocation:
+      return "invocations";
+    case TrapType::linkage:
+      return "linkage";
+    default:
+      return "";
+  }
+}
+
+static std::filesystem::path getRelativeTrapPath(const std::filesystem::path& target,
+                                                 TrapType type,
+                                                 const char* extension = ".trap") {
+  auto trap = typeToStr(type) / target.relative_path();
+  trap += extension;
+  return trap;
+}
+
+std::filesystem::path getTrapPath(const SwiftExtractorState& state,
+                                  const std::filesystem::path& target,
+                                  TrapType type) {
+  return state.configuration.trapDir / getRelativeTrapPath(target, type);
+}
+
+std::optional<TrapDomain> createTargetTrapDomain(SwiftExtractorState& state,
+                                                 const std::filesystem::path& target,
+                                                 TrapType type) {
+  if (target.empty()) {
+    return std::nullopt;
+  }
+  auto trap = getRelativeTrapPath(target, type);
+  auto ret =
+      TargetFile::create(trap, state.configuration.trapDir, state.configuration.getTempTrapDir());
+  state.traps.push_back(std::move(trap));
+  if (ret) {
+    *ret << "/* extractor-args:\n";
+    for (const auto& opt : state.configuration.frontendOptions) {
+      *ret << "  " << std::quoted(opt) << " \\\n";
+    }
+    *ret << "\n*/\n";
+    return TrapDomain{*std::move(ret)};
+  }
+  return std::nullopt;
+}
+
+namespace {
+template <typename Domain>
+std::optional<Domain> createTarget(const SwiftExtractorState& state,
+                                   const std::filesystem::path& target,
+                                   const char* extension) {
+  if (target.empty()) {
+    return std::nullopt;
+  }
+  auto file = getRelativeTrapPath(target, TrapType::linkage, extension);
+  auto ret =
+      TargetFile::create(file, state.configuration.trapDir, state.configuration.getTempTrapDir());
+  if (ret) {
+    return Domain{*std::move(ret)};
+  }
+  return std::nullopt;
+}
+
+}  // namespace
+
+std::optional<LinkDomain> createTargetLinkDomain(const SwiftExtractorState& state,
+                                                 const std::filesystem::path& target) {
+  return createTarget<LinkDomain>(state, target, ".link");
+}
+
+std::optional<ObjectDomain> createTargetObjectDomain(const SwiftExtractorState& state,
+                                                     const std::filesystem::path& target) {
+  return createTarget<ObjectDomain>(state, target, ".odep");
+}
+
+}  // namespace codeql

swift/extractor/infra/TargetDomains.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "swift/extractor/trap/TrapDomain.h"
+#include "swift/extractor/trap/LinkDomain.h"
+#include "swift/extractor/trap/ObjectDomain.h"
+#include "swift/extractor/config/SwiftExtractorState.h"
+
+namespace codeql {
+
+enum class TrapType {
+  source,
+  module,
+  invocation,
+  linkage,
+};
+
+std::filesystem::path getTrapPath(const SwiftExtractorState& state,
+                                  const std::filesystem::path& target,
+                                  TrapType type);
+
+std::optional<TrapDomain> createTargetTrapDomain(SwiftExtractorState& state,
+                                                 const std::filesystem::path& target,
+                                                 TrapType type);
+
+std::optional<LinkDomain> createTargetLinkDomain(const SwiftExtractorState& state,
+                                                 const std::filesystem::path& target);
+
+std::optional<ObjectDomain> createTargetObjectDomain(const SwiftExtractorState& state,
+                                                     const std::filesystem::path& target);
+
+}  // namespace codeql

swift/extractor/infra/file/Path.cpp

@@ -21,7 +21,7 @@ std::filesystem::path resolvePath(const std::filesystem::path& path) {
   std::filesystem::path ret = {};
   static const auto canonicalize = shouldCanonicalize();
   if (canonicalize) {
-    ret = std::filesystem::canonical(path, ec);
+    ret = std::filesystem::weakly_canonical(path, ec);
   } else {
     ret = std::filesystem::absolute(path, ec);
   }

swift/extractor/invocation/BUILD.bazel

@@ -6,6 +6,8 @@ swift_cc_library(
     hdrs = glob(["*.h"]),
     visibility = ["//swift:__subpackages__"],
     deps = [
+        "//swift/extractor/config",
         "//swift/extractor/infra",
+        "//swift/extractor/remapping",
     ],
 )

swift/extractor/invocation/SwiftInvocationExtractor.cpp

@@ -0,0 +1,167 @@
+#include "swift/extractor/invocation/SwiftInvocationExtractor.h"
+#include "swift/extractor/remapping/SwiftFileInterception.h"
+#include "swift/extractor/infra/TargetDomains.h"
+#include "swift/extractor/trap/generated/TrapTags.h"
+#include "swift/extractor/infra/file/TargetFile.h"
+#include "swift/extractor/infra/file/Path.h"
+#include "swift/extractor/trap/LinkDomain.h"
+
+namespace fs = std::filesystem;
+using namespace std::string_literals;
+
+namespace codeql {
+namespace {
+std::string getModuleId(const std::string_view& name, const std::string_view& hash) {
+  auto ret = "module:"s;
+  ret += name;
+  ret += ':';
+  ret += hash;
+  return ret;
+}
+
+std::string getModuleId(const swift::ModuleDecl* module, const std::string_view& hash) {
+  return getModuleId(module->getRealName().str(), hash);
+}
+
+fs::path getModuleTarget(const std::string_view& name, const std::string_view& hash) {
+  fs::path ret{"modules"};
+  ret /= name;
+  ret += '_';
+  ret += hash;
+  ret /= "module";
+  return ret;
+}
+
+std::optional<std::string> getModuleHash(const fs::path& moduleFile) {
+  return getHashOfRealFile(moduleFile);
+}
+
+std::optional<std::string> getModuleHash(const swift::ModuleDecl* module) {
+  return getModuleHash(std::string_view{module->getModuleFilename()});
+}
+
+std::string getSourceId(const fs::path& file) {
+  return "source:"s + file.c_str();
+}
+
+std::string getSourceId(const swift::InputFile& input) {
+  return getSourceId(resolvePath(input.getFileName()));
+}
+
+fs::path getSourceTarget(const fs::path& file) {
+  return "sources" / file.relative_path();
+}
+
+struct ModuleInfo {
+  fs::path target;
+  std::string id;
+};
+
+std::vector<ModuleInfo> emitModuleImplementations(SwiftExtractorState& state,
+                                                  const std::string& moduleName) {
+  std::vector<ModuleInfo> ret;
+  ret.reserve(state.originalOutputModules.size());
+  for (const auto& modulePath : state.originalOutputModules) {
+    if (auto hash = getModuleHash(modulePath)) {
+      auto target = getModuleTarget(moduleName, *hash);
+      if (auto moduleTrap = createTargetTrapDomain(state, target, TrapType::linkage)) {
+        moduleTrap->createLabelWithImplementationId<ModuleDeclTag>(*hash, moduleName);
+        ret.push_back({target, getModuleId(moduleName, *hash)});
+      }
+    }
+  }
+  return ret;
+}
+
+void emitLinkFile(const SwiftExtractorState& state, const fs::path& target, const std::string& id) {
+  if (auto link = createTargetLinkDomain(state, target)) {
+    link->emitTarget(id);
+    link->emitObjectDependency(id);
+  }
+}
+
+void emitSourceObjectDependencies(const SwiftExtractorState& state,
+                                  const fs::path& target,
+                                  const std::string& id) {
+  if (auto object = createTargetObjectDomain(state, target)) {
+    object->emitObject(id);
+    for (auto encounteredModule : state.encounteredModules) {
+      if (auto depHash = getModuleHash(encounteredModule)) {
+        object->emitObjectDependency(getModuleId(encounteredModule, *depHash));
+      }
+    }
+    for (const auto& requestedTrap : state.traps) {
+      object->emitTrapDependency(requestedTrap);
+    }
+  }
+}
+
+void replaceMergedModulesImplementation(const SwiftExtractorState& state,
+                                        const std::string& name,
+                                        const fs::path& mergeTarget,
+                                        const std::string& mergedPartHash) {
+  std::error_code ec;
+  auto mergedPartTarget = getModuleTarget(name, mergedPartHash);
+  fs::copy(getTrapPath(state, mergeTarget, TrapType::linkage),
+           getTrapPath(state, mergedPartTarget, TrapType::linkage),
+           fs::copy_options::overwrite_existing, ec);
+  if (ec) {
+    std::cerr << "unable to replace trap implementation id for merged module '" << name << "' ("
+              << ec.message() << ")";
+  }
+}
+
+void emitModuleObjectDependencies(const SwiftExtractorState& state,
+                                  const swift::FrontendOptions& options,
+                                  const fs::path& target,
+                                  const std::string& id) {
+  if (auto object = createTargetObjectDomain(state, target)) {
+    object->emitObject(id);
+    for (auto encounteredModule : state.encounteredModules) {
+      if (auto depHash = getModuleHash(encounteredModule)) {
+        object->emitObjectDependency(getModuleId(encounteredModule, *depHash));
+      }
+    }
+    if (options.RequestedAction == swift::FrontendOptions::ActionType::MergeModules) {
+      for (const auto& input : options.InputsAndOutputs.getAllInputs()) {
+        if (auto mergedHash = getModuleHash(input.getFileName())) {
+          object->emitObjectDependency(getModuleId(options.ModuleName, *mergedHash));
+          replaceMergedModulesImplementation(state, options.ModuleName, target, *mergedHash);
+        }
+      }
+    } else {
+      for (const auto& input : options.InputsAndOutputs.getAllInputs()) {
+        object->emitObjectDependency(getSourceId(input));
+      }
+    }
+    for (const auto& requestedTrap : state.traps) {
+      object->emitTrapDependency(requestedTrap);
+    }
+  }
+}
+}  // namespace
+
+void extractSwiftInvocation(SwiftExtractorState& state,
+                            swift::CompilerInstance& compiler,
+                            TrapDomain& trap) {
+  const auto& options = compiler.getInvocation().getFrontendOptions();
+
+  // notice that there is only one unique module name per frontend run, even when outputting
+  // multiples `swiftmodule` files
+  // this step must be executed first so that we can capture in `state` the emitted trap files
+  // that will be used in following steps
+  auto modules = emitModuleImplementations(state, options.ModuleName);
+
+  for (const auto& input : state.sourceFiles) {
+    auto path = resolvePath(input);
+    auto target = getSourceTarget(path);
+    auto inputId = getSourceId(path);
+    emitSourceObjectDependencies(state, target, inputId);
+  }
+
+  for (const auto& [target, moduleId] : modules) {
+    emitLinkFile(state, target, moduleId);
+    emitModuleObjectDependencies(state, options, target, moduleId);
+  }
+}
+}  // namespace codeql

swift/extractor/invocation/SwiftInvocationExtractor.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <memory>
+
+#include <swift/AST/SourceFile.h>
+#include <swift/Frontend/Frontend.h>
+
+#include "swift/extractor/config/SwiftExtractorState.h"
+#include "swift/extractor/trap/TrapDomain.h"
+
+namespace codeql {
+
+void extractSwiftInvocation(SwiftExtractorState& state,
+                            swift::CompilerInstance& compiler,
+                            codeql::TrapDomain& trap);
+}  // namespace codeql

swift/extractor/main.cpp

@@ -9,11 +9,13 @@
 
 #include <swift/Basic/LLVMInitialize.h>
 #include <swift/FrontendTool/FrontendTool.h>
+#include <swift/Basic/InitializeSwiftModules.h>
 
 #include "swift/extractor/SwiftExtractor.h"
-#include "swift/extractor/TargetTrapFile.h"
+#include "swift/extractor/infra/TargetDomains.h"
 #include "swift/extractor/remapping/SwiftFileInterception.h"
 #include "swift/extractor/invocation/SwiftDiagnosticsConsumer.h"
+#include "swift/extractor/invocation/SwiftInvocationExtractor.h"
 #include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/infra/file/Path.h"
 #include <swift/Basic/InitializeSwiftModules.h>
@@ -21,21 +23,23 @@
 using namespace std::string_literals;
 
 // must be called before processFrontendOptions modifies output paths
-static void lockOutputSwiftModuleTraps(const codeql::SwiftExtractorConfiguration& config,
+static void lockOutputSwiftModuleTraps(codeql::SwiftExtractorState& state,
                                        const swift::FrontendOptions& options) {
   for (const auto& input : options.InputsAndOutputs.getAllInputs()) {
     if (const auto& module = input.getPrimarySpecificPaths().SupplementaryOutputs.ModuleOutputPath;
         !module.empty()) {
-      if (auto target = codeql::createTargetTrapFile(config, codeql::resolvePath(module))) {
-        *target << "// trap file deliberately empty\n"
-                   "// this swiftmodule was created during the build, so its entities must have"
-                   " been extracted directly from source files";
+      if (auto target = codeql::createTargetTrapDomain(state, codeql::resolvePath(module),
+                                                       codeql::TrapType::module)) {
+        target->emit("// trap file deliberately empty\n"
+                     "// this swiftmodule was created during the build, so its entities must have"
+                     " been extracted directly from source files");
       }
     }
   }
 }
 
-static void processFrontendOptions(swift::FrontendOptions& options) {
+static void processFrontendOptions(codeql::SwiftExtractorState& state,
+                                   swift::FrontendOptions& options) {
   auto& inOuts = options.InputsAndOutputs;
   std::vector<swift::InputFile> inputs;
   inOuts.forEachInput([&](const swift::InputFile& input) {
@@ -53,6 +57,7 @@ static void processFrontendOptions(swift::FrontendOptions& options) {
             input.getPrimarySpecificPaths().SupplementaryOutputs.ModuleOutputPath;
         !module.empty()) {
       psp.SupplementaryOutputs.ModuleOutputPath = codeql::redirect(module);
+      state.originalOutputModules.push_back(module);
     }
     auto inputCopy = input;
     inputCopy.setPrimarySpecificPaths(std::move(psp));
@@ -65,19 +70,19 @@ static void processFrontendOptions(swift::FrontendOptions& options) {
   }
 }
 
+codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state);
+
 // This is part of the swiftFrontendTool interface, we hook into the
 // compilation pipeline and extract files after the Swift frontend performed
 // semantic analysis
 class Observer : public swift::FrontendObserver {
  public:
-  explicit Observer(const codeql::SwiftExtractorConfiguration& config,
-                    codeql::SwiftDiagnosticsConsumer& diagConsumer)
-      : config{config}, diagConsumer{diagConsumer} {}
+  explicit Observer(const codeql::SwiftExtractorConfiguration& config) : state{config} {}
 
   void parsedArgs(swift::CompilerInvocation& invocation) override {
     auto& options = invocation.getFrontendOptions();
-    lockOutputSwiftModuleTraps(config, options);
-    processFrontendOptions(options);
+    lockOutputSwiftModuleTraps(state, options);
+    processFrontendOptions(state, options);
   }
 
   void configuredCompiler(swift::CompilerInstance& instance) override {
@@ -85,12 +90,14 @@ class Observer : public swift::FrontendObserver {
   }
 
   void performedSemanticAnalysis(swift::CompilerInstance& compiler) override {
-    codeql::extractSwiftFiles(config, compiler);
+    codeql::extractSwiftFiles(state, compiler);
+    codeql::extractSwiftInvocation(state, compiler, invocationTrap);
   }
 
  private:
-  const codeql::SwiftExtractorConfiguration& config;
-  codeql::SwiftDiagnosticsConsumer& diagConsumer;
+  codeql::SwiftExtractorState state;
+  codeql::TrapDomain invocationTrap{invocationTrapDomain(state)};
+  codeql::SwiftDiagnosticsConsumer diagConsumer{invocationTrap};
 };
 
 static std::string getenv_or(const char* envvar, const std::string& def) {
@@ -145,16 +152,16 @@ static void checkWhetherToRunUnderTool(int argc, char* const* argv) {
 
 // Creates a target file that should store per-invocation info, e.g. compilation args,
 // compilations, diagnostics, etc.
-codeql::TargetFile invocationTargetFile(const codeql::SwiftExtractorConfiguration& configuration) {
+codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state) {
   auto timestamp = std::chrono::system_clock::now().time_since_epoch().count();
   auto filename = std::to_string(timestamp) + '-' + std::to_string(getpid());
   auto target = std::filesystem::path("invocations") / std::filesystem::path(filename);
-  auto maybeFile = codeql::createTargetTrapFile(configuration, target);
-  if (!maybeFile) {
+  auto maybeDomain = codeql::createTargetTrapDomain(state, target, codeql::TrapType::invocation);
+  if (!maybeDomain) {
     std::cerr << "Cannot create invocation trap file: " << target << "\n";
     abort();
   }
-  return std::move(maybeFile.value());
+  return std::move(maybeDomain.value());
 }
 
 codeql::SwiftExtractorConfiguration configure(int argc, char** argv) {
@@ -183,10 +190,7 @@ int main(int argc, char** argv) {
 
   auto openInterception = codeql::setupFileInterception(configuration);
 
-  auto invocationTrapFile = invocationTargetFile(configuration);
-  codeql::TrapDomain invocationDomain(invocationTrapFile);
-  codeql::SwiftDiagnosticsConsumer diagConsumer(invocationDomain);
-  Observer observer(configuration, diagConsumer);
+  Observer observer(configuration);
   int frontend_rc = swift::performFrontend(configuration.frontendOptions, "swift-extractor",
                                            (void*)main, &observer);
 

swift/extractor/translators/DeclTranslator.cpp

@@ -303,13 +303,15 @@ std::optional<codeql::ModuleDecl> DeclTranslator::translateModuleDecl(
 }
 
 std::string DeclTranslator::mangledName(const swift::ValueDecl& decl) {
+  std::string_view moduleName = decl.getModuleContext()->getRealName().str();
   // ASTMangler::mangleAnyDecl crashes when called on `ModuleDecl`
-  // TODO find a more unique string working also when different modules are compiled with the same
-  // name
-  std::ostringstream ret;
   if (decl.getKind() == swift::DeclKind::Module) {
-    ret << static_cast<const swift::ModuleDecl&>(decl).getRealName().str().str();
-  } else if (decl.getKind() == swift::DeclKind::TypeAlias) {
+    return std::string{moduleName};
+  }
+  std::ostringstream ret;
+  // stamp all declarations with an id-ref of the containing module
+  ret << '{' << ModuleDeclTag::prefix << '_' << moduleName << '}';
+  if (decl.getKind() == swift::DeclKind::TypeAlias) {
     // In cases like this (when coming from PCM)
     //  typealias CFXMLTree = CFTree
     //  typealias CFXMLTreeRef = CFXMLTree
@@ -321,14 +323,6 @@ std::string DeclTranslator::mangledName(const swift::ValueDecl& decl) {
     // prefix adds a couple of special symbols, we don't necessary need them
     ret << mangler.mangleAnyDecl(&decl, /* prefix = */ false);
   }
-  // there can be separate declarations (`VarDecl` or `AccessorDecl`) which are effectively the same
-  // (with equal mangled name) but come from different clang modules. This is the case for example
-  // for glibc constants like `L_SET` that appear in both `SwiftGlibc` and `CDispatch`.
-  // For the moment, we sidestep the problem by making them separate entities in the DB
-  // TODO find a more solid solution
-  if (decl.getModuleContext()->isNonSwiftModule()) {
-    ret << '_' << decl.getModuleContext()->getRealName().str().str();
-  }
   return ret.str();
 }
 

swift/extractor/trap/BUILD.bazel

@@ -30,7 +30,7 @@ filegroup(
 
 swift_cc_library(
     name = "trap",
-    srcs = [
+    srcs = glob(["*.cpp"]) + [
         "generated/%s%s.cpp" % (dir, name)
         for dir in _dirs
         for name in ("TrapEntries", "TrapClasses")
@@ -43,4 +43,7 @@ swift_cc_library(
         for name in ("TrapEntries", "TrapClasses")
     ],
     visibility = ["//visibility:public"],
+    deps = [
+        "//swift/extractor/infra/file",
+    ],
 )

swift/extractor/trap/LinkDomain.cpp

@@ -0,0 +1,37 @@
+#include "swift/extractor/trap/LinkDomain.h"
+
+namespace codeql {
+LinkDomain::LinkDomain(TargetFile out) : out{std::move(out)} {
+  this->out << "Linker invocation 1.0\n";
+}
+
+void LinkDomain::emitTarget(const std::string& target) {
+  ensurePhase(Phase::target);
+  out << target << '\n';
+}
+
+void LinkDomain::emitObjectDependency(const std::string& object) {
+  ensurePhase(Phase::objects);
+  out << object << '\n';
+}
+
+void LinkDomain::ensurePhase(Phase phase) {
+  assert(phase >= current && "wrong order in .link file attributes");
+
+  if (phase > current) {
+    switch (phase) {
+      case Phase::target:
+        out << "TARGET\n";
+        break;
+      case Phase::objects:
+        out << "OBJECTS\n";
+        break;
+      default:
+        assert(false && "unexpected phase");
+        break;
+    }
+  }
+  current = phase;
+}
+
+}  // namespace codeql

swift/extractor/trap/LinkDomain.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <cassert>
+#include "swift/extractor/infra/file/TargetFile.h"
+
+namespace codeql {
+
+class LinkDomain {
+  TargetFile out;
+
+  enum class Phase {
+    header,
+    target,
+    objects,
+    end,
+  };
+
+  Phase current{Phase::header};
+
+ public:
+  explicit LinkDomain(TargetFile out);
+
+  void emitTarget(const std::string& target);
+  void emitObjectDependency(const std::string& object);
+
+ private:
+  void ensurePhase(Phase phase);
+};
+
+}  // namespace codeql

swift/extractor/trap/ObjectDomain.cpp

@@ -0,0 +1,45 @@
+#include "swift/extractor/trap/ObjectDomain.h"
+
+namespace codeql {
+ObjectDomain::ObjectDomain(TargetFile out) : out{std::move(out)} {
+  this->out << "TRAP dependencies 1.2\n";
+}
+
+void ObjectDomain::emitObject(const std::string& object) {
+  ensurePhase(Phase::objects);
+  out << object << '\n';
+}
+
+void ObjectDomain::emitObjectDependency(const std::string& object) {
+  ensurePhase(Phase::input_objects);
+  out << object << '\n';
+}
+
+void ObjectDomain::emitTrapDependency(const std::filesystem::path& trap) {
+  ensurePhase(Phase::traps);
+  out << trap.c_str() << '\n';
+}
+
+void ObjectDomain::ensurePhase(Phase phase) {
+  assert(phase >= current && "wrong order in .odep file attributes");
+
+  if (phase > current) {
+    switch (phase) {
+      case Phase::objects:
+        out << "OBJECTS\n";
+        break;
+      case Phase::input_objects:
+        out << "INPUT_OBJECTS\n";
+        break;
+      case Phase::traps:
+        out << "TRAPS\n";
+        break;
+      default:
+        assert(false && "unexpected phase");
+        break;
+    }
+  }
+  current = phase;
+}
+
+}  // namespace codeql

swift/extractor/trap/ObjectDomain.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <cassert>
+#include <filesystem>
+#include "swift/extractor/infra/file/TargetFile.h"
+
+namespace codeql {
+
+class ObjectDomain {
+  TargetFile out;
+
+  enum class Phase {
+    header,
+    objects,
+    input_objects,
+    traps,
+    end,
+  };
+
+  Phase current{Phase::header};
+
+ public:
+  explicit ObjectDomain(TargetFile out);
+
+  void emitObject(const std::string& object);
+  void emitObjectDependency(const std::string& object);
+  void emitTrapDependency(const std::filesystem::path& trap);
+
+ private:
+  void ensurePhase(Phase phase);
+};
+
+}  // namespace codeql

swift/extractor/trap/TrapDomain.h

@@ -10,27 +10,28 @@ namespace codeql {
 
 // Abstracts a given trap output file, with its own universe of trap labels
 class TrapDomain {
-  TargetFile& out_;
+  TargetFile out;
 
  public:
-  explicit TrapDomain(TargetFile& out) : out_{out} {}
+  explicit TrapDomain(TargetFile&& out) : out{std::move(out)} {}
 
   template <typename Entry>
   void emit(const Entry& e) {
-    print(e);
+    out << e << '\n';
   }
 
   template <typename... Args>
   void debug(const Args&... args) {
-    print("/* DEBUG:");
-    print(args...);
-    print("*/");
+    out << "/* DEBUG:\n";
+    (out << ... << args);
+    out << "\n*/\n";
   }
 
   template <typename Tag>
   TrapLabel<Tag> createLabel() {
     auto ret = allocateLabel<Tag>();
     assignStar(ret);
+    out << '\n';
     return ret;
   }
 
@@ -38,6 +39,16 @@ class TrapDomain {
   TrapLabel<Tag> createLabel(Args&&... args) {
     auto ret = allocateLabel<Tag>();
     assignKey(ret, std::forward<Args>(args)...);
+    out << '\n';
+    return ret;
+  }
+
+  template <typename Tag, typename... Args>
+  TrapLabel<Tag> createLabelWithImplementationId(const std::string_view& implementationId,
+                                                 Args&&... args) {
+    auto ret = allocateLabel<Tag>();
+    assignKey(ret, std::forward<Args>(args)...);
+    out << " .implementation " << trapQuoted(implementationId) << '\n';
     return ret;
   }
 
@@ -49,21 +60,16 @@ class TrapDomain {
     return TrapLabel<Tag>::unsafeCreateFromExplicitId(id_++);
   }
 
-  template <typename... Args>
-  void print(const Args&... args) {
-    (out_ << ... << args) << '\n';
-  }
-
   template <typename Tag>
   void assignStar(TrapLabel<Tag> label) {
-    print(label, "=*");
+    out << label << "=*";
   }
 
   template <typename Tag>
   void assignKey(TrapLabel<Tag> label, const std::string& key) {
     // prefix the key with the id to guarantee the same key is not used wrongly with different tags
     auto prefixed = std::string(Tag::prefix) + '_' + key;
-    print(label, "=@", trapQuoted(prefixed));
+    out << label << "=@" << trapQuoted(prefixed);
   }
 
   template <typename Tag, typename... Args>

swift/extractor/trap/TrapLabel.h

@@ -86,7 +86,7 @@ struct TrapLabelVectorWrapper {
   }
 };
 
-inline auto trapQuoted(const std::string& s) {
+inline auto trapQuoted(const std::string_view& s) {
   return std::quoted(s, '"', '"');
 }
 

swift/integration-tests/posix-only/cross-references/Module.expected

@@ -0,0 +1 @@
+| file://:0:0:0:0 | cross_references |

swift/integration-tests/posix-only/cross-references/Module.ql

@@ -0,0 +1,6 @@
+import swift
+
+// check that the `cross_references` module has exactly one entity in the DB
+from ModuleDecl m
+where m.getName() = "cross_references"
+select m

swift/integration-tests/posix-only/frontend-invocations/F1.swift

@@ -1 +1 @@
-func foo() {}
+public func foo() {}

swift/integration-tests/posix-only/frontend-invocations/F2.swift

@@ -1 +1 @@
-func bar() {}
+public func bar() {}

swift/integration-tests/posix-only/frontend-invocations/F3.swift

@@ -0,0 +1 @@
+public func baz() {}

swift/integration-tests/posix-only/frontend-invocations/F4.swift

@@ -0,0 +1 @@
+public func bazz() {}

swift/integration-tests/posix-only/frontend-invocations/F5.swift

@@ -0,0 +1,9 @@
+import F
+import Fs
+
+func use_imported() {
+    F.foo()
+    F.bar()
+    Fs.baz()
+    Fs.bazz()
+}

swift/integration-tests/posix-only/frontend-invocations/Files.expected

@@ -5,7 +5,11 @@
 | E.swift:0:0:0:0 | E.swift |
 | F1.swift:0:0:0:0 | F1.swift |
 | F2.swift:0:0:0:0 | F2.swift |
+| F3.swift:0:0:0:0 | F3.swift |
+| F5.swift:0:0:0:0 | F5.swift |
 | G.swift:0:0:0:0 | G.swift |
 | H1.swift:0:0:0:0 | H1.swift |
 | H2.swift:0:0:0:0 | H2.swift |
+| I1.swift:0:0:0:0 | I1.swift |
+| I2.swift:0:0:0:0 | I2.swift |
 | file://:0:0:0:0 |  |

swift/integration-tests/posix-only/frontend-invocations/I1.swift

@@ -0,0 +1 @@
+let I1 = ""

swift/integration-tests/posix-only/frontend-invocations/I2.swift

@@ -0,0 +1 @@
+let I2 = ""

swift/integration-tests/posix-only/frontend-invocations/Modules.expected

@@ -0,0 +1,15 @@
+| file://:0:0:0:0 | A |
+| file://:0:0:0:0 | B |
+| file://:0:0:0:0 | Builtin |
+| file://:0:0:0:0 | C |
+| file://:0:0:0:0 | D |
+| file://:0:0:0:0 | E |
+| file://:0:0:0:0 | F |
+| file://:0:0:0:0 | F5 |
+| file://:0:0:0:0 | Fs |
+| file://:0:0:0:0 | G |
+| file://:0:0:0:0 | H1 |
+| file://:0:0:0:0 | I1 |
+| file://:0:0:0:0 | Swift |
+| file://:0:0:0:0 | SwiftOnoneSupport |
+| file://:0:0:0:0 | SwiftShims |

swift/integration-tests/posix-only/frontend-invocations/Modules.ql

@@ -0,0 +1,5 @@
+import swift
+
+from ModuleDecl m
+where m.getName().charAt(0) != "_"
+select m

swift/integration-tests/posix-only/frontend-invocations/build.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -x
 
 if [[ "$(uname)" == Darwin ]]; then
   SDK="-sdk $(xcrun -show-sdk-path)"
@@ -8,16 +8,22 @@ else
   FRONTEND="swift-frontend"
 fi
 
+function invoke() {
+  $FRONTEND -frontend "$@" $SDK
+}
+
 rm -rf *.swiftmodule *.o
 
-$FRONTEND -frontend -c A.swift $SDK
-$FRONTEND -frontend -c B.swift -o B.o $SDK
-$FRONTEND -frontend -c -primary-file C.swift $SDK
-$FRONTEND -frontend -c -primary-file D.swift -o D.o $SDK
-$FRONTEND -frontend -c -primary-file E.swift Esup.swift -o E.o $SDK
-$FRONTEND -frontend -emit-module -primary-file F1.swift F2.swift -module-name F -o F1.swiftmodule $SDK
-$FRONTEND -frontend -emit-module F1.swift -primary-file F2.swift -module-name F -o F2.swiftmodule $SDK
-$FRONTEND -frontend -emit-module F1.swift F2.swift -o Fs.swiftmodule $SDK
-$FRONTEND -frontend -merge-modules F1.swiftmodule F2.swiftmodule -o F.swiftmodule $SDK
-( cd dir; $FRONTEND -frontend -c ../G.swift $SDK )
-$FRONTEND -frontend -c -primary-file H1.swift -primary-file H2.swift H3.swift -emit-module-path H1.swiftmodule -emit-module-path H2.swiftmodule -o H1.o -o H2.o $SDK
+invoke -c A.swift
+invoke -c B.swift -o B.o
+invoke -c -primary-file C.swift
+invoke -c -primary-file D.swift -o D.o
+invoke -c -primary-file E.swift Esup.swift -o E.o
+invoke -emit-module -primary-file F1.swift F2.swift -module-name F -o F1.swiftmodule
+invoke -emit-module F1.swift -primary-file F2.swift -module-name F -o F2.swiftmodule
+invoke -emit-module F3.swift F4.swift -o Fs.swiftmodule
+invoke -emit-module -merge-modules F1.swiftmodule F2.swiftmodule -o F.swiftmodule
+invoke -c F5.swift -o F5.o -I.
+( cd dir; invoke -c ../G.swift )
+invoke -c -primary-file H1.swift -primary-file H2.swift H3.swift -emit-module-path H1.swiftmodule -emit-module-path H2.swiftmodule -o H1.o -o H2.o
+invoke -emit-module -primary-file I1.swift -primary-file I2.swift -o I1.swiftmodule -o I2.swiftmodule

swift/integration-tests/posix-only/linkage-awareness/Bodies.expected

@@ -0,0 +1,2 @@
+| Foo1/Sources/foo/main.swift:1:1:1:31 | foo() | Foo1/Sources/foo/main.swift:1:19:1:31 | { ... } |
+| Foo2/Sources/foo/main.swift:2:1:2:32 | foo() | Foo2/Sources/foo/main.swift:2:19:2:32 | { ... } |

swift/integration-tests/posix-only/linkage-awareness/Bodies.ql

@@ -0,0 +1,5 @@
+import swift
+
+from ConcreteFuncDecl decl, BraceStmt body
+where decl.getName() = "foo()" and decl.getBody() = body
+select decl, body

swift/integration-tests/posix-only/linkage-awareness/Foo1/Package.swift

@@ -0,0 +1,17 @@
+// swift-tools-version: 5.7
+
+import PackageDescription
+
+let package = Package(
+        name: "Foo",
+        products: [
+            .executable(
+                    name: "foo",
+                    targets: ["foo"]),
+        ],
+        targets: [
+            .executableTarget(
+                    name: "foo",
+                    dependencies: []),
+        ]
+)

swift/integration-tests/posix-only/linkage-awareness/Foo1/Sources/foo/main.swift

@@ -0,0 +1,3 @@
+func foo() -> Int { return 42 }
+
+print(foo())

swift/integration-tests/posix-only/linkage-awareness/Foo2/Package.swift

@@ -0,0 +1,17 @@
+// swift-tools-version: 5.7
+
+import PackageDescription
+
+let package = Package(
+        name: "Foo",
+        products: [
+            .executable(
+                    name: "foo",
+                    targets: ["foo"]),
+        ],
+        targets: [
+            .executableTarget(
+                    name: "foo",
+                    dependencies: []),
+        ]
+)

swift/integration-tests/posix-only/linkage-awareness/Foo2/Sources/foo/main.swift

@@ -0,0 +1,5 @@
+
+func foo() -> Int { return 404 }
+
+
+print(foo())

swift/integration-tests/posix-only/linkage-awareness/build.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+for dir in Foo1 Foo2; do
+  (
+    cd $dir
+    swift package clean
+    swift build
+  )
+done

swift/integration-tests/posix-only/linkage-awareness/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+
+run_codeql_database_create([
+    './build.sh',
+], lang='swift', keep_trap=True)

swift/integration-tests/posix-only/partial-modules/test.py

@@ -1,7 +1,6 @@
 from create_database_utils import *
 
 run_codeql_database_create([
-    'env',
     'swift package clean',
     'swift build'
 ], lang='swift', keep_trap=True)