Merge branch 'main' into redsun82/swift-decls

swift/extractor/BUILD.bazel

@@ -10,6 +10,7 @@ swift_cc_binary(
     visibility = ["//swift:__pkg__"],
     deps = [
         "//swift/extractor/infra",
+        "//swift/extractor/invocation",
         "//swift/extractor/remapping",
         "//swift/extractor/translators",
         "//swift/third_party/swift-llvm-support",

swift/extractor/infra/SwiftDispatcher.h

@@ -10,7 +10,7 @@
 #include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/infra/SwiftTagTraits.h"
 #include "swift/extractor/trap/generated/TrapClasses.h"
-#include "swift/extractor/infra/file/PathHash.h"
+#include "swift/extractor/infra/SwiftLocationExtractor.h"
 
 namespace codeql {
 
@@ -29,8 +29,7 @@ class SwiftDispatcher {
                                const swift::Expr*,
                                const swift::Pattern*,
                                const swift::TypeRepr*,
-                               const swift::TypeBase*,
-                               std::filesystem::path>;
+                               const swift::TypeBase*>;
 
   template <typename E>
   static constexpr bool IsStorable = std::is_constructible_v<Store::Handle, const E&>;
@@ -48,10 +47,11 @@ class SwiftDispatcher {
       : sourceManager{sourceManager},
         trap{trap},
         currentModule{currentModule},
-        currentPrimarySourceFile{currentPrimarySourceFile} {
+        currentPrimarySourceFile{currentPrimarySourceFile},
+        locationExtractor(trap) {
     if (currentPrimarySourceFile) {
       // we make sure the file is in the trap output even if the source is empty
-      fetchLabel(getFilePath(currentPrimarySourceFile->getFilename()));
+      locationExtractor.emitFile(currentPrimarySourceFile->getFilename());
     }
   }
 
@@ -325,20 +325,7 @@ class SwiftDispatcher {
   void attachLocation(swift::SourceLoc start,
                       swift::SourceLoc end,
                       TrapLabel<LocatableTag> locatableLabel) {
-    if (!start.isValid() || !end.isValid()) {
-      // invalid locations seem to come from entities synthesized by the compiler
-      return;
-    }
-    auto file = getFilePath(sourceManager.getDisplayNameForLoc(start));
-    DbLocation entry{{}};
-    entry.file = fetchLabel(file);
-    std::tie(entry.start_line, entry.start_column) = sourceManager.getLineAndColumnInBuffer(start);
-    std::tie(entry.end_line, entry.end_column) = sourceManager.getLineAndColumnInBuffer(end);
-    entry.id = trap.createLabel<DbLocationTag>('{', entry.file, "}:", entry.start_line, ':',
-                                               entry.start_column, ':', entry.end_line, ':',
-                                               entry.end_column);
-    emit(entry);
-    emit(LocatableLocationsTrap{locatableLabel, entry.id});
+    locationExtractor.attachLocation(sourceManager, start, end, locatableLabel);
   }
 
   template <typename Tag, typename... Ts>
@@ -391,12 +378,6 @@ class SwiftDispatcher {
   virtual void visit(const swift::TypeRepr* typeRepr, swift::Type type) = 0;
   virtual void visit(const swift::TypeBase* type) = 0;
 
-  void visit(const std::filesystem::path& file) {
-    auto entry = createEntry(file, file.string());
-    entry.name = file.string();
-    emit(entry);
-  }
-
   const swift::SourceManager& sourceManager;
   TrapDomain& trap;
   Store store;
@@ -404,6 +385,7 @@ class SwiftDispatcher {
   swift::ModuleDecl& currentModule;
   swift::SourceFile* currentPrimarySourceFile;
   std::unordered_set<swift::ModuleDecl*> encounteredModules;
+  SwiftLocationExtractor locationExtractor;
 };
 
 }  // namespace codeql

swift/extractor/infra/SwiftLocationExtractor.cpp

@@ -0,0 +1,59 @@
+#include <swift/AST/SourceFile.h>
+#include <swift/Basic/SourceManager.h>
+
+#include "swift/extractor/trap/TrapDomain.h"
+#include "swift/extractor/trap/generated/TrapEntries.h"
+#include "swift/extractor/trap/generated/TrapClasses.h"
+#include "swift/extractor/infra/SwiftLocationExtractor.h"
+
+using namespace codeql;
+
+static std::filesystem::path getFilePath(std::string_view path) {
+  // TODO: this needs more testing
+  // TODO: check canonicalization of names on a case insensitive filesystems
+  // TODO: make symlink resolution conditional on CODEQL_PRESERVE_SYMLINKS=true
+  std::error_code ec;
+  auto ret = std::filesystem::canonical(path, ec);
+  if (ec) {
+    std::cerr << "Cannot get real path: " << std::quoted(path) << ": " << ec.message() << "\n";
+    return {};
+  }
+  return ret;
+}
+
+void SwiftLocationExtractor::attachLocation(const swift::SourceManager& sourceManager,
+                                            swift::SourceLoc start,
+                                            swift::SourceLoc end,
+                                            TrapLabel<LocatableTag> locatableLabel) {
+  if (!start.isValid() || !end.isValid()) {
+    // invalid locations seem to come from entities synthesized by the compiler
+    return;
+  }
+  auto file = getFilePath(sourceManager.getDisplayNameForLoc(start));
+  DbLocation entry{{}};
+  entry.file = fetchFileLabel(file);
+  std::tie(entry.start_line, entry.start_column) = sourceManager.getLineAndColumnInBuffer(start);
+  std::tie(entry.end_line, entry.end_column) = sourceManager.getLineAndColumnInBuffer(end);
+  entry.id = trap.createLabel<DbLocationTag>('{', entry.file, "}:", entry.start_line, ':',
+                                             entry.start_column, ':', entry.end_line, ':',
+                                             entry.end_column);
+  trap.emit(entry);
+  trap.emit(LocatableLocationsTrap{locatableLabel, entry.id});
+}
+
+void SwiftLocationExtractor::emitFile(llvm::StringRef path) {
+  fetchFileLabel(getFilePath(path));
+}
+
+TrapLabel<FileTag> SwiftLocationExtractor::fetchFileLabel(const std::filesystem::path& file) {
+  if (store.count(file)) {
+    return store[file];
+  }
+
+  DbFile entry({});
+  entry.id = trap.createLabel<DbFileTag>(file.string());
+  entry.name = file.string();
+  trap.emit(entry);
+  store[file] = entry.id;
+  return entry.id;
+}

swift/extractor/infra/SwiftLocationExtractor.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <swift/Basic/SourceManager.h>
+#include <unordered_map>
+#include <filesystem>
+
+#include "swift/extractor/trap/generated/TrapEntries.h"
+#include "swift/extractor/infra/file/PathHash.h"
+
+namespace codeql {
+
+class TrapDomain;
+
+class SwiftLocationExtractor {
+ public:
+  explicit SwiftLocationExtractor(TrapDomain& trap) : trap(trap) {}
+
+  void attachLocation(const swift::SourceManager& sourceManager,
+                      swift::SourceLoc start,
+                      swift::SourceLoc end,
+                      TrapLabel<LocatableTag> locatableLabel);
+
+  void emitFile(llvm::StringRef path);
+
+ private:
+  TrapLabel<FileTag> fetchFileLabel(const std::filesystem::path& file);
+  TrapDomain& trap;
+  std::unordered_map<std::filesystem::path, TrapLabel<FileTag>> store;
+};
+
+}  // namespace codeql

swift/extractor/invocation/BUILD.bazel

@@ -0,0 +1,11 @@
+load("//swift:rules.bzl", "swift_cc_library")
+
+swift_cc_library(
+    name = "invocation",
+    srcs = glob(["*.cpp"]),
+    hdrs = glob(["*.h"]),
+    visibility = ["//swift:__subpackages__"],
+    deps = [
+        "//swift/extractor/infra",
+    ],
+)

swift/extractor/invocation/SwiftDiagnosticsConsumer.cpp

@@ -0,0 +1,44 @@
+#include "swift/extractor/invocation/SwiftDiagnosticsConsumer.h"
+#include "swift/extractor/trap/generated/TrapEntries.h"
+#include "swift/extractor/trap/TrapDomain.h"
+
+#include <swift/AST/DiagnosticEngine.h>
+#include <swift/Basic/SourceManager.h>
+#include <llvm/ADT/SmallString.h>
+#include <llvm/Support/raw_ostream.h>
+#include <string>
+
+using namespace codeql;
+
+static int diagnosticsKind(const swift::DiagnosticInfo& diagInfo) {
+  switch (diagInfo.Kind) {
+    case swift::DiagnosticKind::Error:
+      return 1;
+    case swift::DiagnosticKind::Warning:
+      return 2;
+    case swift::DiagnosticKind::Note:
+      return 3;
+    case swift::DiagnosticKind::Remark:
+      return 4;
+  }
+  return 0;
+}
+
+void SwiftDiagnosticsConsumer::handleDiagnostic(swift::SourceManager& sourceManager,
+                                                const swift::DiagnosticInfo& diagInfo) {
+  auto message = getDiagMessage(sourceManager, diagInfo);
+  DiagnosticsTrap diag{};
+  diag.id = trap.createLabel<DiagnosticsTag>();
+  diag.kind = diagnosticsKind(diagInfo);
+  diag.text = message;
+  trap.emit(diag);
+  locationExtractor.attachLocation(sourceManager, diagInfo.Loc, diagInfo.Loc, diag.id);
+}
+
+std::string SwiftDiagnosticsConsumer::getDiagMessage(swift::SourceManager& sourceManager,
+                                                     const swift::DiagnosticInfo& diagInfo) {
+  llvm::SmallString<256> text;
+  llvm::raw_svector_ostream out(text);
+  swift::DiagnosticEngine::formatDiagnosticText(out, diagInfo.FormatString, diagInfo.FormatArgs);
+  return text.str().str();
+}

swift/extractor/invocation/SwiftDiagnosticsConsumer.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <swift/AST/DiagnosticConsumer.h>
+#include "swift/extractor/infra/SwiftLocationExtractor.h"
+
+namespace codeql {
+
+class TrapDomain;
+
+class SwiftDiagnosticsConsumer : public swift::DiagnosticConsumer {
+ public:
+  explicit SwiftDiagnosticsConsumer(TrapDomain& targetFile)
+      : trap(targetFile), locationExtractor(targetFile) {}
+  void handleDiagnostic(swift::SourceManager& sourceManager,
+                        const swift::DiagnosticInfo& diagInfo) override;
+
+ private:
+  static std::string getDiagMessage(swift::SourceManager& sourceManager,
+                                    const swift::DiagnosticInfo& diagInfo);
+  TrapDomain& trap;
+  SwiftLocationExtractor locationExtractor;
+};
+
+}  // namespace codeql

swift/extractor/main.cpp

@@ -1,12 +1,11 @@
 #include <fstream>
 #include <iomanip>
 #include <stdlib.h>
-#include <unordered_set>
 #include <vector>
-#include <string>
 #include <iostream>
 #include <regex>
 #include <unistd.h>
+#include <chrono>
 
 #include <swift/Basic/LLVMInitialize.h>
 #include <swift/FrontendTool/FrontendTool.h>
@@ -15,6 +14,8 @@
 #include "swift/extractor/TargetTrapFile.h"
 #include "swift/extractor/remapping/SwiftOutputRewrite.h"
 #include "swift/extractor/remapping/SwiftOpenInterception.h"
+#include "swift/extractor/invocation/SwiftDiagnosticsConsumer.h"
+#include "swift/extractor/trap/TrapDomain.h"
 
 using namespace std::string_literals;
 
@@ -23,7 +24,13 @@ using namespace std::string_literals;
 // semantic analysis
 class Observer : public swift::FrontendObserver {
  public:
-  explicit Observer(const codeql::SwiftExtractorConfiguration& config) : config{config} {}
+  explicit Observer(const codeql::SwiftExtractorConfiguration& config,
+                    codeql::SwiftDiagnosticsConsumer& diagConsumer)
+      : config{config}, diagConsumer{diagConsumer} {}
+
+  void configuredCompiler(swift::CompilerInstance& instance) override {
+    instance.addDiagnosticConsumer(&diagConsumer);
+  }
 
   void performedSemanticAnalysis(swift::CompilerInstance& compiler) override {
     codeql::extractSwiftFiles(config, compiler);
@@ -31,6 +38,7 @@ class Observer : public swift::FrontendObserver {
 
  private:
   const codeql::SwiftExtractorConfiguration& config;
+  codeql::SwiftDiagnosticsConsumer& diagConsumer;
 };
 
 static std::string getenv_or(const char* envvar, const std::string& def) {
@@ -96,6 +104,20 @@ static void checkWhetherToRunUnderTool(int argc, char* const* argv) {
   execvp(args[0], args.data());
 }
 
+// Creates a target file that should store per-invocation info, e.g. compilation args,
+// compilations, diagnostics, etc.
+codeql::TargetFile invocationTargetFile(codeql::SwiftExtractorConfiguration& configuration) {
+  auto timestamp = std::chrono::system_clock::now().time_since_epoch().count();
+  auto filename = std::to_string(timestamp) + '-' + std::to_string(getpid());
+  auto target = std::filesystem::path("invocations") / std::filesystem::path(filename);
+  auto maybeFile = codeql::createTargetTrapFile(configuration, target);
+  if (!maybeFile) {
+    std::cerr << "Cannot create invocation trap file: " << target << "\n";
+    abort();
+  }
+  return std::move(maybeFile.value());
+}
+
 int main(int argc, char** argv) {
   checkWhetherToRunUnderTool(argc, argv);
 
@@ -131,7 +153,10 @@ int main(int argc, char** argv) {
     args.push_back(arg.c_str());
   }
 
-  Observer observer(configuration);
+  auto invocationTrapFile = invocationTargetFile(configuration);
+  codeql::TrapDomain invocationDomain(invocationTrapFile);
+  codeql::SwiftDiagnosticsConsumer diagConsumer(invocationDomain);
+  Observer observer(configuration, diagConsumer);
   int frontend_rc = swift::performFrontend(args, "swift-extractor", (void*)main, &observer);
 
   codeql::finalizeRemapping(remapping);

swift/ql/lib/codeql/swift/elements.qll

@@ -4,6 +4,7 @@ import codeql.swift.elements.Callable
 import codeql.swift.elements.Comment
 import codeql.swift.elements.DbFile
 import codeql.swift.elements.DbLocation
+import codeql.swift.elements.Diagnostics
 import codeql.swift.elements.Element
 import codeql.swift.elements.File
 import codeql.swift.elements.Locatable

swift/ql/lib/codeql/swift/elements/Diagnostics.qll

@@ -0,0 +1,5 @@
+private import codeql.swift.generated.Diagnostics
+
+class Diagnostics extends Generated::Diagnostics {
+  override string toString() { result = getText() }
+}

swift/ql/lib/codeql/swift/elements/DiagnosticsConstructor.qll

@@ -0,0 +1,4 @@
+// generated by codegen/codegen.py, remove this comment if you wish to edit this file
+private import codeql.swift.generated.Raw
+
+predicate constructDiagnostics(Raw::Diagnostics id) { any() }

swift/ql/lib/codeql/swift/generated/Diagnostics.qll

@@ -0,0 +1,20 @@
+// generated by codegen/codegen.py
+private import codeql.swift.generated.Synth
+private import codeql.swift.generated.Raw
+import codeql.swift.elements.Locatable
+
+module Generated {
+  class Diagnostics extends Synth::TDiagnostics, Locatable {
+    override string getAPrimaryQlClass() { result = "Diagnostics" }
+
+    /**
+     * Gets the text of this diagnostics.
+     */
+    string getText() { result = Synth::convertDiagnosticsToRaw(this).(Raw::Diagnostics).getText() }
+
+    /**
+     * Gets the kind of this diagnostics.
+     */
+    int getKind() { result = Synth::convertDiagnosticsToRaw(this).(Raw::Diagnostics).getKind() }
+  }
+}

swift/ql/lib/codeql/swift/generated/ParentChild.qll

@@ -135,6 +135,21 @@ private module Impl {
     )
   }
 
+  private Element getImmediateChildOfDiagnostics(
+    Diagnostics e, int index, string partialPredicateCall
+  ) {
+    exists(int b, int bLocatable, int n |
+      b = 0 and
+      bLocatable = b + 1 + max(int i | i = -1 or exists(getImmediateChildOfLocatable(e, i, _)) | i) and
+      n = bLocatable and
+      (
+        none()
+        or
+        result = getImmediateChildOfLocatable(e, index - b, partialPredicateCall)
+      )
+    )
+  }
+
   private Element getImmediateChildOfUnknownFile(
     UnknownFile e, int index, string partialPredicateCall
   ) {
@@ -4885,6 +4900,8 @@ private module Impl {
     or
     result = getImmediateChildOfDbLocation(e, index, partialAccessor)
     or
+    result = getImmediateChildOfDiagnostics(e, index, partialAccessor)
+    or
     result = getImmediateChildOfUnknownFile(e, index, partialAccessor)
     or
     result = getImmediateChildOfUnknownLocation(e, index, partialAccessor)

swift/ql/lib/codeql/swift/generated/Raw.qll

@@ -51,6 +51,14 @@ module Raw {
     override string toString() { result = "DbLocation" }
   }
 
+  class Diagnostics extends @diagnostics, Locatable {
+    override string toString() { result = "Diagnostics" }
+
+    string getText() { diagnostics(this, result, _) }
+
+    int getKind() { diagnostics(this, _, result) }
+  }
+
   class UnspecifiedElement extends @unspecified_element, Locatable {
     override string toString() { result = "UnspecifiedElement" }
 

swift/ql/lib/codeql/swift/generated/Synth.qll

@@ -8,6 +8,7 @@ module Synth {
     TComment(Raw::Comment id) { constructComment(id) } or
     TDbFile(Raw::DbFile id) { constructDbFile(id) } or
     TDbLocation(Raw::DbLocation id) { constructDbLocation(id) } or
+    TDiagnostics(Raw::Diagnostics id) { constructDiagnostics(id) } or
     TUnknownFile() or
     TUnknownLocation() or
     TUnspecifiedElement(Raw::UnspecifiedElement id) { constructUnspecifiedElement(id) } or
@@ -322,7 +323,7 @@ module Synth {
 
   class TFile = TDbFile or TUnknownFile;
 
-  class TLocatable = TArgument or TAstNode or TComment or TUnspecifiedElement;
+  class TLocatable = TArgument or TAstNode or TComment or TDiagnostics or TUnspecifiedElement;
 
   class TLocation = TDbLocation or TUnknownLocation;
 
@@ -494,6 +495,9 @@ module Synth {
   cached
   TDbLocation convertDbLocationFromRaw(Raw::Element e) { result = TDbLocation(e) }
 
+  cached
+  TDiagnostics convertDiagnosticsFromRaw(Raw::Element e) { result = TDiagnostics(e) }
+
   cached
   TUnknownFile convertUnknownFileFromRaw(Raw::Element e) { none() }
 
@@ -1471,6 +1475,8 @@ module Synth {
     or
     result = convertCommentFromRaw(e)
     or
+    result = convertDiagnosticsFromRaw(e)
+    or
     result = convertUnspecifiedElementFromRaw(e)
   }
 
@@ -2201,6 +2207,9 @@ module Synth {
   cached
   Raw::Element convertDbLocationToRaw(TDbLocation e) { e = TDbLocation(result) }
 
+  cached
+  Raw::Element convertDiagnosticsToRaw(TDiagnostics e) { e = TDiagnostics(result) }
+
   cached
   Raw::Element convertUnknownFileToRaw(TUnknownFile e) { none() }
 
@@ -3176,6 +3185,8 @@ module Synth {
     or
     result = convertCommentToRaw(e)
     or
+    result = convertDiagnosticsToRaw(e)
+    or
     result = convertUnspecifiedElementToRaw(e)
   }
 

swift/ql/lib/codeql/swift/generated/SynthConstructors.qll

@@ -2,6 +2,7 @@
 import codeql.swift.elements.CommentConstructor
 import codeql.swift.elements.DbFileConstructor
 import codeql.swift.elements.DbLocationConstructor
+import codeql.swift.elements.DiagnosticsConstructor
 import codeql.swift.elements.UnspecifiedElementConstructor
 import codeql.swift.elements.decl.AccessorDeclConstructor
 import codeql.swift.elements.decl.AssociatedTypeDeclConstructor

swift/ql/lib/codeql/swift/printast/PrintAstNode.qll

@@ -20,7 +20,7 @@ class PrintAstConfiguration extends TPrintAstConfiguration {
   /**
    * Holds if the AST for `e` should be printed. By default, holds for all.
    */
-  predicate shouldPrint(Locatable e) { any() }
+  predicate shouldPrint(Locatable e) { not e instanceof Diagnostics }
 }
 
 private predicate shouldPrint(Locatable e) { any(PrintAstConfiguration config).shouldPrint(e) }

swift/ql/lib/swift.dbscheme

@@ -66,6 +66,7 @@ files(
   @argument
 | @ast_node
 | @comment
+| @diagnostics
 | @unspecified_element
 ;
 
@@ -125,6 +126,12 @@ db_locations(
   unique int id: @db_location
 );
 
+diagnostics(
+  unique int id: @diagnostics,
+  string text: string ref,
+  int kind: int ref
+);
+
 unknown_files(
   unique int id: @unknown_file
 );

swift/ql/test/extractor-tests/generated/Diagnostics/Diagnostics.expected

@@ -0,0 +1,3 @@
+| diags.swift:1:10:1:10 | Hello, warning | getText: | Hello, warning | getKind: | 2 |
+| error.swift:2:14:2:14 | cannot convert value of type 'String' to specified type 'Int' | getText: | cannot convert value of type 'String' to specified type 'Int' | getKind: | 1 |
+| import-error.swift:2:8:2:8 | no such module 'FooBar' | getText: | no such module 'FooBar' | getKind: | 1 |

swift/ql/test/extractor-tests/generated/Diagnostics/Diagnostics.ql

@@ -0,0 +1,11 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from Diagnostics x, string getText, int getKind
+where
+  toBeTested(x) and
+  not x.isUnknown() and
+  getText = x.getText() and
+  getKind = x.getKind()
+select x, "getText:", getText, "getKind:", getKind

swift/ql/test/extractor-tests/generated/Diagnostics/diags.swift

@@ -0,0 +1 @@
+#warning("Hello, warning")

swift/ql/test/extractor-tests/generated/Diagnostics/error.swift

@@ -0,0 +1,2 @@
+//codeql-extractor-expected-status: 1
+let x: Int = "Hello"

swift/ql/test/extractor-tests/generated/Diagnostics/import-error.swift

@@ -0,0 +1,2 @@
+//codeql-extractor-expected-status: 1
+import FooBar

swift/schema.py

@@ -47,6 +47,10 @@ class UnspecifiedElement(Locatable):
 class Comment(Locatable):
     text: string
 
+class Diagnostics(Locatable):
+    text: string
+    kind: int
+
 class DbFile(File):
     pass