Merge pull request #12860 from github/redsun82/swift-logging-assertions-and-prints

Swift: replace assertions and direct prints with proper logging

misc/codegen/templates/cpp_classes_h.mustache

@@ -42,34 +42,12 @@ struct {{name}}{{#has_bases}} : {{#bases}}{{^first}}, {{/first}}{{ref.name}}{{/b
  {{/final}}
   template <typename F>
   void forEachLabel(F f) {
-    {{#final}}
-    f("id", -1, id);
-    {{/final}}
-    {{#bases}}
-    {{ref.name}}::forEachLabel(f);
-    {{/bases}}
-    {{#fields}}
-    {{#is_label}}
-    {{#is_repeated}}
-    for (auto i = 0u; i < {{field_name}}.size(); ++i) {
-      {{#is_optional}}
-      if ({{field_name}}[i]) f("{{field_name}}", i, *{{field_name}}[i]);
-      {{/is_optional}}
-      {{^is_optional}}
-      f("{{field_name}}", i, {{field_name}}[i]);
-      {{/is_optional}}
-    }
-    {{/is_repeated}}
-    {{^is_repeated}}
-    {{#is_optional}}
-    if ({{field_name}}) f("{{field_name}}", -1, *{{field_name}});
-    {{/is_optional}}
-    {{^is_optional}}
-    f("{{field_name}}", -1, {{field_name}});
-    {{/is_optional}}
-    {{/is_repeated}}
-    {{/is_label}}
-    {{/fields}}
+    {{>cpp_for_each_label_body}}
+  }
+
+  template <typename F>
+  void forEachLabel(F f) const {
+    {{>cpp_for_each_label_body}}
   }
 
  protected:

misc/codegen/templates/cpp_for_each_label_body.mustache

@@ -0,0 +1,28 @@
+{{#final}}
+f("id", -1, id);
+{{/final}}
+{{#bases}}
+{{ref.name}}::forEachLabel(f);
+{{/bases}}
+{{#fields}}
+{{#is_label}}
+{{#is_repeated}}
+for (auto i = 0u; i < {{field_name}}.size(); ++i) {
+  {{#is_optional}}
+  if ({{field_name}}[i]) f("{{field_name}}", i, *{{field_name}}[i]);
+  {{/is_optional}}
+  {{^is_optional}}
+  f("{{field_name}}", i, {{field_name}}[i]);
+  {{/is_optional}}
+}
+{{/is_repeated}}
+{{^is_repeated}}
+{{#is_optional}}
+if ({{field_name}}) f("{{field_name}}", -1, *{{field_name}});
+{{/is_optional}}
+{{^is_optional}}
+f("{{field_name}}", -1, {{field_name}});
+{{/is_optional}}
+{{/is_repeated}}
+{{/is_label}}
+{{/fields}}

misc/codegen/templates/trap_traps_h.mustache

@@ -23,11 +23,12 @@ struct {{name}}Trap {
 
   template <typename F>
   void forEachLabel(F f) {
-    {{#fields}}
-    {{#is_label}}
-    f("{{field_name}}", -1, {{field_name}});
-    {{/is_label}}
-    {{/fields}}
+    {{>cpp_for_each_label_body}}
+  }
+
+  template <typename F>
+  void forEachLabel(F f) const {
+    {{>cpp_for_each_label_body}}
   }
 };
 

swift/extractor/SwiftExtractor.cpp

@@ -5,6 +5,7 @@
 #include <unordered_set>
 #include <queue>
 
+#include <picosha2.h>
 #include <swift/AST/SourceFile.h>
 #include <swift/AST/Builtins.h>
 
@@ -14,19 +15,23 @@
 #include "swift/extractor/infra/SwiftLocationExtractor.h"
 #include "swift/extractor/infra/SwiftBodyEmissionStrategy.h"
 #include "swift/extractor/mangler/SwiftMangler.h"
-#include <picosha2.h>
+#include "swift/extractor/infra/log/SwiftAssert.h"
 
 using namespace codeql;
 using namespace std::string_literals;
 namespace fs = std::filesystem;
 
+Logger& main_logger::logger() {
+  static Logger ret{"main"};
+  return ret;
+}
+
+using namespace main_logger;
+
 static void ensureDirectory(const char* label, const fs::path& dir) {
   std::error_code ec;
   fs::create_directories(dir, ec);
-  if (ec) {
-    std::cerr << "Cannot create " << label << " directory: " << ec.message() << "\n";
-    std::abort();
-  }
+  CODEQL_ASSERT(!ec, "Cannot create {} directory ({})", label, ec);
 }
 
 static void archiveFile(const SwiftExtractorConfiguration& config, swift::SourceFile& file) {
@@ -37,10 +42,10 @@ static void archiveFile(const SwiftExtractorConfiguration& config, swift::Source
 
   std::error_code ec;
   fs::copy(source, destination, fs::copy_options::overwrite_existing, ec);
-
   if (ec) {
-    std::cerr << "Cannot archive source file " << source << " -> " << destination << ": "
-              << ec.message() << "\n";
+    LOG_INFO(
+        "Cannot archive source file {} -> {}, probably a harmless race with another process ({})",
+        source, destination, ec);
   }
 }
 
@@ -73,7 +78,7 @@ static fs::path getFilename(swift::ModuleDecl& module,
   if (lazyDeclaration) {
     // this code will be thrown away in the near future
     auto decl = llvm::dyn_cast<swift::ValueDecl>(lazyDeclaration);
-    assert(decl);
+    CODEQL_ASSERT(decl, "not a ValueDecl");
     auto mangled = mangledDeclName(*decl);
     // mangled name can be too long to use as a file name, so we can't use it directly
     mangled = picosha2::hash256_hex_string(mangled);
@@ -276,11 +281,9 @@ void codeql::extractExtractLazyDeclarations(SwiftExtractorState& state,
   // Just in case
   const int upperBound = 100;
   int iteration = 0;
-  while (!state.pendingDeclarations.empty() && iteration++ < upperBound) {
+  while (!state.pendingDeclarations.empty()) {
+    CODEQL_ASSERT(iteration++ < upperBound,
+                  "Swift extractor reached upper bound while extracting lazy declarations");
     extractLazy(state, compiler);
   }
-  if (iteration >= upperBound) {
-    std::cerr << "Swift extractor reached upper bound while extracting lazy declarations\n";
-    abort();
-  }
 }

swift/extractor/SwiftExtractor.h

@@ -8,4 +8,9 @@
 namespace codeql {
 void extractSwiftFiles(SwiftExtractorState& state, swift::CompilerInstance& compiler);
 void extractExtractLazyDeclarations(SwiftExtractorState& state, swift::CompilerInstance& compiler);
+
+class Logger;
+namespace main_logger {
+Logger& logger();
+}
 }  // namespace codeql

swift/extractor/infra/SwiftDispatcher.h

@@ -13,7 +13,7 @@
 #include "swift/extractor/infra/SwiftBodyEmissionStrategy.h"
 #include "swift/extractor/infra/SwiftMangledName.h"
 #include "swift/extractor/config/SwiftExtractorState.h"
-#include "swift/extractor/infra/log/SwiftLogging.h"
+#include "swift/extractor/infra/log/SwiftAssert.h"
 
 namespace codeql {
 
@@ -74,21 +74,20 @@ class SwiftDispatcher {
     entry.forEachLabel([&valid, &entry, this](const char* field, int index, auto& label) {
       using Label = std::remove_reference_t<decltype(label)>;
       if (!label.valid()) {
-        std::cerr << entry.NAME << " has undefined " << field;
-        if (index >= 0) {
-          std::cerr << '[' << index << ']';
-        }
+        const char* action;
         if constexpr (std::is_base_of_v<typename Label::Tag, UnspecifiedElementTag>) {
-          std::cerr << ", replacing with unspecified element\n";
+          action = "replacing with unspecified element";
           label = emitUnspecified(idOf(entry), field, index);
         } else {
-          std::cerr << ", skipping emission\n";
+          action = "skipping emission";
           valid = false;
         }
+        LOG_ERROR("{} has undefined field {}{}, {}", entry.NAME, field,
+                  index >= 0 ? ('[' + std::to_string(index) + ']') : "", action);
       }
     });
     if (valid) {
-      trap.emit(entry);
+      trap.emit(entry, /* check */ false);
     }
   }
 
@@ -153,7 +152,7 @@ class SwiftDispatcher {
     auto& stored = store[e];
     if (!stored.valid()) {
       auto inserted = fetching.insert(e);
-      assert(inserted.second && "detected infinite fetchLabel recursion");
+      CODEQL_ASSERT(inserted.second, "detected infinite fetchLabel recursion");
       stored = createLabel(e, type);
       fetching.erase(e);
     }
@@ -165,7 +164,13 @@ class SwiftDispatcher {
   TrapLabel<TypeTag> fetchLabel(swift::Type t) { return fetchLabel(t.getPointer()); }
 
   TrapLabel<AstNodeTag> fetchLabel(swift::ASTNode node) {
-    return fetchLabelFromUnion<AstNodeTag>(node);
+    auto ret = fetchLabelFromUnion<AstNodeTag>(node);
+    if (!ret.valid()) {
+      // TODO to be more useful, we need a generic way of attaching original source location info
+      // to logs, this will come in upcoming work
+      LOG_ERROR("Unable to fetch label for ASTNode");
+    }
+    return ret;
   }
 
   template <typename E, std::enable_if_t<IsFetchable<E*>>* = nullptr>
@@ -177,7 +182,7 @@ class SwiftDispatcher {
   template <typename E>
   auto createEntry(const E& e) {
     auto found = store.find(&e);
-    assert(found != store.end() && "createEntry called on non-fetched label");
+    CODEQL_ASSERT(found != store.end(), "createEntry called on non-fetched label");
     auto label = TrapLabel<ConcreteTrapTagOf<E>>::unsafeCreateFromUntyped(found->second);
     if constexpr (IsLocatable<E>) {
       locationExtractor.attachLocation(sourceManager, e, label);
@@ -307,7 +312,6 @@ class SwiftDispatcher {
     // with logical op short-circuiting, this will stop trying on the first successful fetch
     bool unionCaseFound = (... || fetchLabelFromUnionCase<Tag, Ts>(u, ret));
     if (!unionCaseFound) {
-      // TODO emit error/warning here
       return undefined_label;
     }
     return ret;
@@ -357,7 +361,10 @@ class SwiftDispatcher {
   SwiftLocationExtractor& locationExtractor;
   SwiftBodyEmissionStrategy& bodyEmissionStrategy;
   std::unordered_set<swift::ModuleDecl*> encounteredModules;
-  Logger logger{"dispatcher"};
+  Logger& logger() {
+    static Logger ret{"dispatcher"};
+    return ret;
+  }
 };
 
 }  // namespace codeql

swift/extractor/infra/file/BUILD.bazel

@@ -2,9 +2,10 @@ load("//swift:rules.bzl", "swift_cc_library")
 
 swift_cc_library(
     name = "file",
-    srcs = glob(["*.cpp"]),
-    hdrs = glob(["*.h"]) + [":path_hash_workaround"],
+    srcs = glob(["*.cpp", "FsLogger.h"]),
+    hdrs = glob(["*.h"], exclude=["FsLogger.h"]) + [":path_hash_workaround"],
     visibility = ["//swift:__subpackages__"],
+    deps = ["//swift/extractor/infra/log"],
 )
 
 genrule(

swift/extractor/infra/file/FsLogger.h

@@ -0,0 +1,10 @@
+#include "swift/extractor/infra/log/SwiftLogging.h"
+
+namespace codeql {
+namespace fs_logger {
+inline Logger& logger() {
+  static Logger ret{"fs"};
+  return ret;
+}
+}  // namespace fs_logger
+}  // namespace codeql

swift/extractor/infra/file/Path.cpp

@@ -1,17 +1,21 @@
 #include "swift/extractor/infra/file/Path.h"
+
 #include <iostream>
 #include <unistd.h>
 
+#include "swift/extractor/infra/file/FsLogger.h"
+
+using namespace std::string_view_literals;
+
 namespace codeql {
 
+using namespace fs_logger;
+
 static bool shouldCanonicalize() {
-  auto preserve = getenv("CODEQL_PRESERVE_SYMLINKS");
-  if (preserve && std::string(preserve) == "true") {
-    return false;
-  }
-  preserve = getenv("SEMMLE_PRESERVE_SYMLINKS");
-  if (preserve && std::string(preserve) == "true") {
-    return false;
+  for (auto var : {"CODEQL_PRESERVE_SYMLINKS", "SEMMLE_PRESERVE_SYMLINKS"}) {
+    if (auto preserve = getenv(var); preserve && preserve == "true"sv) {
+      return false;
+    }
   }
   return true;
 }
@@ -26,8 +30,13 @@ std::filesystem::path resolvePath(const std::filesystem::path& path) {
     ret = std::filesystem::absolute(path, ec);
   }
   if (ec) {
-    std::cerr << "Cannot get " << (canonicalize ? "canonical" : "absolute") << " path: " << path
-              << ": " << ec.message() << "\n";
+    if (ec.value() == ENOENT) {
+      // this is pretty normal, nothing to spam about
+      LOG_DEBUG("resolving non-existing {}", path);
+    } else {
+      LOG_WARNING("cannot get {} path for {} ({})", canonicalize ? "canonical" : "absolute", path,
+                  ec);
+    }
     return path;
   }
   return ret;

swift/extractor/infra/file/TargetFile.cpp

@@ -1,6 +1,8 @@
 #include "swift/extractor/infra/file/TargetFile.h"
+#include "swift/extractor/infra/file/FsLogger.h"
+#include "swift/extractor/infra/log/SwiftLogging.h"
+#include "swift/extractor/infra/log/SwiftAssert.h"
 
-#include <iostream>
 #include <cassert>
 #include <cstdio>
 #include <cerrno>
@@ -10,32 +12,24 @@
 namespace fs = std::filesystem;
 
 namespace codeql {
+
+using namespace fs_logger;
+
 namespace {
-[[noreturn]] void error(const char* action, const fs::path& arg, std::error_code ec) {
-  std::cerr << "Unable to " << action << ": " << arg << " (" << ec.message() << ")\n";
-  std::abort();
-}
-
-[[noreturn]] void error(const char* action, const fs::path& arg) {
-  error(action, arg, {errno, std::system_category()});
-}
-
-void check(const char* action, const fs::path& arg, std::error_code ec) {
-  if (ec) {
-    error(action, arg, ec);
-  }
+std::error_code currentErrorCode() {
+  return {errno, std::system_category()};
 }
 
 void ensureParentDir(const fs::path& path) {
   auto parent = path.parent_path();
   std::error_code ec;
   fs::create_directories(parent, ec);
-  check("create directory", parent, ec);
+  CODEQL_ASSERT(!ec, "Unable to create directory {} ({})", parent, ec);
 }
 
 fs::path initPath(const std::filesystem::path& target, const std::filesystem::path& dir) {
   fs::path ret{dir};
-  assert(!target.empty() && "target must be a non-empty path");
+  CODEQL_ASSERT(!target.empty());
   ret /= target.relative_path();
   ensureParentDir(ret);
   return ret;
@@ -53,13 +47,12 @@ bool TargetFile::init() {
   if (auto f = std::fopen(targetPath.c_str(), "wx")) {
     std::fclose(f);
     out.open(workingPath);
-    checkOutput("open file for writing");
+    checkOutput("open");
     return true;
   }
-  if (errno != EEXIST) {
-    error("open file for writing", targetPath);
-  }
-  // else we just lost the race
+  CODEQL_ASSERT(errno == EEXIST, "Unable to open {} for writing ({})", targetPath,
+                currentErrorCode());
+  // else the file already exists and we just lost the race
   return false;
 }
 
@@ -76,13 +69,11 @@ void TargetFile::commit() {
     out.close();
     std::error_code ec;
     fs::rename(workingPath, targetPath, ec);
-    check("rename file", targetPath, ec);
+    CODEQL_ASSERT(!ec, "Unable to rename {} -> {} ({})", workingPath, targetPath, ec);
   }
 }
 
 void TargetFile::checkOutput(const char* action) {
-  if (!out) {
-    error(action, workingPath);
-  }
+  CODEQL_ASSERT(out, "Unable to {} {} ({})", action, workingPath, currentErrorCode());
 }
 }  // namespace codeql

swift/extractor/infra/file/TargetFile.h

@@ -32,7 +32,7 @@ class TargetFile {
   TargetFile& operator<<(T&& value) {
     errno = 0;
     out << value;
-    checkOutput("write to file");
+    checkOutput("write to");
     return *this;
   }
 

swift/extractor/infra/log/SwiftAssert.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <cstdlib>
+
+#include "swift/extractor/infra/log/SwiftLogging.h"
+
+// assert CONDITION, which is always evaluated (once) regardless of the build type. If
+// CONDITION is not satisfied, emit a critical log optionally using provided format and arguments,
+// abort the program
+#define CODEQL_ASSERT(CONDITION, ...) \
+  CODEQL_ASSERT_IMPL(CRITICAL, std::abort(), CONDITION, __VA_ARGS__)
+
+// If CONDITION is not satisfied, emit an error log optionally using provided format and arguments,
+// but continue execution
+#define CODEQL_EXPECT(CONDITION, ...) CODEQL_EXPECT_OR(void(), CONDITION, __VA_ARGS__)
+
+// If CONDITION is not satisfied, emit an error log optionally using provided format and arguments,
+// and execute ACTION (for example return)
+#define CODEQL_EXPECT_OR(ACTION, CONDITION, ...) \
+  CODEQL_ASSERT_IMPL(ERROR, ACTION, CONDITION, __VA_ARGS__)
+
+#define CODEQL_ASSERT_IMPL(LEVEL, ACTION, CONDITION, ...)                           \
+  do {                                                                              \
+    if (!(CONDITION)) {                                                             \
+      [[unlikely]] LOG_##LEVEL("assertion failed on " #CONDITION ". " __VA_ARGS__); \
+      codeql::Log::flush();                                                         \
+      ACTION;                                                                       \
+    }                                                                               \
+  } while (false)

swift/extractor/infra/log/SwiftLogging.cpp

@@ -134,6 +134,12 @@ void Log::configure() {
 
 void Log::flushImpl() {
   session.consume(*this);
+  if (text) {
+    textFile.flush();
+  }
+  if (binary) {
+    binary.output.flush();
+  }
 }
 
 Log::LoggerConfiguration Log::getLoggerConfigurationImpl(std::string_view name) {

swift/extractor/infra/log/SwiftLogging.h

@@ -9,6 +9,7 @@
 #include <binlog/TextOutputStream.hpp>
 #include <binlog/EventFilter.hpp>
 #include <binlog/adapt_stdfilesystem.hpp>
+#include <binlog/adapt_stderrorcode.hpp>
 #include <binlog/adapt_stdoptional.hpp>
 #include <binlog/adapt_stdvariant.hpp>
 

swift/extractor/invocation/SwiftInvocationExtractor.cpp

@@ -4,6 +4,7 @@
 #include "swift/extractor/trap/generated/TrapTags.h"
 #include "swift/extractor/infra/file/TargetFile.h"
 #include "swift/extractor/infra/file/Path.h"
+#include "swift/extractor/infra/log/SwiftAssert.h"
 #include "swift/extractor/mangler/SwiftMangler.h"
 
 namespace fs = std::filesystem;
@@ -11,6 +12,12 @@ using namespace std::string_literals;
 
 namespace codeql {
 namespace {
+
+Logger& logger() {
+  static Logger ret{"invocation"};
+  return ret;
+}
+
 std::string getModuleId(const std::string_view& name, const std::string_view& hash) {
   auto ret = "module:"s;
   ret += name;
@@ -109,10 +116,8 @@ void replaceMergedModulesImplementation(const SwiftExtractorState& state,
   fs::copy(getTrapPath(state, mergeTarget, TrapType::linkage),
            getTrapPath(state, mergedPartTarget, TrapType::linkage),
            fs::copy_options::overwrite_existing, ec);
-  if (ec) {
-    std::cerr << "unable to replace trap implementation id for merged module '" << name << "' ("
-              << ec.message() << ")";
-  }
+  CODEQL_ASSERT(!ec, "Unable to replace trap implementation id for merged module '{}' ({})", name,
+                ec);
 }
 
 void emitModuleObjectDependencies(const SwiftExtractorState& state,

swift/extractor/main.cpp

@@ -18,9 +18,10 @@
 #include "swift/extractor/invocation/SwiftInvocationExtractor.h"
 #include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/infra/file/Path.h"
-#include "swift/extractor/infra/log/SwiftLogging.h"
+#include "swift/extractor/infra/log/SwiftAssert.h"
 
 using namespace std::string_literals;
+using namespace codeql::main_logger;
 
 const std::string_view codeql::logRootName = "extractor";
 
@@ -32,9 +33,10 @@ static void lockOutputSwiftModuleTraps(codeql::SwiftExtractorState& state,
         !module.empty()) {
       if (auto target = codeql::createTargetTrapDomain(state, codeql::resolvePath(module),
                                                        codeql::TrapType::module)) {
-        target->emit("// trap file deliberately empty\n"
-                     "// this swiftmodule was created during the build, so its entities must have"
-                     " been extracted directly from source files");
+        target->emitComment(
+            "trap file deliberately empty\n"
+            " * this swiftmodule was created during the build, so its entities must have\n"
+            " * been extracted directly from source files\n");
       }
     }
   }
@@ -45,7 +47,6 @@ static void processFrontendOptions(codeql::SwiftExtractorState& state,
   auto& inOuts = options.InputsAndOutputs;
   std::vector<swift::InputFile> inputs;
   inOuts.forEachInput([&](const swift::InputFile& input) {
-    std::cerr << input.getFileName() << ":\n";
     swift::PrimarySpecificPaths psp{};
     if (std::filesystem::path output = input.getPrimarySpecificPaths().OutputFilename;
         !output.empty()) {
@@ -142,7 +143,7 @@ static bool checkRunUnderFilter(int argc, char* const* argv) {
 // An example usage is to run the extractor under `gdbserver :1234` when the
 // arguments match a given source file.
 static void checkWhetherToRunUnderTool(int argc, char* const* argv) {
-  assert(argc > 0);
+  if (argc == 0) return;
 
   auto runUnder = getenv("CODEQL_EXTRACTOR_SWIFT_RUN_UNDER");
   if (runUnder == nullptr || !checkRunUnderFilter(argc, argv)) {
@@ -168,10 +169,7 @@ codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state) {
   auto filename = std::to_string(timestamp) + '-' + std::to_string(getpid());
   auto target = std::filesystem::path("invocations") / std::filesystem::path(filename);
   auto maybeDomain = codeql::createTargetTrapDomain(state, target, codeql::TrapType::invocation);
-  if (!maybeDomain) {
-    std::cerr << "Cannot create invocation trap file: " << target << "\n";
-    abort();
-  }
+  CODEQL_ASSERT(maybeDomain, "Cannot create invocation trap file for {}", target);
   return std::move(maybeDomain.value());
 }
 
@@ -216,11 +214,8 @@ int main(int argc, char** argv, char** envp) {
   initializeSwiftModules();
 
   const auto configuration = configure(argc, argv);
-  {
-    codeql::Logger logger{"main"};
-    LOG_INFO("calling extractor with arguments \"{}\"", argDump(argc, argv));
-    LOG_DEBUG("environment:\n{}\n", envDump(envp));
-  }
+  LOG_INFO("calling extractor with arguments \"{}\"", argDump(argc, argv));
+  LOG_DEBUG("environment:\n{}\n", envDump(envp));
 
   auto openInterception = codeql::setupFileInterception(configuration);
 

swift/extractor/remapping/SwiftFileInterception.cpp

@@ -14,6 +14,7 @@
 
 #include "swift/extractor/infra/file/PathHash.h"
 #include "swift/extractor/infra/file/Path.h"
+#include "swift/extractor/infra/log/SwiftAssert.h"
 
 #ifdef __APPLE__
 // path is hardcoded as otherwise redirection could break when setting DYLD_FALLBACK_LIBRARY_PATH
@@ -27,14 +28,22 @@
 namespace fs = std::filesystem;
 
 namespace {
+
+namespace {
+codeql::Logger& logger() {
+  static codeql::Logger ret{"open_interception"};
+  return ret;
+}
+}  // namespace
+
 namespace original {
 
 void* openLibC() {
   if (auto ret = dlopen(SHARED_LIBC, RTLD_LAZY)) {
     return ret;
   }
-  std::cerr << "Unable to dlopen " SHARED_LIBC "!\n";
-  std::abort();
+  LOG_CRITICAL("Unable to dlopen " SHARED_LIBC "!");
+  abort();
 }
 
 void* libc() {
@@ -71,8 +80,12 @@ bool mayBeRedirected(const char* path, int flags = O_RDONLY) {
 std::optional<std::string> hashFile(const fs::path& path) {
   auto fd = original::open(path.c_str(), O_RDONLY | O_CLOEXEC);
   if (fd < 0) {
-    auto ec = std::make_error_code(static_cast<std::errc>(errno));
-    std::cerr << "unable to open " << path << " for reading (" << ec.message() << ")\n";
+    if (errno == ENOENT) {
+      LOG_DEBUG("ignoring non-existing module {}", path);
+    } else {
+      LOG_ERROR("unable to open {} for hashing ({})", path,
+                std::make_error_code(static_cast<std::errc>(errno)));
+    }
     return std::nullopt;
   }
   auto hasher = picosha2::hash256_one_by_one();
@@ -102,7 +115,7 @@ class FileInterceptor {
 
   int open(const char* path, int flags, mode_t mode = 0) const {
     fs::path fsPath{path};
-    assert((flags & O_ACCMODE) == O_RDONLY);
+    CODEQL_ASSERT((flags & O_ACCMODE) == O_RDONLY, "We should only be intercepting file reads");
     // try to use the hash map first
     errno = 0;
     if (auto hashed = hashPath(path)) {
@@ -114,15 +127,15 @@ class FileInterceptor {
   }
 
   fs::path redirect(const fs::path& target) const {
-    assert(mayBeRedirected(target.c_str()));
+    CODEQL_ASSERT(mayBeRedirected(target.c_str()), "Trying to redirect {} which is unsupported",
+                  target);
     auto redirected = redirectedPath(target);
     fs::create_directories(redirected.parent_path());
     if (auto hashed = hashPath(target)) {
       std::error_code ec;
       fs::create_symlink(*hashed, redirected, ec);
-      if (ec) {
-        std::cerr << "Cannot remap file " << *hashed << " -> " << redirected << ": " << ec.message()
-                  << "\n";
+      if (ec && ec.value() != ENOENT) {
+        LOG_WARNING("Cannot remap file {} -> {} ({})", *hashed, redirected, ec);
       }
       return *hashed;
     }

swift/extractor/translators/DeclTranslator.cpp

@@ -85,7 +85,6 @@ codeql::ParamDecl DeclTranslator::translateParamDecl(const swift::ParamDecl& dec
 codeql::TopLevelCodeDecl DeclTranslator::translateTopLevelCodeDecl(
     const swift::TopLevelCodeDecl& decl) {
   auto entry = createEntry(decl);
-  assert(decl.getBody() && "Expect top level code to have body");
   entry.body = dispatcher.fetchLabel(decl.getBody());
   return entry;
 }
@@ -95,7 +94,6 @@ codeql::PatternBindingDecl DeclTranslator::translatePatternBindingDecl(
   auto entry = createEntry(decl);
   for (unsigned i = 0; i < decl.getNumPatternEntries(); ++i) {
     auto pattern = decl.getPattern(i);
-    assert(pattern && "Expect pattern binding decl to have all patterns");
     entry.patterns.push_back(dispatcher.fetchLabel(pattern));
     entry.inits.push_back(dispatcher.fetchOptionalLabel(decl.getInit(i)));
   }
@@ -252,9 +250,10 @@ codeql::ModuleDecl DeclTranslator::translateModuleDecl(const swift::ModuleDecl&
 
 void DeclTranslator::fillFunction(const swift::AbstractFunctionDecl& decl,
                                   codeql::Function& entry) {
-  assert(decl.hasParameterList() && "Expect functions to have a parameter list");
   entry.name = !decl.hasName() ? "(unnamed function decl)" : constructName(decl.getName());
   entry.body = dispatcher.fetchOptionalLabel(decl.getBody());
+  CODEQL_EXPECT_OR(return, decl.hasParameterList(), "Function {} has no parameter list",
+                         entry.name);
   entry.params = dispatcher.fetchRepeatedLabels(*decl.getParameters());
   auto self = const_cast<swift::ParamDecl* const>(decl.getImplicitSelfDecl());
   entry.self_param = dispatcher.fetchOptionalLabel(self);
@@ -321,7 +320,6 @@ void DeclTranslator::fillGenericContext(const swift::GenericContext& decl,
 }
 
 void DeclTranslator::fillValueDecl(const swift::ValueDecl& decl, codeql::ValueDecl& entry) {
-  assert(decl.getInterfaceType() && "Expect ValueDecl to have InterfaceType");
   entry.interface_type = dispatcher.fetchLabel(decl.getInterfaceType());
 }
 

swift/extractor/translators/DeclTranslator.h

@@ -14,6 +14,8 @@ namespace codeql {
 // "forward declarations" while our extraction is incomplete
 class DeclTranslator : public AstTranslatorBase<DeclTranslator> {
  public:
+  static constexpr std::string_view name = "decl";
+
   using AstTranslatorBase<DeclTranslator>::AstTranslatorBase;
 
   codeql::NamedFunction translateFuncDecl(const swift::FuncDecl& decl);

swift/extractor/translators/ExprTranslator.cpp

@@ -355,11 +355,11 @@ codeql::IsExpr ExprTranslator::translateIsExpr(const swift::IsExpr& expr) {
 codeql::SubscriptExpr ExprTranslator::translateSubscriptExpr(const swift::SubscriptExpr& expr) {
   auto entry = createExprEntry(expr);
   fillAccessorSemantics(expr, entry);
-  assert(expr.getArgs() && "SubscriptExpr has getArgs");
+  fillLookupExpr(expr, entry);
+  CODEQL_EXPECT_OR(return entry, expr.getArgs(), "SubscriptExpr has null getArgs");
   for (const auto& arg : *expr.getArgs()) {
     entry.arguments.push_back(emitArgument(arg));
   }
-  fillLookupExpr(expr, entry);
   return entry;
 }
 
@@ -384,9 +384,10 @@ codeql::KeyPathExpr ExprTranslator::translateKeyPathExpr(const swift::KeyPathExp
     }
     if (auto rootTypeRepr = expr.getRootType()) {
       auto keyPathType = expr.getType()->getAs<swift::BoundGenericClassType>();
-      assert(keyPathType && "KeyPathExpr must have BoundGenericClassType");
+      CODEQL_EXPECT_OR(return entry, keyPathType, "KeyPathExpr must have BoundGenericClassType");
       auto keyPathTypeArgs = keyPathType->getGenericArgs();
-      assert(keyPathTypeArgs.size() != 0 && "KeyPathExpr type must have generic args");
+      CODEQL_EXPECT_OR(return entry, keyPathTypeArgs.size() != 0,
+                              "KeyPathExpr type must have generic args");
       entry.root = dispatcher.fetchLabel(rootTypeRepr, keyPathTypeArgs[0]);
     }
   }
@@ -474,10 +475,10 @@ codeql::ErrorExpr ExprTranslator::translateErrorExpr(const swift::ErrorExpr& exp
 
 void ExprTranslator::fillClosureExpr(const swift::AbstractClosureExpr& expr,
                                      codeql::ClosureExpr& entry) {
-  assert(expr.getParameters() && "AbstractClosureExpr has getParameters()");
-  entry.params = dispatcher.fetchRepeatedLabels(*expr.getParameters());
   entry.body = dispatcher.fetchLabel(expr.getBody());
   entry.captures = dispatcher.fetchRepeatedLabels(expr.getCaptureInfo().getCaptures());
+  CODEQL_EXPECT_OR(return, expr.getParameters(), "AbstractClosureExpr has null getParameters()");
+  entry.params = dispatcher.fetchRepeatedLabels(*expr.getParameters());
 }
 
 TrapLabel<ArgumentTag> ExprTranslator::emitArgument(const swift::Argument& arg) {
@@ -524,7 +525,7 @@ void ExprTranslator::fillAnyTryExpr(const swift::AnyTryExpr& expr, codeql::AnyTr
 
 void ExprTranslator::fillApplyExpr(const swift::ApplyExpr& expr, codeql::ApplyExpr& entry) {
   entry.function = dispatcher.fetchLabel(expr.getFn());
-  assert(expr.getArgs() && "ApplyExpr has getArgs");
+  CODEQL_EXPECT_OR(return, expr.getArgs(), "ApplyExpr has null getArgs");
   for (const auto& arg : *expr.getArgs()) {
     entry.arguments.push_back(emitArgument(arg));
   }

swift/extractor/translators/ExprTranslator.h

@@ -7,6 +7,8 @@ namespace codeql {
 
 class ExprTranslator : public AstTranslatorBase<ExprTranslator> {
  public:
+  static constexpr std::string_view name = "expr";
+
   using AstTranslatorBase<ExprTranslator>::AstTranslatorBase;
 
   codeql::IntegerLiteralExpr translateIntegerLiteralExpr(const swift::IntegerLiteralExpr& expr);

swift/extractor/translators/PatternTranslator.h

@@ -7,6 +7,8 @@ namespace codeql {
 
 class PatternTranslator : public AstTranslatorBase<PatternTranslator> {
  public:
+  static constexpr std::string_view name = "pattern";
+
   using AstTranslatorBase<PatternTranslator>::AstTranslatorBase;
 
   codeql::NamedPattern translateNamedPattern(const swift::NamedPattern& pattern);

swift/extractor/translators/StmtTranslator.h

@@ -7,6 +7,8 @@ namespace codeql {
 
 class StmtTranslator : public AstTranslatorBase<StmtTranslator> {
  public:
+  static constexpr std::string_view name = "stmt";
+
   using AstTranslatorBase<StmtTranslator>::AstTranslatorBase;
   using AstTranslatorBase<StmtTranslator>::translateAndEmit;
 

swift/extractor/translators/TranslatorBase.h

@@ -11,10 +11,11 @@ namespace detail {
 class TranslatorBase {
  protected:
   SwiftDispatcher& dispatcher;
+  Logger logger;
 
- public:
   // SwiftDispatcher should outlive this instance
-  TranslatorBase(SwiftDispatcher& dispatcher) : dispatcher{dispatcher} {}
+  TranslatorBase(SwiftDispatcher& dispatcher, std::string_view name)
+      : dispatcher{dispatcher}, logger{"translator/" + std::string(name)} {}
 };
 
 // define by macro metaprogramming member checkers
@@ -90,7 +91,7 @@ enum class TranslatorPolicy {
   void visit##CLASS##KIND(swift::CLASS##KIND* e) {                                   \
     constexpr auto policy = getPolicyFor##CLASS##KIND();                             \
     if constexpr (policy == TranslatorPolicy::ignore) {                              \
-      std::cerr << "Unexpected " #CLASS #KIND "\n";                                  \
+      LOG_ERROR("Unexpected " #CLASS #KIND);                                         \
       return;                                                                        \
     } else if constexpr (policy == TranslatorPolicy::translate) {                    \
       dispatcher.emit(static_cast<CrtpSubclass*>(this)->translate##CLASS##KIND(*e)); \
@@ -108,7 +109,7 @@ template <typename CrtpSubclass>
 class AstTranslatorBase : private swift::ASTVisitor<CrtpSubclass>,
                           protected detail::TranslatorBase {
  public:
-  using TranslatorBase::TranslatorBase;
+  AstTranslatorBase(SwiftDispatcher& dispatcher) : TranslatorBase(dispatcher, CrtpSubclass::name) {}
 
   // swift does not provide const visitors. The following const_cast is safe, as we privately
   // route the visit to translateXXX functions only if they take const references to swift
@@ -145,7 +146,8 @@ template <typename CrtpSubclass>
 class TypeTranslatorBase : private swift::TypeVisitor<CrtpSubclass>,
                            protected detail::TranslatorBase {
  public:
-  using TranslatorBase::TranslatorBase;
+  TypeTranslatorBase(SwiftDispatcher& dispatcher)
+      : TranslatorBase(dispatcher, CrtpSubclass::name) {}
 
   // swift does not provide const visitors. The following const_cast is safe, as we privately
   // route the visit to translateXXX functions only if they take const references to swift

swift/extractor/translators/TypeTranslator.h

@@ -6,6 +6,8 @@
 namespace codeql {
 class TypeTranslator : public TypeTranslatorBase<TypeTranslator> {
  public:
+  static constexpr std::string_view name = "type";
+
   using TypeTranslatorBase<TypeTranslator>::TypeTranslatorBase;
   using TypeTranslatorBase<TypeTranslator>::translateAndEmit;
 

swift/extractor/trap/TrapDomain.h

@@ -21,11 +21,26 @@ class TrapDomain {
   }
 
   template <typename Entry>
-  void emit(const Entry& e) {
+  void emit(const Entry& e, bool check = true) {
     LOG_TRACE("{}", e);
+    if (check) {
+      e.forEachLabel([&e, this](const char* field, int index, auto& label) {
+        if (!label.valid()) {
+          LOG_ERROR("{} has undefined field {}{}", e.NAME, field,
+                    index >= 0 ? ('[' + std::to_string(index) + ']') : "");
+        }
+      });
+    }
     out << e << '\n';
   }
 
+  template <typename... Args>
+  void emitComment(const Args&... args) {
+    out << "/* ";
+    (out << ... << args);
+    out << " */\n";
+  }
+
   template <typename... Args>
   void debug(const Args&... args) {
     out << "/* DEBUG:\n";

swift/extractor/trap/TrapLabel.h

@@ -35,10 +35,6 @@ class UntypedTrapLabel {
   explicit operator bool() const { return valid(); }
 
   friend std::ostream& operator<<(std::ostream& out, UntypedTrapLabel l) {
-    // TODO: this is a temporary fix to catch us from outputting undefined labels to trap
-    // this should be moved to a validity check, probably aided by code generation and carried out
-    // by `SwiftDispatcher`
-    assert(l && "outputting an undefined label!");
     out << '#' << std::hex << l.id_ << std::dec;
     return out;
   }

swift/tools/qltest.sh

@@ -6,6 +6,7 @@ QLTEST_LOG="$CODEQL_EXTRACTOR_SWIFT_LOG_DIR"/qltest.log
 
 EXTRACTOR="$CODEQL_EXTRACTOR_SWIFT_ROOT/tools/$CODEQL_PLATFORM/extractor"
 SDK="$CODEQL_EXTRACTOR_SWIFT_ROOT/qltest/$CODEQL_PLATFORM/sdk"
+export CODEQL_EXTRACTOR_SWIFT_LOG_LEVELS=${CODEQL_EXTRACTOR_SWIFT_LOG_LEVELS:-out:text:no_logs,out:console:info}
 
 for src in *.swift; do
   env=()
@@ -14,7 +15,7 @@ for src in *.swift; do
   expected_status=$(sed -n 's=//codeql-extractor-expected-status:[[:space:]]*==p' $src)
   expected_status=${expected_status:-0}
   env+=($(sed -n '1 s=//codeql-extractor-env:==p' $src))
-  echo -e "calling extractor with flags: ${opts[@]}\n" >> $QLTEST_LOG
+  echo >> $QLTEST_LOG
   env "${env[@]}" "$EXTRACTOR" "${opts[@]}" >> $QLTEST_LOG 2>&1
   actual_status=$?
   if [[ $actual_status != $expected_status ]]; then