hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13979 from github/alexdenisov/autobuilder-spm

Browse Source 
Swift: teach autobuilder about SPM, CocoaPods, and Carthage
This commit is contained in:
AlexDenisov
2023-08-24 08:50:04 +02:00
committed by GitHub
parent 96e9dfc7b2 5cce37baa9
commit d89a86fea4
12 changed files with 159 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
swift/integration-tests/osx-only/autobuilder/failure/diagnostics.expected
View File

@@ -1,5 +1,5 @@
{
"markdownMessage": "`autobuild` failed to run the detected build command:\n\n```\n/usr/bin/xcodebuild build -project <test-root-directory>/hello-failure.xcodeproj -target hello-failure CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO\n```\n\nSet up a [manual build command][1] or [check the logs of the autobuild step][2].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language\n[2]: https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/using-workflow-run-logs",
"markdownMessage": "`autobuild` failed to run the build command:\n\n```\n/usr/bin/xcodebuild build -project <test-root-directory>/hello-failure.xcodeproj -target hello-failure CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO\n```\n\nSet up a [manual build command][1] or [check the logs of the autobuild step][2].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language\n[2]: https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/using-workflow-run-logs",
"severity": "error",
"source": {
"extractorName": "swift",

2
swift/integration-tests/osx-only/autobuilder/no-build-system/diagnostics.expected
View File

@@ -1,5 +1,5 @@
{
"markdownMessage": "`autobuild` could not detect an Xcode project or workspace.\n\nSet up a [manual build command][1].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language",
"markdownMessage": "`autobuild` detected neither an Xcode project or workspace, nor a Swift package\n\nSet up a [manual build command][1].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language",
"severity": "error",
"source": {
"extractorName": "swift",

6
swift/integration-tests/osx-only/autobuilder/no-swift-with-spm/diagnostics.expected
View File

@@ -1,10 +1,10 @@
{
"markdownMessage": "A Swift package was detected, but no viable Xcode target was found.\n\nSwift Package Manager builds are not currently supported by `autobuild`. Set up a [manual build command][1].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language",
"markdownMessage": "`autobuild` failed to run the build command:\n\n```\n/usr/bin/swift build --package-path <test-root-directory>/hello-objective\n```\n\nSet up a [manual build command][1] or [check the logs of the autobuild step][2].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language\n[2]: https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/using-workflow-run-logs",
"severity": "error",
"source": {
"extractorName": "swift",
"id": "swift/autobuilder/spm-not-supported",
"name": "Swift Package Manager is not supported"
"id": "swift/autobuilder/build-command-failed",
"name": "Detected build command failed"
},
"visibility": {
"cliSummaryTable": true,

6
swift/integration-tests/osx-only/autobuilder/no-xcode-with-spm/diagnostics.expected
View File

@@ -1,10 +1,10 @@
{
"markdownMessage": "A Swift package was detected, but no viable Xcode target was found.\n\nSwift Package Manager builds are not currently supported by `autobuild`. Set up a [manual build command][1].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language",
"markdownMessage": "`autobuild` failed to run the build command:\n\n```\n/usr/bin/swift build --package-path <test-root-directory>\n```\n\nSet up a [manual build command][1] or [check the logs of the autobuild step][2].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language\n[2]: https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/using-workflow-run-logs",
"severity": "error",
"source": {
"extractorName": "swift",
"id": "swift/autobuilder/spm-not-supported",
"name": "Swift Package Manager is not supported"
"id": "swift/autobuilder/build-command-failed",
"name": "Detected build command failed"
},
"visibility": {
"cliSummaryTable": true,

6
swift/integration-tests/osx-only/autobuilder/only-tests-with-spm/diagnostics.expected
View File

@@ -1,10 +1,10 @@
{
"markdownMessage": "A Swift package was detected, but no viable Xcode target was found.\n\nSwift Package Manager builds are not currently supported by `autobuild`. Set up a [manual build command][1].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language",
"markdownMessage": "`autobuild` failed to run the build command:\n\n```\n/usr/bin/swift build --package-path <test-root-directory>\n```\n\nSet up a [manual build command][1] or [check the logs of the autobuild step][2].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language\n[2]: https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/using-workflow-run-logs",
"severity": "error",
"source": {
"extractorName": "swift",
"id": "swift/autobuilder/spm-not-supported",
"name": "Swift Package Manager is not supported"
"id": "swift/autobuilder/build-command-failed",
"name": "Detected build command failed"
},
"visibility": {
"cliSummaryTable": true,

2
swift/tools/autobuild.sh
View File

@@ -1,6 +1,8 @@
#!/bin/bash
if [[ "$OSTYPE" == "darwin"* ]]; then
export CODEQL_SWIFT_CARTHAGE_EXEC=`which carthage`
export CODEQL_SWIFT_POD_EXEC=`which pod`
exec "${CODEQL_EXTRACTOR_SWIFT_ROOT}/tools/${CODEQL_PLATFORM}/xcode-autobuilder"
else
exec "${CODEQL_EXTRACTOR_SWIFT_ROOT}/tools/${CODEQL_PLATFORM}/autobuilder-incompatible-os"

58
swift/xcode-autobuilder/XcodeBuildRunner.cpp
View File

@@ -51,7 +51,21 @@ static bool exec(const std::vector<std::string>& argv) {
return true;
}
void buildTarget(Target& target, bool dryRun) {
static bool run_build_command(const std::vector<std::string>& argv, bool dryRun) {
if (dryRun) {
std::cout << absl::StrJoin(argv, " ") << "\n";
} else {
if (!exec(argv)) {
DIAGNOSE_ERROR(buildCommandFailed,
"`autobuild` failed to run the build command:\n\n```\n{}\n```",
absl::StrJoin(argv, " "));
return false;
}
}
return true;
}
bool buildXcodeTarget(const XcodeTarget& target, bool dryRun) {
std::vector<std::string> argv({"/usr/bin/xcodebuild", "build"});
if (!target.workspace.empty()) {
argv.push_back("-workspace");
@@ -65,16 +79,40 @@ void buildTarget(Target& target, bool dryRun) {
argv.push_back(target.name);
argv.push_back("CODE_SIGNING_REQUIRED=NO");
argv.push_back("CODE_SIGNING_ALLOWED=NO");
return run_build_command(argv, dryRun);
}
if (dryRun) {
std::cout << absl::StrJoin(argv, " ") << "\n";
} else {
if (!exec(argv)) {
DIAGNOSE_ERROR(buildCommandFailed,
"`autobuild` failed to run the detected build command:\n\n```\n{}\n```",
absl::StrJoin(argv, " "));
codeql::Log::flush();
exit(1);
bool buildSwiftPackage(const std::filesystem::path& packageFile, bool dryRun) {
std::vector<std::string> argv(
{"/usr/bin/swift", "build", "--package-path", packageFile.parent_path()});
return run_build_command(argv, dryRun);
}
static void pod_install(const std::string& pod, const std::filesystem::path& podfile, bool dryRun) {
std::vector<std::string> argv(
{pod, "install", "--project-directory=" + podfile.parent_path().string()});
run_build_command(argv, dryRun);
}
static void carthage_install(const std::string& carthage,
const std::filesystem::path& podfile,
bool dryRun) {
std::vector<std::string> argv(
{carthage, "bootstrap", "--project-directory", podfile.parent_path()});
run_build_command(argv, dryRun);
}
void installDependencies(const ProjectStructure& target, bool dryRun) {
auto pod = std::string(getenv("CODEQL_SWIFT_POD_EXEC") ?: "");
auto carthage = std::string(getenv("CODEQL_SWIFT_CARTHAGE_EXEC") ?: "");
if (!pod.empty() && !target.podfiles.empty()) {
for (auto& podfile : target.podfiles) {
pod_install(pod, podfile, dryRun);
}
}
if (!carthage.empty() && !target.cartfiles.empty()) {
for (auto& cartfile : target.cartfiles) {
carthage_install(carthage, cartfile, dryRun);
}
}
}

6
swift/xcode-autobuilder/XcodeBuildRunner.h
View File

@@ -1,5 +1,9 @@
#pragma once
#include "swift/xcode-autobuilder/XcodeTarget.h"
#include "swift/xcode-autobuilder/XcodeProjectParser.h"
#include <filesystem>
void buildTarget(Target& target, bool dryRun);
void installDependencies(const ProjectStructure& target, bool dryRun);
bool buildXcodeTarget(const XcodeTarget& target, bool dryRun);
bool buildSwiftPackage(const std::filesystem::path& packageFile, bool dryRun);

58
swift/xcode-autobuilder/XcodeProjectParser.cpp
View File

@@ -198,14 +198,30 @@ static std::unordered_map<std::string, TargetData> mapTargetsToWorkspace(
return targetMapping;
}
static std::vector<fs::path> collectFiles(const std::string& workingDir) {
struct ProjectFiles {
std::vector<fs::path> xcodeFiles;
std::vector<fs::path> packageFiles;
std::vector<fs::path> podfiles;
std::vector<fs::path> cartfiles;
};
static ProjectFiles scanWorkingDir(const std::string& workingDir) {
ProjectFiles structure;
fs::path workDir(workingDir);
std::vector<fs::path> files;
auto end = fs::recursive_directory_iterator();
for (auto it = fs::recursive_directory_iterator(workDir); it != end; ++it) {
const auto& p = it->path();
if (p.filename() == "Package.swift") {
files.push_back(p);
structure.packageFiles.push_back(p);
continue;
}
if (p.filename() == "Podfile") {
structure.podfiles.push_back(p);
continue;
}
if (p.filename() == "Cartfile" || p.filename() == "Cartfile.private") {
structure.cartfiles.push_back(p);
continue;
}
if (!it->is_directory()) {
@@ -217,43 +233,31 @@ static std::vector<fs::path> collectFiles(const std::string& workingDir) {
continue;
}
if (p.extension() == ".xcodeproj" || p.extension() == ".xcworkspace") {
files.push_back(p);
structure.xcodeFiles.push_back(p);
}
}
return files;
return structure;
}
static std::unordered_map<std::string, std::vector<std::string>> collectWorkspaces(
const std::string& workingDir,
bool& swiftPackageEncountered) {
const ProjectFiles& projectFiles) {
// Here we are collecting list of all workspaces and Xcode projects corresponding to them
// Projects without workspaces go into the same "empty-workspace" bucket
swiftPackageEncountered = false;
std::unordered_map<std::string, std::vector<std::string>> workspaces;
std::unordered_set<std::string> projectsBelongingToWorkspace;
std::vector<fs::path> files = collectFiles(workingDir);
for (auto& path : files) {
for (auto& path : projectFiles.xcodeFiles) {
if (path.extension() == ".xcworkspace") {
auto projects = readProjectsFromWorkspace(path.string());
for (auto& project : projects) {
projectsBelongingToWorkspace.insert(project.string());
workspaces[path.string()].push_back(project.string());
}
} else if (!swiftPackageEncountered && path.filename() == "Package.swift") {
// a package manifest must begin with a specific header comment
// see https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html
static constexpr std::string_view packageHeader = "// swift-tools-version:";
std::array<char, packageHeader.size()> buffer{};
std::string_view bufferView{buffer.data(), buffer.size()};
if (std::ifstream{path}.read(buffer.data(), buffer.size()) && bufferView == packageHeader) {
swiftPackageEncountered = true;
}
}
}
// Collect all projects not belonging to any workspace into a separate empty bucket
for (auto& path : files) {
for (auto& path : projectFiles.xcodeFiles) {
if (path.extension() == ".xcodeproj") {
if (projectsBelongingToWorkspace.count(path.string())) {
if (projectsBelongingToWorkspace.contains(path.string())) {
continue;
}
workspaces[std::string()].push_back(path.string());
@@ -262,11 +266,15 @@ static std::unordered_map<std::string, std::vector<std::string>> collectWorkspac
return workspaces;
}
Targets collectTargets(const std::string& workingDir) {
Targets ret;
ProjectStructure scanProjectStructure(const std::string& workingDir) {
ProjectStructure ret;
// Getting a list of workspaces and the project that belong to them
auto workspaces = collectWorkspaces(workingDir, ret.swiftPackageEncountered);
auto projectFiles = scanWorkingDir(workingDir);
auto workspaces = collectWorkspaces(projectFiles);
ret.xcodeEncountered = !workspaces.empty();
ret.swiftPackages = std::move(projectFiles.packageFiles);
ret.podfiles = std::move(projectFiles.podfiles);
ret.cartfiles = std::move(projectFiles.cartfiles);
if (!ret.xcodeEncountered) {
return ret;
}
@@ -278,8 +286,8 @@ Targets collectTargets(const std::string& workingDir) {
auto targetFilesMapping = mapTargetsToSourceFiles(workspaces);
for (auto& [targetName, data] : targetMapping) {
ret.targets.push_back(Target{data.workspace, data.project, targetName, data.type,
targetFilesMapping[targetName]});
ret.xcodeTargets.push_back(XcodeTarget{data.workspace, data.project, targetName, data.type,
targetFilesMapping[targetName]});
}
return ret;
}

14
swift/xcode-autobuilder/XcodeProjectParser.h
View File

@@ -3,11 +3,17 @@
#include "swift/xcode-autobuilder/XcodeTarget.h"
#include <vector>
#include <string>
#include <filesystem>
struct Targets {
std::vector<Target> targets;
struct ProjectStructure {
std::vector<XcodeTarget> xcodeTargets;
bool xcodeEncountered;
bool swiftPackageEncountered;
// Swift Package Manager support
std::vector<std::filesystem::path> swiftPackages;
// CocoaPods support
std::vector<std::filesystem::path> podfiles;
// Carthage support
std::vector<std::filesystem::path> cartfiles;
};
Targets collectTargets(const std::string& workingDir);
ProjectStructure scanProjectStructure(const std::string& workingDir);

4
swift/xcode-autobuilder/XcodeTarget.h
View File

@@ -3,7 +3,7 @@
#include <string>
#include <binlog/adapt_struct.hpp>
struct Target {
struct XcodeTarget {
std::string workspace;
std::string project;
std::string name;
@@ -11,4 +11,4 @@ struct Target {
size_t fileCount;
};
BINLOG_ADAPT_STRUCT(Target, workspace, project, name, type, fileCount);
BINLOG_ADAPT_STRUCT(XcodeTarget, workspace, project, name, type, fileCount);

74
swift/xcode-autobuilder/xcode-autobuilder.cpp
View File

@@ -24,12 +24,6 @@ constexpr codeql::SwiftDiagnostic noSwiftTarget{
.action = "To analyze a custom set of source files, set up a [manual build "
"command][1].\n\n[1]: " MANUAL_BUILD_COMMAND_HELP_LINK};
constexpr codeql::SwiftDiagnostic spmNotSupported{
.id = "spm-not-supported",
.name = "Swift Package Manager is not supported",
.action = "Swift Package Manager builds are not currently supported by `autobuild`. Set up a "
"[manual build command][1].\n\n[1]: " MANUAL_BUILD_COMMAND_HELP_LINK};
static codeql::Logger& logger() {
static codeql::Logger ret{"main"};
return ret;
@@ -44,38 +38,63 @@ static bool endsWith(std::string_view s, std::string_view suffix) {
return s.size() >= suffix.size() && s.substr(s.size() - suffix.size()) == suffix;
}
static bool isNonSwiftOrTestTarget(const Target& t) {
static bool isNonSwiftOrTestTarget(const XcodeTarget& t) {
return t.fileCount == 0 || t.type == uiTest || t.type == unitTest ||
// unknown target types can be legitimate, let's do a name-based heuristic then
(t.type == unknownType && (endsWith(t.name, "Tests") || endsWith(t.name, "Test")));
}
static void autobuild(const CLIArgs& args) {
auto collected = collectTargets(args.workingDir);
auto& targets = collected.targets;
for (const auto& t : targets) {
static void buildSwiftPackages(const std::vector<std::filesystem::path>& swiftPackages,
bool dryRun) {
auto any_successful =
std::any_of(std::begin(swiftPackages), std::end(swiftPackages), [&](auto& packageFile) {
LOG_INFO("Building Swift package: {}", packageFile);
return buildSwiftPackage(packageFile, dryRun);
});
if (!any_successful) {
codeql::Log::flush();
exit(1);
}
}
static bool autobuild(const CLIArgs& args) {
auto structure = scanProjectStructure(args.workingDir);
auto& xcodeTargets = structure.xcodeTargets;
auto& swiftPackages = structure.swiftPackages;
for (const auto& t : xcodeTargets) {
LOG_INFO("{}", t);
}
// Filter out targets that are tests or have no swift source files
targets.erase(std::remove_if(std::begin(targets), std::end(targets), isNonSwiftOrTestTarget),
std::end(targets));
xcodeTargets.erase(
std::remove_if(std::begin(xcodeTargets), std::end(xcodeTargets), isNonSwiftOrTestTarget),
std::end(xcodeTargets));
// Sort targets by the amount of files in each
std::sort(std::begin(targets), std::end(targets),
[](Target& lhs, Target& rhs) { return lhs.fileCount > rhs.fileCount; });
if ((!collected.xcodeEncountered || targets.empty()) && collected.swiftPackageEncountered) {
DIAGNOSE_ERROR(spmNotSupported,
"A Swift package was detected, but no viable Xcode target was found.");
} else if (!collected.xcodeEncountered) {
DIAGNOSE_ERROR(noProjectFound, "`autobuild` could not detect an Xcode project or workspace.");
} else if (targets.empty()) {
std::sort(std::begin(xcodeTargets), std::end(xcodeTargets),
[](XcodeTarget& lhs, XcodeTarget& rhs) { return lhs.fileCount > rhs.fileCount; });
if (structure.xcodeEncountered && xcodeTargets.empty() && swiftPackages.empty()) {
// Report error only when there are no Xcode targets and no Swift packages
DIAGNOSE_ERROR(noSwiftTarget, "All targets found within Xcode projects or workspaces either "
"contain no Swift source files, or are tests.");
} else {
LOG_INFO("Selected {}", targets.front());
buildTarget(targets.front(), args.dryRun);
return;
return false;
} else if (!structure.xcodeEncountered && swiftPackages.empty()) {
DIAGNOSE_ERROR(noProjectFound,
"`autobuild` detected neither an Xcode project or workspace, nor a Swift package");
return false;
} else if (!xcodeTargets.empty()) {
LOG_INFO("Building Xcode target: {}", xcodeTargets.front());
installDependencies(structure, args.dryRun);
auto buildSucceeded = buildXcodeTarget(xcodeTargets.front(), args.dryRun);
// If build failed, try to build Swift packages
if (!buildSucceeded && !swiftPackages.empty()) {
buildSwiftPackages(swiftPackages, args.dryRun);
}
return buildSucceeded;
} else if (!swiftPackages.empty()) {
buildSwiftPackages(swiftPackages, args.dryRun);
}
return true;
}
static CLIArgs parseCLIArgs(int argc, char** argv) {
@@ -96,7 +115,10 @@ static CLIArgs parseCLIArgs(int argc, char** argv) {
int main(int argc, char** argv) {
auto args = parseCLIArgs(argc, argv);
autobuild(args);
auto success = autobuild(args);
codeql::Log::flush();
if (!success) {
return 1;
}
return 0;
}