Merge pull request #10786 from github/alexdenisov/xcode-autobuilder

Swift: introduce Xcode autobuilder
This commit is contained in:
AlexDenisov
2022-10-19 10:19:49 +02:00
committed by GitHub
14 changed files with 605 additions and 4 deletions

View File

@@ -24,8 +24,8 @@ pkg_files(
)
pkg_files(
name = "qltest",
srcs = ["tools/qltest.sh"],
name = "scripts",
srcs = ["tools/qltest.sh", "tools/autobuild.sh"],
attributes = pkg_attributes(mode = "0755"),
prefix = "tools",
)
@@ -46,8 +46,8 @@ pkg_filegroup(
srcs = [
":dbscheme_files",
":manifest",
":qltest",
":tracing-config",
":scripts",
],
visibility = ["//visibility:public"],
)
@@ -58,6 +58,12 @@ pkg_runfiles(
prefix = "tools/" + codeql_platform,
)
pkg_runfiles(
name = "xcode-autobuilder",
srcs = ["//swift/xcode-autobuilder"],
prefix = "tools/" + codeql_platform,
)
pkg_files(
name = "swift-test-sdk-arch",
srcs = ["//swift/tools/prebuilt:swift-test-sdk"],
@@ -70,7 +76,12 @@ pkg_filegroup(
srcs = [
":extractor",
":swift-test-sdk-arch",
],
] + select({
"@platforms//os:linux": [],
"@platforms//os:macos": [
":xcode-autobuilder"
],
}),
visibility = ["//visibility:public"],
)

View File

@@ -14,3 +14,4 @@ project(codeql)
include(../misc/bazel/cmake/setup.cmake)
include_generated(//swift/extractor:cmake)
include_generated(//swift/xcode-autobuilder:cmake)

8
swift/tools/autobuild.sh Executable file
View File

@@ -0,0 +1,8 @@
#!/bin/bash
if [[ "$OSTYPE" == "darwin"* ]]; then
exec "${CODEQL_EXTRACTOR_SWIFT_ROOT}/tools/${CODEQL_PLATFORM}/xcode-autobuilder"
else
echo "Not implemented yet"
exit 1
fi

View File

@@ -0,0 +1,22 @@
load("//swift:rules.bzl", "swift_cc_binary")
load("//misc/bazel/cmake:cmake.bzl", "generate_cmake")
swift_cc_binary(
name = "xcode-autobuilder",
srcs = glob([
"*.cpp",
"*.h",
]),
visibility = ["//swift:__pkg__"],
linkopts = [
"-lxml2",
"-framework CoreFoundation",
],
target_compatible_with = ["@platforms//os:macos"],
)
generate_cmake(
name = "cmake",
targets = [":xcode-autobuilder"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,41 @@
#include "swift/xcode-autobuilder/CFHelpers.h"
#include <iostream>
typedef CFTypeID (*cf_get_type_id)();
template <typename CFType, cf_get_type_id get_type_id>
CFType cf_cast(const void* ptr) {
if (!ptr) {
return nullptr;
}
if (CFGetTypeID(ptr) != get_type_id()) {
std::cerr << "Unexpected type: ";
CFShow(ptr);
abort();
}
return static_cast<CFType>(ptr);
}
CFStringRef cf_string_ref(const void* ptr) {
return cf_cast<CFStringRef, CFStringGetTypeID>(ptr);
}
CFArrayRef cf_array_ref(const void* ptr) {
return cf_cast<CFArrayRef, CFArrayGetTypeID>(ptr);
}
CFDictionaryRef cf_dictionary_ref(const void* ptr) {
return cf_cast<CFDictionaryRef, CFDictionaryGetTypeID>(ptr);
}
std::string stringValueForKey(CFDictionaryRef dict, CFStringRef key) {
auto cfValue = cf_string_ref(CFDictionaryGetValue(dict, key));
if (cfValue) {
const int bufferSize = 256;
char buf[bufferSize];
if (CFStringGetCString(cfValue, buf, bufferSize, kCFStringEncodingUTF8)) {
return {buf};
}
}
return {};
}

View File

@@ -0,0 +1,24 @@
#pragma once
#include <CoreFoundation/CoreFoundation.h>
#include <string>
#include <vector>
CFStringRef cf_string_ref(const void* ptr);
CFArrayRef cf_array_ref(const void* ptr);
CFDictionaryRef cf_dictionary_ref(const void* ptr);
std::string stringValueForKey(CFDictionaryRef dict, CFStringRef key);
struct CFKeyValues {
static CFKeyValues fromDictionary(CFDictionaryRef dict) {
auto size = CFDictionaryGetCount(dict);
CFKeyValues ret(size);
CFDictionaryGetKeysAndValues(dict, ret.keys.data(), ret.values.data());
return ret;
}
explicit CFKeyValues(size_t size) : size(size), keys(size), values(size) {}
size_t size;
std::vector<const void*> keys;
std::vector<const void*> values;
};

View File

@@ -0,0 +1,65 @@
#include "swift/xcode-autobuilder/XcodeBuildRunner.h"
#include <vector>
#include <iostream>
#include <spawn.h>
static int waitpid_status(pid_t child) {
int status;
while (waitpid(child, &status, 0) == -1) {
if (errno != EINTR) break;
}
return status;
}
extern char** environ;
static bool exec(const std::vector<std::string>& argv) {
const char** c_argv = (const char**)calloc(argv.size() + 1, sizeof(char*));
for (size_t i = 0; i < argv.size(); i++) {
c_argv[i] = argv[i].c_str();
}
c_argv[argv.size()] = nullptr;
pid_t pid = 0;
if (posix_spawn(&pid, argv.front().c_str(), nullptr, nullptr, (char* const*)c_argv, environ) !=
0) {
std::cerr << "[xcode autobuilder] posix_spawn failed: " << strerror(errno) << "\n";
free(c_argv);
return false;
}
free(c_argv);
int status = waitpid_status(pid);
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
return false;
}
return true;
}
void buildTarget(Target& target, bool dryRun) {
std::vector<std::string> argv({"/usr/bin/xcodebuild", "build"});
if (!target.workspace.empty()) {
argv.push_back("-workspace");
argv.push_back(target.workspace);
argv.push_back("-scheme");
} else {
argv.push_back("-project");
argv.push_back(target.project);
argv.push_back("-target");
}
argv.push_back(target.name);
argv.push_back("CODE_SIGNING_REQUIRED=NO");
argv.push_back("CODE_SIGNING_ALLOWED=NO");
if (dryRun) {
for (auto& arg : argv) {
std::cout << arg + " ";
}
std::cout << "\n";
} else {
if (!exec(argv)) {
std::cerr << "Build failed\n";
exit(1);
}
}
}

View File

@@ -0,0 +1,5 @@
#pragma once
#include "swift/xcode-autobuilder/XcodeTarget.h"
void buildTarget(Target& target, bool dryRun);

View File

@@ -0,0 +1,273 @@
#include "swift/xcode-autobuilder/XcodeProjectParser.h"
#include "swift/xcode-autobuilder/XcodeWorkspaceParser.h"
#include "swift/xcode-autobuilder/CFHelpers.h"
#include <iostream>
#include <filesystem>
#include <unordered_map>
#include <unordered_set>
#include <fstream>
#include <CoreFoundation/CoreFoundation.h>
namespace fs = std::filesystem;
struct TargetData {
std::string workspace;
std::string project;
std::string type;
};
typedef std::unordered_map<std::string, CFDictionaryRef> Targets;
typedef std::unordered_map<std::string, std::vector<std::string>> Dependencies;
typedef std::unordered_map<std::string, std::vector<std::pair<std::string, CFDictionaryRef>>>
BuildFiles;
static size_t totalFilesCount(const std::string& target,
const Dependencies& dependencies,
const BuildFiles& buildFiles) {
size_t sum = buildFiles.at(target).size();
for (auto& dep : dependencies.at(target)) {
sum += totalFilesCount(dep, dependencies, buildFiles);
}
return sum;
}
static bool objectIsTarget(CFDictionaryRef object) {
auto isa = cf_string_ref(CFDictionaryGetValue(object, CFSTR("isa")));
if (isa) {
for (auto target :
{CFSTR("PBXAggregateTarget"), CFSTR("PBXNativeTarget"), CFSTR("PBXLegacyTarget")}) {
if (CFStringCompare(isa, target, 0) == kCFCompareEqualTo) {
return true;
}
}
}
return false;
}
static void mapTargetsToSourceFiles(CFDictionaryRef objects,
std::unordered_map<std::string, size_t>& fileCounts) {
Targets targets;
Dependencies dependencies;
BuildFiles buildFiles;
auto kv = CFKeyValues::fromDictionary(objects);
for (size_t i = 0; i < kv.size; i++) {
auto object = cf_dictionary_ref(kv.values[i]);
if (objectIsTarget(object)) {
auto name = stringValueForKey(object, CFSTR("name"));
dependencies[name] = {};
buildFiles[name] = {};
targets.emplace(name, object);
}
}
for (auto& [targetName, targetObject] : targets) {
auto deps = cf_array_ref(CFDictionaryGetValue(targetObject, CFSTR("dependencies")));
auto size = CFArrayGetCount(deps);
for (CFIndex i = 0; i < size; i++) {
auto dependencyID = cf_string_ref(CFArrayGetValueAtIndex(deps, i));
auto dependency = cf_dictionary_ref(CFDictionaryGetValue(objects, dependencyID));
auto targetID = cf_string_ref(CFDictionaryGetValue(dependency, CFSTR("target")));
if (!targetID) {
// Skipping non-targets (e.g., productRef)
continue;
}
auto targetDependency = cf_dictionary_ref(CFDictionaryGetValue(objects, targetID));
auto dependencyName = stringValueForKey(targetDependency, CFSTR("name"));
if (!dependencyName.empty()) {
dependencies[targetName].push_back(dependencyName);
}
}
}
for (auto& [targetName, targetObject] : targets) {
auto buildPhases = cf_array_ref(CFDictionaryGetValue(targetObject, CFSTR("buildPhases")));
auto buildPhaseCount = CFArrayGetCount(buildPhases);
for (CFIndex buildPhaseIndex = 0; buildPhaseIndex < buildPhaseCount; buildPhaseIndex++) {
auto buildPhaseID = cf_string_ref(CFArrayGetValueAtIndex(buildPhases, buildPhaseIndex));
auto buildPhase = cf_dictionary_ref(CFDictionaryGetValue(objects, buildPhaseID));
auto fileRefs = cf_array_ref(CFDictionaryGetValue(buildPhase, CFSTR("files")));
if (!fileRefs) {
continue;
}
auto fileRefsCount = CFArrayGetCount(fileRefs);
for (CFIndex fileRefIndex = 0; fileRefIndex < fileRefsCount; fileRefIndex++) {
auto fileRefID = cf_string_ref(CFArrayGetValueAtIndex(fileRefs, fileRefIndex));
auto fileRef = cf_dictionary_ref(CFDictionaryGetValue(objects, fileRefID));
auto fileID = cf_string_ref(CFDictionaryGetValue(fileRef, CFSTR("fileRef")));
if (!fileID) {
// FileRef is not a reference to a file (e.g., PBXBuildFile)
continue;
}
auto file = cf_dictionary_ref(CFDictionaryGetValue(objects, fileID));
if (!file) {
// Sometimes the references file belongs to another project, which is not present for
// various reasons
continue;
}
auto isa = stringValueForKey(file, CFSTR("isa"));
if (isa != "PBXFileReference") {
// Skipping anything that is not a 'file', e.g. PBXVariantGroup
continue;
}
auto fileType = stringValueForKey(file, CFSTR("lastKnownFileType"));
auto path = stringValueForKey(file, CFSTR("path"));
if (fileType == "sourcecode.swift" && !path.empty()) {
buildFiles[targetName].emplace_back(path, file);
}
}
}
}
for (auto& [targetName, _] : targets) {
fileCounts[targetName] = totalFilesCount(targetName, dependencies, buildFiles);
}
}
static CFDictionaryRef xcodeProjectObjects(const std::string& xcodeProject) {
auto allocator = CFAllocatorGetDefault();
auto pbxproj = fs::path(xcodeProject) / "project.pbxproj";
if (!fs::exists(pbxproj)) {
return CFDictionaryCreate(allocator, nullptr, nullptr, 0, nullptr, nullptr);
}
std::ifstream ifs(pbxproj, std::ios::in);
std::string content((std::istreambuf_iterator<char>(ifs)), (std::istreambuf_iterator<char>()));
auto data = CFDataCreate(allocator, reinterpret_cast<UInt8*>(content.data()), content.size());
CFErrorRef error = nullptr;
auto plist = CFPropertyListCreateWithData(allocator, data, 0, nullptr, &error);
if (error) {
std::cerr << "[xcode autobuilder] Cannot read Xcode project: ";
CFShow(error);
std::cerr << ": " << pbxproj << "\n";
return CFDictionaryCreate(allocator, nullptr, nullptr, 0, nullptr, nullptr);
}
return cf_dictionary_ref(CFDictionaryGetValue((CFDictionaryRef)plist, CFSTR("objects")));
}
// Maps each target to the number of Swift source files it contains transitively
static std::unordered_map<std::string, size_t> mapTargetsToSourceFiles(
const std::unordered_map<std::string, std::vector<std::string>>& workspaces) {
std::unordered_map<std::string, size_t> fileCounts;
for (auto& [workspace, projects] : workspaces) {
// All targets/dependencies should be resolved in the context of the same workspace
// As different projects in the same workspace may reference each other for dependencies
auto allocator = CFAllocatorGetDefault();
auto allObjects = CFDictionaryCreateMutable(allocator, 0, nullptr, nullptr);
for (auto& project : projects) {
CFDictionaryRef objects = xcodeProjectObjects(project);
auto kv = CFKeyValues::fromDictionary(objects);
for (size_t i = 0; i < kv.size; i++) {
CFDictionaryAddValue(allObjects, kv.keys[i], kv.values[i]);
}
}
mapTargetsToSourceFiles(allObjects, fileCounts);
}
return fileCounts;
}
static std::vector<std::pair<std::string, std::string>> readTargets(const std::string& project) {
auto objects = xcodeProjectObjects(project);
std::vector<std::pair<std::string, std::string>> targets;
auto kv = CFKeyValues::fromDictionary(objects);
for (size_t i = 0; i < kv.size; i++) {
auto object = (CFDictionaryRef)kv.values[i];
if (objectIsTarget(object)) {
auto name = stringValueForKey(object, CFSTR("name"));
auto type = stringValueForKey(object, CFSTR("productType"));
targets.emplace_back(name, type.empty() ? "<unknown_target_type>" : type);
}
}
return targets;
}
static std::unordered_map<std::string, TargetData> mapTargetsToWorkspace(
const std::unordered_map<std::string, std::vector<std::string>>& workspaces) {
std::unordered_map<std::string, TargetData> targetMapping;
for (auto& [workspace, projects] : workspaces) {
for (auto& project : projects) {
auto targets = readTargets(project);
for (auto& [target, type] : targets) {
targetMapping[target] = TargetData{workspace, project, type};
}
}
}
return targetMapping;
}
static std::vector<fs::path> collectFiles(const std::string& workingDir) {
fs::path workDir(workingDir);
std::vector<fs::path> files;
auto iterator = fs::recursive_directory_iterator(workDir);
auto end = fs::recursive_directory_iterator();
for (; iterator != end; iterator++) {
auto filename = iterator->path().filename();
if (filename == "DerivedData" || filename == ".git" || filename == "build") {
// Skip these folders
iterator.disable_recursion_pending();
continue;
}
auto dirEntry = *iterator;
if (!dirEntry.is_directory()) {
continue;
}
if (dirEntry.path().extension() != fs::path(".xcodeproj") &&
dirEntry.path().extension() != fs::path(".xcworkspace")) {
continue;
}
files.push_back(dirEntry.path());
}
return files;
}
static std::unordered_map<std::string, std::vector<std::string>> collectWorkspaces(
const std::string& workingDir) {
// Here we are collecting list of all workspaces and Xcode projects corresponding to them
// Projects without workspaces go into the same "empty-workspace" bucket
std::unordered_map<std::string, std::vector<std::string>> workspaces;
std::unordered_set<std::string> projectsBelongingToWorkspace;
std::vector<fs::path> files = collectFiles(workingDir);
for (auto& path : files) {
if (path.extension() == ".xcworkspace") {
auto projects = readProjectsFromWorkspace(path.string());
for (auto& project : projects) {
projectsBelongingToWorkspace.insert(project.string());
workspaces[path.string()].push_back(project.string());
}
}
}
// Collect all projects not belonging to any workspace into a separate empty bucket
for (auto& path : files) {
if (path.extension() == ".xcodeproj") {
if (projectsBelongingToWorkspace.count(path.string())) {
continue;
}
workspaces[std::string()].push_back(path.string());
}
}
return workspaces;
}
std::vector<Target> collectTargets(const std::string& workingDir) {
// Getting a list of workspaces and the project that belong to them
auto workspaces = collectWorkspaces(workingDir);
if (workspaces.empty()) {
std::cerr << "[xcode autobuilder] Xcode project or workspace not found\n";
exit(1);
}
// Mapping each target to the workspace/project it belongs to
auto targetMapping = mapTargetsToWorkspace(workspaces);
// Mapping each target to the number of source files it contains
auto targetFilesMapping = mapTargetsToSourceFiles(workspaces);
std::vector<Target> targets;
for (auto& [targetName, data] : targetMapping) {
targets.push_back(Target{data.workspace, data.project, targetName, data.type,
targetFilesMapping[targetName]});
}
return targets;
}

View File

@@ -0,0 +1,7 @@
#pragma once
#include "swift/xcode-autobuilder/XcodeTarget.h"
#include <vector>
#include <string>
std::vector<Target> collectTargets(const std::string& workingDir);

View File

@@ -0,0 +1,11 @@
#pragma once
#include <string>
struct Target {
std::string workspace;
std::string project;
std::string name;
std::string type;
size_t fileCount;
};

View File

@@ -0,0 +1,62 @@
#include <libxml/tree.h>
#include <libxml/parser.h>
#include <iostream>
#include "swift/xcode-autobuilder/XcodeWorkspaceParser.h"
/*
Extracts FileRef locations from an XML of the following form:
<?xml version="1.0" encoding="UTF-8"?>
<Workspace version = "1.0">
<FileRef location = "group:PathToProject.xcodeproj">
</FileRef>
</Workspace>
*/
std::vector<fs::path> readProjectsFromWorkspace(const std::string& workspace) {
fs::path workspacePath(workspace);
auto workspaceData = workspacePath / "contents.xcworkspacedata";
if (!fs::exists(workspaceData)) {
std::cerr << "[xcode autobuilder] Cannot read workspace: file does not exist '" << workspaceData
<< "\n";
return {};
}
auto xmlDoc = xmlParseFile(workspaceData.c_str());
if (!xmlDoc) {
std::cerr << "[xcode autobuilder] Cannot parse workspace file '" << workspaceData << "\n";
return {};
}
auto root = xmlDocGetRootElement(xmlDoc);
auto first = xmlFirstElementChild(root);
auto last = xmlLastElementChild(root);
std::vector<xmlNodePtr> children;
for (; first != last; first = xmlNextElementSibling(first)) {
children.push_back(first);
}
children.push_back(first);
std::vector<std::string> locations;
for (auto child : children) {
if (child) {
auto prop = xmlGetProp(child, xmlCharStrdup("location"));
if (prop) {
locations.emplace_back((char*)prop);
}
}
}
xmlFreeDoc(xmlDoc);
std::vector<fs::path> projects;
for (auto& location : locations) {
auto colon = location.find(':');
if (colon != std::string::npos) {
auto project = location.substr(colon + 1);
if (!project.empty()) {
auto fullPath = workspacePath.parent_path() / project;
projects.push_back(fullPath);
}
}
}
return projects;
}

View File

@@ -0,0 +1,9 @@
#pragma once
#include <string>
#include <vector>
#include <filesystem>
namespace fs = std::filesystem;
std::vector<fs::path> readProjectsFromWorkspace(const std::string& workspace);

View File

@@ -0,0 +1,62 @@
#include <iostream>
#include <vector>
#include <filesystem>
#include "swift/xcode-autobuilder/XcodeTarget.h"
#include "swift/xcode-autobuilder/XcodeBuildRunner.h"
#include "swift/xcode-autobuilder/XcodeProjectParser.h"
static const char* Application = "com.apple.product-type.application";
static const char* Framework = "com.apple.product-type.framework";
struct CLIArgs {
std::string workingDir;
bool dryRun;
};
static void autobuild(const CLIArgs& args) {
auto targets = collectTargets(args.workingDir);
// Filter out non-application/framework targets
targets.erase(std::remove_if(std::begin(targets), std::end(targets),
[&](Target& t) -> bool {
return t.type != Application && t.type != Framework;
}),
std::end(targets));
// Sort targets by the amount of files in each
std::sort(std::begin(targets), std::end(targets),
[](Target& lhs, Target& rhs) { return lhs.fileCount > rhs.fileCount; });
for (auto& t : targets) {
std::cerr << t.workspace << " " << t.project << " " << t.type << " " << t.name << " "
<< t.fileCount << "\n";
}
if (targets.empty()) {
std::cerr << "[xcode autobuilder] Suitable targets not found\n";
exit(1);
}
buildTarget(targets.front(), args.dryRun);
}
static CLIArgs parseCLIArgs(int argc, char** argv) {
bool dryRun = false;
std::string path;
if (argc == 3) {
path = argv[2];
if (std::string(argv[1]) == "-dry-run") {
dryRun = true;
}
} else if (argc == 2) {
path = argv[1];
} else {
path = std::filesystem::current_path();
}
return CLIArgs{path, dryRun};
}
int main(int argc, char** argv) {
auto args = parseCLIArgs(argc, argv);
autobuild(args);
return 0;
}