From c45cc2aa2f22e25fc0ee036864cb95e4ddf86d4d Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 23 Jul 2020 17:37:01 +0200 Subject: [PATCH] Python: CG trace: Add helper.sh to run tracing against real projects --- .../recorded-call-graph-metrics/.gitignore | 2 + .../recorded-call-graph-metrics/helper.sh | 159 ++++++++++++++++++ .../recorded-call-graph-metrics/projects.json | 18 ++ 3 files changed, 179 insertions(+) create mode 100755 python/tools/recorded-call-graph-metrics/helper.sh create mode 100644 python/tools/recorded-call-graph-metrics/projects.json diff --git a/python/tools/recorded-call-graph-metrics/.gitignore b/python/tools/recorded-call-graph-metrics/.gitignore index 5d5e32913d8..7b0abc5caa2 100644 --- a/python/tools/recorded-call-graph-metrics/.gitignore +++ b/python/tools/recorded-call-graph-metrics/.gitignore @@ -8,4 +8,6 @@ tests/cg-trace-test-db # Artifact from building `pip install -e .` src/cg_trace.egg-info/ +projects/ + venv/ diff --git a/python/tools/recorded-call-graph-metrics/helper.sh b/python/tools/recorded-call-graph-metrics/helper.sh new file mode 100755 index 00000000000..cb072347a8d --- /dev/null +++ b/python/tools/recorded-call-graph-metrics/helper.sh @@ -0,0 +1,159 @@ +#!/bin/bash + +set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +PROJECTS_FILE="$SCRIPTDIR/projects.json" + +PROJECT_BASE_DIR="$SCRIPTDIR/projects" + +repo_dir() { + echo "$PROJECT_BASE_DIR/$1/repo" +} + +venv_dir() { + echo "$PROJECT_BASE_DIR/$1/venv" +} + +trace_dir() { + echo "$PROJECT_BASE_DIR/$1/traces" +} + +db_path() { + echo "$PROJECT_BASE_DIR/$1/$1-db" +} + +help() { + echo -n """\ +$0 help This message +$0 projects List projects +$0 repo Fetch repo for projects +$0 setup Perform setup steps for projects (install dependencies) +$0 trace Trace projects +$0 db Build databases for projects +$0 all Perform all the above steps for projects +""" +} + +projects() { + jq -r 'keys[]' "$PROJECTS_FILE" +} + +check_project_exists() { + if ! jq -e ".\"$1\"" "$PROJECTS_FILE" &>/dev/null; then + echo "ERROR: '$1' not a known project, see '$0 projects'" + exit 1 + fi +} + +repo() { + for project in $@; do + check_project_exists $project + + echo "Cloning repo for $project" + + REPO_DIR=$(repo_dir $project) + + if [[ -d "$REPO_DIR" ]]; then + echo "Repo already cloned in $REPO_DIR" + continue; + fi + + REPO_URL=$(jq -e -r ".\"$project\".repo" "$PROJECTS_FILE") + SHA=$(jq -e -r ".\"$project\".sha" "$PROJECTS_FILE") + + mkdir -p "$REPO_DIR" + cd "$REPO_DIR" + git init + git remote add origin $REPO_URL + git fetch --depth 1 origin $SHA + git -c advice.detachedHead=False checkout FETCH_HEAD + done +} + +setup() { + for project in $@; do + check_project_exists $project + + echo "Setting up $project" + + python3 -m venv $(venv_dir $project) + source $(venv_dir $project)/bin/activate + + pip install -e "$SCRIPTDIR" + + IFS=$'\n' + setup_commands=($(jq -r ".\"$project\".setup[]" $PROJECTS_FILE)) + unset IFS + for setup_command in "${setup_commands[@]}"; do + echo "Running '$setup_command'" + $setup_command + done + + # deactivate venv again + deactivate + done +} + +trace() { + for project in $@; do + check_project_exists $project + + echo "Tracing '$project" + + source $(venv_dir $project)/bin/activate + + REPO_DIR=$(repo_dir $project) + cd "$REPO_DIR" + + mkdir -p $(trace_dir $project) + + MODULE_COMMAND=$(jq -r ".\"$project\".module_command" $PROJECTS_FILE) + + cg-trace --xml $(trace_dir $project)/trace.xml --module $MODULE_COMMAND + done +} + +db() { + for project in $@; do + check_project_exists $project + + echo "Creating CodeQL database for '$project" + + DB=$(db_path $project) + SRC=$(repo_dir $project) + PYTHON_EXTRACTOR=$(codeql resolve extractor --language=python) + + # Source venv so we can extract dependencies + source $(venv_dir $project)/bin/activate + + rm -rf "$DB" + + codeql database init --source-root="$SRC" --language=python "$DB" + codeql database trace-command --working-dir="$SRC" "$DB" "$PYTHON_EXTRACTOR/tools/autobuild.sh" + codeql database index-files --language xml --include-extension .xml --working-dir="$(trace_dir $project)" "$DB" + codeql database finalize "$DB" + + echo "Created database in '$DB'" + + # deactivate venv again + deactivate + done +} + +all() { + for project in $@; do + check_project_exists $project + + repo $project + setup $project + trace $project + db $project + done +} + + +COMMAND=${1:-"help"} +shift + +$COMMAND $@ diff --git a/python/tools/recorded-call-graph-metrics/projects.json b/python/tools/recorded-call-graph-metrics/projects.json new file mode 100644 index 00000000000..e0754474034 --- /dev/null +++ b/python/tools/recorded-call-graph-metrics/projects.json @@ -0,0 +1,18 @@ +{ + "wcwidth": { + "repo": "https://github.com/jquast/wcwidth.git", + "sha": "b29897e5a1b403a0e36f7fc991614981cbc42475", + "module_command": "pytest -c /dev/null", + "setup": [ + "pip install pytest" + ] + }, + "youtube-dl": { + "repo": "https://github.com/ytdl-org/youtube-dl.git", + "sha": "a115e07594ccb7749ca108c889978510c7df126e", + "module_command": "nose -v test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py --exclude test_socks.py", + "setup": [ + "pip install nose" + ] + } +}