From fc165db8acb30401569073f8e0564e880749d092 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Fri, 30 May 2025 12:17:03 +0100 Subject: [PATCH] Bulk generator: Specify 'with-summaries', 'with-sources', and 'with-sinks' in the config file. --- cpp/misc/bulk_generation_targets.json | 4 +- .../models-as-data/bulk_generate_mad.py | 38 ++++++----- rust/misc/bulk_generation_targets.json | 65 +++++++++++++++---- 3 files changed, 75 insertions(+), 32 deletions(-) diff --git a/cpp/misc/bulk_generation_targets.json b/cpp/misc/bulk_generation_targets.json index 5f74b094d35..6cc2223b5e9 100644 --- a/cpp/misc/bulk_generation_targets.json +++ b/cpp/misc/bulk_generation_targets.json @@ -1,8 +1,8 @@ { "strategy": "dca", "targets": [ - { "name": "openssl" }, - { "name": "sqlite" } + { "name": "openssl", "with_summaries": true }, + { "name": "sqlite", "with_summaries": true } ], "destination": "cpp/ql/lib/ext/generated" } \ No newline at end of file diff --git a/misc/scripts/models-as-data/bulk_generate_mad.py b/misc/scripts/models-as-data/bulk_generate_mad.py index 8b548262874..bed3442f790 100644 --- a/misc/scripts/models-as-data/bulk_generate_mad.py +++ b/misc/scripts/models-as-data/bulk_generate_mad.py @@ -41,7 +41,18 @@ class Project(TypedDict): name: str git_repo: NotRequired[str] git_tag: NotRequired[str] + with_sinks: NotRequired[bool] + with_sinks: NotRequired[bool] + with_summaries: NotRequired[bool] +def shouldGenerateSinks(project: Project) -> bool: + return project.get("with_sinks", False) + +def shouldGenerateSources(project: Project) -> bool: + return project.get("with_sources", False) + +def shouldGenerateSummaries(project: Project) -> bool: + return project.get("with_summaries", False) def clone_project(project: Project) -> str: """ @@ -185,7 +196,7 @@ def build_database( return database_dir -def generate_models(args, project: Project, database_dir: str) -> None: +def generate_models(language: str, config, project: Project, database_dir: str) -> None: """ Generate models for a project. @@ -196,10 +207,11 @@ def generate_models(args, project: Project, database_dir: str) -> None: """ name = project["name"] - generator = mad.Generator(args.lang) - generator.generateSinks = args.with_sinks - generator.generateSources = args.with_sources - generator.generateSummaries = args.with_summaries + generator = mad.Generator(language) + # Note: The argument parser converts with-sinks to with_sinks, etc. + generator.generateSinks = shouldGenerateSinks(project) + generator.generateSources = shouldGenerateSources(project) + generator.generateSummaries = shouldGenerateSummaries(project) generator.setenvironment(database=database_dir, folder=name) generator.run() @@ -309,13 +321,14 @@ def download_dca_databases( pat, ) targets = response["targets"] + project_map = {project["name"]: project for project in projects} for data in targets.values(): downloads = data["downloads"] analyzed_database = downloads["analyzed_database"] artifact_name = analyzed_database["artifact_name"] pretty_name = pretty_name_from_artifact_name(artifact_name) - if not pretty_name in [project["name"] for project in projects]: + if not pretty_name in project_map: print(f"Skipping {pretty_name} as it is not in the list of projects") continue @@ -350,7 +363,7 @@ def download_dca_databases( tar_ref.extractall(artifact_unzipped_location) database_results.append( ( - {"name": pretty_name}, + project_map[pretty_name], os.path.join( artifact_unzipped_location, remove_extension(entry) ), @@ -451,7 +464,7 @@ To avoid loss of data, please commit your changes.""" for project, database_dir in database_results: if database_dir is not None: - generate_models(args, project, database_dir) + generate_models(language, config, project, database_dir) if __name__ == "__main__": @@ -474,15 +487,6 @@ if __name__ == "__main__": parser.add_argument( "--lang", type=str, help="The language to generate models for", required=True ) - parser.add_argument( - "--with-sources", action="store_true", help="Generate sources", required=False - ) - parser.add_argument( - "--with-sinks", action="store_true", help="Generate sinks", required=False - ) - parser.add_argument( - "--with-summaries", action="store_true", help="Generate sinks", required=False - ) args = parser.parse_args() # Load config file diff --git a/rust/misc/bulk_generation_targets.json b/rust/misc/bulk_generation_targets.json index ca30b76eb12..85c28c7f817 100644 --- a/rust/misc/bulk_generation_targets.json +++ b/rust/misc/bulk_generation_targets.json @@ -4,67 +4,106 @@ { "name": "libc", "git_repo": "https://github.com/rust-lang/libc", - "git_tag": "0.2.172" + "git_tag": "0.2.172", + "with-sources": true, + "with-sinks": true, + "with-summaries": true }, { "name": "log", "git_repo": "https://github.com/rust-lang/log", - "git_tag": "0.4.27" + "git_tag": "0.4.27", + "with-sources": true, + "with-sinks": true, + "with-summaries": true }, { "name": "memchr", "git_repo": "https://github.com/BurntSushi/memchr", - "git_tag": "2.7.4" + "git_tag": "2.7.4", + "with-sources": true, + "with-sinks": true, + "with-summaries": true }, { "name": "once_cell", "git_repo": "https://github.com/matklad/once_cell", - "git_tag": "v1.21.3" + "git_tag": "v1.21.3", + "with-sources": true, + "with-sinks": true, + "with-summaries": true }, { "name": "rand", "git_repo": "https://github.com/rust-random/rand", - "git_tag": "0.9.1" + "git_tag": "0.9.1", + "with-sources": true, + "with-sinks": true, + "with-summaries": true }, { "name": "smallvec", "git_repo": "https://github.com/servo/rust-smallvec", - "git_tag": "v1.15.0" + "git_tag": "v1.15.0", + "with-sources": true, + "with-sinks": true, + "with-summaries": true }, { "name": "serde", "git_repo": "https://github.com/serde-rs/serde", - "git_tag": "v1.0.219" + "git_tag": "v1.0.219", + "with-sources": true, + "with-sinks": true, + "with-summaries": true }, { "name": "tokio", "git_repo": "https://github.com/tokio-rs/tokio", - "git_tag": "tokio-1.45.0" + "git_tag": "tokio-1.45.0", + "with-sources": true, + "with-sinks": true, + "with-summaries": true }, { "name": "reqwest", "git_repo": "https://github.com/seanmonstar/reqwest", - "git_tag": "v0.12.15" + "git_tag": "v0.12.15", + "with-sources": true, + "with-sinks": true, + "with-summaries": true }, { "name": "rocket", "git_repo": "https://github.com/SergioBenitez/Rocket", - "git_tag": "v0.5.1" + "git_tag": "v0.5.1", + "with-sources": true, + "with-sinks": true, + "with-summaries": true }, { "name": "actix-web", "git_repo": "https://github.com/actix/actix-web", - "git_tag": "web-v4.11.0" + "git_tag": "web-v4.11.0", + "with-sources": true, + "with-sinks": true, + "with-summaries": true }, { "name": "hyper", "git_repo": "https://github.com/hyperium/hyper", - "git_tag": "v1.6.0" + "git_tag": "v1.6.0", + "with-sources": true, + "with-sinks": true, + "with-summaries": true }, { "name": "clap", "git_repo": "https://github.com/clap-rs/clap", - "git_tag": "v4.5.38" + "git_tag": "v4.5.38", + "with-sources": true, + "with-sinks": true, + "with-summaries": true } ], "destination": "rust/ql/lib/ext/generated",