mirror of
https://github.com/github/codeql.git
synced 2025-12-17 01:03:14 +01:00
Bulk generator: Specify 'with-summaries', 'with-sources', and 'with-sinks' in the config file.
This commit is contained in:
Mathias Vorreiter Pedersen
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"strategy": "dca",
|
||||
"targets": [
|
||||
{ "name": "openssl" },
|
||||
{ "name": "sqlite" }
|
||||
{ "name": "openssl", "with_summaries": true },
|
||||
{ "name": "sqlite", "with_summaries": true }
|
||||
],
|
||||
"destination": "cpp/ql/lib/ext/generated"
|
||||
}
|
||||
@@ -41,7 +41,18 @@ class Project(TypedDict):
|
||||
name: str
|
||||
git_repo: NotRequired[str]
|
||||
git_tag: NotRequired[str]
|
||||
with_sinks: NotRequired[bool]
|
||||
with_sinks: NotRequired[bool]
|
||||
with_summaries: NotRequired[bool]
|
||||
|
||||
def shouldGenerateSinks(project: Project) -> bool:
|
||||
return project.get("with_sinks", False)
|
||||
|
||||
def shouldGenerateSources(project: Project) -> bool:
|
||||
return project.get("with_sources", False)
|
||||
|
||||
def shouldGenerateSummaries(project: Project) -> bool:
|
||||
return project.get("with_summaries", False)
|
||||
|
||||
def clone_project(project: Project) -> str:
|
||||
"""
|
||||
@@ -185,7 +196,7 @@ def build_database(
|
||||
return database_dir
|
||||
|
||||
|
||||
def generate_models(args, project: Project, database_dir: str) -> None:
|
||||
def generate_models(language: str, config, project: Project, database_dir: str) -> None:
|
||||
"""
|
||||
Generate models for a project.
|
||||
|
||||
@@ -196,10 +207,11 @@ def generate_models(args, project: Project, database_dir: str) -> None:
|
||||
"""
|
||||
name = project["name"]
|
||||
|
||||
generator = mad.Generator(args.lang)
|
||||
generator.generateSinks = args.with_sinks
|
||||
generator.generateSources = args.with_sources
|
||||
generator.generateSummaries = args.with_summaries
|
||||
generator = mad.Generator(language)
|
||||
# Note: The argument parser converts with-sinks to with_sinks, etc.
|
||||
generator.generateSinks = shouldGenerateSinks(project)
|
||||
generator.generateSources = shouldGenerateSources(project)
|
||||
generator.generateSummaries = shouldGenerateSummaries(project)
|
||||
generator.setenvironment(database=database_dir, folder=name)
|
||||
generator.run()
|
||||
|
||||
@@ -309,13 +321,14 @@ def download_dca_databases(
|
||||
pat,
|
||||
)
|
||||
targets = response["targets"]
|
||||
project_map = {project["name"]: project for project in projects}
|
||||
for data in targets.values():
|
||||
downloads = data["downloads"]
|
||||
analyzed_database = downloads["analyzed_database"]
|
||||
artifact_name = analyzed_database["artifact_name"]
|
||||
pretty_name = pretty_name_from_artifact_name(artifact_name)
|
||||
|
||||
if not pretty_name in [project["name"] for project in projects]:
|
||||
if not pretty_name in project_map:
|
||||
print(f"Skipping {pretty_name} as it is not in the list of projects")
|
||||
continue
|
||||
|
||||
@@ -350,7 +363,7 @@ def download_dca_databases(
|
||||
tar_ref.extractall(artifact_unzipped_location)
|
||||
database_results.append(
|
||||
(
|
||||
{"name": pretty_name},
|
||||
project_map[pretty_name],
|
||||
os.path.join(
|
||||
artifact_unzipped_location, remove_extension(entry)
|
||||
),
|
||||
@@ -451,7 +464,7 @@ To avoid loss of data, please commit your changes."""
|
||||
|
||||
for project, database_dir in database_results:
|
||||
if database_dir is not None:
|
||||
generate_models(args, project, database_dir)
|
||||
generate_models(language, config, project, database_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
@@ -474,15 +487,6 @@ if __name__ == "__main__":
|
||||
parser.add_argument(
|
||||
"--lang", type=str, help="The language to generate models for", required=True
|
||||
)
|
||||
parser.add_argument(
|
||||
"--with-sources", action="store_true", help="Generate sources", required=False
|
||||
)
|
||||
parser.add_argument(
|
||||
"--with-sinks", action="store_true", help="Generate sinks", required=False
|
||||
)
|
||||
parser.add_argument(
|
||||
"--with-summaries", action="store_true", help="Generate sinks", required=False
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load config file
|
||||
|
||||
@@ -4,67 +4,106 @@
|
||||
{
|
||||
"name": "libc",
|
||||
"git_repo": "https://github.com/rust-lang/libc",
|
||||
"git_tag": "0.2.172"
|
||||
"git_tag": "0.2.172",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
},
|
||||
{
|
||||
"name": "log",
|
||||
"git_repo": "https://github.com/rust-lang/log",
|
||||
"git_tag": "0.4.27"
|
||||
"git_tag": "0.4.27",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
},
|
||||
{
|
||||
"name": "memchr",
|
||||
"git_repo": "https://github.com/BurntSushi/memchr",
|
||||
"git_tag": "2.7.4"
|
||||
"git_tag": "2.7.4",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
},
|
||||
{
|
||||
"name": "once_cell",
|
||||
"git_repo": "https://github.com/matklad/once_cell",
|
||||
"git_tag": "v1.21.3"
|
||||
"git_tag": "v1.21.3",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
},
|
||||
{
|
||||
"name": "rand",
|
||||
"git_repo": "https://github.com/rust-random/rand",
|
||||
"git_tag": "0.9.1"
|
||||
"git_tag": "0.9.1",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
},
|
||||
{
|
||||
"name": "smallvec",
|
||||
"git_repo": "https://github.com/servo/rust-smallvec",
|
||||
"git_tag": "v1.15.0"
|
||||
"git_tag": "v1.15.0",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
},
|
||||
{
|
||||
"name": "serde",
|
||||
"git_repo": "https://github.com/serde-rs/serde",
|
||||
"git_tag": "v1.0.219"
|
||||
"git_tag": "v1.0.219",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
},
|
||||
{
|
||||
"name": "tokio",
|
||||
"git_repo": "https://github.com/tokio-rs/tokio",
|
||||
"git_tag": "tokio-1.45.0"
|
||||
"git_tag": "tokio-1.45.0",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
},
|
||||
{
|
||||
"name": "reqwest",
|
||||
"git_repo": "https://github.com/seanmonstar/reqwest",
|
||||
"git_tag": "v0.12.15"
|
||||
"git_tag": "v0.12.15",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
},
|
||||
{
|
||||
"name": "rocket",
|
||||
"git_repo": "https://github.com/SergioBenitez/Rocket",
|
||||
"git_tag": "v0.5.1"
|
||||
"git_tag": "v0.5.1",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
},
|
||||
{
|
||||
"name": "actix-web",
|
||||
"git_repo": "https://github.com/actix/actix-web",
|
||||
"git_tag": "web-v4.11.0"
|
||||
"git_tag": "web-v4.11.0",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
},
|
||||
{
|
||||
"name": "hyper",
|
||||
"git_repo": "https://github.com/hyperium/hyper",
|
||||
"git_tag": "v1.6.0"
|
||||
"git_tag": "v1.6.0",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
},
|
||||
{
|
||||
"name": "clap",
|
||||
"git_repo": "https://github.com/clap-rs/clap",
|
||||
"git_tag": "v4.5.38"
|
||||
"git_tag": "v4.5.38",
|
||||
"with-sources": true,
|
||||
"with-sinks": true,
|
||||
"with-summaries": true
|
||||
}
|
||||
],
|
||||
"destination": "rust/ql/lib/ext/generated",
|
||||
|
||||
Reference in New Issue
Block a user