hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into patch-1

Browse Source
This commit is contained in:
Chad Bentz
2024-01-17 09:25:28 -05:00
committed by GitHub
parent ee3085e15e 8610c950e1
commit b12f4d97f8
6226 changed files with 615113 additions and 173628 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
misc/scripts/library-coverage/comment-pr.py
View File

@@ -56,6 +56,20 @@ def comment_pr(repo, run_id):
if os.path.isdir("pr"):
shutil.rmtree("pr")
try:
utils.download_artifact(repo, "comment", "comment", run_id)
with open("comment/ID") as file:
raw_comment_id = int(file.read().strip())
except Exception as e:
# If there is no existing comment, the `comment/ID` artifact
# will not exist. This will cause `utils.download_artifact`
# to fail, so we catch that and set `raw_comment_id` to `None`.
print("Could not retrieve an existing comment ID. \n", e)
raw_comment_id = None
finally:
if os.path.isdir("comment"):
shutil.rmtree("comment")
# Try storing diff for previous run:
prev_run_id = 0
prev_diff_exists = False
@@ -95,14 +109,37 @@ def comment_pr(repo, run_id):
comment = comment_first_line + \
"A recent commit removed the previously reported differences."
post_comment(comment, repo, pr_number)
if raw_comment_id is None:
post_initial_comment(comment, repo, pr_number)
else:
update_existing_comment(comment, repo, pr_number, raw_comment_id)
def post_comment(comment, repo, pr_number):
def post_initial_comment(comment, repo, pr_number):
print(f"Posting comment to PR #{pr_number}")
utils.subprocess_run(["gh", "pr", "comment", str(pr_number),
"--repo", repo, "--body", comment])
def update_existing_comment(comment, repo, pr_number, raw_comment_id):
# Fetch existing comment, and validate:
# - comment belongs to the PR with number `pr_number`
# - comment starts with the expected prefix `comment_first_line`
# - comment author is github-actions[bot]
comment_author = "github-actions[bot]"
filter = f"select(.issue_url | endswith(\"{repo}/issues/{pr_number}\")) " + \
f"| select(.body | startswith(\"{comment_first_line}\")) " + \
f"| select(.user.login == \"{comment_author}\") " + \
"| .id"
comment_id = utils.subprocess_check_output(["gh", "api", f"repos/{repo}/issues/comments/{raw_comment_id}",
"--jq", filter]).strip()
if comment_id:
print(f"Updating comment {comment_id} on PR #{pr_number}")
utils.subprocess_run(["gh", "api", f"repos/{repo}/issues/comments/{comment_id}",
"-X", "PATCH", "-f", f"body={comment}"])
else:
print(f"Comment {raw_comment_id} did not pass validations: not editing.")
def get_previous_run_id(repo, run_id, pr_number):
"""
@@ -118,7 +155,7 @@ def get_previous_run_id(repo, run_id, pr_number):
pr_repo = this_run["head_repository"]
# Get all previous runs that match branch, repo and workflow name:
output = utils.subprocess_check_output(["gh", "api", "-X", "GET", f"repos/{repo}/actions/runs", "-f", "event=pull_request", "-f", "status=success", "-f", f"branch='{pr_branch}'", "--paginate",
output = utils.subprocess_check_output(["gh", "api", "-X", "GET", f"repos/{repo}/actions/runs", "-f", "event=pull_request", "-f", "status=success", "-f", f"branch={pr_branch}", "--paginate",
"--jq", f'[.workflow_runs.[] | select(.head_repository.full_name=="{pr_repo}" and .name=="{artifacts_workflow_name}")] | sort_by(.id) | reverse | [.[].id]'])
ids = []

37
misc/scripts/models-as-data/convert_extensions.py
View File

@@ -7,7 +7,6 @@ import subprocess
import sys
import tempfile
def quote_if_needed(v):
# string columns
if type(v) is str:
@@ -15,24 +14,13 @@ def quote_if_needed(v):
# bool column
return str(v)
def insert_update(rows, key, value):
if key in rows:
rows[key] += value
else:
rows[key] = value
def merge(*dicts):
merged = {}
for d in dicts:
for entry in d:
insert_update(merged, entry, d[entry])
return merged
def parseData(data):
rows = { }
rows = [{ }, { }]
for row in data:
d = map(quote_if_needed, row)
insert_update(rows, row[0], " - [" + ', '.join(d) + ']\n')
provenance = row[-1]
targetRows = rows[1] if provenance.endswith("generated") else rows[0]
helpers.insert_update(targetRows, row[0], " - [" + ', '.join(d) + ']\n')
return rows
@@ -57,9 +45,10 @@ class Converter:
def asAddsTo(self, rows, predicate):
extensions = { }
for key in rows:
extensions[key] = helpers.addsToTemplate.format(f"codeql/{self.language}-all", predicate, rows[key])
extensions = [{ }, { }]
for i in range(2):
for key in rows[i]:
extensions[i][key] = helpers.addsToTemplate.format(f"codeql/{self.language}-all", predicate, rows[i][key])
return extensions
@@ -75,7 +64,7 @@ class Converter:
sources = self.getAddsTo("ExtractSources.ql", helpers.sourceModelPredicate)
sinks = self.getAddsTo("ExtractSinks.ql", helpers.sinkModelPredicate)
neutrals = self.getAddsTo("ExtractNeutrals.ql", helpers.neutralModelPredicate)
return merge(sources, sinks, summaries, neutrals)
return [helpers.merge(sources[0], sinks[0], summaries[0], neutrals[0]), helpers.merge(sources[1], sinks[1], summaries[1], neutrals[1])]
def save(self, extensions):
@@ -85,9 +74,13 @@ class Converter:
# Create a file for each namespace and save models.
extensionTemplate = """extensions:
{0}"""
for entry in extensions:
for entry in extensions[0]:
with open(self.extDir + "/" + entry + self.modelFileExtension, "w") as f:
f.write(extensionTemplate.format(extensions[entry]))
f.write(extensionTemplate.format(extensions[0][entry]))
for entry in extensions[1]:
with open(self.extDir + "/generated/" + entry + self.modelFileExtension, "w") as f:
f.write(extensionTemplate.format(extensions[1][entry]))
def run(self):
extensions = self.makeContent()

98
misc/scripts/models-as-data/generate_flow_model.py
View File

@@ -13,14 +13,16 @@ def quote_if_needed(row):
if row != "true" and row != "false":
return "\"" + row + "\""
# subtypes column
return row
return row[0].upper() + row[1:]
def parseData(data):
rows = ""
for (row) in data:
rows = { }
for row in data:
d = row[0].split(';')
namespace = d[0]
d = map(quote_if_needed, d)
rows += " - [" + ', '.join(d) + ']\n'
helpers.insert_update(rows, namespace, " - [" + ', '.join(d) + ']\n')
return rows
@@ -38,12 +40,10 @@ class Generator:
def printHelp(self):
print(f"""Usage:
python3 GenerateFlowModel.py <library-database> <outputYml> [<friendlyFrameworkName>] [--with-sinks] [--with-sources] [--with-summaries] [--with-typebased-summaries] [--dry-run]
python3 GenerateFlowModel.py <library-database> [DIR] [--with-sinks] [--with-sources] [--with-summaries] [--with-neutrals] [--with-typebased-summaries] [--dry-run]
This generates summary, source and sink models for the code in the database.
The files will be placed in `{self.language}/ql/lib/ext/generated/<outputYml>.model.yml` where
outputYml is the name (and path) of the output YAML file. Usually, models are grouped by their
respective frameworks.
This generates summary, source, sink and neutral models for the code in the database.
The files will be placed in `{self.language}/ql/lib/ext/generated/DIR`
Which models are generated is controlled by the flags:
--with-sinks
@@ -56,29 +56,20 @@ If none of these flags are specified, all models are generated except for the ty
--dry-run: Only run the queries, but don't write to file.
Example invocations:
$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db mylibrary
$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db mylibrary "Friendly Name of Framework"
$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db
$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db --with-sinks
$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db --with-sinks my_directory
Requirements: `codeql` should both appear on your path.
""")
def setenvironment(self, target, database, friendlyName):
def setenvironment(self, database, folder):
self.codeQlRoot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
if not target.endswith(".model.yml"):
target += ".model.yml"
filename = os.path.basename(target)
if friendlyName is not None:
self.friendlyname = friendlyName
else:
self.friendlyname = filename[:-10]
self.shortname = filename[:-10]
self.database = database
self.generatedFrameworks = os.path.join(
self.codeQlRoot, f"{self.language}/ql/lib/ext/generated/")
self.frameworkTarget = os.path.join(self.generatedFrameworks, filename)
self.typeBasedFrameworkTarget = os.path.join(self.generatedFrameworks, "TypeBased" + filename)
self.codeQlRoot, f"{self.language}/ql/lib/ext/generated/{folder}")
self.workDir = tempfile.mkdtemp()
os.makedirs(self.generatedFrameworks, exist_ok=True)
@@ -117,15 +108,15 @@ Requirements: `codeql` should both appear on your path.
if not generator.generateSinks and not generator.generateSources and not generator.generateSummaries and not generator.generateNeutrals and not generator.generateTypeBasedSummaries:
generator.generateSinks = generator.generateSources = generator.generateSummaries = generator.generateNeutrals = True
if len(sys.argv) < 3 or len(sys.argv) > 4:
n = len(sys.argv)
if n < 2:
generator.printHelp()
sys.exit(1)
elif n == 2:
generator.setenvironment(sys.argv[1], "")
else:
generator.setenvironment(sys.argv[1], sys.argv[2])
friendlyName = None
if len(sys.argv) == 4:
friendlyName = sys.argv[3]
generator.setenvironment(sys.argv[2], sys.argv[1], friendlyName)
return generator
@@ -141,58 +132,57 @@ Requirements: `codeql` should both appear on your path.
def asAddsTo(self, rows, predicate):
if rows.strip() == "":
return ""
return helpers.addsToTemplate.format(f"codeql/{self.language}-all", predicate, rows)
extensions = { }
for key in rows:
extensions[key] = helpers.addsToTemplate.format(f"codeql/{self.language}-all", predicate, rows[key])
return extensions
def getAddsTo(self, query, predicate):
data = self.runQuery(query)
rows = parseData(data)
return self.asAddsTo(rows, predicate)
def makeContent(self):
if self.generateSummaries:
summaryAddsTo = self.getAddsTo("CaptureSummaryModels.ql", helpers.summaryModelPredicate)
else:
summaryAddsTo = ""
summaryAddsTo = { }
if self.generateSinks:
sinkAddsTo = self.getAddsTo("CaptureSinkModels.ql", helpers.sinkModelPredicate)
else:
sinkAddsTo = ""
sinkAddsTo = { }
if self.generateSources:
sourceAddsTo = self.getAddsTo("CaptureSourceModels.ql", helpers.sourceModelPredicate)
else:
sourceAddsTo = ""
sourceAddsTo = {}
if self.generateNeutrals:
neutralAddsTo = self.getAddsTo("CaptureNeutralModels.ql", helpers.neutralModelPredicate)
else:
neutralAddsTo = ""
neutralAddsTo = { }
return f"""# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
# Definitions of models for the {self.friendlyname} framework.
extensions:
{sinkAddsTo}{sourceAddsTo}{summaryAddsTo}{neutralAddsTo}"""
return helpers.merge(summaryAddsTo, sinkAddsTo, sourceAddsTo, neutralAddsTo)
def makeTypeBasedContent(self):
if self.generateTypeBasedSummaries:
typeBasedSummaryAddsTo = self.getAddsTo("CaptureTypeBasedSummaryModels.ql", "extSummaryModel")
typeBasedSummaryAddsTo = self.getAddsTo("CaptureTypeBasedSummaryModels.ql", helpers.summaryModelPredicate)
else:
typeBasedSummaryAddsTo = ""
typeBasedSummaryAddsTo = { }
return f"""# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
# Definitions of type based summaries in the {self.friendlyname} framework.
return typeBasedSummaryAddsTo
def save(self, extensions, extension):
# Create a file for each namespace and save models.
extensionTemplate = """# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
extensions:
{typeBasedSummaryAddsTo}"""
def save(self, content, target):
with open(target, "w") as targetYml:
targetYml.write(content)
print("Models as data extensions written to " + target)
{0}"""
for entry in extensions:
target = os.path.join(self.generatedFrameworks, entry + extension)
with open(target, "w") as f:
f.write(extensionTemplate.format(extensions[entry]))
print("Models as data extensions written to " + target)
def run(self):
@@ -204,7 +194,7 @@ extensions:
sys.exit(0)
if self.generateSinks or self.generateSinks or self.generateSummaries:
self.save(content, self.frameworkTarget)
self.save(content, ".model.yml")
if self.generateTypeBasedSummaries:
self.save(typeBasedContent, self.typeBasedFrameworkTarget)
self.save(typeBasedContent, ".typebased.model.yml")

13
misc/scripts/models-as-data/helpers.py
View File

@@ -38,3 +38,16 @@ def readData(workDir, bqrsFile):
except KeyError:
print('Unexpected JSON output - no tuples found')
exit(1)
def insert_update(rows, key, value):
if key in rows:
rows[key] += value
else:
rows[key] = value
def merge(*dicts):
merged = {}
for d in dicts:
for entry in d:
insert_update(merged, entry, d[entry])
return merged