mirror of
https://github.com/github/codeql.git
synced 2026-05-03 04:39:29 +02:00
Java: Move Generator creation into class definition.
This commit is contained in:
Michael Nebel
200
java/ql/src/utils/model-generator/generate_flow_model.py
Executable file → Normal file
200
java/ql/src/utils/model-generator/generate_flow_model.py
Executable file → Normal file
@@ -13,6 +13,10 @@ language = "java"
|
||||
class Generator:
|
||||
def __init__ (self, language):
|
||||
self.language = language
|
||||
self.generateSinks = False
|
||||
self.generateSources = False
|
||||
self.generateSummaries = False
|
||||
self.dryRun = False
|
||||
|
||||
def printHelp(self):
|
||||
print(f"""Usage:
|
||||
@@ -38,99 +42,97 @@ $ python3 GenerateFlowModel.py /tmp/dbs/my_library_db "mylibrary/FrameworkSinks.
|
||||
Requirements: `codeql` should both appear on your path.
|
||||
""")
|
||||
|
||||
generator = Generator(language)
|
||||
|
||||
if any(s == "--help" for s in sys.argv):
|
||||
generator.printHelp()
|
||||
sys.exit(0)
|
||||
def setenvironment(self, target, database):
|
||||
self.codeQlRoot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
|
||||
if not target.endswith(".qll"):
|
||||
target += ".qll"
|
||||
self.filename = os.path.basename(target)
|
||||
self.shortname = self.filename[:-4]
|
||||
self.database = database
|
||||
self.generatedFrameworks = os.path.join(
|
||||
self.codeQlRoot, f"{self.language}/ql/lib/semmle/code/{self.language}/frameworks/")
|
||||
self.frameworkTarget = os.path.join(self.generatedFrameworks, target)
|
||||
|
||||
generateSinks = False
|
||||
generateSources = False
|
||||
generateSummaries = False
|
||||
dryRun = False
|
||||
self.workDir = tempfile.mkdtemp()
|
||||
os.makedirs(self.generatedFrameworks, exist_ok=True)
|
||||
|
||||
if "--with-sinks" in sys.argv:
|
||||
sys.argv.remove("--with-sinks")
|
||||
generateSinks = True
|
||||
@staticmethod
|
||||
def make(language):
|
||||
generator = Generator(language)
|
||||
if any(s == "--help" for s in sys.argv):
|
||||
generator.printHelp()
|
||||
sys.exit(0)
|
||||
|
||||
if "--with-sources" in sys.argv:
|
||||
sys.argv.remove("--with-sources")
|
||||
generateSources = True
|
||||
if "--with-sinks" in sys.argv:
|
||||
sys.argv.remove("--with-sinks")
|
||||
generator.generateSinks = True
|
||||
|
||||
if "--with-summaries" in sys.argv:
|
||||
sys.argv.remove("--with-summaries")
|
||||
generateSummaries = True
|
||||
if "--with-sources" in sys.argv:
|
||||
sys.argv.remove("--with-sources")
|
||||
generator.generateSources = True
|
||||
|
||||
if "--dry-run" in sys.argv:
|
||||
sys.argv.remove("--dry-run")
|
||||
dryRun = True
|
||||
if "--with-summaries" in sys.argv:
|
||||
sys.argv.remove("--with-summaries")
|
||||
generator.generateSummaries = True
|
||||
|
||||
if not generateSinks and not generateSources and not generateSummaries:
|
||||
generateSinks = generateSources = generateSummaries = True
|
||||
if "--dry-run" in sys.argv:
|
||||
sys.argv.remove("--dry-run")
|
||||
generator.dryRun = True
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
generator.printHelp()
|
||||
sys.exit(1)
|
||||
if not generator.generateSinks and not generator.generateSources and not generator.generateSummaries:
|
||||
generator.generateSinks = generator.generateSources = generator.generateSummaries = True
|
||||
|
||||
codeQlRoot = subprocess.check_output(
|
||||
["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
|
||||
targetQll = sys.argv[2]
|
||||
if not targetQll.endswith(".qll"):
|
||||
targetQll += ".qll"
|
||||
filename = os.path.basename(targetQll)
|
||||
shortname = filename[:-4]
|
||||
generatedFrameworks = os.path.join(
|
||||
codeQlRoot, f"{language}/ql/lib/semmle/code/{language}/frameworks/")
|
||||
frameworkTarget = os.path.join(generatedFrameworks, targetQll)
|
||||
if len(sys.argv) != 3:
|
||||
generator.printHelp()
|
||||
sys.exit(1)
|
||||
|
||||
generator.setenvironment(sys.argv[2], sys.argv[1])
|
||||
return generator
|
||||
|
||||
workDir = tempfile.mkdtemp()
|
||||
os.makedirs(generatedFrameworks, exist_ok=True)
|
||||
def runQuery(self, infoMessage, query):
|
||||
print("########## Querying " + infoMessage + "...")
|
||||
queryFile = os.path.join(os.path.dirname(
|
||||
__file__), query)
|
||||
resultBqrs = os.path.join(self.workDir, "out.bqrs")
|
||||
cmd = ['codeql', 'query', 'run', queryFile, '--database',
|
||||
self.database, '--output', resultBqrs, '--threads', '8']
|
||||
|
||||
ret = subprocess.call(cmd)
|
||||
if ret != 0:
|
||||
print("Failed to generate " + infoMessage +
|
||||
". Failed command was: " + shlex.join(cmd))
|
||||
sys.exit(1)
|
||||
return self.readRows(resultBqrs)
|
||||
|
||||
|
||||
def runQuery(infoMessage, query):
|
||||
print("########## Querying " + infoMessage + "...")
|
||||
database = sys.argv[1]
|
||||
queryFile = os.path.join(os.path.dirname(
|
||||
__file__), query)
|
||||
resultBqrs = os.path.join(workDir, "out.bqrs")
|
||||
cmd = ['codeql', 'query', 'run', queryFile, '--database',
|
||||
database, '--output', resultBqrs, '--threads', '8']
|
||||
def readRows(self, bqrsFile):
|
||||
generatedJson = os.path.join(self.workDir, "out.json")
|
||||
cmd = ['codeql', 'bqrs', 'decode', bqrsFile,
|
||||
'--format=json', '--output', generatedJson]
|
||||
ret = subprocess.call(cmd)
|
||||
if ret != 0:
|
||||
print("Failed to decode BQRS. Failed command was: " + shlex.join(cmd))
|
||||
sys.exit(1)
|
||||
|
||||
ret = subprocess.call(cmd)
|
||||
if ret != 0:
|
||||
print("Failed to generate " + infoMessage +
|
||||
". Failed command was: " + shlex.join(cmd))
|
||||
sys.exit(1)
|
||||
return readRows(resultBqrs)
|
||||
with open(generatedJson) as f:
|
||||
results = json.load(f)
|
||||
|
||||
try:
|
||||
results['#select']['tuples']
|
||||
except KeyError:
|
||||
print('Unexpected JSON output - no tuples found')
|
||||
exit(1)
|
||||
|
||||
rows = ""
|
||||
for (row) in results['#select']['tuples']:
|
||||
rows += " \"" + row[0] + "\",\n"
|
||||
|
||||
return rows[:-2]
|
||||
|
||||
|
||||
def readRows(bqrsFile):
|
||||
generatedJson = os.path.join(workDir, "out.json")
|
||||
cmd = ['codeql', 'bqrs', 'decode', bqrsFile,
|
||||
'--format=json', '--output', generatedJson]
|
||||
ret = subprocess.call(cmd)
|
||||
if ret != 0:
|
||||
print("Failed to decode BQRS. Failed command was: " + shlex.join(cmd))
|
||||
sys.exit(1)
|
||||
|
||||
with open(generatedJson) as f:
|
||||
results = json.load(f)
|
||||
|
||||
try:
|
||||
results['#select']['tuples']
|
||||
except KeyError:
|
||||
print('Unexpected JSON output - no tuples found')
|
||||
exit(1)
|
||||
|
||||
rows = ""
|
||||
for (row) in results['#select']['tuples']:
|
||||
rows += " \"" + row[0] + "\",\n"
|
||||
|
||||
return rows[:-2]
|
||||
|
||||
|
||||
def asCsvModel(superclass, kind, rows):
|
||||
classTemplate = """
|
||||
def asCsvModel(self, superclass, kind, rows):
|
||||
classTemplate = """
|
||||
private class {0}{1}Csv extends {2} {{
|
||||
override predicate row(string row) {{
|
||||
row =
|
||||
@@ -139,35 +141,37 @@ private class {0}{1}Csv extends {2} {{
|
||||
]
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
if rows.strip() == "":
|
||||
return ""
|
||||
return classTemplate.format(shortname[0].upper() + shortname[1:], kind.capitalize(), superclass, rows)
|
||||
"""
|
||||
if rows.strip() == "":
|
||||
return ""
|
||||
return classTemplate.format(self.shortname[0].upper() + self.shortname[1:], kind.capitalize(), superclass, rows)
|
||||
|
||||
|
||||
if generateSummaries:
|
||||
summaryRows = runQuery("summary models", "CaptureSummaryModels.ql")
|
||||
summaryCsv = asCsvModel("SummaryModelCsv", "summary", summaryRows)
|
||||
generator = Generator.make(language)
|
||||
|
||||
if generator.generateSummaries:
|
||||
summaryRows = generator.runQuery("summary models", "CaptureSummaryModels.ql")
|
||||
summaryCsv = generator.asCsvModel("SummaryModelCsv", "summary", summaryRows)
|
||||
else:
|
||||
summaryCsv = ""
|
||||
|
||||
if generateSinks:
|
||||
sinkRows = runQuery("sink models", "CaptureSinkModels.ql")
|
||||
sinkCsv = asCsvModel("SinkModelCsv", "sinks", sinkRows)
|
||||
if generator.generateSinks:
|
||||
sinkRows = generator.runQuery("sink models", "CaptureSinkModels.ql")
|
||||
sinkCsv = generator.asCsvModel("SinkModelCsv", "sinks", sinkRows)
|
||||
else:
|
||||
sinkCsv = ""
|
||||
|
||||
if generateSources:
|
||||
sourceRows = runQuery("source models", "CaptureSourceModels.ql")
|
||||
sourceCsv = asCsvModel("SourceModelCsv", "sources", sourceRows)
|
||||
if generator.generateSources:
|
||||
sourceRows = generator.runQuery("source models", "CaptureSourceModels.ql")
|
||||
sourceCsv = generator.asCsvModel("SourceModelCsv", "sources", sourceRows)
|
||||
else:
|
||||
sourceCsv = ""
|
||||
|
||||
qllContents = f"""
|
||||
/** Definitions of taint steps in the {shortname} framework */
|
||||
/** Definitions of taint steps in the {generator.shortname} framework */
|
||||
|
||||
import {language}
|
||||
private import semmle.code.{language}.dataflow.ExternalFlow
|
||||
import {generator.language}
|
||||
private import semmle.code.{generator.language}.dataflow.ExternalFlow
|
||||
|
||||
{sinkCsv}
|
||||
{sourceCsv}
|
||||
@@ -175,18 +179,18 @@ private import semmle.code.{language}.dataflow.ExternalFlow
|
||||
|
||||
"""
|
||||
|
||||
if dryRun:
|
||||
if generator.dryRun:
|
||||
print("CSV Models generated, but not written to file.")
|
||||
sys.exit(0)
|
||||
|
||||
with open(frameworkTarget, "w") as frameworkQll:
|
||||
with open(generator.frameworkTarget, "w") as frameworkQll:
|
||||
frameworkQll.write(qllContents)
|
||||
|
||||
cmd = ['codeql', 'query', 'format', '--in-place', frameworkTarget]
|
||||
cmd = ['codeql', 'query', 'format', '--in-place', generator.frameworkTarget]
|
||||
ret = subprocess.call(cmd)
|
||||
if ret != 0:
|
||||
print("Failed to format query. Failed command was: " + shlex.join(cmd))
|
||||
sys.exit(1)
|
||||
|
||||
print("")
|
||||
print("CSV model written to " + frameworkTarget)
|
||||
print("CSV model written to " + generator.frameworkTarget)
|
||||
|
||||
Reference in New Issue
Block a user