hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Shared: Remove the language-specific model generator scripts

Browse Source
This commit is contained in:
Simon Friis Vindum
2025-05-02 13:21:10 +02:00
parent 74669cb0cb
commit c6d95ceeb0
9 changed files with 54 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/mad_modelDiff.yml vendored
View File

@@ -68,7 +68,7 @@ jobs:
DATABASE=$2
cd codeql-$QL_VARIANT
SHORTNAME=`basename $DATABASE`
python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT
python misc/scripts/models-as-data/generate_mad.py --language java --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT
mkdir -p $MODELS/$SHORTNAME
mv java/ql/lib/ext/generated/$SHORTNAME/$QL_VARIANT $MODELS/$SHORTNAME
cd ..

15
cpp/ql/src/utils/modelgenerator/GenerateFlowModel.py
View File

@@ -1,15 +0,0 @@
#!/usr/bin/python3
import sys
import os.path
import subprocess
# Add Model as Data script directory to sys.path.
gitroot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
madpath = os.path.join(gitroot, "misc/scripts/models-as-data/")
sys.path.append(madpath)
import generate_flow_model as model
language = "cpp"
model.Generator.make(language).run()

8
csharp/ql/src/change-notes/2025-05-02-mad-generator-renamed.md Normal file
View File

@@ -0,0 +1,8 @@
---
category: minorAnalysis
---
* Changes to the MaD model generation infrastructure:
* The `csharp/ql/src/utils/modelgenerator/GenerateFlowModel.py` script has
been removed. The `/misc/scripts/models-as-data/generate_mad.py` script now
supports being called directly and should be used instead. The script
requires a `--language` argument but otherwise functions identically.

15
csharp/ql/src/utils/modelgenerator/GenerateFlowModel.py
View File

@@ -1,15 +0,0 @@
#!/usr/bin/python3
import sys
import os.path
import subprocess
# Add Model as Data script directory to sys.path.
gitroot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
madpath = os.path.join(gitroot, "misc/scripts/models-as-data/")
sys.path.append(madpath)
import generate_flow_model as model
language = "csharp"
model.Generator.make(language).run()

8
java/ql/src/change-notes/2025-05-02-mad-generator-renamed.md Normal file
View File

@@ -0,0 +1,8 @@
---
category: minorAnalysis
---
* Changes to the MaD model generation infrastructure:
* The `java/ql/src/utils/modelgenerator/GenerateFlowModel.py` script has
been removed. The `/misc/scripts/models-as-data/generate_mad.py` script now
supports being called directly and should be used instead. The script
requires a `--language` argument but otherwise functions identically.

15
java/ql/src/utils/modelgenerator/GenerateFlowModel.py
View File

@@ -1,15 +0,0 @@
#!/usr/bin/python3
import sys
import os.path
import subprocess
# Add Model as Data script directory to sys.path.
gitroot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
madpath = os.path.join(gitroot, "misc/scripts/models-as-data/")
sys.path.append(madpath)
import generate_flow_model as model
language = "java"
model.Generator.make(language).run()

4
java/ql/src/utils/modelgenerator/RegenerateModels.py
View File

@@ -35,8 +35,8 @@ def regenerateModel(lgtmSlug, extractedDb):
sys.exit(1)
modelFile = lgtmSlugToModelFile[lgtmSlug]
codeQlRoot = findGitRoot()
subprocess.check_call([codeQlRoot + "/java/ql/src/utils/modelgenerator/GenerateFlowModel.py",
"--with-summaries", "--with-sinks", "--with-neutrals",
subprocess.check_call([codeQlRoot + "/misc/scripts/models-as-data/generate_mad.py",
"--language", "java", "--with-summaries", "--with-sinks", "--with-neutrals",
extractedDb, modelFile])
print("Regenerated " + modelFile)
shutil.rmtree(tmpDir)

57
misc/scripts/models-as-data/generate_flow_model.py → misc/scripts/models-as-data/generate_mad.py
View File

@@ -1,10 +1,8 @@
#!/usr/bin/python3
import helpers
import json
import os
import os.path
import shlex
import subprocess
import sys
import tempfile
@@ -27,24 +25,13 @@ def parseData(data):
return rows
class Generator:
def __init__ (self, language):
self.language = language
self.generateSinks = False
self.generateSources = False
self.generateSummaries = False
self.generateNeutrals = False
self.generateTypeBasedSummaries = False
self.dryRun = False
self.dirname = "modelgenerator"
def printHelp(self):
def printHelp():
print(f"""Usage:
python3 GenerateFlowModel.py <library-database> [DIR] [--with-sinks] [--with-sources] [--with-summaries] [--with-neutrals] [--with-typebased-summaries] [--dry-run]
python3 generate_mad.py <library-database> [DIR] --language LANGUAGE [--with-sinks] [--with-sources] [--with-summaries] [--with-neutrals] [--with-typebased-summaries] [--dry-run]
This generates summary, source, sink and neutral models for the code in the database.
The files will be placed in `{self.language}/ql/lib/ext/generated/DIR`
The files will be placed in `LANGUAGE/ql/lib/ext/generated/DIR`
Which models are generated is controlled by the flags:
--with-sinks
@@ -57,14 +44,25 @@ If none of these flags are specified, all models are generated except for the ty
--dry-run: Only run the queries, but don't write to file.
Example invocations:
$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db
$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db --with-sinks
$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db --with-sinks my_directory
$ python3 generate_mad.py /tmp/dbs/my_library_db
$ python3 generate_mad.py /tmp/dbs/my_library_db --with-sinks
$ python3 generate_mad.py /tmp/dbs/my_library_db --with-sinks my_directory
Requirements: `codeql` should appear on your path.
""")
class Generator:
def __init__(self, language):
self.language = language
self.generateSinks = False
self.generateSources = False
self.generateSummaries = False
self.generateNeutrals = False
self.generateTypeBasedSummaries = False
self.dryRun = False
self.dirname = "modelgenerator"
def setenvironment(self, database, folder):
self.codeQlRoot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
@@ -76,12 +74,22 @@ Requirements: `codeql` should appear on your path.
@staticmethod
def make(language):
generator = Generator(language)
def make():
'''Create a generator instance based on command line arguments.'''
if any(s == "--help" for s in sys.argv):
generator.printHelp()
printHelp()
sys.exit(0)
if "--language" in sys.argv:
language = sys.argv[sys.argv.index("--language") + 1]
sys.argv.remove("--language")
sys.argv.remove(language)
else:
print("Error: Language not specified. Use --language <language>.")
sys.exit(0)
generator = Generator(language=language)
if "--with-sinks" in sys.argv:
sys.argv.remove("--with-sinks")
generator.generateSinks = True
@@ -115,7 +123,7 @@ Requirements: `codeql` should appear on your path.
n = len(sys.argv)
if n < 2:
generator.printHelp()
printHelp()
sys.exit(1)
elif n == 2:
generator.setenvironment(sys.argv[1], "")
@@ -204,3 +212,6 @@ extensions:
if self.generateTypeBasedSummaries:
self.save(typeBasedContent, ".typebased.model.yml")
if __name__ == '__main__':
Generator.make().run()

15
rust/ql/src/utils/modelgenerator/GenerateFlowModel.py
View File

@@ -1,15 +0,0 @@
#!/usr/bin/python3
import sys
import os.path
import subprocess
# Add Model as Data script directory to sys.path.
gitroot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
madpath = os.path.join(gitroot, "misc/scripts/models-as-data/")
sys.path.append(madpath)
import generate_flow_model as model
language = "rust"
model.Generator.make(language).run()