hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

QL: delete old copy of the identical files scripts

Browse Source
This commit is contained in:
Erik Krogh Kristensen
2022-04-26 10:37:14 +02:00
parent 6738270b65
commit ba3aa4f186
3 changed files with 0 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
ql/scripts/identical-files.json
View File

@@ -1,18 +0,0 @@
{
"SSA": [
"codeql/csharp/ql/src/semmle/code/csharp/dataflow/internal/SsaImplCommon.qll",
"ql/src/codeql_ql/dataflow/internal/SsaImplCommon.qll"
],
"DataFlow Common": [
"codeql/csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
"ql/src/codeql_ql/dataflow/internal/DataFlowImplCommon.qll"
],
"DataFlow": [
"codeql/csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
"ql/src/codeql_ql/dataflow/internal/DataFlowImpl.qll"
],
"TypeTracker": [
"codeql/python/ql/src/experimental/typetracking/TypeTracker.qll",
"ql/src/codeql_ql/typetracking/TypeTracker.qll"
]
}

90
ql/scripts/merge_stats.py
View File

@@ -1,90 +0,0 @@
#!/usr/bin/python
# This script merges a number of stats files to produce a single stats file.
import sys
from lxml import etree
import argparse
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--output', required=True, help="Path of the output file.")
parser.add_argument('--normalise', required=True, help="Name of the relation to normalise the sizes on.")
parser.add_argument('--unscaled-stats', default=[], action='append', help="A stats file which should not be normalised.")
parser.add_argument('inputs', nargs='*', help="The other stats files")
return parser.parse_args()
def die(msg):
sys.stderr.write('Error: ' + msg + '\n')
sys.exit(1)
def main():
args = parse_args()
inputs = args.inputs
output = args.output
normalise = args.normalise
unscaled_stats = args.unscaled_stats
print("Merging %s into %s normalising on '%s'." % (', '.join(inputs), output, normalise))
do_xml_files(output, inputs, unscaled_stats, normalise)
def read_sized_xml(xml_file, name):
# Take the size of the named table as the size of the codebase
xml = etree.parse(xml_file)
ns = xml.xpath("stats/relation[name='%s']/cardinality" % name)
if len(ns) == 0:
die('Sized stats file ' + xml_file + ' does not have a cardinality for normalisation relation ' + name + '.')
n = ns[0]
size = int(n.text)
return (xml, size)
def scale(xml, size, max_size):
# Scale up the contents of all the <v> and <cardinality> tags
for v in xml.xpath(".//v|.//cardinality"):
v.text = str((int(v.text) * max_size) // size)
def do_xml_files(output, scaled_xml_files, unscaled_xml_files, name):
# The result starts off empty
result = etree.Element("dbstats")
# Scale all of the stats so that they might have come code bases of
# the same size
sized_xmls = [read_sized_xml(xml_file, name)
for xml_file in scaled_xml_files]
if sized_xmls != []:
max_size = max([size for (xml, size) in sized_xmls])
for (xml, size) in sized_xmls:
scale(xml, size, max_size)
unsized_xmls = list(map(etree.parse, unscaled_xml_files))
xmls = [xml for (xml, size) in sized_xmls] + unsized_xmls
# Put all the stats in a single XML doc so that we can search them
# more easily
merged_xml = etree.Element("merged")
for xml in xmls:
merged_xml.append(xml.getroot())
# For each value of <e><k>, take the <e> tag with the biggest <e><v>
typesizes = etree.SubElement(result, "typesizes")
typenames = sorted(set ([ typesize.find("k").text for typesize in merged_xml.xpath("dbstats/typesizes/e")]))
for typename in typenames:
xs = merged_xml.xpath("dbstats/typesizes/e[k='" + typename + "']")
sized_xs = [(int(x.find("v").text), x) for x in xs]
(_, x) = max(sized_xs, key = lambda p: p[0])
typesizes.append(x)
# For each value of <relation><name>, take the <relation> tag with
# the biggest <relation><cardinality>
stats = etree.SubElement(result, "stats")
relnames = sorted(set ([relation.find("name").text for relation in merged_xml.xpath("dbstats/stats/relation") ]))
for relname in relnames:
rels = merged_xml.xpath("dbstats/stats/relation[name='" + relname + "']")
sized_rels = [(int(rel.find("cardinality").text), rel) for rel in rels]
(_, rel) = max(sized_rels, key = lambda p: p[0])
stats.append(rel)
with open(output, 'wb') as f:
f.write(etree.tostring(result, pretty_print=True))
main()

46
ql/scripts/sync-identical-files.py
View File

@@ -1,46 +0,0 @@
#!/usr/bin/env python3
# Due to various technical limitations, we sometimes have files that need to be
# kept identical in the repository. This script loads a database of such
# files and can perform two functions: check whether they are still identical,
# and overwrite the others with a master copy if needed.
# The script that does the actual work is `sync-files.py`, which lives in the `codeql` submodule.
import sys
import os
sys.path.append(os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../codeql/config')))
import importlib
syncfiles = importlib.import_module('sync-files')
def chdir_repo_root():
root_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
os.chdir(root_path)
def sync_identical_files():
if len(sys.argv) == 1:
master_file_picker = lambda files: None
elif len(sys.argv) == 2:
if sys.argv[1] == "--latest":
master_file_picker = syncfiles.choose_latest_file
elif os.path.isfile(sys.argv[1]):
master_file_picker = lambda files: syncfiles.choose_master_file(sys.argv[1], files)
else:
raise Exception("File not found")
else:
raise Exception("Bad command line or file not found")
chdir_repo_root()
syncfiles.load_if_exists('.', 'scripts/identical-files.json')
for group_name, files in syncfiles.file_groups.items():
syncfiles.check_group(group_name, files, master_file_picker, syncfiles.emit_local_error)
def main():
sync_identical_files()
if syncfiles.local_error_count > 0:
exit(1)
else:
print(__file__ +": All checks OK.")
if __name__ == "__main__":
main()