diff --git a/ql/scripts/identical-files.json b/ql/scripts/identical-files.json deleted file mode 100644 index ee52deb2fb6..00000000000 --- a/ql/scripts/identical-files.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "SSA": [ - "codeql/csharp/ql/src/semmle/code/csharp/dataflow/internal/SsaImplCommon.qll", - "ql/src/codeql_ql/dataflow/internal/SsaImplCommon.qll" - ], - "DataFlow Common": [ - "codeql/csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll", - "ql/src/codeql_ql/dataflow/internal/DataFlowImplCommon.qll" - ], - "DataFlow": [ - "codeql/csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll", - "ql/src/codeql_ql/dataflow/internal/DataFlowImpl.qll" - ], - "TypeTracker": [ - "codeql/python/ql/src/experimental/typetracking/TypeTracker.qll", - "ql/src/codeql_ql/typetracking/TypeTracker.qll" - ] -} \ No newline at end of file diff --git a/ql/scripts/merge_stats.py b/ql/scripts/merge_stats.py deleted file mode 100644 index d2cbe0a3e64..00000000000 --- a/ql/scripts/merge_stats.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/python - -# This script merges a number of stats files to produce a single stats file. - -import sys -from lxml import etree -import argparse - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--output', required=True, help="Path of the output file.") - parser.add_argument('--normalise', required=True, help="Name of the relation to normalise the sizes on.") - parser.add_argument('--unscaled-stats', default=[], action='append', help="A stats file which should not be normalised.") - parser.add_argument('inputs', nargs='*', help="The other stats files") - return parser.parse_args() - -def die(msg): - sys.stderr.write('Error: ' + msg + '\n') - sys.exit(1) - -def main(): - args = parse_args() - inputs = args.inputs - output = args.output - normalise = args.normalise - unscaled_stats = args.unscaled_stats - - print("Merging %s into %s normalising on '%s'." % (', '.join(inputs), output, normalise)) - do_xml_files(output, inputs, unscaled_stats, normalise) - -def read_sized_xml(xml_file, name): - # Take the size of the named table as the size of the codebase - xml = etree.parse(xml_file) - ns = xml.xpath("stats/relation[name='%s']/cardinality" % name) - if len(ns) == 0: - die('Sized stats file ' + xml_file + ' does not have a cardinality for normalisation relation ' + name + '.') - n = ns[0] - size = int(n.text) - return (xml, size) - -def scale(xml, size, max_size): - # Scale up the contents of all the and tags - for v in xml.xpath(".//v|.//cardinality"): - v.text = str((int(v.text) * max_size) // size) - -def do_xml_files(output, scaled_xml_files, unscaled_xml_files, name): - # The result starts off empty - result = etree.Element("dbstats") - - # Scale all of the stats so that they might have come code bases of - # the same size - sized_xmls = [read_sized_xml(xml_file, name) - for xml_file in scaled_xml_files] - if sized_xmls != []: - max_size = max([size for (xml, size) in sized_xmls]) - for (xml, size) in sized_xmls: - scale(xml, size, max_size) - unsized_xmls = list(map(etree.parse, unscaled_xml_files)) - xmls = [xml for (xml, size) in sized_xmls] + unsized_xmls - - # Put all the stats in a single XML doc so that we can search them - # more easily - merged_xml = etree.Element("merged") - for xml in xmls: - merged_xml.append(xml.getroot()) - - # For each value of , take the tag with the biggest - typesizes = etree.SubElement(result, "typesizes") - typenames = sorted(set ([ typesize.find("k").text for typesize in merged_xml.xpath("dbstats/typesizes/e")])) - for typename in typenames: - xs = merged_xml.xpath("dbstats/typesizes/e[k='" + typename + "']") - sized_xs = [(int(x.find("v").text), x) for x in xs] - (_, x) = max(sized_xs, key = lambda p: p[0]) - typesizes.append(x) - - # For each value of , take the tag with - # the biggest - stats = etree.SubElement(result, "stats") - - relnames = sorted(set ([relation.find("name").text for relation in merged_xml.xpath("dbstats/stats/relation") ])) - for relname in relnames: - rels = merged_xml.xpath("dbstats/stats/relation[name='" + relname + "']") - sized_rels = [(int(rel.find("cardinality").text), rel) for rel in rels] - (_, rel) = max(sized_rels, key = lambda p: p[0]) - stats.append(rel) - - with open(output, 'wb') as f: - f.write(etree.tostring(result, pretty_print=True)) - -main() diff --git a/ql/scripts/sync-identical-files.py b/ql/scripts/sync-identical-files.py deleted file mode 100755 index 17fe22808e9..00000000000 --- a/ql/scripts/sync-identical-files.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 - -# Due to various technical limitations, we sometimes have files that need to be -# kept identical in the repository. This script loads a database of such -# files and can perform two functions: check whether they are still identical, -# and overwrite the others with a master copy if needed. -# The script that does the actual work is `sync-files.py`, which lives in the `codeql` submodule. -import sys -import os - -sys.path.append(os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../codeql/config'))) - -import importlib -syncfiles = importlib.import_module('sync-files') - -def chdir_repo_root(): - root_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..') - os.chdir(root_path) - -def sync_identical_files(): - if len(sys.argv) == 1: - master_file_picker = lambda files: None - elif len(sys.argv) == 2: - if sys.argv[1] == "--latest": - master_file_picker = syncfiles.choose_latest_file - elif os.path.isfile(sys.argv[1]): - master_file_picker = lambda files: syncfiles.choose_master_file(sys.argv[1], files) - else: - raise Exception("File not found") - else: - raise Exception("Bad command line or file not found") - chdir_repo_root() - syncfiles.load_if_exists('.', 'scripts/identical-files.json') - for group_name, files in syncfiles.file_groups.items(): - syncfiles.check_group(group_name, files, master_file_picker, syncfiles.emit_local_error) - -def main(): - sync_identical_files() - - if syncfiles.local_error_count > 0: - exit(1) - else: - print(__file__ +": All checks OK.") - -if __name__ == "__main__": - main()