Move files to ruby subfolder

ruby/scripts/create-extractor-pack.ps1

@@ -0,0 +1,12 @@
+cargo build --release
+
+cargo run --release -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+codeql query format -i ql\lib\codeql/ruby\ast\internal\TreeSitter.qll
+
+rm -Recurse -Force extractor-pack
+mkdir extractor-pack | Out-Null
+cp codeql-extractor.yml, ql\lib\ruby.dbscheme, ql\lib\ruby.dbscheme.stats extractor-pack
+cp -Recurse tools extractor-pack
+mkdir extractor-pack\tools\win64 | Out-Null
+cp target\release\ruby-extractor.exe extractor-pack\tools\win64\extractor.exe
+cp target\release\ruby-autobuilder.exe extractor-pack\tools\win64\autobuilder.exe

ruby/scripts/create-extractor-pack.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -eux
+
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+  platform="linux64"
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+  platform="osx64"
+else
+  echo "Unknown OS"
+  exit 1
+fi
+
+cargo build --release
+
+cargo run --release -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+codeql query format -i ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+
+rm -rf extractor-pack
+mkdir -p extractor-pack
+cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme ql/lib/ruby.dbscheme.stats extractor-pack/
+mkdir -p extractor-pack/tools/${platform}
+cp target/release/ruby-extractor extractor-pack/tools/${platform}/extractor
+cp target/release/ruby-autobuilder extractor-pack/tools/${platform}/autobuilder

ruby/scripts/identical-files.json

@@ -0,0 +1,48 @@
+{
+    "SSA": [
+        "codeql/csharp/ql/lib/semmle/code/csharp/dataflow/internal/SsaImplCommon.qll",
+        "ql/lib/codeql/ruby/dataflow/internal/SsaImplCommon.qll"
+    ],
+    "DataFlow Common": [
+        "codeql/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
+        "ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll"
+    ],
+    "DataFlow": [
+        "codeql/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
+        "ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
+        "ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll"
+    ],
+    "DataFlow2": [
+        "codeql/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll",
+        "ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll"
+    ],
+    "DataFlow Consistency": [
+        "codeql/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll",
+        "ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll"
+    ],
+    "DataFlow Summaries": [
+        "codeql/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll",
+        "ql/lib/codeql/ruby/dataflow/internal/FlowSummaryImpl.qll"
+    ],
+    "TaintTracking": [
+        "codeql/csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
+        "ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
+    ],
+    "TypeTracker": [
+        "codeql/python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll",
+        "ql/lib/codeql/ruby/typetracking/TypeTracker.qll"
+    ],
+    "Inline Test Expectations": [
+        "codeql/python/ql/test/TestUtilities/InlineExpectationsTest.qll",
+        "ql/test/TestUtilities/InlineExpectationsTest.qll"
+    ],
+    "CFG": [
+        "codeql/csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll",
+        "ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll"
+    ],
+    "ReDoS Polynomial Ruby/Python/JS": [
+        "codeql/javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll",
+        "codeql/python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll",
+        "ql/lib/codeql/ruby/regexp/SuperlinearBackTracking.qll"
+    ]
+}

ruby/scripts/merge_stats.py

@@ -0,0 +1,90 @@
+#!/usr/bin/python
+
+# This script merges a number of stats files to produce a single stats file.
+
+import sys
+from lxml import etree
+import argparse
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--output',              required=True,               help="Path of the output file.")
+    parser.add_argument('--normalise',           required=True,               help="Name of the relation to normalise the sizes on.")
+    parser.add_argument('--unscaled-stats',      default=[], action='append', help="A stats file which should not be normalised.")
+    parser.add_argument('inputs',                nargs='*',                   help="The other stats files")
+    return parser.parse_args()
+
+def die(msg):
+    sys.stderr.write('Error: ' + msg + '\n')
+    sys.exit(1)
+
+def main():
+    args = parse_args()
+    inputs    = args.inputs
+    output    = args.output
+    normalise = args.normalise
+    unscaled_stats = args.unscaled_stats
+
+    print("Merging %s into %s normalising on '%s'." % (', '.join(inputs), output, normalise))
+    do_xml_files(output, inputs, unscaled_stats, normalise)
+
+def read_sized_xml(xml_file, name):
+    # Take the size of the named table as the size of the codebase
+    xml = etree.parse(xml_file)
+    ns = xml.xpath("stats/relation[name='%s']/cardinality" % name)
+    if len(ns) == 0:
+        die('Sized stats file ' + xml_file + ' does not have a cardinality for normalisation relation ' + name + '.')
+    n = ns[0]
+    size = int(n.text)
+    return (xml, size)
+
+def scale(xml, size, max_size):
+    # Scale up the contents of all the <v> and <cardinality> tags
+    for v in xml.xpath(".//v|.//cardinality"):
+        v.text = str((int(v.text) * max_size) // size)
+
+def do_xml_files(output, scaled_xml_files, unscaled_xml_files, name):
+    # The result starts off empty
+    result = etree.Element("dbstats")
+
+    # Scale all of the stats so that they might have come code bases of
+    # the same size
+    sized_xmls = [read_sized_xml(xml_file, name)
+                  for xml_file in scaled_xml_files]
+    if sized_xmls != []:
+        max_size = max([size for (xml, size) in sized_xmls])
+        for (xml, size) in sized_xmls:
+            scale(xml, size, max_size)
+    unsized_xmls = list(map(etree.parse, unscaled_xml_files))
+    xmls = [xml for (xml, size) in sized_xmls] + unsized_xmls
+
+    # Put all the stats in a single XML doc so that we can search them
+    # more easily
+    merged_xml = etree.Element("merged")
+    for xml in xmls:
+        merged_xml.append(xml.getroot())
+
+    # For each value of <e><k>, take the <e> tag with the biggest <e><v>
+    typesizes = etree.SubElement(result, "typesizes")
+    typenames = sorted(set ([ typesize.find("k").text for typesize in merged_xml.xpath("dbstats/typesizes/e")]))
+    for typename in typenames:
+        xs = merged_xml.xpath("dbstats/typesizes/e[k='" + typename + "']")
+        sized_xs = [(int(x.find("v").text), x) for x in xs]
+        (_, x) = max(sized_xs, key = lambda p: p[0])
+        typesizes.append(x)
+
+    # For each value of <relation><name>, take the <relation> tag with
+    # the biggest <relation><cardinality>
+    stats = etree.SubElement(result, "stats")
+    
+    relnames = sorted(set ([relation.find("name").text for relation in merged_xml.xpath("dbstats/stats/relation") ]))
+    for relname in relnames:
+        rels = merged_xml.xpath("dbstats/stats/relation[name='" + relname + "']")
+        sized_rels = [(int(rel.find("cardinality").text), rel) for rel in rels]
+        (_, rel) = max(sized_rels, key = lambda p: p[0])
+        stats.append(rel)
+
+    with open(output, 'wb') as f:
+        f.write(etree.tostring(result, pretty_print=True))
+
+main()

ruby/scripts/prepare-db-upgrade.sh

@@ -0,0 +1,106 @@
+#!/bin/sh
+#
+# Prepare the upgrade script directory for a Ruby database schema upgrade.
+
+set -e
+set -u
+
+app_name="$(basename "$0")"
+
+usage()
+{
+  exit_code="$1"
+  shift
+
+  cat >&2 <<EOF
+${app_name}: $@
+${app_name}: Generate skeleton upgrade script.
+Usage: ${app_name} [--prev_hash <COMMITISH>]"
+
+--prev-hash <COMMITISH>
+        Hash/branch to use to get SHA1 for previous DB scheme.
+        Default: origin/main
+
+Must be run within the git repo needing an update.
+EOF
+  exit "${exit_code}"
+}
+
+prev_hash="origin/main"
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    -x)
+      set -x
+      ;;
+    -h | --help)
+      usage 0
+      ;;
+    --prev-hash)
+      if [ $# -eq 1 ]; then
+        usage 2 "--prev-hash requires Commit/Branch option"
+      fi
+      shift
+      prev_hash="$1"
+      ;;
+    --)
+      shift
+      break
+      ;;
+    -*)
+      usage 2 "Unrecognised option: $1"
+      ;;
+    *)
+      break
+      ;;
+  esac
+  shift
+done
+
+if [ $# -gt 0 ]; then
+  usage 2 "Unrecognised operand: $1"
+fi
+
+scheme_file="ql/lib/ruby.dbscheme"
+upgrade_root="ql/lib/upgrades"
+
+check_hash_valid()
+{
+  if [ ${#2} -ne 40 ]; then
+    echo "Did not get expected $1 hash: $2" >&2
+    exit 2
+  fi
+}
+
+# Get the hash of the previous and current DB Schema files
+prev_hash="$(git show "${prev_hash}:${scheme_file}" | git hash-object --stdin)"
+check_hash_valid previous "${prev_hash}"
+current_hash="$(git hash-object "${scheme_file}")"
+check_hash_valid current "${current_hash}"
+if [ "${current_hash}" = "${prev_hash}" ]; then
+  echo "No work to be done."
+  exit
+fi
+
+# Copy current and new dbscheme into the upgrade dir
+upgradedir="${upgrade_root}/${prev_hash}"
+mkdir -p "${upgradedir}"
+
+cp "${scheme_file}" "${upgradedir}"
+git cat-file blob "${prev_hash}" > "${upgradedir}/old.dbscheme"
+
+# Create the template upgrade.properties file.
+cat <<EOF > "${upgradedir}/upgrade.properties"
+description: <INSERT DESCRIPTION HERE>
+compatibility: full|backwards|partial|breaking
+EOF
+
+# Tell user what we've done
+cat <<EOF
+Created upgrade directory here:
+  ${upgradedir}
+
+Please update:
+  ${upgradedir}/upgrade.properties
+with appropriate upgrade instructions
+EOF

ruby/scripts/sync-identical-files.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+# Due to various technical limitations, we sometimes have files that need to be
+# kept identical in the repository. This script loads a database of such
+# files and can perform two functions: check whether they are still identical,
+# and overwrite the others with a master copy if needed.
+# The script that does the actual work is `sync-files.py`, which lives in the `codeql` submodule.
+import sys
+import os
+
+sys.path.append(os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../codeql/config')))
+
+import importlib
+syncfiles = importlib.import_module('sync-files')
+
+def chdir_repo_root():
+    root_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
+    os.chdir(root_path)
+
+def sync_identical_files():
+    if len(sys.argv) == 1:
+        master_file_picker = lambda files: None
+    elif len(sys.argv) == 2:
+        if sys.argv[1] == "--latest":
+            master_file_picker = syncfiles.choose_latest_file
+        elif os.path.isfile(sys.argv[1]):
+            master_file_picker = lambda files: syncfiles.choose_master_file(sys.argv[1], files)
+        else:
+            raise Exception("File not found")
+    else:
+        raise Exception("Bad command line or file not found")
+    chdir_repo_root()
+    syncfiles.load_if_exists('.', 'scripts/identical-files.json')
+    for group_name, files in syncfiles.file_groups.items():
+        syncfiles.check_group(group_name, files, master_file_picker, syncfiles.emit_local_error)
+
+def main():
+    sync_identical_files()
+
+    if syncfiles.local_error_count > 0:
+        exit(1)
+    else:
+        print(__file__ +": All checks OK.")
+
+if __name__ == "__main__":
+    main()