From 5e98ff48ff702b5a72ae2087d6d3747109988bce Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen <rasmuswl@github.com>
Date: Tue, 12 Sep 2023 16:22:27 +0200
Subject: [PATCH] Python: Add script to process results from MRVA (bqrs files)

Also makes `empty.model.yml` empty once again
---
 .../frameworks/data/internal/empty.model.yml  | 29 +-------
 .../ClassHierarchy/process-mrva-results.py    | 68 +++++++++++++++++++
 2 files changed, 69 insertions(+), 28 deletions(-)
 create mode 100644 python/ql/src/meta/ClassHierarchy/process-mrva-results.py

diff --git a/python/ql/lib/semmle/python/frameworks/data/internal/empty.model.yml b/python/ql/lib/semmle/python/frameworks/data/internal/empty.model.yml
index 4219c8ac175..49a1f01983f 100644
--- a/python/ql/lib/semmle/python/frameworks/data/internal/empty.model.yml
+++ b/python/ql/lib/semmle/python/frameworks/data/internal/empty.model.yml
@@ -23,34 +23,7 @@ extensions:
   - addsTo:
       pack: codeql/python-all
       extensible: typeModel
-    data:
-      - ["flask.MethodView~Subclass","flask_restplus","Member[api].Member[SwaggerView]"]
-      - ["flask.MethodView~Subclass","flask_restplus","Member[resource].Member[Resource]"]
-      - ["flask.MethodView~Subclass","flask_restplus","Member[api].Member[Resource]"]
-      - ["flask.MethodView~Subclass","flask_restplus","Member[resource].Member[MethodView]"]
-      - ["flask.MethodView~Subclass","flask_restplus","Member[Resource]"]
-
-      - ["flask.MethodView~Subclass","flask_restx","Member[api].Member[SwaggerView]"]
-      - ["flask.MethodView~Subclass","flask_restx","Member[resource].Member[Resource]"]
-      - ["flask.MethodView~Subclass","flask_restx","Member[api].Member[Resource]"]
-      - ["flask.MethodView~Subclass","flask_restx","Member[resource].Member[MethodView]"]
-      - ["flask.MethodView~Subclass","flask_restx","Member[Resource]"]
-
-      - ["flask.MethodView~Subclass", "flask_restful", "Member[Resource]"]
-
-      - ["fastapi.APIRouter~Subclass","fastapi_utils","Member[inferring_router].Member[InferringRouter]"]
-      - ["fastapi.APIRouter~Subclass","fastapi_utils","Member[inferring_router].Member[APIRouter]"]
-      - ["fastapi.APIRouter~Subclass","fastapi_utils","Member[cbv].Member[APIRouter]"]
-
-      - ["django.forms.BaseForm~Subclass","haystack","Member[forms].Member[ModelSearchForm]"]
-      - ["django.forms.BaseForm~Subclass","haystack","Member[forms].Member[SearchForm]"]
-      - ["django.forms.BaseForm~Subclass","haystack","Member[forms].Member[FacetedSearchForm]"]
-      - ["django.forms.BaseForm~Subclass","haystack","Member[forms].Member[HighlightedSearchForm]"]
-      - ["django.forms.BaseForm~Subclass","haystack","Member[forms].Member[HighlightedModelSearchForm]"]
-      - ["django.forms.BaseForm~Subclass","haystack","Member[forms].Member[FacetedModelSearchForm]"]
-      - ["django.forms.BaseForm~Subclass","haystack","Member[views].Member[FacetedSearchForm]"]
-      - ["django.forms.BaseForm~Subclass","haystack","Member[views].Member[ModelSearchForm]"]
-
+    data: []
 
   - addsTo:
       pack: codeql/python-all
diff --git a/python/ql/src/meta/ClassHierarchy/process-mrva-results.py b/python/ql/src/meta/ClassHierarchy/process-mrva-results.py
new file mode 100644
index 00000000000..d5b25f24e21
--- /dev/null
+++ b/python/ql/src/meta/ClassHierarchy/process-mrva-results.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+import sys
+import glob
+from pathlib import Path
+import json
+import subprocess
+from collections import defaultdict
+import yaml
+
+VERSION = "process-mrva-results 0.0.1"
+
+mad_path = Path(__file__).parent.parent.parent.parent / "lib/semmle/python/frameworks/data/internal/"
+
+assert mad_path.exists(), mad_path
+
+package_data = defaultdict(set)
+
+# gather data
+for f in glob.glob(f"{sys.argv[1]}/**/results/results.bqrs", recursive=True):
+    print(f"Processing {f}")
+
+    json_data = subprocess.check_output(["codeql", "bqrs", "decode", "--format=json", f])
+    select = json.loads(json_data)
+
+    for t in select["#select"]["tuples"]:
+        pkg = t[1]
+        package_data[pkg].add(tuple(t))
+
+# process data
+
+def wrap_in_template(data):
+    return {
+        "extensions": [
+            {
+                "addsTo": {
+                    "pack": "codeql/python-all",
+                    "extensible": "typeModel",
+                },
+                "data": data,
+            }
+        ]
+    }
+
+def parse_from_file(path):
+    if not path.exists():
+        return set()
+
+    text = path.read_text()
+    assert text.startswith(f"# {VERSION}\n"), f"{path}: {text[:100]}"
+
+    raw_data = yaml.safe_load(text)
+    assert len(raw_data["extensions"]) == 1, path
+    assert raw_data["extensions"][0]["addsTo"]["extensible"] == "typeModel", path
+
+    return set(tuple(x) for x in raw_data["extensions"][0]["data"])
+
+for pkg in package_data:
+    pkg_path = mad_path / f"auto-{pkg}.model.yml"
+
+    all_data = parse_from_file(pkg_path)
+    all_data.update(package_data[pkg])
+
+    as_lists = [list(t) for t in all_data]
+
+    data_for_yaml = wrap_in_template(as_lists)
+
+    pkg_path.write_text(f"# {VERSION}\n" + yaml.dump(data_for_yaml, indent=2))