Merge branch 'main' into alexdenisov/swift-first-extractor-test

swift/codegen/codegen.py

@@ -3,6 +3,7 @@
 
 from lib import generator
 import dbschemegen
+import qlgen
 
 if __name__ == "__main__":
-    generator.run(dbschemegen.generate)
+    generator.run(dbschemegen.generate, qlgen.generate)

swift/codegen/lib/generator.py

@@ -23,7 +23,5 @@ def run(*generators, tags=None):
     `generators` should be callables taking as input an option namespace and a `render.Renderer` instance
     """
     opts = _parse(tags)
-    renderer = render.Renderer(dryrun=opts.check)
     for g in generators:
-        g(opts, renderer)
-    sys.exit(1 if opts.check and renderer.done_something else 0)
+        g(opts, render.Renderer())

swift/codegen/lib/options.py

@@ -9,10 +9,12 @@ from . import paths
 
 
 def _init_options():
-    Option("--check", "-c", action="store_true")
     Option("--verbose", "-v", action="store_true")
     Option("--schema", tags=["schema"], type=pathlib.Path, default=paths.swift_dir / "codegen/schema.yml")
     Option("--dbscheme", tags=["dbscheme"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/swift.dbscheme")
+    Option("--ql-output", tags=["ql"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
+    Option("--ql-stub-output", tags=["ql"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
+    Option("--codeql-binary", tags=["ql"], default="codeql")
 
 
 _options = collections.defaultdict(list)

swift/codegen/lib/render.py

@@ -5,7 +5,6 @@
 https://mustache.github.io/
 """
 
-import hashlib
 import logging
 import pathlib
 
@@ -16,29 +15,13 @@ from . import paths
 log = logging.getLogger(__name__)
 
 
-def _md5(data):
-    return hashlib.md5(data).digest()
-
-
 class Renderer:
     """ Template renderer using mustache templates in the `templates` directory """
 
-    def __init__(self, dryrun=False):
-        """ Construct the renderer, which will not write anything if `dryrun` is `True` """
+    def __init__(self):
         self.r = pystache.Renderer(search_dirs=str(paths.lib_dir / "templates"), escape=lambda u: u)
         self.generator = paths.exe_file.relative_to(paths.swift_dir)
-        self.dryrun = dryrun
         self.written = set()
-        self.skipped = set()
-        self.erased = set()
-
-    @property
-    def done_something(self):
-        return bool(self.written or self.erased)
-
-    @property
-    def rendered(self):
-        return self.written | self.skipped
 
     def render(self, data, output: pathlib.Path):
         """ Render `data` to `output`.
@@ -50,27 +33,14 @@ class Renderer:
         mnemonic = type(data).__name__
         output.parent.mkdir(parents=True, exist_ok=True)
         data = self.r.render_name(data.template, data, generator=self.generator)
-        if output.is_file():
-            with open(output, "rb") as file:
-                if _md5(data.encode()) == _md5(file.read()):
-                    log.debug(f"skipped {output.name}")
-                    self.skipped.add(output)
-                    return
-        if self.dryrun:
-            log.error(f"would have generated {mnemonic} {output.name}")
-        else:
-            with open(output, "w") as out:
-                out.write(data)
-            log.info(f"generated {mnemonic} {output.name}")
+        with open(output, "w") as out:
+            out.write(data)
+        log.debug(f"generated {mnemonic} {output.name}")
         self.written.add(output)
 
     def cleanup(self, existing):
         """ Remove files in `existing` for which no `render` has been called """
-        for f in existing - self.written - self.skipped:
+        for f in existing - self.written:
             if f.is_file():
-                if self.dryrun:
-                    log.error(f"would have removed {f.name}")
-                else:
-                    f.unlink()
-                    log.info(f"removed {f.name}")
-                self.erased.add(f)
+                f.unlink()
+                log.info(f"removed {f.name}")

swift/codegen/lib/templates/ql_class.mustache

@@ -0,0 +1,47 @@
+// generated by {{generator}}
+{{#imports}}
+import {{.}}
+{{/imports}}
+
+class {{name}}Base extends {{db_id}}{{#bases}}, {{.}}{{/bases}} {
+  {{#root}}
+  string toString() { none() } // overridden by subclasses
+
+  {{name}}Base getResolveStep() { none() } // overridden by subclasses
+
+  {{name}}Base resolve() {
+    not exists(getResolveStep()) and result = this
+    or
+    result = getResolveStep().resolve()
+  }
+  {{/root}}
+  {{#final}}
+  override string toString() { result = "{{name}}" }
+  {{/final}}
+  {{#properties}}
+
+  {{#type_is_class}}
+  {{type}} get{{singular}}({{#params}}{{^first}}, {{/first}}{{type}} {{param}}{{/params}}) {
+    exists({{type}} {{local_var}} |
+      {{tablename}}({{#tableparams}}{{^first}}, {{/first}}{{param}}{{/tableparams}})
+      and
+      result = {{local_var}}.resolve())
+  }
+  {{/type_is_class}}
+  {{^type_is_class}}
+  {{type}} get{{singular}}({{#params}}{{^first}}, {{/first}}{{type}} {{param}}{{/params}}) {
+    {{tablename}}({{#tableparams}}{{^first}}, {{/first}}{{param}}{{/tableparams}})
+  }
+  {{/type_is_class}}
+  {{#indefinite_article}}
+
+  {{type}} get{{.}}{{singular}}() {
+    result = get{{singular}}({{#params}}{{^first}}, {{/first}}_{{/params}})
+  }
+
+  int getNumberOf{{plural}}() {
+    result = count(get{{.}}{{singular}}())
+  }
+  {{/indefinite_article}}
+  {{/properties}}
+}

swift/codegen/lib/templates/ql_imports.mustache

@@ -0,0 +1,4 @@
+// generated by {{generator}}
+{{#imports}}
+import {{.}}
+{{/imports}}

swift/codegen/lib/templates/ql_stub.mustache

@@ -0,0 +1,4 @@
+// generated by {{generator}}, remove this comment if you wish to edit this file
+private import {{base_import}}
+
+class {{name}} extends {{name}}Base { }

swift/codegen/qlgen.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+
+import logging
+import pathlib
+import subprocess
+from dataclasses import dataclass, field
+from typing import List, ClassVar
+
+import inflection
+
+from lib import schema, paths, generator
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class QlParam:
+    param: str
+    type: str = None
+    first: bool = False
+
+
+@dataclass
+class QlProperty:
+    singular: str
+    type: str
+    tablename: str
+    tableparams: List[QlParam]
+    plural: str = None
+    params: List[QlParam] = field(default_factory=list)
+    first: bool = False
+    local_var: str = "x"
+
+    def __post_init__(self):
+        if self.params:
+            self.params[0].first = True
+        while self.local_var in (p.param for p in self.params):
+            self.local_var += "_"
+        assert self.tableparams
+        if self.type_is_class:
+            self.tableparams = [x if x != "result" else self.local_var for x in self.tableparams]
+        self.tableparams = [QlParam(x) for x in self.tableparams]
+        self.tableparams[0].first = True
+
+    @property
+    def indefinite_article(self):
+        if self.plural:
+            return "An" if self.singular[0] in "AEIO" else "A"
+
+    @property
+    def type_is_class(self):
+        return self.type[0].isupper()
+
+
+@dataclass
+class QlClass:
+    template: ClassVar = 'ql_class'
+
+    name: str
+    bases: List[str]
+    final: bool
+    properties: List[QlProperty]
+    dir: pathlib.Path
+    imports: List[str] = field(default_factory=list)
+
+    def __post_init__(self):
+        self.bases = sorted(self.bases)
+        if self.properties:
+            self.properties[0].first = True
+
+    @property
+    def db_id(self):
+        return "@" + inflection.underscore(self.name)
+
+    @property
+    def root(self):
+        return not self.bases
+
+    @property
+    def path(self):
+        return self.dir / self.name
+
+
+@dataclass
+class QlStub:
+    template: ClassVar = 'ql_stub'
+
+    name: str
+    base_import: str
+
+
+@dataclass
+class QlImportList:
+    template: ClassVar = 'ql_imports'
+
+    imports: List[str] = field(default_factory=list)
+
+
+def get_ql_property(cls: schema.Class, prop: schema.Property):
+    if prop.is_single:
+        return QlProperty(
+            singular=inflection.camelize(prop.name),
+            type=prop.type,
+            tablename=inflection.tableize(cls.name),
+            tableparams=["this"] + ["result" if p is prop else "_" for p in cls.properties if p.is_single],
+        )
+    elif prop.is_optional:
+        return QlProperty(
+            singular=inflection.camelize(prop.name),
+            type=prop.type,
+            tablename=inflection.tableize(f"{cls.name}_{prop.name}"),
+            tableparams=["this", "result"],
+        )
+    elif prop.is_repeated:
+        return QlProperty(
+            singular=inflection.singularize(inflection.camelize(prop.name)),
+            plural=inflection.pluralize(inflection.camelize(prop.name)),
+            type=prop.type,
+            tablename=inflection.tableize(f"{cls.name}_{prop.name}"),
+            tableparams=["this", "index", "result"],
+            params=[QlParam("index", type="int")],
+        )
+
+
+def get_ql_class(cls: schema.Class):
+    return QlClass(
+        name=cls.name,
+        bases=cls.bases,
+        final=not cls.derived,
+        properties=[get_ql_property(cls, p) for p in cls.properties],
+        dir=cls.dir,
+    )
+
+
+def get_import(file):
+    stem = file.relative_to(paths.swift_dir / "ql/lib").with_suffix("")
+    return str(stem).replace("/", ".")
+
+
+def get_types_used_by(cls: QlClass):
+    for b in cls.bases:
+        yield b
+    for p in cls.properties:
+        yield p.type
+        for param in p.params:
+            yield param.type
+
+
+def get_classes_used_by(cls: QlClass):
+    return sorted(set(t for t in get_types_used_by(cls) if t[0].isupper()))
+
+
+def is_generated(file):
+    with open(file) as contents:
+        return next(contents).startswith("// generated")
+
+
+def format(codeql, files):
+    format_cmd = [codeql, "query", "format", "--in-place", "--"]
+    format_cmd.extend(str(f) for f in files)
+    res = subprocess.run(format_cmd, check=True, stderr=subprocess.PIPE, text=True)
+    for line in res.stderr.splitlines():
+        log.debug(line.strip())
+
+
+def generate(opts, renderer):
+    input = opts.schema.resolve()
+    out = opts.ql_output.resolve()
+    stub_out = opts.ql_stub_output.resolve()
+    existing = {q for q in out.rglob("*.qll")}
+    existing |= {q for q in stub_out.rglob("*.qll") if is_generated(q)}
+
+    with open(input) as src:
+        data = schema.load(src)
+
+    classes = [get_ql_class(cls) for cls in data.classes.values()]
+    imports = {}
+
+    for c in classes:
+        imports[c.name] = get_import(stub_out / c.path)
+
+    for c in classes:
+        assert not c.final or c.bases, c.name
+        qll = (out / c.path).with_suffix(".qll")
+        c.imports = [imports[t] for t in get_classes_used_by(c)]
+        renderer.render(c, qll)
+        stub_file = (stub_out / c.path).with_suffix(".qll")
+        if not stub_file.is_file() or is_generated(stub_file):
+            stub = QlStub(name=c.name, base_import=get_import(qll))
+            renderer.render(stub, stub_file)
+
+    # for example path/to/syntax/generated -> path/to/syntax.qll
+    include_file = stub_out.with_suffix(".qll")
+    all_imports = QlImportList(v for _, v in sorted(imports.items()))
+    renderer.render(all_imports, include_file)
+
+    renderer.cleanup(existing)
+    format(opts.codeql_binary, renderer.written)
+
+
+if __name__ == "__main__":
+    generator.run(generate, tags=["schema", "ql"])

swift/codegen/schema.yml

@@ -308,7 +308,7 @@ ArrowExpr:
 AssignExpr:
   _extends: Expr
   dest: Expr
-  src: Expr
+  source: Expr
 
 BindOptionalExpr:
   _extends: Expr
@@ -661,7 +661,7 @@ PrefixUnaryExpr:
 
 SelfApplyExpr:
   _extends: ApplyExpr
-  base: Expr
+  base_expr: Expr
 
 ArrayExpr:
   _extends: CollectionExpr
@@ -843,12 +843,12 @@ StmtCondition:
 
 RepeatWhileStmt:
   _extends: LabeledStmt
-  cond: Expr
+  condition: Expr
   body: Stmt
 
 SwitchStmt:
   _extends: LabeledStmt
-  subject_expr: Expr
+  expr: Expr
   cases: CaseStmt*
 
 BoundGenericClassType: