Rust: archiving + skeleton def translator

misc/codegen/codegen.py

@@ -30,8 +30,8 @@ def _parse_args() -> argparse.Namespace:
 
     p = argparse.ArgumentParser(description="Code generation suite")
     p.add_argument("--generate", type=lambda x: x.split(","),
-                   help="specify what targets to generate as a comma separated list, choosing among dbscheme, ql, trap "
-                        "and cpp")
+                   help="specify what targets to generate as a comma separated list, choosing among dbscheme, ql, "
+                        "trap, cpp and rust")
     p.add_argument("--verbose", "-v", action="store_true", help="print more information")
     p.add_argument("--quiet", "-q", action="store_true", help="only print errors")
     p.add_argument("--configuration-file", "-c", type=_abspath, default=conf,
@@ -57,6 +57,9 @@ def _parse_args() -> argparse.Namespace:
         p.add_argument("--cpp-output",
                        help="output directory for generated C++ files, required if trap or cpp is provided to "
                             "--generate"),
+        p.add_argument("--rust-output",
+                       help="output directory for generated Rust files, required if rust is provided to "
+                            "--generate"),
         p.add_argument("--generated-registry",
                        help="registry file containing information about checked-in generated code. A .gitattributes"
                             "file is generated besides it to mark those files with linguist-generated=true. Must"

misc/codegen/generators/__init__.py

@@ -1,4 +1,4 @@
-from . import dbschemegen, qlgen, trapgen, cppgen
+from . import dbschemegen, qlgen, trapgen, cppgen, rustgen
 
 
 def generate(target, opts, renderer):

misc/codegen/generators/rustgen.py

@@ -0,0 +1,102 @@
+"""
+Rust trap class generation
+"""
+
+import functools
+import typing
+
+import inflection
+
+from misc.codegen.lib import rust, schema
+from misc.codegen.loaders import schemaloader
+
+
+def _get_type(t: str) -> str:
+    match t:
+        case None | "boolean":  # None means a predicate
+            return "bool"
+        case "string":
+            return "String"
+        case "int":
+            return "i32"
+        case _ if t[0].isupper():
+            return "TrapLabel"
+        case _:
+            return t
+
+
+def _get_field(cls: schema.Class, p: schema.Property) -> rust.Field:
+    table_name = None
+    if not p.is_single:
+        table_name = f"{cls.name}_{p.name}"
+        if p.is_predicate:
+            table_name = inflection.underscore(table_name)
+        else:
+            table_name = inflection.tableize(table_name)
+    args = dict(
+        field_name=p.name + ("_" if p.name in rust.keywords else ""),
+        base_type=_get_type(p.type),
+        is_optional=p.is_optional,
+        is_repeated=p.is_repeated,
+        is_predicate=p.is_predicate,
+        is_unordered=p.is_unordered,
+        table_name=table_name,
+    )
+    args.update(rust.get_field_override(p.name))
+    return rust.Field(**args)
+
+
+def _get_properties(
+    cls: schema.Class, lookup: dict[str, schema.Class]
+) -> typing.Iterable[schema.Property]:
+    for b in cls.bases:
+        yield from _get_properties(lookup[b], lookup)
+    yield from cls.properties
+
+
+class Processor:
+    def __init__(self, data: schema.Schema):
+        self._classmap = data.classes
+
+    def _get_class(self, name: str) -> rust.Class:
+        cls = self._classmap[name]
+        return rust.Class(
+            name=name,
+            fields=[
+                _get_field(cls, p)
+                for p in _get_properties(cls, self._classmap)
+                if "rust_skip" not in p.pragmas and not p.synth
+            ],
+            table_name=inflection.tableize(cls.name),
+        )
+
+    def get_classes(self):
+        ret = {"": []}
+        for k, cls in self._classmap.items():
+            if not cls.synth and not cls.derived:
+                ret.setdefault(cls.group, []).append(self._get_class(cls.name))
+        return ret
+
+
+def generate(opts, renderer):
+    assert opts.rust_output
+    processor = Processor(schemaloader.load_file(opts.schema))
+    out = opts.rust_output
+    groups = set()
+    for group, classes in processor.get_classes().items():
+        group = group or "top"
+        groups.add(group)
+        renderer.render(
+            rust.ClassList(
+                classes,
+                opts.schema,
+            ),
+            out / f"{group}.rs",
+        )
+    renderer.render(
+        rust.ModuleList(
+            groups,
+            opts.schema,
+        ),
+        out / f"mod.rs",
+    )

misc/codegen/lib/rust.py

@@ -0,0 +1,141 @@
+import dataclasses
+import re
+import typing
+
+# taken from https://doc.rust-lang.org/reference/keywords.html
+keywords = {
+    "as",
+    "break",
+    "const",
+    "continue",
+    "crate",
+    "else",
+    "enum",
+    "extern",
+    "false",
+    "fn",
+    "for",
+    "if",
+    "impl",
+    "in",
+    "let",
+    "loop",
+    "match",
+    "mod",
+    "move",
+    "mut",
+    "pub",
+    "ref",
+    "return",
+    "self",
+    "Self",
+    "static",
+    "struct",
+    "super",
+    "trait",
+    "true",
+    "type",
+    "unsafe",
+    "use",
+    "where",
+    "while",
+    "async",
+    "await",
+    "dyn",
+    "abstract",
+    "become",
+    "box",
+    "do",
+    "final",
+    "macro",
+    "override",
+    "priv",
+    "typeof",
+    "unsized",
+    "virtual",
+    "yield",
+    "try",
+}
+
+_field_overrides = [
+    (
+        re.compile(r"(start|end)_(line|column)|(.*_)?index|width|num_.*"),
+        {"base_type": "usize"},
+    ),
+    (re.compile(r"(.*)_"), lambda m: {"field_name": m[1]}),
+]
+
+
+def get_field_override(field: str):
+    for r, o in _field_overrides:
+        m = r.fullmatch(field)
+        if m:
+            return o(m) if callable(o) else o
+    return {}
+
+
+@dataclasses.dataclass
+class Field:
+    field_name: str
+    base_type: str
+    table_name: str = None
+    is_optional: bool = False
+    is_repeated: bool = False
+    is_unordered: bool = False
+    is_predicate: bool = False
+    first: bool = False
+
+    def __post_init__(self):
+        if self.field_name in keywords:
+            self.field_name += "_"
+
+    @property
+    def type(self) -> str:
+        type = self.base_type
+        if self.is_optional:
+            type = f"Option<{type}>"
+        if self.is_repeated:
+            type = f"Vec<{type}>"
+        return type
+
+    # using @property breaks pystache internals here
+    def emitter(self):
+        if self.type == "String":
+            return lambda x: f"quoted(&{x})"
+        else:
+            return lambda x: x
+
+    @property
+    def is_single(self):
+        return not (self.is_optional or self.is_repeated or self.is_predicate)
+
+    @property
+    def is_label(self):
+        return self.base_type == "TrapLabel"
+
+
+@dataclasses.dataclass
+class Class:
+    name: str
+    table_name: str
+    fields: list[Field] = dataclasses.field(default_factory=list)
+
+    @property
+    def single_fields(self):
+        return [f for f in self.fields if f.is_single]
+
+
+@dataclasses.dataclass
+class ClassList:
+    template: typing.ClassVar[str] = "rust_classes"
+
+    classes: list[Class]
+    source: str
+
+
+@dataclasses.dataclass
+class ModuleList:
+    template: typing.ClassVar[str] = "rust_module"
+
+    modules: list[str]
+    source: str

misc/codegen/schema_documentation.md

@@ -0,0 +1,114 @@
+# Documenting `schema.py` entities
+
+## Classes
+
+Classes can be documented with plain python docstrings, for example
+
+```
+class ErrorElement(Locatable):
+    '''The superclass of all elements indicating some kind of error.'''
+    pass
+```
+
+This gets copied verbatim as QL doc comments for the class (with some internal handling for preservation of indentation,
+as explained in https://peps.python.org/pep-0257/#handling-docstring-indentation).
+
+## Properties
+
+Properties by default get a generated doc comment created from the name of the property and the enclosing class. So for
+example property `name` in class `File` will get documented as
+
+```
+/**
+ * Gets the name of this file.
+ */
+```
+
+This documentation generation will expand common abbreviations. The list of expanded abbreviations can be found
+in [`codegen/generators/qlgen.py`](./codegen/generators/qlgen.py) as a dictionary under the `abbreviations` variable.
+
+The `name of this file` part in the example above can be customized by appending `| doc("<replacement>")` to the
+property specification, for example
+
+```
+class Locatable(Element):
+    location: optional[Location] | doc("location associated with this element in the code")
+```
+
+When keeping the default documentation header, the name used for the class (for example `file` above) can be customized
+at the class level by applying to the class the `@ql.default_doc_name("<replacement>")` decorator, for example
+
+```
+@ql.default_doc_name("function type")
+class AnyFunctionType(Type):
+  ...
+```
+
+Additionally, a description can be given which will be added after the documentation header
+using `| desc("<description>")`. For example
+
+```
+class PoundDiagnosticDecl(Decl):
+    kind: int | desc("This is 1 for `#error` and 2 for `#warning`.")
+```
+
+will result in
+
+```
+/**
+ * Gets the kind of this pound diagnostic declaration.
+ *
+ * This is 1 for `#error` and 2 for `#warning`.
+ */
+```
+
+### Plural/singular
+
+Notice that for repeated properties both the plural and the singular forms will be present in documentation. What term
+is taken to be pluralized/singularized depends on customization:
+
+* for auto-generated documentation headers, the _last_ word of the property name will be taken;
+* for headers overridden with `doc("<override>")`, the _first_ word of the override will be taken.
+
+So for example:
+
+```
+generic_type_params: list[GenericTypeParamDecl]
+   -> generic type parameter/generic type parameters of this generic context
+arguments: list[Argument] | doc("arguments passed to the applied function")
+   -> argument/arguments passed to the applied function
+```
+
+If this behaviour is not wanted, this can be overridden by enclosing the term to be pluralized/singularized in `{ }`
+within `doc`. So for example
+
+```
+class Foo:
+    names_of_the_things: list[string] | doc("{names} of the things in this foo")
+    silly_cats_or_dogs: list[Animal] | doc("silly {cats} or {dogs} in this foo")
+```
+
+## Predicates
+
+Similarly as properties, predicates get by default an automatically generated doc comment from the class name and the
+predicate name. For example
+
+```
+class ImportDecl(Decl):
+    is_exported: predicate
+```
+
+will generate the doc
+
+```
+/**
+ * Holds if this import declaration is exported.
+ */
+```
+
+And similarly to properties, one can:
+
+* customize everything that comes strictly after `if` with `| doc("<replacement>")`;
+* customize the default name for used for the class (`import declaration` in the example above) with
+  the `@ql.default_doc_name("<replacement>")` class decorator;
+* add a more in-depth description with `| desc("<description>")`.

misc/codegen/templates/rust_classes.mustache

@@ -0,0 +1,54 @@
+// generated by {{generator}}
+
+use crate::trap::{TrapLabel, TrapEntry, quoted};
+use std::io::Write;
+
+{{#classes}}
+#[derive(Debug)]
+pub struct {{name}} {
+    pub key: Option<String>,
+    {{#fields}}
+    pub {{field_name}}: {{type}},
+    {{/fields}}
+}
+
+impl TrapEntry for {{name}} {
+    type Label = TrapLabel;
+
+    fn prefix() -> &'static str { "{{name}}_" }
+
+    fn key(&self) -> Option<&str> { self.key.as_ref().map(String::as_str) }
+
+    fn emit<W: Write>(&self, id: &Self::Label, out: &mut W) -> std::io::Result<()> {
+        write!(out, "{{table_name}}({id}{{#single_fields}}, {}{{/single_fields}})\n"{{#single_fields}}, {{#emitter}}self.{{field_name}}{{/emitter}}{{/single_fields}})?;
+        {{#fields}}
+        {{#is_predicate}}
+        if self.{{field_name}} {
+            write!(out, "{{table_name}}({id})\n")?;
+        }
+        {{/is_predicate}}
+        {{#is_optional}}
+        {{^is_repeated}}
+        if let Some(ref v) = &self.{{field_name}} {
+            write!(out, "{{table_name}}({id}, {})\n", {{#emitter}}v{{/emitter}})?;
+        }
+        {{/is_repeated}}
+        {{/is_optional}}
+        {{#is_repeated}}
+        for (i, &ref v) in self.{{field_name}}.iter().enumerate() {
+            {{^is_optional}}
+            write!(out, "{{table_name}}({id}, {{^is_unordered}}{}, {{/is_unordered}}{})\n", {{^is_unordered}}i, {{/is_unordered}}{{#emitter}}v{{/emitter}})?;
+            {{/is_optional}}
+            {{#is_optional}}
+            if let Some(ref vv) = &v {
+                write!(out, "{{table_name}}({id}, {{^is_unordered}}{}, {{/is_unordered}}{})\n", {{^is_unordered}}i, {{/is_unordered}}{{#emitter}}vv{{/emitter}})?;
+            }
+            {{/is_optional}}
+        }
+        {{/is_repeated}}
+        {{/fields}}
+        Ok(())
+    }
+}
+
+{{/classes}}

misc/codegen/templates/rust_module.mustache

@@ -0,0 +1,7 @@
+// generated by {{generator}}
+
+{{#modules}}
+mod {{.}};
+pub use {{.}}::*;
+
+{{/modules}}