Merge pull request #17543 from github/aibaars/rust-gen-extractor

Rust: generate the extractor

rust/extractor/src/main.rs

@@ -1,11 +1,43 @@
 use anyhow::Context;
 use ra_ap_ide_db::line_index::LineIndex;
+use ra_ap_parser::Edition;
 mod archive;
 mod config;
 pub mod generated;
 mod translate;
 pub mod trap;
+use ra_ap_syntax::ast::SourceFile;
+use ra_ap_syntax::AstNode;
 
+fn extract(
+    archiver: &archive::Archiver,
+    traps: &trap::TrapFileProvider,
+    file: std::path::PathBuf,
+) -> anyhow::Result<()> {
+    let file = std::path::absolute(&file).unwrap_or(file);
+    let file = std::fs::canonicalize(&file).unwrap_or(file);
+    archiver.archive(&file);
+    let input = std::fs::read(&file)?;
+    let input = String::from_utf8(input)?;
+    let line_index = LineIndex::new(&input);
+    let display_path = file.to_string_lossy();
+    let mut trap = traps.create("source", &file);
+    let label = trap.emit_file(&file);
+    let mut translator = translate::Translator::new(trap, label, line_index);
+
+    let parse = ra_ap_syntax::ast::SourceFile::parse(&input, Edition::CURRENT);
+    for err in parse.errors() {
+        let (start, _) = translator.location(err.range());
+        log::warn!("{}:{}:{}: {}", display_path, start.line, start.col, err);
+    }
+    if let Some(ast) = SourceFile::cast(parse.syntax_node()) {
+        translator.emit_source_file(ast);
+        translator.trap.commit()?
+    } else {
+        log::warn!("Skipped {}", display_path);
+    }
+    Ok(())
+}
 fn main() -> anyhow::Result<()> {
     let cfg = config::Config::extract().context("failed to load configuration")?;
     stderrlog::new()
@@ -18,18 +50,7 @@ fn main() -> anyhow::Result<()> {
         root: cfg.source_archive_dir,
     };
     for file in cfg.inputs {
-        let file = std::path::absolute(&file).unwrap_or(file);
-        let file = std::fs::canonicalize(&file).unwrap_or(file);
-        archiver.archive(&file);
-        let input = std::fs::read(&file)?;
-        let input = String::from_utf8(input)?;
-        let line_index = LineIndex::new(&input);
-        let display_path = file.to_string_lossy();
-        let mut trap = traps.create("source", &file);
-        let label = trap.emit_file(&file);
-        translate::SourceFileTranslator::new(trap, label, line_index)
-            .extract(&display_path, &input)
-            .context("writing trap file")?;
+        extract(&archiver, &traps, file)?;
     }
 
     Ok(())

rust/extractor/src/translate/.gitattributes

@@ -0,0 +1 @@
+/generated.rs linguist-generated

rust/extractor/src/translate/base.rs

@@ -0,0 +1,86 @@
+use crate::trap::TrapFile;
+use crate::trap::{Label, TrapClass};
+use codeql_extractor::trap::{self};
+use ra_ap_ide_db::line_index::{LineCol, LineIndex};
+use ra_ap_syntax::ast::RangeItem;
+use ra_ap_syntax::TextSize;
+use ra_ap_syntax::{ast, TextRange};
+pub trait TextValue {
+    fn try_get_text(&self) -> Option<String>;
+}
+
+impl TextValue for ast::Lifetime {
+    fn try_get_text(&self) -> Option<String> {
+        self.text().to_string().into()
+    }
+}
+impl TextValue for ast::Name {
+    fn try_get_text(&self) -> Option<String> {
+        self.text().to_string().into()
+    }
+}
+impl TextValue for ast::Literal {
+    fn try_get_text(&self) -> Option<String> {
+        self.token().text().to_string().into()
+    }
+}
+impl TextValue for ast::NameRef {
+    fn try_get_text(&self) -> Option<String> {
+        self.text().to_string().into()
+    }
+}
+impl TextValue for ast::Abi {
+    fn try_get_text(&self) -> Option<String> {
+        self.abi_string().map(|x| x.to_string())
+    }
+}
+
+impl TextValue for ast::BinExpr {
+    fn try_get_text(&self) -> Option<String> {
+        self.op_token().map(|x| x.text().to_string())
+    }
+}
+impl TextValue for ast::PrefixExpr {
+    fn try_get_text(&self) -> Option<String> {
+        self.op_token().map(|x| x.text().to_string())
+    }
+}
+impl TextValue for ast::RangeExpr {
+    fn try_get_text(&self) -> Option<String> {
+        self.op_token().map(|x| x.text().to_string())
+    }
+}
+impl TextValue for ast::RangePat {
+    fn try_get_text(&self) -> Option<String> {
+        self.op_token().map(|x| x.text().to_string())
+    }
+}
+pub struct Translator {
+    pub trap: TrapFile,
+    label: trap::Label,
+    line_index: LineIndex,
+}
+
+impl Translator {
+    pub fn new(trap: TrapFile, label: trap::Label, line_index: LineIndex) -> Translator {
+        Translator {
+            trap,
+            label,
+            line_index,
+        }
+    }
+    pub fn location(&self, range: TextRange) -> (LineCol, LineCol) {
+        let start = self.line_index.line_col(range.start());
+        let end = self.line_index.line_col(
+            range
+                .end()
+                .checked_sub(TextSize::new(1))
+                .unwrap_or(range.end()),
+        );
+        (start, end)
+    }
+    pub fn emit_location<T: TrapClass>(&mut self, label: Label<T>, node: impl ast::AstNode) {
+        let (start, end) = self.location(node.syntax().text_range());
+        self.trap.emit_location(self.label, label, start, end)
+    }
+}

rust/generate-schema/src/main.rs

@@ -53,6 +53,10 @@ fn write_schema(
     super_types: BTreeMap<String, BTreeSet<String>>,
 ) -> std::io::Result<String> {
     let mut buf: Vec<u8> = Vec::new();
+    writeln!(
+        buf,
+        "# Generated by `cargo generate-schema`, do not edit by hand.\n"
+    )?;
     writeln!(buf, "from .prelude import *\n")?;
 
     for node in &grammar.enums {
@@ -402,14 +406,33 @@ fn get_fields(node: &AstNodeSrc) -> Vec<FieldInfo> {
 
 fn write_extractor(grammar: &AstSrc) -> std::io::Result<String> {
     let mut buf: Vec<u8> = Vec::new();
+    writeln!(
+        buf,
+        "//! Generated by `cargo generate-schema`, do not edit by hand.\n
+use crate::generated;
+use super::base::{{TextValue, Translator}};
+use crate::trap::{{Label, TrapId}};
+use ra_ap_syntax::ast;
+use ra_ap_syntax::ast::{{
+    HasArgList, HasAttrs, HasGenericArgs, HasGenericParams, HasLoopBody, HasModuleItem, HasName,
+    HasTypeBounds, HasVisibility, RangeItem,
+}};
 
+impl Translator {{
+    fn emit_else_branch(&mut self, node: ast::ElseBranch) -> Label<generated::Expr> {{
+        match node {{
+            ast::ElseBranch::IfExpr(inner) => self.emit_if_expr(inner).into(),
+            ast::ElseBranch::Block(inner) => self.emit_block_expr(inner).into(),
+        }}
+    }}\n"
+    )?;
     for node in &grammar.enums {
         let type_name = &node.name;
         let class_name = class_name(&node.name);
 
         writeln!(
             buf,
-            "    fn emit_{}(&mut self, node: ast::{}) -> Label<generated::{}> {{",
+            "    pub(crate) fn emit_{}(&mut self, node: ast::{}) -> Label<generated::{}> {{",
             to_lower_snake_case(type_name),
             type_name,
             class_name
@@ -434,7 +457,7 @@ fn write_extractor(grammar: &AstSrc) -> std::io::Result<String> {
 
         writeln!(
             buf,
-            "    fn emit_{}(&mut self, node: ast::{}) -> Label<generated::{}> {{",
+            "    pub(crate) fn emit_{}(&mut self, node: ast::{}) -> Label<generated::{}> {{",
             to_lower_snake_case(type_name),
             type_name,
             class_name
@@ -491,6 +514,7 @@ fn write_extractor(grammar: &AstSrc) -> std::io::Result<String> {
 
         writeln!(buf, "    }}\n")?;
     }
+    writeln!(buf, "}}")?;
     Ok(String::from_utf8_lossy(&buf).into_owned())
 }
 
@@ -523,13 +547,21 @@ fn main() -> std::io::Result<()> {
     });
     let schema = write_schema(&grammar, super_types)?;
     let schema_path = PathBuf::from("../schema/ast.py");
-    let extractor = write_extractor(&grammar)?;
-    print!("{}", extractor);
     codegen::ensure_file_contents(
         crate::flags::CodegenType::Grammar,
         &schema_path,
         &schema,
         false,
     );
+
+    let extractor = write_extractor(&grammar)?;
+    let extractor_path = PathBuf::from("../extractor/src/translate/generated.rs");
+    codegen::ensure_file_contents(
+        crate::flags::CodegenType::Grammar,
+        &extractor_path,
+        &extractor,
+        false,
+    );
+
     Ok(())
 }

rust/schema/.gitattributes

@@ -0,0 +1 @@
+/ast.py linguist-generated

rust/schema/ast.py

@@ -1,3 +1,5 @@
+# Generated by `cargo generate-schema`, do not edit by hand.
+
 from .prelude import *
 
 class AssocItem(AstNode):