hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Ruby: write errors to json log

Browse Source
This commit is contained in:
Arthur Baars
2023-01-20 18:46:01 +01:00
parent d5e60dfb22
commit 2b9bc3c7e3
4 changed files with 201 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
ruby/extractor/src/diagnostics.rs
View File

@@ -6,6 +6,7 @@ use std::path::PathBuf;
pub enum Severity {
Error,
Warning,
#[allow(unused)]
Note,
}
@@ -42,13 +43,13 @@ pub struct Location {
/** Path to the affected file if appropriate, relative to the source root */
pub file: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub start_line: Option<i32>,
pub start_line: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub start_column: Option<i32>,
pub start_column: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub end_line: Option<i32>,
pub end_line: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub end_column: Option<i32>,
pub end_column: Option<usize>,
}
#[derive(Serialize)]
@@ -81,12 +82,45 @@ fn is_default_visibility(v: &Visibility) -> bool {
}
pub struct LogWriter {
extractor: String,
path: Option<PathBuf>,
inner: Option<std::io::BufWriter<std::fs::File>>,
}
impl LogWriter {
pub fn write(&mut self, mesg: &DiagnosticMessage) -> std::io::Result<()> {
pub fn message(&self, id: &str, name: &str) -> DiagnosticMessage {
DiagnosticMessage {
timestamp: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("")
.as_millis() as u64,
source: Source {
id: format!("{}/{}", self.extractor, id),
name: name.to_owned(),
extractor_name: Some(self.extractor.to_owned()),
},
markdown_message: String::new(),
plaintext_message: String::new(),
help_links: vec![],
severity: None,
internal: false,
visibility: Visibility {
cli_summary_table: false,
status_page: false,
telemetry: false,
},
location: None,
}
}
pub fn write(&mut self, mesg: &DiagnosticMessage) {
let full_error_message = mesg.full_error_message();
match mesg.severity {
Some(Severity::Error) => tracing::error!("{}", full_error_message),
Some(Severity::Warning) => tracing::warn!("{}", full_error_message),
Some(Severity::Note) => tracing::info!("{}", full_error_message),
None => tracing::debug!("{}", full_error_message),
}
if self.inner.is_none() {
let mut open_failed = false;
self.inner = self.path.as_ref().and_then(|path| {
@@ -113,10 +147,12 @@ impl LogWriter {
}
}
if let Some(mut writer) = self.inner.as_mut() {
serde_json::to_writer(&mut writer, mesg)?;
&mut writer.write_all(b"\n")?;
serde_json::to_writer(&mut writer, mesg)
.unwrap_or_else(|e| tracing::debug!("Failed to write log entry: {}", e));
&mut writer
.write_all(b"\n")
.unwrap_or_else(|e| tracing::debug!("Failed to write log entry: {}", e));
}
Ok(())
}
}
@@ -134,7 +170,7 @@ impl DiagnosticLoggers {
let root = match std::env::var(&env_var) {
Err(e) => {
tracing::error!("{}: {}", &env_var, e);
tracing::error!("{}: {}", e, &env_var);
None
}
Ok(dir) => {
@@ -160,6 +196,7 @@ impl DiagnosticLoggers {
};
}
THREAD_NUM.with(|n| LogWriter {
extractor: self.extractor.to_owned(),
inner: None,
path: self
.root
@@ -167,30 +204,6 @@ impl DiagnosticLoggers {
.map(|root| root.to_owned().join(format!("extractor_{}.jsonl", n))),
})
}
pub fn message(&self, id: &str, name: &str) -> DiagnosticMessage {
DiagnosticMessage {
timestamp: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("")
.as_millis() as u64,
source: Source {
id: id.to_owned(),
name: name.to_owned(),
extractor_name: Some(self.extractor.to_owned()),
},
markdown_message: String::new(),
plaintext_message: String::new(),
help_links: vec![],
severity: None,
internal: false,
visibility: Visibility {
cli_summary_table: false,
status_page: false,
telemetry: false,
},
location: None,
}
}
}
static EMPTY_LOCATION: Location = Location {
file: None,
@@ -200,10 +213,23 @@ static EMPTY_LOCATION: Location = Location {
end_column: None,
};
impl DiagnosticMessage {
pub fn full_error_message(&self) -> String {
match &self.location {
Some(Location {
file: Some(path),
start_line: Some(line),
..
}) => format!("{}:{}: {}", path, line, self.plaintext_message),
_ => self.plaintext_message.to_owned(),
}
}
pub fn text<'a>(&'a mut self, text: &str) -> &'a mut Self {
self.plaintext_message = text.to_owned();
self
}
#[allow(unused)]
pub fn markdown<'a>(&'a mut self, text: &str) -> &'a mut Self {
self.markdown_message = text.to_owned();
self
@@ -212,14 +238,17 @@ impl DiagnosticMessage {
self.severity = Some(severity);
self
}
#[allow(unused)]
pub fn help_link<'a>(&'a mut self, link: &str) -> &'a mut Self {
self.help_links.push(link.to_owned());
self
}
#[allow(unused)]
pub fn internal<'a>(&'a mut self) -> &'a mut Self {
self.internal = true;
self
}
#[allow(unused)]
pub fn cli_summary_table<'a>(&'a mut self) -> &'a mut Self {
self.visibility.cli_summary_table = true;
self
@@ -228,36 +257,25 @@ impl DiagnosticMessage {
self.visibility.status_page = true;
self
}
#[allow(unused)]
pub fn telemetry<'a>(&'a mut self) -> &'a mut Self {
self.visibility.telemetry = true;
self
}
pub fn file<'a>(&'a mut self, path: &str) -> &'a mut Self {
self.location.get_or_insert(EMPTY_LOCATION.to_owned()).file = Some(path.to_owned());
self
}
pub fn start_line<'a>(&'a mut self, start_line: i32) -> &'a mut Self {
self.location
.get_or_insert(EMPTY_LOCATION.to_owned())
.start_line = Some(start_line);
self
}
pub fn start_column<'a>(&'a mut self, start_column: i32) -> &'a mut Self {
self.location
.get_or_insert(EMPTY_LOCATION.to_owned())
.start_column = Some(start_column);
self
}
pub fn end_line<'a>(&'a mut self, end_line: i32) -> &'a mut Self {
self.location
.get_or_insert(EMPTY_LOCATION.to_owned())
.end_line = Some(end_line);
self
}
pub fn end_column<'a>(&'a mut self, end_column: i32) -> &'a mut Self {
self.location
.get_or_insert(EMPTY_LOCATION.to_owned())
.end_column = Some(end_column);
pub fn location<'a>(
&'a mut self,
path: &str,
start_line: usize,
start_column: usize,
end_line: usize,
end_column: usize,
) -> &'a mut Self {
let loc = self.location.get_or_insert(EMPTY_LOCATION.to_owned());
loc.file = Some(path.to_owned());
loc.start_line = Some(start_line);
loc.start_column = Some(start_column);
loc.end_line = Some(end_line);
loc.end_column = Some(end_column);
self
}
}

133
ruby/extractor/src/extractor.rs
View File

@@ -6,7 +6,6 @@ use std::collections::BTreeSet as Set;
use std::fmt;
use std::path::Path;
use tracing::{error, info, span, Level};
use tree_sitter::{Language, Node, Parser, Range, Tree};
pub fn populate_file(writer: &mut trap::Writer, absolute_path: &Path) -> trap::Label {
@@ -122,15 +121,15 @@ pub fn extract(
ranges: &[Range],
) -> std::io::Result<()> {
let path_str = format!("{}", path.display());
let span = span!(
Level::TRACE,
let span = tracing::span!(
tracing::Level::TRACE,
"extract",
file = %path_str
);
let _enter = span.enter();
info!("extracting: {}", path_str);
tracing::info!("extracting: {}", path_str);
let mut parser = Parser::new();
parser.set_language(language).unwrap();
@@ -252,34 +251,31 @@ impl<'a> Visitor<'a> {
}
}
fn record_parse_error(
&mut self,
error_message: String,
full_error_message: String,
loc: trap::Label,
) {
error!("{}", full_error_message);
fn record_parse_error(&mut self, loc: trap::Label, mesg: &diagnostics::DiagnosticMessage) {
self.diagnostics_writer.write(mesg);
let id = self.trap_writer.fresh_id();
let full_error_message = mesg.full_error_message();
let severity_code = match mesg.severity {
Some(diagnostics::Severity::Error) => 40,
Some(diagnostics::Severity::Warning) => 30,
Some(diagnostics::Severity::Note) => 20,
None => 10,
};
self.trap_writer.add_tuple(
"diagnostics",
vec![
trap::Arg::Label(id),
trap::Arg::Int(40), // severity 40 = error
trap::Arg::Int(severity_code),
trap::Arg::String("parse_error".to_string()),
trap::Arg::String(error_message),
trap::Arg::String(mesg.plaintext_message.to_owned()),
trap::Arg::String(full_error_message),
trap::Arg::Label(loc),
],
);
}
fn record_parse_error_for_node(
&mut self,
error_message: String,
full_error_message: String,
node: Node,
) {
let (start_line, start_column, end_line, end_column) = location_for(self.source, node);
fn record_parse_error_for_node(&mut self, error_message: String, node: Node) {
let (start_line, start_column, end_line, end_column) = location_for(self, node);
let loc = location(
self.trap_writer,
self.file_label,
@@ -288,7 +284,14 @@ impl<'a> Visitor<'a> {
end_line,
end_column,
);
self.record_parse_error(error_message, full_error_message, loc);
self.record_parse_error(
loc,
self.diagnostics_writer
.message("parse-error", "Parse error")
.severity(diagnostics::Severity::Error)
.location(self.path, start_line, start_column, end_line, end_column)
.text(&error_message),
);
}
fn enter_node(&mut self, node: Node) -> bool {
@@ -298,13 +301,7 @@ impl<'a> Visitor<'a> {
} else {
"parse error".to_string()
};
let full_error_message = format!(
"{}:{}: {}",
&self.path,
node.start_position().row + 1,
error_message
);
self.record_parse_error_for_node(error_message, full_error_message, node);
self.record_parse_error_for_node(error_message, node);
return false;
}
@@ -319,7 +316,7 @@ impl<'a> Visitor<'a> {
return;
}
let (id, _, child_nodes) = self.stack.pop().expect("Vistor: empty stack");
let (start_line, start_column, end_line, end_column) = location_for(self.source, node);
let (start_line, start_column, end_line, end_column) = location_for(self, node);
let loc = location(
self.trap_writer,
self.file_label,
@@ -387,13 +384,15 @@ impl<'a> Visitor<'a> {
}
_ => {
let error_message = format!("unknown table type: '{}'", node.kind());
let full_error_message = format!(
"{}:{}: {}",
&self.path,
node.start_position().row + 1,
error_message
self.record_parse_error(
loc,
self.diagnostics_writer
.message("parse-error", "Parse error")
.severity(diagnostics::Severity::Error)
.location(self.path, start_line, start_column, end_line, end_column)
.text(&error_message)
.status_page(),
);
self.record_parse_error(error_message, full_error_message, loc);
valid = false;
}
@@ -447,13 +446,7 @@ impl<'a> Visitor<'a> {
child_node.type_name,
field.type_info
);
let full_error_message = format!(
"{}:{}: {}",
&self.path,
node.start_position().row + 1,
error_message
);
self.record_parse_error_for_node(error_message, full_error_message, *node);
self.record_parse_error_for_node(error_message, *node);
}
} else if child_node.field_name.is_some() || child_node.type_name.named {
let error_message = format!(
@@ -462,13 +455,7 @@ impl<'a> Visitor<'a> {
&child_node.field_name.unwrap_or("child"),
&child_node.type_name
);
let full_error_message = format!(
"{}:{}: {}",
&self.path,
node.start_position().row + 1,
error_message
);
self.record_parse_error_for_node(error_message, full_error_message, *node);
self.record_parse_error_for_node(error_message, *node);
}
}
let mut args = Vec::new();
@@ -491,13 +478,7 @@ impl<'a> Visitor<'a> {
node.kind(),
column_name
);
let full_error_message = format!(
"{}:{}: {}",
&self.path,
node.start_position().row + 1,
error_message
);
self.record_parse_error_for_node(error_message, full_error_message, *node);
self.record_parse_error_for_node(error_message, *node);
}
}
Storage::Table {
@@ -512,17 +493,8 @@ impl<'a> Visitor<'a> {
node.kind(),
table_name,
);
let full_error_message = format!(
"{}:{}: {}",
&self.path,
node.start_position().row + 1,
error_message
);
self.record_parse_error_for_node(
error_message,
full_error_message,
*node,
);
self.record_parse_error_for_node(error_message, *node);
break;
}
let mut args = vec![trap::Arg::Label(parent_id)];
@@ -589,7 +561,7 @@ fn sliced_source_arg(source: &[u8], n: Node) -> trap::Arg {
// Emit a pair of `TrapEntry`s for the provided node, appropriately calibrated.
// The first is the location and label definition, and the second is the
// 'Located' entry.
fn location_for(source: &[u8], n: Node) -> (usize, usize, usize, usize) {
fn location_for(visitor: &mut Visitor, n: Node) -> (usize, usize, usize, usize) {
// Tree-sitter row, column values are 0-based while CodeQL starts
// counting at 1. In addition Tree-sitter's row and column for the
// end position are exclusive while CodeQL's end positions are inclusive.
@@ -607,6 +579,7 @@ fn location_for(source: &[u8], n: Node) -> (usize, usize, usize, usize) {
end_line = start_line;
end_col = start_col - 1;
} else if end_col == 0 {
let source = visitor.source;
// end_col = 0 means that we are at the start of a line
// unfortunately 0 is invalid as column number, therefore
// we should update the end location to be the end of the
@@ -615,7 +588,14 @@ fn location_for(source: &[u8], n: Node) -> (usize, usize, usize, usize) {
if index > 0 && index <= source.len() {
index -= 1;
if source[index] != b'\n' {
error!("expecting a line break symbol, but none found while correcting end column value");
&visitor.diagnostics_writer.write(
&visitor
.diagnostics_writer
.message("internal-error", "Internal error")
.text("expecting a line break symbol, but none found while correcting end column value")
.status_page()
.severity(diagnostics::Severity::Error),
);
}
end_line -= 1;
end_col = 1;
@@ -624,10 +604,17 @@ fn location_for(source: &[u8], n: Node) -> (usize, usize, usize, usize) {
end_col += 1;
}
} else {
error!(
"cannot correct end column value: end_byte index {} is not in range [1,{}]",
index,
source.len()
&visitor.diagnostics_writer.write(
&visitor
.diagnostics_writer
.message("internal-error", "Internal error")
.text(&format!(
"cannot correct end column value: end_byte index {} is not in range [1,{}]",
index,
source.len()
))
.status_page()
.severity(diagnostics::Severity::Error),
);
}
}

82
ruby/extractor/src/main.rs
View File

@@ -25,22 +25,19 @@ use tree_sitter::{Language, Parser, Range};
* of cores available on the machine to determine how many threads to use
* (minimum of 1). If unspecified, should be considered as set to -1."
*/
fn num_codeql_threads() -> usize {
fn num_codeql_threads() -> Result<usize, String> {
let threads_str = std::env::var("CODEQL_THREADS").unwrap_or_else(|_| "-1".to_owned());
match threads_str.parse::<i32>() {
Ok(num) if num <= 0 => {
let reduction = -num as usize;
std::cmp::max(1, num_cpus::get() - reduction)
Ok(std::cmp::max(1, num_cpus::get() - reduction))
}
Ok(num) => num as usize,
Ok(num) => Ok(num as usize),
Err(_) => {
tracing::error!(
"Unable to parse CODEQL_THREADS value '{}'; defaulting to 1 thread.",
&threads_str
);
1
}
Err(_) => Err(format!(
"Unable to parse CODEQL_THREADS value '{}'",
&threads_str
)),
}
}
@@ -60,7 +57,6 @@ fn encoding_from_name(encoding_name: &str) -> Option<&(dyn encoding::Encoding +
}
fn main() -> std::io::Result<()> {
let diagnostics = diagnostics::DiagnosticLoggers::new("ruby");
tracing_subscriber::fmt()
.with_target(false)
.without_time()
@@ -70,7 +66,21 @@ fn main() -> std::io::Result<()> {
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("ruby_extractor=warn")),
)
.init();
let num_threads = num_codeql_threads();
let diagnostics = diagnostics::DiagnosticLoggers::new("ruby");
let logger = &mut diagnostics.logger();
let num_threads = match num_codeql_threads() {
Ok(num) => num,
Err(e) => {
logger.write(
&logger
.message("configuration-error", "Configuration error")
.text(&format!("{}; defaulting to 1 thread.", e))
.status_page()
.severity(diagnostics::Severity::Warning),
);
1
}
};
tracing::info!(
"Using {} {}",
num_threads,
@@ -80,6 +90,20 @@ fn main() -> std::io::Result<()> {
"threads"
}
);
let trap_compression = match trap::Compression::from_env("CODEQL_RUBY_TRAP_COMPRESSION") {
Ok(x) => x,
Err(e) => {
logger.write(
&logger
.message("configuration-error", "Configuration error")
.text(&format!("{}; using gzip.", e))
.status_page()
.severity(diagnostics::Severity::Warning),
);
trap::Compression::Gzip
}
};
rayon::ThreadPoolBuilder::new()
.num_threads(num_threads)
.build_global()
@@ -102,7 +126,6 @@ fn main() -> std::io::Result<()> {
.value_of("output-dir")
.expect("missing --output-dir");
let trap_dir = PathBuf::from(trap_dir);
let trap_compression = trap::Compression::from_env("CODEQL_RUBY_TRAP_COMPRESSION");
let file_list = matches.value_of("file-list").expect("missing --file-list");
let file_list = fs::File::open(file_list)?;
@@ -174,20 +197,35 @@ fn main() -> std::io::Result<()> {
Ok(str) => source = str.as_bytes().to_owned(),
Err(msg) => {
needs_conversion = false;
tracing::warn!(
"{}: character decoding failure: {} ({})",
&path.to_string_lossy(),
msg,
&encoding_name
diagnostics_writer.write(
&logger
.message(
"character-encoding-error",
"Character encoding error",
)
.text(&format!(
"{}: character decoding failure: {} ({})",
&path.to_string_lossy(),
msg,
&encoding_name
))
.status_page()
.severity(diagnostics::Severity::Warning),
);
}
}
}
} else {
tracing::warn!(
"{}: unknown character encoding: '{}'",
&path.to_string_lossy(),
&encoding_name
diagnostics_writer.write(
&logger
.message("character-encoding-error", "Character encoding error")
.text(&format!(
"{}: unknown character encoding: '{}'",
&path.to_string_lossy(),
&encoding_name
))
.status_page()
.severity(diagnostics::Severity::Warning),
);
}
}

11
ruby/extractor/src/trap.rs
View File

@@ -224,17 +224,14 @@ pub enum Compression {
}
impl Compression {
pub fn from_env(var_name: &str) -> Compression {
pub fn from_env(var_name: &str) -> Result<Compression, String> {
match std::env::var(var_name) {
Ok(method) => match Compression::from_string(&method) {
Some(c) => c,
None => {
tracing::error!("Unknown compression method '{}'; using gzip.", &method);
Compression::Gzip
}
Some(c) => Ok(c),
None => Err(format!("Unknown compression method '{}'", &method)),
},
// Default compression method if the env var isn't set:
Err(_) => Compression::Gzip,
Err(_) => Ok(Compression::Gzip),
}
}