From 2cb2be93fc078048faf7675ce49c8fcfec95f5e6 Mon Sep 17 00:00:00 2001 From: lcdr Date: Thu, 30 May 2024 13:55:55 +0200 Subject: [PATCH] Add GraphQL query API endpoint --- Cargo.toml | 1 + src/api/graphql.rs | 522 +++++++++++++++++++++++++++++++++++++++++++++ src/api/mod.rs | 35 +++ 3 files changed, 558 insertions(+) create mode 100644 src/api/graphql.rs diff --git a/Cargo.toml b/Cargo.toml index 27721b8..6a287e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,7 @@ tracing = "0.1.37" form_urlencoded = "1.1.0" latin1str = "0.1.3" once_cell = "1.16.0" +graphql-parser = "0.4" [dependencies.clap] version = "4.0.29" diff --git a/src/api/graphql.rs b/src/api/graphql.rs new file mode 100644 index 0000000..b6d287d --- /dev/null +++ b/src/api/graphql.rs @@ -0,0 +1,522 @@ +use std::collections::HashMap; +use std::fmt; +use std::fmt::Write; +use std::{borrow::Borrow, path::Path}; + +use rusqlite::{types::ValueRef, Connection, OpenFlags}; + +use graphql_parser::{ + parse_query, + query::{Definition, Field, OperationDefinition, Selection}, +}; + +use super::PercentDecoded; + +#[derive(Debug)] +pub struct QueryError { + pub error: String, + pub message: String, +} + +impl fmt::Display for QueryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + write!(f, "{}: {}", self.error, self.message) + } +} + +impl std::error::Error for QueryError {} + +impl From for QueryError { + fn from(value: graphql_parser::query::ParseError) -> Self { + QueryError { + error: String::from("GraphQL parsing error"), + message: format!("{}", value), + } + } +} + +impl From for QueryError { + fn from(value: rusqlite::Error) -> Self { + QueryError { + error: String::from("rusqlite error"), + message: format!("{}", value), + } + } +} + +fn invalid_query(message: String) -> QueryError { + QueryError { + error: "invalid graphql query".to_string(), + message, + } +} + +/// A relation between two SQL tables, a foreign key or its reverse. +#[derive(Clone, Debug, Default)] +pub struct TableRel { + unique: bool, + from_col: String, + to_table: String, + to_col: String, +} + +pub type TableRels = HashMap>; + +#[derive(Debug)] +struct TableQuery { + name: String, + cols: Vec, + constraints: Vec, + joins: Vec, + // buffer for table_to_json + rowid: i64, + // buffer for table_to_json + flushed_outputs: Vec, +} + +#[derive(Debug)] +struct Column { + name: String, + alias: Option, + // buffer for table_to_json + value: Option, +} + +#[derive(Debug)] +struct Join { + unique: bool, + graphql_name: String, + from_col: String, + to_col: String, + to_table: TableQuery, +} + +/// Reads out foreign key and reverse relations from an SQLite DB. +pub fn read_out_table_rels(sqlite_path: &Path) -> Result { + let conn = Connection::open_with_flags(sqlite_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; + + let mut table_rels: TableRels = HashMap::new(); + + let mut tables_stmt = conn.prepare("select name from sqlite_master")?; + let mut tables_rows = tables_stmt.query([])?; + + // for each table in the SQLite DB: + while let Some(tables_row) = tables_rows.next()? { + let from_table: String = tables_row.get(0)?; + + // temporary store for reverse relations + // we don't insert them into table_rels immediately so we can insert shorthands when there is only one rel for a table + let mut rev_rels: TableRels = HashMap::new(); + + let mut fk_stmt = + conn.prepare("select \"from\", \"table\", \"to\" from pragma_foreign_key_list(?1)")?; + let mut fk_rows = fk_stmt.query([&from_table])?; + + // for each foreign key in the table: + while let Some(fk_row) = fk_rows.next()? { + let from_col: String = fk_row.get(0)?; + let to_table: String = fk_row.get(1)?; + let to_col: String = fk_row.get(2)?; + table_rels.entry(from_table.clone()).or_default().insert( + from_col.clone(), + TableRel { + unique: true, + from_col: from_col.clone(), + to_table: to_table.clone(), + to_col: to_col.clone(), + }, + ); + + // reverse relation + + let rev_col = format!("{}_{}", from_table, from_col); + + rev_rels.entry(to_table.clone()).or_default().insert( + rev_col, + TableRel { + unique: false, + from_col: to_col.clone(), + to_table: from_table.clone(), + to_col: from_col, + }, + ); + } + + // merge rev rels into main rels + for (table_name, rels) in rev_rels { + let t_rels = table_rels.entry(table_name).or_default(); + if let Some(rel) = rels.values().next() { + t_rels.insert(rel.to_table.clone(), rel.clone()); + } + for (col_name, rel) in rels { + t_rels.insert(col_name, rel); + } + } + } + + Ok(table_rels) +} + +/// Parses a GraphQl query, transforms it into equivalent SQL, runs it against the DB, and returns the output transformed to matching json. +pub(super) fn graphql( + sqlite_path: &Path, + table_rels: &TableRels, + query: PercentDecoded, +) -> Result { + let doc = parse_query::(query.borrow())?; + + let def = match doc.definitions.len() { + 0 => { + return Err(invalid_query("empty query".to_string())); + } + 1 => match &doc.definitions[0] { + Definition::Operation(op_def) => op_def, + Definition::Fragment(_) => { + return Err(invalid_query( + "TODO: fragment definition not supported".to_string(), + )); + } + }, + n => { + return Err(invalid_query(format!( + "only 1 definition allowed, got: {n}" + ))); + } + }; + + let selections = &match def { + OperationDefinition::Query(query) => &query.selection_set, + OperationDefinition::SelectionSet(sel) => sel, + _ => { + return Err(invalid_query(format!("Unsupported operation: {def}"))); + } + } + .items; + + let conn = Connection::open_with_flags(sqlite_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; + + let mut kv = vec![]; + for selection in selections { + match selection { + Selection::FragmentSpread(_) => { + return Err(invalid_query( + "TODO: Selection::FragmentSpread not supported".to_string(), + )); + } + Selection::InlineFragment(_) => { + return Err(invalid_query( + "TODO: Selection::InlineFragment not supported".to_string(), + )); + } + Selection::Field(f) => { + let mut table_query = field_to_table_query(table_rels, f)?; + + let query = table_query_to_sql(&table_query); + + let mut stmt = conn.prepare(&query)?; + let mut rows = stmt.query([])?; + let key = if let Some(alias) = &f.alias { + alias + } else { + &f.name + }; + kv.push(format!( + "\"{}\":{}", + key, + table_to_json(&mut table_query, &mut rows)? + )); + } + } + } + Ok(format!("{{{}}}", kv.join(","))) +} + +/// Recursively parses a GraphQL field into an abstract TableQuery. +fn field_to_table_query( + table_rels: &TableRels, + field: &Field, +) -> Result { + field_to_table_query_inner(table_rels, field, field.name.as_ref()) +} + +fn field_to_table_query_inner( + table_rels: &TableRels, + field: &Field, + table_name: &str, +) -> Result { + let mut table_query = TableQuery { + name: table_name.to_string(), + cols: vec![], + constraints: vec![], + rowid: 0, + joins: vec![], + flushed_outputs: vec![], + }; + + for (key, value) in &field.arguments { + table_query + .constraints + .push(format!("{} == {}", key, value)); + } + + let this_table_rels = if let Some(rels) = table_rels.get(&table_query.name) { + rels + } else { + return Err(QueryError { + error: format!("invalid table name {}", table_query.name), + message: "table does not exist".to_string(), + }); + }; + + for selection in &field.selection_set.items { + match selection { + Selection::Field(f) => { + if f.selection_set.items.is_empty() { + // no curly braces, this is a normal column + // if it ends in _loc, query the column with the appropriate localization + table_query.cols.push(if f.name.ends_with("_loc") { + let mut localized_name = f.name.clone(); + localized_name.truncate(localized_name.len() - "loc".len()); + localized_name.push_str("en_US"); // todo: proper localization support + Column { + name: localized_name, + alias: Some(f.name.clone()), + value: None, + } + } else { + // normal column + Column { + name: f.name.clone(), + alias: f.alias.clone(), + value: None, + } + }); + } else { + // curly braces, this requires the field to be a valid relation + let rel = if let Some(rel) = this_table_rels.get(&f.name) { + rel + } else { + return Err(QueryError { + error: format!("field {} is not a foreign key", &f.name), + message: "field has items but is not a FK according to the DB" + .to_string(), + }); + }; + + // recurse with the fields in curly braces + let tq = field_to_table_query_inner(table_rels, f, rel.to_table.as_ref())?; + + // link the parent query to the child query + table_query.joins.push(Join { + unique: rel.unique, + graphql_name: f.name.clone(), + from_col: rel.from_col.clone(), + to_col: rel.to_col.clone(), + to_table: tq, + }); + } + } + _ => { + return Err(invalid_query( + "selection set in query should only contain Fields".to_string(), + )); + } + } + } + + Ok(table_query) +} + +/// Generates equivalent SQL from a parsed TableQuery. +fn table_query_to_sql(table_query: &TableQuery) -> String { + let mut cols = vec![]; + let mut tables = vec![]; + let mut constraints = vec![]; + + tables.push(format!("{} as t0", table_query.name)); + + table_query_to_sql_inner(table_query, &mut cols, &mut tables, &mut constraints); + + let mut query = format!("select {} from {}", cols.join(", "), tables.join(" ")); + if !constraints.is_empty() { + write!(query, " where {}", constraints.join(" and ")).unwrap(); + } + write!( + query, + " order by {}", + (0..tables.len()) + .map(|x| format!("t{}.rowid", x)) + .collect::>() + .join(", ") + ) + .unwrap(); + query +} + +fn table_query_to_sql_inner( + table_query: &TableQuery, + cols: &mut Vec, + tables: &mut Vec, + constraints: &mut Vec, +) { + let table_n = tables.len() - 1; + let mut table_n2 = table_n; + + cols.push(format!("t{}.rowid", table_n)); + + for col in &table_query.cols { + cols.push(format!("t{}.{}", table_n, col.name)); + } + + for constraint in &table_query.constraints { + constraints.push(format!("t{}.{}", table_n, constraint)); + } + + for join in &table_query.joins { + tables.push(format!( + "left join {} as t{} on t{}.{} = t{}.{}", + join.to_table.name, + table_n2 + 1, + table_n, + join.from_col, + table_n2 + 1, + join.to_col + )); + table_query_to_sql_inner(&join.to_table, cols, tables, constraints); + table_n2 = tables.len() - 1; + } +} + +/// Formats a sqlite value as json. +fn valueref_to_json(valueref: &ValueRef) -> Result { + let mut str = String::new(); + match valueref { + ValueRef::Null => str.push_str("null"), + ValueRef::Integer(x) => str.push_str(&x.to_string()), + ValueRef::Real(x) => str.push_str(&x.to_string()), + ValueRef::Text(x) | ValueRef::Blob(x) => { + str.push('"'); + str.push_str( + &std::str::from_utf8(x) + .map_err(rusqlite::Error::Utf8Error)? + .replace('\\', "\\\\") + .replace('"', "\\\""), + ); + str.push('"'); + } + } + Ok(str) +} + +/// Given an SQLite query result `rows`, use the structure info from `table_query` to transform it to hierarchical JSON. +fn table_to_json( + table_query: &mut TableQuery, + rows: &mut rusqlite::Rows, +) -> Result { + while let Some(row) = rows.next()? { + let mut icol = 0; + // read in the data into the right buffers... + table_to_json_inner(table_query, &mut icol, false, row)?; + } + // ...and convert it to json + if table_query.rowid > 0 { + let out = flush_table_data(table_query); + table_query.flushed_outputs.push(out); + } + Ok(format!("[{}]", table_query.flushed_outputs.join(","))) +} + +/// Given a single SQLite query result row, recursively stores the data in the appropriate TableQuery buffers. +fn table_to_json_inner( + table_query: &mut TableQuery, + // Index of current column to read from. + icol: &mut usize, + // Whether to skip reading data, but still advance the column index for all subqueries. + mut skip: bool, + row: &rusqlite::Row, +) -> Result<(), rusqlite::Error> { + if !skip { + let rowid_col = row.get_ref(*icol)?; + *icol += 1; + + if let rusqlite::types::ValueRef::Integer(rowid) = rowid_col { + // since SQL joins duplicate values, we use the sorted rowid to deduplicate + if rowid > table_query.rowid { + // new row, read in data + if table_query.rowid > 0 { + // we read a row before, whose subbuffers need to be flushed out + let out = flush_table_data(table_query); + table_query.flushed_outputs.push(out); + } + // keep track of this rowid for the next row'scheck + table_query.rowid = rowid; + + for col in &mut table_query.cols { + col.value = Some(valueref_to_json(&row.get_ref(*icol)?)?); + *icol += 1; + } + } else if rowid == table_query.rowid { + // already encountered this entry in a previous join, skip this table but not subtables, which might have new subentries + *icol += table_query.cols.len(); + } else { + // already encountered this entry in a previous join, skip this and all subtables + skip = true; + *icol += table_query.cols.len(); + } + } else { + // left join is null, skip + if table_query.rowid > 0 { + // ...but null joins still need to flush out old rows + let out = flush_table_data(table_query); + table_query.flushed_outputs.push(out); + } + *icol += table_query.cols.len(); + } + } else { + *icol += table_query.cols.len() + 1; + } + + // recurse for each subquery + for join in &mut table_query.joins { + table_to_json_inner(&mut join.to_table, icol, skip, row)?; + } + + Ok(()) +} + +/// Serializes this query's buffers to JSON and then clears them, returns the JSON. +fn flush_table_data(table_query: &mut TableQuery) -> String { + let mut kv = vec![]; + for col in &table_query.cols { + let key = if let Some(alias) = &col.alias { + alias + } else { + &col.name + }; + kv.push(format!("\"{}\":{}", key, col.value.as_ref().unwrap())); + } + + for join in &mut table_query.joins { + if join.unique { + let out = if join.to_table.rowid > 0 { + flush_table_data(&mut join.to_table) + } else { + String::from("null") + }; + kv.push(format!("\"{}\":{}", join.graphql_name, out)); + } else { + if join.to_table.rowid > 0 { + let out = flush_table_data(&mut join.to_table); + join.to_table.flushed_outputs.push(out); + } + kv.push(format!( + "\"{}\":[{}]", + join.graphql_name, + join.to_table.flushed_outputs.join(",") + )); + } + join.to_table.flushed_outputs.clear(); + } + table_query.rowid = 0; + format!("{{{}}}", kv.join(",")) +} diff --git a/src/api/mod.rs b/src/api/mod.rs index e0e87e7..14c87af 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -41,6 +41,7 @@ use self::{ pub mod adapter; pub mod docs; pub mod files; +mod graphql; mod locale; mod query; pub mod rev; @@ -81,6 +82,7 @@ impl ToString for PercentDecoded { pub enum ApiError { DB(CastError), Sqlite(rusqlite::Error), + GraphQl(graphql::QueryError), Json(serde_json::Error), Yaml(serde_yaml::Error), } @@ -100,6 +102,12 @@ impl From for ApiError { } } +impl From for ApiError { + fn from(value: graphql::QueryError) -> Self { + Self::GraphQl(value) + } +} + impl From for ApiError { fn from(value: serde_json::Error) -> Self { Self::Json(value) @@ -117,6 +125,7 @@ impl From for io::Error { match value { ApiError::DB(e) => into_other_io_error(e), ApiError::Sqlite(e) => into_other_io_error(e), + ApiError::GraphQl(e) => into_other_io_error(e), ApiError::Json(e) => into_other_io_error(e), ApiError::Yaml(e) => into_other_io_error(e), } @@ -151,6 +160,7 @@ enum ApiRoute<'r> { AllTableRows(&'r str), TableRowsByPK(&'r str, &'r str), Query(PercentDecoded), + GraphQl(PercentDecoded), Locale(RestPath<'r>), Crc(u32), Rev(rev::Route), @@ -187,6 +197,12 @@ impl<'r> ApiRoute<'r> { )), None => Err(()), }, + Some("graphql") => match parts.next() { + Some(query) => Ok(Self::GraphQl( + PercentDecoded::from_str(query).map_err(|_e| ())?, + )), + None => Err(()), + }, Some("locale") => Ok(Self::Locale(RestPath(parts))), Some("rev") => rev::Route::from_parts(parts).map(ApiRoute::Rev), Some("crc") => match parts.next() { @@ -368,6 +384,7 @@ pub struct ApiService { rev: rev::RevService, res: EventSender, sqlite_path: PathBuf, + db_table_rels: graphql::TableRels, } #[allow(clippy::declare_interior_mutable_const)] // c.f. https://github.com/rust-lang/rust-clippy/issues/5812 @@ -393,6 +410,7 @@ impl ApiService { sqlite_path: PathBuf, ) -> Self { let api_url = HeaderValue::from_str(&api_uri.to_string()).unwrap(); + let db_table_rels = graphql::read_out_table_rels(&sqlite_path).unwrap(); Self { pack, db, @@ -402,6 +420,7 @@ impl ApiService { res: spawn_handler(res_path), rev: RevService::new(tydb, locale_root, rev), sqlite_path, + db_table_rels, } } @@ -432,6 +451,17 @@ impl ApiService { )) } + fn graphql_api( + &self, + f: impl FnOnce(&Path, &graphql::TableRels) -> Result, + ) -> Result, ApiError> { + Ok(reply_string( + f(&self.sqlite_path, &self.db_table_rels)?, + APPLICATION_JSON, + StatusCode::OK, + )) + } + /// Get data from `locale.xml` fn locale(&self, accept: Accept, rest: RestPath) -> Result, ApiError> { match locale::select_node(self.locale_root.root.node(), rest) { @@ -554,6 +584,11 @@ where (Method::GET, ApiRoute::Query(query)) => { self.query_api(|sqlite_path| query::query(sqlite_path, query)) } + (Method::GET, ApiRoute::GraphQl(query)) => { + self.graphql_api(|sqlite_path, table_rels| { + graphql::graphql(sqlite_path, table_rels, query) + }) + } (method, ApiRoute::Locale(rest)) => match method { Method::GET => self.locale(accept, rest), m if m.as_str() == "QUERY" => {