From 5ffefa7b60a8eb0b5fbefbdd90c8512652909c3a Mon Sep 17 00:00:00 2001 From: Sebastian Jeltsch Date: Mon, 16 Feb 2026 22:36:16 +0100 Subject: [PATCH] Overhaul schema metadata handling in preparation for geometry columns. --- crates/core/Cargo.toml | 1 + crates/core/src/admin/logs/list_logs.rs | 2 +- crates/core/src/admin/logs/stats.rs | 2 +- crates/core/src/admin/rows/delete_rows.rs | 9 +- crates/core/src/admin/rows/list_rows.rs | 15 +- crates/core/src/admin/rows/read_files.rs | 23 +- crates/core/src/admin/rows/update_row.rs | 5 +- crates/core/src/admin/user/list_users.rs | 2 +- crates/core/src/auth/api/avatar.rs | 9 +- crates/core/src/connection.rs | 9 +- crates/core/src/listing.rs | 11 +- crates/core/src/records/create_record.rs | 8 +- crates/core/src/records/delete_record.rs | 4 +- crates/core/src/records/expand.rs | 143 ++++++----- crates/core/src/records/filter.rs | 15 +- crates/core/src/records/list_records.rs | 28 +-- crates/core/src/records/params.rs | 155 ++++++++---- crates/core/src/records/read_queries.rs | 32 +-- crates/core/src/records/read_record.rs | 159 ++++++------ crates/core/src/records/record_api.rs | 174 ++++++------- crates/core/src/records/subscribe.rs | 2 +- crates/core/src/records/transaction.rs | 17 +- crates/core/src/records/update_record.rs | 4 +- crates/core/src/records/validate.rs | 19 +- crates/core/src/schema_metadata.rs | 2 +- crates/schema/src/json_schema.rs | 16 +- crates/schema/src/metadata.rs | 291 +++++++++++++--------- 27 files changed, 613 insertions(+), 544 deletions(-) diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 4694b921..a2880235 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -48,6 +48,7 @@ ed25519-dalek = { version = "2.1.1", features = ["pkcs8", "pem", "rand_core"] } fallible-iterator = "0.3.0" form_urlencoded = "1.2.1" futures-util = { version = "0.3", default-features = false, features = ["alloc"] } +# geos = { version = "10.0.0", default-features = false, features = ["geo", "json"] } http-body-util = "0.1.3" hyper = "1.6.0" hyper-util = "0.1.7" diff --git a/crates/core/src/admin/logs/list_logs.rs b/crates/core/src/admin/logs/list_logs.rs index e220e060..c123d42f 100644 --- a/crates/core/src/admin/logs/list_logs.rs +++ b/crates/core/src/admin/logs/list_logs.rs @@ -55,7 +55,7 @@ pub async fn list_logs_handler( &[table], )?; - build_filter_where_clause(TABLE_ALIAS, &table_metadata.schema.columns, filter_params)? + build_filter_where_clause(TABLE_ALIAS, &table_metadata.column_metadata, filter_params)? }; let total_row_count: i64 = conn diff --git a/crates/core/src/admin/logs/stats.rs b/crates/core/src/admin/logs/stats.rs index a1b336e5..e7b9e544 100644 --- a/crates/core/src/admin/logs/stats.rs +++ b/crates/core/src/admin/logs/stats.rs @@ -52,7 +52,7 @@ pub async fn fetch_stats_handler( &[table], )?; - build_filter_where_clause(TABLE_ALIAS, &table_metadata.schema.columns, filter_params)? + build_filter_where_clause(TABLE_ALIAS, &table_metadata.column_metadata, filter_params)? }; let now = Utc::now(); diff --git a/crates/core/src/admin/rows/delete_rows.rs b/crates/core/src/admin/rows/delete_rows.rs index a911f9fe..dd562bee 100644 --- a/crates/core/src/admin/rows/delete_rows.rs +++ b/crates/core/src/admin/rows/delete_rows.rs @@ -5,7 +5,7 @@ use axum::{ response::{IntoResponse, Response}, }; use serde::{Deserialize, Serialize}; -use trailbase_schema::{QualifiedName, QualifiedNameEscaped}; +use trailbase_schema::{QualifiedName, QualifiedNameEscaped, metadata::find_file_column_indexes}; use trailbase_sqlvalue::SqlValue; use ts_rs::TS; @@ -54,21 +54,24 @@ pub(crate) async fn delete_row( ))); }; - let Some((_index, column)) = table_metadata.column_by_name(pk_col) else { + let Some(meta) = table_metadata.column_by_name(pk_col) else { return Err(Error::Precondition(format!("Missing column: {pk_col}"))); }; + let column = &meta.column; if !column.is_primary() { return Err(Error::Precondition(format!("Not a primary key: {pk_col}"))); } + let has_file_columns = !find_file_column_indexes(&table_metadata.column_metadata).is_empty(); + run_delete_query( &conn, state.objectstore(), &QualifiedNameEscaped::from(&table_metadata.schema.name), pk_col, pk_value.try_into()?, - table_metadata.json_metadata.has_file_columns(), + has_file_columns, ) .await?; diff --git a/crates/core/src/admin/rows/list_rows.rs b/crates/core/src/admin/rows/list_rows.rs index b8128782..8eaaf4cc 100644 --- a/crates/core/src/admin/rows/list_rows.rs +++ b/crates/core/src/admin/rows/list_rows.rs @@ -90,7 +90,7 @@ pub async fn list_rows_handler( let cursor_column = table_or_view.record_pk_column(); let cursor = match (cursor, cursor_column) { - (Some(cursor), Some((_idx, c))) => Some(parse_cursor(&cursor, c)?), + (Some(cursor), Some(meta)) => Some(parse_cursor(&cursor, &meta.column)?), _ => None, }; let (rows, columns) = fetch_rows( @@ -99,7 +99,7 @@ pub async fn list_rows_handler( filter_where_clause, &order, Pagination { - cursor_column: cursor_column.map(|(_idx, c)| c), + cursor_column: cursor_column.map(|meta| &meta.column), cursor, offset, limit: limit_or_default(limit, None).map_err(|err| Error::BadRequest(err.into()))?, @@ -108,10 +108,10 @@ pub async fn list_rows_handler( .await?; let next_cursor = if order.is_none() { - cursor_column.and_then(|(col_idx, _col)| { + cursor_column.and_then(|meta| { let row = rows.last()?; - assert!(row.len() > col_idx); - match &row[col_idx] { + assert!(row.len() > meta.index); + match &row[meta.index] { SqlValue::Integer(n) => Some(n.to_string()), SqlValue::Blob(b) => { // Should be a base64 encoded [u8; 16] id. @@ -130,7 +130,10 @@ pub async fn list_rows_handler( // NOTE: in the view case we don't have a good way of extracting the columns from the "CREATE // VIEW" query so we fall back to columns constructed from the returned data. columns: match table_or_view.columns() { - Some(schema_columns) if !schema_columns.is_empty() => schema_columns.to_vec(), + Some(schema_columns) if !schema_columns.is_empty() => schema_columns + .iter() + .map(|meta| meta.column.clone()) + .collect(), _ => { // VIRTUAL TABLE or VIEW case. debug!("Falling back to inferred cols for view: {table_name:?}"); diff --git a/crates/core/src/admin/rows/read_files.rs b/crates/core/src/admin/rows/read_files.rs index a6b786c5..a0acb178 100644 --- a/crates/core/src/admin/rows/read_files.rs +++ b/crates/core/src/admin/rows/read_files.rs @@ -3,7 +3,6 @@ use axum::{ response::Response, }; use serde::Deserialize; -use trailbase_schema::metadata::TableOrViewMetadata; use trailbase_schema::{FileUploads, QualifiedName}; use ts_rs::TS; @@ -39,45 +38,31 @@ pub async fn read_files_handler( "Table {table_name:?} not found" ))); }; - let Some((_index, pk_col)) = table_or_view.column_by_name(&query.pk_column) else { + let Some(pk_meta) = table_or_view.column_by_name(&query.pk_column) else { return Err(Error::Precondition(format!( "Missing PK column: {}", query.pk_column ))); }; + let pk_col = &pk_meta.column; if !pk_col.is_primary() { return Err(Error::Precondition(format!( "Not a primary key: {pk_col:?}" ))); } - let Some((index, file_col_metadata)) = table_or_view.column_by_name(&query.file_column_name) - else { + let Some(file_column_meta) = table_or_view.column_by_name(&query.file_column_name) else { return Err(Error::Precondition(format!( "Missing file column: {}", query.file_column_name ))); }; - let Some(file_col_json_metadata) = (match table_or_view { - TableOrViewMetadata::Table(t) => t.json_metadata.columns[index].as_ref(), - TableOrViewMetadata::View(v) => v - .json_metadata - .as_ref() - .and_then(|j| j.columns[index].as_ref()), - }) else { - return Err(Error::Precondition(format!( - "Not a JSON column: {}", - query.file_column_name - ))); - }; - let FileUploads(mut file_uploads) = run_get_files_query( &conn, &table_name.into(), - file_col_metadata, - file_col_json_metadata, + file_column_meta, &query.pk_column, trailbase_schema::json::parse_string_to_sqlite_value(pk_col.data_type, query.pk_value)?, ) diff --git a/crates/core/src/admin/rows/update_row.rs b/crates/core/src/admin/rows/update_row.rs index 496d5772..29d1c16c 100644 --- a/crates/core/src/admin/rows/update_row.rs +++ b/crates/core/src/admin/rows/update_row.rs @@ -46,16 +46,17 @@ pub async fn update_row_handler( }; let pk_col = &request.primary_key_column; - let Some((index, column)) = table_metadata.column_by_name(pk_col) else { + let Some(meta) = table_metadata.column_by_name(pk_col) else { return Err(Error::Precondition(format!("Missing column: {pk_col}"))); }; if let Some(pk_index) = table_metadata.record_pk_column - && index != pk_index + && meta.index != pk_index { return Err(Error::Precondition(format!("Pk column mismatch: {pk_col}"))); } + let column = &meta.column; if !column.is_primary() { return Err(Error::Precondition(format!("Not a primary key: {pk_col}"))); } diff --git a/crates/core/src/admin/user/list_users.rs b/crates/core/src/admin/user/list_users.rs index 091025c9..5faf30a6 100644 --- a/crates/core/src/admin/user/list_users.rs +++ b/crates/core/src/admin/user/list_users.rs @@ -81,7 +81,7 @@ pub async fn list_users_handler( // Where clause contains column filters and offset depending on what's present in the url query // string. let filter_where_clause = - build_filter_where_clause("_ROW_", &table_metadata.schema.columns, filter_params)?; + build_filter_where_clause("_ROW_", &table_metadata.column_metadata, filter_params)?; let total_row_count: i64 = conn .read_query_row_f( diff --git a/crates/core/src/auth/api/avatar.rs b/crates/core/src/auth/api/avatar.rs index 53ca37cb..80a08ba0 100644 --- a/crates/core/src/auth/api/avatar.rs +++ b/crates/core/src/auth/api/avatar.rs @@ -39,19 +39,14 @@ pub async fn get_avatar_handler( return Err(AuthError::Internal("missing table".into())); }; - let Some((index, file_column)) = table_metadata.column_by_name("file") else { + let Some(file_column_meta) = table_metadata.column_by_name("file") else { return Err(AuthError::Internal("missing column".into())); }; - let Some(ref column_json_metadata) = table_metadata.json_metadata.columns[index] else { - return Err(AuthError::Internal("missing metadata".into())); - }; - let file_upload = run_get_file_query( &conn, &trailbase_schema::QualifiedNameEscaped::new(&AVATAR_TABLE_NAME), - file_column, - column_json_metadata, + file_column_meta, "user", rusqlite::types::Value::Blob(user_id.into()), ) diff --git a/crates/core/src/connection.rs b/crates/core/src/connection.rs index 80da02e0..48075ee3 100644 --- a/crates/core/src/connection.rs +++ b/crates/core/src/connection.rs @@ -462,7 +462,11 @@ fn setup_file_deletion_triggers_sync( metadata: &ConnectionMetadata, ) -> Result<(), trailbase_sqlite::Error> { for metadata in metadata.tables.values() { - for idx in metadata.json_metadata.file_column_indexes() { + for column_meta in &metadata.column_metadata { + if !column_meta.is_file { + continue; + } + let table_name = &metadata.schema.name; let unqualified_name = &metadata.schema.name.name; let db = metadata @@ -472,8 +476,7 @@ fn setup_file_deletion_triggers_sync( .as_deref() .unwrap_or("main"); - let col = &metadata.schema.columns[*idx]; - let column_name = &col.name; + let column_name = &column_meta.column.name; conn.execute_batch(&format!( "\ diff --git a/crates/core/src/listing.rs b/crates/core/src/listing.rs index 939e4b8b..e148a926 100644 --- a/crates/core/src/listing.rs +++ b/crates/core/src/listing.rs @@ -1,7 +1,7 @@ use std::borrow::Cow; use thiserror::Error; use trailbase_qs::ValueOrComposite; -use trailbase_schema::sqlite::Column; +use trailbase_schema::metadata::ColumnMetadata; #[derive(Debug, Error)] pub enum WhereClauseError { @@ -23,7 +23,7 @@ pub struct WhereClause { pub(crate) fn build_filter_where_clause( table_name: &str, - columns: &[Column], + column_metadata: &[ColumnMetadata], filter_params: Option, ) -> Result { let Some(filter_params) = filter_params else { @@ -42,14 +42,17 @@ pub(crate) fn build_filter_where_clause( ))); } - let Some(column) = columns.iter().find(|c| c.name == column_name) else { + let Some(meta) = column_metadata + .iter() + .find(|meta| meta.column.name == column_name) + else { return Err(WhereClauseError::UnrecognizedParam(format!( "Unrecognized parameter: {column_name}" ))); }; // TODO: Improve hacky error handling. - return crate::records::filter::qs_value_to_sql_with_constraints(column, value) + return crate::records::filter::qs_value_to_sql_with_constraints(&meta.column, value) .map_err(|err| WhereClauseError::UnrecognizedParam(err.to_string())); }; diff --git a/crates/core/src/records/create_record.rs b/crates/core/src/records/create_record.rs index ceddcb65..600121bd 100644 --- a/crates/core/src/records/create_record.rs +++ b/crates/core/src/records/create_record.rs @@ -105,7 +105,7 @@ pub async fn create_record_handler( && let Some(ref user) = user { for column_index in api.user_id_columns() { - let col_name = &api.columns()[*column_index].name; + let col_name = &api.columns()[*column_index].column.name; if !record.contains_key(col_name) { record.insert( col_name.to_owned(), @@ -136,7 +136,7 @@ pub async fn create_record_handler( ); } - let (_index, pk_column) = api.record_pk_column(); + let pk_meta = api.record_pk_column(); let record_ids: Vec = match params_list.len() { 0 => { return Err(RecordError::BadRequest("no values provided")); @@ -147,7 +147,7 @@ pub async fn create_record_handler( state.objectstore(), api.table_name(), api.insert_conflict_resolution_strategy(), - &pk_column.name, + &pk_meta.column.name, params_list.swap_remove(0), ) .await?; @@ -161,7 +161,7 @@ pub async fn create_record_handler( let table_name: QualifiedNameEscaped = api.table_name().clone(); let (query, files) = WriteQuery::new_insert( &table_name, - &pk_column.name, + &pk_meta.column.name, api.insert_conflict_resolution_strategy(), params, ) diff --git a/crates/core/src/records/delete_record.rs b/crates/core/src/records/delete_record.rs index e4899bbc..25303e49 100644 --- a/crates/core/src/records/delete_record.rs +++ b/crates/core/src/records/delete_record.rs @@ -36,13 +36,13 @@ pub async fn delete_record_handler( .check_record_level_access(Permission::Delete, Some(&record_id), None, user.as_ref()) .await?; - let (_index, pk_column) = api.record_pk_column(); + let pk_meta = api.record_pk_column(); run_delete_query( api.conn(), state.objectstore(), api.table_name(), - &pk_column.name, + &pk_meta.column.name, record_id, api.has_file_columns(), ) diff --git a/crates/core/src/records/expand.rs b/crates/core/src/records/expand.rs index 37b56279..1e6914cc 100644 --- a/crates/core/src/records/expand.rs +++ b/crates/core/src/records/expand.rs @@ -4,7 +4,8 @@ use std::collections::HashMap; use thiserror::Error; use trailbase_schema::QualifiedName; use trailbase_schema::json::value_to_flat_json; -use trailbase_schema::sqlite::{Column, ColumnOption}; +use trailbase_schema::metadata::ColumnMetadata; +use trailbase_schema::sqlite::ColumnOption; use crate::records::RecordError; use crate::records::record_api::RecordApi; @@ -56,79 +57,84 @@ fn is_foreign_key(options: &[ColumnOption]) -> bool { /// Serialize SQL row to json. pub(crate) fn row_to_json_expand( - columns: &[Column], - json_metadata: &[Option], + column_metadata: &[ColumnMetadata], row: &trailbase_sqlite::Row, column_filter: fn(&str) -> bool, expand: Option<&HashMap>, ) -> Result { - // Row may contain extra columns like trailing "_rowid_". - assert!(columns.len() <= row.column_count()); - assert_eq!(columns.len(), json_metadata.len()); + // Row may contain extra columns like trailing "_rowid_" or excluded columns. + if column_metadata.len() > row.column_count() { + return Err(JsonError::NotSupported); + } return Ok(serde_json::Value::Object( - (0..columns.len()) - .filter(|i| column_filter(&columns[*i].name)) - .map(|i| -> Result<(String, serde_json::Value), JsonError> { - let column = &columns[i]; + column_metadata + .iter() + .enumerate() + .filter(|(_i, meta)| column_filter(&meta.column.name)) + .map( + |(i, meta)| -> Result<(String, serde_json::Value), JsonError> { + let column = &meta.column; + if column.name.as_str() != row.column_name(i).unwrap_or_default() { + return Err(JsonError::NotSupported); + } - assert_eq!(Some(column.name.as_str()), row.column_name(i)); + let value = row.get_value(i).ok_or(JsonError::ValueNotFound)?; + if matches!(value, types::Value::Null) { + return Ok((column.name.clone(), serde_json::Value::Null)); + } - let value = row.get_value(i).ok_or(JsonError::ValueNotFound)?; - if matches!(value, types::Value::Null) { - return Ok((column.name.clone(), serde_json::Value::Null)); - } + if let Some(foreign_value) = expand.and_then(|e| e.get(&column.name)) + && is_foreign_key(&column.options) + { + let id = value_to_flat_json(value)?; - if let Some(foreign_value) = expand.and_then(|e| e.get(&column.name)) - && is_foreign_key(&column.options) - { - let id = value_to_flat_json(value)?; + return Ok(match foreign_value { + serde_json::Value::Null => ( + column.name.clone(), + serde_json::json!({ + "id": id, + }), + ), + value => ( + column.name.clone(), + serde_json::json!({ + "id": id, + "data": value, + }), + ), + }); + } - return Ok(match foreign_value { - serde_json::Value::Null => ( - column.name.clone(), - serde_json::json!({ - "id": id, - }), - ), - value => ( - column.name.clone(), - serde_json::json!({ - "id": id, - "data": value, - }), - ), - }); - } - - // Deserialize JSON. - if let types::Value::Text(str) = value { - match json_metadata[i].as_ref() { - Some(JsonColumnMetadata::SchemaName(x)) if x == "std.FileUpload" => { - #[allow(unused_mut)] - let mut value: serde_json::Value = serde_json::from_str(str)?; - #[cfg(not(test))] - value.as_object_mut().map(|o| o.remove("id")); - return Ok((column.name.clone(), value)); - } - Some(JsonColumnMetadata::SchemaName(x)) if x == "std.FileUploads" => { - #[allow(unused_mut)] - let mut values: Vec = serde_json::from_str(str)?; - #[cfg(not(test))] - for value in &mut values { + // Deserialize JSON. + if let types::Value::Text(str) = value { + match meta.json.as_ref() { + Some(JsonColumnMetadata::SchemaName(x)) if x == "std.FileUpload" => { + #[allow(unused_mut)] + let mut value: serde_json::Value = serde_json::from_str(str)?; + #[cfg(not(test))] value.as_object_mut().map(|o| o.remove("id")); + return Ok((column.name.clone(), value)); } - return Ok((column.name.clone(), serde_json::Value::Array(values))); - } - Some(JsonColumnMetadata::SchemaName(_)) | Some(JsonColumnMetadata::Pattern(_)) => { - return Ok((column.name.clone(), serde_json::from_str(str)?)); - } - None => {} - }; - } + Some(JsonColumnMetadata::SchemaName(x)) if x == "std.FileUploads" => { + #[allow(unused_mut)] + let mut values: Vec = serde_json::from_str(str)?; + #[cfg(not(test))] + for value in &mut values { + value.as_object_mut().map(|o| o.remove("id")); + } + return Ok((column.name.clone(), serde_json::Value::Array(values))); + } + Some(JsonColumnMetadata::SchemaName(_)) | Some(JsonColumnMetadata::Pattern(_)) => { + return Ok((column.name.clone(), serde_json::from_str(str)?)); + } + None => {} + }; + } - return Ok((column.name.clone(), value_to_flat_json(value)?)); - }) + return Ok((column.name.clone(), value_to_flat_json(value)?)); + }, + ) .collect::>()?, )); } @@ -155,7 +161,7 @@ pub(crate) fn expand_tables<'s, T: AsRef>( continue; } - let Some(column) = record_api + let Some(meta) = record_api .column_index_by_name(col_name) .map(|idx| &record_api.columns()[idx]) else { @@ -167,7 +173,8 @@ pub(crate) fn expand_tables<'s, T: AsRef>( foreign_table: foreign_table_name, referred_columns: _, .. - }) = column + }) = meta + .column .options .iter() .find_or_first(|o| matches!(o, ColumnOption::ForeignKey { .. })) @@ -295,15 +302,7 @@ mod tests { let parsed = rows .iter() - .map(|row| { - row_to_json_expand( - &metadata.schema.columns, - &metadata.json_metadata.columns, - row, - |_| true, - None, - ) - }) + .map(|row| row_to_json_expand(&metadata.column_metadata, row, |_| true, None)) .collect::, _>>() .unwrap(); diff --git a/crates/core/src/records/filter.rs b/crates/core/src/records/filter.rs index e4246792..53bfafca 100644 --- a/crates/core/src/records/filter.rs +++ b/crates/core/src/records/filter.rs @@ -1,6 +1,9 @@ use regex::Regex; use trailbase_qs::{Combiner, CompareOp}; -use trailbase_schema::sqlite::{Column, ColumnDataType}; +use trailbase_schema::{ + metadata::ColumnMetadata, + sqlite::{Column, ColumnDataType}, +}; use crate::records::RecordError; @@ -76,20 +79,20 @@ pub(crate) fn qs_value_to_sql_with_constraints( } pub(crate) fn qs_filter_to_record_filter( - columns: &[Column], + column_metadata: &[ColumnMetadata], filter: trailbase_qs::ValueOrComposite, ) -> Result { return match filter { trailbase_qs::ValueOrComposite::Value(col_op_value) => { - let column = columns + let meta = column_metadata .iter() - .find(|c| c.name == col_op_value.column) + .find(|meta| meta.column.name == col_op_value.column) .ok_or_else(|| RecordError::BadRequest("Invalid query"))?; Ok(ValueOrComposite::Value(ColumnOpValue { column: col_op_value.column, op: col_op_value.op, - value: qs_value_to_sql_with_constraints(column, col_op_value.value)?, + value: qs_value_to_sql_with_constraints(&meta.column, col_op_value.value)?, })) } trailbase_qs::ValueOrComposite::Composite(combiner, expressions) => { @@ -97,7 +100,7 @@ pub(crate) fn qs_filter_to_record_filter( combiner, expressions .into_iter() - .map(|value_or_composite| qs_filter_to_record_filter(columns, value_or_composite)) + .map(|value_or_composite| qs_filter_to_record_filter(column_metadata, value_or_composite)) .collect::, _>>()?, )) } diff --git a/crates/core/src/records/list_records.rs b/crates/core/src/records/list_records.rs index f90441ce..d3ef4ab0 100644 --- a/crates/core/src/records/list_records.rs +++ b/crates/core/src/records/list_records.rs @@ -72,7 +72,8 @@ pub async fn list_records_handler( api.check_table_level_access(Permission::Read, user.as_ref())?; let table_name = api.table_name(); - let (_pk_index, pk_column) = api.record_pk_column(); + let pk_meta = api.record_pk_column(); + let pk_column = &pk_meta.column; let is_table = api.is_table(); let Query { @@ -208,7 +209,7 @@ pub async fn list_records_handler( column_names: &api .columns() .iter() - .map(|c| c.name.as_str()) + .map(|meta| meta.column.name.as_str()) .collect::>(), // NOTE: We're using the read access rule to filter accessible rows as opposed to blocking // access early as we do for READs. @@ -278,15 +279,7 @@ pub async fn list_records_handler( let records = if expanded_tables.is_empty() { rows .into_iter() - .map(|row| { - row_to_json_expand( - api.columns(), - api.json_column_metadata(), - &row, - column_filter, - api.expand(), - ) - }) + .map(|row| row_to_json_expand(api.columns(), &row, column_filter, api.expand())) .collect::, JsonError>>() .map_err(|err| RecordError::Internal(err.into()))? } else { @@ -306,8 +299,7 @@ pub async fn list_records_handler( let next = curr.split_off(expanded.num_columns); let foreign_value = row_to_json_expand( - &expanded.metadata.schema.columns, - &expanded.metadata.json_metadata.columns, + &expanded.metadata.column_metadata, &curr, column_filter, None, @@ -320,14 +312,8 @@ pub async fn list_records_handler( curr = next; } - return row_to_json_expand( - api.columns(), - api.json_column_metadata(), - &row, - column_filter, - Some(&expand), - ) - .map_err(|err| RecordError::Internal(err.into())); + return row_to_json_expand(api.columns(), &row, column_filter, Some(&expand)) + .map_err(|err| RecordError::Internal(err.into())); }) .collect::, RecordError>>()? }; diff --git a/crates/core/src/records/params.rs b/crates/core/src/records/params.rs index 631643c6..31fa7093 100644 --- a/crates/core/src/records/params.rs +++ b/crates/core/src/records/params.rs @@ -2,6 +2,7 @@ use parking_lot::RwLock; use std::collections::HashSet; use std::sync::Arc; use trailbase_schema::json::flat_json_to_value; +use trailbase_schema::metadata::ColumnMetadata; use trailbase_schema::registry::JsonSchemaRegistry; use trailbase_schema::sqlite::{Column, ColumnDataType}; use trailbase_schema::{FileUpload, FileUploadInput, FileUploads}; @@ -41,6 +42,8 @@ pub enum ParamsError { Storage(Arc), #[error("SqlValueDecode: {0}")] SqlValueDecode(#[from] trailbase_sqlvalue::DecodeError), + // #[error("Geos: {0}")] + // Geos(#[from] geos::Error), } impl From for ParamsError { @@ -76,40 +79,23 @@ pub(crate) type FileMetadataContents = Vec<(FileUpload, Vec)>; pub(crate) type JsonRow = serde_json::Map; -pub trait SchemaAccessor { - fn column_by_name( - &self, - field_name: &str, - ) -> Option<(usize, &Column, Option<&JsonColumnMetadata>)>; +pub trait ColumnAccessor { + fn column_by_name(&self, field_name: &str) -> Option<&ColumnMetadata>; } /// Implementation to build insert/update Params for admin APIs. -impl SchemaAccessor for TableMetadata { +impl ColumnAccessor for TableMetadata { #[inline] - fn column_by_name( - &self, - field_name: &str, - ) -> Option<(usize, &Column, Option<&JsonColumnMetadata>)> { - return self - .column_by_name(field_name) - .map(|(index, col)| (index, col, self.json_metadata.columns[index].as_ref())); + fn column_by_name(&self, field_name: &str) -> Option<&ColumnMetadata> { + return self.column_by_name(field_name); } } /// Implementation to build insert/update Params for record APIs. -impl SchemaAccessor for RecordApi { +impl ColumnAccessor for RecordApi { #[inline] - fn column_by_name( - &self, - field_name: &str, - ) -> Option<(usize, &Column, Option<&JsonColumnMetadata>)> { - return self.column_index_by_name(field_name).map(|index| { - return ( - index, - &self.columns()[index], - self.json_column_metadata()[index].as_ref(), - ); - }); + fn column_by_name(&self, field_name: &str) -> Option<&ColumnMetadata> { + return self.column_metadata_by_name(field_name); } } @@ -149,7 +135,7 @@ pub enum Params { impl Params { /// Converts a Json object + optional MultiPart files into trailbase_sqlite::Values and extracted /// files. - pub fn for_insert( + pub fn for_insert( accessor: &S, json_schema_registry: &JsonSchemaRegistry, row: JsonRow, @@ -165,12 +151,24 @@ impl Params { for (key, value) in row { // We simply skip unknown columns, this could simply be malformed input or version skew. This // is similar in spirit to protobuf's unknown fields behavior. - let Some((index, col, json_meta)) = accessor.column_by_name(&key) else { + let Some(ColumnMetadata { + index, + column, + json, + is_file: _, + is_geometry, + }) = accessor.column_by_name(&key) + else { continue; }; - let (param, json_files) = - extract_params_and_files_from_json(json_schema_registry, col, json_meta, value)?; + let (param, json_files) = extract_params_and_files_from_json( + json_schema_registry, + column, + json.as_ref(), + *is_geometry, + value, + )?; if let Some(json_files) = json_files { // Note: files provided as a multipart form upload are handled below. They need more // special handling to establish the field.name to column mapping. @@ -179,7 +177,7 @@ impl Params { named_params.push((prefix_colon(&key).into(), param)); column_names.push(key); - column_indexes.push(index); + column_indexes.push(*index); } // Note: files provided as part of a JSON request are handled above. @@ -201,7 +199,7 @@ impl Params { }); } - pub fn for_admin_insert( + pub fn for_admin_insert( accessor: &S, row: indexmap::IndexMap, ) -> Result { @@ -213,13 +211,20 @@ impl Params { for (key, value) in row { // We simply skip unknown columns, this could simply be malformed input or version skew. This // is similar in spirit to protobuf's unknown fields behavior. - let Some((index, _col, _json_meta)) = accessor.column_by_name(&key) else { + let Some(ColumnMetadata { + index, + column: _, + json: _, + is_file: _, + is_geometry: _, + }) = accessor.column_by_name(&key) + else { continue; }; named_params.push((prefix_colon(&key).into(), value.try_into()?)); column_names.push(key); - column_indexes.push(index); + column_indexes.push(*index); } return Ok(Params::Insert { @@ -230,7 +235,7 @@ impl Params { }); } - pub fn for_update( + pub fn for_update( accessor: &S, json_schema_registry: &JsonSchemaRegistry, row: JsonRow, @@ -248,12 +253,24 @@ impl Params { for (key, value) in row { // We simply skip unknown columns, this could simply be malformed input or version skew. This // is similar in spirit to protobuf's unknown fields behavior. - let Some((index, col, json_meta)) = accessor.column_by_name(&key) else { + let Some(ColumnMetadata { + index, + column, + json, + is_file: _, + is_geometry, + }) = accessor.column_by_name(&key) + else { continue; }; - let (param, json_files) = - extract_params_and_files_from_json(json_schema_registry, col, json_meta, value)?; + let (param, json_files) = extract_params_and_files_from_json( + json_schema_registry, + column, + json.as_ref(), + *is_geometry, + value, + )?; if let Some(json_files) = json_files { // Note: files provided as a multipart form upload are handled below. They need more // special handling to establish the field.name to column mapping. @@ -268,7 +285,7 @@ impl Params { named_params.push((prefix_colon(&key).into(), param)); column_names.push(key); - column_indexes.push(index); + column_indexes.push(*index); } // Inject the pk_value. It may already be present, if redundantly provided both in the API path @@ -295,7 +312,7 @@ impl Params { }); } - pub fn for_admin_update( + pub fn for_admin_update( accessor: &S, row: indexmap::IndexMap, pk_column_name: String, @@ -311,7 +328,14 @@ impl Params { for (key, value) in row { // We simply skip unknown columns, this could simply be malformed input or version skew. This // is similar in spirit to protobuf's unknown fields behavior. - let Some((index, _col, _json_meta)) = accessor.column_by_name(&key) else { + let Some(ColumnMetadata { + index, + column: _, + json: _, + is_file: _, + is_geometry: _, + }) = accessor.column_by_name(&key) + else { continue; }; @@ -325,7 +349,7 @@ impl Params { named_params.push((prefix_colon(&key).into(), param)); column_names.push(key); - column_indexes.push(index); + column_indexes.push(*index); } // Inject the pk_value. It may already be present, if redundantly provided both in the API path @@ -354,7 +378,7 @@ pub enum LazyParams<'a> { } impl<'a> LazyParams<'a> { - pub fn for_insert( + pub fn for_insert( accessor: &'a S, json_schema_registry: Arc>, json_row: JsonRow, @@ -370,7 +394,7 @@ impl<'a> LazyParams<'a> { }))); } - pub fn for_update( + pub fn for_update( accessor: &'a S, json_schema_registry: Arc>, json_row: JsonRow, @@ -418,7 +442,7 @@ impl<'a> LazyParams<'a> { } } -fn extract_files_from_multipart( +fn extract_files_from_multipart( accessor: &S, multipart_files: Vec, named_params: &mut NamedParams, @@ -443,36 +467,43 @@ fn extract_files_from_multipart( for (field_name, file_metadata, _content) in &files { // We simply skip unknown columns, this could simply be malformed input or version skew. This // is similar in spirit to protobuf's unknown fields behavior. - let Some((index, col, json_meta)) = accessor.column_by_name(field_name) else { + let Some(ColumnMetadata { + index, + column, + json, + is_file: _, + is_geometry: _, + }) = accessor.column_by_name(field_name) + else { continue; }; - let Some(JsonColumnMetadata::SchemaName(schema_name)) = &json_meta else { + let Some(JsonColumnMetadata::SchemaName(schema_name)) = &json else { return Err(ParamsError::Column("Expected json file column")); }; match schema_name.as_str() { "std.FileUpload" => { - if !uploaded_files.insert(&col.name) { + if !uploaded_files.insert(&column.name) { return Err(ParamsError::Column( "Collision: too many files for std.FileUpload", )); } named_params.push(( - prefix_colon(&col.name).into(), + prefix_colon(&column.name).into(), Value::Text(serde_json::to_string(&file_metadata)?), )); - column_names.push(col.name.to_string()); - column_indexes.push(index); + column_names.push(column.name.to_string()); + column_indexes.push(*index); } "std.FileUploads" => { named_params.push(( - prefix_colon(&col.name).into(), + prefix_colon(&column.name).into(), Value::Text(serde_json::to_string(&file_metadata)?), )); - column_names.push(col.name.to_string()); - column_indexes.push(index); + column_names.push(column.name.to_string()); + column_indexes.push(*index); } _ => { return Err(ParamsError::Column("Mismatching JSON schema")); @@ -492,10 +523,28 @@ fn extract_params_and_files_from_json( json_schema_registry: &JsonSchemaRegistry, col: &Column, json_metadata: Option<&JsonColumnMetadata>, + is_geometry: bool, value: serde_json::Value, ) -> Result<(Value, Option), ParamsError> { // If this is *not* a JSON column convert the value trivially. let Some(json_metadata) = json_metadata else { + // if is_geometry && col.data_type == ColumnDataType::Blob { + // use geos::Geom; + // + // let json_geometry = geos::geojson::Geometry::from_json_value(value) + // .map_err(|err| ParamsError::UnexpectedType("", format!("GeoJSON: {err}")))?; + // let geometry: geos::Geometry = json_geometry.try_into()?; + // + // let mut writer = geos::WKBWriter::new()?; + // if let Some(_) = geometry.get_srid().ok() { + // writer.set_include_SRID(true); + // } + // + // return Ok((Value::Blob(writer.write_wkb(&geometry)?.into()), None)); + // } + + debug_assert!(!is_geometry); + return Ok((flat_json_to_value(col.data_type, value)?, None)); }; diff --git a/crates/core/src/records/read_queries.rs b/crates/core/src/records/read_queries.rs index ff7f72a1..d62184aa 100644 --- a/crates/core/src/records/read_queries.rs +++ b/crates/core/src/records/read_queries.rs @@ -1,5 +1,5 @@ use askama::Template; -use trailbase_schema::sqlite::Column; +use trailbase_schema::metadata::ColumnMetadata; use trailbase_schema::{FileUpload, FileUploads, QualifiedNameEscaped}; use trailbase_sqlite::Value; @@ -70,14 +70,13 @@ pub(crate) async fn run_expanded_select_query<'a>( pub(crate) async fn run_get_file_query( conn: &trailbase_sqlite::Connection, table_name: &QualifiedNameEscaped, - file_column: &Column, - json_metadata: &JsonColumnMetadata, + file_column_meta: &ColumnMetadata, pk_column: &str, pk_value: Value, ) -> Result { - return match &json_metadata { - JsonColumnMetadata::SchemaName(name) if name == "std.FileUpload" => { - let column_name = &file_column.name; + return match &file_column_meta.json { + Some(JsonColumnMetadata::SchemaName(name)) if name == "std.FileUpload" => { + let column_name = &file_column_meta.column.name; let Some(row) = conn .read_query_row( @@ -104,14 +103,13 @@ pub(crate) async fn run_get_file_query( pub(crate) async fn run_get_files_query( conn: &trailbase_sqlite::Connection, table_name: &QualifiedNameEscaped, - file_column: &Column, - json_metadata: &JsonColumnMetadata, + file_column_meta: &ColumnMetadata, pk_column: &str, pk_value: Value, ) -> Result { - return match &json_metadata { - JsonColumnMetadata::SchemaName(name) if name == "std.FileUploads" => { - let column_name = &file_column.name; + return match &file_column_meta.json { + Some(JsonColumnMetadata::SchemaName(name)) if name == "std.FileUploads" => { + let column_name = &file_column_meta.column.name; let Some(row) = conn .read_query_row( @@ -131,17 +129,9 @@ pub(crate) async fn run_get_files_query( serde_json::from_str(&contents).map_err(|err| RecordError::Internal(err.into()))?; Ok(file_uploads) } - JsonColumnMetadata::SchemaName(name) if name == "std.FileUpload" => { + Some(JsonColumnMetadata::SchemaName(name)) if name == "std.FileUpload" => { return Ok(FileUploads(vec![ - run_get_file_query( - conn, - table_name, - file_column, - json_metadata, - pk_column, - pk_value, - ) - .await?, + run_get_file_query(conn, table_name, file_column_meta, pk_column, pk_value).await?, ])); } _ => Err(RecordError::BadRequest("Not a files list")), diff --git a/crates/core/src/records/read_record.rs b/crates/core/src/records/read_record.rs index 903f1127..c752faa6 100644 --- a/crates/core/src/records/read_record.rs +++ b/crates/core/src/records/read_record.rs @@ -11,7 +11,6 @@ use crate::auth::user::User; use crate::records::expand::expand_tables; use crate::records::expand::row_to_json_expand; use crate::records::files::read_file_into_response; -use crate::records::params::SchemaAccessor; use crate::records::read_queries::{ ExpandedSelectQueryResult, run_expanded_select_query, run_get_file_query, run_get_files_query, run_select_query, @@ -51,89 +50,80 @@ pub async fn read_record_handler( .check_record_level_access(Permission::Read, Some(&record_id), None, user.as_ref()) .await?; - let (_index, pk_column) = api.record_pk_column(); - let column_names: Vec<_> = api.columns().iter().map(|c| c.name.as_str()).collect(); + let pk_meta = api.record_pk_column(); + let column_names = || { + api + .columns() + .iter() + .map(|meta| meta.column.name.as_str()) + .collect::>() + }; - return Ok(Json(match query.expand { - Some(query_expand) if !query_expand.is_empty() => { - let Some(expand) = api.expand() else { + if let Some(query_expand) = query.expand + && !query_expand.is_empty() + { + let Some(expand) = api.expand() else { + return Err(RecordError::BadRequest("Invalid expansion")); + }; + + // Input validation, i.e. only accept columns that are also configured. + let query_expand: Vec<_> = query_expand.split(",").collect(); + for col_name in &query_expand { + if !query_expand.contains(col_name) { return Err(RecordError::BadRequest("Invalid expansion")); - }; - - // Input validation, i.e. only accept columns that are also configured. - let query_expand: Vec<_> = query_expand.split(",").collect(); - for col_name in &query_expand { - if !query_expand.contains(col_name) { - return Err(RecordError::BadRequest("Invalid expansion")); - } } + } - let metadata = api.connection_metadata(); - let expanded_tables = expand_tables(&api, &metadata, &query_expand)?; + let metadata = api.connection_metadata(); + let expanded_tables = expand_tables(&api, &metadata, &query_expand)?; - let Some(ExpandedSelectQueryResult { root, foreign_rows }) = run_expanded_select_query( - api.conn(), - api.table_name(), - &column_names, - &pk_column.name, - record_id, - &expanded_tables, - ) - .await? - else { - return Err(RecordError::RecordNotFound); - }; + let Some(ExpandedSelectQueryResult { root, foreign_rows }) = run_expanded_select_query( + api.conn(), + api.table_name(), + &column_names(), + &pk_meta.column.name, + record_id, + &expanded_tables, + ) + .await? + else { + return Err(RecordError::RecordNotFound); + }; - // Alloc a map from column name to value that's pre-filled with with Value::Null for all - // expandable columns. - let mut expand = expand.clone(); + // Alloc a map from column name to value that's pre-filled with with Value::Null for all + // expandable columns. + let mut expand = expand.clone(); - for (col_name, (metadata, row)) in std::iter::zip(query_expand, foreign_rows) { - let foreign_value = row_to_json_expand( - &metadata.schema.columns, - &metadata.json_metadata.columns, - &row, - prefix_filter, - None, - ) + for (col_name, (metadata, row)) in std::iter::zip(query_expand, foreign_rows) { + let foreign_value = row_to_json_expand(&metadata.column_metadata, &row, prefix_filter, None) .map_err(|err| RecordError::Internal(err.into()))?; - let result = expand.insert(col_name.to_string(), foreign_value); - assert!(result.is_some()); - } - - row_to_json_expand( - api.columns(), - api.json_column_metadata(), - &root, - prefix_filter, - Some(&expand), - ) - .map_err(|err| RecordError::Internal(err.into()))? + let result = expand.insert(col_name.to_string(), foreign_value); + assert!(result.is_some()); } - Some(_) | None => { - let Some(row) = run_select_query( - api.conn(), - api.table_name(), - &column_names, - &pk_column.name, - record_id, - ) - .await? - else { - return Err(RecordError::RecordNotFound); - }; - row_to_json_expand( - api.columns(), - api.json_column_metadata(), - &row, - prefix_filter, - api.expand(), - ) - .map_err(|err| RecordError::Internal(err.into()))? - } - })); + return Ok(Json( + row_to_json_expand(api.columns(), &root, prefix_filter, Some(&expand)) + .map_err(|err| RecordError::Internal(err.into()))?, + )); + } + + let Some(row) = run_select_query( + api.conn(), + api.table_name(), + &column_names(), + &pk_meta.column.name, + record_id, + ) + .await? + else { + return Err(RecordError::RecordNotFound); + }; + + return Ok(Json( + row_to_json_expand(api.columns(), &row, prefix_filter, api.expand()) + .map_err(|err| RecordError::Internal(err.into()))?, + )); } type GetUploadedFileFromRecordPath = Path<( @@ -172,22 +162,17 @@ pub async fn get_uploaded_file_from_record_handler( return Err(RecordError::Forbidden); }; - let (_index, pk_column) = api.record_pk_column(); - let Some(index) = api.column_index_by_name(&column_name) else { - return Err(RecordError::BadRequest("Invalid field/column name")); - }; + let pk_meta = api.record_pk_column(); - let column = &api.columns()[index]; - let Some(ref column_json_metadata) = api.json_column_metadata()[index] else { - return Err(RecordError::BadRequest("Invalid column")); + let Some(column_metadata) = api.column_metadata_by_name(&column_name) else { + return Err(RecordError::BadRequest("Invalid field/column name")); }; let file_upload = run_get_file_query( api.conn(), api.table_name(), - column, - column_json_metadata, - &pk_column.name, + column_metadata, + &pk_meta.column.name, record_id, ) .await?; @@ -229,16 +214,14 @@ pub async fn get_uploaded_files_from_record_handler( .check_record_level_access(Permission::Read, Some(&record_id), None, user.as_ref()) .await?; - let Some((_index, column, Some(column_json_metadata))) = api.column_by_name(&column_name) else { + let Some(column_metadata) = api.column_metadata_by_name(&column_name) else { return Err(RecordError::BadRequest("Invalid field/column name")); }; - let FileUploads(file_uploads) = run_get_files_query( api.conn(), api.table_name(), - column, - column_json_metadata, - &api.record_pk_column().1.name, + column_metadata, + &api.record_pk_column().column.name, record_id, ) .await?; diff --git a/crates/core/src/records/record_api.rs b/crates/core/src/records/record_api.rs index 7739d71e..62037e7b 100644 --- a/crates/core/src/records/record_api.rs +++ b/crates/core/src/records/record_api.rs @@ -5,10 +5,9 @@ use std::borrow::Cow; use std::collections::HashMap; use std::sync::Arc; use trailbase_schema::metadata::{ - ConnectionMetadata, JsonColumnMetadata, TableMetadata, ViewMetadata, find_file_column_indexes, + ColumnMetadata, ConnectionMetadata, TableMetadata, ViewMetadata, find_file_column_indexes, find_user_id_foreign_key_columns, }; -use trailbase_schema::sqlite::Column; use trailbase_schema::{QualifiedName, QualifiedNameEscaped}; use trailbase_sqlite::{Connection, NamedParams, Params as _, Value}; @@ -30,15 +29,20 @@ struct RecordApiSchema { attached_databases: Vec, is_table: bool, - record_pk_column: (usize, Column), - columns: Vec, - json_column_metadata: Vec>, + record_pk_column: ColumnMetadata, + column_metadata: Vec, + has_file_columns: bool, user_id_columns: Vec, - // Helpers - column_name_to_index: HashMap, + // Helpers: + + // SQLite parameter emplate named_params_template: NamedParams, + // Mapping form column name to *API column index*. + // NOTE: The index may be different from the column's index in the underlying TABLE or VIEW + // due to excluded columns. + column_name_to_index: HashMap, } type DeferredAclCheck = dyn (FnOnce(&rusqlite::Connection) -> Result<(), RecordError>) + Send; @@ -47,32 +51,26 @@ impl RecordApiSchema { fn from_table(table_metadata: &TableMetadata, config: &RecordApiConfig) -> Result { assert_name(config, table_metadata.name()); - let Some((pk_index, pk_column)) = table_metadata.record_pk_column() else { + let Some(record_pk_column) = table_metadata.record_pk_column() else { return Err("RecordApi requires integer/UUIDv7 primary key column".into()); }; - let record_pk_column = (pk_index, pk_column.clone()); - let (columns, json_column_metadata) = filter_columns( - config, - &table_metadata.schema.columns, - &table_metadata.json_metadata.columns, - ); - - let has_file_columns = !find_file_column_indexes(&json_column_metadata).is_empty(); - let user_id_columns = find_user_id_foreign_key_columns(&columns, USER_TABLE); + let column_metadata = filter_excluded_columns(config, &table_metadata.column_metadata); + let has_file_columns = !find_file_column_indexes(&column_metadata).is_empty(); + let user_id_columns = find_user_id_foreign_key_columns(&column_metadata, USER_TABLE); let column_name_to_index = HashMap::::from_iter( - columns + column_metadata .iter() .enumerate() - .map(|(index, col)| (col.name.clone(), index)), + .map(|(index, meta)| (meta.column.name.clone(), index)), ); - let named_params_template: NamedParams = columns + let named_params_template: NamedParams = column_metadata .iter() - .map(|column| { + .map(|meta| { ( - Cow::Owned(prefix_colon(&column.name)), + Cow::Owned(prefix_colon(&meta.column.name)), trailbase_sqlite::Value::Null, ) }) @@ -83,9 +81,8 @@ impl RecordApiSchema { table_name: QualifiedNameEscaped::new(&table_metadata.schema.name), attached_databases: config.attached_databases.clone(), is_table: true, - record_pk_column, - columns, - json_column_metadata, + record_pk_column: record_pk_column.clone(), + column_metadata, has_file_columns, user_id_columns, column_name_to_index, @@ -96,30 +93,25 @@ impl RecordApiSchema { fn from_view(view_metadata: &ViewMetadata, config: &RecordApiConfig) -> Result { assert_name(config, view_metadata.name()); - let Some((pk_index, pk_column)) = view_metadata.record_pk_column() else { + let Some(record_pk_column) = view_metadata.record_pk_column() else { return Err(format!( "RecordApi requires integer/UUIDv7 primary key column: {config:?}" )); }; - let record_pk_column = (pk_index, pk_column.clone()); - let Some(columns) = view_metadata.columns() else { + let Some(ref column_metadata) = view_metadata.column_metadata else { return Err("RecordApi requires schema".to_string()); }; - let Some(ref json_metadata) = view_metadata.json_metadata else { - return Err("RecordApi requires json metadata".to_string()); - }; - let (columns, json_column_metadata) = filter_columns(config, columns, &json_metadata.columns); - - let has_file_columns = !find_file_column_indexes(&json_column_metadata).is_empty(); - let user_id_columns = find_user_id_foreign_key_columns(&columns, USER_TABLE); + let column_metadata = filter_excluded_columns(config, column_metadata); + let has_file_columns = !find_file_column_indexes(&column_metadata).is_empty(); + let user_id_columns = find_user_id_foreign_key_columns(&column_metadata, USER_TABLE); let column_name_to_index = HashMap::::from_iter( - columns + column_metadata .iter() .enumerate() - .map(|(index, col)| (col.name.clone(), index)), + .map(|(index, meta)| (meta.column.name.clone(), index)), ); return Ok(Self { @@ -127,9 +119,8 @@ impl RecordApiSchema { table_name: QualifiedNameEscaped::new(&view_metadata.schema.name), attached_databases: config.attached_databases.clone(), is_table: false, - record_pk_column, - columns, - json_column_metadata, + record_pk_column: record_pk_column.clone(), + column_metadata: column_metadata.clone(), has_file_columns, user_id_columns, column_name_to_index, @@ -224,22 +215,27 @@ impl RecordApi { schema: RecordApiSchema, config: RecordApiConfig, ) -> Result { - assert_eq!(schema.columns.len(), schema.json_column_metadata.len()); - let Some(api_name) = config.name.clone() else { return Err(format!("RecordApi misses name: {config:?}")); }; let (read_access_query, subscription_read_access_query) = match &config.read_access_rule { Some(rule) => { - let read_access_query = - build_read_delete_schema_query(&schema.table_name, &schema.record_pk_column.1.name, rule); + let read_access_query = build_read_delete_schema_query( + &schema.table_name, + &schema.record_pk_column.column.name, + rule, + ); let subscription_read_access_query = if schema.is_table { Some( SubscriptionRecordReadTemplate { read_access_rule: rule, - column_names: schema.columns.iter().map(|c| c.name.as_str()).collect(), + column_names: schema + .column_metadata + .iter() + .map(|m| m.column.name.as_str()) + .collect(), } .render() .map_err(|err| err.to_string())?, @@ -254,17 +250,25 @@ impl RecordApi { }; let delete_access_query = config.delete_access_rule.as_ref().map(|rule| { - build_read_delete_schema_query(&schema.table_name, &schema.record_pk_column.1.name, rule) + build_read_delete_schema_query( + &schema.table_name, + &schema.record_pk_column.column.name, + rule, + ) }); let schema_access_query = config.schema_access_rule.as_ref().map(|rule| { - build_read_delete_schema_query(&schema.table_name, &schema.record_pk_column.1.name, rule) + build_read_delete_schema_query( + &schema.table_name, + &schema.record_pk_column.column.name, + rule, + ) }); let create_access_query = match &config.create_access_rule { Some(rule) => { if schema.is_table { - Some(build_create_access_query(&schema.columns, rule)?) + Some(build_create_access_query(&schema.column_metadata, rule)?) } else { None } @@ -277,8 +281,8 @@ impl RecordApi { if schema.is_table { Some(build_update_access_query( &schema.table_name, - &schema.columns, - &schema.record_pk_column.1.name, + &schema.column_metadata, + &schema.record_pk_column.column.name, rule, )?) } else { @@ -397,18 +401,13 @@ impl RecordApi { } #[inline] - pub fn record_pk_column(&self) -> &(usize, Column) { + pub fn record_pk_column(&self) -> &ColumnMetadata { return &self.state.schema.record_pk_column; } #[inline] - pub fn columns(&self) -> &[Column] { - return &self.state.schema.columns; - } - - #[inline] - pub fn json_column_metadata(&self) -> &[Option] { - return &self.state.schema.json_column_metadata; + pub fn columns(&self) -> &[ColumnMetadata] { + return &self.state.schema.column_metadata; } #[inline] @@ -417,14 +416,19 @@ impl RecordApi { } #[inline] - pub fn column_index_by_name(&self, key: &str) -> Option { - return self.state.schema.column_name_to_index.get(key).copied(); + pub fn column_index_by_name(&self, name: &str) -> Option { + return self.state.schema.column_name_to_index.get(name).copied(); + } + + #[inline] + pub fn column_metadata_by_name(&self, name: &str) -> Option<&ColumnMetadata> { + return Some(&self.state.schema.column_metadata[self.column_index_by_name(name)?]); } pub fn primary_key_to_value(&self, pk: String) -> Result { // NOTE: loosly parse - will convert STRING to INT/REAL. return trailbase_schema::json::parse_string_to_sqlite_value( - self.state.schema.record_pk_column.1.data_type, + self.state.schema.record_pk_column.column.data_type, pk, ) .map_err(|_| RecordError::BadRequest("Invalid id")); @@ -769,10 +773,13 @@ struct CreateRecordAccessQueryTemplate<'a> { /// /// Assumes access_rule is an expression: https://www.sqlite.org/syntax/expr.html fn build_create_access_query( - columns: &[Column], + column_metadata: &[ColumnMetadata], create_access_rule: &str, ) -> Result, String> { - let column_names: Vec<&str> = columns.iter().map(|c| c.name.as_str()).collect(); + let column_names: Vec<&str> = column_metadata + .iter() + .map(|m| m.column.name.as_str()) + .collect(); return Ok( CreateRecordAccessQueryTemplate { @@ -803,11 +810,14 @@ struct UpdateRecordAccessQueryTemplate<'a> { /// Assumes access_rule is an expression: https://www.sqlite.org/syntax/expr.html fn build_update_access_query( table_name: &QualifiedNameEscaped, - columns: &[Column], + column_metadata: &[ColumnMetadata], pk_column_name: &str, update_access_rule: &str, ) -> Result, String> { - let column_names: Vec<&str> = columns.iter().map(|c| c.name.as_str()).collect(); + let column_names: Vec<&str> = column_metadata + .iter() + .map(|m| m.column.name.as_str()) + .collect(); return Ok( UpdateRecordAccessQueryTemplate { @@ -839,33 +849,23 @@ enum Entity { Authenticated = 1, } -fn filter_columns( +fn filter_excluded_columns( config: &RecordApiConfig, - columns: &[Column], - json_column_metadata: &[Option], -) -> (Vec, Vec>) { - assert_eq!(columns.len(), json_column_metadata.len()); + column_metadata: &[ColumnMetadata], +) -> Vec { if config.excluded_columns.is_empty() { - return (columns.to_vec(), json_column_metadata.to_vec()); + return column_metadata.to_vec(); } - let excluded_indexes = config - .excluded_columns + return column_metadata .iter() - .filter_map(|name| columns.iter().position(|col| col.name == *name)); - assert_eq!( - excluded_indexes.clone().count(), - config.excluded_columns.len() - ); - - let mut columns_vec = columns.to_vec(); - let mut json_column_metadata_vec = json_column_metadata.to_vec(); - for idx in excluded_indexes.rev() { - columns_vec.remove(idx); - json_column_metadata_vec.remove(idx); - } - - return (columns_vec, json_column_metadata_vec); + .filter_map(|meta| { + if config.excluded_columns.contains(&meta.column.name) { + return None; + } + return Some(meta.clone()); + }) + .collect(); } #[inline] diff --git a/crates/core/src/records/subscribe.rs b/crates/core/src/records/subscribe.rs index 47103a5a..47a66062 100644 --- a/crates/core/src/records/subscribe.rs +++ b/crates/core/src/records/subscribe.rs @@ -323,7 +323,7 @@ impl PerConnectionState { let table_name = api.table_name(); let qualified_table_name = api.qualified_name().clone(); - let pk_column = &api.record_pk_column().1.name; + let pk_column = &api.record_pk_column().column.name; let Some(row_id): Option = api .conn() diff --git a/crates/core/src/records/transaction.rs b/crates/core/src/records/transaction.rs index 6a631290..6d943a3f 100644 --- a/crates/core/src/records/transaction.rs +++ b/crates/core/src/records/transaction.rs @@ -105,7 +105,7 @@ pub async fn record_transactions_handler( && let Some(ref user) = user { for column_index in api.user_id_columns() { - let col_name = &api.columns()[*column_index].name; + let col_name = &api.columns()[*column_index].column.name; if !record.contains_key(col_name) { record.insert( col_name.to_owned(), @@ -126,7 +126,7 @@ pub async fn record_transactions_handler( let (query, _files) = WriteQuery::new_insert( api.table_name(), - &api.record_pk_column().1.name, + &api.record_pk_column().column.name, api.insert_conflict_resolution_strategy(), lazy_params .consume() @@ -154,14 +154,14 @@ pub async fn record_transactions_handler( let api = get_api(&state, &api_name, idx)?; let record = extract_record(value)?; let record_id = api.primary_key_to_value(record_id)?; - let (_index, pk_column) = api.record_pk_column(); + let pk_meta = api.record_pk_column(); let mut lazy_params = LazyParams::for_update( &api, state.json_schema_registry().clone(), record, None, - pk_column.name.clone(), + pk_meta.column.name.clone(), record_id.clone(), ); @@ -203,9 +203,12 @@ pub async fn record_transactions_handler( user.as_ref(), )?; - let query = - WriteQuery::new_delete(api.table_name(), &api.record_pk_column().1.name, record_id) - .map_err(|err| RecordError::Internal(err.into()))?; + let query = WriteQuery::new_delete( + api.table_name(), + &api.record_pk_column().column.name, + record_id, + ) + .map_err(|err| RecordError::Internal(err.into()))?; Ok(Box::new(move |conn| { acl_check(conn)?; diff --git a/crates/core/src/records/update_record.rs b/crates/core/src/records/update_record.rs index 9855823b..ece239c8 100644 --- a/crates/core/src/records/update_record.rs +++ b/crates/core/src/records/update_record.rs @@ -38,14 +38,14 @@ pub async fn update_record_handler( }; let record_id = api.primary_key_to_value(record)?; - let (_index, pk_column) = api.record_pk_column(); + let pk_meta = api.record_pk_column(); let mut lazy_params = LazyParams::for_update( &api, state.json_schema_registry().clone(), request, multipart_files, - pk_column.name.clone(), + pk_meta.column.name.clone(), record_id.clone(), ); diff --git a/crates/core/src/records/validate.rs b/crates/core/src/records/validate.rs index e0ae5697..dcf596ef 100644 --- a/crates/core/src/records/validate.rs +++ b/crates/core/src/records/validate.rs @@ -93,7 +93,7 @@ pub(crate) fn validate_record_api_config( "Must be STRICT for strong end-to-end type-safety.", )); } - table.schema.columns.as_slice() + table.column_metadata.as_slice() } TableOrViewMetadata::View(view) => { prefix.entity = Entity::View; @@ -111,7 +111,7 @@ pub(crate) fn validate_record_api_config( } }; - let Some((pk_index, _)) = table_or_view.record_pk_column() else { + let Some(pk_meta) = table_or_view.record_pk_column() else { return Err(invalid_prefixed( &prefix, "Does not have a suitable PRIMARY KEY column. At this point TrailBase requires PRIMARY KEYS to be of type INTEGER or UUID (i.e. BLOB with `is_uuid.*` CHECK constraint).", @@ -121,7 +121,7 @@ pub(crate) fn validate_record_api_config( for excluded_column_name in &api_config.excluded_columns { let Some(excluded_index) = columns .iter() - .position(|col| col.name == *excluded_column_name) + .position(|meta| meta.column.name == *excluded_column_name) else { return Err(invalid_prefixed( &prefix, @@ -129,14 +129,14 @@ pub(crate) fn validate_record_api_config( )); }; - if excluded_index == pk_index { + if excluded_index == pk_meta.index { return Err(invalid_prefixed( &prefix, format!("PK column '{excluded_column_name}' must not be excluded.",), )); } - let excluded_column = &columns[excluded_index]; + let excluded_column = &columns[excluded_index].column; if excluded_column.is_not_null() && !excluded_column.has_default() { return Err(invalid_prefixed( &prefix, @@ -155,7 +155,7 @@ pub(crate) fn validate_record_api_config( )); } - let Some(column) = columns.iter().find(|c| c.name == *expand) else { + let Some(meta) = columns.iter().find(|meta| meta.column.name == *expand) else { return Err(invalid_prefixed( &prefix, format!("Expands unknown column '{expand}'."), @@ -166,7 +166,8 @@ pub(crate) fn validate_record_api_config( foreign_table: foreign_table_name, referred_columns, .. - }) = column + }) = meta + .column .options .iter() .find_or_first(|o| matches!(o, ColumnOption::ForeignKey { .. })) @@ -192,7 +193,7 @@ pub(crate) fn validate_record_api_config( )); }; - let Some((_idx, foreign_pk_column)) = foreign_table.record_pk_column() else { + let Some(foreign_pk_meta) = foreign_table.record_pk_column() else { return Err(invalid_prefixed( &prefix, format!("Expanded foreign table '{foreign_table_name}' lacks suitable PRIMARY KEY."), @@ -202,7 +203,7 @@ pub(crate) fn validate_record_api_config( match referred_columns.len() { 0 => {} 1 => { - if referred_columns[0] != foreign_pk_column.name { + if referred_columns[0] != foreign_pk_meta.column.name { return Err(invalid_prefixed( &prefix, format!("Expanded column '{expand}' references non-PK."), diff --git a/crates/core/src/schema_metadata.rs b/crates/core/src/schema_metadata.rs index 06d38240..259f6d9a 100644 --- a/crates/core/src/schema_metadata.rs +++ b/crates/core/src/schema_metadata.rs @@ -275,7 +275,7 @@ mod tests { let (validator, schema) = build_json_schema_expanded( &state.json_schema_registry().read(), &table_name.name, - &table_metadata.schema.columns, + &table_metadata.column_metadata, JsonSchemaMode::Select, Some(Expand { tables: &metadata.tables.values().collect::>(), diff --git a/crates/schema/src/json_schema.rs b/crates/schema/src/json_schema.rs index 1c3e285f..499a53e1 100644 --- a/crates/schema/src/json_schema.rs +++ b/crates/schema/src/json_schema.rs @@ -5,9 +5,10 @@ use serde_json::Value; use trailbase_extension::jsonschema::JsonSchemaRegistry; use crate::metadata::{ - JsonColumnMetadata, JsonSchemaError, TableMetadata, extract_json_metadata, is_pk_column, + ColumnMetadata, JsonColumnMetadata, JsonSchemaError, TableMetadata, extract_json_metadata, + is_pk_column, }; -use crate::sqlite::{Column, ColumnDataType, ColumnOption}; +use crate::sqlite::{ColumnDataType, ColumnOption}; /// Influeces the generated JSON schema. In `Insert` mode columns with default values will be /// optional. @@ -32,7 +33,7 @@ pub enum JsonSchemaMode { pub fn build_json_schema( registry: &JsonSchemaRegistry, title: &str, - columns: &[Column], + columns: &[ColumnMetadata], mode: JsonSchemaMode, ) -> Result<(Validator, serde_json::Value), JsonSchemaError> { return build_json_schema_expanded(registry, title, columns, mode, None); @@ -49,7 +50,7 @@ pub struct Expand<'a> { pub fn build_json_schema_expanded( registry: &JsonSchemaRegistry, title: &str, - columns: &[Column], + columns_metadata: &[ColumnMetadata], mode: JsonSchemaMode, expand: Option>, ) -> Result<(Validator, serde_json::Value), JsonSchemaError> { @@ -57,7 +58,8 @@ pub fn build_json_schema_expanded( let mut defs = serde_json::Map::new(); let mut required_cols: Vec = vec![]; - for col in columns { + for meta in columns_metadata { + let col = &meta.column; let mut def_name: Option = None; let mut not_null = false; let mut default = false; @@ -158,7 +160,7 @@ pub fn build_json_schema_expanded( }; let (_validator, schema) = - build_json_schema(registry, foreign_table, &table.schema.columns, mode)?; + build_json_schema(registry, foreign_table, &table.column_metadata, mode)?; let new_def_name = foreign_table.clone(); defs.insert( @@ -434,7 +436,7 @@ mod tests { let (schema, _) = build_json_schema( ®istry, &table_metadata.name().name, - &table_metadata.schema.columns, + &table_metadata.column_metadata, JsonSchemaMode::Insert, ) .unwrap(); diff --git a/crates/schema/src/metadata.rs b/crates/schema/src/metadata.rs index b4319b5a..1f11a0cc 100644 --- a/crates/schema/src/metadata.rs +++ b/crates/schema/src/metadata.rs @@ -65,38 +65,19 @@ impl JsonColumnMetadata { } } -#[derive(Debug, Clone, PartialEq)] -pub struct JsonMetadata { - pub columns: Vec>, - - // Contains both, 'std.FileUpload' and 'std.FileUpload'. - file_column_indexes: Vec, -} - -impl JsonMetadata { - fn from_columns( - registry: &JsonSchemaRegistry, - columns: &[Column], - ) -> Result { - let columns = columns - .iter() - .map(|c| build_json_metadata(registry, c)) - .collect::, _>>()?; - - return Ok(Self { - file_column_indexes: find_file_column_indexes(&columns), - columns, - }); - } - - pub fn has_file_columns(&self) -> bool { - return !self.file_column_indexes.is_empty(); - } - - /// Contains both, 'std.FileUpload' and 'std.FileUpload'. - pub fn file_column_indexes(&self) -> &[usize] { - return &self.file_column_indexes; - } +#[derive(Debug, Clone)] +pub struct ColumnMetadata { + /// The original table or view index. May be different from column index in a RecordApi after + /// filtering out excluded columns. + pub index: usize, + /// Column schema. + pub column: Column, + /// JSON metadata if the column has a schema defined, i.e. `CHECK(jsonschema(_))`. + pub json: Option, + /// Whether the JSON schema happens to be `std.FileUpload[s]`. + pub is_file: bool, + /// Whether the column has an ST_Valid geometry check constaint. + pub is_geometry: bool, } /// A data class describing a sqlite Table and additional meta data useful for TrailBase. @@ -108,8 +89,7 @@ pub struct TableMetadata { /// If and which column on this table qualifies as a record PK column, i.e. integer or UUIDv7. pub record_pk_column: Option, - /// Metadata for CHECK(jsonschema()) columns. - pub json_metadata: JsonMetadata, + pub column_metadata: Vec, name_to_index: HashMap, // TODO: Add triggers once sqlparser supports a sqlite "CREATE TRIGGER" statements. @@ -125,16 +105,31 @@ impl TableMetadata { table: Table, tables: &[Table], ) -> Result { + let column_metadata: Vec<_> = table + .columns + .iter() + .enumerate() + .map(|(i, c)| { + let json_metadata = build_json_metadata(registry, c)?; + + return Ok(ColumnMetadata { + index: i, + is_file: is_file_column(&json_metadata), + json: json_metadata, + is_geometry: is_geometry_column(c), + column: c.clone(), + }); + }) + .collect::>()?; + return Ok(TableMetadata { record_pk_column: find_record_pk_column_index_for_table(&table, tables), - json_metadata: JsonMetadata::from_columns(registry, &table.columns)?, name_to_index: HashMap::::from_iter( - table - .columns + column_metadata .iter() - .enumerate() - .map(|(index, col)| (col.name.clone(), index)), + .map(|meta| (meta.column.name.clone(), meta.index)), ), + column_metadata, schema: table, }); } @@ -149,14 +144,14 @@ impl TableMetadata { return self.name_to_index.get(key).copied(); } - pub fn column_by_name(&self, key: &str) -> Option<(usize, &Column)> { + pub fn column_by_name(&self, key: &str) -> Option<&ColumnMetadata> { let index = self.column_index_by_name(key)?; - return Some((index, &self.schema.columns[index])); + return Some(&self.column_metadata[index]); } - pub fn record_pk_column(&self) -> Option<(usize, &Column)> { + pub fn record_pk_column(&self) -> Option<&ColumnMetadata> { let index = self.record_pk_column?; - return self.schema.columns.get(index).map(|c| (index, c)); + return Some(&self.column_metadata[index]); } } @@ -165,13 +160,11 @@ impl TableMetadata { pub struct ViewMetadata { pub schema: View, - // QUESTION: Why do we have copy of the columns here? Right now it's duplicate from `.schema`. - // This probably only exists because we have a trait impl that returns Option<&[Column]>. - columns: Option>, + /// If and which column on this table qualifies as a record PK column, i.e. integer or UUIDv7. + pub record_pk_column: Option, + pub column_metadata: Option>, name_to_index: HashMap, - record_pk_column: Option, - pub json_metadata: Option, } impl ViewMetadata { @@ -187,9 +180,8 @@ impl ViewMetadata { let Some(column_mapping) = &view.column_mapping else { return Ok(ViewMetadata { name_to_index: HashMap::::default(), - columns: None, record_pk_column: None, - json_metadata: None, + column_metadata: None, schema: view, }); }; @@ -200,17 +192,31 @@ impl ViewMetadata { .map(|m| m.column.clone()) .collect(); + let column_metadata: Vec<_> = columns + .into_iter() + .enumerate() + .map(|(i, c)| { + let json_metadata = build_json_metadata(registry, &c)?; + + return Ok(ColumnMetadata { + index: i, + is_file: is_file_column(&json_metadata), + json: json_metadata, + is_geometry: is_geometry_column(&c), + column: c, + }); + }) + .collect::>()?; + let name_to_index = HashMap::::from_iter( - columns + column_metadata .iter() - .enumerate() - .map(|(index, col)| (col.name.clone(), index)), + .map(|meta| (meta.column.name.clone(), meta.index)), ); return Ok(ViewMetadata { name_to_index, - json_metadata: Some(JsonMetadata::from_columns(registry, &columns)?), - columns: Some(columns), + column_metadata: Some(column_metadata), record_pk_column: find_record_pk_column_index_for_view(column_mapping, tables), schema: view, }); @@ -222,8 +228,8 @@ impl ViewMetadata { } #[inline] - pub fn columns(&self) -> Option<&[Column]> { - return self.columns.as_deref(); + pub fn columns(&self) -> Option<&[ColumnMetadata]> { + return self.column_metadata.as_deref(); } #[inline] @@ -231,15 +237,16 @@ impl ViewMetadata { self.name_to_index.get(key).copied() } - pub fn column_by_name(&self, key: &str) -> Option<(usize, &Column)> { + pub fn column_by_name(&self, key: &str) -> Option<&ColumnMetadata> { let index = self.column_index_by_name(key)?; - let mapping = self.schema.column_mapping.as_ref()?; - return Some((index, &mapping.columns[index].column)); + let meta = self.column_metadata.as_ref()?; + return Some(&meta[index]); } - pub fn record_pk_column(&self) -> Option<(usize, &Column)> { + pub fn record_pk_column(&self) -> Option<&ColumnMetadata> { let index = self.record_pk_column?; - return self.columns.as_ref()?.get(index).map(|c| (index, c)); + let meta = self.column_metadata.as_ref()?; + return Some(&meta[index]); } } @@ -296,21 +303,21 @@ impl<'a> TableOrViewMetadata<'a> { }; } - pub fn columns(&self) -> Option<&'a [Column]> { + pub fn columns(&self) -> Option<&'a [ColumnMetadata]> { return match self { - Self::Table(t) => Some(&t.schema.columns), - Self::View(v) => v.columns(), + Self::Table(t) => Some(&t.column_metadata), + Self::View(v) => v.column_metadata.as_deref(), }; } - pub fn record_pk_column(&self) -> Option<(usize, &Column)> { + pub fn record_pk_column(&self) -> Option<&ColumnMetadata> { return match self { Self::Table(t) => t.record_pk_column(), Self::View(v) => v.record_pk_column(), }; } - pub fn column_by_name(&self, name: &str) -> Option<(usize, &Column)> { + pub fn column_by_name(&self, name: &str) -> Option<&ColumnMetadata> { return match self { Self::Table(t) => t.column_by_name(name), Self::View(v) => v.column_by_name(name), @@ -434,44 +441,81 @@ pub(crate) fn extract_json_metadata( return Ok(None); } -pub fn find_file_column_indexes(json_column_metadata: &[Option]) -> Vec { - let mut indexes: Vec = vec![]; - - for (index, column) in json_column_metadata.iter().enumerate() { - if let Some(metadata) = column { - match metadata { - JsonColumnMetadata::SchemaName(name) if name == "std.FileUpload" => { - indexes.push(index); - } - JsonColumnMetadata::SchemaName(name) if name == "std.FileUploads" => { - indexes.push(index); - } - _ => {} - }; - } +fn is_file_column(json: &Option) -> bool { + if let Some(metadata) = json + && let JsonColumnMetadata::SchemaName(name) = metadata + && (name == "std.FileUpload" || name == "std.FileUploads") + { + return true; } - - return indexes; + return false; } -pub fn find_user_id_foreign_key_columns(columns: &[Column], user_table_name: &str) -> Vec { - let mut indexes: Vec = vec![]; - for (index, col) in columns.iter().enumerate() { - for opt in &col.options { - if let ColumnOption::ForeignKey { - foreign_table, - referred_columns, - .. - } = opt - && foreign_table == user_table_name - && referred_columns.len() == 1 - && referred_columns[0] == "id" +pub fn find_file_column_indexes(column_metadata: &[ColumnMetadata]) -> Vec { + return column_metadata + .iter() + .flat_map(|meta| { + if is_file_column(&meta.json) { + return Some(meta.index); + } + return None; + }) + .collect(); +} + +fn is_geometry_column(column: &Column) -> bool { + lazy_static! { + static ref GEOMETRY_CHECK_RE: Regex = Regex::new(r"^ST_IsValid\s*\(").expect("infallible"); + } + + if column.data_type == ColumnDataType::Blob { + for opt in &column.options { + if let ColumnOption::Check(expr) = opt + && GEOMETRY_CHECK_RE.is_match(expr) { - indexes.push(index); + return true; } } } - return indexes; + return false; +} + +pub fn find_geometry_column_indexes(columns: &[Column]) -> Vec { + return columns + .iter() + .enumerate() + .flat_map(|(index, column)| { + if is_geometry_column(column) { + return Some(index); + } + return None; + }) + .collect(); +} + +pub fn find_user_id_foreign_key_columns( + column_metadata: &[ColumnMetadata], + user_table_name: &str, +) -> Vec { + return column_metadata + .iter() + .flat_map(|meta| { + for opt in &meta.column.options { + if let ColumnOption::ForeignKey { + foreign_table, + referred_columns, + .. + } = opt + && foreign_table == user_table_name + && referred_columns.len() == 1 + && referred_columns[0] == "id" + { + return Some(meta.index); + } + } + return None; + }) + .collect(); } pub(crate) fn is_pk_column(column: &Column) -> bool { @@ -694,6 +738,21 @@ mod tests { ); } + #[test] + fn test_find_geometry_columns() { + let table = parse_create_table( + "CREATE TABLE t ( + id INT PRIMARY KEY, + not_geom TEXT, + geom0 BLOB CHECK(ST_IsValid(geom0)), + geom1 BLOB CHECK(ST_IsValid(geom1)) NOT NULL + ) STRICT;", + ); + + let geometry_columns = find_geometry_column_indexes(&table.columns); + assert_eq!(vec![2, 3], geometry_columns); + } + #[test] fn test_parse_create_view() { let table = parse_create_table( @@ -754,7 +813,7 @@ mod tests { let uuidv7_col = view_metadata.record_pk_column().unwrap(); let columns = view_metadata.columns().unwrap(); assert_eq!(columns.len(), 3); - assert_eq!(columns[uuidv7_col.0].name, "id"); + assert_eq!(columns[uuidv7_col.index].column.name, "id"); } } @@ -784,9 +843,9 @@ mod tests { assert_eq!(view_columns[1].column.data_type, ColumnDataType::Text); let metadata = ViewMetadata::new(®istry, view, &tables).unwrap(); - let (pk_index, pk_col) = metadata.record_pk_column().unwrap(); - assert_eq!(pk_index, 0); - assert_eq!(pk_col.name, "id"); + let meta = metadata.record_pk_column().unwrap(); + assert_eq!(meta.index, 0); + assert_eq!(meta.column.name, "id"); } { @@ -801,9 +860,9 @@ mod tests { assert_eq!(view_columns[0].column.data_type, ColumnDataType::Integer); let metadata = ViewMetadata::new(®istry, view, &tables).unwrap(); - let (pk_index, pk_col) = metadata.record_pk_column().unwrap(); - assert_eq!(pk_index, 0); - assert_eq!(pk_col.name, "id"); + let meta = metadata.record_pk_column().unwrap(); + assert_eq!(meta.index, 0); + assert_eq!(meta.column.name, "id"); } { @@ -818,9 +877,9 @@ mod tests { assert_eq!(view_columns[0].column.data_type, ColumnDataType::Integer); let metadata = ViewMetadata::new(®istry, view, &tables).unwrap(); - let (pk_index, pk_col) = metadata.record_pk_column().unwrap(); - assert_eq!(pk_index, 0); - assert_eq!(pk_col.name, "id"); + let meta = metadata.record_pk_column().unwrap(); + assert_eq!(meta.index, 0); + assert_eq!(meta.column.name, "id"); } { @@ -869,9 +928,9 @@ mod tests { assert_eq!(view_columns[1].column.data_type, ColumnDataType::Integer); let metadata = ViewMetadata::new(®istry, view, &tables).unwrap(); - let (pk_index, pk_col) = metadata.record_pk_column().unwrap(); - assert_eq!(pk_index, 2); - assert_eq!(pk_col.name, "id"); + let meta = metadata.record_pk_column().unwrap(); + assert_eq!(meta.index, 2); + assert_eq!(meta.column.name, "id"); } { @@ -893,7 +952,7 @@ mod tests { let metadata = ViewMetadata::new(®istry, view, &tables).unwrap(); assert_eq!( expected, - metadata.record_pk_column().map(|c| c.0), + metadata.record_pk_column().map(|c| c.index), "{join_type}" ); } @@ -929,7 +988,7 @@ mod tests { assert!(view.column_mapping.is_some(), "{i}: {sql}"); let metadata = ViewMetadata::new(®istry, view, &tables).unwrap(); - assert_eq!(Some(1), metadata.record_pk_column().map(|c| c.0)); + assert_eq!(Some(1), metadata.record_pk_column().map(|c| c.index)); } } @@ -941,7 +1000,7 @@ mod tests { .unwrap(); let metadata = ViewMetadata::new(®istry, view, &tables).unwrap(); - assert_eq!(Some(1), metadata.record_pk_column().map(|c| c.0)); + assert_eq!(Some(1), metadata.record_pk_column().map(|c| c.index)); } } } @@ -984,7 +1043,7 @@ mod tests { .unwrap(); let metadata = ViewMetadata::new(®istry, view, &tables).unwrap(); - assert_eq!(Some(0), metadata.record_pk_column().map(|c| c.0)); + assert_eq!(Some(0), metadata.record_pk_column().map(|c| c.index)); } { @@ -1001,7 +1060,7 @@ mod tests { .unwrap(); let metadata = ViewMetadata::new(®istry, view, &tables).unwrap(); - assert_eq!(Some(0), metadata.record_pk_column().map(|c| c.0)); + assert_eq!(Some(0), metadata.record_pk_column().map(|c| c.index)); } } }