mirror/trailbase
1
0
Fork 0
mirror of https://github.com/trailbaseio/trailbase.git synced 2026-02-12 20:49:21 -06:00
Code Issues Packages Projects Releases Wiki Activity

Allow expanding the schema of a foreign table into the local one. There are only stubs for configuring the expansion and the row_to_json output builder for the record read/list handlers .

Browse Source
This commit is contained in:
Sebastian Jeltsch
2025-02-10 00:10:44 +01:00
parent 1ac32acbee
commit 2b1d87a7ab
10 changed files with 349 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
trailbase-core/proto/config.proto
View File

@@ -138,6 +138,9 @@ message RecordApiConfig {
optional string update_access_rule = 13;
optional string delete_access_rule = 14;
optional string schema_access_rule = 15;
// A list of foreign key columns that can be expanded on read/list.
// repeated string expand = 21;
}
message JsonSchemaConfig {

33
trailbase-core/src/admin/rows/list_rows.rs
View File

@@ -12,7 +12,7 @@ use crate::listing::{
};
use crate::records::sql_to_json::rows_to_json_arrays;
use crate::schema::Column;
use crate::table_metadata::TableOrViewMetadata;
use crate::table_metadata::{TableMetadata, TableOrViewMetadata};
#[derive(Debug, Serialize, TS)]
#[ts(export)]
@@ -42,11 +42,14 @@ pub async fn list_rows_handler(
} = parse_query(raw_url_query.as_deref())
.map_err(|err| Error::Precondition(format!("Invalid query '{err}': {raw_url_query:?}")))?;
let (virtual_table, table_or_view_metadata): (bool, Arc<dyn TableOrViewMetadata + Sync + Send>) = {
let (table_metadata, table_or_view_metadata): (
Option<Arc<TableMetadata>>,
Arc<dyn TableOrViewMetadata + Sync + Send>,
) = {
if let Some(table_metadata) = state.table_metadata().get(&table_name) {
(table_metadata.schema.virtual_table, table_metadata)
(Some(table_metadata.clone()), table_metadata)
} else if let Some(view_metadata) = state.table_metadata().get_view(&table_name) {
(false, view_metadata)
(None, view_metadata)
} else {
return Err(Error::Precondition(format!(
"Table or view '{table_name}' not found"
@@ -103,12 +106,24 @@ pub async fn list_rows_handler(
cursor: next_cursor,
// NOTE: in the view case we don't have a good way of extracting the columns from the "CREATE
// VIEW" query so we fall back to columns constructed from the returned data.
columns: match virtual_table {
true => columns.unwrap_or_else(Vec::new),
false => table_or_view_metadata.columns().unwrap_or_else(|| {
debug!("Falling back to inferred cols for view: '{table_name}'");
columns: match table_metadata {
Some(ref metadata) if metadata.schema.virtual_table => {
// Virtual TABLE case.
columns.unwrap_or_else(Vec::new)
}),
}
Some(ref metadata) => {
// Non-virtual TABLE case.
metadata.schema.columns.clone()
}
_ => {
// VIEW-case
if let Some(columns) = table_or_view_metadata.columns() {
columns.clone()
} else {
debug!("Falling back to inferred cols for view: '{table_name}'");
columns.unwrap_or_else(Vec::new)
}
}
},
rows,
}));

1
trailbase-core/src/config.rs
View File

@@ -170,6 +170,7 @@ pub mod proto {
update_access_rule: Some("_ROW_.user = _USER_.id".to_string()),
delete_access_rule: Some("_ROW_.user = _USER_.id".to_string()),
schema_access_rule: None,
// expand: vec![],
}];
return config;

24
trailbase-core/src/records/json_schema.rs
View File

@@ -3,7 +3,7 @@ use serde::Deserialize;
use crate::auth::user::User;
use crate::records::{Permission, RecordError};
use crate::table_metadata::build_json_schema;
use crate::table_metadata::{build_json_schema, build_json_schema_recursive, Expand};
use crate::{api::JsonSchemaMode, app_state::AppState};
#[derive(Debug, Clone, Deserialize)]
@@ -34,7 +34,23 @@ pub async fn json_schema_handler(
.await?;
let mode = request.mode.unwrap_or(JsonSchemaMode::Insert);
let (_schema, json) = build_json_schema(api.table_name(), api.metadata(), mode)
.map_err(|err| RecordError::Internal(err.into()))?;
return Ok(Json(json));
let expand = api.expand();
match (expand.len(), mode) {
(n, JsonSchemaMode::Select) if n > 0 => {
let expand = Expand {
table_metadata: state.table_metadata(),
foreign_key_columns: expand,
};
let (_schema, json) =
build_json_schema_recursive(api.table_name(), api.metadata(), mode, Some(expand))
.map_err(|err| RecordError::Internal(err.into()))?;
return Ok(Json(json));
}
_ => {
let (_schema, json) = build_json_schema(api.table_name(), api.metadata(), mode)
.map_err(|err| RecordError::Internal(err.into()))?;
return Ok(Json(json));
}
}
}

11
trailbase-core/src/records/list_records.rs
View File

@@ -51,6 +51,9 @@ pub async fn list_records_handler(
api.check_table_level_access(Permission::Read, user.as_ref())?;
let metadata = api.metadata();
let Some(columns) = metadata.columns() else {
return Err(RecordError::Internal("missing columns".into()));
};
let Some((pk_index, _pk_column)) = metadata.record_pk_column() else {
return Err(RecordError::Internal("missing pk column".into()));
};
@@ -191,9 +194,11 @@ pub async fn list_records_handler(
None
};
let records = rows_to_json(metadata, rows, |col_name| !col_name.starts_with("_"))
.await
.map_err(|err| RecordError::Internal(err.into()))?;
let records = rows_to_json(columns, metadata.column_metadata(), rows, |col_name| {
!col_name.starts_with("_")
})
.await
.map_err(|err| RecordError::Internal(err.into()))?;
return Ok(Json(ListResponse {
cursor,

1
trailbase-core/src/records/mod.rs
View File

@@ -135,6 +135,7 @@ pub async fn add_record_api(
update_access_rule: access_rules.update,
delete_access_rule: access_rules.delete,
schema_access_rule: access_rules.schema,
// expand: vec![],
});
return state.validate_and_update_config(config, None).await;

10
trailbase-core/src/records/read_record.rs
View File

@@ -27,6 +27,10 @@ pub async fn read_record_handler(
let Some(api) = state.lookup_record_api(&api_name) else {
return Err(RecordError::ApiNotFound);
};
let metadata = api.metadata();
let Some(columns) = metadata.columns() else {
return Err(RecordError::ApiNotFound);
};
let record_id = api.id_to_sql(&record)?;
@@ -46,8 +50,10 @@ pub async fn read_record_handler(
};
return Ok(Json(
row_to_json(api.metadata(), &row, |col_name| !col_name.starts_with("_"))
.map_err(|err| RecordError::Internal(err.into()))?,
row_to_json(columns, metadata.column_metadata(), &row, |col_name| {
!col_name.starts_with("_")
})
.map_err(|err| RecordError::Internal(err.into()))?,
));
}

9
trailbase-core/src/records/record_api.rs
View File

@@ -61,6 +61,8 @@ struct RecordApiState {
insert_conflict_resolution_strategy: Option<ConflictResolutionStrategy>,
insert_autofill_missing_user_id_columns: bool,
expand: Vec<String>,
create_access_rule: Option<String>,
create_access_query: Option<String>,
@@ -198,6 +200,8 @@ impl RecordApi {
.autofill_missing_user_id_columns
.unwrap_or(false),
expand: vec![], // config.expand,
// Access control lists.
acl: [
convert_acl(&config.acl_world),
@@ -239,6 +243,11 @@ impl RecordApi {
return self.state.metadata.metadata();
}
#[inline]
pub fn expand(&self) -> &[String] {
return &self.state.expand;
}
pub fn table_metadata(&self) -> Option<&TableMetadata> {
match &self.state.metadata {
RecordApiMetadata::Table(ref table) => Some(table),

103
trailbase-core/src/records/sql_to_json.rs
View File

@@ -3,7 +3,7 @@ use log::*;
use thiserror::Error;
use crate::schema::{Column, ColumnDataType};
use crate::table_metadata::TableOrViewMetadata;
use crate::table_metadata::ColumnMetadata;
#[derive(Debug, Error)]
pub enum JsonError {
@@ -17,6 +17,8 @@ pub enum JsonError {
Finite,
#[error("Value not found")]
ValueNotFound,
#[error("Missing col name")]
MissingColumnName,
}
pub(crate) fn valueref_to_json(
@@ -40,49 +42,85 @@ pub(crate) fn valueref_to_json(
/// Serialize SQL row to json.
pub fn row_to_json(
metadata: &(dyn TableOrViewMetadata + Send + Sync),
columns: &[Column],
column_metadata: &[ColumnMetadata],
row: &trailbase_sqlite::Row,
column_filter: fn(&str) -> bool,
) -> Result<serde_json::Value, JsonError> {
let mut map = serde_json::Map::<String, serde_json::Value>::default();
for i in 0..(row.column_count()) {
let Some(col_name) = row.column_name(i) else {
error!("Missing column name for {i} in {row:?}");
continue;
};
if !column_filter(col_name) {
continue;
}
let value = row.get_value(i).ok_or(JsonError::ValueNotFound)?;
if let rusqlite::types::Value::Text(str) = &value {
if let Some((_col, col_meta)) = metadata.column_by_name(col_name) {
if col_meta.json.is_some() {
map.insert(col_name.to_string(), serde_json::from_str(str)?);
continue;
}
} else {
warn!("Missing col: {col_name}");
let map = (0..row.column_count())
.filter_map(|i| {
let Some(column_name) = row.column_name(i) else {
return Some(Err(JsonError::MissingColumnName));
};
if !column_filter(column_name) {
return None;
}
}
map.insert(col_name.to_string(), valueref_to_json(value.into())?);
}
assert!(i < columns.len());
assert!(i < column_metadata.len());
let column = &columns[i];
assert_eq!(column_name, column.name);
let Some(value) = row.get_value(i) else {
return Some(Err(JsonError::ValueNotFound));
};
// TODO: Follow references for extended columns.
// TODO: Should this only expand if mentioned in the config or should we always pull out IDs.
// use crate::schema::ColumnOption;
// use itertools::Itertools;
// if let Some(ColumnOption::ForeignKey {
// foreign_table: _,
// referred_columns: _,
// ..
// }) = column
// .options
// .iter()
// .find_or_first(|o| matches!(o, ColumnOption::ForeignKey { .. }))
// {
// return match valueref_to_json(value.into()) {
// Ok(value) => Some(Ok((
// column_name.to_string(),
// serde_json::json!({
// "id": value,
// // "data": serde_json::Value::Null,
// }),
// ))),
// Err(err) => Some(Err(err)),
// };
// }
if let rusqlite::types::Value::Text(str) = &value {
let metadata = &column_metadata[i];
if metadata.json.is_some() {
return match serde_json::from_str(str) {
Ok(json) => Some(Ok((column_name.to_string(), json))),
Err(err) => Some(Err(err.into())),
};
}
}
return match valueref_to_json(value.into()) {
Ok(value) => Some(Ok((column_name.to_string(), value))),
Err(err) => Some(Err(err)),
};
})
.collect::<Result<serde_json::Map<_, _>, JsonError>>()?;
return Ok(serde_json::Value::Object(map));
}
/// Turns rows into a list of json objects.
pub async fn rows_to_json(
metadata: &(dyn TableOrViewMetadata + Send + Sync),
columns: &[Column],
column_metadata: &[ColumnMetadata],
rows: trailbase_sqlite::Rows,
column_filter: fn(&str) -> bool,
) -> Result<Vec<serde_json::Value>, JsonError> {
let mut objects: Vec<serde_json::Value> = vec![];
for row in rows.iter() {
objects.push(row_to_json(metadata, row, column_filter)?);
objects.push(row_to_json(columns, column_metadata, row, column_filter)?);
}
return Ok(objects);
@@ -159,7 +197,7 @@ mod tests {
use super::*;
use crate::app_state::*;
use crate::table_metadata::{lookup_and_parse_table_schema, TableMetadata};
use crate::table_metadata::{lookup_and_parse_table_schema, TableMetadata, TableOrViewMetadata};
#[tokio::test]
async fn test_read_rows() {
@@ -220,7 +258,14 @@ mod tests {
insert(object.clone()).await.unwrap();
let rows = conn.query("SELECT * FROM test_table", ()).await.unwrap();
let parsed = rows_to_json(&metadata, rows, |_| true).await.unwrap();
let parsed = rows_to_json(
metadata.columns().unwrap(),
metadata.column_metadata(),
rows,
|_| true,
)
.await
.unwrap();
assert_eq!(parsed.len(), 1);
let serde_json::Value::Object(map) = parsed.first().unwrap() else {

227
trailbase-core/src/table_metadata.rs
View File

@@ -247,7 +247,8 @@ pub trait TableOrViewMetadata {
fn column_by_name(&self, key: &str) -> Option<(&Column, &ColumnMetadata)>;
// Impl detail: only used by admin and list
fn columns(&self) -> Option<Vec<Column>>;
fn columns(&self) -> Option<&Vec<Column>>;
fn column_metadata(&self) -> &Vec<ColumnMetadata>;
fn record_pk_column(&self) -> Option<(usize, &Column)>;
}
@@ -256,8 +257,12 @@ impl TableOrViewMetadata for TableMetadata {
self.column_by_name(key)
}
fn columns(&self) -> Option<Vec<Column>> {
Some(self.schema.columns.clone())
fn columns(&self) -> Option<&Vec<Column>> {
Some(&self.schema.columns)
}
fn column_metadata(&self) -> &Vec<ColumnMetadata> {
return &self.metadata;
}
fn record_pk_column(&self) -> Option<(usize, &Column)> {
@@ -271,8 +276,12 @@ impl TableOrViewMetadata for ViewMetadata {
self.column_by_name(key)
}
fn columns(&self) -> Option<Vec<Column>> {
return self.schema.columns.clone();
fn columns(&self) -> Option<&Vec<Column>> {
return self.schema.columns.as_ref();
}
fn column_metadata(&self) -> &Vec<ColumnMetadata> {
return &self.metadata;
}
fn record_pk_column(&self) -> Option<(usize, &Column)> {
@@ -750,7 +759,12 @@ pub fn build_json_schema(
metadata: &(dyn TableOrViewMetadata + Send + Sync),
mode: JsonSchemaMode,
) -> Result<(Validator, serde_json::Value), JsonSchemaError> {
return build_json_schema_recursive(table_or_view_name, metadata, mode, &[]);
return build_json_schema_recursive(table_or_view_name, metadata, mode, None);
}
pub(crate) struct Expand<'a> {
pub(crate) table_metadata: &'a TableMetadataCache,
pub(crate) foreign_key_columns: &'a [String],
}
/// NOTE: Foreign keys can only reference tables not view, so the inline schemas don't need to be
@@ -759,7 +773,7 @@ pub(crate) fn build_json_schema_recursive(
table_or_view_name: &str,
metadata: &(dyn TableOrViewMetadata + Send + Sync),
mode: JsonSchemaMode,
inline: &[&TableMetadata],
expand: Option<Expand<'_>>,
) -> Result<(Validator, serde_json::Value), JsonSchemaError> {
let mut properties = serde_json::Map::new();
let mut defs = serde_json::Map::new();
@@ -789,12 +803,10 @@ pub(crate) fn build_json_schema_recursive(
};
defs.insert(col.name.clone(), schema.schema);
found_def = true;
break;
}
JsonColumnMetadata::Pattern(pattern) => {
defs.insert(col.name.clone(), pattern.clone());
found_def = true;
break;
}
}
}
@@ -819,9 +831,40 @@ pub(crate) fn build_json_schema_recursive(
referred_columns: _,
..
} => {
for metadata in inline {
if metadata.name() == foreign_table {
if let (Some(expand), JsonSchemaMode::Select) = (&expand, mode) {
for metadata in expand.foreign_key_columns {
if metadata != foreign_table {
continue;
}
// TODO: Implement nesting.
let Some(table) = expand.table_metadata.get(foreign_table) else {
warn!("Failed to find table: {foreign_table}");
continue;
};
let Some((_idx, pk_column)) = table.record_pk_column() else {
warn!("Missing pk column for table: {foreign_table}");
continue;
};
let (_validator, schema) = build_json_schema(foreign_table, &*table, mode)?;
defs.insert(
col.name.clone(),
serde_json::json!({
"type": "object",
"properties": {
"id": serde_json::json!({
"type": column_data_type_to_json_type(pk_column.data_type),
}),
// "id": { "type" : "string" },
"data": schema,
},
"required": ["id"],
}),
);
found_def = true;
}
}
}
@@ -851,24 +894,32 @@ pub(crate) fn build_json_schema_recursive(
"$ref": format!("#/$defs/{name}")
}),
);
continue;
} else {
properties.insert(
col.name.clone(),
serde_json::json!({
"type": column_data_type_to_json_type(col.data_type),
}),
);
}
properties.insert(
col.name.clone(),
serde_json::json!({
"type": column_data_type_to_json_type(col.data_type),
}),
);
}
let schema = serde_json::json!({
"title": table_or_view_name,
"type": "object",
"properties": serde_json::Value::Object(properties),
"required": serde_json::Value::Array(required_cols.into_iter().map(serde_json::Value::String).collect()),
"$defs":serde_json::Value::Object(defs),
});
let schema = if defs.is_empty() {
serde_json::json!({
"title": table_or_view_name,
"type": "object",
"properties": serde_json::Value::Object(properties),
"required": serde_json::json!(required_cols),
})
} else {
serde_json::json!({
"title": table_or_view_name,
"type": "object",
"properties": serde_json::Value::Object(properties),
"required": serde_json::json!(required_cols),
"$defs": serde_json::Value::Object(defs),
})
};
return Ok((
Validator::new(&schema).map_err(|err| JsonSchemaError::SchemaCompile(err.to_string()))?,
@@ -1039,6 +1090,130 @@ mod tests {
})));
}
#[tokio::test]
async fn test_expanded_foreign_key() {
let state = test_state(None).await.unwrap();
let conn = state.conn();
conn
.execute(
"CREATE TABLE foreign_table (id INTEGER PRIMARY KEY) STRICT",
(),
)
.await
.unwrap();
let table_name = "test_table";
conn
.execute(
&format!(
r#"CREATE TABLE {table_name} (
id INTEGER PRIMARY KEY,
fk INTEGER REFERENCES foreign_table(id)
) STRICT"#
),
(),
)
.await
.unwrap();
state.table_metadata().invalidate_all().await.unwrap();
use crate::config::proto::PermissionFlag;
use crate::records::read_record::read_record_handler;
use crate::records::*;
add_record_api(
&state,
"test_table_api",
table_name,
Acls {
world: vec![PermissionFlag::Create, PermissionFlag::Read],
..Default::default()
},
AccessRules::default(),
)
.await
.unwrap();
let test_table_metadata = state.table_metadata().get(table_name).unwrap();
let (_validator, schema) = build_json_schema_recursive(
table_name,
&*test_table_metadata,
JsonSchemaMode::Select,
Some(Expand {
table_metadata: state.table_metadata(),
foreign_key_columns: &["foreign_table".to_string()],
}),
)
.unwrap();
assert_eq!(
schema,
json!({
"title": table_name,
"type": "object",
"properties": {
"id": { "type": "integer" },
"fk": { "$ref": "#/$defs/fk" },
},
"required": ["id"],
"$defs": {
"fk": {
"type": "object",
"properties": {
"id" : { "type": "integer"},
"data": {
"title": "foreign_table",
"type": "object",
"properties": {
"id" : { "type": "integer" },
},
"required": ["id"],
},
},
"required": ["id"],
},
},
})
);
conn
.execute("INSERT INTO foreign_table (id) VALUES (1);", ())
.await
.unwrap();
conn
.execute(
&format!("INSERT INTO {table_name} (id, fk) VALUES (1, 1);"),
(),
)
.await
.unwrap();
use axum::extract::{Json, Path, State};
let Json(value) = read_record_handler(
State(state.clone()),
Path(("test_table_api".to_string(), "1".to_string())),
None,
)
.await
.unwrap();
{
let (validator, _schema) = build_json_schema_recursive(
table_name,
&*test_table_metadata,
JsonSchemaMode::Select,
None,
)
.unwrap();
validator.validate(&value).expect(&format!("{value}"));
}
}
#[test]
fn test_parse_alter_table() {
let sql = "ALTER TABLE foo RENAME TO bar";