Control expansion of FKs in RecordApi.(read|list) via query parameters.

This commit is contained in:
Sebastian Jeltsch
2025-02-12 23:56:50 +01:00
parent fd318cf328
commit c06ceee31b
7 changed files with 399 additions and 164 deletions
+2
View File
@@ -85,6 +85,7 @@ pub struct QueryParseResult {
pub cursor: Option<[u8; 16]>,
pub offset: Option<usize>,
pub count: Option<bool>,
pub expand: Option<Vec<String>>,
// Ordering. It's a vector for &order=-col0,+col1,col2
pub order: Option<Vec<(String, Order)>>,
@@ -129,6 +130,7 @@ pub fn parse_query(query: Option<&str>) -> Result<QueryParseResult, String> {
"cursor" => result.cursor = b64_to_id(value.as_ref()).ok(),
"offset" => result.offset = value.parse::<usize>().ok(),
"count" => result.count = parse_bool(&value),
"expand" => result.expand = Some(value.split(",").map(|s| s.to_owned()).collect()),
"order" => {
let order: Vec<(String, Order)> = value
.split(",")
+50 -20
View File
@@ -311,30 +311,26 @@ impl Params {
}
}
pub(crate) struct SelectQueryBuilder;
pub(crate) struct Expansions {
/// Contains the indexes on where to cut the resulting Row.
///
/// The joins will lead to a row schema that looks something like:
/// (root_table..., foreign_table0..., foreign_table1...).
pub indexes: Vec<(usize, Arc<TableMetadata>)>,
/// The actual join statements.
pub joins: Vec<String>,
impl SelectQueryBuilder {
pub(crate) async fn run(
state: &AppState,
table_name: &str,
pk_column: &str,
pk_value: Value,
) -> Result<Option<trailbase_sqlite::Row>, trailbase_sqlite::Error> {
return state
.conn()
.query_row(
&format!(r#"SELECT * FROM "{table_name}" WHERE "{pk_column}" = $1"#),
[pk_value],
)
.await;
}
/// Select clauses in case the joins are aliased, i.e. a `prefix` is given.
pub selects: Option<Vec<String>>,
}
pub(crate) fn build_joins(
impl Expansions {
pub(crate) fn build(
table_metadata: &TableMetadataCache,
table_name: &str,
expand: &[String],
prefix: Option<&str>,
) -> Result<(Vec<(usize, Arc<TableMetadata>)>, Vec<String>), RecordError> {
) -> Result<Expansions, RecordError> {
let Some(root_table) = table_metadata.get(table_name) else {
return Err(RecordError::ApiRequiresTable);
};
@@ -376,7 +372,40 @@ impl SelectQueryBuilder {
indexes.push((foreign_table.schema.columns.len(), foreign_table));
}
return Ok((indexes, joins));
let selects = if prefix.is_none() {
None
} else {
Some(
(0..joins.len())
.map(|idx| format!("F{idx}.*"))
.collect::<Vec<_>>(),
)
};
return Ok(Expansions {
indexes,
joins,
selects,
});
}
}
pub(crate) struct SelectQueryBuilder;
impl SelectQueryBuilder {
pub(crate) async fn run(
state: &AppState,
table_name: &str,
pk_column: &str,
pk_value: Value,
) -> Result<Option<trailbase_sqlite::Row>, trailbase_sqlite::Error> {
return state
.conn()
.query_row(
&format!(r#"SELECT * FROM "{table_name}" WHERE "{pk_column}" = $1"#),
[pk_value],
)
.await;
}
pub(crate) async fn run_expanded(
@@ -387,7 +416,8 @@ impl SelectQueryBuilder {
expand: &[String],
) -> Result<Vec<(Arc<TableMetadata>, trailbase_sqlite::Row)>, RecordError> {
let table_metadata = state.table_metadata();
let (indexes, joins) = Self::build_joins(table_metadata, table_name, expand, None)?;
let Expansions { indexes, joins, .. } =
Expansions::build(table_metadata, table_name, expand, None)?;
let sql = format!(
r#"SELECT * FROM "{table_name}" AS R {} WHERE R.{pk_column} = $1"#,
+52 -19
View File
@@ -14,8 +14,10 @@ use crate::auth::user::User;
use crate::listing::{
build_filter_where_clause, limit_or_default, parse_query, Order, QueryParseResult, WhereClause,
};
use crate::records::json_to_sql::SelectQueryBuilder;
use crate::records::sql_to_json::{row_to_json, row_to_json_expand, rows_to_json, JsonError};
use crate::records::json_to_sql::Expansions;
use crate::records::sql_to_json::{
row_to_json, row_to_json_expand, rows_to_json_expand, JsonError,
};
use crate::records::{Permission, RecordError};
use crate::util::uuid_to_b64;
@@ -66,6 +68,7 @@ pub async fn list_records_handler(
limit,
order,
count,
expand: query_expand,
..
} = parse_query(raw_url_query.as_deref()).map_err(|_err| {
return RecordError::BadRequest("Invalid query");
@@ -122,14 +125,18 @@ pub async fn list_records_handler(
.join(", ");
let table_name = api.table_name();
let expand = api.expand();
let (indexes, joins, selects) = if expand.is_empty() {
(vec![], vec![], vec![])
} else {
let (indexes, joins) =
SelectQueryBuilder::build_joins(state.table_metadata(), table_name, expand, Some("_ROW_"))?;
let selects: Vec<String> = (0..joins.len()).map(|idx| format!("F{idx}.*")).collect();
(indexes, joins, selects)
let (indexes, joins, selects) = match query_expand {
Some(ref expand) if !expand.is_empty() => {
let Expansions {
indexes,
joins,
selects,
} = Expansions::build(state.table_metadata(), table_name, expand, Some("_ROW_"))?;
(Some(indexes), Some(joins), selects)
}
_ => (None, None, None),
};
let get_total_count = count.unwrap_or(false);
@@ -159,8 +166,8 @@ pub async fn list_records_handler(
{order_clause}
LIMIT :limit
"#,
selects = selects.iter().map(|s| format!("{s}, ")).join(" "),
joins = joins.join(" "),
selects = selects.map_or(EMPTY, |v| format!("{}, ", v.join(", "))),
joins = joins.map_or(EMPTY, |j| j.join(" ")),
)
} else {
formatdoc!(
@@ -177,8 +184,8 @@ pub async fn list_records_handler(
{order_clause}
LIMIT :limit
"#,
selects = selects.iter().map(|s| format!(", {s}")).join(" "),
joins = joins.join(" "),
selects = selects.map_or(EMPTY, |v| format!(", {}", v.join(", "))),
joins = joins.map_or(EMPTY, |j| j.join(" ")),
)
};
@@ -216,10 +223,8 @@ pub async fn list_records_handler(
return !col_name.starts_with("_");
}
let records = if expand.is_empty() {
rows_to_json(columns, metadata.column_metadata(), rows, filter)
.map_err(|err| RecordError::Internal(err.into()))?
} else {
let config_expand = api.expand();
let records = if let (Some(query_expand), Some(indexes)) = (query_expand, indexes) {
rows
.into_iter()
.map(|row| {
@@ -233,7 +238,7 @@ pub async fn list_records_handler(
let foreign_rows = result.split_off(1);
let foreign_values = std::iter::zip(expand, foreign_rows)
let mut foreign_values = std::iter::zip(&query_expand, foreign_rows)
.map(|(col_name, (metadata, row))| {
let value = row_to_json(
&metadata.schema.columns,
@@ -246,6 +251,12 @@ pub async fn list_records_handler(
.collect::<Result<HashMap<&str, serde_json::Value>, JsonError>>()
.map_err(|err| RecordError::Internal(err.into()))?;
for col_name in config_expand {
foreign_values
.entry(col_name)
.or_insert(serde_json::Value::Null);
}
return row_to_json_expand(
columns,
metadata.column_metadata(),
@@ -256,6 +267,26 @@ pub async fn list_records_handler(
.map_err(|err| RecordError::Internal(err.into()));
})
.collect::<Result<Vec<_>, RecordError>>()?
} else {
let expand: Option<HashMap<&str, serde_json::Value>> = if config_expand.is_empty() {
None
} else {
Some(
config_expand
.iter()
.map(|col_name| (col_name.as_str(), serde_json::Value::Null))
.collect(),
)
};
rows_to_json_expand(
columns,
metadata.column_metadata(),
rows,
filter,
expand.as_ref(),
)
.map_err(|err| RecordError::Internal(err.into()))?
};
return Ok(Json(ListResponse {
@@ -265,6 +296,8 @@ pub async fn list_records_handler(
}));
}
const EMPTY: String = String::new();
#[cfg(test)]
mod tests {
use serde::Deserialize;
+102 -51
View File
@@ -1,10 +1,10 @@
use std::collections::HashMap;
use axum::{
extract::{Path, State},
extract::{Path, Query, State},
response::Response,
Json,
};
use serde::Deserialize;
use std::collections::HashMap;
use crate::auth::user::User;
use crate::records::files::read_file_into_response;
@@ -13,6 +13,11 @@ use crate::records::sql_to_json::{row_to_json, JsonError};
use crate::records::{Permission, RecordError};
use crate::{app_state::AppState, records::sql_to_json::row_to_json_expand};
#[derive(Debug, Default, Deserialize)]
pub struct ReadRecordQuery {
pub expand: Option<Vec<String>>,
}
/// Read record.
#[utoipa::path(
get,
@@ -24,6 +29,7 @@ use crate::{app_state::AppState, records::sql_to_json::row_to_json_expand};
pub async fn read_record_handler(
State(state): State<AppState>,
Path((api_name, record)): Path<(String, String)>,
Query(query): Query<ReadRecordQuery>,
user: Option<User>,
) -> Result<Json<serde_json::Value>, RecordError> {
let Some(api) = state.lookup_record_api(&api_name) else {
@@ -44,53 +50,50 @@ pub async fn read_record_handler(
return !col_name.starts_with("_");
}
let expand = api.expand();
if expand.is_empty() {
let Some(row) = SelectQueryBuilder::run(
&state,
api.table_name(),
&api.record_pk_column().name,
record_id,
)
.await?
else {
return Err(RecordError::RecordNotFound);
};
let config_expand = api.expand();
return Ok(Json(match query.expand {
Some(query_expand) if !query_expand.is_empty() => {
for col_name in &query_expand {
if !config_expand.iter().any(|e| e == col_name) {
return Err(RecordError::BadRequest("Invalid expansion"));
}
}
return Ok(Json(
row_to_json(columns, metadata.column_metadata(), &row, filter)
.map_err(|err| RecordError::Internal(err.into()))?,
));
} else {
let mut rows = SelectQueryBuilder::run_expanded(
&state,
api.table_name(),
&api.record_pk_column().name,
record_id,
expand,
)
.await?;
let mut rows = SelectQueryBuilder::run_expanded(
&state,
api.table_name(),
&api.record_pk_column().name,
record_id,
&query_expand,
)
.await?;
if rows.is_empty() {
return Err(RecordError::RecordNotFound);
}
if rows.is_empty() {
return Err(RecordError::RecordNotFound);
}
let foreign_rows = rows.split_off(1);
let foreign_rows = rows.split_off(1);
let foreign_values = std::iter::zip(expand, foreign_rows)
.map(|(col_name, (metadata, row))| {
let value = row_to_json(
&metadata.schema.columns,
metadata.column_metadata(),
&row,
filter,
)?;
return Ok((col_name.as_str(), value));
})
.collect::<Result<HashMap<&str, serde_json::Value>, JsonError>>()
.map_err(|err| RecordError::Internal(err.into()))?;
let mut foreign_values = std::iter::zip(&query_expand, foreign_rows)
.map(|(col_name, (metadata, row))| {
let value = row_to_json(
&metadata.schema.columns,
metadata.column_metadata(),
&row,
filter,
)?;
return Ok((col_name.as_str(), value));
})
.collect::<Result<HashMap<&str, serde_json::Value>, JsonError>>()
.map_err(|err| RecordError::Internal(err.into()))?;
// Set missing foreign values to null.
for col_name in config_expand {
foreign_values
.entry(col_name)
.or_insert(serde_json::Value::Null);
}
return Ok(Json(
row_to_json_expand(
columns,
metadata.column_metadata(),
@@ -98,9 +101,41 @@ pub async fn read_record_handler(
filter,
Some(&foreign_values),
)
.map_err(|err| RecordError::Internal(err.into()))?,
));
}
.map_err(|err| RecordError::Internal(err.into()))?
}
_ => {
let Some(row) = SelectQueryBuilder::run(
&state,
api.table_name(),
&api.record_pk_column().name,
record_id,
)
.await?
else {
return Err(RecordError::RecordNotFound);
};
let expand: Option<HashMap<&str, serde_json::Value>> = if config_expand.is_empty() {
None
} else {
Some(
config_expand
.iter()
.map(|col_name| (col_name.as_str(), serde_json::Value::Null))
.collect(),
)
};
row_to_json_expand(
columns,
metadata.column_metadata(),
&row,
filter,
expand.as_ref(),
)
.map_err(|err| RecordError::Internal(err.into()))?
}
}));
}
type GetUploadedFileFromRecordPath = Path<(
@@ -319,6 +354,7 @@ mod test {
assert!(read_record_handler(
State(state.clone()),
Path(("messages_api".to_string(), id_to_b64(&message_id),)),
Query(ReadRecordQuery::default()),
None
)
.await
@@ -329,6 +365,7 @@ mod test {
let response = read_record_handler(
State(state.clone()),
Path(("messages_api".to_string(), id_to_b64(&message_id))),
Query(ReadRecordQuery::default()),
User::from_auth_token(&state, &user_x_token.auth_token),
)
.await;
@@ -340,6 +377,7 @@ mod test {
let response = read_record_handler(
State(state.clone()),
Path(("messages_api".to_string(), id_to_b64(&message_id))),
Query(ReadRecordQuery::default()),
User::from_auth_token(&state, &user_y_token.auth_token),
)
.await;
@@ -355,6 +393,7 @@ mod test {
let response = read_record_handler(
State(state.clone()),
Path(("messages_api".to_string(), id_to_b64(&message_id))),
Query(ReadRecordQuery::default()),
User::from_auth_token(&state, &user_y_token.auth_token),
)
.await;
@@ -454,8 +493,13 @@ mod test {
let record_path = Path((API_NAME.to_string(), create_response.ids[0].clone()));
let Json(value) =
read_record_handler(State(state.clone()), Path(record_path.clone()), None).await?;
let Json(value) = read_record_handler(
State(state.clone()),
Path(record_path.clone()),
Query(ReadRecordQuery::default()),
None,
)
.await?;
let serde_json::Value::Object(map) = value else {
panic!("Not a map");
@@ -547,7 +591,13 @@ mod test {
let record_path = Path((API_NAME.to_string(), resp.ids[0].clone()));
let Json(value) = read_record_handler(State(state.clone()), record_path, None).await?;
let Json(value) = read_record_handler(
State(state.clone()),
record_path,
Query(ReadRecordQuery::default()),
None,
)
.await?;
let serde_json::Value::Object(map) = value else {
panic!("Not a map");
@@ -634,6 +684,7 @@ mod test {
let response = read_record_handler(
State(state.clone()),
Path(("messages_api".to_string(), id_to_b64(&message_id))),
Query(ReadRecordQuery::default()),
User::from_auth_token(&state, &user_x_token.auth_token),
)
.await;
+39 -28
View File
@@ -1,5 +1,4 @@
use crate::schema::ColumnOption;
use itertools::Itertools;
use std::collections::HashMap;
use base64::prelude::*;
@@ -54,6 +53,13 @@ pub fn row_to_json(
return row_to_json_expand(columns, column_metadata, row, column_filter, None);
}
#[inline]
fn is_foreign_key(options: &[ColumnOption]) -> bool {
return options
.iter()
.any(|o| matches!(o, ColumnOption::ForeignKey { .. }));
}
/// Serialize SQL row to json.
pub fn row_to_json_expand(
columns: &[Column],
@@ -84,39 +90,30 @@ pub fn row_to_json_expand(
return Some(Ok((column_name.to_string(), serde_json::Value::Null)));
}
// TODO: Should IDs be pulled out only if explicitly requested? Implications for schema
// migrations.
// WARN: this probably brakes admin UI right now.
if let Some(ColumnOption::ForeignKey {
foreign_table: _,
referred_columns: _,
..
}) = column
.options
.iter()
.find_or_first(|o| matches!(o, ColumnOption::ForeignKey { .. }))
{
if let Some(ref foreign_value) = expand.as_ref().and_then(|m| m.get(column_name)) {
return match valueref_to_json(value.into()) {
Ok(value) => Some(Ok((
if let Some(foreign_value) = expand.and_then(|e| e.get(column_name)) {
if is_foreign_key(&column.options) {
let id = match valueref_to_json(value.into()) {
Ok(value) => value,
Err(err) => {
return Some(Err(err));
}
};
return Some(Ok(match foreign_value {
serde_json::Value::Null => (
column_name.to_string(),
serde_json::json!({
"id": value,
"data": foreign_value,
"id": id,
}),
))),
Err(err) => Some(Err(err)),
};
} else {
return match valueref_to_json(value.into()) {
Ok(value) => Some(Ok((
),
value => (
column_name.to_string(),
serde_json::json!({
"id": value,
"id": id,
"data": value,
}),
))),
Err(err) => Some(Err(err)),
};
),
}));
}
}
@@ -153,6 +150,20 @@ pub fn rows_to_json(
.collect::<Result<Vec<_>, JsonError>>();
}
/// Turns rows into a list of json objects.
pub fn rows_to_json_expand(
columns: &[Column],
column_metadata: &[ColumnMetadata],
rows: trailbase_sqlite::Rows,
column_filter: fn(&str) -> bool,
expand: Option<&HashMap<&str, serde_json::Value>>,
) -> Result<Vec<serde_json::Value>, JsonError> {
return rows
.iter()
.map(|row| row_to_json_expand(columns, column_metadata, row, column_filter, expand))
.collect::<Result<Vec<_>, JsonError>>();
}
pub fn row_to_json_array(row: &trailbase_sqlite::Row) -> Result<Vec<serde_json::Value>, JsonError> {
let cols = row.column_count();
let mut json_row = Vec::<serde_json::Value>::with_capacity(cols);
+145 -42
View File
@@ -937,7 +937,7 @@ pub(crate) fn build_json_schema_recursive(
#[cfg(test)]
mod tests {
use axum::extract::{Json, Path, RawQuery, State};
use axum::extract::{Json, Path, Query, RawQuery, State};
use indoc::indoc;
use serde_json::json;
use trailbase_sqlite::schema::FileUpload;
@@ -946,7 +946,7 @@ mod tests {
use crate::app_state::*;
use crate::config::proto::PermissionFlag;
use crate::records::list_records::list_records_handler;
use crate::records::read_record::read_record_handler;
use crate::records::read_record::{read_record_handler, ReadRecordQuery};
use crate::records::*;
use crate::schema::ColumnOption;
@@ -1202,6 +1202,29 @@ mod tests {
.await
.unwrap();
{
let Json(value) = read_record_handler(
State(state.clone()),
Path(("test_table_api".to_string(), "1".to_string())),
Query(ReadRecordQuery::default()),
None,
)
.await
.unwrap();
validator.validate(&value).expect(&format!("{value}"));
assert_eq!(
json!({
"id": 1,
"fk":{
"id": 1,
},
}),
value
);
}
let expected = json!({
"id": 1,
"fk":{
@@ -1212,23 +1235,28 @@ mod tests {
},
});
let Json(value) = read_record_handler(
State(state.clone()),
Path(("test_table_api".to_string(), "1".to_string())),
None,
)
.await
.unwrap();
{
let Json(value) = read_record_handler(
State(state.clone()),
Path(("test_table_api".to_string(), "1".to_string())),
Query(ReadRecordQuery {
expand: Some(vec!["fk".to_string()]),
}),
None,
)
.await
.unwrap();
validator.validate(&value).expect(&format!("{value}"));
validator.validate(&value).expect(&format!("{value}"));
assert_eq!(expected, value);
assert_eq!(expected, value);
}
{
let Json(list_response) = list_records_handler(
State(state.clone()),
Path("test_table_api".to_string()),
RawQuery(None),
RawQuery(Some("expand=fk".to_string())),
None,
)
.await
@@ -1241,7 +1269,7 @@ mod tests {
let Json(list_response) = list_records_handler(
State(state.clone()),
Path("test_table_api".to_string()),
RawQuery(Some("count=1".to_string())),
RawQuery(Some("count=1&expand=fk".to_string())),
None,
)
.await
@@ -1313,38 +1341,26 @@ mod tests {
.await
.unwrap();
let expected = json!({
"id": 1,
"fk0": {
"id": 1,
"data": {
"id": 1,
},
},
"fk0_null": serde_json::Value::Null,
"fk1": {
"id": 1,
"data": {
"id": 1,
},
},
});
let Json(value) = read_record_handler(
State(state.clone()),
Path(("test_table_api".to_string(), "1".to_string())),
None,
)
.await
.unwrap();
assert_eq!(expected, value);
exec(&format!("INSERT INTO {table_name} (id) VALUES (2);"))
// Expand none
{
let Json(value) = read_record_handler(
State(state.clone()),
Path(("test_table_api".to_string(), "1".to_string())),
Query(ReadRecordQuery { expand: None }),
None,
)
.await
.unwrap();
{
let expected = json!({
"id": 1,
"fk0": { "id": 1 },
"fk0_null": serde_json::Value::Null,
"fk1": { "id": 1 },
});
assert_eq!(expected, value);
let Json(list_response) = list_records_handler(
State(state.clone()),
Path("test_table_api".to_string()),
@@ -1354,6 +1370,93 @@ mod tests {
.await
.unwrap();
assert_eq!(vec![expected], list_response.records);
}
// Expand one
{
let expected = json!({
"id": 1,
"fk0": { "id": 1 },
"fk0_null": serde_json::Value::Null,
"fk1": {
"id": 1,
"data": {
"id": 1,
},
},
});
let Json(value) = read_record_handler(
State(state.clone()),
Path(("test_table_api".to_string(), "1".to_string())),
Query(ReadRecordQuery {
expand: Some(vec!["fk1".to_string()]),
}),
None,
)
.await
.unwrap();
assert_eq!(expected, value);
let Json(list_response) = list_records_handler(
State(state.clone()),
Path("test_table_api".to_string()),
RawQuery(Some("expand=fk1".to_string())),
None,
)
.await
.unwrap();
assert_eq!(vec![expected], list_response.records);
}
// Expand all.
{
let expected = json!({
"id": 1,
"fk0": {
"id": 1,
"data": {
"id": 1,
},
},
"fk0_null": serde_json::Value::Null,
"fk1": {
"id": 1,
"data": {
"id": 1,
},
},
});
let Json(value) = read_record_handler(
State(state.clone()),
Path(("test_table_api".to_string(), "1".to_string())),
Query(ReadRecordQuery {
expand: Some(vec!["fk0".to_string(), "fk1".to_string()]),
}),
None,
)
.await
.unwrap();
assert_eq!(expected, value);
exec(&format!("INSERT INTO {table_name} (id) VALUES (2);"))
.await
.unwrap();
let Json(list_response) = list_records_handler(
State(state.clone()),
Path("test_table_api".to_string()),
RawQuery(Some("expand=fk0,fk1".to_string())),
None,
)
.await
.unwrap();
assert_eq!(
vec![
json!({
+9 -4
View File
@@ -72,10 +72,6 @@ impl Rows {
return self.0.iter();
}
pub fn into_iter(self) -> std::vec::IntoIter<Row> {
return self.0.into_iter();
}
pub fn get(&self, idx: usize) -> Option<&Row> {
return self.0.get(idx);
}
@@ -123,6 +119,15 @@ impl Index<usize> for Rows {
}
}
impl IntoIterator for Rows {
type Item = Row;
type IntoIter = std::vec::IntoIter<Self::Item>;
fn into_iter(self) -> Self::IntoIter {
return self.0.into_iter();
}
}
#[derive(Debug)]
pub struct Row(Vec<types::Value>, Arc<Vec<Column>>);