Add an experimental /api/transaction/v1/execute endpoint for mutating (create, update and delete) records across multiple APIs in a single transaction. #124

The endpoint is disabled by default and needs to be enabled in the configuration. Indepdently, the various client libraries do not yet support this new API.

crates/core/proto/config.proto

@@ -113,6 +113,9 @@ message ServerConfig {
 
   /// If present will use S3 setup over local file-system based storage.
   optional S3StorageConfig s3_storage_config = 13;
+
+  /// If enabled, batches of transactions can be submitted for attomic execution
+  optional bool enable_record_transactions = 14;
 }
 
 enum SystemJobId {

crates/core/src/constants.rs

@@ -30,6 +30,7 @@ pub(crate) const REFRESH_TOKEN_LENGTH: usize = 32;
 
 // Public APIs
 pub const RECORD_API_PATH: &str = "api/records/v1";
+pub const TRANSACTION_API_PATH: &str = "api/transaction/v1";
 pub const QUERY_API_PATH: &str = "api/query/v1";
 pub const AUTH_API_PATH: &str = "api/auth/v1";
 pub const ADMIN_API_PATH: &str = "api/_admin";

crates/core/src/records/create_record.rs

@@ -24,7 +24,7 @@ pub struct CreateRecordQuery {
 
 #[derive(Clone, Debug, Deserialize, Serialize, ToSchema)]
 pub struct CreateRecordResponse {
-  /// Safe-url base64 encoded id of the newly created record.
+  /// Url-Safe base64 encoded ids of the newly created record.
   pub ids: Vec<String>,
 }
 
@@ -41,9 +41,7 @@ pub(crate) fn extract_record(value: serde_json::Value) -> Result<JsonRow, Record
 pub(crate) type RecordAndFiles = (JsonRow, Option<Vec<FileUploadInput>>);
 
 #[inline]
-pub(crate) fn extract_records(
-  value: serde_json::Value,
-) -> Result<Vec<RecordAndFiles>, RecordError> {
+fn extract_records(value: serde_json::Value) -> Result<Vec<RecordAndFiles>, RecordError> {
   return match value {
     serde_json::Value::Object(record) => Ok(vec![(record, None)]),
     serde_json::Value::Array(records) => {
@@ -70,18 +68,6 @@ pub(crate) fn extract_records(
   };
 }
 
-#[inline]
-fn extract_record_id(value: rusqlite::types::Value) -> Result<String, trailbase_sqlite::Error> {
-  return match value {
-    rusqlite::types::Value::Blob(blob) => Ok(BASE64_URL_SAFE.encode(blob)),
-    rusqlite::types::Value::Text(text) => Ok(text),
-    rusqlite::types::Value::Integer(i) => Ok(i.to_string()),
-    _ => Err(trailbase_sqlite::Error::Other(
-      "Unexpected data type".into(),
-    )),
-  };
-}
-
 /// Create new record.
 #[utoipa::path(
   post,
@@ -90,7 +76,7 @@ fn extract_record_id(value: rusqlite::types::Value) -> Result<String, trailbase_
   params(CreateRecordQuery),
   request_body = serde_json::Value,
   responses(
-    (status = 200, description = "Record id of successful insertion.", body = CreateRecordResponse),
+    (status = 200, description = "Ids of successfully created records.", body = CreateRecordResponse),
   )
 )]
 pub async fn create_record_handler(
@@ -200,6 +186,18 @@ pub async fn create_record_handler(
   return Ok(Json(CreateRecordResponse { ids: record_ids }).into_response());
 }
 
+#[inline]
+fn extract_record_id(value: rusqlite::types::Value) -> Result<String, trailbase_sqlite::Error> {
+  return match value {
+    rusqlite::types::Value::Blob(blob) => Ok(BASE64_URL_SAFE.encode(blob)),
+    rusqlite::types::Value::Text(text) => Ok(text),
+    rusqlite::types::Value::Integer(i) => Ok(i.to_string()),
+    _ => Err(trailbase_sqlite::Error::Other(
+      "Unexpected data type".into(),
+    )),
+  };
+}
+
 #[cfg(test)]
 mod test {
   use super::*;

crates/core/src/records/mod.rs

@@ -6,20 +6,22 @@ use utoipa::OpenApi;
 
 pub(crate) mod create_record;
 pub(crate) mod delete_record;
-mod error;
-mod expand;
 pub(crate) mod files;
 pub(crate) mod json_schema;
 pub(crate) mod list_records;
 pub(crate) mod params;
 pub(crate) mod read_queries;
 pub(crate) mod read_record;
-mod record_api;
 pub(crate) mod subscribe;
-pub mod test_utils;
+pub(crate) mod test_utils;
+pub(crate) mod write_queries;
+
+mod error;
+mod expand;
+mod record_api;
+mod transaction;
 mod update_record;
 mod validate;
-pub mod write_queries;
 
 pub(crate) use error::RecordError;
 pub use record_api::RecordApi;
@@ -27,7 +29,7 @@ pub(crate) use validate::validate_record_api_config;
 
 use crate::AppState;
 use crate::config::proto::PermissionFlag;
-use crate::constants::RECORD_API_PATH;
+use crate::constants::{RECORD_API_PATH, TRANSACTION_API_PATH};
 
 #[allow(unused)]
 #[derive(OpenApi)]
@@ -44,8 +46,8 @@ use crate::constants::RECORD_API_PATH;
 ))]
 pub(super) struct RecordOpenApi;
 
-pub(crate) fn router() -> Router<AppState> {
-  return Router::new()
+pub(crate) fn router(enable_transactions: bool) -> Router<AppState> {
+  let router = Router::new()
     .route(
       &format!("/{RECORD_API_PATH}/{{name}}/{{record}}"),
       get(read_record::read_record_handler),
@@ -82,6 +84,15 @@ pub(crate) fn router() -> Router<AppState> {
       &format!("/{RECORD_API_PATH}/{{name}}/subscribe/{{record}}"),
       get(subscribe::add_subscription_sse_handler),
     );
+
+  if enable_transactions {
+    return router.route(
+      &format!("/{TRANSACTION_API_PATH}/execute"),
+      post(transaction::record_transactions_handler),
+    );
+  }
+
+  return router;
 }
 
 // Since this is for APIs access control, we'll use the API- space CRUD terminology instead of

crates/core/src/records/record_api.rs

@@ -41,6 +41,8 @@ struct RecordApiSchema {
   named_params_template: NamedParams,
 }
 
+type DeferredAclCheck = dyn (FnOnce(&rusqlite::Connection) -> Result<(), RecordError>) + Send;
+
 impl RecordApiSchema {
   fn from_table(schema_metadata: &TableMetadata, config: &RecordApiConfig) -> Result<Self, String> {
     assert_eq!(
@@ -444,26 +446,33 @@ impl RecordApi {
         self
           .state
           .conn
-          .read_query_row_f(access_query, params, |row| row.get(0))
+          .call_reader(move |conn| {
+            Ok(Self::check_record_level_access_impl(
+              conn,
+              &access_query,
+              params,
+            )?)
+          })
           .await
       }
       _ => {
         self
           .state
           .conn
-          .query_row_f(access_query, params, |row| row.get(0))
+          .call(move |conn| {
+            Ok(Self::check_record_level_access_impl(
+              conn,
+              &access_query,
+              params,
+            )?)
+          })
           .await
       }
     };
-    // let allowed_result = self
-    //   .state
-    //   .conn
-    //   .read_query_row_f(access_query, params, |row| row.get(0))
-    //   .await;
 
     match allowed_result {
       Ok(allowed) => {
-        if allowed.unwrap_or(false) {
+        if allowed {
           return Ok(());
         }
       }
@@ -478,6 +487,43 @@ impl RecordApi {
     return Err(RecordError::Forbidden);
   }
 
+  pub fn build_record_level_access_check(
+    &self,
+    p: Permission,
+    record_id: Option<&Value>,
+    request_params: Option<&mut LazyParams<'_>>,
+    user: Option<&User>,
+  ) -> Result<Box<DeferredAclCheck>, RecordError> {
+    // First check table level access and if present check row-level access based on access rule.
+    self.check_table_level_access(p, user)?;
+
+    let Some(access_query) = self.state.cached_access_query(p) else {
+      return Ok(Box::new(|_conn| Ok(())));
+    };
+
+    let params = self.build_named_params(p, record_id, request_params, user)?;
+    return Ok(Box::new(move |conn| {
+      return match Self::check_record_level_access_impl(conn, &access_query, params) {
+        Ok(allowed) if allowed => Ok(()),
+        _ => Err(RecordError::Forbidden),
+      };
+    }));
+  }
+
+  #[inline]
+  fn check_record_level_access_impl(
+    conn: &rusqlite::Connection,
+    query: &str,
+    named_params: NamedParams,
+  ) -> Result<bool, rusqlite::Error> {
+    let mut stmt = conn.prepare_cached(query)?;
+    named_params.bind(&mut stmt)?;
+    if let Some(row) = stmt.raw_query().next()? {
+      return row.get(0);
+    }
+    return Err(rusqlite::Error::QueryReturnedNoRows);
+  }
+
   /// Check if the given user (if any) can access a record given the request and the operation.
   ///
   /// NOTE: We could inline this in `SubscriptionManager::broker_subscriptions` and reduce some

crates/core/src/records/transaction.rs

@@ -0,0 +1,338 @@
+use axum::extract::{Json, State};
+use base64::prelude::*;
+use serde::{Deserialize, Serialize};
+use utoipa::ToSchema;
+
+use crate::app_state::AppState;
+use crate::auth::user::User;
+use crate::records::params::LazyParams;
+use crate::records::record_api::RecordApi;
+use crate::records::write_queries::WriteQuery;
+use crate::records::{Permission, RecordError};
+use crate::util::uuid_to_b64;
+
+#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)]
+pub enum Operation {
+  Create {
+    api_name: String,
+    value: serde_json::Value,
+  },
+  Update {
+    api_name: String,
+    record_id: String,
+    value: serde_json::Value,
+  },
+  Delete {
+    api_name: String,
+    record_id: String,
+  },
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)]
+pub struct TransactionRequest {
+  operations: Vec<Operation>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)]
+pub struct TransactionResponse {
+  /// Url-Safe base64 encoded ids of the newly created record.
+  pub ids: Vec<String>,
+}
+
+/// Execute a batch of transactions.
+#[utoipa::path(
+  post,
+  path = "/api/transaction/v1/execute",
+  tag = "transactions",
+  params(),
+  request_body = TransactionRequest,
+  responses(
+    (status = 200, description = "Ids of successfully created records.", body = TransactionResponse),
+  )
+)]
+pub async fn record_transactions_handler(
+  State(state): State<AppState>,
+  user: Option<User>,
+  Json(request): Json<TransactionRequest>,
+) -> Result<Json<TransactionResponse>, RecordError> {
+  if request.operations.len() > 128 {
+    return Err(RecordError::BadRequest("Transactions exceed limit: 128"));
+  }
+
+  type Op = dyn (FnOnce(&rusqlite::Connection) -> Result<Option<String>, RecordError>) + Send;
+
+  let operations: Vec<Box<Op>> = request
+    .operations
+    .into_iter()
+    .map(|op| -> Result<Box<Op>, RecordError> {
+      return match op {
+        Operation::Create { api_name, value } => {
+          let api = get_api(&state, &api_name)?;
+          let mut record = extract_record(value)?;
+
+          if api.insert_autofill_missing_user_id_columns() {
+            if let Some(ref user) = user {
+              for column_index in api.user_id_columns() {
+                let col_name = &api.columns()[*column_index].name;
+                if !record.contains_key(col_name) {
+                  record.insert(
+                    col_name.to_owned(),
+                    serde_json::Value::String(uuid_to_b64(&user.uuid)),
+                  );
+                }
+              }
+            }
+          }
+
+          let mut lazy_params = LazyParams::for_insert(&api, record, None);
+          let acl_check = api.build_record_level_access_check(
+            Permission::Create,
+            None,
+            Some(&mut lazy_params),
+            user.as_ref(),
+          )?;
+
+          let (query, _files) = WriteQuery::new_insert(
+            api.table_name(),
+            &api.record_pk_column().1.name,
+            api.insert_conflict_resolution_strategy(),
+            lazy_params
+              .consume()
+              .map_err(|_| RecordError::BadRequest("Invalid Parameters"))?,
+          )
+          .map_err(|err| RecordError::Internal(err.into()))?;
+
+          Ok(Box::new(move |conn| {
+            acl_check(conn)?;
+            let result = query
+              .apply(conn)
+              .map_err(|err| RecordError::Internal(err.into()))?;
+
+            return Ok(Some(
+              extract_record_id(result.pk_value.expect("insert"))
+                .map_err(|err| RecordError::Internal(err.into()))?,
+            ));
+          }))
+        }
+        Operation::Update {
+          api_name,
+          record_id,
+          value,
+        } => {
+          let api = get_api(&state, &api_name)?;
+          let record = extract_record(value)?;
+          let record_id = api.primary_key_to_value(record_id)?;
+
+          let mut lazy_params = LazyParams::for_insert(&api, record, None);
+
+          let acl_check = api.build_record_level_access_check(
+            Permission::Update,
+            Some(&record_id),
+            Some(&mut lazy_params),
+            user.as_ref(),
+          )?;
+
+          let (query, _files) = WriteQuery::new_update(
+            api.table_name(),
+            lazy_params
+              .consume()
+              .map_err(|_| RecordError::BadRequest("Invalid Parameters"))?,
+          )
+          .map_err(|err| RecordError::Internal(err.into()))?;
+
+          Ok(Box::new(move |conn| {
+            acl_check(conn)?;
+            let _ = query
+              .apply(conn)
+              .map_err(|err| RecordError::Internal(err.into()))?;
+
+            return Ok(None);
+          }))
+        }
+        Operation::Delete {
+          api_name,
+          record_id,
+        } => {
+          let api = get_api(&state, &api_name)?;
+          let record_id = api.primary_key_to_value(record_id)?;
+
+          let acl_check = api.build_record_level_access_check(
+            Permission::Delete,
+            Some(&record_id),
+            None,
+            user.as_ref(),
+          )?;
+
+          let query =
+            WriteQuery::new_delete(api.table_name(), &api.record_pk_column().1.name, record_id)
+              .map_err(|err| RecordError::Internal(err.into()))?;
+
+          Ok(Box::new(move |conn| {
+            acl_check(conn)?;
+            let _ = query
+              .apply(conn)
+              .map_err(|err| RecordError::Internal(err.into()))?;
+
+            return Ok(None);
+          }))
+        }
+      };
+    })
+    .collect::<Result<Vec<_>, _>>()?;
+
+  let ids = state
+    .conn()
+    .call(
+      move |conn: &mut rusqlite::Connection| -> Result<Vec<String>, trailbase_sqlite::Error> {
+        let tx = conn.transaction()?;
+
+        let mut ids: Vec<String> = vec![];
+        for op in operations {
+          if let Some(id) = op(&tx).map_err(|err| trailbase_sqlite::Error::Other(err.into()))? {
+            ids.push(id);
+          }
+        }
+
+        tx.commit()?;
+
+        return Ok(ids);
+      },
+    )
+    .await?;
+
+  return Ok(Json(TransactionResponse { ids }));
+}
+
+#[inline]
+fn extract_record_id(value: rusqlite::types::Value) -> Result<String, trailbase_sqlite::Error> {
+  return match value {
+    rusqlite::types::Value::Blob(blob) => Ok(BASE64_URL_SAFE.encode(blob)),
+    rusqlite::types::Value::Text(text) => Ok(text),
+    rusqlite::types::Value::Integer(i) => Ok(i.to_string()),
+    _ => Err(trailbase_sqlite::Error::Other(
+      "Unexpected data type".into(),
+    )),
+  };
+}
+
+#[inline]
+fn get_api(state: &AppState, api_name: &str) -> Result<RecordApi, RecordError> {
+  let Some(api) = state.lookup_record_api(api_name) else {
+    return Err(RecordError::ApiNotFound);
+  };
+  if !api.is_table() {
+    return Err(RecordError::ApiRequiresTable);
+  }
+  return Ok(api);
+}
+
+#[inline]
+fn extract_record(
+  value: serde_json::Value,
+) -> Result<serde_json::Map<String, serde_json::Value>, RecordError> {
+  let serde_json::Value::Object(record) = value else {
+    return Err(RecordError::BadRequest("Not a record"));
+  };
+  return Ok(record);
+}
+
+#[cfg(test)]
+mod tests {
+  use serde_json::json;
+
+  use super::*;
+  use crate::app_state::*;
+  use crate::config::proto::{ConflictResolutionStrategy, PermissionFlag, RecordApiConfig};
+  use crate::records::test_utils::*;
+
+  #[tokio::test]
+  async fn test_transactions() {
+    let state = test_state(None).await.unwrap();
+
+    state
+      .conn()
+      .execute_batch(
+        r#"
+          CREATE TABLE test (
+            id      INTEGER PRIMARY KEY,
+            value   INTEGER
+          ) STRICT;
+        "#,
+      )
+      .await
+      .unwrap();
+
+    state.rebuild_schema_cache().await.unwrap();
+
+    add_record_api_config(
+      &state,
+      RecordApiConfig {
+        name: Some("test_api".to_string()),
+        table_name: Some("test".to_string()),
+        conflict_resolution: Some(ConflictResolutionStrategy::Replace as i32),
+        acl_world: [
+          PermissionFlag::Create as i32,
+          PermissionFlag::Create as i32,
+          PermissionFlag::Delete as i32,
+          PermissionFlag::Read as i32,
+        ]
+        .into(),
+        ..Default::default()
+      },
+    )
+    .await
+    .unwrap();
+
+    let response = record_transactions_handler(
+      State(state.clone()),
+      None,
+      Json(TransactionRequest {
+        operations: vec![
+          Operation::Create {
+            api_name: "test_api".to_string(),
+            value: json!({"value": 1}),
+          },
+          Operation::Create {
+            api_name: "test_api".to_string(),
+            value: json!({"value": 2}),
+          },
+        ],
+      }),
+    )
+    .await
+    .unwrap();
+
+    assert_eq!(2, response.ids.len());
+
+    let response = record_transactions_handler(
+      State(state.clone()),
+      None,
+      Json(TransactionRequest {
+        operations: vec![
+          Operation::Delete {
+            api_name: "test_api".to_string(),
+            record_id: response.ids[0].clone(),
+          },
+          Operation::Create {
+            api_name: "test_api".to_string(),
+            value: json!({"value": 3}),
+          },
+        ],
+      }),
+    )
+    .await
+    .unwrap();
+
+    assert_eq!(1, response.ids.len());
+
+    assert_eq!(
+      2,
+      state
+        .conn()
+        .read_query_value::<i64>("SELECT COUNT(*) FROM test;", ())
+        .await
+        .unwrap()
+        .unwrap()
+    );
+  }
+}

crates/core/src/server/mod.rs

@@ -335,9 +335,12 @@ impl Server {
     opts: &ServerOptions,
     custom_router: Option<Router<AppState>>,
   ) -> (String, Router<()>) {
+    let enable_transactions =
+      state.access_config(|conn| conn.server.enable_record_transactions.unwrap_or(false));
+
     let mut router = Router::new()
       // Public, stable and versioned APIs.
-      .merge(records::router())
+      .merge(records::router(enable_transactions))
       .merge(auth::router())
       .route("/api/healthcheck", get(healthcheck_handler));
 

crates/sqlite/src/connection.rs

@@ -242,7 +242,7 @@ impl Connection {
   }
 
   #[inline]
-  async fn call_reader<F, R>(&self, function: F) -> Result<R>
+  pub async fn call_reader<F, R>(&self, function: F) -> Result<R>
   where
     F: FnOnce(&rusqlite::Connection) -> Result<R> + Send + 'static,
     R: Send + 'static,