From 26869c83f16fd77915e51e1cf9ec9b947f4c0f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dr=2E=20Patrick=20Urbanke=20=28=E5=8A=89=E8=87=AA=E6=88=90?= =?UTF-8?q?=29?= Date: Sat, 13 Sep 2025 08:29:26 +0200 Subject: [PATCH] Added support for JSON fields (#48) --- docs/README.md | 1 + docs/json.md | 133 +++++++++++++++++++++++++ include/sqlgen.hpp | 1 + include/sqlgen/JSON.hpp | 109 ++++++++++++++++++++ include/sqlgen/dynamic/Type.hpp | 7 +- include/sqlgen/dynamic/types.hpp | 4 + include/sqlgen/parsing/Parser.hpp | 1 + include/sqlgen/parsing/Parser_json.hpp | 34 +++++++ src/sqlgen/mysql/to_sql.cpp | 12 ++- src/sqlgen/postgres/to_sql.cpp | 6 +- src/sqlgen/sqlite/to_sql.cpp | 6 +- tests/mysql/test_json.cpp | 53 ++++++++++ tests/postgres/test_json.cpp | 53 ++++++++++ tests/sqlite/test_json.cpp | 43 ++++++++ vcpkg | 2 +- 15 files changed, 456 insertions(+), 9 deletions(-) create mode 100644 docs/json.md create mode 100644 include/sqlgen/JSON.hpp create mode 100644 include/sqlgen/parsing/Parser_json.hpp create mode 100644 tests/mysql/test_json.cpp create mode 100644 tests/postgres/test_json.cpp create mode 100644 tests/sqlite/test_json.cpp diff --git a/docs/README.md b/docs/README.md index 999adca..d4fad81 100644 --- a/docs/README.md +++ b/docs/README.md @@ -43,6 +43,7 @@ Welcome to the sqlgen documentation. This guide provides detailed information ab - [sqlgen::Dynamic](dynamic.md) - How to define custom SQL types not natively supported by sqlgen - [sqlgen::ForeignKey](foreign_key.md) - How to establish referential integrity between tables +- [sqlgen::JSON](json.md) - How to store and work with JSON fields - [sqlgen::Pattern](pattern.md) - How to add regex pattern validation to avoid SQL injection - [sqlgen::Timestamp](timestamp.md) - How timestamps work in sqlgen - [sqlgen::Unique](unique.md) - How to enforce uniqueness constraints on table columns diff --git a/docs/json.md b/docs/json.md new file mode 100644 index 0000000..bb0e316 --- /dev/null +++ b/docs/json.md @@ -0,0 +1,133 @@ +# `sqlgen::JSON` + +`sqlgen::JSON` stores structured JSON data in your database while preserving full C++ type-safety. It integrates with reflectcpp (`rfl`) so you can read and write arbitrary structured data without manual (de)serialization. + +Critically, `rfl::JSON` is fully serializable with reflectcpp and supports any data type also supported by reflectcpp (including nested structs, `std::optional`, containers like `std::vector`, and more), enabling seamless end-to-end integration. + +## Usage + +### Basic Definition + +Define a JSON field in your struct by wrapping the underlying C++ type with `sqlgen::JSON`: + +```cpp +struct Person { + std::string first_name; + std::string last_name; + int age; + sqlgen::JSON>> children; // Nested, optional JSON +}; +``` + +This generates a table with a JSON-compatible column type (dialect-specific): + +```sql +-- PostgreSQL +CREATE TABLE IF NOT EXISTS "Person"( + "first_name" TEXT NOT NULL, + "last_name" TEXT NOT NULL, + "age" INTEGER NOT NULL, + "children" JSONB +); + +-- MySQL / MariaDB +CREATE TABLE IF NOT EXISTS `Person`( + `first_name` TEXT NOT NULL, + `last_name` TEXT NOT NULL, + `age` INT NOT NULL, + `children` JSON +); + +-- SQLite +CREATE TABLE IF NOT EXISTS "Person"( + "first_name" TEXT NOT NULL, + "last_name" TEXT NOT NULL, + "age" INTEGER NOT NULL, + "children" JSONB +); +``` + +### Construction and Assignment + +Assign any reflectcpp-supported value to the JSON field. For example, a nested vector of the same type: + +```cpp +const auto children = std::vector({ + Person{.first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{.first_name = "Maggie", .last_name = "Simpson", .age = 0} +}); + +const auto homer = Person{ + .first_name = "Homer", + .last_name = "Simpson", + .age = 45, + .children = children // Automatically serialized to JSON +}; +``` + +You can store any `T` that reflectcpp can serialize, such as: +- `std::optional` +- `std::vector`, `std::map`, and other standard containers +- Nested `struct` types reflected with reflectcpp + +### Reading and Writing + +Use the regular `sqlgen::write` and `sqlgen::read` APIs. JSON values are transparently serialized/deserialized. + +```cpp +using namespace sqlgen; +using namespace sqlgen::literals; + +// PostgreSQL example (analogous for MySQL/SQLite) +const auto credentials = sqlgen::postgres::Credentials{ + .user = "postgres", .password = "password", .host = "localhost", .dbname = "postgres"}; + +const auto people = sqlgen::postgres::connect(credentials) + .and_then(drop | if_exists) + .and_then(write(std::ref(homer))) + .and_then(sqlgen::read>) + .value(); +``` + +### Accessing Values + +Access the underlying C++ value via familiar methods: + +```cpp +// Underlying value access +people[0].children(); +people[0].children.get(); +people[0].children.value(); + +// Serialize entire result to JSON with reflectcpp +const std::string json = rfl::json::write(people); +``` + +### Template Parameters + +The `sqlgen::JSON` template takes one parameter: + +1. T: Any reflectcpp-supported C++ type to store as JSON + +```cpp +sqlgen::JSON field_name; +``` + +### Database Integration + +`sqlgen::JSON` selects the appropriate JSON-capable storage per dialect: +- **PostgreSQL**: `JSONB` +- **MySQL/MariaDB**: `JSON` +- **SQLite**: `JSONB` + +All (de)serialization is handled by reflectcpp (`rfl`) underneath, ensuring type-safe transformations without manual glue code. + +## Notes + +- Works with any reflectcpp-serializable type (`rfl::JSON` support), including deeply nested structures +- Integrates with `sqlgen::read` and `sqlgen::write` like any other field +- Database JSON type is chosen automatically per dialect +- Supports move and copy semantics +- Provides multiple access methods for the underlying value +- Ideal for flexible, schema-evolving nested attributes diff --git a/include/sqlgen.hpp b/include/sqlgen.hpp index 88ebafa..1172fad 100644 --- a/include/sqlgen.hpp +++ b/include/sqlgen.hpp @@ -6,6 +6,7 @@ #include "sqlgen/ForeignKey.hpp" #include "sqlgen/Iterator.hpp" #include "sqlgen/IteratorBase.hpp" +#include "sqlgen/JSON.hpp" #include "sqlgen/Literal.hpp" #include "sqlgen/Pattern.hpp" #include "sqlgen/PrimaryKey.hpp" diff --git a/include/sqlgen/JSON.hpp b/include/sqlgen/JSON.hpp new file mode 100644 index 0000000..d1c4f90 --- /dev/null +++ b/include/sqlgen/JSON.hpp @@ -0,0 +1,109 @@ +#ifndef SQLGEN_JSON_HPP_ +#define SQLGEN_JSON_HPP_ + +#include +#include + +#include "transpilation/is_nullable.hpp" + +namespace sqlgen { + +template +class JSON { + public: + using ReflectionType = T; + + JSON() : value_(T()) {} + + JSON(const T& _value) : value_(_value) {} + + JSON(JSON&& _other) noexcept = default; + + JSON(const JSON& _other) = default; + + template + JSON(const JSON& _other) : value_(_other.get()) {} + + template + JSON(JSON&& _other) : value_(_other.get()) {} + + template , + bool>::type = true> + JSON(const U& _value) : value_(_value) {} + + template , + bool>::type = true> + JSON(U&& _value) noexcept : value_(std::forward(_value)) {} + + template , + bool>::type = true> + JSON(const JSON& _other) : value_(_other.value()) {} + + ~JSON() = default; + + /// Returns the underlying object. + T& get() { return value_; } + + /// Returns the underlying object. + const T& get() const { return value_; } + + /// Returns the underlying object. + T& operator()() { return value_; } + + /// Returns the underlying object. + const T& operator()() const { return value_; } + + /// Assigns the underlying object. + auto& operator=(const T& _value) { + value_ = _value; + return *this; + } + + /// Assigns the underlying object. + template , + bool>::type = true> + auto& operator=(const U& _value) { + value_ = _value; + return *this; + } + + /// Assigns the underlying object. + JSON& operator=(const JSON& _other) = default; + + /// Assigns the underlying object. + JSON& operator=(JSON&& _other) = default; + + /// Assigns the underlying object. + template + auto& operator=(const JSON& _other) { + value_ = _other.get(); + return *this; + } + + /// Assigns the underlying object. + template + auto& operator=(JSON&& _other) { + value_ = std::move(_other.value_); + return *this; + } + + /// Necessary for the automated transpilation to work. + const T& reflection() const { return value_; } + + /// Assigns the underlying object. + void set(const T& _value) { value_ = _value; } + + /// Returns the underlying object. + T& value() { return value_; } + + /// Returns the underlying object. + const T& value() const { return value_; } + + private: + /// The underlying value. + T value_; +}; + +} // namespace sqlgen + +#endif diff --git a/include/sqlgen/dynamic/Type.hpp b/include/sqlgen/dynamic/Type.hpp index 7f8b8ea..7180993 100644 --- a/include/sqlgen/dynamic/Type.hpp +++ b/include/sqlgen/dynamic/Type.hpp @@ -11,9 +11,10 @@ namespace sqlgen::dynamic { using Type = rfl::TaggedUnion<"type", types::Unknown, types::Boolean, types::Dynamic, types::Float32, types::Float64, types::Int8, types::Int16, - types::Int32, types::Int64, types::UInt8, types::UInt16, - types::UInt32, types::UInt64, types::Text, types::Date, - types::Timestamp, types::TimestampWithTZ, types::VarChar>; + types::Int32, types::Int64, types::JSON, types::UInt8, + types::UInt16, types::UInt32, types::UInt64, types::Text, + types::Date, types::Timestamp, types::TimestampWithTZ, + types::VarChar>; } // namespace sqlgen::dynamic diff --git a/include/sqlgen/dynamic/types.hpp b/include/sqlgen/dynamic/types.hpp index c5fdbb8..ca7aba7 100644 --- a/include/sqlgen/dynamic/types.hpp +++ b/include/sqlgen/dynamic/types.hpp @@ -58,6 +58,10 @@ struct Int64 { Properties properties; }; +struct JSON { + Properties properties; +}; + struct UInt8 { Properties properties; }; diff --git a/include/sqlgen/parsing/Parser.hpp b/include/sqlgen/parsing/Parser.hpp index 2179d93..8e3af5c 100644 --- a/include/sqlgen/parsing/Parser.hpp +++ b/include/sqlgen/parsing/Parser.hpp @@ -4,6 +4,7 @@ #include "Parser_base.hpp" #include "Parser_default.hpp" #include "Parser_foreign_key.hpp" +#include "Parser_json.hpp" #include "Parser_optional.hpp" #include "Parser_primary_key.hpp" #include "Parser_shared_ptr.hpp" diff --git a/include/sqlgen/parsing/Parser_json.hpp b/include/sqlgen/parsing/Parser_json.hpp new file mode 100644 index 0000000..f846ee0 --- /dev/null +++ b/include/sqlgen/parsing/Parser_json.hpp @@ -0,0 +1,34 @@ +#ifndef SQLGEN_PARSING_PARSER_JSON_HPP_ +#define SQLGEN_PARSING_PARSER_JSON_HPP_ + +#include +#include +#include + +#include "../JSON.hpp" +#include "../Result.hpp" +#include "../dynamic/Type.hpp" +#include "Parser_base.hpp" + +namespace sqlgen::parsing { + +template +struct Parser> { + static Result> read(const std::optional& _str) noexcept { + if (!_str) { + return error("NULL value encounted: JSON value cannot be NULL."); + } + return rfl::json::read(*_str).transform( + [](auto&& _t) { return JSON(std::move(_t)); }); + } + + static std::optional write(const JSON& _j) noexcept { + return rfl::json::write(_j.value()); + } + + static dynamic::Type to_type() noexcept { return dynamic::types::JSON{}; } +}; + +} // namespace sqlgen::parsing + +#endif diff --git a/src/sqlgen/mysql/to_sql.cpp b/src/sqlgen/mysql/to_sql.cpp index 46c6303..8236ae5 100644 --- a/src/sqlgen/mysql/to_sql.cpp +++ b/src/sqlgen/mysql/to_sql.cpp @@ -140,7 +140,8 @@ std::string cast_type_to_sql(const dynamic::Type& _type) noexcept { return "DECIMAL"; } else if constexpr (std::is_same_v || - std::is_same_v) { + std::is_same_v || + std::is_same_v) { return "CHAR"; } else if constexpr (std::is_same_v) { @@ -541,13 +542,15 @@ std::string insert_or_write_to_sql(const InsertOrWrite& _stmt) noexcept { stream << " VALUES ("; stream << internal::strings::join( - ", ", internal::collect::vector(_stmt.columns | transform(to_questionmark))); + ", ", + internal::collect::vector(_stmt.columns | transform(to_questionmark))); stream << ")"; if constexpr (std::is_same_v) { if (_stmt.or_replace) { stream << " ON DUPLICATE KEY UPDATE "; stream << internal::strings::join( - ", ", internal::collect::vector(_stmt.columns | transform(as_values))); + ", ", + internal::collect::vector(_stmt.columns | transform(as_values))); } } @@ -879,6 +882,9 @@ std::string type_to_sql(const dynamic::Type& _type) noexcept { } else if constexpr (std::is_same_v) { return "VARCHAR(" + std::to_string(_t.length) + ")"; + } else if constexpr (std::is_same_v) { + return "JSON"; + } else if constexpr (std::is_same_v) { return "DATE"; diff --git a/src/sqlgen/postgres/to_sql.cpp b/src/sqlgen/postgres/to_sql.cpp index 12e0827..95cab03 100644 --- a/src/sqlgen/postgres/to_sql.cpp +++ b/src/sqlgen/postgres/to_sql.cpp @@ -424,7 +424,8 @@ std::string insert_to_sql(const dynamic::Insert& _stmt) noexcept { stream << " DO UPDATE SET "; stream << internal::strings::join( - ", ", internal::collect::vector(_stmt.columns | transform(as_excluded))); + ", ", + internal::collect::vector(_stmt.columns | transform(as_excluded))); } stream << ";"; @@ -761,6 +762,9 @@ std::string type_to_sql(const dynamic::Type& _type) noexcept { } else if constexpr (std::is_same_v) { return "VARCHAR(" + std::to_string(_t.length) + ")"; + } else if constexpr (std::is_same_v) { + return "JSONB"; + } else if constexpr (std::is_same_v) { return "DATE"; diff --git a/src/sqlgen/sqlite/to_sql.cpp b/src/sqlgen/sqlite/to_sql.cpp index d0c2ba9..472c902 100644 --- a/src/sqlgen/sqlite/to_sql.cpp +++ b/src/sqlgen/sqlite/to_sql.cpp @@ -420,7 +420,8 @@ std::string insert_or_write_to_sql(const InsertOrWrite& _stmt) noexcept { stream << " DO UPDATE SET "; stream << internal::strings::join( - ", ", internal::collect::vector(_stmt.columns | transform(as_excluded))); + ", ", + internal::collect::vector(_stmt.columns | transform(as_excluded))); } } @@ -763,6 +764,9 @@ std::string type_to_sql(const dynamic::Type& _type) noexcept { std::is_same_v) { return "TEXT"; + } else if constexpr (std::is_same_v) { + return "JSONB"; + } else if constexpr (std::is_same_v) { return _t.type_name; diff --git a/tests/mysql/test_json.cpp b/tests/mysql/test_json.cpp new file mode 100644 index 0000000..a4a3243 --- /dev/null +++ b/tests/mysql/test_json.cpp @@ -0,0 +1,53 @@ +#ifndef SQLGEN_BUILD_DRY_TESTS_ONLY + +#include + +#include +#include +#include +#include +#include + +namespace test_json { + +struct Person { + std::string first_name; + std::string last_name; + int age; + sqlgen::JSON>> children; +}; + +TEST(mysql, test_json) { + const auto children = std::vector( + {Person{.first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{.first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto homer = Person{.first_name = "Homer", + .last_name = "Simpson", + .age = 45, + .children = children}; + + const auto credentials = sqlgen::mysql::Credentials{.host = "localhost", + .user = "sqlgen", + .password = "password", + .dbname = "mysql"}; + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = sqlgen::mysql::connect(credentials) + .and_then(drop | if_exists) + .and_then(write(std::ref(homer))) + .and_then(sqlgen::read>) + .value(); + + const std::string expected = + R"([{"first_name":"Homer","last_name":"Simpson","age":45,"children":[{"first_name":"Bart","last_name":"Simpson","age":10},{"first_name":"Lisa","last_name":"Simpson","age":8},{"first_name":"Maggie","last_name":"Simpson","age":0}]}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_json + +#endif diff --git a/tests/postgres/test_json.cpp b/tests/postgres/test_json.cpp new file mode 100644 index 0000000..66c5155 --- /dev/null +++ b/tests/postgres/test_json.cpp @@ -0,0 +1,53 @@ +#ifndef SQLGEN_BUILD_DRY_TESTS_ONLY + +#include + +#include +#include +#include +#include +#include + +namespace test_json { + +struct Person { + std::string first_name; + std::string last_name; + int age; + sqlgen::JSON>> children; +}; + +TEST(postgres, test_json) { + const auto children = std::vector( + {Person{.first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{.first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto homer = Person{.first_name = "Homer", + .last_name = "Simpson", + .age = 45, + .children = children}; + + const auto credentials = sqlgen::postgres::Credentials{.user = "postgres", + .password = "password", + .host = "localhost", + .dbname = "postgres"}; + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = sqlgen::postgres::connect(credentials) + .and_then(drop | if_exists) + .and_then(write(std::ref(homer))) + .and_then(sqlgen::read>) + .value(); + + const std::string expected = + R"([{"first_name":"Homer","last_name":"Simpson","age":45,"children":[{"first_name":"Bart","last_name":"Simpson","age":10},{"first_name":"Lisa","last_name":"Simpson","age":8},{"first_name":"Maggie","last_name":"Simpson","age":0}]}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_json + +#endif diff --git a/tests/sqlite/test_json.cpp b/tests/sqlite/test_json.cpp new file mode 100644 index 0000000..51a0dd4 --- /dev/null +++ b/tests/sqlite/test_json.cpp @@ -0,0 +1,43 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_json { + +struct Person { + std::string first_name; + std::string last_name; + int age; + sqlgen::JSON>> children; +}; + +TEST(sqlite, test_json) { + const auto children = std::vector( + {Person{.first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{.first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto homer = Person{.first_name = "Homer", + .last_name = "Simpson", + .age = 45, + .children = children}; + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = sqlgen::sqlite::connect() + .and_then(write(std::ref(homer))) + .and_then(sqlgen::read>) + .value(); + + const std::string expected = + R"([{"first_name":"Homer","last_name":"Simpson","age":45,"children":[{"first_name":"Bart","last_name":"Simpson","age":10},{"first_name":"Lisa","last_name":"Simpson","age":8},{"first_name":"Maggie","last_name":"Simpson","age":0}]}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_json diff --git a/vcpkg b/vcpkg index 7354f1c..7213cf8 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit 7354f1c8a0a276072e8d73d7eb6df6ca0ce8ccb1 +Subproject commit 7213cf8135c329c37c7e2778e40774489a0583a8