diff --git a/include/sqlgen.hpp b/include/sqlgen.hpp index e28c190..425c811 100644 --- a/include/sqlgen.hpp +++ b/include/sqlgen.hpp @@ -3,8 +3,10 @@ #include "sqlgen/Connection.hpp" #include "sqlgen/Iterator.hpp" +#include "sqlgen/IteratorBase.hpp" #include "sqlgen/Literal.hpp" #include "sqlgen/PrimaryKey.hpp" +#include "sqlgen/Range.hpp" #include "sqlgen/Ref.hpp" #include "sqlgen/Result.hpp" #include "sqlgen/write.hpp" diff --git a/include/sqlgen/Connection.hpp b/include/sqlgen/Connection.hpp index 205efd4..b6a0038 100644 --- a/include/sqlgen/Connection.hpp +++ b/include/sqlgen/Connection.hpp @@ -5,7 +5,7 @@ #include #include -#include "Iterator.hpp" +#include "IteratorBase.hpp" #include "Ref.hpp" #include "Result.hpp" #include "dynamic/Insert.hpp" @@ -27,7 +27,7 @@ struct Connection { virtual Result execute(const std::string& _sql) = 0; /// Reads the results of a SelectFrom statement. - virtual Result> read(const dynamic::SelectFrom& _query) = 0; + virtual Result> read(const dynamic::SelectFrom& _query) = 0; /// Transpiles a statement to a particular SQL dialect. virtual std::string to_sql(const dynamic::Statement& _stmt) = 0; diff --git a/include/sqlgen/Iterator.hpp b/include/sqlgen/Iterator.hpp index 3370f44..48eaead 100644 --- a/include/sqlgen/Iterator.hpp +++ b/include/sqlgen/Iterator.hpp @@ -1,24 +1,70 @@ #ifndef SQLGEN_ITERATOR_HPP_ #define SQLGEN_ITERATOR_HPP_ -#include -#include -#include +#include +#include +#include "Ref.hpp" #include "Result.hpp" +#include "internal/batch_size.hpp" +#include "internal/collect/vector.hpp" +#include "internal/from_str_vec.hpp" namespace sqlgen { -/// Abstract base class for an iterator to be returned by Connection::read(...). -struct Iterator { - /// Whether the end of the available data has been reached. - virtual bool end() const = 0; +/// An input_iterator that returns the underlying type. +template +class Iterator { + public: + using difference_type = std::ptrdiff_t; + using value_type = Result; - /// Returns the next batch of rows. - /// If _batch_size is greater than the number of rows left, returns all - /// of the rows left. - virtual Result>>> next( - const size_t _batch_size) = 0; + struct End {}; + + Iterator(const Ref& _it) + : current_batch_(get_next_batch()), it_(_it), ix_(0) {} + + ~Iterator() = default; + + Result& operator*() const noexcept { return (*current_batch_)[ix_]; } + + bool operator==(const End&) noexcept { + return ix_ >= current_batch_->size() && it_->end(); + } + + bool operator!=(const End& _end) noexcept { return !(*this == _end); } + + Iterator& operator++() noexcept { + if (ix_ >= current_batch_->size() && !it_->end()) { + current_batch_ = get_next_batch(); + ix_ = 0; + } else { + ++ix_; + } + } + + void operator++(int) noexcept { ++*this; } + + private: + Ref>> get_next_batch() const noexcept { + using namespace std::ranges::views; + return it_->next(SQLGEN_BATCH_SIZE) + .transform([](auto str_vec) { + return Ref>>::make(internal::collect::vector( + str_vec | transform(internal::from_str_vec))); + }) + .value_or(Ref>>()); + } + + private: + /// The current batch of data. + Ref>> current_batch_; + + /// The underlying database iterator. + Ref it_; + + /// The current index in the current batch. + size_t ix_; }; } // namespace sqlgen diff --git a/include/sqlgen/IteratorBase.hpp b/include/sqlgen/IteratorBase.hpp new file mode 100644 index 0000000..aabac61 --- /dev/null +++ b/include/sqlgen/IteratorBase.hpp @@ -0,0 +1,27 @@ +#ifndef SQLGEN_ITERATORBASE_HPP_ +#define SQLGEN_ITERATORBASE_HPP_ + +#include +#include +#include + +#include "Result.hpp" + +namespace sqlgen { + +/// Abstract base class for an iterator to be returned by Connection::read(...). +struct IteratorBase { + /// Whether the end of the available data has been reached. + virtual bool end() const = 0; + + /// Returns the next batch of rows. + /// If _batch_size is greater than the number of rows left, returns all + /// of the rows left. + virtual Result>>> next( + const size_t _batch_size) = 0; +}; + +} // namespace sqlgen + +#endif + diff --git a/include/sqlgen/Range.hpp b/include/sqlgen/Range.hpp new file mode 100644 index 0000000..b76d179 --- /dev/null +++ b/include/sqlgen/Range.hpp @@ -0,0 +1,33 @@ +#ifndef SQLGEN_RANGE_HPP_ +#define SQLGEN_RANGE_HPP_ + +#include +#include + +#include "Iterator.hpp" + +namespace sqlgen { + +/// This class is meant to provide a way to iterate through the data in the +/// database efficiently that is compatible with std::ranges. +template +class Range { + public: + struct End {}; + + Range(const Ref& _it) : it_(_it) {} + + ~Range() = default; + + auto begin() const { return Iterator(it_); } + + auto end() const { return typename Iterator::End{}; } + + private: + /// The underlying database iterator. + Ref it_; +}; + +} // namespace sqlgen + +#endif diff --git a/include/sqlgen/internal/batch_size.hpp b/include/sqlgen/internal/batch_size.hpp new file mode 100644 index 0000000..5ae8960 --- /dev/null +++ b/include/sqlgen/internal/batch_size.hpp @@ -0,0 +1,3 @@ +#ifndef SQLGEN_BATCH_SIZE +#define SQLGEN_BATCH_SIZE 50000 +#endif diff --git a/include/sqlgen/internal/from_str_vec.hpp b/include/sqlgen/internal/from_str_vec.hpp index ecd042e..c9b3b4c 100644 --- a/include/sqlgen/internal/from_str_vec.hpp +++ b/include/sqlgen/internal/from_str_vec.hpp @@ -17,7 +17,8 @@ Result from_str_vec( const std::vector>& _str_vec) { T t; auto view = rfl::to_view(t); - auto view_reader = parsing::ViewReader>(&view); + auto view_reader = + parsing::ViewReader>(&view); const auto [err, num_fields_assigned] = view_reader.read(_str_vec); if (err) { return error(err->what()); diff --git a/include/sqlgen/parsing/ViewReader.hpp b/include/sqlgen/parsing/ViewReader.hpp index e079738..d4eb039 100644 --- a/include/sqlgen/parsing/ViewReader.hpp +++ b/include/sqlgen/parsing/ViewReader.hpp @@ -31,7 +31,7 @@ class ViewReader { std::stringstream stream; stream << "Expected exactly " << std::to_string(size_) << " fields, but got " << _row.size() << "."; - return std::make_pair(error(stream.str()), 0); + return std::make_pair(Error(stream.str()), 0); } for (size_t i = 0; i < size_; ++i) { const auto err = assign_to_field_i( diff --git a/include/sqlgen/read.hpp b/include/sqlgen/read.hpp new file mode 100644 index 0000000..2a07e21 --- /dev/null +++ b/include/sqlgen/read.hpp @@ -0,0 +1,63 @@ +#ifndef SQLGEN_READ_HPP_ +#define SQLGEN_READ_HPP_ + +#include +#include +#include +#include +#include +#include + +#include "Connection.hpp" +#include "Ref.hpp" +#include "Result.hpp" +#include "internal/to_str_vec.hpp" +#include "parsing/to_create_table.hpp" +#include "parsing/to_insert.hpp" + +namespace sqlgen { + +template +Result read(const Ref& _conn) { + using T = std::remove_cvref_t; + + const auto start_write = [&](const auto&) -> Result { + const auto insert_stmt = parsing::to_insert(); + return _conn->start_write(insert_stmt); + }; + + const auto write = [&](const auto&) -> Result { + try { + std::vector>> data; + for (auto it = _begin; it != _end; ++it) { + data.emplace_back(internal::to_str_vec(*it)); + if (data.size() == 50000) { + _conn->write(data).value(); + data.clear(); + } + } + if (data.size() != 0) { + _conn->write(data).value(); + } + return Nothing{}; + } catch (std::exception& e) { + _conn->end_write(); + return error(e.what()); + } + }; + + const auto end_write = [&](const auto&) -> Result { + return _conn->end_write(); + }; + + const auto create_table_stmt = parsing::to_create_table(); + + return _conn->execute(_conn->to_sql(create_table_stmt)) + .and_then(start_write) + .and_then(write) + .and_then(end_write); +} + +} // namespace sqlgen + +#endif diff --git a/include/sqlgen/sqlite/Connection.hpp b/include/sqlgen/sqlite/Connection.hpp index 2acfb6d..c8f5d6d 100644 --- a/include/sqlgen/sqlite/Connection.hpp +++ b/include/sqlgen/sqlite/Connection.hpp @@ -9,6 +9,7 @@ #include #include "../Connection.hpp" +#include "../IteratorBase.hpp" #include "../Ref.hpp" #include "../Result.hpp" #include "../dynamic/Column.hpp" @@ -37,7 +38,7 @@ class Connection : public sqlgen::Connection { Connection& operator=(const Connection& _other) = delete; - Result> read(const dynamic::SelectFrom& _query) final { + Result> read(const dynamic::SelectFrom& _query) final { return error("TODO"); } diff --git a/include/sqlgen/write.hpp b/include/sqlgen/write.hpp index f0ef9e0..cc3ab4e 100644 --- a/include/sqlgen/write.hpp +++ b/include/sqlgen/write.hpp @@ -11,6 +11,7 @@ #include "Connection.hpp" #include "Ref.hpp" #include "Result.hpp" +#include "internal/batch_size.hpp" #include "internal/to_str_vec.hpp" #include "parsing/to_create_table.hpp" #include "parsing/to_insert.hpp" @@ -33,7 +34,7 @@ Result write(const Ref& _conn, ItBegin _begin, std::vector>> data; for (auto it = _begin; it != _end; ++it) { data.emplace_back(internal::to_str_vec(*it)); - if (data.size() == 50000) { + if (data.size() == SQLGEN_BATCH_SIZE) { _conn->write(data).value(); data.clear(); }