Added a range

include/sqlgen.hpp

@@ -3,8 +3,10 @@
 
 #include "sqlgen/Connection.hpp"
 #include "sqlgen/Iterator.hpp"
+#include "sqlgen/IteratorBase.hpp"
 #include "sqlgen/Literal.hpp"
 #include "sqlgen/PrimaryKey.hpp"
+#include "sqlgen/Range.hpp"
 #include "sqlgen/Ref.hpp"
 #include "sqlgen/Result.hpp"
 #include "sqlgen/write.hpp"

include/sqlgen/Connection.hpp

@@ -5,7 +5,7 @@
 #include <string>
 #include <vector>
 
-#include "Iterator.hpp"
+#include "IteratorBase.hpp"
 #include "Ref.hpp"
 #include "Result.hpp"
 #include "dynamic/Insert.hpp"
@@ -27,7 +27,7 @@ struct Connection {
   virtual Result<Nothing> execute(const std::string& _sql) = 0;
 
   /// Reads the results of a SelectFrom statement.
-  virtual Result<Ref<Iterator>> read(const dynamic::SelectFrom& _query) = 0;
+  virtual Result<Ref<IteratorBase>> read(const dynamic::SelectFrom& _query) = 0;
 
   /// Transpiles a statement to a particular SQL dialect.
   virtual std::string to_sql(const dynamic::Statement& _stmt) = 0;

include/sqlgen/Iterator.hpp

@@ -1,24 +1,70 @@
 #ifndef SQLGEN_ITERATOR_HPP_
 #define SQLGEN_ITERATOR_HPP_
 
-#include <optional>
-#include <string>
-#include <vector>
+#include <memory>
+#include <ranges>
 
+#include "Ref.hpp"
 #include "Result.hpp"
+#include "internal/batch_size.hpp"
+#include "internal/collect/vector.hpp"
+#include "internal/from_str_vec.hpp"
 
 namespace sqlgen {
 
-/// Abstract base class for an iterator to be returned by Connection::read(...).
-struct Iterator {
-  /// Whether the end of the available data has been reached.
-  virtual bool end() const = 0;
+/// An input_iterator that returns the underlying type.
+template <class T>
+class Iterator {
+ public:
+  using difference_type = std::ptrdiff_t;
+  using value_type = Result<T>;
 
-  /// Returns the next batch of rows.
-  /// If _batch_size is greater than the number of rows left, returns all
-  /// of the rows left.
-  virtual Result<std::vector<std::vector<std::optional<std::string>>>> next(
-      const size_t _batch_size) = 0;
+  struct End {};
+
+  Iterator(const Ref<IteratorBase>& _it)
+      : current_batch_(get_next_batch()), it_(_it), ix_(0) {}
+
+  ~Iterator() = default;
+
+  Result<T>& operator*() const noexcept { return (*current_batch_)[ix_]; }
+
+  bool operator==(const End&) noexcept {
+    return ix_ >= current_batch_->size() && it_->end();
+  }
+
+  bool operator!=(const End& _end) noexcept { return !(*this == _end); }
+
+  Iterator<T>& operator++() noexcept {
+    if (ix_ >= current_batch_->size() && !it_->end()) {
+      current_batch_ = get_next_batch();
+      ix_ = 0;
+    } else {
+      ++ix_;
+    }
+  }
+
+  void operator++(int) noexcept { ++*this; }
+
+ private:
+  Ref<std::vector<Result<T>>> get_next_batch() const noexcept {
+    using namespace std::ranges::views;
+    return it_->next(SQLGEN_BATCH_SIZE)
+        .transform([](auto str_vec) {
+          return Ref<std::vector<Result<T>>>::make(internal::collect::vector(
+              str_vec | transform(internal::from_str_vec<T>)));
+        })
+        .value_or(Ref<std::vector<Result<T>>>());
+  }
+
+ private:
+  /// The current batch of data.
+  Ref<std::vector<Result<T>>> current_batch_;
+
+  /// The underlying database iterator.
+  Ref<IteratorBase> it_;
+
+  /// The current index in the current batch.
+  size_t ix_;
 };
 
 }  // namespace sqlgen

include/sqlgen/IteratorBase.hpp

@@ -0,0 +1,27 @@
+#ifndef SQLGEN_ITERATORBASE_HPP_
+#define SQLGEN_ITERATORBASE_HPP_
+
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "Result.hpp"
+
+namespace sqlgen {
+
+/// Abstract base class for an iterator to be returned by Connection::read(...).
+struct IteratorBase {
+  /// Whether the end of the available data has been reached.
+  virtual bool end() const = 0;
+
+  /// Returns the next batch of rows.
+  /// If _batch_size is greater than the number of rows left, returns all
+  /// of the rows left.
+  virtual Result<std::vector<std::vector<std::optional<std::string>>>> next(
+      const size_t _batch_size) = 0;
+};
+
+}  // namespace sqlgen
+
+#endif
+

include/sqlgen/Range.hpp

@@ -0,0 +1,33 @@
+#ifndef SQLGEN_RANGE_HPP_
+#define SQLGEN_RANGE_HPP_
+
+#include <memory>
+#include <ranges>
+
+#include "Iterator.hpp"
+
+namespace sqlgen {
+
+/// This class is meant to provide a way to iterate through the data in the
+/// database efficiently that is compatible with std::ranges.
+template <class T>
+class Range {
+ public:
+  struct End {};
+
+  Range(const Ref<IteratorBase>& _it) : it_(_it) {}
+
+  ~Range() = default;
+
+  auto begin() const { return Iterator<T>(it_); }
+
+  auto end() const { return typename Iterator<T>::End{}; }
+
+ private:
+  /// The underlying database iterator.
+  Ref<IteratorBase> it_;
+};
+
+}  // namespace sqlgen
+
+#endif

include/sqlgen/internal/batch_size.hpp

@@ -0,0 +1,3 @@
+#ifndef SQLGEN_BATCH_SIZE
+#define SQLGEN_BATCH_SIZE 50000
+#endif

include/sqlgen/internal/from_str_vec.hpp

@@ -17,7 +17,8 @@ Result<T> from_str_vec(
     const std::vector<std::optional<std::string>>& _str_vec) {
   T t;
   auto view = rfl::to_view(t);
-  auto view_reader = parsing::ViewReader<remove_cvref_t<decltype(view)>>(&view);
+  auto view_reader =
+      parsing::ViewReader<std::remove_cvref_t<decltype(view)>>(&view);
   const auto [err, num_fields_assigned] = view_reader.read(_str_vec);
   if (err) {
     return error(err->what());

include/sqlgen/parsing/ViewReader.hpp

@@ -31,7 +31,7 @@ class ViewReader {
       std::stringstream stream;
       stream << "Expected exactly " << std::to_string(size_)
              << " fields, but got " << _row.size() << ".";
-      return std::make_pair(error(stream.str()), 0);
+      return std::make_pair(Error(stream.str()), 0);
     }
     for (size_t i = 0; i < size_; ++i) {
       const auto err = assign_to_field_i(

include/sqlgen/read.hpp

@@ -0,0 +1,63 @@
+#ifndef SQLGEN_READ_HPP_
+#define SQLGEN_READ_HPP_
+
+#include <iterator>
+#include <optional>
+#include <rfl.hpp>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include "Connection.hpp"
+#include "Ref.hpp"
+#include "Result.hpp"
+#include "internal/to_str_vec.hpp"
+#include "parsing/to_create_table.hpp"
+#include "parsing/to_insert.hpp"
+
+namespace sqlgen {
+
+template <class ContainerType>
+Result<ContainerType> read(const Ref<Connection>& _conn) {
+  using T = std::remove_cvref_t<typename ContainerType::value_type>;
+
+  const auto start_write = [&](const auto&) -> Result<Nothing> {
+    const auto insert_stmt = parsing::to_insert<T>();
+    return _conn->start_write(insert_stmt);
+  };
+
+  const auto write = [&](const auto&) -> Result<Nothing> {
+    try {
+      std::vector<std::vector<std::optional<std::string>>> data;
+      for (auto it = _begin; it != _end; ++it) {
+        data.emplace_back(internal::to_str_vec(*it));
+        if (data.size() == 50000) {
+          _conn->write(data).value();
+          data.clear();
+        }
+      }
+      if (data.size() != 0) {
+        _conn->write(data).value();
+      }
+      return Nothing{};
+    } catch (std::exception& e) {
+      _conn->end_write();
+      return error(e.what());
+    }
+  };
+
+  const auto end_write = [&](const auto&) -> Result<Nothing> {
+    return _conn->end_write();
+  };
+
+  const auto create_table_stmt = parsing::to_create_table<T>();
+
+  return _conn->execute(_conn->to_sql(create_table_stmt))
+      .and_then(start_write)
+      .and_then(write)
+      .and_then(end_write);
+}
+
+}  // namespace sqlgen
+
+#endif

include/sqlgen/sqlite/Connection.hpp

@@ -9,6 +9,7 @@
 #include <string>
 
 #include "../Connection.hpp"
+#include "../IteratorBase.hpp"
 #include "../Ref.hpp"
 #include "../Result.hpp"
 #include "../dynamic/Column.hpp"
@@ -37,7 +38,7 @@ class Connection : public sqlgen::Connection {
 
   Connection& operator=(const Connection& _other) = delete;
 
-  Result<Ref<Iterator>> read(const dynamic::SelectFrom& _query) final {
+  Result<Ref<IteratorBase>> read(const dynamic::SelectFrom& _query) final {
     return error("TODO");
   }
 

include/sqlgen/write.hpp

@@ -11,6 +11,7 @@
 #include "Connection.hpp"
 #include "Ref.hpp"
 #include "Result.hpp"
+#include "internal/batch_size.hpp"
 #include "internal/to_str_vec.hpp"
 #include "parsing/to_create_table.hpp"
 #include "parsing/to_insert.hpp"
@@ -33,7 +34,7 @@ Result<Nothing> write(const Ref<Connection>& _conn, ItBegin _begin,
       std::vector<std::vector<std::optional<std::string>>> data;
       for (auto it = _begin; it != _end; ++it) {
         data.emplace_back(internal::to_str_vec(*it));
-        if (data.size() == 50000) {
+        if (data.size() == SQLGEN_BATCH_SIZE) {
           _conn->write(data).value();
           data.clear();
         }