From b41046e6b6955f9222fa8073cf0bae7f151863b5 Mon Sep 17 00:00:00 2001
From: Martin Kleusberg <github@mkleusberg.de>
Date: Sat, 9 Jan 2021 12:19:08 +0100
Subject: [PATCH] Further speed up loading of data from SQL queries

This further improves the performance of SELECT statements by not
triggering the row count determiniation if it is not needed. So for
queries which return less rows than the configured prefetch block size
we save an extra query. Because this requires querying the data first
and the row count second (before this the order was reversed) the data
also appears faster in the UI.

This commit also fixes another issue which was introduced in commit
89587a7d67928388d57a092817816b4971cf6e4b: When setting a filter which
made the query not return any data, the column names and the filter row
became invisible. Because of that it was also impossible to change the
filter again.
---
 src/RowCache.h           | 17 +++++++++++++++++
 src/RowLoader.cpp        | 38 +++++++++++++++++++++++++-------------
 src/sqlitetablemodel.cpp |  2 +-
 3 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/src/RowCache.h b/src/RowCache.h
index 182c9865..65163f8b 100644
--- a/src/RowCache.h
+++ b/src/RowCache.h
@@ -61,6 +61,18 @@ public:
     /// to remove already-loaded rows from both ends.
     void smallestNonAvailableRange (size_t & row_begin, size_t & row_end) const;
 
+    /// \returns whether this cache object is marked as initialised
+    bool initialised() const
+    {
+        return is_initialised;
+    }
+
+    /// mark the cache object as intialised. This is reset by the clear() function
+    void setInitialised()
+    {
+        is_initialised = true;
+    }
+
 private:
     /// a single segment containing contiguous entries
     struct Segment
@@ -77,6 +89,9 @@ private:
     using Segments = std::vector<Segment>;
     Segments segments;
 
+    // Set to true when the cache is first initialised no matter whether it contains any data or represents an empty table
+    bool is_initialised;
+
     // ------------------------------------------------------------------------------
 
     /// \returns first segment that definitely cannot contain 'pos',
@@ -112,6 +127,7 @@ private:
 
 template <typename T>
 RowCache<T>::RowCache ()
+    : is_initialised(false)
 {
 }
 
@@ -234,6 +250,7 @@ void RowCache<T>::erase (size_t pos)
 template <typename T>
 void RowCache<T>::clear ()
 {
+    is_initialised = false;
     segments.clear();
 }
 
diff --git a/src/RowLoader.cpp b/src/RowLoader.cpp
index 50f18750..b639a4a6 100644
--- a/src/RowLoader.cpp
+++ b/src/RowLoader.cpp
@@ -229,24 +229,22 @@ void RowLoader::process (Task & t)
     auto row = t.row_begin;
     if(sqlite3_prepare_v2(pDb.get(), utf8Query, utf8Query.size(), &stmt, nullptr) == SQLITE_OK)
     {
-        bool fill_headers = headers.empty();
+        // If the cache is uninitialised, retrieve the column names and types
+        const bool first_chunk = !cache_data.initialised();
+        if(first_chunk)
+        {
+            size_t num_columns = static_cast<size_t>(sqlite3_column_count(stmt));
+            for(size_t i=0;i<num_columns;++i)
+            {
+                headers.push_back(sqlite3_column_name(stmt, static_cast<int>(i)));
+                data_types.push_back(sqlite3_column_type(stmt, static_cast<int>(i)));
+            }
+        }
 
         while(!t.cancel && sqlite3_step(stmt) == SQLITE_ROW)
         {
             size_t num_columns = static_cast<size_t>(sqlite3_data_count(stmt));
 
-            // For the first row, determine the column names and types
-            if(fill_headers)
-            {
-                for(size_t i=0;i<num_columns;++i)
-                {
-                    headers.push_back(sqlite3_column_name(stmt, static_cast<int>(i)));
-                    data_types.push_back(sqlite3_column_type(stmt, static_cast<int>(i)));
-                }
-
-                fill_headers = false;
-            }
-
             // Construct a new row object with the right number of columns
             Cache::value_type rowdata(num_columns);
             for(size_t i=0;i<num_columns;++i)
@@ -266,6 +264,20 @@ void RowLoader::process (Task & t)
         }
 
         sqlite3_finalize(stmt);
+
+        // Query the total row count if and only if:
+        // - this is the first batch of data we load for this query
+        // - we got exactly the number of rows back we queried (which indicates there might be more rows)
+        // If there is no need to query the row count this means the number of rows we just got is the total row count.
+        if(first_chunk)
+        {
+            cache_data.setInitialised();
+
+            if(row == t.row_end)
+                triggerRowCountDetermination(t.token);
+            else
+                emit rowCountComplete(t.token, static_cast<int>(row-t.row_begin));
+        }
     }
 
     emit fetched(t.token, t.row_begin, row);
diff --git a/src/sqlitetablemodel.cpp b/src/sqlitetablemodel.cpp
index 2c56dbcc..ce7befe0 100644
--- a/src/sqlitetablemodel.cpp
+++ b/src/sqlitetablemodel.cpp
@@ -171,7 +171,6 @@ void SqliteTableModel::setQuery(const QString& sQuery, const QString& sCountQuer
     removeCommentsFromQuery(m_sQuery);
 
     worker->setQuery(m_sQuery, sCountQuery);
-    worker->triggerRowCountDetermination(m_lifeCounter);
 
     // now fetch the first entries
     triggerCacheLoad(static_cast<int>(m_chunkSize / 2) - 1);
@@ -851,6 +850,7 @@ void SqliteTableModel::clearCache()
     }
 
     m_cache.clear();
+    m_headers.clear();
     m_currentRowCount = 0;
     m_rowCountAvailable = RowCount::Unknown;
 }