Further speed up loading of data from SQL queries

This further improves the performance of SELECT statements by not
triggering the row count determiniation if it is not needed. So for
queries which return less rows than the configured prefetch block size
we save an extra query. Because this requires querying the data first
and the row count second (before this the order was reversed) the data
also appears faster in the UI.

This commit also fixes another issue which was introduced in commit
89587a7d67: When setting a filter which
made the query not return any data, the column names and the filter row
became invisible. Because of that it was also impossible to change the
filter again.

src/RowCache.h

@@ -61,6 +61,18 @@ public:
     /// to remove already-loaded rows from both ends.
     void smallestNonAvailableRange (size_t & row_begin, size_t & row_end) const;
 
+    /// \returns whether this cache object is marked as initialised
+    bool initialised() const
+    {
+        return is_initialised;
+    }
+
+    /// mark the cache object as intialised. This is reset by the clear() function
+    void setInitialised()
+    {
+        is_initialised = true;
+    }
+
 private:
     /// a single segment containing contiguous entries
     struct Segment
@@ -77,6 +89,9 @@ private:
     using Segments = std::vector<Segment>;
     Segments segments;
 
+    // Set to true when the cache is first initialised no matter whether it contains any data or represents an empty table
+    bool is_initialised;
+
     // ------------------------------------------------------------------------------
 
     /// \returns first segment that definitely cannot contain 'pos',
@@ -112,6 +127,7 @@ private:
 
 template <typename T>
 RowCache<T>::RowCache ()
+    : is_initialised(false)
 {
 }
 
@@ -234,6 +250,7 @@ void RowCache<T>::erase (size_t pos)
 template <typename T>
 void RowCache<T>::clear ()
 {
+    is_initialised = false;
     segments.clear();
 }
 

src/RowLoader.cpp

@@ -229,24 +229,22 @@ void RowLoader::process (Task & t)
     auto row = t.row_begin;
     if(sqlite3_prepare_v2(pDb.get(), utf8Query, utf8Query.size(), &stmt, nullptr) == SQLITE_OK)
     {
-        bool fill_headers = headers.empty();
+        // If the cache is uninitialised, retrieve the column names and types
+        const bool first_chunk = !cache_data.initialised();
+        if(first_chunk)
+        {
+            size_t num_columns = static_cast<size_t>(sqlite3_column_count(stmt));
+            for(size_t i=0;i<num_columns;++i)
+            {
+                headers.push_back(sqlite3_column_name(stmt, static_cast<int>(i)));
+                data_types.push_back(sqlite3_column_type(stmt, static_cast<int>(i)));
+            }
+        }
 
         while(!t.cancel && sqlite3_step(stmt) == SQLITE_ROW)
         {
             size_t num_columns = static_cast<size_t>(sqlite3_data_count(stmt));
 
-            // For the first row, determine the column names and types
-            if(fill_headers)
-            {
-                for(size_t i=0;i<num_columns;++i)
-                {
-                    headers.push_back(sqlite3_column_name(stmt, static_cast<int>(i)));
-                    data_types.push_back(sqlite3_column_type(stmt, static_cast<int>(i)));
-                }
-
-                fill_headers = false;
-            }
-
             // Construct a new row object with the right number of columns
             Cache::value_type rowdata(num_columns);
             for(size_t i=0;i<num_columns;++i)
@@ -266,6 +264,20 @@ void RowLoader::process (Task & t)
         }
 
         sqlite3_finalize(stmt);
+
+        // Query the total row count if and only if:
+        // - this is the first batch of data we load for this query
+        // - we got exactly the number of rows back we queried (which indicates there might be more rows)
+        // If there is no need to query the row count this means the number of rows we just got is the total row count.
+        if(first_chunk)
+        {
+            cache_data.setInitialised();
+
+            if(row == t.row_end)
+                triggerRowCountDetermination(t.token);
+            else
+                emit rowCountComplete(t.token, static_cast<int>(row-t.row_begin));
+        }
     }
 
     emit fetched(t.token, t.row_begin, row);

src/sqlitetablemodel.cpp

@@ -171,7 +171,6 @@ void SqliteTableModel::setQuery(const QString& sQuery, const QString& sCountQuer
     removeCommentsFromQuery(m_sQuery);
 
     worker->setQuery(m_sQuery, sCountQuery);
-    worker->triggerRowCountDetermination(m_lifeCounter);
 
     // now fetch the first entries
     triggerCacheLoad(static_cast<int>(m_chunkSize / 2) - 1);
@@ -851,6 +850,7 @@ void SqliteTableModel::clearCache()
     }
 
     m_cache.clear();
+    m_headers.clear();
     m_currentRowCount = 0;
     m_rowCountAvailable = RowCount::Unknown;
 }