Use a specialised count query for browsing tables/views

For queries built from a table and filters, use a specialised count query
that only takes into account the where part of the statement. This will
speed-up the Browse Data tab for big tables.

See issue #1666

src/RowLoader.cpp

@@ -26,6 +26,7 @@ RowLoader::RowLoader (
     : db_getter(db_getter_), statement_logger(statement_logger_), headers(headers_)
     , cache_mutex(cache_mutex_), cache_data(cache_data_)
     , query()
+    , countQuery()
     , num_tasks(0)
     , pDb(nullptr)
     , stop_requested(false)
@@ -34,10 +35,15 @@ RowLoader::RowLoader (
 {
 }
 
-void RowLoader::setQuery (QString new_query)
+void RowLoader::setQuery (QString new_query, QString newCountQuery)
 {
     std::lock_guard<std::mutex> lk(m);
     query = new_query;
+    if (newCountQuery.isEmpty())
+        // If it is a normal query - hopefully starting with SELECT - just do a COUNT on it and return the results
+        countQuery = QString("SELECT COUNT(*) FROM (%1);").arg(rtrimChar(query, ';'));
+    else
+        countQuery = newCountQuery;
 }
 
 void RowLoader::triggerRowCountDetermination(int token)
@@ -89,10 +95,8 @@ int RowLoader::countRows()
             return retval;
         }
     } else {
-        // If it is a normal query - hopefully starting with SELECT - just do a COUNT on it and return the results
-        QString sCountQuery = QString("SELECT COUNT(*) FROM (%1);").arg(rtrimChar(query, ';'));
-        statement_logger(sCountQuery);
-        QByteArray utf8Query = sCountQuery.toUtf8();
+        statement_logger(countQuery);
+        QByteArray utf8Query = countQuery.toUtf8();
 
         sqlite3_stmt* stmt;
         int status = sqlite3_prepare_v2(pDb.get(), utf8Query, utf8Query.size(), &stmt, nullptr);
@@ -106,7 +110,7 @@ int RowLoader::countRows()
             }
             sqlite3_finalize(stmt);
         } else {
-            qWarning() << "Count query failed: " << sCountQuery;
+            qWarning() << "Count query failed: " << countQuery;
         }
     }
 

src/RowLoader.h

@@ -34,7 +34,7 @@ public:
         Cache & cache_data
         );
 
-    void setQuery (QString);
+    void setQuery (QString new_query, QString newCountQuery = QString());
 
     void triggerRowCountDetermination (int token);
 
@@ -77,6 +77,7 @@ private:
     mutable std::condition_variable cv;
 
     QString query;
+    QString countQuery;
 
     mutable std::future<void> row_counter;
 

src/sql/Query.cpp

@@ -18,6 +18,28 @@ void Query::clear()
     m_sort.clear();
 }
 
+std::string Query::buildWherePart() const
+{
+    std::string where;
+    if(m_where.size())
+    {
+        where = "WHERE ";
+
+        for(auto i=m_where.cbegin();i!=m_where.cend();++i)
+        {
+            const auto it = findSelectedColumnByName(m_column_names.at(i->first));
+            std::string column = sqlb::escapeIdentifier(m_column_names.at(i->first));
+            if(it != m_selected_columns.cend() && it->selector != column)
+                column = it->selector;
+            where += column + " " + i->second + " AND ";
+        }
+
+        // Remove last ' AND '
+        where.erase(where.size() - 5);
+    }
+    return where;
+}
+
 std::string Query::buildQuery(bool withRowid) const
 {
     // Selector and display formats
@@ -40,23 +62,7 @@ std::string Query::buildQuery(bool withRowid) const
     }
 
     // Filter
-    std::string where;
-    if(m_where.size())
-    {
-        where = "WHERE ";
-
-        for(auto i=m_where.cbegin();i!=m_where.cend();++i)
-        {
-            const auto it = findSelectedColumnByName(m_column_names.at(i->first));
-            std::string column = sqlb::escapeIdentifier(m_column_names.at(i->first));
-            if(it != m_selected_columns.cend() && it->selector != column)
-                column = it->selector;
-            where += column + " " + i->second + " AND ";
-        }
-
-        // Remove last ' AND '
-        where.erase(where.size() - 5);
-    }
+    std::string where = buildWherePart();
 
     // Sorting
     std::string order_by;
@@ -72,6 +78,12 @@ std::string Query::buildQuery(bool withRowid) const
     return "SELECT " + selector + " FROM " + m_table.toString().toStdString() + " " + where + " " + order_by;
 }
 
+std::string Query::buildCountQuery() const
+{
+    // Build simplest count query for this (filtered) table
+    return "SELECT COUNT(*) FROM " + m_table.toString().toStdString() + " " + buildWherePart();
+}
+
 std::vector<SelectedColumn>::iterator Query::findSelectedColumnByName(const std::string& name)
 {
     return std::find_if(m_selected_columns.begin(), m_selected_columns.end(), [name](const SelectedColumn& c) {

src/sql/Query.h

@@ -48,6 +48,7 @@ public:
 
     void clear();
     std::string buildQuery(bool withRowid) const;
+    std::string buildCountQuery() const;
 
     void setColumNames(const std::vector<std::string>& column_names) { m_column_names = column_names; }
     std::vector<std::string> columnNames() const { return m_column_names; }
@@ -77,6 +78,7 @@ private:
 
     std::vector<SelectedColumn>::iterator findSelectedColumnByName(const std::string& name);
     std::vector<SelectedColumn>::const_iterator findSelectedColumnByName(const std::string& name) const;
+    std::string buildWherePart() const;
 };
 
 }

src/sqlitetablemodel.cpp

@@ -176,7 +176,7 @@ void SqliteTableModel::setQuery(const sqlb::Query& query)
     buildQuery();
 }
 
-void SqliteTableModel::setQuery(const QString& sQuery, bool dontClearHeaders)
+void SqliteTableModel::setQuery(const QString& sQuery, const QString& sCountQuery, bool dontClearHeaders)
 {
     // clear
     if(!dontClearHeaders)
@@ -190,7 +190,7 @@ void SqliteTableModel::setQuery(const QString& sQuery, bool dontClearHeaders)
     m_sQuery = sQuery.trimmed();
     removeCommentsFromQuery(m_sQuery);
 
-    worker->setQuery(m_sQuery);
+    worker->setQuery(m_sQuery, sCountQuery);
     worker->triggerRowCountDetermination(m_lifeCounter);
 
     if(!dontClearHeaders)
@@ -593,7 +593,7 @@ QModelIndex SqliteTableModel::dittoRecord(int old_row)
 
 void SqliteTableModel::buildQuery()
 {
-    setQuery(QString::fromStdString(m_query.buildQuery(true)), true);
+    setQuery(QString::fromStdString(m_query.buildQuery(true)), QString::fromStdString(m_query.buildCountQuery()), true);
 }
 
 void SqliteTableModel::removeCommentsFromQuery(QString& query)

src/sqlitetablemodel.h

@@ -77,7 +77,7 @@ public:
     QModelIndex dittoRecord(int old_row);
 
     /// configure for browsing results of specified query
-    void setQuery(const QString& sQuery, bool dontClearHeaders = false);
+    void setQuery(const QString& sQuery, const QString& sCountQuery = QString(), bool dontClearHeaders = false);
 
     QString query() const { return m_sQuery; }
     QString customQuery(bool withRowid) const { return QString::fromStdString(m_query.buildQuery(withRowid)); }