Speed up executing SQL queries

This improves the performance of running SQL queries in the
SqliteTableModel class by avoiding an extra query for figuring out the
column names and data types of the returned data.

See issue #2165.

src/RowLoader.cpp

@@ -20,10 +20,12 @@ RowLoader::RowLoader (
     std::function<std::shared_ptr<sqlite3>(void)> db_getter_,
     std::function<void(QString)> statement_logger_,
     std::vector<std::string> & headers_,
+    std::vector<int>& data_types_,
     std::mutex & cache_mutex_,
     Cache & cache_data_
     )
-    : db_getter(db_getter_), statement_logger(statement_logger_), headers(headers_)
+    : db_getter(db_getter_), statement_logger(statement_logger_)
+    , headers(headers_), data_types(data_types_)
     , cache_mutex(cache_mutex_), cache_data(cache_data_)
     , query()
     , countQuery()
@@ -227,10 +229,25 @@ void RowLoader::process (Task & t)
     auto row = t.row_begin;
     if(sqlite3_prepare_v2(pDb.get(), utf8Query, utf8Query.size(), &stmt, nullptr) == SQLITE_OK)
     {
-        const size_t num_columns = headers.size();
+        size_t num_columns = 0;
+
+        bool first_row = true;
 
         while(!t.cancel && sqlite3_step(stmt) == SQLITE_ROW)
         {
+            // For the first row, determine the column names and types
+            if(first_row)
+            {
+                num_columns = static_cast<size_t>(sqlite3_data_count(stmt));
+                for(size_t i=0;i<num_columns;++i)
+                {
+                    headers.push_back(sqlite3_column_name(stmt, static_cast<int>(i)));
+                    data_types.push_back(sqlite3_column_type(stmt, static_cast<int>(i)));
+                }
+
+                first_row = false;
+            }
+
             // Construct a new row object with the right number of columns
             Cache::value_type rowdata(num_columns);
             for(size_t i=0;i<num_columns;++i)

src/RowLoader.h

@@ -30,6 +30,7 @@ public:
         std::function<std::shared_ptr<sqlite3>(void)> db_getter,
         std::function<void(QString)> statement_logger,
         std::vector<std::string> & headers,
+        std::vector<int>& data_types,
         std::mutex & cache_mutex,
         Cache & cache_data
         );
@@ -70,6 +71,7 @@ private:
     const std::function<std::shared_ptr<sqlite3>()> db_getter;
     const std::function<void(QString)> statement_logger;
     std::vector<std::string> & headers;
+    std::vector<int> & data_types;
     std::mutex & cache_mutex;
     Cache & cache_data;
 

src/TableBrowser.cpp

@@ -348,6 +348,15 @@ TableBrowser::TableBrowser(DBBrowserDB* _db, QWidget* parent) :
 
     // Connect slots
     connect(m_model, &SqliteTableModel::finishedFetch, this, &TableBrowser::fetchedData);
+    connect(m_model, &SqliteTableModel::columnsChanged, this, [this]() {
+        // Apply all settings
+        const sqlb::ObjectIdentifier tablename = currentlyBrowsedTableName();
+        const BrowseDataTableSettings& storedData = m_settings[tablename];
+
+        applyModelSettings(storedData);
+        applyViewportSettings(storedData, tablename);
+        updateRecordsetLabel();
+    });
 
     // Load initial settings
     reloadSettings();
@@ -505,10 +514,8 @@ void TableBrowser::refresh()
     // Current table changed
     emit currentTableChanged(tablename);
 
-    // Build query and apply settings
-    applyModelSettings(storedData, buildQuery(storedData, tablename));
-    applyViewportSettings(storedData, tablename);
-    updateRecordsetLabel();
+    // Set query which also resets the model
+    m_model->setQuery(buildQuery(storedData, tablename));
 }
 
 void TableBrowser::clearFilters()
@@ -785,11 +792,8 @@ sqlb::Query TableBrowser::buildQuery(const BrowseDataTableSettings& storedData,
     return query;
 }
 
-void TableBrowser::applyModelSettings(const BrowseDataTableSettings& storedData, const sqlb::Query& query)
+void TableBrowser::applyModelSettings(const BrowseDataTableSettings& storedData)
 {
-    // Set query which also resets the model
-    m_model->setQuery(query);
-
     // Regular conditional formats
     for(auto formatIt=storedData.condFormats.cbegin(); formatIt!=storedData.condFormats.cend(); ++formatIt)
         m_model->setCondFormats(false, formatIt->first, formatIt->second);

src/TableBrowser.h

@@ -171,7 +171,7 @@ private:
     void modifyFormat(std::function<void(CondFormat&)> changeFunction);
 
     sqlb::Query buildQuery(const BrowseDataTableSettings& storedData, const sqlb::ObjectIdentifier& tablename) const;
-    void applyModelSettings(const BrowseDataTableSettings& storedData, const sqlb::Query& query);
+    void applyModelSettings(const BrowseDataTableSettings& storedData);
     void applyViewportSettings(const BrowseDataTableSettings& storedData, const sqlb::ObjectIdentifier& tablename);
     void generateFilters();
 };

src/sqlitetablemodel.cpp

@@ -35,7 +35,7 @@ SqliteTableModel::SqliteTableModel(DBBrowserDB& db, QObject* parent, const QStri
     worker = new RowLoader(
         [this, force_wait](){ return m_db.get(tr("reading rows"), force_wait); },
         [this](QString stmt){ return m_db.logSQL(stmt, kLogMsg_App); },
-        m_headers, m_mutexDataCache, m_cache
+        m_headers, m_vDataTypes, m_mutexDataCache, m_cache
         );
 
     worker->start();
@@ -67,6 +67,15 @@ void SqliteTableModel::handleFinishedFetch (int life_id, unsigned int fetched_ro
 
     Q_ASSERT(fetched_row_end >= fetched_row_begin);
 
+    // Tell the query object about the column names. We also use this property to determine if the number of columns changed for some
+    // reason and if so, we tell the view to update the table layout.
+    if(m_query.columnNames().size() != m_headers.size())
+    {
+        emit layoutChanged();
+        emit columnsChanged();
+    }
+    m_query.setColumNames(m_headers);
+
     auto old_row_count = m_currentRowCount;
 
     auto new_row_count = std::max(old_row_count, fetched_row_begin);
@@ -129,38 +138,20 @@ void SqliteTableModel::setQuery(const sqlb::Query& query)
     m_query = query;
     m_table_of_query = m_db.getObjectByName<sqlb::Table>(query.table());
 
-    // The first column is the rowid column and therefore is always of type integer
-    m_vDataTypes.emplace_back(SQLITE_INTEGER);
-
-    // Get the data types of all other columns as well as the column names
+    // Set the row id columns
     if(m_table_of_query && m_table_of_query->fields.size()) // It is a table and parsing was OK
     {
         sqlb::StringVector rowids = m_table_of_query->rowidColumns();
         m_query.setRowIdColumns(rowids);
-        m_headers.push_back(sqlb::joinStringVector(rowids, ","));
-
-        // Store field names and affinity data types
-        for(const sqlb::Field& fld : m_table_of_query->fields)
-        {
-            m_headers.push_back(fld.name());
-            m_vDataTypes.push_back(fld.affinity());
-        }
     } else {
         // If for one reason or another (either it's a view or we couldn't parse the table statement) we couldn't get the field
         // information we retrieve it from SQLite using an extra query.
         // NOTE: It would be nice to eventually get rid of this piece here. As soon as the grammar parser is good enough...
 
-        std::string sColumnQuery = "SELECT * FROM " + query.table().toString() + ";";
         if(m_query.rowIdColumns().empty())
             m_query.setRowIdColumn("_rowid_");
-        m_headers.emplace_back("_rowid_");
-        auto columns = getColumns(nullptr, sColumnQuery, m_vDataTypes);
-        m_headers.insert(m_headers.end(), columns.begin(), columns.end());
     }
 
-    // Tell the query object about the column names
-    m_query.setColumNames(m_headers);
-
     // Apply new query and update view
     buildQuery();
 }
@@ -182,16 +173,8 @@ void SqliteTableModel::setQuery(const QString& sQuery, const QString& sCountQuer
     worker->setQuery(m_sQuery, sCountQuery);
     worker->triggerRowCountDetermination(m_lifeCounter);
 
-    if(!dontClearHeaders)
-    {
-        auto columns = getColumns(worker->getDb(), sQuery.toStdString(), m_vDataTypes);
-        m_headers.insert(m_headers.end(), columns.begin(), columns.end());
-    }
-
     // now fetch the first entries
     triggerCacheLoad(static_cast<int>(m_chunkSize / 2) - 1);
-
-    emit layoutChanged();
 }
 
 int SqliteTableModel::rowCount(const QModelIndex&) const
@@ -792,30 +775,6 @@ void SqliteTableModel::removeCommentsFromQuery(QString& query)
     }
 }
 
-std::vector<std::string> SqliteTableModel::getColumns(std::shared_ptr<sqlite3> pDb, const std::string& sQuery, std::vector<int>& fieldsTypes) const
-{
-    if(!pDb)
-        pDb = m_db.get(tr("retrieving list of columns"));
-
-    sqlite3_stmt* stmt;
-    std::vector<std::string> listColumns;
-    if(sqlite3_prepare_v2(pDb.get(), sQuery.c_str(), static_cast<int>(sQuery.size()), &stmt, nullptr) == SQLITE_OK)
-    {
-        if(sqlite3_step(stmt) == SQLITE_ROW)
-        {
-            int columns = sqlite3_data_count(stmt);
-            for(int i = 0; i < columns; ++i)
-            {
-                listColumns.push_back(sqlite3_column_name(stmt, i));
-                fieldsTypes.push_back(sqlite3_column_type(stmt, i));
-            }
-        }
-    }
-    sqlite3_finalize(stmt);
-
-    return listColumns;
-}
-
 void addCondFormatToMap(std::map<size_t, std::vector<CondFormat>>& mCondFormats, size_t column, const CondFormat& condFormat)
 {
     // If the condition is already present in the vector, update that entry and respect the order, since two entries with the same

src/sqlitetablemodel.h

@@ -149,6 +149,7 @@ public slots:
 signals:
     void finishedFetch(int fetched_row_begin, int fetched_row_end);
     void finishedRowCount();
+    void columnsChanged();
 
 protected:
     Qt::DropActions supportedDropActions() const override;
@@ -167,9 +168,6 @@ private:
 
     void buildQuery();
 
-    /// \param pDb connection to query; if null, obtains it from 'm_db'.
-    std::vector<std::string> getColumns(std::shared_ptr<sqlite3> pDb, const std::string& sQuery, std::vector<int>& fieldsTypes) const;
-
     QByteArray encode(const QByteArray& str) const;
     QByteArray decode(const QByteArray& str) const;