Rework 89587a7d67

This partially reverts 89587a7d67 to fix
some issues it introduced. Using the knowledge we gained in the further
optimisation process, this commit now gets us the best of both worlds:
good performance when executing complex queries as well as a more
straightforward way to deal with the multithreaded nature of data
loading.

See issue #2537.

src/RowLoader.cpp

@@ -16,16 +16,14 @@ namespace {
 } // anon ns
 
 
-RowLoader::RowLoader (
-    std::function<std::shared_ptr<sqlite3>(void)> db_getter_,
+RowLoader::RowLoader (std::function<std::shared_ptr<sqlite3>(void)> db_getter_,
     std::function<void(QString)> statement_logger_,
     std::vector<std::string> & headers_,
-    std::vector<int>& data_types_,
     std::mutex & cache_mutex_,
     Cache & cache_data_
     )
     : db_getter(db_getter_), statement_logger(statement_logger_)
-    , headers(headers_), data_types(data_types_)
+    , headers(headers_)
     , cache_mutex(cache_mutex_), cache_data(cache_data_)
     , query()
     , countQuery()
@@ -229,18 +227,6 @@ void RowLoader::process (Task & t)
     auto row = t.row_begin;
     if(sqlite3_prepare_v2(pDb.get(), utf8Query, utf8Query.size(), &stmt, nullptr) == SQLITE_OK)
     {
-        // If the cache is uninitialised, retrieve the column names and types
-        const bool first_chunk = !cache_data.initialised();
-        if(first_chunk)
-        {
-            int num_columns = sqlite3_column_count(stmt);
-            for(int i=0;i<num_columns;++i)
-            {
-                headers.push_back(sqlite3_column_name(stmt, i));
-                data_types.push_back(sqlite3_column_type(stmt, i));
-            }
-        }
-
         while(!t.cancel && sqlite3_step(stmt) == SQLITE_ROW)
         {
             size_t num_columns = static_cast<size_t>(sqlite3_data_count(stmt));
@@ -269,8 +255,9 @@ void RowLoader::process (Task & t)
         // - this is the first batch of data we load for this query
         // - we got exactly the number of rows back we queried (which indicates there might be more rows)
         // If there is no need to query the row count this means the number of rows we just got is the total row count.
-        if(first_chunk)
+        if(!cache_data.initialised())
         {
+            cache_data.setInitialised();
             if(row == t.row_end)
                 triggerRowCountDetermination(t.token);
             else

src/RowLoader.h

@@ -30,7 +30,6 @@ public:
         std::function<std::shared_ptr<sqlite3>(void)> db_getter,
         std::function<void(QString)> statement_logger,
         std::vector<std::string> & headers,
-        std::vector<int>& data_types,
         std::mutex & cache_mutex,
         Cache & cache_data
         );
@@ -71,7 +70,6 @@ private:
     const std::function<std::shared_ptr<sqlite3>()> db_getter;
     const std::function<void(QString)> statement_logger;
     std::vector<std::string> & headers;
-    std::vector<int> & data_types;
     std::mutex & cache_mutex;
     Cache & cache_data;
 

src/TableBrowser.cpp

@@ -348,16 +348,6 @@ TableBrowser::TableBrowser(DBBrowserDB* _db, QWidget* parent) :
 
     // Connect slots
     connect(m_model, &SqliteTableModel::finishedFetch, this, &TableBrowser::fetchedData);
-    connect(m_model, &SqliteTableModel::columnsChanged, this, [this]() {
-        // Apply all settings
-        const sqlb::ObjectIdentifier tablename = currentlyBrowsedTableName();
-        const BrowseDataTableSettings& storedData = m_settings[tablename];
-
-        applyModelSettings(storedData);
-        applyViewportSettings(storedData, tablename);
-        updateRecordsetLabel();
-        emit updatePlot(ui->dataTable, m_model, &m_settings[tablename], true);
-    });
 
     // Load initial settings
     reloadSettings();
@@ -515,8 +505,10 @@ void TableBrowser::refresh()
     // Current table changed
     emit currentTableChanged(tablename);
 
-    // Set query which also resets the model
-    m_model->setQuery(buildQuery(storedData, tablename));
+    // Build query and apply settings
+   applyModelSettings(storedData, buildQuery(storedData, tablename));
+   applyViewportSettings(storedData, tablename);
+   updateRecordsetLabel();
 }
 
 void TableBrowser::clearFilters()
@@ -793,8 +785,11 @@ sqlb::Query TableBrowser::buildQuery(const BrowseDataTableSettings& storedData,
     return query;
 }
 
-void TableBrowser::applyModelSettings(const BrowseDataTableSettings& storedData)
+void TableBrowser::applyModelSettings(const BrowseDataTableSettings& storedData, const sqlb::Query& query)
 {
+    // Set query which also resets the model
+    m_model->setQuery(query);
+
     // Regular conditional formats
     for(auto formatIt=storedData.condFormats.cbegin(); formatIt!=storedData.condFormats.cend(); ++formatIt)
         m_model->setCondFormats(false, formatIt->first, formatIt->second);

src/TableBrowser.h

@@ -171,7 +171,7 @@ private:
     void modifyFormat(std::function<void(CondFormat&)> changeFunction);
 
     sqlb::Query buildQuery(const BrowseDataTableSettings& storedData, const sqlb::ObjectIdentifier& tablename) const;
-    void applyModelSettings(const BrowseDataTableSettings& storedData);
+    void applyModelSettings(const BrowseDataTableSettings& storedData, const sqlb::Query& query);
     void applyViewportSettings(const BrowseDataTableSettings& storedData, const sqlb::ObjectIdentifier& tablename);
     void generateFilters();
 };

src/sqlitetablemodel.cpp

@@ -35,7 +35,7 @@ SqliteTableModel::SqliteTableModel(DBBrowserDB& db, QObject* parent, const QStri
     worker = new RowLoader(
         [this, force_wait](){ return m_db.get(tr("reading rows"), force_wait); },
         [this](QString stmt){ return m_db.logSQL(stmt, kLogMsg_App); },
-        m_headers, m_vDataTypes, m_mutexDataCache, m_cache
+        m_headers, m_mutexDataCache, m_cache
         );
 
     worker->start();
@@ -67,18 +67,6 @@ void SqliteTableModel::handleFinishedFetch (int life_id, unsigned int fetched_ro
 
     Q_ASSERT(fetched_row_end >= fetched_row_begin);
 
-    // Tell the query object about the column names
-    m_query.setColumNames(m_headers);
-
-    // If the cache has been uninitialised so far we set it to initialised now and
-    // tell the view to update the table layout.
-    if(!m_cache.initialised())
-    {
-        m_cache.setInitialised();
-        emit layoutChanged();
-        emit columnsChanged();
-    }
-
     auto old_row_count = m_currentRowCount;
     auto new_row_count = std::max(old_row_count, fetched_row_begin);
     new_row_count = std::max(new_row_count, fetched_row_end);
@@ -140,20 +128,40 @@ void SqliteTableModel::setQuery(const sqlb::Query& query)
     m_query = query;
     m_table_of_query = m_db.getObjectByName<sqlb::Table>(query.table());
 
+    // The first column is the rowid column and therefore is always of type integer
+    m_vDataTypes.emplace_back(SQLITE_INTEGER);
+
+    // Get the data types of all other columns as well as the column names
     // Set the row id columns
     if(m_table_of_query && m_table_of_query->fields.size()) // It is a table and parsing was OK
     {
         sqlb::StringVector rowids = m_table_of_query->rowidColumns();
         m_query.setRowIdColumns(rowids);
+
+        m_headers.push_back(sqlb::joinStringVector(rowids, ","));
+
+        // Store field names and affinity data types
+        for(const sqlb::Field& fld : m_table_of_query->fields)
+        {
+            m_headers.push_back(fld.name());
+            m_vDataTypes.push_back(fld.affinity());
+        }
     } else {
         // If for one reason or another (either it's a view or we couldn't parse the table statement) we couldn't get the field
         // information we retrieve it from SQLite using an extra query.
         // NOTE: It would be nice to eventually get rid of this piece here. As soon as the grammar parser is good enough...
 
+        std::string sColumnQuery = "SELECT * FROM " + query.table().toString() + ";";
         if(m_query.rowIdColumns().empty())
             m_query.setRowIdColumn("_rowid_");
+        m_headers.emplace_back("_rowid_");
+                auto columns = getColumns(nullptr, sColumnQuery, m_vDataTypes);
+                m_headers.insert(m_headers.end(), columns.begin(), columns.end());
     }
 
+    // Tell the query object about the column names
+    m_query.setColumNames(m_headers);
+
     // Apply new query and update view
     buildQuery();
 }
@@ -174,8 +182,16 @@ void SqliteTableModel::setQuery(const QString& sQuery, const QString& sCountQuer
 
     worker->setQuery(m_sQuery, sCountQuery);
 
+    if(!dontClearHeaders)
+    {
+        auto columns = getColumns(worker->getDb(), sQuery.toStdString(), m_vDataTypes);
+        m_headers.insert(m_headers.end(), columns.begin(), columns.end());
+    }
+
     // now fetch the first entries
     triggerCacheLoad(static_cast<int>(m_chunkSize / 2) - 1);
+
+    emit layoutChanged();
 }
 
 int SqliteTableModel::rowCount(const QModelIndex&) const
@@ -776,6 +792,27 @@ void SqliteTableModel::removeCommentsFromQuery(QString& query)
     }
 }
 
+std::vector<std::string> SqliteTableModel::getColumns(std::shared_ptr<sqlite3> pDb, const std::string& sQuery, std::vector<int>& fieldsTypes) const
+{
+    if(!pDb)
+        pDb = m_db.get(tr("retrieving list of columns"));
+
+    sqlite3_stmt* stmt;
+    std::vector<std::string> listColumns;
+    if(sqlite3_prepare_v2(pDb.get(), sQuery.c_str(), static_cast<int>(sQuery.size()), &stmt, nullptr) == SQLITE_OK)
+    {
+        int columns = sqlite3_column_count(stmt);
+        for(int i = 0; i < columns; ++i)
+        {
+            listColumns.push_back(sqlite3_column_name(stmt, i));
+            fieldsTypes.push_back(sqlite3_column_type(stmt, i));
+        }
+    }
+    sqlite3_finalize(stmt);
+
+    return listColumns;
+}
+
 void addCondFormatToMap(std::map<size_t, std::vector<CondFormat>>& mCondFormats, size_t column, const CondFormat& condFormat)
 {
     // If the condition is already present in the vector, update that entry and respect the order, since two entries with the same

src/sqlitetablemodel.h

@@ -149,7 +149,6 @@ public slots:
 signals:
     void finishedFetch(int fetched_row_begin, int fetched_row_end);
     void finishedRowCount();
-    void columnsChanged();
 
 protected:
     Qt::DropActions supportedDropActions() const override;
@@ -168,6 +167,9 @@ private:
 
     void buildQuery();
 
+    /// \param pDb connection to query; if null, obtains it from 'm_db'.
+    std::vector<std::string> getColumns(std::shared_ptr<sqlite3> pDb, const std::string& sQuery, std::vector<int>& fieldsTypes) const;
+
     QByteArray encode(const QByteArray& str) const;
     QByteArray decode(const QByteArray& str) const;