From 89587a7d67928388d57a092817816b4971cf6e4b Mon Sep 17 00:00:00 2001 From: Martin Kleusberg Date: Tue, 29 Dec 2020 20:51:04 +0100 Subject: [PATCH] Speed up executing SQL queries This improves the performance of running SQL queries in the SqliteTableModel class by avoiding an extra query for figuring out the column names and data types of the returned data. See issue #2165. --- src/RowLoader.cpp | 21 ++++++++++++-- src/RowLoader.h | 2 ++ src/TableBrowser.cpp | 20 ++++++++----- src/TableBrowser.h | 2 +- src/sqlitetablemodel.cpp | 63 +++++++--------------------------------- src/sqlitetablemodel.h | 4 +-- 6 files changed, 46 insertions(+), 66 deletions(-) diff --git a/src/RowLoader.cpp b/src/RowLoader.cpp index e7d4b2cf..02b158cd 100644 --- a/src/RowLoader.cpp +++ b/src/RowLoader.cpp @@ -20,10 +20,12 @@ RowLoader::RowLoader ( std::function(void)> db_getter_, std::function statement_logger_, std::vector & headers_, + std::vector& data_types_, std::mutex & cache_mutex_, Cache & cache_data_ ) - : db_getter(db_getter_), statement_logger(statement_logger_), headers(headers_) + : db_getter(db_getter_), statement_logger(statement_logger_) + , headers(headers_), data_types(data_types_) , cache_mutex(cache_mutex_), cache_data(cache_data_) , query() , countQuery() @@ -227,10 +229,25 @@ void RowLoader::process (Task & t) auto row = t.row_begin; if(sqlite3_prepare_v2(pDb.get(), utf8Query, utf8Query.size(), &stmt, nullptr) == SQLITE_OK) { - const size_t num_columns = headers.size(); + size_t num_columns = 0; + + bool first_row = true; while(!t.cancel && sqlite3_step(stmt) == SQLITE_ROW) { + // For the first row, determine the column names and types + if(first_row) + { + num_columns = static_cast(sqlite3_data_count(stmt)); + for(size_t i=0;i(i))); + data_types.push_back(sqlite3_column_type(stmt, static_cast(i))); + } + + first_row = false; + } + // Construct a new row object with the right number of columns Cache::value_type rowdata(num_columns); for(size_t i=0;i(void)> db_getter, std::function statement_logger, std::vector & headers, + std::vector& data_types, std::mutex & cache_mutex, Cache & cache_data ); @@ -70,6 +71,7 @@ private: const std::function()> db_getter; const std::function statement_logger; std::vector & headers; + std::vector & data_types; std::mutex & cache_mutex; Cache & cache_data; diff --git a/src/TableBrowser.cpp b/src/TableBrowser.cpp index 0a27e429..09f18266 100644 --- a/src/TableBrowser.cpp +++ b/src/TableBrowser.cpp @@ -348,6 +348,15 @@ TableBrowser::TableBrowser(DBBrowserDB* _db, QWidget* parent) : // Connect slots connect(m_model, &SqliteTableModel::finishedFetch, this, &TableBrowser::fetchedData); + connect(m_model, &SqliteTableModel::columnsChanged, this, [this]() { + // Apply all settings + const sqlb::ObjectIdentifier tablename = currentlyBrowsedTableName(); + const BrowseDataTableSettings& storedData = m_settings[tablename]; + + applyModelSettings(storedData); + applyViewportSettings(storedData, tablename); + updateRecordsetLabel(); + }); // Load initial settings reloadSettings(); @@ -505,10 +514,8 @@ void TableBrowser::refresh() // Current table changed emit currentTableChanged(tablename); - // Build query and apply settings - applyModelSettings(storedData, buildQuery(storedData, tablename)); - applyViewportSettings(storedData, tablename); - updateRecordsetLabel(); + // Set query which also resets the model + m_model->setQuery(buildQuery(storedData, tablename)); } void TableBrowser::clearFilters() @@ -785,11 +792,8 @@ sqlb::Query TableBrowser::buildQuery(const BrowseDataTableSettings& storedData, return query; } -void TableBrowser::applyModelSettings(const BrowseDataTableSettings& storedData, const sqlb::Query& query) +void TableBrowser::applyModelSettings(const BrowseDataTableSettings& storedData) { - // Set query which also resets the model - m_model->setQuery(query); - // Regular conditional formats for(auto formatIt=storedData.condFormats.cbegin(); formatIt!=storedData.condFormats.cend(); ++formatIt) m_model->setCondFormats(false, formatIt->first, formatIt->second); diff --git a/src/TableBrowser.h b/src/TableBrowser.h index daebe4f3..2d973254 100644 --- a/src/TableBrowser.h +++ b/src/TableBrowser.h @@ -171,7 +171,7 @@ private: void modifyFormat(std::function changeFunction); sqlb::Query buildQuery(const BrowseDataTableSettings& storedData, const sqlb::ObjectIdentifier& tablename) const; - void applyModelSettings(const BrowseDataTableSettings& storedData, const sqlb::Query& query); + void applyModelSettings(const BrowseDataTableSettings& storedData); void applyViewportSettings(const BrowseDataTableSettings& storedData, const sqlb::ObjectIdentifier& tablename); void generateFilters(); }; diff --git a/src/sqlitetablemodel.cpp b/src/sqlitetablemodel.cpp index 38cd4122..2c56dbcc 100644 --- a/src/sqlitetablemodel.cpp +++ b/src/sqlitetablemodel.cpp @@ -35,7 +35,7 @@ SqliteTableModel::SqliteTableModel(DBBrowserDB& db, QObject* parent, const QStri worker = new RowLoader( [this, force_wait](){ return m_db.get(tr("reading rows"), force_wait); }, [this](QString stmt){ return m_db.logSQL(stmt, kLogMsg_App); }, - m_headers, m_mutexDataCache, m_cache + m_headers, m_vDataTypes, m_mutexDataCache, m_cache ); worker->start(); @@ -67,6 +67,15 @@ void SqliteTableModel::handleFinishedFetch (int life_id, unsigned int fetched_ro Q_ASSERT(fetched_row_end >= fetched_row_begin); + // Tell the query object about the column names. We also use this property to determine if the number of columns changed for some + // reason and if so, we tell the view to update the table layout. + if(m_query.columnNames().size() != m_headers.size()) + { + emit layoutChanged(); + emit columnsChanged(); + } + m_query.setColumNames(m_headers); + auto old_row_count = m_currentRowCount; auto new_row_count = std::max(old_row_count, fetched_row_begin); @@ -129,38 +138,20 @@ void SqliteTableModel::setQuery(const sqlb::Query& query) m_query = query; m_table_of_query = m_db.getObjectByName(query.table()); - // The first column is the rowid column and therefore is always of type integer - m_vDataTypes.emplace_back(SQLITE_INTEGER); - - // Get the data types of all other columns as well as the column names + // Set the row id columns if(m_table_of_query && m_table_of_query->fields.size()) // It is a table and parsing was OK { sqlb::StringVector rowids = m_table_of_query->rowidColumns(); m_query.setRowIdColumns(rowids); - m_headers.push_back(sqlb::joinStringVector(rowids, ",")); - - // Store field names and affinity data types - for(const sqlb::Field& fld : m_table_of_query->fields) - { - m_headers.push_back(fld.name()); - m_vDataTypes.push_back(fld.affinity()); - } } else { // If for one reason or another (either it's a view or we couldn't parse the table statement) we couldn't get the field // information we retrieve it from SQLite using an extra query. // NOTE: It would be nice to eventually get rid of this piece here. As soon as the grammar parser is good enough... - std::string sColumnQuery = "SELECT * FROM " + query.table().toString() + ";"; if(m_query.rowIdColumns().empty()) m_query.setRowIdColumn("_rowid_"); - m_headers.emplace_back("_rowid_"); - auto columns = getColumns(nullptr, sColumnQuery, m_vDataTypes); - m_headers.insert(m_headers.end(), columns.begin(), columns.end()); } - // Tell the query object about the column names - m_query.setColumNames(m_headers); - // Apply new query and update view buildQuery(); } @@ -182,16 +173,8 @@ void SqliteTableModel::setQuery(const QString& sQuery, const QString& sCountQuer worker->setQuery(m_sQuery, sCountQuery); worker->triggerRowCountDetermination(m_lifeCounter); - if(!dontClearHeaders) - { - auto columns = getColumns(worker->getDb(), sQuery.toStdString(), m_vDataTypes); - m_headers.insert(m_headers.end(), columns.begin(), columns.end()); - } - // now fetch the first entries triggerCacheLoad(static_cast(m_chunkSize / 2) - 1); - - emit layoutChanged(); } int SqliteTableModel::rowCount(const QModelIndex&) const @@ -792,30 +775,6 @@ void SqliteTableModel::removeCommentsFromQuery(QString& query) } } -std::vector SqliteTableModel::getColumns(std::shared_ptr pDb, const std::string& sQuery, std::vector& fieldsTypes) const -{ - if(!pDb) - pDb = m_db.get(tr("retrieving list of columns")); - - sqlite3_stmt* stmt; - std::vector listColumns; - if(sqlite3_prepare_v2(pDb.get(), sQuery.c_str(), static_cast(sQuery.size()), &stmt, nullptr) == SQLITE_OK) - { - if(sqlite3_step(stmt) == SQLITE_ROW) - { - int columns = sqlite3_data_count(stmt); - for(int i = 0; i < columns; ++i) - { - listColumns.push_back(sqlite3_column_name(stmt, i)); - fieldsTypes.push_back(sqlite3_column_type(stmt, i)); - } - } - } - sqlite3_finalize(stmt); - - return listColumns; -} - void addCondFormatToMap(std::map>& mCondFormats, size_t column, const CondFormat& condFormat) { // If the condition is already present in the vector, update that entry and respect the order, since two entries with the same diff --git a/src/sqlitetablemodel.h b/src/sqlitetablemodel.h index 4ecbd7d9..8b56100e 100644 --- a/src/sqlitetablemodel.h +++ b/src/sqlitetablemodel.h @@ -149,6 +149,7 @@ public slots: signals: void finishedFetch(int fetched_row_begin, int fetched_row_end); void finishedRowCount(); + void columnsChanged(); protected: Qt::DropActions supportedDropActions() const override; @@ -167,9 +168,6 @@ private: void buildQuery(); - /// \param pDb connection to query; if null, obtains it from 'm_db'. - std::vector getColumns(std::shared_ptr pDb, const std::string& sQuery, std::vector& fieldsTypes) const; - QByteArray encode(const QByteArray& str) const; QByteArray decode(const QByteArray& str) const;