From 51dbe72e23bd13cc1881fe6a6fa7938937f1510e Mon Sep 17 00:00:00 2001 From: Martin Kleusberg Date: Mon, 21 May 2018 18:13:56 +0200 Subject: [PATCH] Multi-threading patch This was done by Michael Krause. https://lists.sqlitebrowser.org/pipermail/db4s-dev/2018-February/000305.html In this commit I only fixed two compiler warnings, some whitespace issues and removed some debug messages. --- CMakeLists.txt | 6 + src/DbStructureModel.cpp | 2 +- src/EditTableDialog.cpp | 8 +- src/ExportDataDialog.cpp | 7 +- src/ExtendedTableWidget.cpp | 26 +- src/ImportCsvDialog.cpp | 3 +- src/MainWindow.cpp | 91 +++++-- src/MainWindow.h | 6 +- src/PlotDock.cpp | 26 +- src/RowCache.h | 264 +++++++++++++++++++ src/RowLoader.cpp | 264 +++++++++++++++++++ src/RowLoader.h | 121 +++++++++ src/sqlitedb.cpp | 101 +++++++- src/sqlitedb.h | 78 +++++- src/sqlitetablemodel.cpp | 490 ++++++++++++++++++----------------- src/sqlitetablemodel.h | 124 ++++++--- src/src.pro | 3 + src/tests/CMakeLists.txt | 11 +- src/tests/test_row_cache.cpp | 179 +++++++++++++ 19 files changed, 1466 insertions(+), 344 deletions(-) create mode 100644 src/RowCache.h create mode 100644 src/RowLoader.cpp create mode 100644 src/RowLoader.h create mode 100644 src/tests/test_row_cache.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 485e09c6..17696e00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,6 +78,8 @@ endif() add_subdirectory(${QHEXEDIT_DIR}) add_subdirectory(${QCUSTOMPLOT_DIR}) +add_subdirectory(/home/michael/dev/gtest/git/googletest /home/michael/dev/gtest/build) + find_package(Qt5Widgets REQUIRED) find_package(Qt5LinguistTools REQUIRED) find_package(Qt5Network REQUIRED) @@ -117,6 +119,8 @@ set(SQLB_MOC_HDR src/SqlExecutionArea.h src/VacuumDialog.h src/sqlitetablemodel.h + src/RowLoader.h + src/RowCache.h src/sqltextedit.h src/docktextedit.h src/DbStructureModel.h @@ -154,6 +158,7 @@ set(SQLB_SRC src/VacuumDialog.cpp src/sqlitedb.cpp src/sqlitetablemodel.cpp + src/RowLoader.cpp src/sqlitetypes.cpp src/sqltextedit.cpp src/docktextedit.cpp @@ -371,6 +376,7 @@ endif() target_link_libraries(${PROJECT_NAME} qhexedit qcustomplot + pthread ${QT_LIBRARIES} ${WIN32_STATIC_LINK} ${LIBSQLITE} diff --git a/src/DbStructureModel.cpp b/src/DbStructureModel.cpp index 040eed3a..305297be 100644 --- a/src/DbStructureModel.cpp +++ b/src/DbStructureModel.cpp @@ -219,7 +219,7 @@ QMimeData* DbStructureModel::mimeData(const QModelIndexList& indices) const sqlb::ObjectIdentifier objid(data(index.sibling(index.row(), ColumnSchema), Qt::DisplayRole).toString(), data(index.sibling(index.row(), ColumnName), Qt::DisplayRole).toString()); tableModel.setTable(objid); - tableModel.waitForFetchingFinished(); + tableModel.completeCache(); for(int i=0; i < tableModel.rowCount(); ++i) { QString insertStatement = "INSERT INTO " + objid.toString() + " VALUES("; diff --git a/src/EditTableDialog.cpp b/src/EditTableDialog.cpp index e7e04361..8512fa2e 100644 --- a/src/EditTableDialog.cpp +++ b/src/EditTableDialog.cpp @@ -388,7 +388,7 @@ void EditTableDialog::itemChanged(QTreeWidgetItem *item, int column) .arg(sqlb::escapeIdentifier(pdb.getObjectByName(curTable).dynamicCast()->rowidColumn())) .arg(curTable.toString()) .arg(sqlb::escapeIdentifier(field->name()))); - m.waitForFetchingFinished(); + m.completeCache(); if(m.data(m.index(0, 0)).toInt() > 0) { // There is a NULL value, so print an error message, uncheck the combobox, and return here @@ -416,7 +416,7 @@ void EditTableDialog::itemChanged(QTreeWidgetItem *item, int column) .arg(curTable.toString()) .arg(sqlb::escapeIdentifier(field->name())) .arg(sqlb::escapeIdentifier(field->name()))); - m.waitForFetchingFinished(); + m.completeCache(); if(m.data(m.index(0, 0)).toInt() > 0) { // There is a non-integer value, so print an error message, uncheck the combobox, and return here @@ -458,10 +458,10 @@ void EditTableDialog::itemChanged(QTreeWidgetItem *item, int column) // Because our renameColumn() function fails when setting a column to unique when it already contains the same values SqliteTableModel m(pdb, this); m.setQuery(QString("SELECT COUNT(%2) FROM %1;").arg(curTable.toString()).arg(sqlb::escapeIdentifier(field->name()))); - m.waitForFetchingFinished(); + m.completeCache(); int rowcount = m.data(m.index(0, 0)).toInt(); m.setQuery(QString("SELECT COUNT(DISTINCT %2) FROM %1;").arg(curTable.toString()).arg(sqlb::escapeIdentifier(field->name()))); - m.waitForFetchingFinished(); + m.completeCache(); int uniquecount = m.data(m.index(0, 0)).toInt(); if(rowcount != uniquecount) { diff --git a/src/ExportDataDialog.cpp b/src/ExportDataDialog.cpp index 8580da03..f2f21420 100644 --- a/src/ExportDataDialog.cpp +++ b/src/ExportDataDialog.cpp @@ -117,7 +117,8 @@ bool ExportDataDialog::exportQueryCsv(const QString& sQuery, const QString& sFil QByteArray utf8Query = sQuery.toUtf8(); sqlite3_stmt *stmt; - int status = sqlite3_prepare_v2(pdb._db, utf8Query.data(), utf8Query.size(), &stmt, nullptr); + auto pDb = pdb.get("exporting CSV"); + int status = sqlite3_prepare_v2(pDb.get(), utf8Query.data(), utf8Query.size(), &stmt, nullptr); if(SQLITE_OK == status) { if(ui->checkHeader->isChecked()) @@ -199,7 +200,9 @@ bool ExportDataDialog::exportQueryJson(const QString& sQuery, const QString& sFi { QByteArray utf8Query = sQuery.toUtf8(); sqlite3_stmt *stmt; - int status = sqlite3_prepare_v2(pdb._db, utf8Query.data(), utf8Query.size(), &stmt, nullptr); + + auto pDb = pdb.get("exporting JSON"); + int status = sqlite3_prepare_v2(pDb.get(), utf8Query.data(), utf8Query.size(), &stmt, nullptr); QJsonArray json_table; diff --git a/src/ExtendedTableWidget.cpp b/src/ExtendedTableWidget.cpp index d26e73e5..c6b0a9bd 100644 --- a/src/ExtendedTableWidget.cpp +++ b/src/ExtendedTableWidget.cpp @@ -1,3 +1,5 @@ +#include + #include "ExtendedTableWidget.h" #include "sqlitetablemodel.h" #include "FilterTableHeader.h" @@ -604,19 +606,29 @@ void ExtendedTableWidget::updateGeometries() // If so and if it is a SqliteTableModel and if the parent implementation of this method decided that a scrollbar is needed, update its maximum value SqliteTableModel* m = qobject_cast(model()); if(m && verticalScrollBar()->maximum()) - verticalScrollBar()->setMaximum(m->totalRowCount() - numVisibleRows() + 1); + verticalScrollBar()->setMaximum(m->rowCount() - numVisibleRows() + 1); } } void ExtendedTableWidget::vscrollbarChanged(int value) { + //std::cout << "ExtendedTableWidget::vscrollbarChanged " << value << std::endl; + // Cancel if there is no model set yet - this shouldn't happen (because without a model there should be no scrollbar) but just to be sure... if(!model()) return; // Fetch more data from the DB if necessary - if((value + numVisibleRows()) >= model()->rowCount() && model()->canFetchMore(QModelIndex())) - model()->fetchMore(QModelIndex()); + const auto nrows = model()->rowCount(); + if(nrows == 0) + return; + + if(auto * m = dynamic_cast(model())) + { + int row_begin = std::min(value, nrows - 1); + int row_end = std::min(value + numVisibleRows(), nrows); + m->triggerCacheLoad(row_begin, row_end); + } } int ExtendedTableWidget::numVisibleRows() @@ -682,13 +694,11 @@ void ExtendedTableWidget::selectTableLine(int lineToSelect) SqliteTableModel* m = qobject_cast(model()); // Are there even that many lines? - if(lineToSelect >= m->totalRowCount()) + if(lineToSelect >= m->rowCount()) return; QApplication::setOverrideCursor( Qt::WaitCursor ); - // Make sure this line has already been fetched - while(lineToSelect >= m->rowCount() && m->canFetchMore()) - m->fetchMore(); + m->triggerCacheLoad(lineToSelect); // Select it clearSelection(); @@ -703,7 +713,7 @@ void ExtendedTableWidget::selectTableLines(int firstLine, int count) int lastLine = firstLine+count-1; // Are there even that many lines? - if(lastLine >= m->totalRowCount()) + if(lastLine >= m->rowCount()) return; selectTableLine(firstLine); diff --git a/src/ImportCsvDialog.cpp b/src/ImportCsvDialog.cpp index daa5ad3d..9d5fdc92 100644 --- a/src/ImportCsvDialog.cpp +++ b/src/ImportCsvDialog.cpp @@ -607,7 +607,8 @@ bool ImportCsvDialog::importCsv(const QString& fileName, const QString& name) sQuery.chop(1); // Remove last comma sQuery.append(")"); sqlite3_stmt* stmt; - sqlite3_prepare_v2(pdb->_db, sQuery.toUtf8(), sQuery.toUtf8().length(), &stmt, nullptr); + auto pDb = pdb->get("importing CSV"); + sqlite3_prepare_v2(pDb.get(), sQuery.toUtf8(), sQuery.toUtf8().length(), &stmt, nullptr); // Parse entire file size_t lastRowNum = 0; diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp index 4dc5d71c..4c5eac0d 100644 --- a/src/MainWindow.cpp +++ b/src/MainWindow.cpp @@ -1,3 +1,5 @@ +#include + #include "MainWindow.h" #include "ui_MainWindow.h" @@ -57,6 +59,7 @@ MainWindow::MainWindow(QWidget* parent) : QMainWindow(parent), ui(new Ui::MainWindow), + db(), m_browseTableModel(new SqliteTableModel(db, this, Settings::getValue("db", "prefetchsize").toInt())), m_currentTabTableModel(m_browseTableModel), m_remoteDb(new RemoteDatabase), @@ -292,6 +295,11 @@ void MainWindow::init() connect(m_browseTableModel, &SqliteTableModel::finishedFetch, this, &MainWindow::setRecordsetLabel); connect(ui->dataTable, &ExtendedTableWidget::selectedRowsToBeDeleted, this, &MainWindow::deleteRecord); + connect(m_browseTableModel, &SqliteTableModel::finishedFetch, [this](){ + auto & settings = browseTableSettings[currentlyBrowsedTableName()]; + plotDock->updatePlot(m_browseTableModel, &settings, true, true); + }); + // Lambda function for keyboard shortcuts for selecting next/previous table in Browse Data tab connect(ui->dataTable, &ExtendedTableWidget::switchTable, [this](bool next) { int index = ui->comboBrowseTable->currentIndex(); @@ -505,13 +513,18 @@ void MainWindow::populateTable() { connect(ui->dataTable->selectionModel(), SIGNAL(currentChanged(QModelIndex,QModelIndex)), this, SLOT(dataTableSelectionChanged(QModelIndex))); - // Lambda function for updating the delete record button to reflect number of selected records + // Lambda function for updating the delete record button to reflect number of selected records. -- TODO actual + // call this once to disable Delete button... connect(ui->dataTable->selectionModel(), &QItemSelectionModel::selectionChanged, [this](const QItemSelection&, const QItemSelection&) { - // NOTE: We're assuming here that the selection is always contiguous, i.e. that there are never two selected rows with a non-selected - // row in between. + // NOTE: We're assuming here that the selection is always contiguous, i.e. that there are never two selected + // rows with a non-selected row in between. int rows = 0; - if(ui->dataTable->selectionModel()->selectedIndexes().count()) - rows = ui->dataTable->selectionModel()->selectedIndexes().last().row() - ui->dataTable->selectionModel()->selectedIndexes().first().row() + 1; + + const auto & sel = ui->dataTable->selectionModel()->selectedIndexes(); + if(sel.count()) + rows = sel.last().row() - sel.first().row() + 1; + + ui->buttonDeleteRecord->setEnabled(rows != 0); if(rows > 1) ui->buttonDeleteRecord->setText(tr("Delete records")); @@ -552,6 +565,8 @@ void MainWindow::populateTable() // Encoding m_browseTableModel->setEncoding(defaultBrowseTableEncoding); + setRecordsetLabel(); + // Plot attachPlot(ui->dataTable, m_browseTableModel, &browseTableSettings[tablename]); @@ -609,6 +624,8 @@ void MainWindow::populateTable() // Encoding m_browseTableModel->setEncoding(storedData.encoding); + setRecordsetLabel(); + // Plot attachPlot(ui->dataTable, m_browseTableModel, &browseTableSettings[tablename], false); } @@ -710,8 +727,8 @@ void MainWindow::deleteRecord() } } - if(old_row > m_browseTableModel->totalRowCount()) - old_row = m_browseTableModel->totalRowCount(); + if(old_row > m_browseTableModel->rowCount()) + old_row = m_browseTableModel->rowCount(); selectTableLine(old_row); } else { QMessageBox::information( this, QApplication::applicationName(), tr("Please select a record first")); @@ -750,8 +767,8 @@ void MainWindow::navigateNext() { int curRow = ui->dataTable->currentIndex().row(); curRow += ui->dataTable->numVisibleRows() - 1; - if(curRow >= m_browseTableModel->totalRowCount()) - curRow = m_browseTableModel->totalRowCount() - 1; + if(curRow >= m_browseTableModel->rowCount()) + curRow = m_browseTableModel->rowCount() - 1; selectTableLine(curRow); } @@ -762,7 +779,7 @@ void MainWindow::navigateBegin() void MainWindow::navigateEnd() { - selectTableLine(m_browseTableModel->totalRowCount()-1); + selectTableLine(m_browseTableModel->rowCount()-1); } @@ -771,8 +788,8 @@ void MainWindow::navigateGoto() int row = ui->editGoto->text().toInt(); if(row <= 0) row = 1; - if(row > m_browseTableModel->totalRowCount()) - row = m_browseTableModel->totalRowCount(); + if(row > m_browseTableModel->rowCount()) + row = m_browseTableModel->rowCount(); selectTableLine(row - 1); ui->editGoto->setText(QString::number(row)); @@ -782,7 +799,7 @@ void MainWindow::setRecordsetLabel() { // Get all the numbers, i.e. the number of the first row and the last row as well as the total number of rows int from = ui->dataTable->verticalHeader()->visualIndexAt(0) + 1; - int total = m_browseTableModel->totalRowCount(); + int total = m_browseTableModel->rowCount(); int to = ui->dataTable->verticalHeader()->visualIndexAt(ui->dataTable->height()) - 1; if (to == -2) to = total; @@ -791,7 +808,23 @@ void MainWindow::setRecordsetLabel() gotoValidator->setRange(0, total); // Update the label showing the current position - ui->labelRecordset->setText(tr("%1 - %2 of %3").arg(from).arg(to).arg(total)); + QString txt; + switch(m_browseTableModel->rowCountAvailable()) + { + case SqliteTableModel::RowCount::Unknown: + txt = tr("determining row count..."); + break; + case SqliteTableModel::RowCount::Partial: + txt = tr("%1 - %2 of >= %3").arg(from).arg(to).arg(total); + break; + case SqliteTableModel::RowCount::Complete: + default: + txt = tr("%1 - %2 of %3").arg(from).arg(to).arg(total); + break; + } + ui->labelRecordset->setText(txt); + + ui->dataTable->setDisabled( m_browseTableModel->rowCountAvailable() == SqliteTableModel::RowCount::Unknown ); } void MainWindow::refresh() @@ -1119,7 +1152,8 @@ void MainWindow::executeQuery() // Execute next statement int tail_length_before = tail_length; const char* qbegin = tail; - sql3status = sqlite3_prepare_v2(db._db,tail, tail_length, &vm, &tail); + auto pDb = db.get("executing query"); + sql3status = sqlite3_prepare_v2(pDb.get(), tail, tail_length, &vm, &tail); QString queryPart = QString::fromUtf8(qbegin, tail - qbegin); tail_length -= (tail - qbegin); int execution_end_index = execution_start_index + tail_length_before - tail_length; @@ -1144,12 +1178,20 @@ void MainWindow::executeQuery() { // If we get here, the SQL statement returns some sort of data. So hand it over to the model for display. Don't set the modified flag // because statements that display data don't change data as well. + pDb = nullptr; - sqlWidget->getModel()->setQuery(queryPart); + auto * model = sqlWidget->getModel(); + model->setQuery(queryPart); + + // Wait until the initial loading of data (= first chunk and row count) has been performed. I have the + // feeling that a lot of stuff would need rewriting if we wanted to become more asynchronous here: + // essentially the entire loop over the commands would need to be signal-driven. + model->waitUntilIdle(); + qApp->processEvents(); // to make row count available // The query takes the last placeholder as it may itself contain the sequence '%' + number statusMessage = tr("%1 rows returned in %2ms from: %3").arg( - sqlWidget->getModel()->totalRowCount()).arg(timer.elapsed()).arg(queryPart.trimmed()); + model->rowCount()).arg(timer.elapsed()).arg(queryPart.trimmed()); ok = true; ui->actionSqlResultsSave->setEnabled(true); ui->actionSqlResultsSaveAsView->setEnabled(!db.readOnly()); @@ -1165,7 +1207,7 @@ void MainWindow::executeQuery() QString stmtHasChangedDatabase; if(query_part_type == InsertStatement || query_part_type == UpdateStatement || query_part_type == DeleteStatement) - stmtHasChangedDatabase = tr(", %1 rows affected").arg(sqlite3_changes(db._db)); + stmtHasChangedDatabase = tr(", %1 rows affected").arg(sqlite3_changes(pDb.get())); // Attach/Detach statements don't modify the original database if(query_part_type != StatementType::AttachStatement && query_part_type != StatementType::DetachStatement) @@ -1178,18 +1220,20 @@ void MainWindow::executeQuery() case SQLITE_MISUSE: continue; default: - statusMessage = QString::fromUtf8(sqlite3_errmsg(db._db)) + ": " + queryPart; + statusMessage = QString::fromUtf8(sqlite3_errmsg(pDb.get())) + ": " + queryPart; ok = true; break; } timer.restart(); } else { - statusMessage = QString::fromUtf8(sqlite3_errmsg(db._db)) + ": " + queryPart; + statusMessage = QString::fromUtf8(sqlite3_errmsg(pDb.get())) + ": " + queryPart; sqlWidget->getEditor()->setErrorIndicator(execution_start_line, execution_start_index, execution_start_line, execution_end_index); ok = false; } + pDb = nullptr; // release db + execution_start_index = execution_end_index; // Revert to save point now if it wasn't needed. We need to do this here because there are some rare cases where the next statement might @@ -1206,6 +1250,7 @@ void MainWindow::executeQuery() // Process events to keep the UI responsive qApp->processEvents(); } + sqlWidget->finishExecution(statusMessage, ok); attachPlot(sqlWidget->getTableResult(), sqlWidget->getModel()); @@ -2907,8 +2952,10 @@ void MainWindow::requestCollation(const QString& name, int eTextRep) "that this application can't provide without further knowledge.\n" "If you choose to proceed, be aware bad things can happen to your database.\n" "Create a backup!").arg(name), QMessageBox::Yes | QMessageBox::No); - if(reply == QMessageBox::Yes) - sqlite3_create_collation(db._db, name.toUtf8(), eTextRep, nullptr, collCompare); + if(reply == QMessageBox::Yes) { + auto pDb = db.get("creating collation"); + sqlite3_create_collation(pDb.get(), name.toUtf8(), eTextRep, nullptr, collCompare); + } } void MainWindow::renameSqlTab(int index) diff --git a/src/MainWindow.h b/src/MainWindow.h index da2bf8f1..f80627b8 100644 --- a/src/MainWindow.h +++ b/src/MainWindow.h @@ -135,7 +135,12 @@ private: Ui::MainWindow* ui; + DBBrowserDB db; + + /// the table model used in the "Browse Data" page (re-used and + /// re-initialized when switching to another table) SqliteTableModel* m_browseTableModel; + SqliteTableModel* m_currentTabTableModel; QMenu* popupTableMenu; @@ -166,7 +171,6 @@ private: QIntValidator* gotoValidator; - DBBrowserDB db; QString defaultBrowseTableEncoding; void init(); diff --git a/src/PlotDock.cpp b/src/PlotDock.cpp index 0be88aa2..3df62579 100644 --- a/src/PlotDock.cpp +++ b/src/PlotDock.cpp @@ -127,8 +127,6 @@ void PlotDock::updatePlot(SqliteTableModel* model, BrowseDataTableSettings* sett if(model) { - model->waitForFetchingFinished(); - // Add each column with a supported data type to the column selection view for(int i=0;icolumnCount();++i) { @@ -292,9 +290,12 @@ void PlotDock::updatePlot(SqliteTableModel* model, BrowseDataTableSettings* sett // prepare the data vectors for qcustomplot // possible improvement might be a QVector subclass that directly // access the model data, to save memory, we are copying here - QVector xdata(model->rowCount()), ydata(model->rowCount()), tdata(model->rowCount()); + + auto nrows = model->rowCount(); + + QVector xdata(nrows), ydata(nrows), tdata(nrows); QVector labels; - for(int i = 0; i < model->rowCount(); ++i) + for(int i = 0; i < nrows; ++i) { tdata[i] = i; // convert x type axis if it's datetime @@ -423,7 +424,7 @@ void PlotDock::updatePlot(SqliteTableModel* model, BrowseDataTableSettings* sett ui->plotWidget->replot(); // Warn user if not all data has been fetched and hint about the button for loading all the data - if (ui->plotWidget->plottableCount() > 0 && model->canFetchMore()) { + if (model->rowCountAvailable() != SqliteTableModel::RowCount::Complete || !model->isCacheComplete()) { ui->buttonLoadAllData->setEnabled(true); ui->buttonLoadAllData->setStyleSheet("QToolButton {color: white; background-color: rgb(255, 102, 102)}"); ui->buttonLoadAllData->setToolTip(tr("Load all data and redraw plot.\n" @@ -735,23 +736,14 @@ void PlotDock::fetchAllData() { // Show progress dialog because fetching all data might take some time QProgressDialog progress(tr("Fetching all data..."), - tr("Cancel"), m_currentPlotModel->rowCount(), m_currentPlotModel->totalRowCount()); + tr("Cancel"), m_currentPlotModel->rowCount(), m_currentPlotModel->rowCount()); progress.setWindowModality(Qt::ApplicationModal); progress.show(); qApp->processEvents(); // Make sure all data is loaded - while(m_currentPlotModel->canFetchMore()) - { - // Fetch the next bunch of data - m_currentPlotModel->fetchMore(); - - // Update the progress dialog and stop loading data when the cancel button was pressed - progress.setValue(m_currentPlotModel->rowCount()); - qApp->processEvents(); - if(progress.wasCanceled()) - break; - } + // TODO make this cancellable & show progress + m_currentPlotModel->completeCache(); // Update plot updatePlot(m_currentPlotModel, m_currentTableSettings); diff --git a/src/RowCache.h b/src/RowCache.h new file mode 100644 index 00000000..b0348fcd --- /dev/null +++ b/src/RowCache.h @@ -0,0 +1,264 @@ +#ifndef ROW_CACHE_H +#define ROW_CACHE_H + +#include +#include +#include +#include + +/** + + cache structure adapted to the existing access patterns in + SqliteTableModel. handles many large segments with gaps in between + well. + + logical structure resembling a std::vector>, but + implementation avoids actually allocating space for the non-empty + optionals, and supports (hopefully) more efficient insertion / + deletion. + + actually, this is not even a "cache" - once set, elements are never + thrown away to make space for new elements. + + TODO introduce maximum segment size? + +**/ +template +class RowCache +{ +public: + typedef T value_type; + + /// constructs an empty cache + explicit RowCache (); + + /// \returns number of cached rows + size_t numSet () const; + + /// \returns number of segments + size_t numSegments () const; + + /// \returns 1 if specified row is loaded, 0 otherwise + size_t count (size_t pos) const; + + /// \returns specified row. \throws if not available + const T & at (size_t pos) const; + T & at (size_t pos); + + /// assigns value to specified row; may increase numSet() by one + void set (size_t pos, T && value); + + /// insert new element; increases numSet() by one + void insert (size_t pos, T && value); + + /// delete element; decreases numSet() by one + void erase (size_t pos); + + /// reset to state after construction + void clear (); + + /// given a range of rows (end is exclusive), narrow it in order + /// to remove already-loaded rows from both ends. + void smallestNonAvailableRange (size_t & row_begin, size_t & row_end) const; + +private: + /// a single segment containing contiguous entries + struct Segment + { + size_t pos_begin; + std::vector entries; + + /// returns past-the-end position of this segment + size_t pos_end () const { return pos_begin + entries.size(); } + }; + + /// collection of non-overlapping segments, in order of increasing + /// position + using Segments = std::vector; + Segments segments; + + // ------------------------------------------------------------------------------ + + /// \returns first segment that definitely cannot contain 'pos', + /// because it starts at some later position. + typename Segments::const_iterator getSegmentBeyond (size_t pos) const { + // first segment whose pos_begin > pos (so can't contain pos itself): + return std::upper_bound(segments.begin(), segments.end(), pos, pred); + } + + typename Segments::iterator getSegmentBeyond (size_t pos) { + return std::upper_bound(segments.begin(), segments.end(), pos, pred); + } + + static bool pred (size_t pos, const Segment & s) { return pos < s.pos_begin; } + + // ------------------------------------------------------------------------------ + + /// \returns segment containing 'pos' + typename Segments::const_iterator getSegmentContaining (size_t pos) const + { + auto it = getSegmentBeyond(pos); + + if(it != segments.begin()) { + auto prev_it = it - 1; + if(pos < prev_it->pos_end()) + return prev_it; + } + + return segments.end(); + } + +}; + +template +RowCache::RowCache () +{ +} + +template +size_t RowCache::numSet () const +{ + return std::accumulate(segments.begin(), segments.end(), size_t(0), + [](size_t r, const Segment & s) { return r + s.entries.size(); }); +} + +template +size_t RowCache::numSegments () const +{ + return segments.size(); +} + +template +size_t RowCache::count (size_t pos) const +{ + return getSegmentContaining(pos) != segments.end(); +} + +template +const T & RowCache::at (size_t pos) const +{ + auto it = getSegmentContaining(pos); + + if(it != segments.end()) + return it->entries[pos - it->pos_begin]; + + throw std::out_of_range("no matching segment found"); +} + +template +T & RowCache::at (size_t pos) +{ + return const_cast(static_cast(*this).at(pos)); +} + +template +void RowCache::set (size_t pos, T && value) +{ + auto it = getSegmentBeyond(pos); + + if(it != segments.begin()) + { + auto prev_it = it - 1; + auto d = pos - prev_it->pos_begin; // distance from segment start (>=0) + + if(d < prev_it->entries.size()) + { + // replace value + prev_it->entries[d] = std::move(value); + return; + } + + if(d == prev_it->entries.size()) + { + // extend existing segment + prev_it->entries.insert(prev_it->entries.end(), std::move(value)); + return; + } + } + + // make new segment + segments.insert(it, { pos, { std::move(value) } }); +} + +template +void RowCache::insert (size_t pos, T && value) +{ + auto it = getSegmentBeyond(pos); + + if(it != segments.begin()) + { + auto prev_it = it - 1; + auto d = pos - prev_it->pos_begin; // distance from segment start (>=0) + + if(d <= prev_it->entries.size()) + { + // can extend existing segment + prev_it->entries.insert(prev_it->entries.begin() + d, std::move(value)); + goto push; + } + } + + // make new segment + it = segments.insert(it, { pos, { std::move(value) } }) + 1; + +push: + // push back all later segments + std::for_each(it, segments.end(), [](Segment &s){ s.pos_begin++; }); +} + +template +void RowCache::erase (size_t pos) +{ + auto it = getSegmentBeyond(pos); + + // if previous segment actually contains pos, shorten it + if(it != segments.begin()) + { + auto prev_it = it - 1; + auto d = pos - prev_it->pos_begin; // distance from segment start (>=0) + + if(d < prev_it->entries.size()) + { + prev_it->entries.erase(prev_it->entries.begin() + d); + if(prev_it->entries.empty()) + { + it = segments.erase(prev_it); + } + } + } + + // pull forward all later segments + std::for_each(it, segments.end(), [](Segment &s){ s.pos_begin--; }); +} + +template +void RowCache::clear () +{ + segments.clear(); +} + +template +void RowCache::smallestNonAvailableRange (size_t & row_begin, size_t & row_end) const +{ + if(row_end < row_begin) + throw std::invalid_argument("end must be >= begin"); + + while(row_begin < row_end) { + auto it = getSegmentContaining(row_begin); + if(it == segments.end()) + break; + row_begin = it->pos_end(); + } + + while(row_end > row_begin) { + auto it = getSegmentContaining(row_end - 1); + if(it == segments.end()) + break; + row_end = it->pos_begin; + } + + if(row_end < row_begin) + row_end = row_begin; +} + +#endif // SEGMENTING_CACHE_H diff --git a/src/RowLoader.cpp b/src/RowLoader.cpp new file mode 100644 index 00000000..7acde9fa --- /dev/null +++ b/src/RowLoader.cpp @@ -0,0 +1,264 @@ +#include + +#include "RowLoader.h" +#include "sqlite.h" + +namespace { + + QString rtrimChar(const QString& s, QChar c) + { + QString r = s.trimmed(); + while(r.endsWith(c)) + r.chop(1); + return r; + } + +} // anon ns + + +RowLoader::RowLoader ( + std::function(void)> db_getter_, + std::function statement_logger_, + QStringList & headers_, + QMutex & cache_mutex_, + Cache & cache_data_ + ) + : db_getter(db_getter_), statement_logger(statement_logger_), headers(headers_) + , cache_mutex(cache_mutex_), cache_data(cache_data_) + , query() + , num_tasks(0) + , pDb(nullptr) + , stop_requested(false) + , current_task(nullptr) + , next_task(nullptr) +{ +} + +void RowLoader::setQuery (QString new_query) +{ + std::lock_guard lk(m); + query = new_query; +} + +void RowLoader::triggerRowCountDetermination(int token) +{ + std::unique_lock lk(m); + + num_tasks++; + nosync_ensureDbAccess(); + + // do a count query to get the full row count in a fast manner + row_counter = std::async(std::launch::async, [this, token]() { + auto nrows = countRows(); + if(nrows >= 0) + emit rowCountComplete(token, nrows); + + std::lock_guard lk(m); + nosync_taskDone(); + }); +} + +void RowLoader::nosync_ensureDbAccess () +{ + if(!pDb) + pDb = db_getter(); +} + +std::shared_ptr RowLoader::getDb () const +{ + std::lock_guard lk(m); + return pDb; +} + +int RowLoader::countRows() +{ + int retval = -1; + + // Use a different approach of determining the row count when a EXPLAIN or a PRAGMA statement is used because a COUNT fails on these queries + if(query.startsWith("EXPLAIN", Qt::CaseInsensitive) || query.startsWith("PRAGMA", Qt::CaseInsensitive)) + { + // So just execute the statement as it is and fetch all results counting the rows + sqlite3_stmt* stmt; + QByteArray utf8Query = query.toUtf8(); + if(sqlite3_prepare_v2(pDb.get(), utf8Query, utf8Query.size(), &stmt, NULL) == SQLITE_OK) + { + retval = 0; + while(sqlite3_step(stmt) == SQLITE_ROW) + retval++; + sqlite3_finalize(stmt); + return retval; + } + } else { + // If it is a normal query - hopefully starting with SELECT - just do a COUNT on it and return the results + QString sCountQuery = QString("SELECT COUNT(*) FROM (%1);").arg(rtrimChar(query, ';')); + statement_logger(sCountQuery); + QByteArray utf8Query = sCountQuery.toUtf8(); + + sqlite3_stmt* stmt; + int status = sqlite3_prepare_v2(pDb.get(), utf8Query, utf8Query.size(), &stmt, NULL); + if(status == SQLITE_OK) + { + status = sqlite3_step(stmt); + if(status == SQLITE_ROW) + { + QString sCount = QString::fromUtf8((const char*)sqlite3_column_text(stmt, 0)); + retval = sCount.toInt(); + } + sqlite3_finalize(stmt); + } else { + qWarning() << "Count query failed: " << sCountQuery; + } + } + + return retval; +} + +void RowLoader::triggerFetch (int token, size_t row_begin, size_t row_end) +{ + std::unique_lock lk(m); + + if(pDb) { + if(!row_counter.valid() || row_counter.wait_for(std::chrono::seconds(0)) == std::future_status::ready) { + // only if row count is complete, we can safely interrupt SQLite to speed up cancellation + sqlite3_interrupt(pDb.get()); + } + } + + if(current_task) + current_task->cancel = true; + + nosync_ensureDbAccess(); + + // (forget a possibly already existing "next task") + next_task.reset(new Task{ *this, token, row_begin, row_end }); + + lk.unlock(); + cv.notify_all(); +} + +void RowLoader::nosync_taskDone() +{ + if(--num_tasks == 0) { + pDb = nullptr; + } +} + +void RowLoader::cancel () +{ + std::unique_lock lk(m); + + if(pDb) + sqlite3_interrupt(pDb.get()); + + if(current_task) + current_task->cancel = true; + + next_task = nullptr; + cv.notify_all(); +} + +void RowLoader::stop () +{ + cancel(); + std::unique_lock lk(m); + stop_requested = true; + cv.notify_all(); +} + +bool RowLoader::readingData () const +{ + std::unique_lock lk(m); + return pDb != nullptr; +} + +void RowLoader::waitUntilIdle () const +{ + if(row_counter.valid()) + row_counter.wait(); + std::unique_lock lk(m); + cv.wait(lk, [this](){ return stop_requested || (!current_task && !next_task); }); +} + +void RowLoader::run () +{ + for(;;) + { + std::unique_lock lk(m); + current_task = nullptr; + cv.notify_all(); + + cv.wait(lk, [this](){ return stop_requested || next_task; }); + + if(stop_requested) + return; + + current_task = std::move(next_task); + lk.unlock(); + + process(*current_task); + } +} + +void RowLoader::process (Task & t) +{ + //std::cout << "RowLoader new task: " << t.row_begin << " --> " << t.row_end << std::endl; + + QString sLimitQuery; + if(query.startsWith("PRAGMA", Qt::CaseInsensitive) || query.startsWith("EXPLAIN", Qt::CaseInsensitive)) + { + sLimitQuery = query; + } else { + // Remove trailing trailing semicolon + QString queryTemp = rtrimChar(query, ';'); + + // If the query ends with a LIMIT statement take it as it is, if not append our own LIMIT part for lazy population + if(queryTemp.contains(QRegExp("LIMIT\\s+.+\\s*((,|\\b(OFFSET)\\b)\\s*.+\\s*)?$", Qt::CaseInsensitive))) + sLimitQuery = queryTemp; + else + sLimitQuery = queryTemp + QString(" LIMIT %1, %2;").arg(t.row_begin).arg(t.row_end-t.row_begin); + } + statement_logger(sLimitQuery); + + QByteArray utf8Query = sLimitQuery.toUtf8(); + sqlite3_stmt *stmt; + + int status = sqlite3_prepare_v2(pDb.get(), utf8Query, utf8Query.size(), &stmt, NULL); + + auto row = t.row_begin; + + if(SQLITE_OK == status) + { + const int num_columns = headers.size(); + + while(!t.cancel && sqlite3_step(stmt) == SQLITE_ROW) + { + Cache::value_type rowdata; + for(int i=0;i(sqlite3_column_blob(stmt, i)), bytes)); + else + rowdata.append(QByteArray("")); + } + } + QMutexLocker lk(&cache_mutex); + cache_data.set(row++, std::move(rowdata)); + } + } + sqlite3_finalize(stmt); + + if(row != t.row_begin) + emit fetched(t.token, t.row_begin, row); + +#if 0 + if(t.cancel) + std::cout << "RowLoader task was CANCELLED\n"; + else + std::cout << "RowLoader task done\n"; +#endif +} diff --git a/src/RowLoader.h b/src/RowLoader.h new file mode 100644 index 00000000..1fae02fa --- /dev/null +++ b/src/RowLoader.h @@ -0,0 +1,121 @@ +#ifndef ROW_LOADER_H +#define ROW_LOADER_H + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "sqlite.h" +#include "RowCache.h" + +class RowLoader : public QThread +{ + Q_OBJECT + + void run() override; + +public: + typedef RowCache> Cache; + + /// set up worker thread to handle row loading + explicit RowLoader ( + std::function(void)> db_getter, + std::function statement_logger, + QStringList & headers, + QMutex & cache_mutex, + Cache & cache_data + ); + + void setQuery (QString); + + void triggerRowCountDetermination (int token); + + /// trigger asynchronous reading of specified row range, + /// cancelling previous tasks; 'row_end' is exclusive; \param + /// token is eventually returned through the 'fetched' + /// signal. depending on how and when tasks are cancelled, not + /// every triggerFetch() will result in a 'fetched' signal, or the + /// 'fetched' signal may be for a narrower row range. + void triggerFetch (int token, size_t row_begin, size_t row_end); + + /// cancel everything + void cancel (); + + /// cancel everything and terminate worker thread + void stop (); + + /// currently reading any data, or anything in "queue"? + bool readingData () const; + + /// wait until not reading any data + void waitUntilIdle () const; + + /// get current database - note that the worker thread might be + /// working on it, too... \returns current db, or nullptr. + std::shared_ptr getDb () const; + +signals: + void fetched(int token, size_t row_begin, size_t row_end); + void rowCountComplete(int token, int num_rows); + +private: + const std::function()> db_getter; + const std::function statement_logger; + QStringList & headers; + QMutex & cache_mutex; + Cache & cache_data; + + mutable std::mutex m; + mutable std::condition_variable cv; + + QString query; + + mutable std::future row_counter; + + size_t num_tasks; + std::shared_ptr pDb; //< exclusive access while held... + + bool stop_requested; + + struct Task + { + RowLoader & row_loader; + int token; + size_t row_begin; + size_t row_end; //< exclusive + std::atomic cancel; + + Task(RowLoader & row_loader_, int t, size_t a, size_t b) + : row_loader(row_loader_), token(t), row_begin(a), row_end(b), cancel(false) + { + row_loader.num_tasks++; + } + + ~Task() + { + // (... mutex being held ...) + row_loader.nosync_taskDone(); + } + }; + + std::unique_ptr current_task; + std::unique_ptr next_task; + + int countRows (); + + void process (Task &); + + void nosync_ensureDbAccess (); + void nosync_taskDone (); + +}; + +#endif // ROW_LOADER_H diff --git a/src/sqlitedb.cpp b/src/sqlitedb.cpp index 82a66eb6..b563d80c 100644 --- a/src/sqlitedb.cpp +++ b/src/sqlitedb.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -173,6 +174,11 @@ bool DBBrowserDB::open(const QString& db, bool readOnly) bool DBBrowserDB::attach(const QString& filename, QString attach_as) { + if(!_db) + return false; + + waitForDbRelease(); + // Check if this file has already been attached and abort if this is the case QString sql = "PRAGMA database_list;"; logSQL(sql, kLogMsg_App); @@ -371,6 +377,11 @@ bool DBBrowserDB::revertToSavepoint(const QString& pointname) bool DBBrowserDB::releaseAllSavepoints() { + if(!_db) + return false; + + waitForDbRelease(); + for(const QString& point : savepointList) { if(!releaseSavepoint(point)) @@ -396,7 +407,8 @@ bool DBBrowserDB::revertAll() bool DBBrowserDB::create ( const QString & db) { - if (isOpen()) close(); + if (isOpen()) + close(); // read encoding from settings and open with sqlite3_open for utf8 and sqlite3_open16 for utf16 QString sEncoding = Settings::getValue("db", "defaultencoding").toString(); @@ -452,9 +464,10 @@ bool DBBrowserDB::create ( const QString & db) } } - bool DBBrowserDB::close() { + waitForDbRelease(); + if(_db) { if (getDirty()) @@ -476,15 +489,56 @@ bool DBBrowserDB::close() revertAll(); //not really necessary, I think... but will not hurt. } sqlite3_close(_db); + _db = nullptr; } - _db = nullptr; + schemata.clear(); savepointList.clear(); emit dbChanged(getDirty()); emit structureUpdated(); - // Return true to tell the calling function that the closing wasn't cancelled by the user - return true; + return true; //< not cancelled +} + +DBBrowserDB::db_pointer_type DBBrowserDB::get(QString user) +{ + if(!_db) + return nullptr; + + auto lk = waitForDbRelease(); + + db_user = user; + db_used = true; + + return db_pointer_type(_db, DatabaseReleaser(this)); +} + +std::unique_lock DBBrowserDB::waitForDbRelease() +{ + if(!_db) + return std::unique_lock(); + + std::unique_lock lk(m); + while(db_used) { + // notify user, give him the opportunity to cancel that + auto str = db_user; + lk.unlock(); + + QMessageBox msgBox; + msgBox.setText("The database is currently busy: " + str); + msgBox.setInformativeText("Do you want to abort that other operation?"); + msgBox.setStandardButtons(QMessageBox::Yes | QMessageBox::No); + msgBox.setDefaultButton(QMessageBox::No); + int ret = msgBox.exec(); + + if(ret == QMessageBox::Yes) + sqlite3_interrupt(_db); + + lk.lock(); + cv.wait(lk, [this](){ return !db_used; }); + } + + return std::move(lk); } bool DBBrowserDB::dump(const QString& filename, @@ -495,6 +549,8 @@ bool DBBrowserDB::dump(const QString& filename, bool exportData, bool keepOldSchema) { + waitForDbRelease(); + // Open file QFile file(filename); if(file.open(QIODevice::WriteOnly)) @@ -518,7 +574,7 @@ bool DBBrowserDB::dump(const QString& filename, // Otherwise get the number of records in this table SqliteTableModel tableModel(*this); tableModel.setTable(sqlb::ObjectIdentifier("main", it.value()->name())); - numRecordsTotal += tableModel.totalRowCount(); + numRecordsTotal += tableModel.rowCount(); } } @@ -564,7 +620,7 @@ bool DBBrowserDB::dump(const QString& filename, sqlite3_stmt *stmt; QString lineSep(QString(")%1\n").arg(insertNewSyntx?',':';')); - int status = sqlite3_prepare_v2(this->_db, utf8Query.data(), utf8Query.size(), &stmt, nullptr); + int status = sqlite3_prepare_v2(_db, utf8Query.data(), utf8Query.size(), &stmt, nullptr); if(SQLITE_OK == status) { int columns = sqlite3_column_count(stmt); @@ -686,7 +742,8 @@ bool DBBrowserDB::dump(const QString& filename, bool DBBrowserDB::executeSQL(QString statement, bool dirtyDB, bool logsql) { - if (!isOpen()) + waitForDbRelease(); + if(!_db) { lastErrorMessage = tr("No database file opened"); return false; @@ -718,8 +775,8 @@ bool DBBrowserDB::executeSQL(QString statement, bool dirtyDB, bool logsql) bool DBBrowserDB::executeMultiSQL(const QString& statement, bool dirty, bool log) { - // First check if a DB is opened - if(!isOpen()) + waitForDbRelease(); + if(!_db) { lastErrorMessage = tr("No database file opened"); return false; @@ -831,6 +888,10 @@ bool DBBrowserDB::executeMultiSQL(const QString& statement, bool dirty, bool log bool DBBrowserDB::getRow(const sqlb::ObjectIdentifier& table, const QString& rowid, QVector& rowdata) { + waitForDbRelease(); + if(!_db) + return false; + QString sQuery = QString("SELECT * FROM %1 WHERE %2='%3';") .arg(table.toString()) .arg(sqlb::escapeIdentifier(getObjectByName(table).dynamicCast()->rowidColumn())) @@ -947,7 +1008,9 @@ QString DBBrowserDB::emptyInsertStmt(const QString& schemaName, const sqlb::Tabl QString DBBrowserDB::addRecord(const sqlb::ObjectIdentifier& tablename) { - if (!isOpen()) return QString(); + waitForDbRelease(); + if(!_db) + return QString(); sqlb::TablePtr table = getObjectByName(tablename).dynamicCast(); if(!table) @@ -972,8 +1035,9 @@ QString DBBrowserDB::addRecord(const sqlb::ObjectIdentifier& tablename) } else { if(table->isWithoutRowidTable()) return pk_value; - else + else { return QString::number(sqlite3_last_insert_rowid(_db)); + } } } @@ -1006,8 +1070,10 @@ bool DBBrowserDB::deleteRecords(const sqlb::ObjectIdentifier& table, const QStri } } -bool DBBrowserDB::updateRecord(const sqlb::ObjectIdentifier& table, const QString& column, const QString& rowid, const QByteArray& value, bool itsBlob, const QString& pseudo_pk) +bool DBBrowserDB::updateRecord(const sqlb::ObjectIdentifier& table, const QString& column, + const QString& rowid, const QByteArray& value, bool itsBlob, const QString& pseudo_pk) { + waitForDbRelease(); if (!isOpen()) return false; // Get primary key of the object to edit. @@ -1401,6 +1467,8 @@ void DBBrowserDB::logSQL(QString statement, int msgtype) void DBBrowserDB::updateSchema() { + waitForDbRelease(); + schemata.clear(); // Exit here is no DB is opened @@ -1505,6 +1573,8 @@ void DBBrowserDB::updateSchema() QString DBBrowserDB::getPragma(const QString& pragma) { + waitForDbRelease(); + if(!isOpen()) return QString(); @@ -1584,7 +1654,8 @@ bool DBBrowserDB::setPragma(const QString& pragma, int value, int& originalvalue bool DBBrowserDB::loadExtension(const QString& filename) { - if(!isOpen()) + waitForDbRelease(); + if(!_db) return false; // Check if file exists @@ -1608,6 +1679,8 @@ bool DBBrowserDB::loadExtension(const QString& filename) QVector> DBBrowserDB::queryColumnInformation(const QString& schema_name, const QString& object_name) { + waitForDbRelease(); + QVector> result; QString statement = QString("PRAGMA %1.TABLE_INFO(%2);").arg(sqlb::escapeIdentifier(schema_name)).arg(sqlb::escapeIdentifier(object_name)); logSQL(statement, kLogMsg_App); diff --git a/src/sqlitedb.h b/src/sqlitedb.h index b1f34ca6..84ce8964 100644 --- a/src/sqlitedb.h +++ b/src/sqlitedb.h @@ -3,6 +3,10 @@ #include "sqlitetypes.h" +#include +#include +#include + #include #include #include @@ -21,17 +25,64 @@ typedef QMap schemaMap; // Maps from the sch int collCompare(void* pArg, int sizeA, const void* sA, int sizeB, const void* sB); +/// represents a single SQLite database. except when noted otherwise, +/// all member functions are to be called from the main UI thread +/// only. class DBBrowserDB : public QObject { Q_OBJECT +private: + /// custom unique_ptr deleter releases database for further use by others + struct DatabaseReleaser + { + DatabaseReleaser(DBBrowserDB * pParent_ = nullptr) : pParent(pParent_) {} + + DBBrowserDB * pParent; + + void operator() (sqlite3 * db) const + { + if(!db || !pParent) + return; + + std::unique_lock lk(pParent->m); + pParent->db_used = false; + lk.unlock(); + pParent->cv.notify_one(); + } + }; + public: - explicit DBBrowserDB () : _db(nullptr), isEncrypted(false), isReadOnly(false), dontCheckForStructureUpdates(false) {} + explicit DBBrowserDB () : _db(nullptr), db_used(false), isEncrypted(false), isReadOnly(false), dontCheckForStructureUpdates(false) {} virtual ~DBBrowserDB (){} + bool open(const QString& db, bool readOnly = false); bool attach(const QString& filename, QString attach_as = ""); bool create ( const QString & db); bool close(); + + typedef std::unique_ptr db_pointer_type; + + /** + borrow exclusive address to the currently open database, until + releasing the returned unique_ptr. + + the intended use case is that the main UI thread can call this + any time, and then optionally pass the obtained pointer to a + background worker, or release it after doing work immediately. + + if database is currently used by somebody else, opens a dialog + box and gives user the opportunity to sqlite3_interrupt() the + operation of the current owner, then tries again. + + \param user a string that identifies the new user, and which + can be displayed in the dialog box. + + \returns a unique_ptr containing the SQLite database handle, or + nullptr in case no database is open. + **/ + db_pointer_type get (QString user); + bool setSavepoint(const QString& pointname = "RESTOREPOINT"); bool releaseSavepoint(const QString& pointname = "RESTOREPOINT"); bool revertToSavepoint(const QString& pointname = "RESTOREPOINT"); @@ -53,6 +104,7 @@ public: */ bool getRow(const sqlb::ObjectIdentifier& table, const QString& rowid, QVector& rowdata); +private: /** * @brief max Queries the table t for the max value of field. * @param tableName Table to query @@ -61,9 +113,10 @@ public: */ QString max(const sqlb::ObjectIdentifier& tableName, sqlb::FieldPtr field) const; +public: void updateSchema(); - QString addRecord(const sqlb::ObjectIdentifier& tablename); +private: /** * @brief Creates an empty insert statement. * @param schemaName The name of the database schema in which to find the table @@ -71,6 +124,9 @@ public: * @return An sqlite conform INSERT INTO statement with empty values. (NULL,'',0) */ QString emptyInsertStmt(const QString& schemaName, const sqlb::Table& t, const QString& pk_value = QString()) const; + +public: + QString addRecord(const sqlb::ObjectIdentifier& tablename); bool deleteRecords(const sqlb::ObjectIdentifier& table, const QStringList& rowids, const QString& pseudo_pk = QString()); bool updateRecord(const sqlb::ObjectIdentifier& table, const QString& column, const QString& rowid, const QByteArray& value, bool itsBlob, const QString& pseudo_pk = QString()); @@ -98,6 +154,8 @@ public: bool readOnly() const { return isReadOnly; } bool getDirty() const; QString currentFile() const { return curDBFilename; } + + /// log an SQL statement [thread-safe] void logSQL(QString statement, int msgtype); QString getPragma(const QString& pragma); @@ -107,15 +165,15 @@ public: bool loadExtension(const QString& filename); +private: QVector> queryColumnInformation(const QString& schema_name, const QString& object_name); +public: QString generateSavepointName(const QString& identifier = QString()) const; // This function generates the name for a temporary table. It guarantees that there is no table with this name yet QString generateTemporaryTableName(const QString& schema) const; - sqlite3 * _db; - schemaMap schemata; signals: @@ -125,6 +183,18 @@ signals: void requestCollation(QString name, int eTextRep); private: + /// external code needs to go through get() to obtain access to the database + sqlite3 * _db; + std::mutex m; + std::condition_variable cv; + bool db_used; + QString db_user; + + /// wait for release of the DB locked through a previous get(), + /// giving users the option to discard running task through a + /// message box. \returns active lock. + std::unique_lock waitForDbRelease (); + QString curDBFilename; QString lastErrorMessage; QStringList savepointList; diff --git a/src/sqlitetablemodel.cpp b/src/sqlitetablemodel.cpp index f4acf250..c9a97ee2 100644 --- a/src/sqlitetablemodel.cpp +++ b/src/sqlitetablemodel.cpp @@ -1,3 +1,4 @@ + #include "sqlitetablemodel.h" #include "sqlitedb.h" #include "sqlite.h" @@ -12,32 +13,92 @@ #include #include +#include "RowLoader.h" + SqliteTableModel::SqliteTableModel(DBBrowserDB& db, QObject* parent, size_t chunkSize, const QString& encoding) : QAbstractTableModel(parent) , m_db(db) - , m_rowCountAdjustment(0) + , m_lifeCounter(0) + , m_currentRowCount(0) , m_chunkSize(chunkSize) , m_encoding(encoding) { + worker = new RowLoader( + [this](){ return m_db.get("reading rows"); }, + [this](QString stmt){ return m_db.logSQL(stmt, kLogMsg_App); }, + m_headers, m_mutexDataCache, m_cache + ); + + worker->start(); + + // any UI updates must be performed in the UI thread, not in the worker thread: + connect(worker, &RowLoader::fetched, this, &SqliteTableModel::handleFinishedFetch, Qt::QueuedConnection); + connect(worker, &RowLoader::rowCountComplete, this, &SqliteTableModel::handleRowCountComplete, Qt::QueuedConnection); + reset(); } SqliteTableModel::~SqliteTableModel() { - m_futureFetch.cancel(); - m_futureFetch.waitForFinished(); + worker->stop(); + worker->wait(); + worker->disconnect(); + delete worker; +} + +SqliteTableModel::RowCount SqliteTableModel::rowCountAvailable () const +{ + return m_rowCountAvailable; +} + +void SqliteTableModel::handleFinishedFetch (int life_id, unsigned int fetched_row_begin, unsigned int fetched_row_end) +{ + if(life_id < m_lifeCounter) + return; + + Q_ASSERT(fetched_row_end >= fetched_row_begin); + + auto old_row_count = m_currentRowCount; + + auto new_row_count = std::max(old_row_count, fetched_row_begin); + new_row_count = std::max(new_row_count, fetched_row_end); + Q_ASSERT(new_row_count >= old_row_count); + + if(new_row_count != old_row_count) + { + beginInsertRows(QModelIndex(), old_row_count, new_row_count - 1); + m_currentRowCount = new_row_count; + endInsertRows(); + } + + if(fetched_row_end != fetched_row_begin) + { + // TODO optimize + int num_columns = m_headers.size(); + emit dataChanged(createIndex(fetched_row_begin, 0), createIndex(fetched_row_end - 1, num_columns - 1)); + } + + if(m_rowCountAvailable != RowCount::Complete) + m_rowCountAvailable = RowCount::Partial; + + emit finishedFetch(fetched_row_begin, fetched_row_end); +} + +void SqliteTableModel::handleRowCountComplete (int life_id, int num_rows) +{ + if(life_id < m_lifeCounter) + return; + + m_rowCountAvailable = RowCount::Complete; + handleFinishedFetch(life_id, num_rows, num_rows); } void SqliteTableModel::reset() { - m_futureFetch.cancel(); - m_futureFetch.waitForFinished(); - m_rowCount = QtConcurrent::run([=]() { - // Make sure we report 0 rows if anybody asks - return 0; - }); + //std::cout << "\n\nSqliteTableModel::reset()\n"; + + clearCache(); - m_rowCountAdjustment = 0; m_sTable.clear(); m_sRowidColumn.clear(); m_iSortColumn = 0; @@ -102,7 +163,7 @@ void SqliteTableModel::setTable(const sqlb::ObjectIdentifier& table, int sortCol QString sColumnQuery = QString::fromUtf8("SELECT * FROM %1;").arg(table.toString()); m_sRowidColumn = "rowid"; m_headers.push_back("rowid"); - m_headers.append(getColumns(sColumnQuery, m_vDataTypes)); + m_headers.append(getColumns(nullptr, sColumnQuery, m_vDataTypes)); } // Set sort parameters. We're setting the sort column to an invalid value before calling sort() because this way, in sort() the @@ -112,103 +173,35 @@ void SqliteTableModel::setTable(const sqlb::ObjectIdentifier& table, int sortCol sort(sortColumn, sortOrder); } -namespace { -QString rtrimChar(const QString& s, QChar c) { - QString r = s.trimmed(); - while(r.endsWith(c)) - r.chop(1); - return r; -} -} - void SqliteTableModel::setQuery(const QString& sQuery, bool dontClearHeaders) { // clear if(!dontClearHeaders) reset(); + else + clearCache(); if(!m_db.isOpen()) return; m_sQuery = sQuery.trimmed(); - removeCommentsFromQuery(m_sQuery); - // do a count query to get the full row count in a fast manner - m_rowCountAdjustment = 0; - m_rowCount = QtConcurrent::run([=]() { - return getQueryRowCount(); - }); + worker->setQuery(m_sQuery); + worker->triggerRowCountDetermination(m_lifeCounter); - // headers if(!dontClearHeaders) - { - m_headers.append(getColumns(sQuery, m_vDataTypes)); - } + m_headers.append(getColumns(worker->getDb(), sQuery, m_vDataTypes)); // now fetch the first entries - clearCache(); - fetchData(0, m_chunkSize); + triggerCacheLoad(m_chunkSize / 2 - 1); emit layoutChanged(); } -int SqliteTableModel::getQueryRowCount() -{ - // Return -1 if there is an error - int retval = -1; - - // Use a different approach of determining the row count when a EXPLAIN or a PRAGMA statement is used because a COUNT fails on these queries - if(m_sQuery.startsWith("EXPLAIN", Qt::CaseInsensitive) || m_sQuery.startsWith("PRAGMA", Qt::CaseInsensitive)) - { - // So just execute the statement as it is and fetch all results counting the rows - sqlite3_stmt* stmt; - QByteArray utf8Query = m_sQuery.toUtf8(); - if(sqlite3_prepare_v2(m_db._db, utf8Query, utf8Query.size(), &stmt, nullptr) == SQLITE_OK) - { - retval = 0; - while(sqlite3_step(stmt) == SQLITE_ROW) - retval++; - sqlite3_finalize(stmt); - - // Return the results but also set the chunk size the number of rows to prevent the lazy population mechanism to kick in as using LIMIT - // fails on this kind of queries as well - m_chunkSize = retval; - return retval; - } - } else { - // If it is a normal query - hopefully starting with SELECT - just do a COUNT on it and return the results - QString sCountQuery = QString("SELECT COUNT(*) FROM (%1);").arg(rtrimChar(m_sQuery, ';')); - m_db.logSQL(sCountQuery, kLogMsg_App); - QByteArray utf8Query = sCountQuery.toUtf8(); - - sqlite3_stmt* stmt; - int status = sqlite3_prepare_v2(m_db._db, utf8Query, utf8Query.size(), &stmt, nullptr); - if(status == SQLITE_OK) - { - status = sqlite3_step(stmt); - if(status == SQLITE_ROW) - { - QString sCount = QString::fromUtf8((const char*)sqlite3_column_text(stmt, 0)); - retval = sCount.toInt(); - } - sqlite3_finalize(stmt); - } else { - qWarning() << "Count query failed: " << sCountQuery; - } - } - - return retval; -} - int SqliteTableModel::rowCount(const QModelIndex&) const { - return m_data.size(); // current fetched row count -} - -int SqliteTableModel::totalRowCount() const -{ - return m_rowCount + m_rowCountAdjustment; + return m_currentRowCount; } int SqliteTableModel::columnCount(const QModelIndex&) const @@ -243,25 +236,43 @@ QVariant SqliteTableModel::data(const QModelIndex &index, int role) const if (!index.isValid()) return QVariant(); - if (index.row() >= (m_rowCount + m_rowCountAdjustment)) + if (index.row() >= rowCount()) return QVariant(); QMutexLocker lock(&m_mutexDataCache); + Row blank_data; + bool row_available; + + const Row * cached_row; + if(m_cache.count(index.row())) + { + cached_row = &m_cache.at(index.row()); + row_available = true; + } + else + { + blank_data = makeDefaultCacheEntry(); + cached_row = &blank_data; + row_available = false; + } + if(role == Qt::DisplayRole || role == Qt::EditRole) { - // If this row is not in the cache yet get it first - while(index.row() >= m_data.size() && canFetchMore()) - const_cast(this)->fetchMore(); // Nothing evil to see here, move along - - if(role == Qt::DisplayRole && m_data.at(index.row()).at(index.column()).isNull()) + if(!row_available) + return decode("loading..."); + if(role == Qt::DisplayRole && cached_row->at(index.column()).isNull()) { return Settings::getValue("databrowser", "null_text").toString(); - } else if(role == Qt::DisplayRole && isBinary(index)) { + } + else if(role == Qt::DisplayRole && nosync_isBinary(index)) + { return Settings::getValue("databrowser", "blob_text").toString(); - } else if(role == Qt::DisplayRole) { + } + else if(role == Qt::DisplayRole) + { int limit = Settings::getValue("databrowser", "symbol_limit").toInt(); - QByteArray displayText = m_data.at(index.row()).at(index.column()); + QByteArray displayText = cached_row->at(index.column()); if (displayText.length() > limit) { // Add "..." to the end of truncated strings return decode(displayText.left(limit).append(" ...")); @@ -269,34 +280,46 @@ QVariant SqliteTableModel::data(const QModelIndex &index, int role) const return decode(displayText); } } else { - return decode(m_data.at(index.row()).at(index.column())); + return decode(cached_row->at(index.column())); } - } else if(role == Qt::FontRole) { + } + else if(role == Qt::FontRole) + { QFont font; - if(m_data.at(index.row()).at(index.column()).isNull() || isBinary(index)) + if(!row_available || cached_row->at(index.column()).isNull() || nosync_isBinary(index)) font.setItalic(true); return font; - } else if(role == Qt::ForegroundRole) { - if(m_data.at(index.row()).at(index.column()).isNull()) + } + else if(role == Qt::ForegroundRole) + { + if(!row_available) + return QColor(100, 100, 100); + if(cached_row->at(index.column()).isNull()) return QColor(Settings::getValue("databrowser", "null_fg_colour").toString()); - else if (isBinary(index)) + else if (nosync_isBinary(index)) return QColor(Settings::getValue("databrowser", "bin_fg_colour").toString()); return QColor(Settings::getValue("databrowser", "reg_fg_colour").toString()); - } else if (role == Qt::BackgroundRole) { - if(m_data.at(index.row()).at(index.column()).isNull()) + } + else if (role == Qt::BackgroundRole) + { + if(!row_available) + return QColor(255, 200, 200); + if(cached_row->at(index.column()).isNull()) return QColor(Settings::getValue("databrowser", "null_bg_colour").toString()); - else if (isBinary(index)) + else if (nosync_isBinary(index)) return QColor(Settings::getValue("databrowser", "bin_bg_colour").toString()); return QColor(Settings::getValue("databrowser", "reg_bg_colour").toString()); - } else if(role == Qt::ToolTipRole) { + } + else if(role == Qt::ToolTipRole) + { sqlb::ForeignKeyClause fk = getForeignKeyClause(index.column()-1); if(fk.isSet()) return tr("References %1(%2)\nHold Ctrl+Shift and click to jump there").arg(fk.table()).arg(fk.columns().join(",")); else return QString(); - } else { - return QVariant(); } + + return QVariant(); } sqlb::ForeignKeyClause SqliteTableModel::getForeignKeyClause(int column) const @@ -343,12 +366,19 @@ bool SqliteTableModel::setData(const QModelIndex& index, const QVariant& value, bool SqliteTableModel::setTypedData(const QModelIndex& index, bool isBlob, const QVariant& value, int role) { + if(readingData()) { + // can't insert rows while reading data in background + return false; + } + if(index.isValid() && role == Qt::EditRole) { QMutexLocker lock(&m_mutexDataCache); + auto & cached_row = m_cache.at(index.row()); + QByteArray newValue = encode(value.toByteArray()); - QByteArray oldValue = m_data.at(index.row()).at(index.column()); + QByteArray oldValue = cached_row.at(index.column()); // Special handling for integer columns: instead of setting an integer column to an empty string, set it to '0' when it is also // used in a primary key. Otherwise SQLite will always output an 'datatype mismatch' error. @@ -368,12 +398,9 @@ bool SqliteTableModel::setTypedData(const QModelIndex& index, bool isBlob, const if(oldValue == newValue && oldValue.isNull() == newValue.isNull()) return true; - if(m_db.updateRecord(m_sTable, m_headers.at(index.column()), m_data[index.row()].at(0), newValue, isBlob, m_pseudoPk)) + if(m_db.updateRecord(m_sTable, m_headers.at(index.column()), cached_row.at(0), newValue, isBlob, m_pseudoPk)) { - // Only update the cache if this row has already been read, if not there's no need to do any changes to the cache - if(index.row() < m_data.size()) - m_data[index.row()].replace(index.column(), newValue); - + cached_row.replace(index.column(), newValue); lock.unlock(); emit dataChanged(index, index); return true; @@ -387,20 +414,6 @@ bool SqliteTableModel::setTypedData(const QModelIndex& index, bool isBlob, const return false; } -bool SqliteTableModel::canFetchMore(const QModelIndex&) const -{ - QMutexLocker lock(&m_mutexDataCache); - return m_data.size() < (m_rowCount + m_rowCountAdjustment); -} - -void SqliteTableModel::fetchMore(const QModelIndex&) -{ - m_futureFetch.waitForFinished(); - QMutexLocker lock(&m_mutexDataCache); - int row = m_data.size(); - fetchData(row, row + m_chunkSize); -} - Qt::ItemFlags SqliteTableModel::flags(const QModelIndex& index) const { if(!index.isValid()) @@ -438,16 +451,34 @@ void SqliteTableModel::sort(int column, Qt::SortOrder order) buildQuery(); } +SqliteTableModel::Row SqliteTableModel::makeDefaultCacheEntry () const +{ + Row blank_data; + + for(int i=0; i < m_headers.size(); ++i) + blank_data.push_back(""); + + return blank_data; +} + +bool SqliteTableModel::readingData() const +{ + return worker->readingData(); +} + bool SqliteTableModel::insertRows(int row, int count, const QModelIndex& parent) { if(!isEditable()) return false; - QByteArrayList blank_data; - for(int i=0; i < m_headers.size(); ++i) - blank_data.push_back(""); + if(readingData()) { + // can't insert rows while reading data in background + return false; + } - DataType tempList; + const auto blank_data = makeDefaultCacheEntry(); + + std::vector tempList; for(int i=row; i < row + count; ++i) { QString rowid = m_db.addRecord(m_sTable); @@ -455,28 +486,29 @@ bool SqliteTableModel::insertRows(int row, int count, const QModelIndex& parent) { return false; } - m_rowCountAdjustment++; - tempList.append(blank_data); - tempList[i - row].replace(0, rowid.toUtf8()); + tempList.push_back(blank_data); + tempList.back().replace(0, rowid.toUtf8()); // update column with default values - QByteArrayList rowdata; + Row rowdata; if(m_db.getRow(m_sTable, rowid, rowdata)) { for(int j=1; j < m_headers.size(); ++j) { - tempList[i - row].replace(j, rowdata[j - 1]); + tempList.back().replace(j, rowdata[j - 1]); } } } beginInsertRows(parent, row, row + count - 1); - QMutexLocker lock(&m_mutexDataCache); - for(int i = 0; i < tempList.size(); ++i) + for(unsigned int i = 0; i < tempList.size(); ++i) { - m_data.insert(i + row, tempList.at(i)); + //std::cout << "inserting at " << i + row << std::endl; + m_cache.insert(i + row, std::move(tempList.at(i))); + m_currentRowCount++; } endInsertRows(); + return true; } @@ -485,16 +517,21 @@ bool SqliteTableModel::removeRows(int row, int count, const QModelIndex& parent) if(!isEditable()) return false; - beginRemoveRows(parent, row, row + count - 1); + if(readingData()) { + // can't delete rows while reading data in background + return false; + } - QMutexLocker lock(&m_mutexDataCache); + beginRemoveRows(parent, row, row + count - 1); QStringList rowids; for(int i=count-1;i>=0;i--) { - rowids.append(m_data.at(row + i).at(0)); - m_data.removeAt(row + i); - --m_rowCountAdjustment; + if(m_cache.count(row+i)) { + rowids.append(m_cache.at(row + i).at(0)); + } + m_cache.erase(row + i); + m_currentRowCount--; } bool ok = m_db.deleteRecords(m_sTable, rowids, m_pseudoPk); @@ -530,69 +567,6 @@ QModelIndex SqliteTableModel::dittoRecord(int old_row) return index(new_row, firstEditedColumn); } -void SqliteTableModel::fetchData(unsigned int from, unsigned to) -{ - // Finish previous loading - m_futureFetch.waitForFinished(); - - // Fetch more data using a separate thread - m_futureFetch = QtConcurrent::run([=]() { - int num_rows_before_insert = m_data.size(); - - QString sLimitQuery; - if(m_sQuery.startsWith("PRAGMA", Qt::CaseInsensitive) || m_sQuery.startsWith("EXPLAIN", Qt::CaseInsensitive)) - { - sLimitQuery = m_sQuery; - } else { - // Remove trailing trailing semicolon - QString queryTemp = rtrimChar(m_sQuery, ';'); - - // If the query ends with a LIMIT statement take it as it is, if not append our own LIMIT part for lazy population - if(queryTemp.contains(QRegExp("LIMIT\\s+.+\\s*((,|\\b(OFFSET)\\b)\\s*.+\\s*)?$", Qt::CaseInsensitive))) - sLimitQuery = queryTemp; - else - sLimitQuery = queryTemp + QString(" LIMIT %1, %2;").arg(from).arg(to-from); - } - m_db.logSQL(sLimitQuery, kLogMsg_App); - QByteArray utf8Query = sLimitQuery.toUtf8(); - sqlite3_stmt *stmt; - int status = sqlite3_prepare_v2(m_db._db, utf8Query, utf8Query.size(), &stmt, nullptr); - - if(SQLITE_OK == status) - { - int num_columns = m_headers.size(); - while(!m_futureFetch.isCanceled() && sqlite3_step(stmt) == SQLITE_ROW) - { - QMutexLocker lock(&m_mutexDataCache); - QByteArrayList rowdata; - for(int i=0;i(sqlite3_column_blob(stmt, i)), bytes)); - else - rowdata.append(QByteArray("")); - } - } - m_data.push_back(rowdata); - } - } - sqlite3_finalize(stmt); - - if(m_data.size() > num_rows_before_insert) - { - beginInsertRows(QModelIndex(), num_rows_before_insert, m_data.size()-1); - endInsertRows(); - } - - emit finishedFetch(); - }); -} - QString SqliteTableModel::customQuery(bool withRowid) { QString where; @@ -714,11 +688,14 @@ void SqliteTableModel::removeCommentsFromQuery(QString& query) } } -QStringList SqliteTableModel::getColumns(const QString& sQuery, QVector& fieldsTypes) +QStringList SqliteTableModel::getColumns(std::shared_ptr pDb, const QString& sQuery, QVector& fieldsTypes) { + if(!pDb) + pDb = m_db.get("retrieving list of columns"); + sqlite3_stmt* stmt; QByteArray utf8Query = sQuery.toUtf8(); - int status = sqlite3_prepare_v2(m_db._db, utf8Query, utf8Query.size(), &stmt, nullptr); + int status = sqlite3_prepare_v2(pDb.get(), utf8Query, utf8Query.size(), &stmt, nullptr); QStringList listColumns; if(SQLITE_OK == status) { @@ -852,27 +829,38 @@ void SqliteTableModel::updateFilter(int column, const QString& value) void SqliteTableModel::clearCache() { - m_futureFetch.cancel(); - m_futureFetch.waitForFinished(); + m_lifeCounter++; - QMutexLocker lock(&m_mutexDataCache); - int size = m_data.size(); - if(size > 0) + if(m_db.isOpen()) { + worker->cancel(); + worker->waitUntilIdle(); + } + + if(m_currentRowCount > 0) { - lock.unlock(); - - beginRemoveRows(QModelIndex(), 0, size - 1); - { - QMutexLocker lock(&m_mutexDataCache); - m_data.clear(); - } + beginRemoveRows(QModelIndex(), 0, m_currentRowCount - 1); endRemoveRows(); } + + m_cache.clear(); + m_currentRowCount = 0; + m_rowCountAvailable = RowCount::Unknown; } bool SqliteTableModel::isBinary(const QModelIndex& index) const { - return !isTextOnly(m_data.at(index.row()).at(index.column()), m_encoding, true); + QMutexLocker lock(&m_mutexDataCache); + return nosync_isBinary(index); +} + +bool SqliteTableModel::nosync_isBinary(const QModelIndex& index) const +{ + if(!m_cache.count(index.row())) + return false; + + const auto & cached_row = m_cache.at(index.row()); + + return !isTextOnly(cached_row.at(index.column()), m_encoding, true); } QByteArray SqliteTableModel::encode(const QByteArray& str) const @@ -942,23 +930,51 @@ bool SqliteTableModel::isEditable() const return !m_sTable.isEmpty() && (m_db.getObjectByName(m_sTable)->type() == sqlb::Object::Types::Table || !m_pseudoPk.isEmpty()); } -void SqliteTableModel::waitForFetchingFinished() +void SqliteTableModel::triggerCacheLoad (int row) const { - if(m_futureFetch.isRunning()) - m_futureFetch.waitForFinished(); -} + size_t row_begin = std::max(0, row - int(m_chunkSize) / 2); + size_t row_end = row + m_chunkSize / 2; -void SqliteTableModel::cancelQuery() -{ - if(m_rowCount.isRunning()) - { - m_rowCount.cancel(); - m_rowCount = QtConcurrent::run([=]() { - // Make sure we report 0 rows if anybody asks - return 0; - }); + if(rowCountAvailable() == RowCount::Complete) { + row_end = std::min(row_end, size_t(rowCount())); + } else { + // will be truncated by reader } - if(m_futureFetch.isRunning()) - m_futureFetch.cancel(); + // avoid re-fetching data + QMutexLocker lk(&m_mutexDataCache); + m_cache.smallestNonAvailableRange(row_begin, row_end); + + if(row_end != row_begin) { + worker->triggerFetch(m_lifeCounter, row_begin, row_end); + } else { + //std::cout << "entire range already loaded\n"; + } +} + +void SqliteTableModel::triggerCacheLoad (int row_begin, int row_end) const +{ + if(row_end == row_begin) + return; + + triggerCacheLoad((row_begin + row_end) / 2); +} + +void SqliteTableModel::completeCache () const +{ + triggerCacheLoad(0, rowCount()); + worker->waitUntilIdle(); +} + +bool SqliteTableModel::isCacheComplete () const +{ + if(readingData()) + return false; + QMutexLocker lock(&m_mutexDataCache); + return m_cache.numSet() == m_currentRowCount; +} + +void SqliteTableModel::waitUntilIdle () const +{ + worker->waitUntilIdle(); } diff --git a/src/sqlitetablemodel.h b/src/sqlitetablemodel.h index 4170013c..5ea08938 100644 --- a/src/sqlitetablemodel.h +++ b/src/sqlitetablemodel.h @@ -1,15 +1,21 @@ #ifndef SQLITETABLEMODEL_H #define SQLITETABLEMODEL_H +#include + #include #include #include -#include +#include +#include #include "sqlitetypes.h" +#include "RowCache.h" +struct sqlite3; class DBBrowserDB; + class SqliteTableModel : public QAbstractTableModel { Q_OBJECT @@ -21,28 +27,65 @@ class SqliteTableModel : public QAbstractTableModel public: explicit SqliteTableModel(DBBrowserDB& db, QObject *parent = nullptr, size_t chunkSize = 50000, const QString& encoding = QString()); ~SqliteTableModel(); + + /// reset to state after construction void reset(); - int rowCount(const QModelIndex &parent = QModelIndex()) const; - int totalRowCount() const; - int columnCount(const QModelIndex &parent = QModelIndex()) const; - int filterCount() const; - QVariant headerData(int section, Qt::Orientation orientation, int role = Qt::DisplayRole) const; - QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const; - bool setData(const QModelIndex& index, const QVariant& value, int role = Qt::EditRole); - bool setTypedData(const QModelIndex& index, bool isBlob, const QVariant& value, int role = Qt::EditRole); - bool canFetchMore(const QModelIndex &parent = QModelIndex()) const; - void fetchMore(const QModelIndex &parent = QModelIndex()); + /// returns logical amount of rows, whether currently cached or not + int rowCount(const QModelIndex &parent = QModelIndex()) const override; - bool insertRows(int row, int count, const QModelIndex& parent = QModelIndex()); - bool removeRows(int row, int count, const QModelIndex& parent = QModelIndex()); + int columnCount(const QModelIndex &parent = QModelIndex()) const override; + int filterCount() const; + QVariant headerData(int section, Qt::Orientation orientation, int role = Qt::DisplayRole) const override; + QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override; + bool setData(const QModelIndex& index, const QVariant& value, int role = Qt::EditRole) override; + bool setTypedData(const QModelIndex& index, bool isBlob, const QVariant& value, int role = Qt::EditRole); + + enum class RowCount + { + Unknown, //< still finding out in background... + Partial, //< some chunk was read and at least a lower bound is thus known + Complete //< total row count of table known + }; + + /// what kind of information is available through rowCount()? + RowCount rowCountAvailable () const; + + /// trigger asynchronous loading of (at least) the specified row + /// into cache. + void triggerCacheLoad (int single_row) const; + + /// trigger asynchronous loading of (at least) the specified rows + /// into cache. \param row_end is exclusive. + void triggerCacheLoad (int row_begin, int row_end) const; + + /// wait until not reading any data (that does not mean data is + /// complete, just that the background reader is idle) + void waitUntilIdle () const; + + /// load all rows into cache, return when done + void completeCache () const; + + /// returns true if all rows are currently available in cache + /// [NOTE: potentially unsafe in case we have a limited-size + /// cache, where entries can vanish again -- however we can't do + /// this for the current implementation of the PlotDock] + bool isCacheComplete () const; + + bool insertRows(int row, int count, const QModelIndex& parent = QModelIndex()) override; + bool removeRows(int row, int count, const QModelIndex& parent = QModelIndex()) override; QModelIndex dittoRecord(int old_row); + /// configure for browsing results of specified query void setQuery(const QString& sQuery, bool dontClearHeaders = false); + QString query() const { return m_sQuery; } QString customQuery(bool withRowid); + + /// configure for browsing specified table void setTable(const sqlb::ObjectIdentifier& table, int sortColumn = 0, Qt::SortOrder sortOrder = Qt::AscendingOrder, const QVector &display_format = QVector()); + void setChunkSize(size_t chunksize); void sort(int column, Qt::SortOrder order = Qt::AscendingOrder); const sqlb::ObjectIdentifier& currentTableName() const { return m_sTable; } @@ -58,8 +101,6 @@ public: void setPseudoPk(const QString& pseudoPk); QString pseudoPk() const { return m_pseudoPk; } - typedef QVector QByteArrayList; - sqlb::ForeignKeyClause getForeignKeyClause(int column) const; // This returns true if the model is set up for editing. The model is able to operate in more or less two different modes, table browsing @@ -70,40 +111,62 @@ public: // Helper function for removing all comments from a SQL query static void removeCommentsFromQuery(QString& query); - // Call this if you want to wait until the thread which is currently fetching more data is finished - void waitForFetchingFinished(); - public slots: void updateFilter(int column, const QString& value); - // This cancels the execution of the current query. It can't guarantee that the query is stopped immediately or after returning but it should - // stop soon afterwards. If some data has already been loaded into the model, that data is not deleted - void cancelQuery(); - signals: - void finishedFetch(); + void finishedFetch(int fetched_row_begin, int fetched_row_end); protected: virtual Qt::DropActions supportedDropActions() const; virtual bool dropMimeData(const QMimeData* data, Qt::DropAction action, int row, int column, const QModelIndex& parent); private: - void fetchData(unsigned int from, unsigned to); + friend class RowLoader; + class RowLoader * worker; + + /// clears the cache, resets row-count to unknown (but keeps table + /// & query info), increase life_counter void clearCache(); + void handleFinishedFetch(int life_id, unsigned int fetched_row_begin, unsigned int fetched_row_end); + void handleRowCountComplete(int life_id, int num_rows); + void buildQuery(); - QStringList getColumns(const QString& sQuery, QVector& fieldsTypes); - int getQueryRowCount(); + + /// \param pDb connection to query; if null, obtains it from 'm_db'. + QStringList getColumns(std::shared_ptr pDb, const QString& sQuery, QVector& fieldsTypes); QByteArray encode(const QByteArray& str) const; QByteArray decode(const QByteArray& str) const; DBBrowserDB& m_db; - QFuture m_rowCount; - int m_rowCountAdjustment; // This needs to be added to the results of the m_rowCount future object to get the actual row count + + /// counts numbers of clearCache() since instantiation; using this + /// to avoid processing of queued signals originating in an era + /// before the most recent reset(). + int m_lifeCounter; + + /// note: the row count can be determined by the row-count query + /// (which yields the "final" row count"), or, if it is faster, by + /// the first chunk reading actual data (in which case the row + /// count will be set to that chunk's size and later updated to + /// the full row count, when the row-count query returns) + RowCount m_rowCountAvailable; + unsigned int m_currentRowCount; + QStringList m_headers; - typedef QList DataType; - DataType m_data; + + /// reading something in background right now? (either counting + /// rows or actually loading data, doesn't matter) + bool readingData() const; + + using Row = QVector; + mutable RowCache m_cache; + + Row makeDefaultCacheEntry () const; + + bool nosync_isBinary(const QModelIndex& index) const; QString m_sQuery; sqlb::ObjectIdentifier m_sTable; @@ -130,7 +193,6 @@ private: /** * These are used for multi-threaded population of the table */ - mutable QFuture m_futureFetch; mutable QMutex m_mutexDataCache; }; diff --git a/src/src.pro b/src/src.pro index b6b6f934..5ce86d5d 100644 --- a/src/src.pro +++ b/src/src.pro @@ -40,6 +40,8 @@ HEADERS += \ grammar/Sqlite3Parser.hpp \ grammar/sqlite3TokenTypes.hpp \ sqlitetablemodel.h \ + RowCache.h \ + RowLoader.h \ FilterTableHeader.h \ version.h \ SqlExecutionArea.h \ @@ -83,6 +85,7 @@ SOURCES += \ grammar/Sqlite3Lexer.cpp \ grammar/Sqlite3Parser.cpp \ sqlitetablemodel.cpp \ + RowLoader.cpp \ FilterTableHeader.cpp \ SqlExecutionArea.cpp \ VacuumDialog.cpp \ diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 8b5672a0..26a6e9d5 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -5,6 +5,7 @@ include_directories("${CMAKE_CURRENT_BINARY_DIR}" ..) set(TESTSQLOBJECTS_SRC ../sqlitedb.cpp ../sqlitetablemodel.cpp + ../RowLoader.cpp ../sqlitetypes.cpp ../csvparser.cpp ../grammar/Sqlite3Lexer.cpp @@ -53,7 +54,7 @@ else() endif() link_directories("${CMAKE_CURRENT_BINARY_DIR}/${QSCINTILLA_DIR}") add_dependencies(test-sqlobjects qscintilla2) -target_link_libraries(test-sqlobjects qscintilla2) +target_link_libraries(test-sqlobjects qscintilla2 pthread) add_test(test-sqlobjects test-sqlobjects) # test-import @@ -80,6 +81,7 @@ add_test(test-import test-import) set(TESTREGEX_SRC ../sqlitedb.cpp ../sqlitetablemodel.cpp + ../RowLoader.cpp ../sqlitetypes.cpp ../grammar/Sqlite3Lexer.cpp ../grammar/Sqlite3Parser.cpp @@ -124,5 +126,10 @@ else() endif() link_directories("${CMAKE_CURRENT_BINARY_DIR}/${QSCINTILLA_DIR}") add_dependencies(test-regex qscintilla2) -target_link_libraries(test-regex qscintilla2) +target_link_libraries(test-regex qscintilla2 pthread) add_test(test-regex test-regex) + + +add_executable(test-cache test_row_cache.cpp) +target_link_libraries(test-cache gtest_main) +add_test(NAME test-cache COMMAND test-cache) diff --git a/src/tests/test_row_cache.cpp b/src/tests/test_row_cache.cpp new file mode 100644 index 00000000..24e9e54f --- /dev/null +++ b/src/tests/test_row_cache.cpp @@ -0,0 +1,179 @@ + +#include "../RowCache.h" + +#include "gtest/gtest.h" + +using C = RowCache; + +TEST(RowCache, Construction) +{ + C c; + + EXPECT_EQ(0u, c.numSet()); + + EXPECT_FALSE(c.count(0)); + EXPECT_FALSE(c.count(1)); + EXPECT_FALSE(c.count(2)); + + EXPECT_THROW(c.at(0), std::out_of_range); +} + +TEST(RowCache, set_get) +{ + C c; + + c.set(1, 10); + c.set(5, 50); + c.set(0, 0); + c.set(6, 60); + c.set(100, 1000); + + EXPECT_EQ(5u, c.numSet()); + EXPECT_EQ(4u, c.numSegments()); // the '0' set after the '1' position does not merge currently + + int cnt = 0; + const C & cc = c; + for(size_t i = 0; i < 200; i++) { + if(c.count(i)) { + EXPECT_EQ(10*i, c.at(i)); + EXPECT_EQ(10*i, cc.at(i)); + cnt++; + } else { + EXPECT_THROW(c.at(i), std::out_of_range); + EXPECT_THROW(cc.at(i), std::out_of_range); + } + } + EXPECT_EQ(5, cnt); +} + +TEST(RowCache, insert) +{ + C c; + + c.insert(3, 30); + EXPECT_EQ(1u, c.numSet()); + EXPECT_EQ(1u, c.numSegments()); + EXPECT_EQ(30, c.at(3)); + + c.insert(3, 31); + EXPECT_EQ(2u, c.numSet()); + EXPECT_EQ(1u, c.numSegments()); + EXPECT_EQ(31, c.at(3)); + EXPECT_EQ(30, c.at(4)); + + c.insert(0, 0); + EXPECT_EQ(3u, c.numSet()); + EXPECT_EQ(2u, c.numSegments()); + EXPECT_EQ(0, c.at(0)); + EXPECT_THROW(c.at(3), std::out_of_range); + EXPECT_EQ(31, c.at(4)); + EXPECT_EQ(30, c.at(5)); + EXPECT_THROW(c.at(6), std::out_of_range); + + c.insert(1, 100); + EXPECT_EQ(4u, c.numSet()); + EXPECT_EQ(2u, c.numSegments()); + EXPECT_EQ(0, c.at(0)); + EXPECT_EQ(100, c.at(1)); + EXPECT_EQ(31, c.at(5)); + EXPECT_EQ(30, c.at(6)); + + c.insert(8, 1); + EXPECT_EQ(5u, c.numSet()); + EXPECT_EQ(3u, c.numSegments()); + EXPECT_EQ(0, c.at(0)); + EXPECT_EQ(100, c.at(1)); + EXPECT_EQ(31, c.at(5)); + EXPECT_EQ(30, c.at(6)); + EXPECT_EQ(1, c.at(8)); +} + +TEST(RowCache, erase) +{ + C c; + c.insert(3, 30); + c.insert(3, 31); + c.insert(0, 0); + c.insert(8, 1); + EXPECT_EQ(4u, c.numSet()); + EXPECT_EQ(3u, c.numSegments()); + EXPECT_EQ(0, c.at(0)); + EXPECT_EQ(31, c.at(4)); + EXPECT_EQ(30, c.at(5)); + EXPECT_EQ(1, c.at(8)); + + // erase entire segment + c.erase(0); + EXPECT_EQ(3u, c.numSet()); + EXPECT_EQ(2u, c.numSegments()); + EXPECT_EQ(31, c.at(3)); + EXPECT_EQ(30, c.at(4)); + EXPECT_EQ(1, c.at(7)); + + // erase inside segment + c.erase(4); + EXPECT_EQ(2u, c.numSet()); + EXPECT_EQ(2u, c.numSegments()); + EXPECT_EQ(31, c.at(3)); + EXPECT_EQ(1, c.at(6)); + + // erase non-filled row + c.erase(5); + EXPECT_EQ(2u, c.numSet()); + EXPECT_EQ(2u, c.numSegments()); + EXPECT_EQ(31, c.at(3)); + EXPECT_EQ(1, c.at(5)); + + c.erase(5); + EXPECT_EQ(1u, c.numSet()); + EXPECT_EQ(1u, c.numSegments()); + EXPECT_EQ(31, c.at(3)); + + c.erase(3); + EXPECT_EQ(0u, c.numSet()); + EXPECT_EQ(0u, c.numSegments()); +} + +TEST(RowCache, smallestNonAvailableRange) +{ + C c; + c.insert(3, 0); + c.insert(3, 0); + c.insert(0, 0); + c.insert(8, 0); + EXPECT_EQ(4u, c.numSet()); + EXPECT_TRUE(c.count(0)); + EXPECT_TRUE(c.count(4)); + EXPECT_TRUE(c.count(5)); + EXPECT_TRUE(c.count(8)); + + using P = std::pair; + + auto test = [&](int begin, int end) { + P p{ begin, end }; + c.smallestNonAvailableRange(p.first, p.second); + return p; + }; + + EXPECT_EQ(P( 0, 0), test( 0, 0)); + EXPECT_EQ(P( 1, 1), test( 0, 1)); + EXPECT_EQ(P( 1, 2), test( 0, 2)); + EXPECT_EQ(P( 1, 3), test( 0, 3)); + EXPECT_EQ(P( 1, 4), test( 0, 4)); + EXPECT_EQ(P( 1, 4), test( 0, 5)); + EXPECT_EQ(P( 1, 4), test( 0, 6)); + EXPECT_EQ(P( 1, 7), test( 0, 7)); + EXPECT_EQ(P( 1, 8), test( 0, 8)); + EXPECT_EQ(P( 1, 8), test( 0, 9)); + EXPECT_EQ(P( 1,10), test( 0,10)); + EXPECT_EQ(P( 1,10), test( 1,10)); + EXPECT_EQ(P( 2,10), test( 2,10)); + EXPECT_EQ(P( 3,10), test( 3,10)); + EXPECT_EQ(P( 6,10), test( 4,10)); + EXPECT_EQ(P( 6,10), test( 5,10)); + EXPECT_EQ(P( 6,10), test( 6,10)); + EXPECT_EQ(P( 7,10), test( 7,10)); + EXPECT_EQ(P( 9,10), test( 8,10)); + EXPECT_EQ(P( 9,10), test( 9,10)); + EXPECT_EQ(P(10,10), test(10,10)); +}