From 28b8652ad6a4d7826995fdae8264f64aefa87eed Mon Sep 17 00:00:00 2001 From: Martin Kleusberg Date: Fri, 8 Jun 2018 18:13:47 +0200 Subject: [PATCH] Improve loading of all data The multithreading patch didn't properly load all data into the cache when this was necessary. It would only do so if the chunk size was sufficiently high. This is fixed in this commit. Show a progress dialog while loading all data which can be cancelled by the user. When cancelling the loading of all data in those cases which require all data to be loaded, stop whatever process needs the data too. --- src/DbStructureModel.cpp | 20 ++++++++++++-------- src/EditTableDialog.cpp | 28 ++++++++++++++++++++++++---- src/PlotDock.cpp | 8 -------- src/sqlitetablemodel.cpp | 27 ++++++++++++++++++++++++--- src/sqlitetablemodel.h | 4 ++-- 5 files changed, 62 insertions(+), 25 deletions(-) diff --git a/src/DbStructureModel.cpp b/src/DbStructureModel.cpp index 305297be..4019c9fb 100644 --- a/src/DbStructureModel.cpp +++ b/src/DbStructureModel.cpp @@ -219,15 +219,19 @@ QMimeData* DbStructureModel::mimeData(const QModelIndexList& indices) const sqlb::ObjectIdentifier objid(data(index.sibling(index.row(), ColumnSchema), Qt::DisplayRole).toString(), data(index.sibling(index.row(), ColumnName), Qt::DisplayRole).toString()); tableModel.setTable(objid); - tableModel.completeCache(); - for(int i=0; i < tableModel.rowCount(); ++i) + if(tableModel.completeCache()) { - QString insertStatement = "INSERT INTO " + objid.toString() + " VALUES("; - for(int j=1; j < tableModel.columnCount(); ++j) - insertStatement += QString("'%1',").arg(tableModel.data(tableModel.index(i, j), Qt::EditRole).toString()); - insertStatement.chop(1); - insertStatement += ");\n"; - sqlData.append(insertStatement); + // Only continue if all data was fetched + + for(int i=0; i < tableModel.rowCount(); ++i) + { + QString insertStatement = "INSERT INTO " + objid.toString() + " VALUES("; + for(int j=1; j < tableModel.columnCount(); ++j) + insertStatement += QString("'%1',").arg(tableModel.data(tableModel.index(i, j), Qt::EditRole).toString()); + insertStatement.chop(1); + insertStatement += ");\n"; + sqlData.append(insertStatement); + } } } } diff --git a/src/EditTableDialog.cpp b/src/EditTableDialog.cpp index 8512fa2e..9de6fe3c 100644 --- a/src/EditTableDialog.cpp +++ b/src/EditTableDialog.cpp @@ -388,7 +388,12 @@ void EditTableDialog::itemChanged(QTreeWidgetItem *item, int column) .arg(sqlb::escapeIdentifier(pdb.getObjectByName(curTable).dynamicCast()->rowidColumn())) .arg(curTable.toString()) .arg(sqlb::escapeIdentifier(field->name()))); - m.completeCache(); + if(!m.completeCache()) + { + // If we couldn't load all data because the cancel button was clicked, just unset the checkbox again and stop. + item->setCheckState(column, Qt::Unchecked); + return; + } if(m.data(m.index(0, 0)).toInt() > 0) { // There is a NULL value, so print an error message, uncheck the combobox, and return here @@ -416,7 +421,12 @@ void EditTableDialog::itemChanged(QTreeWidgetItem *item, int column) .arg(curTable.toString()) .arg(sqlb::escapeIdentifier(field->name())) .arg(sqlb::escapeIdentifier(field->name()))); - m.completeCache(); + if(!m.completeCache()) + { + // If we couldn't load all data because the cancel button was clicked, just unset the checkbox again and stop. + item->setCheckState(column, Qt::Unchecked); + return; + } if(m.data(m.index(0, 0)).toInt() > 0) { // There is a non-integer value, so print an error message, uncheck the combobox, and return here @@ -458,10 +468,20 @@ void EditTableDialog::itemChanged(QTreeWidgetItem *item, int column) // Because our renameColumn() function fails when setting a column to unique when it already contains the same values SqliteTableModel m(pdb, this); m.setQuery(QString("SELECT COUNT(%2) FROM %1;").arg(curTable.toString()).arg(sqlb::escapeIdentifier(field->name()))); - m.completeCache(); + if(!m.completeCache()) + { + // If we couldn't load all data because the cancel button was clicked, just unset the checkbox again and stop. + item->setCheckState(column, Qt::Unchecked); + return; + } int rowcount = m.data(m.index(0, 0)).toInt(); m.setQuery(QString("SELECT COUNT(DISTINCT %2) FROM %1;").arg(curTable.toString()).arg(sqlb::escapeIdentifier(field->name()))); - m.completeCache(); + if(!m.completeCache()) + { + // If we couldn't load all data because the cancel button was clicked, just unset the checkbox again and stop. + item->setCheckState(column, Qt::Unchecked); + return; + } int uniquecount = m.data(m.index(0, 0)).toInt(); if(rowcount != uniquecount) { diff --git a/src/PlotDock.cpp b/src/PlotDock.cpp index 316541dd..36ff9957 100644 --- a/src/PlotDock.cpp +++ b/src/PlotDock.cpp @@ -733,15 +733,7 @@ void PlotDock::fetchAllData() { if(m_currentPlotModel) { - // Show progress dialog because fetching all data might take some time - QProgressDialog progress(tr("Fetching all data..."), - tr("Cancel"), m_currentPlotModel->rowCount(), m_currentPlotModel->rowCount()); - progress.setWindowModality(Qt::ApplicationModal); - progress.show(); - qApp->processEvents(); - // Make sure all data is loaded - // TODO make this cancellable & show progress m_currentPlotModel->completeCache(); // Update plot diff --git a/src/sqlitetablemodel.cpp b/src/sqlitetablemodel.cpp index 4df1de79..710af30f 100644 --- a/src/sqlitetablemodel.cpp +++ b/src/sqlitetablemodel.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "RowLoader.h" @@ -939,10 +940,30 @@ void SqliteTableModel::triggerCacheLoad (int row_begin, int row_end) const triggerCacheLoad((row_begin + row_end) / 2); } -void SqliteTableModel::completeCache () const +bool SqliteTableModel::completeCache () const { - triggerCacheLoad(0, rowCount()); - worker->waitUntilIdle(); + // Show progress dialog because fetching all data might take some time but only show + // cancel button if we allow cancellation here. This isn't + QProgressDialog progress(tr("Fetching data..."), + tr("Cancel"), 0, rowCount()); + progress.setWindowModality(Qt::ApplicationModal); + progress.show(); + + waitUntilIdle(); + + // This loop fetches all data by loading it block by block into the cache + for(int i=0;i(m_chunkSize)/2;i+=m_chunkSize) + { + progress.setValue(i); + qApp->processEvents(); + if(progress.wasCanceled()) + return false; + + triggerCacheLoad(i); + worker->waitUntilIdle(); + } + + return true; } bool SqliteTableModel::isCacheComplete () const diff --git a/src/sqlitetablemodel.h b/src/sqlitetablemodel.h index b261ded9..fd7a9bf3 100644 --- a/src/sqlitetablemodel.h +++ b/src/sqlitetablemodel.h @@ -61,8 +61,8 @@ public: /// complete, just that the background reader is idle) void waitUntilIdle () const; - /// load all rows into cache, return when done - void completeCache () const; + /// load all rows into cache, return when done. Returns true if all data was loaded, false if the loading was cancelled. + bool completeCache() const; /// returns true if all rows are currently available in cache /// [NOTE: potentially unsafe in case we have a limited-size