Improve loading of all data

The multithreading patch didn't properly load all data into the cache
when this was necessary. It would only do so if the chunk size was
sufficiently high. This is fixed in this commit.

Show a progress dialog while loading all data which can be cancelled by
the user.

When cancelling the loading of all data in those cases which require all
data to be loaded, stop whatever process needs the data too.

src/DbStructureModel.cpp

@@ -219,15 +219,19 @@ QMimeData* DbStructureModel::mimeData(const QModelIndexList& indices) const
                     sqlb::ObjectIdentifier objid(data(index.sibling(index.row(), ColumnSchema), Qt::DisplayRole).toString(),
                                                  data(index.sibling(index.row(), ColumnName), Qt::DisplayRole).toString());
                     tableModel.setTable(objid);
-                    tableModel.completeCache();
-                    for(int i=0; i < tableModel.rowCount(); ++i)
+                    if(tableModel.completeCache())
                     {
-                        QString insertStatement = "INSERT INTO " + objid.toString() + " VALUES(";
-                        for(int j=1; j < tableModel.columnCount(); ++j)
-                            insertStatement += QString("'%1',").arg(tableModel.data(tableModel.index(i, j), Qt::EditRole).toString());
-                        insertStatement.chop(1);
-                        insertStatement += ");\n";
-                        sqlData.append(insertStatement);
+                        // Only continue if all data was fetched
+
+                        for(int i=0; i < tableModel.rowCount(); ++i)
+                        {
+                            QString insertStatement = "INSERT INTO " + objid.toString() + " VALUES(";
+                            for(int j=1; j < tableModel.columnCount(); ++j)
+                                insertStatement += QString("'%1',").arg(tableModel.data(tableModel.index(i, j), Qt::EditRole).toString());
+                            insertStatement.chop(1);
+                            insertStatement += ");\n";
+                            sqlData.append(insertStatement);
+                        }
                     }
                 }
             }

src/EditTableDialog.cpp

@@ -388,7 +388,12 @@ void EditTableDialog::itemChanged(QTreeWidgetItem *item, int column)
                            .arg(sqlb::escapeIdentifier(pdb.getObjectByName(curTable).dynamicCast<sqlb::Table>()->rowidColumn()))
                            .arg(curTable.toString())
                            .arg(sqlb::escapeIdentifier(field->name())));
-                m.completeCache();
+                if(!m.completeCache())
+                {
+                    // If we couldn't load all data because the cancel button was clicked, just unset the checkbox again and stop.
+                    item->setCheckState(column, Qt::Unchecked);
+                    return;
+                }
                 if(m.data(m.index(0, 0)).toInt() > 0)
                 {
                     // There is a NULL value, so print an error message, uncheck the combobox, and return here
@@ -416,7 +421,12 @@ void EditTableDialog::itemChanged(QTreeWidgetItem *item, int column)
                                .arg(curTable.toString())
                                .arg(sqlb::escapeIdentifier(field->name()))
                                .arg(sqlb::escapeIdentifier(field->name())));
-                    m.completeCache();
+                    if(!m.completeCache())
+                    {
+                        // If we couldn't load all data because the cancel button was clicked, just unset the checkbox again and stop.
+                        item->setCheckState(column, Qt::Unchecked);
+                        return;
+                    }
                     if(m.data(m.index(0, 0)).toInt() > 0)
                     {
                         // There is a non-integer value, so print an error message, uncheck the combobox, and return here
@@ -458,10 +468,20 @@ void EditTableDialog::itemChanged(QTreeWidgetItem *item, int column)
                 // Because our renameColumn() function fails when setting a column to unique when it already contains the same values
                 SqliteTableModel m(pdb, this);
                 m.setQuery(QString("SELECT COUNT(%2) FROM %1;").arg(curTable.toString()).arg(sqlb::escapeIdentifier(field->name())));
-                m.completeCache();
+                if(!m.completeCache())
+                {
+                    // If we couldn't load all data because the cancel button was clicked, just unset the checkbox again and stop.
+                    item->setCheckState(column, Qt::Unchecked);
+                    return;
+                }
                 int rowcount = m.data(m.index(0, 0)).toInt();
                 m.setQuery(QString("SELECT COUNT(DISTINCT %2) FROM %1;").arg(curTable.toString()).arg(sqlb::escapeIdentifier(field->name())));
-                m.completeCache();
+                if(!m.completeCache())
+                {
+                    // If we couldn't load all data because the cancel button was clicked, just unset the checkbox again and stop.
+                    item->setCheckState(column, Qt::Unchecked);
+                    return;
+                }
                 int uniquecount = m.data(m.index(0, 0)).toInt();
                 if(rowcount != uniquecount)
                 {

src/PlotDock.cpp

@@ -733,15 +733,7 @@ void PlotDock::fetchAllData()
 {
     if(m_currentPlotModel)
     {
-        // Show progress dialog because fetching all data might take some time
-        QProgressDialog progress(tr("Fetching all data..."),
-                                 tr("Cancel"), m_currentPlotModel->rowCount(), m_currentPlotModel->rowCount());
-        progress.setWindowModality(Qt::ApplicationModal);
-        progress.show();
-        qApp->processEvents();
-
         // Make sure all data is loaded
-        // TODO make this cancellable & show progress
         m_currentPlotModel->completeCache();
 
         // Update plot

src/sqlitetablemodel.cpp

@@ -11,6 +11,7 @@
 #include <QFile>
 #include <QUrl>
 #include <QtConcurrent/QtConcurrentRun>
+#include <QProgressDialog>
 
 #include "RowLoader.h"
 
@@ -939,10 +940,30 @@ void SqliteTableModel::triggerCacheLoad (int row_begin, int row_end) const
     triggerCacheLoad((row_begin + row_end) / 2);
 }
 
-void SqliteTableModel::completeCache () const
+bool SqliteTableModel::completeCache () const
 {
-    triggerCacheLoad(0, rowCount());
-    worker->waitUntilIdle();
+    // Show progress dialog because fetching all data might take some time but only show
+    // cancel button if we allow cancellation here. This isn't
+    QProgressDialog progress(tr("Fetching data..."),
+                             tr("Cancel"), 0, rowCount());
+    progress.setWindowModality(Qt::ApplicationModal);
+    progress.show();
+
+    waitUntilIdle();
+
+    // This loop fetches all data by loading it block by block into the cache
+    for(int i=0;i<rowCount()+static_cast<int>(m_chunkSize)/2;i+=m_chunkSize)
+    {
+        progress.setValue(i);
+        qApp->processEvents();
+        if(progress.wasCanceled())
+            return false;
+
+        triggerCacheLoad(i);
+        worker->waitUntilIdle();
+    }
+
+    return true;
 }
 
 bool SqliteTableModel::isCacheComplete () const

src/sqlitetablemodel.h

@@ -61,8 +61,8 @@ public:
     /// complete, just that the background reader is idle)
     void waitUntilIdle () const;
 
-    /// load all rows into cache, return when done
-    void completeCache () const;
+    /// load all rows into cache, return when done. Returns true if all data was loaded, false if the loading was cancelled.
+    bool completeCache() const;
 
     /// returns true if all rows are currently available in cache
     /// [NOTE: potentially unsafe in case we have a limited-size