From 43a5175af10c18fc81dbbb971c39729251dcd3db Mon Sep 17 00:00:00 2001 From: Martin Kleusberg Date: Thu, 23 Mar 2017 19:08:26 +0100 Subject: [PATCH] dbhub: Keep track of the cloned databases This adds a local database to keep track of all the cloned databases. For now we only use this information to prevent exactly the same database being downloaded twice. --- src/MainWindow.cpp | 1 + src/MainWindow.h | 2 +- src/PreferencesDialog.cpp | 1 + src/RemoteDatabase.cpp | 195 +++++++++++++++++++++++++++++++++++++- src/RemoteDatabase.h | 7 ++ 5 files changed, 204 insertions(+), 2 deletions(-) diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp index 18074977..49782ef9 100644 --- a/src/MainWindow.cpp +++ b/src/MainWindow.cpp @@ -22,6 +22,7 @@ #include "ColumnDisplayFormatDialog.h" #include "FilterTableHeader.h" #include "RemoteDock.h" +#include "RemoteDatabase.h" #include #include diff --git a/src/MainWindow.h b/src/MainWindow.h index 93ac5cdc..f0fe0a79 100644 --- a/src/MainWindow.h +++ b/src/MainWindow.h @@ -2,7 +2,6 @@ #define MAINWINDOW_H #include "sqlitedb.h" -#include "RemoteDatabase.h" #include "PlotDock.h" #include @@ -17,6 +16,7 @@ class QPersistentModelIndex; class SqliteTableModel; class DbStructureModel; class RemoteDock; +class RemoteDatabase; namespace Ui { class MainWindow; diff --git a/src/PreferencesDialog.cpp b/src/PreferencesDialog.cpp index 4a794298..4e53f151 100644 --- a/src/PreferencesDialog.cpp +++ b/src/PreferencesDialog.cpp @@ -4,6 +4,7 @@ #include "Settings.h" #include "Application.h" #include "MainWindow.h" +#include "RemoteDatabase.h" #include #include diff --git a/src/RemoteDatabase.cpp b/src/RemoteDatabase.cpp index 212df971..7ab767f5 100644 --- a/src/RemoteDatabase.cpp +++ b/src/RemoteDatabase.cpp @@ -7,15 +7,19 @@ #include #include #include +#include +#include #include "RemoteDatabase.h" #include "version.h" #include "Settings.h" +#include "sqlite.h" RemoteDatabase::RemoteDatabase() : m_manager(new QNetworkAccessManager), m_progress(nullptr), - m_currentReply(nullptr) + m_currentReply(nullptr), + m_dbLocal(nullptr) { // Set up SSL configuration m_sslConfiguration = QSslConfiguration::defaultConfiguration(); @@ -42,6 +46,10 @@ RemoteDatabase::~RemoteDatabase() { delete m_manager; delete m_progress; + + // Close local storage db - but only if it was created/opened in the meantime + if(m_dbLocal) + sqlite3_close(m_dbLocal); } void RemoteDatabase::reloadSettings() @@ -124,6 +132,9 @@ void RemoteDatabase::gotReply(QNetworkReply* reply) QString saveFileAs = Settings::getValue("remote", "clonedirectory").toString() + QString("/%2_%1.remotedb").arg(QDateTime::currentMSecsSinceEpoch()).arg(reply->url().fileName()); + // Add cloned database to list of local databases + localAdd(saveFileAs, reply->property("certfile").toString(), reply->url()); + // Save the downloaded data under the generated file name QFile file(saveFileAs); file.open(QIODevice::WriteOnly); @@ -286,6 +297,19 @@ void RemoteDatabase::fetch(const QString& url, RequestType type, const QString& return; } + // If this is a request for a database there is a chance that we've already cloned that database. So check for that first + if(type == RequestTypeDatabase) + { + QString exists = localExists(url, clientCert); + if(!exists.isEmpty()) + { + // Database has already been cloned! So open the local file instead of fetching the one from the + // server again. + emit openFile(exists); + return; + } + } + // Build network request QNetworkRequest request; request.setUrl(url); @@ -357,3 +381,172 @@ void RemoteDatabase::push(const QString& filename, const QString& url, const QSt // Initialise the progress dialog for this request prepareProgressDialog(true, url); } + +void RemoteDatabase::localAssureOpened() +{ + // This function should be called first in each RemoteDatabase::local* function. It assures the database for storing + // the local database information is opened and ready. If the database file doesn't exist yet it is created by this + // function. If the database file is already created and opened this function does nothing. The reason to open the + // database on first use instead of doing that in the constructor of this class is that this way no database file is + // going to be created and no database handle is held when it's not actually needed. For people not interested in + // the dbhub.io functionality this means no unnecessary files being created. + + // Check if database is already opened and return if it is + if(m_dbLocal) + return; + + // Open file + QString database_file = QStandardPaths::writableLocation(QStandardPaths::AppDataLocation) + "/remotedbs.db"; + if(sqlite3_open_v2(database_file.toUtf8(), &m_dbLocal, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL) != SQLITE_OK) + { + QMessageBox::warning(nullptr, qApp->applicationName(), tr("Error opening local databases list.\n%1").arg(QString::fromUtf8(sqlite3_errmsg(m_dbLocal)))); + return; + } + + // Create local local table if it doesn't exists yet + char* errmsg; + QString statement = QString("CREATE TABLE IF NOT EXISTS \"local\"(" + "\"id\" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT," + "\"identity\" TEXT NOT NULL," + "\"name\" TEXT NOT NULL," + "\"url\" TEXT NOT NULL," + "\"version\" INTEGER NOT NULL," + "\"file\" INTEGER," + "\"modified\" INTEGER DEFAULT 0" + ")"); + if(sqlite3_exec(m_dbLocal, statement.toUtf8(), NULL, NULL, &errmsg) != SQLITE_OK) + { + QMessageBox::warning(nullptr, qApp->applicationName(), tr("Error creating local databases list.\n%1").arg(QString::fromUtf8(errmsg))); + sqlite3_free(errmsg); + sqlite3_close(m_dbLocal); + m_dbLocal = nullptr; + return; + } +} + +void RemoteDatabase::localAdd(QString filename, QString identity, const QUrl& url) +{ + // This function adds a new local database clone to our internal list. It does so by adding a single + // new record to the remote dbs database. All the fields are extracted from the filename, the identity + // and (most importantly) the url parameters. Note that for the version field to be correctly filled we + // require the version to be part of the url parameter. Also note that this function doesn't check if the + // database has already been added to the list before. This needs to be done before calling this function, + // ideally even before sending out a request to the network. + + localAssureOpened(); + + // Insert database into local database list + QString sql = QString("INSERT INTO local(identity, name, url, version, file) VALUES(?, ?, ?, ?, ?)"); + sqlite3_stmt* stmt; + if(sqlite3_prepare_v2(m_dbLocal, sql.toUtf8(), -1, &stmt, 0) != SQLITE_OK) + return; + + QFileInfo f(identity); // Remove the path + identity = f.fileName(); + if(sqlite3_bind_text(stmt, 1, identity.toUtf8(), identity.toUtf8().length(), SQLITE_TRANSIENT)) + { + sqlite3_finalize(stmt); + return; + } + + if(sqlite3_bind_text(stmt, 2, url.fileName().toUtf8(), url.fileName().toUtf8().length(), SQLITE_TRANSIENT)) + { + sqlite3_finalize(stmt); + return; + } + + QUrl url_without_query = url; // Remove the '?version=x' bit from the URL + url_without_query.setQuery(QString()); + if(sqlite3_bind_text(stmt, 3, url_without_query.toString().toUtf8(), url_without_query.toString().toUtf8().length(), SQLITE_TRANSIENT)) + { + sqlite3_finalize(stmt); + return; + } + + if(sqlite3_bind_int(stmt, 4, QUrlQuery(url).queryItemValue("version").toInt())) + { + sqlite3_finalize(stmt); + return; + } + + f = QFileInfo(filename); // Remove the path + filename = f.fileName(); + if(sqlite3_bind_text(stmt, 5, filename.toUtf8(), filename.toUtf8().length(), SQLITE_TRANSIENT)) + { + sqlite3_finalize(stmt); + return; + } + + if(sqlite3_step(stmt) != SQLITE_DONE) + { + sqlite3_finalize(stmt); + return; + } + + sqlite3_finalize(stmt); +} + +QString RemoteDatabase::localExists(const QUrl& url, QString identity) +{ + // This function checks if there already is a clone for the given combination of url and identity. It returns the filename + // of this clone if there is or a null string if there isn't a clone yet. The identity needs to be part of this check because + // with the url alone there could be corner cases where different versions or whatever may not be accessible for all users. + + localAssureOpened(); + + // Extract version from url and remove query part afterwards + int url_version = QUrlQuery(url).queryItemValue("version").toInt(); + QUrl url_without_query = url; + url_without_query.setQuery(QString()); + + // Query version and filename for the given combination of url and identity + QString sql = QString("SELECT id, version, file FROM local WHERE url=? AND identity=?"); + sqlite3_stmt* stmt; + if(sqlite3_prepare_v2(m_dbLocal, sql.toUtf8(), -1, &stmt, 0) != SQLITE_OK) + return QString(); + + if(sqlite3_bind_text(stmt, 1, url_without_query.toString().toUtf8(), url_without_query.toString().toUtf8().length(), SQLITE_TRANSIENT)) + { + sqlite3_finalize(stmt); + return QString(); + } + + QFileInfo f(identity); // Remove the path + identity = f.fileName(); + if(sqlite3_bind_text(stmt, 2, identity.toUtf8(), identity.toUtf8().length(), SQLITE_TRANSIENT)) + { + sqlite3_finalize(stmt); + return QString(); + } + + if(sqlite3_step(stmt) != SQLITE_ROW) + { + // If there was either an error or no record was found for this combination of url and + // identity, stop here. + sqlite3_finalize(stmt); + return QString(); + } + + // Having come here we can assume that at least some local clone for the given combination of + // url and identity exists. So extract all the information we have on it. + //int local_id = sqlite3_column_int(stmt, 0); + int local_version = sqlite3_column_int(stmt, 1); + QString local_file = QString::fromUtf8(reinterpret_cast(sqlite3_column_text(stmt, 2))); + sqlite3_finalize(stmt); + + // There are three possibilities now: either the requested version is the same as the local version, or the requested version + // is newer, or the local version is newer. + if(local_version == url_version) + { + // Both versions are the same. That's the perfect match, so just return the path to the local file + return Settings::getValue("remote", "clonedirectory").toString() + "/" + local_file; + } else { + // In all the other cases just treat the remote database as a completely new database for now. + + // TODO Add some way to update the local clone here. Maybe ask the user what to do because I don't really know what the + // most sensible way to go is in the two remaining cases. We can use the local_id variable (see above) to update the + // record afterwards. + + return QString(); + } +} diff --git a/src/RemoteDatabase.h b/src/RemoteDatabase.h index 4423c773..7f7ef7aa 100644 --- a/src/RemoteDatabase.h +++ b/src/RemoteDatabase.h @@ -10,6 +10,7 @@ class QNetworkReply; class QSslError; class QProgressDialog; class QNetworkRequest; +struct sqlite3; class RemoteDatabase : public QObject { @@ -48,11 +49,17 @@ private: bool prepareSsl(QNetworkRequest* request, const QString& clientCert); void prepareProgressDialog(bool upload, const QString& url); + // Helper functions for managing the list of locally available databases + void localAssureOpened(); + void localAdd(QString filename, QString identity, const QUrl& url); + QString localExists(const QUrl& url, QString identity); + QNetworkAccessManager* m_manager; QProgressDialog* m_progress; QNetworkReply* m_currentReply; QSslConfiguration m_sslConfiguration; QMap m_clientCertFiles; + sqlite3* m_dbLocal; }; #endif