From 549992b5e3e7b609156e63a215cb44cad791d3df Mon Sep 17 00:00:00 2001 From: Alexander Bock Date: Thu, 11 Jun 2015 02:03:50 +0200 Subject: [PATCH] Work to using the request mechanism for downloading --- apps/Launcher/mainwindow.cpp | 1 + apps/Launcher/syncwidget.cpp | 184 +++++++++++++++------ apps/Launcher/syncwidget.h | 29 +++- include/openspace/engine/downloadmanager.h | 22 ++- src/engine/downloadmanager.cpp | 38 ++++- src/engine/openspaceengine.cpp | 3 +- 6 files changed, 215 insertions(+), 62 deletions(-) diff --git a/apps/Launcher/mainwindow.cpp b/apps/Launcher/mainwindow.cpp index a38594d25d..2efa0fc3eb 100644 --- a/apps/Launcher/mainwindow.cpp +++ b/apps/Launcher/mainwindow.cpp @@ -153,6 +153,7 @@ void MainWindow::initialize() { _scenes->addItem(i.fileName()); } _syncWidget.setSceneFiles(_sceneFiles); + _syncWidget.setModulesDirectory(ModulesDirectory); } void MainWindow::shortcutButtonPressed() { diff --git a/apps/Launcher/syncwidget.cpp b/apps/Launcher/syncwidget.cpp index fc12bc950a..c984031480 100644 --- a/apps/Launcher/syncwidget.cpp +++ b/apps/Launcher/syncwidget.cpp @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -41,6 +42,17 @@ namespace { const int nColumns = 3; + + const int DownloadApplicationVersion = 1; + + const std::string FileDownloadKey = "FileDownload"; + const std::string FileRequestKey = "FileRequest"; + const std::string TorrentFilesKey = "TorrentFiles"; + + const std::string UrlKey = "URL"; + const std::string DestinationKey = "Destination"; + const std::string IdentifierKey = "Identifier"; + const std::string VersionKey = "Version"; } SyncWidget::SyncWidget(QWidget* parent) @@ -58,13 +70,18 @@ SyncWidget::SyncWidget(QWidget* parent) } { QPushButton* syncButton = new QPushButton("Synchronize Scenes"); + QObject::connect( + syncButton, SIGNAL(clicked(bool)), + this, SLOT(syncButtonPressed()) + ); + layout->addWidget(syncButton); } setLayout(layout); ghoul::initialize(); - openspace::DownloadManager::initialize("http://openspace.itn.liu.se/request.cgi"); + openspace::DownloadManager::initialize("http://openspace.itn.liu.se/data/request", DownloadApplicationVersion); } SyncWidget::~SyncWidget() { @@ -86,82 +103,147 @@ void SyncWidget::setSceneFiles(QMap sceneFiles) { } } +void SyncWidget::setModulesDirectory(QString modulesDirectory) { + _modulesDirectory = std::move(modulesDirectory); +} + void SyncWidget::clear() { } -void SyncWidget::handleDirectFiles(QString module, QStringList files) { - qDebug() << files; +void SyncWidget::handleDirectFiles(QString module, DirectFiles files) { + qDebug() << "Direct Files"; + for (const DirectFile& f : files) { + qDebug() << f.url << " -> " << f.destination; + + DlManager.downloadFile( + f.url.toStdString(), + fullPath(module, f.destination).toStdString(), + [](const ghoul::filesystem::File& f) { qDebug() << QString::fromStdString(f.filename()) << "finished";} + ); + } } -void SyncWidget::handleTorrentFiles(QString module, QStringList torrents) { - qDebug() << torrents; +void SyncWidget::handleFileRequest(QString module, FileRequests files) { + qDebug() << "File Requests"; + for (const FileRequest& f : files) { + qDebug() << f.identifier << " (" << f.version << ")" << " -> " << f.destination; + + + } +} + +void SyncWidget::handleTorrentFiles(QString module, TorrentFiles files) { + qDebug() << "Torrent Files"; + for (const TorrentFile& f : files) { + qDebug() << f.file; + } } void SyncWidget::syncButtonPressed() { clear(); - + for (const QString& scene : selectedScenes()) { + qDebug() << scene; + ghoul::Dictionary sceneDictionary; + ghoul::lua::loadDictionaryFromFile( + scene.toStdString(), + sceneDictionary + ); - qDebug() << _sceneFiles; + ghoul::Dictionary modules; + bool success = sceneDictionary.getValue("Modules", modules); + qDebug() << success; - ghoul::Dictionary sceneDictionary; - ghoul::lua::loadDictionaryFromFile( - scene.toStdString(), - sceneDictionary - ); + QStringList modulesList; + for (int i = 1; i <= modules.size(); ++i) { + std::string module = modules.value(std::to_string(i)); + modulesList.append(QString::fromStdString(module)); + } + qDebug() << modulesList; - ghoul::Dictionary modules; - bool success = sceneDictionary.getValue("Modules", modules); - qDebug() << success; + QDir sceneDir(scene); + sceneDir.cdUp(); + for (QString module : modulesList) { + QString dataFile = sceneDir.absoluteFilePath(module + "/" + module + ".data"); - QStringList modulesList; - for (int i = 1; i < modules.size(); ++i) { - std::string module = modules.value(std::to_string(i)); - modulesList.append(QString::fromStdString(module)); - } - qDebug() << modulesList; + qDebug() << module; + qDebug() << dataFile << QFileInfo(dataFile).exists(); - QDir sceneDir(scene); - sceneDir.cdUp(); - for (QString module : modulesList) { - QString moduleFile = sceneDir.absoluteFilePath(module + "/" + module + ".mod"); - QString dataFile = sceneDir.absoluteFilePath(module + "/" + module + ".data"); + if (QFileInfo(dataFile).exists()) { + ghoul::Dictionary dataDictionary; + ghoul::lua::loadDictionaryFromFile(dataFile.toStdString(), dataDictionary); - qDebug() << module; - qDebug() << moduleFile << QFileInfo(moduleFile).exists(); - qDebug() << dataFile << QFileInfo(dataFile).exists(); + ghoul::Dictionary directDownloadFiles; + ghoul::Dictionary fileRequests; + ghoul::Dictionary torrentFiles; - if (QFileInfo(dataFile).exists()) { - ghoul::Dictionary dataDictionary; - ghoul::lua::loadDictionaryFromFile(dataFile.toStdString(), dataDictionary); + bool found = dataDictionary.getValue(FileDownloadKey, directDownloadFiles); + if (found) { + DirectFiles files; + //QStringList files; + for (int i = 1; i <= directDownloadFiles.size(); ++i) { + ghoul::Dictionary d = directDownloadFiles.value(std::to_string(i)); + std::string url = d.value(UrlKey); + std::string dest = d.value(DestinationKey); - ghoul::Dictionary directFiles; - ghoul::Dictionary torrentFiles; - - bool found = dataDictionary.getValue("Files", directFiles); - if (found) { - QStringList files; - for (int i = 1; i < directFiles.size(); ++i) { - std::string f = directFiles.value(std::to_string(i)); - files.append(QString::fromStdString(f)); + files.append({ + QString::fromStdString(url), + QString::fromStdString(dest) + }); + } + handleDirectFiles(module, files); } - handleDirectFiles(module, files); - } - found = dataDictionary.getValue("Torrents", torrentFiles); - if (found) { - QStringList torrents; - for (int i = 1; i < torrentFiles.size(); ++i) { - std::string f = torrentFiles.value(std::to_string(i)); - torrents.append(QString::fromStdString(f)); + found = dataDictionary.getValue(FileRequestKey, fileRequests); + if (found) { + FileRequests files; + for (int i = 1; i <= fileRequests.size(); ++i) { + ghoul::Dictionary d = fileRequests.value(std::to_string(i)); + std::string url = d.value(IdentifierKey); + std::string dest = d.value(DestinationKey); + int version = static_cast(d.value(VersionKey)); + + + files.append({ + QString::fromStdString(url), + QString::fromStdString(dest), + version + }); + } + handleFileRequest(module, files); + } + + found = dataDictionary.getValue(TorrentFilesKey, torrentFiles); + if (found) { + TorrentFiles torrents; + for (int i = 1; i <= torrentFiles.size(); ++i) { + std::string f = torrentFiles.value(std::to_string(i)); + + torrents.append({QString::fromStdString(f)}); + } + handleTorrentFiles(module, torrents); } - handleTorrentFiles(module, torrents); } } } } QStringList SyncWidget::selectedScenes() const { - + QStringList result; + int nChildren = _sceneLayout->count(); + for (int i = 0; i < nChildren; ++i) { + QWidget* w = _sceneLayout->itemAt(i)->widget(); + QCheckBox* c = static_cast(w); + if (c->isChecked()) { + QString t = c->text(); + result.append(_sceneFiles[t]); + } + } + qDebug() << result; + return result; +} + +QString SyncWidget::fullPath(QString module, QString destination) const { + return _modulesDirectory + "/" + module + "/" + destination; } diff --git a/apps/Launcher/syncwidget.h b/apps/Launcher/syncwidget.h index 4970488523..0353409a3a 100644 --- a/apps/Launcher/syncwidget.h +++ b/apps/Launcher/syncwidget.h @@ -37,19 +37,42 @@ public: SyncWidget(QWidget* parent); ~SyncWidget(); - void setSceneFiles(QMap sceneFiles); + void setSceneFiles(QMap sceneFiles); + void setModulesDirectory(QString modulesDirectory); private slots: void syncButtonPressed(); private: + struct DirectFile { + QString url; + QString destination; + }; + typedef QList DirectFiles; + + struct FileRequest { + QString identifier; + QString destination; + int version; + }; + typedef QList FileRequests; + + struct TorrentFile { + QString file; + }; + typedef QList TorrentFiles; + void clear(); QStringList selectedScenes() const; - void handleDirectFiles(QString module, QStringList files); - void handleTorrentFiles(QString module, QStringList torrents); + QString fullPath(QString module, QString destination) const; + + void handleDirectFiles(QString module, DirectFiles files); + void handleFileRequest(QString module, FileRequests files); + void handleTorrentFiles(QString module, TorrentFiles files); QMap _sceneFiles; + QString _modulesDirectory; QGridLayout* _sceneLayout; }; diff --git a/include/openspace/engine/downloadmanager.h b/include/openspace/engine/downloadmanager.h index c72ef39c34..13f1f1f7e4 100644 --- a/include/openspace/engine/downloadmanager.h +++ b/include/openspace/engine/downloadmanager.h @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -34,18 +35,33 @@ namespace openspace { class DownloadManager : public ghoul::Singleton { public: - typedef std::function DownloadFinishedCallback; + typedef std::function < + void(const ghoul::filesystem::File&, float progress) + > DownloadProgressCallback; + typedef std::function< + void (const ghoul::filesystem::File&) + > DownloadFinishedCallback; - DownloadManager(std::string requestURL); + DownloadManager(std::string requestURL, int applicationVersion); bool downloadFile( const std::string& url, const ghoul::filesystem::File& file, - DownloadFinishedCallback callback = DownloadFinishedCallback() + DownloadFinishedCallback finishedCallback = DownloadFinishedCallback(), + DownloadProgressCallback progressCallback = DownloadProgressCallback() + ); + + bool downloadRequestFiles( + const std::string& identifier, + const ghoul::filesystem::Directory& destination, + int version, + DownloadFinishedCallback finishedCallback = DownloadFinishedCallback(), + DownloadProgressCallback progressCallback = DownloadProgressCallback() ); private: std::string _requestURL; + int _applicationVersion; }; #define DlManager (openspace::DownloadManager::ref()) diff --git a/src/engine/downloadmanager.cpp b/src/engine/downloadmanager.cpp index 023142cdfb..f6be8283d5 100644 --- a/src/engine/downloadmanager.cpp +++ b/src/engine/downloadmanager.cpp @@ -33,6 +33,10 @@ namespace { const std::string _loggerCat = "DownloadManager"; + + const std::string RequestIdentifier = "identifier"; + const std::string RequestFileVersion = "file_version"; + const std::string RequestApplicationVersion = "application_version"; size_t writeData(void* ptr, size_t size, size_t nmemb, FILE* stream) { size_t written; @@ -43,8 +47,9 @@ namespace { namespace openspace { -DownloadManager::DownloadManager(std::string requestURL) +DownloadManager::DownloadManager(std::string requestURL, int applicationVersion) : _requestURL(std::move(requestURL)) + , _applicationVersion(std::move(applicationVersion)) { curl_global_init(CURL_GLOBAL_ALL); // Check if URL is accessible @@ -53,7 +58,8 @@ DownloadManager::DownloadManager(std::string requestURL) bool DownloadManager::downloadFile( const std::string& url, const ghoul::filesystem::File& file, - DownloadFinishedCallback callback) + DownloadFinishedCallback finishedCallback, + DownloadProgressCallback progressCallback) { CURL* curl = curl_easy_init(); if (curl) { @@ -62,20 +68,44 @@ bool DownloadManager::downloadFile( curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeData); curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); + LDEBUG("Starting download for file: '" << url << + "' into file '" << file.filename() << "'"); CURLcode res = curl_easy_perform(curl); curl_easy_cleanup(curl); fclose(fp); + // TODO: incorporate progressCallback ---abock + // http://curl.haxx.se/libcurl/c/progressfunc.html + if (res != CURLE_OK) { LERROR("Error downloading file 'url': " << curl_easy_strerror(res)); return false; } - if (callback) - callback(file); + if (finishedCallback) + finishedCallback(file); return true; } } +bool DownloadManager::downloadRequestFiles( + const std::string& identifier, + const ghoul::filesystem::Directory& destination, + int version, + DownloadFinishedCallback finishedCallback, + DownloadProgressCallback progressCallback) +{ + // Escaping is necessary ---abock + const std::string fullRequest =_requestURL + "?" + + RequestIdentifier + "=" + identifier + "&" + + RequestFileVersion + "=" + std::to_string(version) + "&" + + RequestApplicationVersion = "=" + std::to_string(_applicationVersion); + + + + + return true; +} + } // namespace openspace diff --git a/src/engine/openspaceengine.cpp b/src/engine/openspaceengine.cpp index e2fff913d9..0b9ec647ba 100644 --- a/src/engine/openspaceengine.cpp +++ b/src/engine/openspaceengine.cpp @@ -87,6 +87,7 @@ namespace { const std::string _sgctConfigArgumentCommand = "-config"; const int CacheVersion = 1; + const int DownloadVersion = 1; struct { std::string configurationName; @@ -292,7 +293,7 @@ bool OpenSpaceEngine::initialize() { std::string requestURL = ""; bool success = configurationManager()->getValue(ConfigurationManager::KeyDownloadRequestURL, requestURL); if (success) - DownloadManager::initialize(requestURL); + DownloadManager::initialize(requestURL, DownloadVersion); // Load SPICE time kernel success = loadSpiceKernels();