Cleanup and better configuration of resource syncs

include/openspace/engine/configurationmanager.h

@@ -115,14 +115,13 @@ public:
     static const std::string KeyDisableMasterRendering;
     /// The key that stores whether the master node should apply the scene transformation
     static const std::string KeyDisableSceneOnMaster;
-    /// The key that sets the request URL that is used to request additional data to be
-    /// downloaded
-    static const std::string KeyDownloadRequestURL;
     /// The key that stores the switch for enabling/disabling the rendering on a master
     /// computer
     static const std::string KeyRenderingMethod;
     /// The key that determines whether a new cache folder is used for each scene file
     static const std::string KeyPerSceneCache;
+    // The key that stores the list of urls to http resource sync repositories
+    static const std::string KeyHttpSynchronizationRepositories;
     /// The key that stores the http proxy settings for the downloadmanager
     static const std::string KeyHttpProxy;
     /// The key that stores the address of the http proxy

include/openspace/engine/downloadmanager.h

@@ -82,8 +82,7 @@ public:
     static bool futureReady(std::future<R> const& f)
     { return f.wait_for(std::chrono::seconds(0)) == std::future_status::ready; }
 
-    DownloadManager(std::string requestURL, int applicationVersion,
-        bool useMultithreadedDownload = true);
+    DownloadManager(bool useMultithreadedDownload = true);
 
     //downloadFile
     // url - specifies the target of the download
@@ -105,25 +104,10 @@ public:
     std::future<MemoryFile> fetchFile(
     const std::string& url,
     SuccessCallback successCallback = SuccessCallback(), ErrorCallback errorCallback = ErrorCallback());
-
-    std::vector<std::shared_ptr<FileFuture>> downloadRequestFiles(const std::string& identifier,
-        const ghoul::filesystem::Directory& destination, int version,
-        bool overrideFiles = true,
-        DownloadFinishedCallback finishedCallback = DownloadFinishedCallback(),
-        DownloadProgressCallback progressCallback = DownloadProgressCallback()
-    );
-
-    void downloadRequestFilesAsync(const std::string& identifier,
-        const ghoul::filesystem::Directory& destination, int version,
-        bool overrideFiles, AsyncDownloadFinishedCallback callback
-    );
-
     void getFileExtension(const std::string& url,
         RequestFinishedCallback finishedCallback = RequestFinishedCallback());
 
 private:
-    std::vector<std::string> _requestURL;
-    int _applicationVersion;
     bool _useMultithreadedDownload;
 };
 

include/openspace/scene/assetloader.h

@@ -29,6 +29,7 @@
 
 #include <openspace/scripting/lualibrary.h>
 #include <openspace/scene/assetsynchronizer.h>
+#include <openspace/util/resourcesynchronization.h>
 
 #include <ghoul/misc/dictionary.h>
 #include <ghoul/lua/luastate.h>
@@ -63,7 +64,7 @@ public:
     AssetLoader(
         ghoul::lua::LuaState& luaState,
         std::string assetRoot,
-        std::string syncRoot
+        ResourceSynchronizationOptions syncOptions
     );
 
     /**
@@ -177,7 +178,7 @@ private:
 
     AssetSynchronizer* _assetSynchronizer;
     std::string _assetRootDirectory;
-    std::string _syncRootDirectory;
+    ResourceSynchronizationOptions _synchronizationOptions;
     ghoul::lua::LuaState* _luaState;
 
     // References to lua values

include/openspace/scene/assetsynchronizer.h

@@ -74,9 +74,6 @@ private:
     std::vector<Asset*> _trivialSynchronizations;
 };
 
-
-
-
 } // namespace openspace
 
 #endif // __OPENSPACE_CORE___ASSETSYNCHRONIZER___H__

include/openspace/util/resourcesynchronization.h

@@ -34,6 +34,11 @@
 
 namespace openspace {
 
+struct ResourceSynchronizationOptions {
+    std::vector<std::string> httpSynchronizationRepositories;
+    std::string synchronizationRoot;
+};
+
 class ResourceSynchronization;
 
 struct SynchronizationProduct {
@@ -58,10 +63,11 @@ public:
         const ghoul::Dictionary& dictionary);
 
     ResourceSynchronization();
+    virtual ~ResourceSynchronization();
     virtual void synchronize() = 0;
     virtual std::string directory() = 0;
 
-    void setSyncRoot(std::string path);
+    void setSynchronizationOptions(ResourceSynchronizationOptions opt);
     void wait();
     bool isResolved();
     void resolve();
@@ -69,7 +75,7 @@ public:
     void updateProgress(float t);
     std::shared_ptr<SynchronizationJob> job();
 protected:
-    std::string _syncRoot;
+    ResourceSynchronizationOptions _synchronizationOptions;
 private:
     std::shared_ptr<SynchronizationJob> _job;
     std::atomic<bool> _started;

modules/sync/syncs/httpsynchronization.cpp

@@ -36,6 +36,11 @@ namespace {
     const char* _loggerCat = "HttpSynchronization";
     const char* KeyIdentifier = "Identifier";
     const char* KeyVersion = "Version";
+
+    const char* QueryKeyIdentifier = "identifier";
+    const char* QueryKeyFileVersion = "file_version";
+    const char* QueryKeyApplicationVersion = "application_version";
+    const int ApplicationVersion = 1;
 }
 
 namespace openspace {
@@ -77,7 +82,7 @@ documentation::Documentation HttpSynchronization::Documentation() {
 
 std::string HttpSynchronization::directory() {
     ghoul::filesystem::Directory d(
-        _syncRoot +
+        _synchronizationOptions.synchronizationRoot +
         ghoul::filesystem::FileSystem::PathSeparator +
         "http" +
         ghoul::filesystem::FileSystem::PathSeparator +
@@ -95,41 +100,64 @@ void HttpSynchronization::synchronize() {
         return;
     }
 
-    std::string listUrl = "http://data.openspaceproject.com/request?identifier=" +
-                       _identifier +
-                       "&file_version=" +
-                        std::to_string(_version) +
-                        "&application_version=" +
-                        std::to_string(1);
+    std::vector<std::string> listUrls = fileListUrls();
+    for (const auto& url : listUrls) {
+        if (trySyncFromUrl(url)) {
+            resolve();
+            return;
+        }
+    }
+}
+    
+std::vector<std::string> HttpSynchronization::fileListUrls() {
+    std::string query = std::string("?") + QueryKeyIdentifier + "=" + _identifier +
+        "&" + QueryKeyFileVersion + "=" + std::to_string(_version) +
+        "&" + QueryKeyApplicationVersion + "=" + std::to_string(ApplicationVersion);
 
+    std::vector<std::string> urls;
+    for (const auto& repoUrl : _synchronizationOptions.httpSynchronizationRepositories) {
+        urls.push_back(repoUrl + query);
+    }
+
+    return urls;
+}
+
+bool HttpSynchronization::hasSyncFile() {
+    std::string path = directory() + ".ossync";
+    return FileSys.fileExists(path);
+}
+
+bool HttpSynchronization::trySyncFromUrl(std::string listUrl) {
     HttpRequest::RequestOptions opt;
     opt.requestTimeoutSeconds = 0;
-    
+
     HttpMemoryDownload fileListDownload(listUrl);
     fileListDownload.download(opt);
 
     const std::vector<char>& buffer = fileListDownload.downloadedData();
 
     std::istringstream fileList(std::string(buffer.begin(), buffer.end()));
+
+    std::vector<std::thread> downloadThreads;
     std::string line = "";
     while (fileList >> line) {
         std::string filename = ghoul::filesystem::File(line, ghoul::filesystem::File::RawPath::Yes).filename();
-
+        
         std::string fileDestination = directory() +
-            ghoul::filesystem::FileSystem::PathSeparator +
-            filename;
+        ghoul::filesystem::FileSystem::PathSeparator +
+        filename;
 
-        HttpFileDownload fileDownload(line, fileDestination);
-        fileDownload.download(opt);
+        std::thread t([opt, line, fileDestination]() {
+            HttpFileDownload fileDownload(line, fileDestination);
+            fileDownload.download(opt);
+        });
+        downloadThreads.push_back(std::move(t));
+    }
+    for (auto& t : downloadThreads) {
+        t.join();
     }
     createSyncFile();
-
-    resolve();
-}
-
-bool HttpSynchronization::hasSyncFile() {
-    std::string path = directory() + ".ossync";
-    return FileSys.fileExists(path);
+    return true;
 }
 
 void HttpSynchronization::createSyncFile() {

modules/sync/syncs/httpsynchronization.h

@@ -43,6 +43,8 @@ public:
     void synchronize() override;
 
 private:
+    std::vector<std::string> fileListUrls();
+    bool trySyncFromUrl(std::string url);
     bool hasSyncFile();
     void createSyncFile();
 

openspace.cfg

@@ -88,7 +88,9 @@ return {
     -- PerSceneCache = true,
     -- DisableRenderingOnMaster = true,
     -- DisableSceneOnMaster = true,
-    DownloadRequestURL = "http://data.openspaceproject.com/request.cgi",
+    HttpSynchronizationRepositories = {
+        "data.openspaceproject.com/request"
+    },
     RenderingMethod = "Framebuffer",
     OpenGLDebugContext = {
        Activate = true,

src/engine/configurationmanager.cpp

@@ -81,12 +81,13 @@ const string ConfigurationManager::KeyCapabilitiesVerbosity =
 const string ConfigurationManager::KeyShutdownCountdown = "ShutdownCountdown";
 const string ConfigurationManager::KeyDisableMasterRendering = "DisableRenderingOnMaster";
 const string ConfigurationManager::KeyDisableSceneOnMaster = "DisableSceneOnMaster";
-const string ConfigurationManager::KeyDownloadRequestURL = "DownloadRequestURL";
 const string ConfigurationManager::KeyPerSceneCache = "PerSceneCache";
 const string ConfigurationManager::KeyRenderingMethod = "RenderingMethod";
 
 const string ConfigurationManager::KeyOnScreenTextScaling = "OnScreenTextScaling";
 
+const string ConfigurationManager::KeyHttpSynchronizationRepositories = "HttpSynchronizationRepositories";
+
 const string ConfigurationManager::KeyHttpProxy = "HttpProxy";
 const string ConfigurationManager::PartHttpProxyAddress = "Address";
 const string ConfigurationManager::PartHttpProxyPort = "Port";

src/engine/configurationmanager_doc.inl

@@ -256,17 +256,6 @@ documentation::Documentation ConfigurationManager::Documentation() {
             "text can either be scaled according to the window size ('window'), or the "
             "rendering resolution ('framebuffer'). This value defaults to 'window'."
         },
-        {
-            ConfigurationManager::KeyDownloadRequestURL,
-            new OrVerifier(
-                new StringVerifier,
-                new StringListVerifier
-            ),
-            Optional::Yes,
-            "The URL from which files will be downloaded by the Launcher. This can "
-            "either be a single URL or a list of possible URLs from which the "
-            "Launcher can then choose."
-        },
         {
             ConfigurationManager::KeyRenderingMethod,
             new StringInListVerifier(
@@ -295,6 +284,13 @@ documentation::Documentation ConfigurationManager::Documentation() {
             "interaction and it is thus desired to disable the transformation. The "
             "default is false."
         },
+        {
+            ConfigurationManager::KeyHttpSynchronizationRepositories,
+            new StringListVerifier("URLs to http synchronization repositories"),
+            Optional::Yes,
+            "Configures the list of http synchronization repositories to load "
+            "resources from. The first URL will be tried first."
+        },
         {
             ConfigurationManager::KeyHttpProxy,
             new TableVerifier({

src/engine/downloadmanager.cpp

@@ -145,17 +145,10 @@ DownloadManager::FileFuture::FileFuture(std::string file)
     , abortDownload(false)
 {}
 
-DownloadManager::DownloadManager(std::string requestURL, int applicationVersion,
-                                 bool useMultithreadedDownload)
-    : _applicationVersion(std::move(applicationVersion))
-    , _useMultithreadedDownload(useMultithreadedDownload)
+DownloadManager::DownloadManager(bool useMultithreadedDownload)
+    : _useMultithreadedDownload(useMultithreadedDownload)
 {
     curl_global_init(CURL_GLOBAL_ALL);
-    
-    _requestURL.push_back(std::move(requestURL));
-    
-    // TODO: Check if URL is accessible ---abock
-    // TODO: Allow for multiple requestURLs
 }
 
 std::shared_ptr<DownloadManager::FileFuture> DownloadManager::downloadFile(
@@ -313,6 +306,7 @@ std::future<DownloadManager::MemoryFile> DownloadManager::fetchFile(
     return std::async(std::launch::async, downloadFunction);
 }
 
+/*
 std::vector<std::shared_ptr<DownloadManager::FileFuture>>
 DownloadManager::downloadRequestFiles(
     const std::string& identifier, const ghoul::filesystem::Directory& destination,
@@ -411,7 +405,7 @@ void DownloadManager::downloadRequestFilesAsync(const std::string& identifier,
     else {
         downloadFunction();
     }
-}
+}*/
 
 void DownloadManager::getFileExtension(const std::string& url,
     RequestFinishedCallback finishedCallback){

src/engine/openspaceengine.cpp

@@ -406,9 +406,23 @@ void OpenSpaceEngine::create(int argc, char** argv,
     sgctArguments.insert(sgctArguments.begin() + 2, absPath(sgctConfigurationPath));
 
     // Set up asset loader
+    ResourceSynchronizationOptions syncOptions;
+    syncOptions.synchronizationRoot = absPath("${SYNC}");
+    if (_engine->_configurationManager->hasKey(
+         ConfigurationManager::KeyHttpSynchronizationRepositories))
+    {
+        ghoul::Dictionary dictionary = _engine->_configurationManager->value<ghoul::Dictionary>(
+            ConfigurationManager::KeyHttpSynchronizationRepositories
+        );
+        for (std::string key : dictionary.keys()) {
+            syncOptions.httpSynchronizationRepositories.push_back(
+                dictionary.value<std::string>(key)
+            );
+        }
+    }
     _engine->_assetManager = std::make_unique<AssetManager>(
-        std::move(std::make_unique<AssetLoader>(*OsEng.scriptEngine().luaState(), "${ASSETS}", "${SYNC}")),
-        std::move(std::make_unique<AssetSynchronizer>(OsEng._resourceSynchronizer.get()))
+        std::make_unique<AssetLoader>(*OsEng.scriptEngine().luaState(), "${ASSETS}", syncOptions),
+        std::make_unique<AssetSynchronizer>(OsEng._resourceSynchronizer.get())
     );
     //_engine->_globalPropertyNamespace->addPropertySubOwner(_engine->_assetLoader->rootAsset());
 }
@@ -500,16 +514,8 @@ void OpenSpaceEngine::initialize() {
         );
     }
 
-    if (configurationManager().hasKey(ConfigurationManager::KeyDownloadRequestURL)) {
-        const std::string requestUrl = configurationManager().value<std::string>(
-            ConfigurationManager::KeyDownloadRequestURL
-        );
+    _downloadManager = std::make_unique<DownloadManager>();
 
-        _downloadManager = std::make_unique<DownloadManager>(
-            requestUrl,
-            DownloadVersion
-        );
-    }
 
     // Register Lua script functions
     LDEBUG("Registering Lua libraries");

src/engine/openspaceengine_lua.inl

@@ -219,7 +219,7 @@ int downloadFile(lua_State* L) {
 
     const std::string _loggerCat = "OpenSpaceEngine";
     LINFO("Downloading file from " << uri);
-    DownloadManager dm = openspace::DownloadManager("", 1, false);
+    DownloadManager dm = openspace::DownloadManager(false);
     std::shared_ptr<openspace::DownloadManager::FileFuture> future =
         dm.downloadFile(uri, absPath("${SCENE}/" + savePath), true, true, 5);
     if (!future || (future && !future->isFinished)) {

src/scene/assetloader.cpp

@@ -72,12 +72,12 @@ namespace openspace {
 AssetLoader::AssetLoader(
     ghoul::lua::LuaState& luaState,
     std::string assetRootDirectory,
-    std::string syncRootDirectory
+    ResourceSynchronizationOptions syncOptions
 )
     : _luaState(&luaState)
     , _rootAsset(std::make_shared<Asset>(this))
     , _assetRootDirectory(assetRootDirectory)
-    , _syncRootDirectory(std::move(syncRootDirectory))
+    , _synchronizationOptions(std::move(syncOptions))
 {
     pushAsset(_rootAsset);
 
@@ -227,7 +227,7 @@ std::shared_ptr<Asset> AssetLoader::rootAsset() const {
 }
 
 const std::string& AssetLoader::syncRootDirectory() {
-    return _syncRootDirectory;
+    return _synchronizationOptions.synchronizationRoot;
 }
 
 const std::string & AssetLoader::assetRootDirectory()
@@ -304,7 +304,7 @@ int AssetLoader::syncedResourceLua(Asset* asset) {
     std::shared_ptr<ResourceSynchronization> sync =
         ResourceSynchronization::createFromDictionary(d);
 
-    sync->setSyncRoot(_syncRootDirectory);
+    sync->setSynchronizationOptions(_synchronizationOptions);
     std::string absolutePath = sync->directory();
 
     asset->addSynchronization(sync);

src/util/httprequest.cpp

@@ -84,7 +84,7 @@ void HttpRequest::perform(RequestOptions opt) {
 
     curl_easy_setopt(curl, CURLOPT_URL, _url.c_str());
     curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
-    
+
     curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlfunctions::writeCallback);
     
@@ -96,12 +96,7 @@ void HttpRequest::perform(RequestOptions opt) {
     }
 
     CURLcode res = curl_easy_perform(curl);
-    if (res == CURLE_OK) {
-        LINFO("CURL is ok");
-    } else {
-        LINFO("CURL failed");
-    }
-
+    setReadyState(res == CURLE_OK ? ReadyState::Success : ReadyState::Fail);
     curl_easy_cleanup(curl);
 }
     

src/util/resourcesynchronization.cpp

@@ -55,7 +55,8 @@ documentation::Documentation ResourceSynchronization::Documentation() {
                 "This key specifies the type of ResourceSyncrhonization that gets created. "
                 "It has to be one of the valid ResourceSyncrhonizations that are available "
                 "for creation (see the FactoryDocumentation for a list of possible "
-                "ResourceSyncrhonizations), which depends on the configration of the application"
+                "ResourceSyncrhonizations), which depends on the configration of the "
+                " application"
             }
         }
     };
@@ -66,18 +67,25 @@ ResourceSynchronization::ResourceSynchronization()
     , _resolved(false)
 {}
 
+ResourceSynchronization::~ResourceSynchronization()
+{}
+
 std::unique_ptr<ResourceSynchronization> ResourceSynchronization::createFromDictionary(
     const ghoul::Dictionary & dictionary)
 {
-    documentation::testSpecificationAndThrow(Documentation(), dictionary, "ResourceSynchronization");
+    documentation::testSpecificationAndThrow(
+        Documentation(), dictionary, "ResourceSynchronization");
 
     std::string synchronizationType = dictionary.value<std::string>(KeyType);
 
     auto factory = FactoryManager::ref().factory<ResourceSynchronization>();
     ghoul_assert(factory, "ResourceSynchronization factory did not exist");
-    std::unique_ptr<ResourceSynchronization> result = factory->create(synchronizationType, dictionary);
+    std::unique_ptr<ResourceSynchronization> result =
+        factory->create(synchronizationType, dictionary);
+
     if (result == nullptr) {
-        LERROR("Failed to create a ResourceSynchronization object of type '" << synchronizationType << "'");
+        LERROR("Failed to create a ResourceSynchronization object of type '" <<
+               synchronizationType << "'");
         return nullptr;
     }
 
@@ -88,10 +96,6 @@ std::shared_ptr<SynchronizationJob> ResourceSynchronization::job() {
     return _job;
 }
 
-void ResourceSynchronization::setSyncRoot(std::string path) {
-    _syncRoot = std::move(path);
-}
-
 void ResourceSynchronization::wait() {
 }
 
@@ -111,6 +115,13 @@ void ResourceSynchronization::updateProgress(float t) {
     _progress = std::min(1.0f, std::max(t, 0.0f));
 }
 
+void ResourceSynchronization::setSynchronizationOptions(
+    openspace::ResourceSynchronizationOptions opt)
+{
+    _synchronizationOptions = std::move(opt);
+}
+
+
 // SynchronizationJob methods
 
 SynchronizationJob::SynchronizationJob(ResourceSynchronization* synchronization) {

src/util/resourcesynchronizer.cpp

@@ -30,7 +30,7 @@ namespace {
 
 namespace openspace {
 
-ResourceSynchronizer::ResourceSynchronizer() 
+ResourceSynchronizer::ResourceSynchronizer()
     : _jobManager(ThreadPool(NumberOfThreads))
 {}