Asset synchronization using HTTP

assets/spice/base.asset

@@ -1,16 +1,16 @@
 local AssetHelper = asset.import("assethelper")
 
-local kernelsDirectory = asset.syncedResource({
-	Type = "HttpSynchronization",
-	Identifier = "base_kernels",
-	Version = 1
+local syncedDirectory = asset.syncedResource({
+    Type = "HttpSynchronization",
+    Identifier = "general_spk",
+    Version = 1
 })
 
 local kernels = {
-	kernelsDirectory .. "/naif0012.tls",
+    asset.localResource("naif0012.tls"),
     -- Leapseconds:
-    kernelsDirectory .. "/pck00010.tpc",
-    kernelsDirectory .. "/de430_1850-2150.bsp"
+    asset.localResource("pck00010.tpc"),
+    syncedDirectory .. "/de430_1850-2150.bsp"
 }
 
 AssetHelper.registerSpiceKernels(asset, kernels)

assets/sun/sun.asset

@@ -6,7 +6,7 @@ local sunParentName = transforms.SunIau.Name;
 local texturesPath = asset.syncedResource({
     Type = "HttpSynchronization",
     Identifier = "sun_textures",
-    Version = 1
+    Version = 3
 })
 
 --asset.addSynchronization({

include/openspace/util/httprequest.h

@@ -0,0 +1,148 @@
+/*****************************************************************************************
+ *                                                                                       *
+ * OpenSpace                                                                             *
+ *                                                                                       *
+ * Copyright (c) 2014-2017                                                               *
+ *                                                                                       *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this  *
+ * software and associated documentation files (the "Software"), to deal in the Software *
+ * without restriction, including without limitation the rights to use, copy, modify,    *
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to    *
+ * permit persons to whom the Software is furnished to do so, subject to the following   *
+ * conditions:                                                                           *
+ *                                                                                       *
+ * The above copyright notice and this permission notice shall be included in all copies *
+ * or substantial portions of the Software.                                              *
+ *                                                                                       *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,   *
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A         *
+ * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT    *
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF  *
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE  *
+ * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                                         *
+ ****************************************************************************************/
+
+#ifndef __OPENSPACE_CORE___HTTPREQUEST___H__
+#define __OPENSPACE_CORE___HTTPREQUEST___H__
+
+#include <ghoul/filesystem/file.h>
+#include <ghoul/filesystem/directory.h>
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace openspace {
+    
+namespace curlfunctions {
+    size_t writeCallback(char *ptr,
+                         size_t size,
+                         size_t nmemb,
+                         void *userdata);
+
+    int progressCallback(void *clientp,
+                         int64_t dltotal,
+                         int64_t dlnow,
+                         int64_t ultotal,
+                         int64_t ulnow);
+}
+
+// Synchronous http request
+class HttpRequest {
+public:
+    enum class ReadyState : unsigned int {
+        Unsent,
+        Loading,
+        ReceivedHeaders,
+        Fail,
+        Success
+    };
+
+    struct Progress {
+        bool totalBytesKnown = false;
+        size_t totalBytes = 0;
+        size_t downloadedBytes = 0;
+    };
+
+    struct Data {
+        char* buffer;
+        size_t size;
+    };
+
+    using ReadyStateChangeCallback = std::function<void(ReadyState)>;
+    using ProgressCallback = std::function<int(Progress)>;
+    using DataCallback = std::function<size_t(Data)>;
+
+    struct RequestOptions {
+        int requestTimeoutSeconds; // 0 for no timeout
+    };
+
+    HttpRequest(std::string url);
+
+    void onReadyStateChange(ReadyStateChangeCallback cb);
+    void onProgress(ProgressCallback cb);
+    void onData(DataCallback cb);
+
+    void perform(RequestOptions opt);
+
+private:
+    void setReadyState(ReadyState state);
+
+    std::string _url;
+
+    ReadyStateChangeCallback _onReadyStateChange;
+    ProgressCallback _onProgress;
+    DataCallback _onData;
+
+    ReadyState _readyState;
+    Progress _progress;
+    RequestOptions _options;
+    
+    friend size_t curlfunctions::writeCallback(
+                                char *ptr,
+                                size_t size,
+                                size_t nmemb,
+                                void *userdata);
+
+    friend int curlfunctions::progressCallback(void *clientp,
+                                               int64_t dltotal,
+                                               int64_t dlnow,
+                                               int64_t ultotal,
+                                               int64_t ulnow);
+};
+    
+class HttpMemoryDownload {
+public:
+    HttpMemoryDownload(std::string url);
+    
+    void onReadyStateChange(HttpRequest::ReadyStateChangeCallback cb);
+    void onProgress(HttpRequest::ProgressCallback cb);
+    
+    void download(HttpRequest::RequestOptions opt);
+    
+    const std::vector<char>& downloadedData();
+
+private:
+    HttpRequest _httpRequest;
+    std::vector<char> _downloadedData;
+};
+
+class HttpFileDownload {
+public:
+    HttpFileDownload(std::string url, std::string destinationPath);
+    
+    void onReadyStateChange(HttpRequest::ReadyStateChangeCallback cb);
+    void onProgress(HttpRequest::ProgressCallback cb);
+    
+    void download(HttpRequest::RequestOptions opt);
+
+private:
+    HttpRequest _httpRequest;
+    std::string _destination;
+};
+    
+
+} // namespace openspace
+
+#endif // __OPENSPACE_CORE___HTTPREQUEST___H__

modules/sync/syncs/httpsynchronization.cpp

@@ -24,11 +24,13 @@
 
 #include "httpsynchronization.h"
 
-#include <openspace/util/asynchttprequest.h>
+#include <openspace/util/httprequest.h>
 #include <ghoul/logging/logmanager.h>
 #include <ghoul/filesystem/filesystem.h>
 #include <openspace/documentation/verifier.h>
 
+#include <sstream>
+
 namespace {
     const char* _loggerCat = "HttpSynchronization";
     const char* KeyIdentifier = "Identifier";
@@ -73,18 +75,48 @@ documentation::Documentation HttpSynchronization::Documentation() {
 }
 
 std::string HttpSynchronization::directory() {
-    return _syncRoot +
+    ghoul::filesystem::Directory d(
+        _syncRoot +
         ghoul::filesystem::FileSystem::PathSeparator +
         _identifier +
         ghoul::filesystem::FileSystem::PathSeparator +
-        std::to_string(_version);
+        std::to_string(_version)
+    );
+
+    return FileSys.absPath(d);
 }
 
 void HttpSynchronization::synchronize() {
-    // TODO: Download files, synchronously.
-    // First check if files exist.
+    // TODO: First check if files exist.
+    std::string listUrl = "http://data.openspaceproject.com/request?identifier=" +
+                       _identifier +
+                       "&file_version=" +
+                        std::to_string(_version) +
+                        "&application_version=" +
+                        std::to_string(1);
+
+    HttpMemoryDownload fileListDownload(listUrl);
+    HttpRequest::RequestOptions opt;
+    opt.requestTimeoutSeconds = 0;
+    fileListDownload.download(opt);
     
-    LINFO("Synchronizing!");
+    const std::vector<char>& buffer = fileListDownload.downloadedData();
+
+    LINFO(std::string(buffer.begin(), buffer.end()));
+
+    std::istringstream fileList(std::string(buffer.begin(), buffer.end()));
+    std::string line = "";
+    while (fileList >> line) {
+        std::string filename = ghoul::filesystem::File(line, ghoul::filesystem::File::RawPath::Yes).filename();
+
+        std::string fileDestination = directory() +
+            ghoul::filesystem::FileSystem::PathSeparator +
+            filename;
+
+        HttpFileDownload fileDownload(line, fileDestination);
+        fileDownload.download(opt);
+    }
+
     resolve();
 }
 

src/CMakeLists.txt

@@ -153,6 +153,7 @@ set(OPENSPACE_SOURCE
     ${OPENSPACE_BASE_DIR}/src/util/boxgeometry.cpp
     ${OPENSPACE_BASE_DIR}/src/util/camera.cpp
     ${OPENSPACE_BASE_DIR}/src/util/factorymanager.cpp
+    ${OPENSPACE_BASE_DIR}/src/util/httprequest.cpp
     ${OPENSPACE_BASE_DIR}/src/util/keys.cpp
     ${OPENSPACE_BASE_DIR}/src/util/openspacemodule.cpp
     ${OPENSPACE_BASE_DIR}/src/util/powerscaledcoordinate.cpp
@@ -317,6 +318,7 @@ set(OPENSPACE_HEADER
     ${OPENSPACE_BASE_DIR}/include/openspace/util/concurrentqueue.inl
     ${OPENSPACE_BASE_DIR}/include/openspace/util/factorymanager.h
     ${OPENSPACE_BASE_DIR}/include/openspace/util/factorymanager.inl
+    ${OPENSPACE_BASE_DIR}/include/openspace/util/httprequest.h
     ${OPENSPACE_BASE_DIR}/include/openspace/util/keys.h
     ${OPENSPACE_BASE_DIR}/include/openspace/util/mouse.h
     ${OPENSPACE_BASE_DIR}/include/openspace/util/openspacemodule.h

src/util/httprequest.cpp

@@ -0,0 +1,193 @@
+/*****************************************************************************************
+ *                                                                                       *
+ * OpenSpace                                                                             *
+ *                                                                                       *
+ * Copyright (c) 2014-2017                                                               *
+ *                                                                                       *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this  *
+ * software and associated documentation files (the "Software"), to deal in the Software *
+ * without restriction, including without limitation the rights to use, copy, modify,    *
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to    *
+ * permit persons to whom the Software is furnished to do so, subject to the following   *
+ * conditions:                                                                           *
+ *                                                                                       *
+ * The above copyright notice and this permission notice shall be included in all copies *
+ * or substantial portions of the Software.                                              *
+ *                                                                                       *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,   *
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A         *
+ * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT    *
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF  *
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE  *
+ * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                                         *
+ ****************************************************************************************/
+
+#include <openspace/util/httprequest.h>
+
+#include <ghoul/filesystem/filesystem.h>
+#include <ghoul/logging/logmanager.h>
+#include <ghoul/misc/assert.h>
+#include <ghoul/misc/thread.h>
+
+#include <chrono>
+#include <cstring>
+#include <fstream>
+#include <stdio.h>
+#include <thread>
+
+#ifdef OPENSPACE_CURL_ENABLED
+#ifdef WIN32
+#pragma warning (push)
+#pragma warning (disable: 4574) // 'INCL_WINSOCK_API_TYPEDEFS' is defined to be '0'
+#endif // WIN32
+
+#include <curl/curl.h>
+
+#ifdef WIN32
+#pragma warning (pop)
+#endif // WIN32
+#endif
+
+namespace {
+    const char* _loggerCat = "HttpRequest";
+}
+
+namespace openspace {
+    
+HttpRequest::HttpRequest(std::string url) 
+    : _url(std::move(url))
+    , _onReadyStateChange([](ReadyState) {})
+    , _onProgress([](Progress) { return 0; })
+    , _onData([](Data d) { return d.size; })
+{}
+
+void HttpRequest::onReadyStateChange(ReadyStateChangeCallback cb) {
+    _onReadyStateChange = std::move(cb);
+}
+
+void HttpRequest::onProgress(ProgressCallback cb) {
+    _onProgress = std::move(cb);
+}
+
+void HttpRequest::onData(DataCallback cb) {
+    _onData = std::move(cb);
+}
+
+void HttpRequest::perform(RequestOptions opt) {
+    setReadyState(ReadyState::Loading);
+
+    CURL* curl = curl_easy_init();
+    if (!curl) {
+        setReadyState(ReadyState::Fail);
+        return;
+    }
+
+    curl_easy_setopt(curl, CURLOPT_URL, _url.c_str());
+    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+    
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlfunctions::writeCallback);
+    
+    curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this);
+    curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, curlfunctions::progressCallback);
+    
+    if (opt.requestTimeoutSeconds > 0) {
+        curl_easy_setopt(curl, CURLOPT_TIMEOUT, opt.requestTimeoutSeconds);
+    }
+
+    CURLcode res = curl_easy_perform(curl);
+    if (res == CURLE_OK) {
+        LINFO("CURL is ok");
+    } else {
+        LINFO("CURL failed");
+    }
+
+    curl_easy_cleanup(curl);
+}
+    
+void HttpRequest::setReadyState(openspace::HttpRequest::ReadyState state) {
+    _readyState = state;
+}
+    
+    
+namespace curlfunctions {
+int progressCallback(void* userData,
+                     int64_t nTotalBytes,
+                     int64_t nDownloadedBytes,
+                     int64_t,
+                     int64_t)
+{
+    HttpRequest* r = reinterpret_cast<HttpRequest*>(userData);
+    return r->_onProgress(
+        HttpRequest::Progress{
+            true,
+            static_cast<size_t>(nTotalBytes),
+            static_cast<size_t>(nDownloadedBytes)
+        }
+    );
+    
+    LINFO("Transfer info from curl: " << nDownloadedBytes << " out of " <<  nTotalBytes);
+    return 0;
+}
+
+size_t writeCallback(char* ptr, size_t size, size_t nmemb, void* userData) {
+    HttpRequest* r = reinterpret_cast<HttpRequest*>(userData);
+    return r->_onData(HttpRequest::Data{ptr, size * nmemb});
+}
+}
+
+
+using namespace openspace;
+
+HttpMemoryDownload::HttpMemoryDownload(std::string url)
+    : _httpRequest(url)
+{}
+
+void HttpMemoryDownload::download(HttpRequest::RequestOptions opt) {
+    _httpRequest.onData([this](HttpRequest::Data d) {
+        _downloadedData.insert(_downloadedData.end(), d.buffer, d.buffer + d.size);
+        return d.size;
+    });
+    _httpRequest.perform(opt);
+}
+    
+const std::vector<char>& HttpMemoryDownload::downloadedData() {
+    return _downloadedData;
+}
+
+HttpFileDownload::HttpFileDownload(std::string url, std::string destinationPath)
+    : _httpRequest(url)
+    , _destination(destinationPath)
+{}
+    
+void HttpFileDownload::download(HttpRequest::RequestOptions opt) {
+    LINFO(_destination);
+    
+    if (FileSys.fileExists(_destination)) {
+        LERROR("File already exists!");
+        return;
+    }
+    
+    {
+        ghoul::filesystem::File f = _destination;
+        ghoul::filesystem::Directory d = f.directoryName();
+        if (!FileSys.directoryExists(d)) {
+            FileSys.createDirectory(d, ghoul::filesystem::Directory::Recursive::Yes);
+        }
+    }
+    
+    std::ofstream f(_destination, std::ofstream::binary);
+    
+    if (f.fail()) {
+        LERROR("Cannot open file!");
+        return;
+    }
+    _httpRequest.onData([this, &f](HttpRequest::Data d) {
+        f.write(d.buffer, d.size);
+        return d.size;
+    });
+    _httpRequest.perform(opt);
+    f.close();
+}
+
+} // namespace openspace