From 03aab55a1ff0f2d5f6b96b10f2e32fc398530272 Mon Sep 17 00:00:00 2001 From: Emil Axelsson Date: Sun, 5 Nov 2017 19:02:31 +0100 Subject: [PATCH] Asset synchronization using HTTP --- assets/spice/base.asset | 14 +- assets/sun/sun.asset | 2 +- include/openspace/util/httprequest.h | 148 ++++++++++++++++ modules/sync/syncs/httpsynchronization.cpp | 44 ++++- src/CMakeLists.txt | 2 + src/util/httprequest.cpp | 193 +++++++++++++++++++++ 6 files changed, 389 insertions(+), 14 deletions(-) create mode 100644 include/openspace/util/httprequest.h create mode 100644 src/util/httprequest.cpp diff --git a/assets/spice/base.asset b/assets/spice/base.asset index a8264d8998..1883ecdc98 100644 --- a/assets/spice/base.asset +++ b/assets/spice/base.asset @@ -1,16 +1,16 @@ local AssetHelper = asset.import("assethelper") -local kernelsDirectory = asset.syncedResource({ - Type = "HttpSynchronization", - Identifier = "base_kernels", - Version = 1 +local syncedDirectory = asset.syncedResource({ + Type = "HttpSynchronization", + Identifier = "general_spk", + Version = 1 }) local kernels = { - kernelsDirectory .. "/naif0012.tls", + asset.localResource("naif0012.tls"), -- Leapseconds: - kernelsDirectory .. "/pck00010.tpc", - kernelsDirectory .. "/de430_1850-2150.bsp" + asset.localResource("pck00010.tpc"), + syncedDirectory .. "/de430_1850-2150.bsp" } AssetHelper.registerSpiceKernels(asset, kernels) \ No newline at end of file diff --git a/assets/sun/sun.asset b/assets/sun/sun.asset index 071ce1e03d..ffdab2a1ce 100644 --- a/assets/sun/sun.asset +++ b/assets/sun/sun.asset @@ -6,7 +6,7 @@ local sunParentName = transforms.SunIau.Name; local texturesPath = asset.syncedResource({ Type = "HttpSynchronization", Identifier = "sun_textures", - Version = 1 + Version = 3 }) --asset.addSynchronization({ diff --git a/include/openspace/util/httprequest.h b/include/openspace/util/httprequest.h new file mode 100644 index 0000000000..0a9f9c2819 --- /dev/null +++ b/include/openspace/util/httprequest.h @@ -0,0 +1,148 @@ +/***************************************************************************************** + * * + * OpenSpace * + * * + * Copyright (c) 2014-2017 * + * * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this * + * software and associated documentation files (the "Software"), to deal in the Software * + * without restriction, including without limitation the rights to use, copy, modify, * + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to * + * permit persons to whom the Software is furnished to do so, subject to the following * + * conditions: * + * * + * The above copyright notice and this permission notice shall be included in all copies * + * or substantial portions of the Software. * + * * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, * + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A * + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF * + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE * + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * + ****************************************************************************************/ + +#ifndef __OPENSPACE_CORE___HTTPREQUEST___H__ +#define __OPENSPACE_CORE___HTTPREQUEST___H__ + +#include +#include + +#include +#include +#include +#include + +namespace openspace { + +namespace curlfunctions { + size_t writeCallback(char *ptr, + size_t size, + size_t nmemb, + void *userdata); + + int progressCallback(void *clientp, + int64_t dltotal, + int64_t dlnow, + int64_t ultotal, + int64_t ulnow); +} + +// Synchronous http request +class HttpRequest { +public: + enum class ReadyState : unsigned int { + Unsent, + Loading, + ReceivedHeaders, + Fail, + Success + }; + + struct Progress { + bool totalBytesKnown = false; + size_t totalBytes = 0; + size_t downloadedBytes = 0; + }; + + struct Data { + char* buffer; + size_t size; + }; + + using ReadyStateChangeCallback = std::function; + using ProgressCallback = std::function; + using DataCallback = std::function; + + struct RequestOptions { + int requestTimeoutSeconds; // 0 for no timeout + }; + + HttpRequest(std::string url); + + void onReadyStateChange(ReadyStateChangeCallback cb); + void onProgress(ProgressCallback cb); + void onData(DataCallback cb); + + void perform(RequestOptions opt); + +private: + void setReadyState(ReadyState state); + + std::string _url; + + ReadyStateChangeCallback _onReadyStateChange; + ProgressCallback _onProgress; + DataCallback _onData; + + ReadyState _readyState; + Progress _progress; + RequestOptions _options; + + friend size_t curlfunctions::writeCallback( + char *ptr, + size_t size, + size_t nmemb, + void *userdata); + + friend int curlfunctions::progressCallback(void *clientp, + int64_t dltotal, + int64_t dlnow, + int64_t ultotal, + int64_t ulnow); +}; + +class HttpMemoryDownload { +public: + HttpMemoryDownload(std::string url); + + void onReadyStateChange(HttpRequest::ReadyStateChangeCallback cb); + void onProgress(HttpRequest::ProgressCallback cb); + + void download(HttpRequest::RequestOptions opt); + + const std::vector& downloadedData(); + +private: + HttpRequest _httpRequest; + std::vector _downloadedData; +}; + +class HttpFileDownload { +public: + HttpFileDownload(std::string url, std::string destinationPath); + + void onReadyStateChange(HttpRequest::ReadyStateChangeCallback cb); + void onProgress(HttpRequest::ProgressCallback cb); + + void download(HttpRequest::RequestOptions opt); + +private: + HttpRequest _httpRequest; + std::string _destination; +}; + + +} // namespace openspace + +#endif // __OPENSPACE_CORE___HTTPREQUEST___H__ diff --git a/modules/sync/syncs/httpsynchronization.cpp b/modules/sync/syncs/httpsynchronization.cpp index a85334e65b..15f289c830 100644 --- a/modules/sync/syncs/httpsynchronization.cpp +++ b/modules/sync/syncs/httpsynchronization.cpp @@ -24,11 +24,13 @@ #include "httpsynchronization.h" -#include +#include #include #include #include +#include + namespace { const char* _loggerCat = "HttpSynchronization"; const char* KeyIdentifier = "Identifier"; @@ -73,18 +75,48 @@ documentation::Documentation HttpSynchronization::Documentation() { } std::string HttpSynchronization::directory() { - return _syncRoot + + ghoul::filesystem::Directory d( + _syncRoot + ghoul::filesystem::FileSystem::PathSeparator + _identifier + ghoul::filesystem::FileSystem::PathSeparator + - std::to_string(_version); + std::to_string(_version) + ); + + return FileSys.absPath(d); } void HttpSynchronization::synchronize() { - // TODO: Download files, synchronously. - // First check if files exist. + // TODO: First check if files exist. + std::string listUrl = "http://data.openspaceproject.com/request?identifier=" + + _identifier + + "&file_version=" + + std::to_string(_version) + + "&application_version=" + + std::to_string(1); + + HttpMemoryDownload fileListDownload(listUrl); + HttpRequest::RequestOptions opt; + opt.requestTimeoutSeconds = 0; + fileListDownload.download(opt); - LINFO("Synchronizing!"); + const std::vector& buffer = fileListDownload.downloadedData(); + + LINFO(std::string(buffer.begin(), buffer.end())); + + std::istringstream fileList(std::string(buffer.begin(), buffer.end())); + std::string line = ""; + while (fileList >> line) { + std::string filename = ghoul::filesystem::File(line, ghoul::filesystem::File::RawPath::Yes).filename(); + + std::string fileDestination = directory() + + ghoul::filesystem::FileSystem::PathSeparator + + filename; + + HttpFileDownload fileDownload(line, fileDestination); + fileDownload.download(opt); + } + resolve(); } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 81347b3a74..678f4e96d7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -153,6 +153,7 @@ set(OPENSPACE_SOURCE ${OPENSPACE_BASE_DIR}/src/util/boxgeometry.cpp ${OPENSPACE_BASE_DIR}/src/util/camera.cpp ${OPENSPACE_BASE_DIR}/src/util/factorymanager.cpp + ${OPENSPACE_BASE_DIR}/src/util/httprequest.cpp ${OPENSPACE_BASE_DIR}/src/util/keys.cpp ${OPENSPACE_BASE_DIR}/src/util/openspacemodule.cpp ${OPENSPACE_BASE_DIR}/src/util/powerscaledcoordinate.cpp @@ -317,6 +318,7 @@ set(OPENSPACE_HEADER ${OPENSPACE_BASE_DIR}/include/openspace/util/concurrentqueue.inl ${OPENSPACE_BASE_DIR}/include/openspace/util/factorymanager.h ${OPENSPACE_BASE_DIR}/include/openspace/util/factorymanager.inl + ${OPENSPACE_BASE_DIR}/include/openspace/util/httprequest.h ${OPENSPACE_BASE_DIR}/include/openspace/util/keys.h ${OPENSPACE_BASE_DIR}/include/openspace/util/mouse.h ${OPENSPACE_BASE_DIR}/include/openspace/util/openspacemodule.h diff --git a/src/util/httprequest.cpp b/src/util/httprequest.cpp new file mode 100644 index 0000000000..9075064fe6 --- /dev/null +++ b/src/util/httprequest.cpp @@ -0,0 +1,193 @@ +/***************************************************************************************** + * * + * OpenSpace * + * * + * Copyright (c) 2014-2017 * + * * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this * + * software and associated documentation files (the "Software"), to deal in the Software * + * without restriction, including without limitation the rights to use, copy, modify, * + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to * + * permit persons to whom the Software is furnished to do so, subject to the following * + * conditions: * + * * + * The above copyright notice and this permission notice shall be included in all copies * + * or substantial portions of the Software. * + * * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, * + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A * + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF * + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE * + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * + ****************************************************************************************/ + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifdef OPENSPACE_CURL_ENABLED +#ifdef WIN32 +#pragma warning (push) +#pragma warning (disable: 4574) // 'INCL_WINSOCK_API_TYPEDEFS' is defined to be '0' +#endif // WIN32 + +#include + +#ifdef WIN32 +#pragma warning (pop) +#endif // WIN32 +#endif + +namespace { + const char* _loggerCat = "HttpRequest"; +} + +namespace openspace { + +HttpRequest::HttpRequest(std::string url) + : _url(std::move(url)) + , _onReadyStateChange([](ReadyState) {}) + , _onProgress([](Progress) { return 0; }) + , _onData([](Data d) { return d.size; }) +{} + +void HttpRequest::onReadyStateChange(ReadyStateChangeCallback cb) { + _onReadyStateChange = std::move(cb); +} + +void HttpRequest::onProgress(ProgressCallback cb) { + _onProgress = std::move(cb); +} + +void HttpRequest::onData(DataCallback cb) { + _onData = std::move(cb); +} + +void HttpRequest::perform(RequestOptions opt) { + setReadyState(ReadyState::Loading); + + CURL* curl = curl_easy_init(); + if (!curl) { + setReadyState(ReadyState::Fail); + return; + } + + curl_easy_setopt(curl, CURLOPT_URL, _url.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + + curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlfunctions::writeCallback); + + curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this); + curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, curlfunctions::progressCallback); + + if (opt.requestTimeoutSeconds > 0) { + curl_easy_setopt(curl, CURLOPT_TIMEOUT, opt.requestTimeoutSeconds); + } + + CURLcode res = curl_easy_perform(curl); + if (res == CURLE_OK) { + LINFO("CURL is ok"); + } else { + LINFO("CURL failed"); + } + + curl_easy_cleanup(curl); +} + +void HttpRequest::setReadyState(openspace::HttpRequest::ReadyState state) { + _readyState = state; +} + + +namespace curlfunctions { +int progressCallback(void* userData, + int64_t nTotalBytes, + int64_t nDownloadedBytes, + int64_t, + int64_t) +{ + HttpRequest* r = reinterpret_cast(userData); + return r->_onProgress( + HttpRequest::Progress{ + true, + static_cast(nTotalBytes), + static_cast(nDownloadedBytes) + } + ); + + LINFO("Transfer info from curl: " << nDownloadedBytes << " out of " << nTotalBytes); + return 0; +} + +size_t writeCallback(char* ptr, size_t size, size_t nmemb, void* userData) { + HttpRequest* r = reinterpret_cast(userData); + return r->_onData(HttpRequest::Data{ptr, size * nmemb}); +} +} + + +using namespace openspace; + +HttpMemoryDownload::HttpMemoryDownload(std::string url) + : _httpRequest(url) +{} + +void HttpMemoryDownload::download(HttpRequest::RequestOptions opt) { + _httpRequest.onData([this](HttpRequest::Data d) { + _downloadedData.insert(_downloadedData.end(), d.buffer, d.buffer + d.size); + return d.size; + }); + _httpRequest.perform(opt); +} + +const std::vector& HttpMemoryDownload::downloadedData() { + return _downloadedData; +} + +HttpFileDownload::HttpFileDownload(std::string url, std::string destinationPath) + : _httpRequest(url) + , _destination(destinationPath) +{} + +void HttpFileDownload::download(HttpRequest::RequestOptions opt) { + LINFO(_destination); + + if (FileSys.fileExists(_destination)) { + LERROR("File already exists!"); + return; + } + + { + ghoul::filesystem::File f = _destination; + ghoul::filesystem::Directory d = f.directoryName(); + if (!FileSys.directoryExists(d)) { + FileSys.createDirectory(d, ghoul::filesystem::Directory::Recursive::Yes); + } + } + + std::ofstream f(_destination, std::ofstream::binary); + + if (f.fail()) { + LERROR("Cannot open file!"); + return; + } + _httpRequest.onData([this, &f](HttpRequest::Data d) { + f.write(d.buffer, d.size); + return d.size; + }); + _httpRequest.perform(opt); + f.close(); +} + +} // namespace openspace