mirror/OpenSpace
1
0
Fork 0
mirror of https://github.com/OpenSpace/OpenSpace.git synced 2026-05-07 03:49:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/data-management' of https://github.com/OpenSpace/OpenSpace into feature/data-management

Browse Source
This commit is contained in:
Alexander Bock
2017-12-23 23:25:38 +01:00
parent 4c92b9ccbb 2fad096a28
commit 079627e4c1
4 changed files with 129 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/openspace/util/httprequest.h
+55 -13
View File
@@ -39,16 +39,27 @@
namespace openspace {
namespace curlfunctions {
size_t writeCallback(char *ptr,
size_t size,
size_t nmemb,
void *userdata);
size_t writeCallback(
char *ptr,
size_t size,
size_t nmemb,
void *userdata
);
int progressCallback(void *clientp,
int64_t dltotal,
int64_t dlnow,
int64_t ultotal,
int64_t ulnow);
int progressCallback(
void *clientp,
int64_t dltotal,
int64_t dlnow,
int64_t ultotal,
int64_t ulnow
);
size_t headerCallback(
char* ptr,
size_t size,
size_t nmemb,
void* userData
);
}
// Synchronous http request
@@ -57,7 +68,7 @@ public:
enum class ReadyState : unsigned int {
Unsent,
Loading,
ReceivedHeaders,
ReceivedHeader,
Fail,
Success
};
@@ -73,6 +84,11 @@ public:
size_t size;
};
struct Header {
char* buffer;
size_t size;
};
using ReadyStateChangeCallback = std::function<void(ReadyState)>;
// ProgressCallback: Return non-zero value to cancel download.
@@ -83,6 +99,11 @@ public:
// the request will fail.
using DataCallback = std::function<size_t(Data)>;
// HeaderCallback: Return number of bytes successfully stored.
// If this does not match the data buffer sice reported in Header,
// the request will fail.
using HeaderCallback = std::function<size_t(Header)>;
struct RequestOptions {
int requestTimeoutSeconds; // 0 for no timeout
};
@@ -90,6 +111,7 @@ public:
HttpRequest(std::string url);
void onReadyStateChange(ReadyStateChangeCallback cb);
void onHeader(HeaderCallback cb);
void onProgress(ProgressCallback cb);
void onData(DataCallback cb);
@@ -103,6 +125,7 @@ private:
ReadyStateChangeCallback _onReadyStateChange;
ProgressCallback _onProgress;
DataCallback _onData;
HeaderCallback _onHeader;
ReadyState _readyState;
Progress _progress;
@@ -120,16 +143,24 @@ private:
int64_t dlnow,
int64_t ultotal,
int64_t ulnow);
friend size_t curlfunctions::headerCallback(
char *ptr,
size_t size,
size_t nmemb,
void *userdata);
};
class HttpDownload {
public:
using ProgressCallback = std::function<bool(HttpRequest::Progress)>;
using HeaderCallback = std::function<bool(HttpRequest::Header)>;
HttpDownload();
HttpDownload(HttpDownload&& d) = default;
virtual ~HttpDownload() = default;
void onProgress(ProgressCallback progressCallback);
void onHeader(HeaderCallback headerCallback);
bool hasStarted();
bool hasFailed();
bool hasSucceeded();
@@ -181,8 +212,10 @@ private:
class HttpFileDownload : public virtual HttpDownload {
public:
using Overwrite = ghoul::Boolean;
HttpFileDownload() = default;
HttpFileDownload(std::string destination);
HttpFileDownload(std::string destination, Overwrite = Overwrite::No);
HttpFileDownload(HttpFileDownload&& d) = default;
virtual ~HttpFileDownload() = default;
protected:
@@ -193,6 +226,7 @@ protected:
static std::mutex _directoryCreationMutex;
private:
std::string _destination;
bool _overwrite;
std::ofstream _file;
};
@@ -220,7 +254,11 @@ public:
// Synchronous download to file
class SyncHttpFileDownload : public SyncHttpDownload, public HttpFileDownload {
public:
SyncHttpFileDownload(std::string url, std::string destinationPath);
SyncHttpFileDownload(
std::string url,
std::string destinationPath,
HttpFileDownload::Overwrite = Overwrite::No
);
virtual ~SyncHttpFileDownload() = default;
};
@@ -234,7 +272,11 @@ public:
// Asynchronous download to file
class AsyncHttpFileDownload : public AsyncHttpDownload, public HttpFileDownload {
public:
AsyncHttpFileDownload(std::string url, std::string destinationPath);
AsyncHttpFileDownload(
std::string url,
std::string destinationPath,
HttpFileDownload::Overwrite = Overwrite::No
);
virtual ~AsyncHttpFileDownload() = default;
};
modules/sync/syncs/httpsynchronization.cpp
+3 -1
View File
@@ -206,7 +206,9 @@ bool HttpSynchronization::trySyncFromUrl(std::string listUrl) {
ghoul::filesystem::FileSystem::PathSeparator +
filename;
downloads.push_back(std::make_unique<AsyncHttpFileDownload>(line, fileDestination));
downloads.push_back(std::make_unique<AsyncHttpFileDownload>(
line, fileDestination, HttpFileDownload::Overwrite::Yes));
auto& fileDownload = downloads.back();
++nDownloads;
src/scene/assetloader.cpp
+22 -14
View File
@@ -39,26 +39,28 @@
#include "assetloader_lua.inl"
namespace {
const char* AssetGlobalVariableName = "asset";
constexpr const char* AssetGlobalVariableName = "asset";
const char* RequireFunctionName = "require";
const char* RequestFunctionName = "request";
const char* ExportFunctionName = "export";
constexpr const char* RequireFunctionName = "require";
constexpr const char* RequestFunctionName = "request";
constexpr const char* ExportFunctionName = "export";
const char* SyncedResourceFunctionName = "syncedResource";
const char* LocalResourceFunctionName = "localResource";
constexpr const char* SyncedResourceFunctionName = "syncedResource";
constexpr const char* LocalResourceFunctionName = "localResource";
const char* OnInitializeFunctionName = "onInitialize";
const char* OnDeinitializeFunctionName = "onDeinitialize";
constexpr const char* OnInitializeFunctionName = "onInitialize";
constexpr const char* OnDeinitializeFunctionName = "onDeinitialize";
const char* ExportsTableName = "_exports";
const char* AssetTableName = "_asset";
const char* DependantsTableName = "_dependants";
constexpr const char* DirectoryConstantName = "directory";
const char* _loggerCat = "AssetLoader";
constexpr const char* ExportsTableName = "_exports";
constexpr const char* AssetTableName = "_asset";
constexpr const char* DependantsTableName = "_dependants";
const char* AssetFileSuffix = "asset";
const char* SceneFileSuffix = "scene";
constexpr const char* _loggerCat = "AssetLoader";
constexpr const char* AssetFileSuffix = "asset";
constexpr const char* SceneFileSuffix = "scene";
enum class PathType : int {
RelativeToAsset = 0,
@@ -133,6 +135,7 @@ void AssetLoader::setUpAssetLuaTable(Asset* asset) {
| |- export
| |- onInitialize
| |- onDeinitialize
| |- directory
|- Dependants (table<dependant, Dependency dep>)
where Dependency is a table:
@@ -201,6 +204,11 @@ void AssetLoader::setUpAssetLuaTable(Asset* asset) {
lua_pushcclosure(*_luaState, &assetloader::onDeinitialize, 1);
lua_setfield(*_luaState, assetTableIndex, OnDeinitializeFunctionName);
// Register directory constant
// string directory
lua_pushstring(*_luaState, asset->assetDirectory().c_str());
lua_setfield(*_luaState, assetTableIndex, DirectoryConstantName);
// Attach Asset table to AssetInfo table
lua_setfield(*_luaState, assetInfoTableIndex, AssetTableName);
src/util/httprequest.cpp
+49 -11
View File
@@ -49,12 +49,21 @@
#endif
namespace {
const char* _loggerCat = "HttpRequest";
constexpr const long StatusCodeOk = 200;
constexpr const char* _loggerCat = "HttpRequest";
}
namespace openspace {
namespace curlfunctions {
size_t headerCallback(char* ptr, size_t size, size_t nmemb, void* userData) {
HttpRequest* r = reinterpret_cast<HttpRequest*>(userData);
size_t nBytes = r->_onHeader(HttpRequest::Header{ptr, size * nmemb});
r->setReadyState(HttpRequest::ReadyState::ReceivedHeader);
return nBytes;
}
int progressCallback(
void* userData,
int64_t nTotalBytes,
@@ -83,6 +92,7 @@ HttpRequest::HttpRequest(std::string url)
, _onReadyStateChange([](ReadyState) {})
, _onProgress([](Progress) { return 0; })
, _onData([](Data d) { return d.size; })
, _onHeader([](Header h) { return h.size; })
{}
void HttpRequest::onReadyStateChange(ReadyStateChangeCallback cb) {
@@ -97,6 +107,10 @@ void HttpRequest::onData(DataCallback cb) {
_onData = std::move(cb);
}
void HttpRequest::onHeader(HeaderCallback cb) {
_onHeader = std::move(cb);
}
void HttpRequest::perform(RequestOptions opt) {
setReadyState(ReadyState::Loading);
@@ -109,18 +123,31 @@ void HttpRequest::perform(RequestOptions opt) {
curl_easy_setopt(curl, CURLOPT_URL, _url.c_str());
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, this);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlfunctions::headerCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlfunctions::writeCallback);
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, curlfunctions::progressCallback);
if (opt.requestTimeoutSeconds > 0) {
curl_easy_setopt(curl, CURLOPT_TIMEOUT, opt.requestTimeoutSeconds);
}
CURLcode res = curl_easy_perform(curl);
setReadyState(res == CURLE_OK ? ReadyState::Success : ReadyState::Fail);
if (res == CURLE_OK) {
long responseCode;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode);
if (responseCode == StatusCodeOk) {
setReadyState(ReadyState::Success);
} else {
setReadyState(ReadyState::Fail);
}
} else {
setReadyState(ReadyState::Fail);
}
curl_easy_cleanup(curl);
}
@@ -279,12 +306,15 @@ size_t HttpMemoryDownload::handleData(HttpRequest::Data d) {
return d.size;
}
HttpFileDownload::HttpFileDownload(std::string destination) {
_destination = std::move(destination);
}
HttpFileDownload::HttpFileDownload(
std::string destination,
HttpFileDownload::Overwrite overwrite)
: _destination(std::move(destination))
, _overwrite(overwrite)
{}
bool HttpFileDownload::initDownload() {
if (FileSys.fileExists(_destination)) {
if (!_overwrite && FileSys.fileExists(_destination)) {
LERROR("File " << _destination << " already exists!");
return false;
}
@@ -325,18 +355,26 @@ SyncHttpMemoryDownload::SyncHttpMemoryDownload(std::string url)
, HttpMemoryDownload()
{}
SyncHttpFileDownload::SyncHttpFileDownload(std::string url, std::string destinationPath)
SyncHttpFileDownload::SyncHttpFileDownload(
std::string url,
std::string destinationPath,
HttpFileDownload::Overwrite overwrite
)
: SyncHttpDownload(std::move(url))
, HttpFileDownload(std::move(destinationPath))
, HttpFileDownload(std::move(destinationPath), overwrite)
{}
AsyncHttpMemoryDownload::AsyncHttpMemoryDownload(std::string url)
: AsyncHttpDownload(std::move(url))
{}
AsyncHttpFileDownload::AsyncHttpFileDownload(std::string url, std::string destinationPath)
AsyncHttpFileDownload::AsyncHttpFileDownload(
std::string url,
std::string destinationPath,
HttpFileDownload::Overwrite overwrite
)
: AsyncHttpDownload(std::move(url))
, HttpFileDownload(std::move(destinationPath))
, HttpFileDownload(std::move(destinationPath), overwrite)
{}