mirror/OpenSpace
1
0
Fork 0
mirror of https://github.com/OpenSpace/OpenSpace.git synced 2026-02-26 14:58:51 -06:00
Code Issues Packages Projects Releases Wiki Activity

Feature/assets (#1784)

Browse Source 
General overhaul of the Asset loading system
This commit is contained in:
Alexander Bock
2021-12-19 21:04:01 +04:00
committed by GitHub
parent f8b5d4b662
commit debcb43ade
167 changed files with 3251 additions and 4441 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
modules/sync/CMakeLists.txt
View File

@@ -28,7 +28,6 @@ set(HEADER_FILES
syncmodule.h
syncs/httpsynchronization.h
syncs/urlsynchronization.h
tasks/syncassettask.h
)
source_group("Header Files" FILES ${HEADER_FILES})
@@ -37,7 +36,6 @@ set(SOURCE_FILES
syncmodule_lua.inl
syncs/httpsynchronization.cpp
syncs/urlsynchronization.cpp
tasks/syncassettask.cpp
)
source_group("Source Files" FILES ${SOURCE_FILES})

62
modules/sync/syncmodule.cpp
View File

@@ -26,7 +26,6 @@
#include <modules/sync/syncs/httpsynchronization.h>
#include <modules/sync/syncs/urlsynchronization.h>
#include <modules/sync/tasks/syncassettask.h>
#include <openspace/documentation/documentation.h>
#include <openspace/engine/globalscallbacks.h>
#include <openspace/rendering/renderable.h>
@@ -40,6 +39,7 @@
#include <ghoul/misc/dictionary.h>
#include <ghoul/misc/memorypool.h>
#include <ghoul/misc/templatefactory.h>
#include <optional>
#include "syncmodule_lua.inl"
@@ -47,6 +47,16 @@ namespace {
constexpr const char* KeyHttpSynchronizationRepositories =
"HttpSynchronizationRepositories";
constexpr const char* KeySynchronizationRoot = "SynchronizationRoot";
struct [[codegen::Dictionary(SyncModule)]] Parameters {
// The list of all repository URLs that are used to fetch data from for
// HTTPSynchronizations
std::optional<std::vector<std::string>> httpSynchronizationRepositories;
// The folder where all of the synchronizations are stored
std::string synchronizationRoot;
};
#include "syncmodule_codegen.cpp"
} // namespace
namespace openspace {
@@ -54,28 +64,13 @@ namespace openspace {
SyncModule::SyncModule() : OpenSpaceModule(Name) {}
void SyncModule::internalInitialize(const ghoul::Dictionary& configuration) {
if (configuration.hasKey(KeyHttpSynchronizationRepositories)) {
ghoul::Dictionary dictionary = configuration.value<ghoul::Dictionary>(
KeyHttpSynchronizationRepositories
);
const Parameters p = codegen::bake<Parameters>(configuration);
for (std::string_view key : dictionary.keys()) {
_synchronizationRepositories.push_back(dictionary.value<std::string>(key));
}
if (p.httpSynchronizationRepositories.has_value()) {
_synchronizationRepositories = *p.httpSynchronizationRepositories;
}
if (configuration.hasKey(KeySynchronizationRoot)) {
_synchronizationRoot = configuration.value<std::string>(KeySynchronizationRoot);
}
else {
LWARNINGC(
"SyncModule",
"No synchronization root specified. Disabling resource synchronization"
);
//_synchronizationEnabled = false;
// TODO: Make it possible to disable synchronization manually.
// Group root and enabled into a sync config object that can be passed to syncs.
}
_synchronizationRoot = absPath(p.synchronizationRoot);
auto fSynchronization = FactoryManager::ref().factory<ResourceSynchronization>();
ghoul_assert(fSynchronization, "ResourceSynchronization factory was not created");
@@ -106,40 +101,23 @@ void SyncModule::internalInitialize(const ghoul::Dictionary& configuration) {
[this](bool, const ghoul::Dictionary& dictionary, ghoul::MemoryPoolBase* pool) {
if (pool) {
void* ptr = pool->allocate(sizeof(UrlSynchronization));
return new (ptr) UrlSynchronization(
dictionary,
_synchronizationRoot
);
return new (ptr) UrlSynchronization(dictionary, _synchronizationRoot);
}
else {
return new UrlSynchronization(
dictionary,
_synchronizationRoot
);
return new UrlSynchronization(dictionary, _synchronizationRoot);
}
}
);
auto fTask = FactoryManager::ref().factory<Task>();
ghoul_assert(fTask, "No task factory existed");
fTask->registerClass<SyncAssetTask>("SyncAssetTask");
}
std::string SyncModule::synchronizationRoot() const {
std::filesystem::path SyncModule::synchronizationRoot() const {
return _synchronizationRoot;
}
void SyncModule::addHttpSynchronizationRepository(std::string repository) {
_synchronizationRepositories.push_back(std::move(repository));
}
std::vector<std::string> SyncModule::httpSynchronizationRepositories() const {
return _synchronizationRepositories;
}
std::vector<documentation::Documentation> SyncModule::documentations() const {
return {
HttpSynchronization::Documentation()
HttpSynchronization::Documentation(),
UrlSynchronization::Documentation()
};
}

9
modules/sync/syncmodule.h
View File

@@ -27,6 +27,8 @@
#include <openspace/util/openspacemodule.h>
#include <filesystem>
namespace openspace {
class SyncModule : public OpenSpaceModule {
@@ -35,10 +37,7 @@ public:
SyncModule();
std::string synchronizationRoot() const;
void addHttpSynchronizationRepository(std::string repository);
std::vector<std::string> httpSynchronizationRepositories() const;
std::filesystem::path synchronizationRoot() const;
std::vector<documentation::Documentation> documentations() const override;
@@ -49,7 +48,7 @@ protected:
private:
std::vector<std::string> _synchronizationRepositories;
std::string _synchronizationRoot;
std::filesystem::path _synchronizationRoot;
};
} // namespace openspace

15
modules/sync/syncmodule_lua.inl
View File

@@ -32,23 +32,20 @@ int syncResource(lua_State* L) {
auto [identifier, version] = ghoul::lua::values<std::string, double>(L);
ghoul::Dictionary dict;
dict.setValue("Type", std::string("HttpSynchronization"));
dict.setValue("Identifier", identifier);
dict.setValue("Version", version);
const SyncModule* module = global::moduleEngine->module<SyncModule>();
HttpSynchronization sync(
dict,
module->synchronizationRoot(),
module->httpSynchronizationRepositories()
);
std::unique_ptr<ResourceSynchronization> sync =
ResourceSynchronization::createFromDictionary(dict);
sync.start();
sync->start();
while (sync.isSyncing()) {
while (sync->isSyncing()) {
std::this_thread::sleep_for(std::chrono::milliseconds(20));
}
ghoul::lua::push(L, sync.isResolved());
ghoul::lua::push(L, sync->isResolved());
return 1;
}

242
modules/sync/syncs/httpsynchronization.cpp
View File

@@ -24,17 +24,10 @@
#include <modules/sync/syncs/httpsynchronization.h>
#include <modules/sync/syncmodule.h>
#include <openspace/documentation/documentation.h>
#include <openspace/documentation/verifier.h>
#include <openspace/util/httprequest.h>
#include <ghoul/fmt.h>
#include <ghoul/filesystem/file.h>
#include <ghoul/filesystem/filesystem.h>
#include <ghoul/logging/logmanager.h>
#include <filesystem>
#include <fstream>
#include <numeric>
namespace {
constexpr const char* _loggerCat = "HttpSynchronization";
@@ -47,10 +40,11 @@ namespace {
constexpr const int ApplicationVersion = 1;
struct [[codegen::Dictionary(HttpSynchronization)]] Parameters {
// A unique identifier for this resource
// The unique identifier for this resource that is used to request a set of files
// from the synchronization servers
std::string identifier;
// The version of this resource
// The version of this resource that should be requested
int version;
};
#include "httpsynchronization_codegen.cpp"
@@ -63,12 +57,11 @@ documentation::Documentation HttpSynchronization::Documentation() {
}
HttpSynchronization::HttpSynchronization(const ghoul::Dictionary& dict,
std::string synchronizationRoot,
std::filesystem::path synchronizationRoot,
std::vector<std::string> synchronizationRepositories
)
: openspace::ResourceSynchronization(dict)
, _synchronizationRoot(std::move(synchronizationRoot))
, _synchronizationRepositories(std::move(synchronizationRepositories))
: ResourceSynchronization(std::move(synchronizationRoot))
, _syncRepositories(std::move(synchronizationRepositories))
{
const Parameters p = codegen::bake<Parameters>(dict);
@@ -83,40 +76,38 @@ HttpSynchronization::~HttpSynchronization() {
}
}
std::string HttpSynchronization::directory() {
std::string d = fmt::format(
"{}/http/{}/{}", _synchronizationRoot, _identifier, _version
);
return absPath(d).string();
std::filesystem::path HttpSynchronization::directory() const {
return _synchronizationRoot / "http" / _identifier / std::to_string(_version);
}
void HttpSynchronization::start() {
if (isSyncing()) {
return;
}
begin();
_state = State::Syncing;
if (hasSyncFile()) {
resolve();
_state = State::Resolved;
return;
}
const std::string& query =
std::string("?") + QueryKeyIdentifier + "=" + _identifier + "&" +
QueryKeyFileVersion + "=" + std::to_string(_version) + "&" +
QueryKeyApplicationVersion + "=" + std::to_string(ApplicationVersion);
std::string query = fmt::format(
"?identifier={}&file_version={}&application_version={}",
_identifier, _version, ApplicationVersion
);
_syncThread = std::thread(
[this](const std::string& q) {
for (const std::string& url : _synchronizationRepositories) {
if (trySyncFromUrl(url + q)) {
for (const std::string& url : _syncRepositories) {
const bool success = trySyncFromUrl(url + q);
if (success) {
createSyncFile();
resolve();
_state = State::Resolved;
return;
}
}
if (!_shouldCancel) {
reject();
_state = State::Rejected;
}
},
query
@@ -125,174 +116,131 @@ void HttpSynchronization::start() {
void HttpSynchronization::cancel() {
_shouldCancel = true;
reset();
}
void HttpSynchronization::clear() {
cancel();
// TODO: Remove all files from directory.
}
size_t HttpSynchronization::nSynchronizedBytes() {
return _nSynchronizedBytes;
}
size_t HttpSynchronization::nTotalBytes() {
return _nTotalBytes;
}
bool HttpSynchronization::nTotalBytesIsKnown() {
return _nTotalBytesKnown;
}
void HttpSynchronization::createSyncFile() {
const std::string& directoryName = directory();
const std::string& filepath = directoryName + ".ossync";
std::filesystem::create_directories(directoryName);
std::ofstream syncFile(filepath, std::ofstream::out);
syncFile << "Synchronized";
syncFile.close();
}
bool HttpSynchronization::hasSyncFile() {
const std::string& path = directory() + ".ossync";
return std::filesystem::is_regular_file(path);
_state = State::Unsynced;
}
bool HttpSynchronization::trySyncFromUrl(std::string listUrl) {
HttpRequest::RequestOptions opt = {};
opt.requestTimeoutSeconds = 0;
SyncHttpMemoryDownload fileListDownload(std::move(listUrl));
fileListDownload.onProgress([&c = _shouldCancel](HttpRequest::Progress) {
HttpMemoryDownload fileListDownload(std::move(listUrl));
fileListDownload.onProgress([&c = _shouldCancel](int64_t, std::optional<int64_t>) {
return !c;
});
fileListDownload.download(opt);
fileListDownload.start();
const bool success = fileListDownload.wait();
if (!fileListDownload.hasSucceeded()) {
const std::vector<char>& buffer = fileListDownload.downloadedData();
if (!success) {
LERRORC("HttpSynchronization", std::string(buffer.begin(), buffer.end()));
return false;
}
const std::vector<char>& buffer = fileListDownload.downloadedData();
_nSynchronizedBytes = 0;
_nTotalBytes = 0;
_nTotalBytesKnown = false;
std::istringstream fileList(std::string(buffer.begin(), buffer.end()));
std::string line;
struct SizeData {
bool totalKnown;
size_t totalBytes;
size_t downloadedBytes;
int64_t downloadedBytes = 0;
std::optional<int64_t> totalBytes;
};
std::unordered_map<std::string, SizeData> sizeData;
std::mutex sizeDataMutex;
std::mutex mutex;
std::atomic_bool startedAllDownloads(false);
std::atomic_bool startedAllDownloads = false;
std::vector<std::unique_ptr<AsyncHttpFileDownload>> downloads;
// Yes, it should be possible to store this in a std::vector<HttpFileDownload> but
// C++ really doesn't like that even though all of the move constructors, move
// assignments and everything is automatically constructed
std::vector<std::unique_ptr<HttpFileDownload>> downloads;
std::string line;
while (fileList >> line) {
size_t lastSlash = line.find_last_of('/');
std::string filename = line.substr(lastSlash + 1);
std::string fileDestination = fmt::format(
"{}/{}{}", directory(), filename, TempSuffix
);
if (sizeData.find(line) != sizeData.end()) {
LWARNING(fmt::format("{}: Duplicate entries: {}", _identifier, line));
if (line.empty() || line[0] == '#') {
// Skip all empty lines and commented out lines
continue;
}
downloads.push_back(std::make_unique<AsyncHttpFileDownload>(
line,
fileDestination,
HttpFileDownload::Overwrite::Yes
));
std::string filename = std::filesystem::path(line).filename().string();
std::filesystem::path destination = directory() / (filename + TempSuffix);
std::unique_ptr<AsyncHttpFileDownload>& fileDownload = downloads.back();
if (sizeData.find(line) != sizeData.end()) {
LWARNING(fmt::format("{}: Duplicate entry for {}", _identifier, line));
continue;
}
sizeData[line] = { false, 0, 0 };
std::unique_ptr<HttpFileDownload> download =
std::make_unique<HttpFileDownload>(
line,
destination,
HttpFileDownload::Overwrite::Yes
);
HttpFileDownload* dl = download.get();
downloads.push_back(std::move(download));
fileDownload->onProgress(
[this, line, &sizeData, &sizeDataMutex,
&startedAllDownloads](HttpRequest::Progress p)
sizeData[line] = SizeData();
dl->onProgress(
[this, line, &sizeData, &mutex, &startedAllDownloads](int64_t downloadedBytes,
std::optional<int64_t> totalBytes)
{
if (!p.totalBytesKnown || !startedAllDownloads) {
if (!totalBytes.has_value() || !startedAllDownloads) {
return !_shouldCancel;
}
std::lock_guard guard(sizeDataMutex);
std::lock_guard guard(mutex);
sizeData[line] = { p.totalBytesKnown, p.totalBytes, p.downloadedBytes };
sizeData[line] = { downloadedBytes, totalBytes };
SizeData size = std::accumulate(
sizeData.begin(),
sizeData.end(),
SizeData{ true, 0, 0 },
[](const SizeData& a, const std::pair<const std::string, SizeData>& b) {
return SizeData {
a.totalKnown && b.second.totalKnown,
a.totalBytes + b.second.totalBytes,
a.downloadedBytes + b.second.downloadedBytes
};
}
);
_nTotalBytesKnown = size.totalKnown;
_nTotalBytes = size.totalBytes;
_nSynchronizedBytes = size.downloadedBytes;
_nTotalBytesKnown = true;
_nTotalBytes = 0;
_nSynchronizedBytes = 0;
for (const std::pair<const std::string, SizeData>& sd : sizeData) {
_nTotalBytesKnown = _nTotalBytesKnown && sd.second.totalBytes.has_value();
_nTotalBytes += sd.second.totalBytes.value_or(0);
_nSynchronizedBytes += sd.second.downloadedBytes;
}
return !_shouldCancel;
});
fileDownload->start(opt);
dl->start();
}
startedAllDownloads = true;
bool failed = false;
for (std::unique_ptr<AsyncHttpFileDownload>& d : downloads) {
for (const std::unique_ptr<HttpFileDownload>& d : downloads) {
d->wait();
if (d->hasSucceeded()) {
// If we are forcing the override, we download to a temporary file
// first, so when we are done here, we need to rename the file to the
// original name
const std::string& tempName = d->destination();
std::string originalName = tempName.substr(
0,
tempName.size() - strlen(TempSuffix)
);
if (std::filesystem::is_regular_file(originalName)) {
std::filesystem::remove(originalName);
}
int success = rename(tempName.c_str(), originalName.c_str());
if (success != 0) {
LERROR(fmt::format(
"Error renaming file {} to {}", tempName, originalName
));
failed = true;
}
}
else {
if (!d->hasSucceeded()) {
LERROR(fmt::format("Error downloading file from URL {}", d->url()));
failed = true;
continue;
}
// If we are forcing the override, we download to a temporary file first, so when
// we are done here, we need to rename the file to the original name
std::filesystem::path tempName = d->destination();
std::filesystem::path originalName = tempName;
// Remove the .tmp extension
originalName.replace_extension("");
if (std::filesystem::is_regular_file(originalName)) {
std::filesystem::remove(originalName);
}
std::error_code ec;
std::filesystem::rename(tempName, originalName, ec);
if (ec) {
LERROR(fmt::format("Error renaming {} to {}", tempName, originalName));
failed = true;
}
}
if (!failed) {
return true;
if (failed) {
for (const std::unique_ptr<HttpFileDownload>& d : downloads) {
d->cancel();
}
}
for (std::unique_ptr<AsyncHttpFileDownload>& d : downloads) {
d->cancel();
}
return false;
return !failed;
}
} // namespace openspace

67
modules/sync/syncs/httpsynchronization.h
View File

@@ -32,40 +32,77 @@
namespace openspace {
/**
* A concreate ResourceSynchronization that will request a list of files from a central
* server (the server list is provided in the constructor) by asking for a specific
* identifier and a file version and application version addition. The server is expected
* to return a flat list of files that can be then directly downloaded into the #directory
* of this synchronization. That list of files can have empty lines and commented out
* lines (starting with a #) that will be ignored. Every other line is URL that will be
* downloaded into the #directory.
* Each requested set of files is identified by a triplet of (identifier, file version,
* application version). The identifier is denoting the group of files that is requested,
* the file version is the specific version of this set of files, and the application
* version is reserved for changes in the data transfer format.
*/
class HttpSynchronization : public ResourceSynchronization {
public:
HttpSynchronization(const ghoul::Dictionary& dict, std::string synchronizationRoot,
/**
* The constructor for this synchronization object. The \p dict contains information
* about the \c identifier and the \version (which is the file version), the
* \p synchronizationRoot is the path to the root folder where the downloaded files
* will be placed, and the \p synchronizationRepositories is a list of the URLs which
* will be asked to resolve the (identifier, version) pair. The first URL in the list
* that can successfully resolve the requested (identifier, version) pair is the one
* that will be used.
*
* \param dict The parameter dictionary (namely the identifier and version)
* \param synchronizationRoot The path to the root from which the complete #directory
* path is constructed
* \param synchronizationRepositories The list of repositories that will be asked to
* resolve the identifier request
*/
HttpSynchronization(const ghoul::Dictionary& dict,
std::filesystem::path synchronizationRoot,
std::vector<std::string> synchronizationRepositories);
/// Destructor that will close the asynchronous file transfer, if it is still ongoing
virtual ~HttpSynchronization();
/**
* Returns the location to which files downloaded through this ResourceSynchronization
* are saved.
*
* \return The location for files created by this class
*/
std::filesystem::path directory() const override;
std::string directory() override;
/**
* Starts the synchronization for this ResourceSynchronization by first trying to find
* a synchronization respository that replies to the request, parsing the result and
* then downloading each of the files that are provided in that result.
*/
void start() override;
void cancel() override;
void clear() override;
size_t nSynchronizedBytes() override;
size_t nTotalBytes() override;
bool nTotalBytesIsKnown() override;
/// Cancels any ongoing synchronization of this ResourceSynchronization
void cancel() override;
static documentation::Documentation Documentation();
private:
void createSyncFile();
bool hasSyncFile();
/// Tries to get a reply from the provided URL and returns that success to the caller
bool trySyncFromUrl(std::string url);
std::atomic_bool _nTotalBytesKnown = false;
std::atomic_size_t _nTotalBytes = 0;
std::atomic_size_t _nSynchronizedBytes = 0;
/// Contains a flag whether the current transfer should be cancelled
std::atomic_bool _shouldCancel = false;
std::string _identifier;
/// The file version for the requested files
int _version = -1;
std::string _synchronizationRoot;
std::vector<std::string> _synchronizationRepositories;
// The list of all repositories that we'll try to sync from
const std::vector<std::string> _syncRepositories;
// The thread that will be doing the synchronization
std::thread _syncThread;
};

253
modules/sync/syncs/urlsynchronization.cpp
View File

@@ -24,20 +24,12 @@
#include <modules/sync/syncs/urlsynchronization.h>
#include <modules/sync/syncmodule.h>
#include <openspace/documentation/documentation.h>
#include <openspace/documentation/verifier.h>
#include <openspace/engine/moduleengine.h>
#include <openspace/util/httprequest.h>
#include <ghoul/fmt.h>
#include <ghoul/logging/logmanager.h>
#include <ghoul/filesystem/file.h>
#include <ghoul/filesystem/filesystem.h>
#include <ghoul/misc/dictionary.h>
#include <filesystem>
#include <fstream>
#include <numeric>
#include <memory>
#include <mutex>
#include <optional>
#include <variant>
@@ -46,14 +38,13 @@ namespace {
struct [[codegen::Dictionary(UrlSynchronization)]] Parameters {
// The URL or urls from where the files are downloaded. If multiple URLs are
// provided, all files will be downloaded to the same directory
// provided, all files will be downloaded to the same directory and the filename
// parameter must not be specified simultaneously
std::variant<std::string, std::vector<std::string>> url;
// This optional identifier will be part of the used folder structure and, if
// provided, can be used to manually find the downloaded folder in the
// synchronization folder. If this value is not specified, 'UseHash' has to be set
// to 'true'
std::optional<std::string> identifier;
// This identifier will be part of the used folder structure and, can be used to
// manually find the downloaded folder in the synchronization folder
std::string identifier;
// If this value is set to 'true' and it is not overwritten by the global
// settings, the file(s) pointed to by this URLSynchronization will always be
@@ -69,7 +60,8 @@ namespace {
std::optional<bool> useHash;
// Optional to provide filename to override the one which is otherwise
// automatically created from the url
// automatically created from the url. If this value is specified, the url
// parameter only only contain exactly one URL
std::optional<std::string> filename;
};
#include "urlsynchronization_codegen.cpp"
@@ -81,12 +73,11 @@ documentation::Documentation UrlSynchronization::Documentation() {
return codegen::doc<Parameters>("sync_synchronization_url");
}
UrlSynchronization::UrlSynchronization(const ghoul::Dictionary& dict,
std::string synchronizationRoot)
: ResourceSynchronization(dict)
, _synchronizationRoot(std::move(synchronizationRoot))
UrlSynchronization::UrlSynchronization(const ghoul::Dictionary& dictionary,
std::filesystem::path synchronizationRoot)
: ResourceSynchronization(std::move(synchronizationRoot))
{
const Parameters p = codegen::bake<Parameters>(dict);
const Parameters p = codegen::bake<Parameters>(dictionary);
if (std::holds_alternative<std::string>(p.url)) {
_urls.push_back(std::get<std::string>(p.url));
@@ -99,38 +90,30 @@ UrlSynchronization::UrlSynchronization(const ghoul::Dictionary& dict,
throw ghoul::MissingCaseException();
}
if (p.filename.has_value() && _urls.size() > 1) {
throw ghoul::RuntimeError(fmt::format(
"UrlSynchronization ({}) requested overwrite filename but specified {} URLs "
"to download, which is not legal",
p.identifier, _urls.size()
));
}
_filename = p.filename.value_or(_filename);
bool useHash = p.useHash.value_or(true);
// We just merge all of the URLs together to generate a hash, it's not as stable to
// reordering URLs, but every other solution would be more error prone
std::string urlConcat = std::accumulate(_urls.begin(), _urls.end(), std::string());
size_t hash = std::hash<std::string>{}(urlConcat);
if (p.identifier.has_value()) {
if (useHash) {
_identifier = *p.identifier + "(" + std::to_string(hash) + ")";
}
else {
_identifier = *p.identifier;
}
}
else {
if (useHash) {
_identifier = std::to_string(hash);
}
else {
documentation::TestResult res;
res.success = false;
documentation::TestResult::Offense o;
o.offender = "Identifier|UseHash";
o.reason = documentation::TestResult::Offense::Reason::MissingKey;
res.offenses.push_back(o);
throw documentation::SpecificationError(std::move(res), "UrlSynchronization");
}
}
_forceOverride = p.forceOverride.value_or(_forceOverride);
const bool useHash = p.useHash.value_or(true);
_identifier = p.identifier;
if (useHash) {
// We just merge all of the URLs together to generate a hash that works for this
std::vector<std::string> urls = _urls;
std::sort(urls.begin(), urls.end());
size_t hash = std::hash<std::string>{}(
std::accumulate(urls.begin(), urls.end(), std::string())
);
_identifier += fmt::format("({})", hash);
}
}
UrlSynchronization::~UrlSynchronization() {
@@ -140,115 +123,107 @@ UrlSynchronization::~UrlSynchronization() {
}
}
std::filesystem::path UrlSynchronization::directory() const {
return _synchronizationRoot / "url" / _identifier / "files";
}
void UrlSynchronization::start() {
if (isSyncing()) {
return;
}
begin();
_state = State::Syncing;
if (hasSyncFile() && !_forceOverride) {
resolve();
_state = State::Resolved;
return;
}
_syncThread = std::thread([this] {
_syncThread = std::thread([this]() {
std::unordered_map<std::string, size_t> fileSizes;
std::mutex fileSizeMutex;
std::atomic_size_t nDownloads(0);
std::atomic_bool startedAllDownloads(false);
std::vector<std::unique_ptr<AsyncHttpFileDownload>> downloads;
size_t nDownloads = 0;
std::atomic_bool startedAllDownloads = false;
std::vector<std::unique_ptr<HttpFileDownload>> downloads;
for (const std::string& url : _urls) {
if (_filename.empty()) {
const size_t lastSlash = url.find_last_of('/');
std::string lastPartOfUrl = url.substr(lastSlash + 1);
std::string name = std::filesystem::path(url).filename().string();
// We can not create filenames with questionmarks
lastPartOfUrl.erase(
std::remove(lastPartOfUrl.begin(), lastPartOfUrl.end(), '?'),
lastPartOfUrl.end()
);
_filename = lastPartOfUrl;
// We can not create filenames with question marks
name.erase(std::remove(name.begin(), name.end(), '?'), name.end());
_filename = name;
}
std::string fileDestination = fmt::format(
"{}/{}{}", directory(), _filename, TempSuffix
);
std::filesystem::path destination = directory() / (_filename + TempSuffix);
std::unique_ptr<AsyncHttpFileDownload> download =
std::make_unique<AsyncHttpFileDownload>(
std::unique_ptr<HttpFileDownload> download =
std::make_unique<HttpFileDownload>(
url,
fileDestination,
destination,
HttpFileDownload::Overwrite::Yes
);
HttpFileDownload* dl = download.get();
downloads.push_back(std::move(download));
std::unique_ptr<AsyncHttpFileDownload>& fileDownload = downloads.back();
++nDownloads;
fileDownload->onProgress(
dl->onProgress(
[this, url, &fileSizes, &fileSizeMutex,
&startedAllDownloads, &nDownloads](HttpRequest::Progress p)
&startedAllDownloads, &nDownloads](int64_t,
std::optional<int64_t> totalBytes)
{
if (p.totalBytesKnown) {
std::lock_guard guard(fileSizeMutex);
fileSizes[url] = p.totalBytes;
if (!totalBytes.has_value()) {
return !_shouldCancel;
}
if (!_nTotalBytesKnown && startedAllDownloads &&
fileSizes.size() == nDownloads)
{
_nTotalBytesKnown = true;
_nTotalBytes = std::accumulate(
fileSizes.begin(),
fileSizes.end(),
size_t(0),
[](size_t a, const std::pair<const std::string, size_t> b) {
return a + b.second;
}
);
std::lock_guard guard(fileSizeMutex);
fileSizes[url] = *totalBytes;
if (!_nTotalBytesKnown && startedAllDownloads &&
fileSizes.size() == nDownloads)
{
_nTotalBytesKnown = true;
_nTotalBytes = 0;
for (const std::pair<const std::string, size_t>& fs : fileSizes) {
_nTotalBytes += fs.second;
}
}
return !_shouldCancel;
});
HttpRequest::RequestOptions opt = {};
opt.requestTimeoutSeconds = 0;
fileDownload->start(opt);
dl->start();
}
startedAllDownloads = true;
bool failed = false;
for (std::unique_ptr<AsyncHttpFileDownload>& d : downloads) {
for (const std::unique_ptr<HttpFileDownload>& d : downloads) {
d->wait();
if (d->hasSucceeded()) {
// If we are forcing the override, we download to a temporary file first,
// so when we are done here, we need to rename the file to the original
// name
if (!d->hasSucceeded()) {
failed = true;
continue;
}
const std::string& tempName = d->destination();
std::string originalName = tempName.substr(
0,
tempName.size() - strlen(TempSuffix)
// If we are forcing the override, we download to a temporary file first, so
// when we are done here, we need to rename the file to the original name
std::filesystem::path tempName = d->destination();
std::filesystem::path originalName = tempName;
// Remove the .tmp extension
originalName.replace_extension("");
if (std::filesystem::is_regular_file(originalName)) {
std::filesystem::remove(originalName);
}
std::error_code ec;
std::filesystem::rename(tempName, originalName, ec);
if (ec) {
LERRORC(
"URLSynchronization",
fmt::format("Error renaming file {} to {}", tempName, originalName)
);
if (std::filesystem::is_regular_file(originalName)) {
std::filesystem::remove(originalName);
}
int success = rename(tempName.c_str(), originalName.c_str());
if (success != 0) {
LERRORC(
"URLSynchronization",
fmt::format(
"Error renaming file {} to {}", tempName, originalName
)
);
failed = true;
}
}
else {
failed = true;
}
}
@@ -257,53 +232,17 @@ void UrlSynchronization::start() {
createSyncFile();
}
else {
for (std::unique_ptr<AsyncHttpFileDownload>& d : downloads) {
for (const std::unique_ptr<HttpFileDownload>& d : downloads) {
d->cancel();
}
}
resolve();
_state = State::Resolved;
});
}
void UrlSynchronization::cancel() {
_shouldCancel = true;
reset();
}
void UrlSynchronization::clear() {
cancel();
// TODO: Remove all files from directory.
}
size_t UrlSynchronization::nSynchronizedBytes() {
return _nSynchronizedBytes;
}
size_t UrlSynchronization::nTotalBytes() {
return _nTotalBytes;
}
bool UrlSynchronization::nTotalBytesIsKnown() {
return _nTotalBytesKnown;
}
void UrlSynchronization::createSyncFile() {
std::string dir = directory();
std::string filepath = dir + ".ossync";
std::filesystem::create_directories(dir);
std::ofstream syncFile(filepath, std::ofstream::out);
syncFile << "Synchronized";
syncFile.close();
}
bool UrlSynchronization::hasSyncFile() {
const std::string& path = directory() + ".ossync";
return std::filesystem::is_regular_file(path);
}
std::string UrlSynchronization::directory() {
std::string d = fmt::format("{}/url/{}/files", _synchronizationRoot, _identifier);
return absPath(d).string();
_state = State::Unsynced;
}
} // namespace openspace

57
modules/sync/syncs/urlsynchronization.h
View File

@@ -28,43 +28,68 @@
#include <openspace/util/resourcesynchronization.h>
#include <atomic>
#include <filesystem>
#include <string>
#include <thread>
#include <vector>
namespace openspace {
/**
* The UrlSynchronization will download one or more files by directly being provided with
* the list of URLs to the files that should be downloaded. The \c Override option in the
* Dictionary determines what should happen in a file with the same name and the same
* identifier has been previously downloaded.
*/
class UrlSynchronization : public ResourceSynchronization {
public:
UrlSynchronization(const ghoul::Dictionary& dict, std::string synchronizationRoot);
/**
* The constructor that takes the parameter \p dictionary and the
* \p synchronizationRoot to the location that is used as the base to compute the
* final storage location. The provided list of URLs must not contain any duplicates.
*
* \param dictionary The parameter dictionary that contains all information that this
* UrlSynchronization needs to download the provided files
* \param synchronizationRoot The base location based off which the final placement
* is calculated
*/
UrlSynchronization(const ghoul::Dictionary& dictionary,
std::filesystem::path synchronizationRoot);
/// Contructor that will terminate the synchronization thread if it is still running
virtual ~UrlSynchronization();
void start() override;
void cancel() override;
void clear() override;
/**
* Returns the location to which files downloaded through this ResourceSynchronization
* are saved.
*
* \return The location for files created by this class
*/
std::filesystem::path directory() const override;
size_t nSynchronizedBytes() override;
size_t nTotalBytes() override;
bool nTotalBytesIsKnown() override;
/// Starts the synchronization for this ResourceSynchronization
void start() override;
/// Cancels any ongoing synchronization of this ResourceSynchronization
void cancel() override;
static documentation::Documentation Documentation();
private:
void createSyncFile();
bool hasSyncFile();
std::string directory() override;
/// The list of URLs that will be downloaded
std::vector<std::string> _urls;
/// Setting whether existing files should be ignored (false) or overwritten (true)
bool _forceOverride = false;
std::string _synchronizationRoot;
std::string _identifier;
/// An optional filename that might overwrite the storage destination. This is only
/// valid if a single URL is specified
std::string _filename;
std::atomic_bool _nTotalBytesKnown = false;
std::atomic_size_t _nTotalBytes = 0;
std::atomic_size_t _nSynchronizedBytes = 0;
/// Contains a flag whether the current transfer should be cancelled
std::atomic_bool _shouldCancel = false;
// The thread that will be doing the synchronization
std::thread _syncThread;
};

119
modules/sync/tasks/syncassettask.cpp
View File

@@ -1,119 +0,0 @@
/*****************************************************************************************
* *
* OpenSpace *
* *
* Copyright (c) 2014-2021 *
* *
* Permission is hereby granted, free of charge, to any person obtaining a copy of this *
* software and associated documentation files (the "Software"), to deal in the Software *
* without restriction, including without limitation the rights to use, copy, modify, *
* merge, publish, distribute, sublicense, and/or sell copies of the Software, and to *
* permit persons to whom the Software is furnished to do so, subject to the following *
* conditions: *
* *
* The above copyright notice and this permission notice shall be included in all copies *
* or substantial portions of the Software. *
* *
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, *
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A *
* PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT *
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF *
* CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE *
* OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. *
****************************************************************************************/
#include <modules/sync/tasks/syncassettask.h>
#include <openspace/openspace.h>
#include <openspace/documentation/core_registration.h>
#include <openspace/documentation/verifier.h>
#include <openspace/engine/moduleengine.h>
#include <openspace/engine/globals.h>
#include <openspace/scene/assetloader.h>
#include <openspace/scripting/scriptengine.h>
#include <openspace/util/openspacemodule.h>
#include <openspace/util/synchronizationwatcher.h>
#include <ghoul/filesystem/filesystem.h>
#include <ghoul/logging/logmanager.h>
#include <ghoul/lua/luastate.h>
#include <chrono>
#include <filesystem>
#include <fstream>
#include <string>
#include <thread>
namespace {
constexpr std::chrono::milliseconds ProgressPollInterval(200);
struct [[codegen::Dictionary(SyncAssetTask)]] Parameters {
// The asset file to sync
std::filesystem::path asset;
};
#include "syncassettask_codegen.cpp"
} // namespace
namespace openspace {
documentation::Documentation SyncAssetTask::documentation() {
return codegen::doc<Parameters>("sync_asset_task");
}
SyncAssetTask::SyncAssetTask(const ghoul::Dictionary& dictionary) {
const Parameters p = codegen::bake<Parameters>(dictionary);
_asset = p.asset.string();
}
std::string SyncAssetTask::description() {
return "Synchronize asset " + _asset;
}
void SyncAssetTask::perform(const Task::ProgressCallback& progressCallback) {
SynchronizationWatcher watcher;
scripting::ScriptEngine scriptEngine;
registerCoreClasses(scriptEngine);
for (OpenSpaceModule* m : global::moduleEngine->modules()) {
scriptEngine.addLibrary(m->luaLibrary());
for (scripting::LuaLibrary& l : m->luaLibraries()) {
scriptEngine.addLibrary(l);
}
}
scriptEngine.initialize();
ghoul::lua::LuaState luaState;
scriptEngine.initializeLuaState(luaState);
AssetLoader loader(&luaState, &watcher, "${ASSETS}");
loader.add(_asset);
loader.rootAsset().startSynchronizations();
std::vector<const Asset*> allAssets = loader.rootAsset().subTreeAssets();
while (true) {
bool inProgress = false;
for (const Asset* asset : allAssets) {
Asset::State state = asset->state();
if (state == Asset::State::Unloaded ||
state == Asset::State::Loaded ||
state == Asset::State::Synchronizing)
{
inProgress = true;
}
}
progressCallback(loader.rootAsset().requestedSynchronizationProgress());
std::this_thread::sleep_for(ProgressPollInterval);
watcher.notify();
if (!inProgress) {
return;
}
}
progressCallback(1.f);
}
} // namespace openspace

49
modules/sync/tasks/syncassettask.h
View File

@@ -1,49 +0,0 @@
/*****************************************************************************************
* *
* OpenSpace *
* *
* Copyright (c) 2014-2021 *
* *
* Permission is hereby granted, free of charge, to any person obtaining a copy of this *
* software and associated documentation files (the "Software"), to deal in the Software *
* without restriction, including without limitation the rights to use, copy, modify, *
* merge, publish, distribute, sublicense, and/or sell copies of the Software, and to *
* permit persons to whom the Software is furnished to do so, subject to the following *
* conditions: *
* *
* The above copyright notice and this permission notice shall be included in all copies *
* or substantial portions of the Software. *
* *
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, *
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A *
* PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT *
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF *
* CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE *
* OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. *
****************************************************************************************/
#ifndef __OPENSPACE_MODULE_SYNC___SYNCASSETTASK___H__
#define __OPENSPACE_MODULE_SYNC___SYNCASSETTASK___H__
#include <openspace/util/task.h>
#include <string>
namespace openspace {
class SyncAssetTask : public Task {
public:
SyncAssetTask(const ghoul::Dictionary& dictionary);
std::string description() override;
void perform(const Task::ProgressCallback& progressCallback) override;
static documentation::Documentation documentation();
private:
std::string _asset;
};
} // namespace openspace
#endif // __OPENSPACE_MODULE_SYNC___SYNCASSETTASK___H__