mirror/OpenSpace
1
0
Fork 0
mirror of https://github.com/OpenSpace/OpenSpace.git synced 2026-03-14 17:40:26 -05:00
Code Issues Packages Projects Releases Wiki Activity

Feature/assets (#1784)

Browse Source 
General overhaul of the Asset loading system
This commit is contained in:
Alexander Bock
2021-12-19 21:04:01 +04:00
committed by GitHub
parent f8b5d4b662
commit debcb43ade
167 changed files with 3251 additions and 4441 deletions
Download Patch File Download Diff File Expand all files Collapse all files

242
modules/sync/syncs/httpsynchronization.cpp
View File

@@ -24,17 +24,10 @@
#include <modules/sync/syncs/httpsynchronization.h>
#include <modules/sync/syncmodule.h>
#include <openspace/documentation/documentation.h>
#include <openspace/documentation/verifier.h>
#include <openspace/util/httprequest.h>
#include <ghoul/fmt.h>
#include <ghoul/filesystem/file.h>
#include <ghoul/filesystem/filesystem.h>
#include <ghoul/logging/logmanager.h>
#include <filesystem>
#include <fstream>
#include <numeric>
namespace {
constexpr const char* _loggerCat = "HttpSynchronization";
@@ -47,10 +40,11 @@ namespace {
constexpr const int ApplicationVersion = 1;
struct [[codegen::Dictionary(HttpSynchronization)]] Parameters {
// A unique identifier for this resource
// The unique identifier for this resource that is used to request a set of files
// from the synchronization servers
std::string identifier;
// The version of this resource
// The version of this resource that should be requested
int version;
};
#include "httpsynchronization_codegen.cpp"
@@ -63,12 +57,11 @@ documentation::Documentation HttpSynchronization::Documentation() {
}
HttpSynchronization::HttpSynchronization(const ghoul::Dictionary& dict,
std::string synchronizationRoot,
std::filesystem::path synchronizationRoot,
std::vector<std::string> synchronizationRepositories
)
: openspace::ResourceSynchronization(dict)
, _synchronizationRoot(std::move(synchronizationRoot))
, _synchronizationRepositories(std::move(synchronizationRepositories))
: ResourceSynchronization(std::move(synchronizationRoot))
, _syncRepositories(std::move(synchronizationRepositories))
{
const Parameters p = codegen::bake<Parameters>(dict);
@@ -83,40 +76,38 @@ HttpSynchronization::~HttpSynchronization() {
}
}
std::string HttpSynchronization::directory() {
std::string d = fmt::format(
"{}/http/{}/{}", _synchronizationRoot, _identifier, _version
);
return absPath(d).string();
std::filesystem::path HttpSynchronization::directory() const {
return _synchronizationRoot / "http" / _identifier / std::to_string(_version);
}
void HttpSynchronization::start() {
if (isSyncing()) {
return;
}
begin();
_state = State::Syncing;
if (hasSyncFile()) {
resolve();
_state = State::Resolved;
return;
}
const std::string& query =
std::string("?") + QueryKeyIdentifier + "=" + _identifier + "&" +
QueryKeyFileVersion + "=" + std::to_string(_version) + "&" +
QueryKeyApplicationVersion + "=" + std::to_string(ApplicationVersion);
std::string query = fmt::format(
"?identifier={}&file_version={}&application_version={}",
_identifier, _version, ApplicationVersion
);
_syncThread = std::thread(
[this](const std::string& q) {
for (const std::string& url : _synchronizationRepositories) {
if (trySyncFromUrl(url + q)) {
for (const std::string& url : _syncRepositories) {
const bool success = trySyncFromUrl(url + q);
if (success) {
createSyncFile();
resolve();
_state = State::Resolved;
return;
}
}
if (!_shouldCancel) {
reject();
_state = State::Rejected;
}
},
query
@@ -125,174 +116,131 @@ void HttpSynchronization::start() {
void HttpSynchronization::cancel() {
_shouldCancel = true;
reset();
}
void HttpSynchronization::clear() {
cancel();
// TODO: Remove all files from directory.
}
size_t HttpSynchronization::nSynchronizedBytes() {
return _nSynchronizedBytes;
}
size_t HttpSynchronization::nTotalBytes() {
return _nTotalBytes;
}
bool HttpSynchronization::nTotalBytesIsKnown() {
return _nTotalBytesKnown;
}
void HttpSynchronization::createSyncFile() {
const std::string& directoryName = directory();
const std::string& filepath = directoryName + ".ossync";
std::filesystem::create_directories(directoryName);
std::ofstream syncFile(filepath, std::ofstream::out);
syncFile << "Synchronized";
syncFile.close();
}
bool HttpSynchronization::hasSyncFile() {
const std::string& path = directory() + ".ossync";
return std::filesystem::is_regular_file(path);
_state = State::Unsynced;
}
bool HttpSynchronization::trySyncFromUrl(std::string listUrl) {
HttpRequest::RequestOptions opt = {};
opt.requestTimeoutSeconds = 0;
SyncHttpMemoryDownload fileListDownload(std::move(listUrl));
fileListDownload.onProgress([&c = _shouldCancel](HttpRequest::Progress) {
HttpMemoryDownload fileListDownload(std::move(listUrl));
fileListDownload.onProgress([&c = _shouldCancel](int64_t, std::optional<int64_t>) {
return !c;
});
fileListDownload.download(opt);
fileListDownload.start();
const bool success = fileListDownload.wait();
if (!fileListDownload.hasSucceeded()) {
const std::vector<char>& buffer = fileListDownload.downloadedData();
if (!success) {
LERRORC("HttpSynchronization", std::string(buffer.begin(), buffer.end()));
return false;
}
const std::vector<char>& buffer = fileListDownload.downloadedData();
_nSynchronizedBytes = 0;
_nTotalBytes = 0;
_nTotalBytesKnown = false;
std::istringstream fileList(std::string(buffer.begin(), buffer.end()));
std::string line;
struct SizeData {
bool totalKnown;
size_t totalBytes;
size_t downloadedBytes;
int64_t downloadedBytes = 0;
std::optional<int64_t> totalBytes;
};
std::unordered_map<std::string, SizeData> sizeData;
std::mutex sizeDataMutex;
std::mutex mutex;
std::atomic_bool startedAllDownloads(false);
std::atomic_bool startedAllDownloads = false;
std::vector<std::unique_ptr<AsyncHttpFileDownload>> downloads;
// Yes, it should be possible to store this in a std::vector<HttpFileDownload> but
// C++ really doesn't like that even though all of the move constructors, move
// assignments and everything is automatically constructed
std::vector<std::unique_ptr<HttpFileDownload>> downloads;
std::string line;
while (fileList >> line) {
size_t lastSlash = line.find_last_of('/');
std::string filename = line.substr(lastSlash + 1);
std::string fileDestination = fmt::format(
"{}/{}{}", directory(), filename, TempSuffix
);
if (sizeData.find(line) != sizeData.end()) {
LWARNING(fmt::format("{}: Duplicate entries: {}", _identifier, line));
if (line.empty() || line[0] == '#') {
// Skip all empty lines and commented out lines
continue;
}
downloads.push_back(std::make_unique<AsyncHttpFileDownload>(
line,
fileDestination,
HttpFileDownload::Overwrite::Yes
));
std::string filename = std::filesystem::path(line).filename().string();
std::filesystem::path destination = directory() / (filename + TempSuffix);
std::unique_ptr<AsyncHttpFileDownload>& fileDownload = downloads.back();
if (sizeData.find(line) != sizeData.end()) {
LWARNING(fmt::format("{}: Duplicate entry for {}", _identifier, line));
continue;
}
sizeData[line] = { false, 0, 0 };
std::unique_ptr<HttpFileDownload> download =
std::make_unique<HttpFileDownload>(
line,
destination,
HttpFileDownload::Overwrite::Yes
);
HttpFileDownload* dl = download.get();
downloads.push_back(std::move(download));
fileDownload->onProgress(
[this, line, &sizeData, &sizeDataMutex,
&startedAllDownloads](HttpRequest::Progress p)
sizeData[line] = SizeData();
dl->onProgress(
[this, line, &sizeData, &mutex, &startedAllDownloads](int64_t downloadedBytes,
std::optional<int64_t> totalBytes)
{
if (!p.totalBytesKnown || !startedAllDownloads) {
if (!totalBytes.has_value() || !startedAllDownloads) {
return !_shouldCancel;
}
std::lock_guard guard(sizeDataMutex);
std::lock_guard guard(mutex);
sizeData[line] = { p.totalBytesKnown, p.totalBytes, p.downloadedBytes };
sizeData[line] = { downloadedBytes, totalBytes };
SizeData size = std::accumulate(
sizeData.begin(),
sizeData.end(),
SizeData{ true, 0, 0 },
[](const SizeData& a, const std::pair<const std::string, SizeData>& b) {
return SizeData {
a.totalKnown && b.second.totalKnown,
a.totalBytes + b.second.totalBytes,
a.downloadedBytes + b.second.downloadedBytes
};
}
);
_nTotalBytesKnown = size.totalKnown;
_nTotalBytes = size.totalBytes;
_nSynchronizedBytes = size.downloadedBytes;
_nTotalBytesKnown = true;
_nTotalBytes = 0;
_nSynchronizedBytes = 0;
for (const std::pair<const std::string, SizeData>& sd : sizeData) {
_nTotalBytesKnown = _nTotalBytesKnown && sd.second.totalBytes.has_value();
_nTotalBytes += sd.second.totalBytes.value_or(0);
_nSynchronizedBytes += sd.second.downloadedBytes;
}
return !_shouldCancel;
});
fileDownload->start(opt);
dl->start();
}
startedAllDownloads = true;
bool failed = false;
for (std::unique_ptr<AsyncHttpFileDownload>& d : downloads) {
for (const std::unique_ptr<HttpFileDownload>& d : downloads) {
d->wait();
if (d->hasSucceeded()) {
// If we are forcing the override, we download to a temporary file
// first, so when we are done here, we need to rename the file to the
// original name
const std::string& tempName = d->destination();
std::string originalName = tempName.substr(
0,
tempName.size() - strlen(TempSuffix)
);
if (std::filesystem::is_regular_file(originalName)) {
std::filesystem::remove(originalName);
}
int success = rename(tempName.c_str(), originalName.c_str());
if (success != 0) {
LERROR(fmt::format(
"Error renaming file {} to {}", tempName, originalName
));
failed = true;
}
}
else {
if (!d->hasSucceeded()) {
LERROR(fmt::format("Error downloading file from URL {}", d->url()));
failed = true;
continue;
}
// If we are forcing the override, we download to a temporary file first, so when
// we are done here, we need to rename the file to the original name
std::filesystem::path tempName = d->destination();
std::filesystem::path originalName = tempName;
// Remove the .tmp extension
originalName.replace_extension("");
if (std::filesystem::is_regular_file(originalName)) {
std::filesystem::remove(originalName);
}
std::error_code ec;
std::filesystem::rename(tempName, originalName, ec);
if (ec) {
LERROR(fmt::format("Error renaming {} to {}", tempName, originalName));
failed = true;
}
}
if (!failed) {
return true;
if (failed) {
for (const std::unique_ptr<HttpFileDownload>& d : downloads) {
d->cancel();
}
}
for (std::unique_ptr<AsyncHttpFileDownload>& d : downloads) {
d->cancel();
}
return false;
return !failed;
}
} // namespace openspace