Fix the map configurations for Mars

Include hotfix for limiting the number of concurrent downloads
This commit is contained in:
Alexander Bock
2017-12-24 18:23:18 +01:00
parent c7c4f01f26
commit 7920c97767
3 changed files with 25 additions and 5 deletions
@@ -14,7 +14,7 @@ local textures = asset.syncedResource({
local marsRadii = { 3396190.0, 3396190.0, 3376200.0 }
local mapServiceConfigs = asset.localResource("mapServiceConfigs")
local mapServiceConfigs = asset.localResource("map_service_configs")
local color_layers = {
{
+12
View File
@@ -28,6 +28,7 @@
#include <ghoul/filesystem/file.h>
#include <ghoul/filesystem/directory.h>
#include <atomic>
#include <fstream>
#include <functional>
#include <memory>
@@ -174,6 +175,7 @@ protected:
void markAsStarted();
void markAsSuccessful();
void markAsFailed();
private:
ProgressCallback _onProgress;
bool _started = false;
@@ -187,6 +189,7 @@ public:
SyncHttpDownload(SyncHttpDownload&& d) = default;
virtual ~SyncHttpDownload() = default;
void download(HttpRequest::RequestOptions opt);
protected:
HttpRequest _httpRequest;
};
@@ -199,8 +202,10 @@ public:
void start(HttpRequest::RequestOptions opt);
void cancel();
void wait();
protected:
void download(HttpRequest::RequestOptions opt);
private:
HttpRequest _httpRequest;
std::thread _downloadThread;
@@ -218,16 +223,21 @@ public:
HttpFileDownload(std::string destination, Overwrite = Overwrite::No);
HttpFileDownload(HttpFileDownload&& d) = default;
virtual ~HttpFileDownload() = default;
protected:
bool initDownload() override;
bool deinitDownload() override;
size_t handleData(HttpRequest::Data d) override;
static std::mutex _directoryCreationMutex;
private:
std::string _destination;
bool _overwrite;
std::ofstream _file;
static const int MaxFilehandles = 35;
static std::atomic_int nCurrentFilehandles;
};
class HttpMemoryDownload : public virtual HttpDownload {
@@ -236,10 +246,12 @@ public:
HttpMemoryDownload(HttpMemoryDownload&& d) = default;
virtual ~HttpMemoryDownload() = default;
const std::vector<char>& downloadedData();
protected:
bool initDownload() override;
bool deinitDownload() override;
size_t handleData(HttpRequest::Data d) override;
private:
std::vector<char> _downloadedData;
};
+12 -4
View File
@@ -51,7 +51,7 @@
namespace {
constexpr const long StatusCodeOk = 200;
constexpr const char* _loggerCat = "HttpRequest";
}
} // namespace
namespace openspace {
@@ -298,9 +298,11 @@ const std::vector<char>& HttpMemoryDownload::downloadedData() {
bool HttpMemoryDownload::initDownload() {
return true;
}
bool HttpMemoryDownload::deinitDownload() {
return true;
}
size_t HttpMemoryDownload::handleData(HttpRequest::Data d) {
_downloadedData.insert(_downloadedData.end(), d.buffer, d.buffer + d.size);
return d.size;
@@ -315,7 +317,7 @@ HttpFileDownload::HttpFileDownload(
bool HttpFileDownload::initDownload() {
if (!_overwrite && FileSys.fileExists(_destination)) {
LERROR("File " << _destination << " already exists!");
LWARNING("File " << _destination << " already exists!");
return false;
}
@@ -329,6 +331,12 @@ bool HttpFileDownload::initDownload() {
}
}
while (nCurrentFilehandles >= MaxFilehandles) {
std::this_thread::sleep_for(std::chrono::milliseconds(500));
}
++nCurrentFilehandles;
_file = std::ofstream(_destination, std::ofstream::binary);
if (_file.fail()) {
@@ -348,8 +356,6 @@ bool HttpFileDownload::initDownload() {
#endif
if (errno) {
#if defined(__unix__)
// We use strerror_r because it's threadsafe.
// GNU's strerror_r returns a string and may ignore buffer completely.
char buffer[255];
LERROR(
"Cannot open file " << destinationFile << ": " <<
@@ -373,6 +379,7 @@ bool HttpFileDownload::initDownload() {
bool HttpFileDownload::deinitDownload() {
_file.close();
--nCurrentFilehandles;
return _file.good();
}
@@ -381,6 +388,7 @@ size_t HttpFileDownload::handleData(HttpRequest::Data d) {
return d.size;
}
std::atomic_int HttpFileDownload::nCurrentFilehandles = 0;
std::mutex HttpFileDownload::_directoryCreationMutex;
SyncHttpMemoryDownload::SyncHttpMemoryDownload(std::string url)