/***************************************************************************************** * * * OpenSpace * * * * Copyright (c) 2014-2017 * * * * Permission is hereby granted, free of charge, to any person obtaining a copy of this * * software and associated documentation files (the "Software"), to deal in the Software * * without restriction, including without limitation the rights to use, copy, modify, * * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to * * permit persons to whom the Software is furnished to do so, subject to the following * * conditions: * * * * The above copyright notice and this permission notice shall be included in all copies * * or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, * * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A * * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE * * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * ****************************************************************************************/ #include #include #include #include #include #include #include #include #include #include #ifdef OPENSPACE_CURL_ENABLED #ifdef WIN32 #pragma warning (push) #pragma warning (disable: 4574) // 'INCL_WINSOCK_API_TYPEDEFS' is defined to be '0' #endif // WIN32 #include #ifdef WIN32 #pragma warning (pop) #endif // WIN32 #endif namespace { const char* _loggerCat = "DownloadManager"; const char* RequestIdentifier = "identifier"; const char* RequestFileVersion = "file_version"; const char* RequestApplicationVersion = "application_version"; struct ProgressInformation { std::shared_ptr future; std::chrono::system_clock::time_point startTime; const openspace::DownloadManager::DownloadProgressCallback* callback; }; size_t writeData(void* ptr, size_t size, size_t nmemb, FILE* stream) { size_t written = fwrite(ptr, size, nmemb, stream); return written; } size_t writeMemoryCallback(void *contents, size_t size, size_t nmemb, void* userp) { size_t realsize = size * nmemb; auto mem = static_cast(userp); mem->buffer = reinterpret_cast( realloc(mem->buffer, mem->size + realsize + 1) ); if (!mem->buffer) { // out of memory printf("not enough memory (realloc returned NULL)\n"); return 0; } std::memcpy(&(mem->buffer[mem->size]), contents, realsize); mem->size += realsize; mem->buffer[mem->size] = 0; return realsize; } int xferinfo(void* p, curl_off_t dltotal, curl_off_t dlnow, curl_off_t, curl_off_t) { if (dltotal == 0) { return 0; } ghoul_assert(p, "Passed progress information is nullptr"); ProgressInformation* i = static_cast(p); ghoul_assert(i, "Passed pointer is not a ProgressInformation"); ghoul_assert(i->future, "FileFuture is not initialized"); ghoul_assert(i->callback, "Callback pointer is nullptr"); if (i->future->abortDownload) { i->future->isAborted = true; return 1; } i->future->currentSize = dlnow; i->future->totalSize = dltotal; i->future->progress = static_cast(dlnow) / static_cast(dltotal); auto now = std::chrono::system_clock::now(); // Compute time spent transferring. auto transferTime = now - i->startTime; // Compute estimated transfer time. auto estimatedTime = transferTime / i->future->progress; // Compute estimated time remaining. auto timeRemaining = estimatedTime - transferTime; float s = static_cast( std::chrono::duration_cast(timeRemaining).count() ); i->future->secondsRemaining = s; if (*(i->callback)) { // The callback function is a pointer to an std::function; that is the reason // for the excessive referencing (*(i->callback))(*(i->future)); } return 0; } } // namespace namespace openspace { DownloadManager::FileFuture::FileFuture(std::string file) : currentSize(-1) , totalSize(-1) , progress(0.f) , secondsRemaining(-1.f) , isFinished(false) , isAborted(false) , filePath(std::move(file)) , errorMessage("") , abortDownload(false) {} DownloadManager::DownloadManager(std::string requestURL, int applicationVersion, bool useMultithreadedDownload) : _applicationVersion(std::move(applicationVersion)) , _useMultithreadedDownload(useMultithreadedDownload) { curl_global_init(CURL_GLOBAL_ALL); _requestURL.push_back(std::move(requestURL)); // TODO: Check if URL is accessible ---abock // TODO: Allow for multiple requestURLs } std::shared_ptr DownloadManager::downloadFile( const std::string& url, const ghoul::filesystem::File& file, bool overrideFile, bool failOnError, unsigned int timeout_secs, DownloadFinishedCallback finishedCallback, DownloadProgressCallback progressCallback) { if (!overrideFile && FileSys.fileExists(file)) { return nullptr; } std::shared_ptr future = std::make_shared(file.filename()); errno = 0; #ifdef WIN32 FILE* fp; errno_t error = fopen_s(&fp, file.path().c_str(), "wb"); if (error != 0) { LERROR( "Could not open/create file:" + file.path() + ". Errno: " + std::to_string(errno) ); } #else FILE* fp = fopen(file.path().c_str(), "wb"); // write binary #endif // WIN32 if (!fp) { LERROR( "Could not open/create file:" + file.path() + ". Errno: " + std::to_string(errno) ); } auto downloadFunction = [url, failOnError, timeout_secs, finishedCallback, progressCallback, future, fp]() { CURL* curl = curl_easy_init(); if (curl) { curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeData); if (timeout_secs) curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout_secs); if (failOnError) curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); ProgressInformation p = { future, std::chrono::system_clock::now(), &progressCallback }; curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, xferinfo); curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &p); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); CURLcode res = curl_easy_perform(curl); curl_easy_cleanup(curl); fclose(fp); if (res == CURLE_OK) { future->isFinished = true; } else { future->errorMessage = curl_easy_strerror(res); } if (finishedCallback) { finishedCallback(*future); } } }; if (_useMultithreadedDownload) { std::thread t = std::thread(downloadFunction); using namespace ghoul::thread; ghoul::thread::setPriority( t, ghoul::thread::ThreadPriorityClass::Idle, ghoul::thread::ThreadPriorityLevel::Lowest ); t.detach(); } else { downloadFunction(); } return future; } std::future DownloadManager::fetchFile( const std::string& url, SuccessCallback successCallback, ErrorCallback errorCallback) { LDEBUG("Start downloading file: '" << url << "' into memory"); auto downloadFunction = [url, successCallback, errorCallback]() { DownloadManager::MemoryFile file; file.buffer = reinterpret_cast(malloc(1)); file.size = 0; file.corrupted = false; CURL* curl = curl_easy_init(); if (!curl) { throw ghoul::RuntimeError("Error initializing cURL"); } curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_WRITEDATA, reinterpret_cast(&file)); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeMemoryCallback); curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false); // Will fail when response status is 400 or above curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); CURLcode res = curl_easy_perform(curl); if (res == CURLE_OK){ // ask for the content-type char *ct; res = curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ct); if (res == CURLE_OK){ std::string extension = std::string(ct); std::stringstream ss(extension); getline(ss, extension ,'/'); getline(ss, extension); file.format = extension; } else { LWARNING("Could not get extension from file downloaded from: " + url); } successCallback(file); curl_easy_cleanup(curl); return file; } else { std::string err = curl_easy_strerror(res); if (errorCallback) { errorCallback(err); } else { LWARNING("Error downloading '" << url << "': " << err); } curl_easy_cleanup(curl); // Throw an error and use try-catch around call to future.get() //throw std::runtime_error( err ); // or set a boolean variable in MemoryFile to determine if it is // valid/corrupted or not. // Return MemoryFile even if it is not valid, and check if it is after // future.get() call. file.corrupted = true; return file; } }; return std::async(std::launch::async, downloadFunction); } std::vector> DownloadManager::downloadRequestFiles( const std::string& identifier, const ghoul::filesystem::Directory& destination, int version, bool overrideFiles, DownloadFinishedCallback, DownloadProgressCallback) { std::vector> futures; FileSys.createDirectory(destination, ghoul::filesystem::FileSystem::Recursive::Yes); // TODO: Check s ---abock // TODO: Escaping is necessary ---abock const std::string fullRequest =_requestURL.back() + "?" + RequestIdentifier + "=" + identifier + "&" + RequestFileVersion + "=" + std::to_string(version) + "&" + RequestApplicationVersion + "=" + std::to_string(_applicationVersion); LDEBUG("Request: " << fullRequest); std::string requestFile = absPath("${TEMPORARY}/" + identifier); LDEBUG("Request File: " << requestFile); bool isFinished = false; auto callback = [&](const FileFuture&) { LDEBUG("Finished: " << requestFile); std::ifstream temporary(requestFile); std::string line; int nFiles = 0; while (std::getline(temporary, line)) { if (line.empty()) { // This might occur if someone added multiple newlines // or mixing carriage return and newlines continue; } ++nFiles; #ifdef __APPLE__ // @TODO: Fix this so that the ifdef is not necessary anymore ---abock std::string file = ghoul::filesystem::File(line, ghoul::filesystem::File::RawPath::Yes).filename(); #else std::string file = ghoul::filesystem::File(line).filename(); #endif LDEBUG("\tLine: " << line << " ; Dest: " << destination.path() + "/" + file); std::shared_ptr future = downloadFile( line, destination.path() + "/" + file, overrideFiles, false, 0, [](const FileFuture& f) { LDEBUG("Finished: " << f.filePath); } ); if (future) { futures.push_back(future); } } isFinished = true; }; std::shared_ptr f = downloadFile( fullRequest, requestFile, true, false, 0, callback ); while (!isFinished) {} return futures; } void DownloadManager::downloadRequestFilesAsync(const std::string& identifier, const ghoul::filesystem::Directory& destination, int version, bool overrideFiles, AsyncDownloadFinishedCallback callback) { auto downloadFunction = [this, identifier, destination, version, overrideFiles, callback]() { std::vector> f = downloadRequestFiles( identifier, destination, version, overrideFiles ); callback(f); }; if (_useMultithreadedDownload) { using namespace ghoul::thread; std::thread t = std::thread(downloadFunction); ghoul::thread::setPriority( t, ThreadPriorityClass::Idle, ThreadPriorityLevel::Lowest ); t.detach(); } else { downloadFunction(); } } void DownloadManager::getFileExtension(const std::string& url, RequestFinishedCallback finishedCallback){ auto requestFunction = [url, finishedCallback]() { CURL* curl = curl_easy_init(); if (curl) { curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); //USING CURLOPT NOBODY curl_easy_setopt(curl, CURLOPT_NOBODY,1); CURLcode res = curl_easy_perform(curl); if (CURLE_OK == res) { char* ct; // ask for the content-type res = curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ct); if ((res == CURLE_OK) && ct && finishedCallback) { finishedCallback(std::string(ct)); } } curl_easy_cleanup(curl); } }; if (_useMultithreadedDownload) { using namespace ghoul::thread; std::thread t = std::thread(requestFunction); ghoul::thread::setPriority( t, ThreadPriorityClass::Idle, ThreadPriorityLevel::Lowest ); t.detach(); } else { requestFunction(); } } } // namespace openspace