From b57c27787d17d2d36b476a89be41bc2818303eba Mon Sep 17 00:00:00 2001 From: Emil Axelsson Date: Fri, 11 Oct 2019 12:57:19 +0200 Subject: [PATCH] Start working on helioviewer download task --- modules/solarbrowsing/CMakeLists.txt | 3 + .../shaders/spacecraftimageprojection_fs.glsl | 6 +- .../shaders/spacecraftimageprojection_vs.glsl | 5 +- modules/solarbrowsing/solarbrowsingmodule.cpp | 8 + .../tasks/helioviewerdownloadtask.cpp | 173 ++++++++++++++++++ .../tasks/helioviewerdownloadtask.h | 58 ++++++ 6 files changed, 249 insertions(+), 4 deletions(-) create mode 100644 modules/solarbrowsing/tasks/helioviewerdownloadtask.cpp create mode 100644 modules/solarbrowsing/tasks/helioviewerdownloadtask.h diff --git a/modules/solarbrowsing/CMakeLists.txt b/modules/solarbrowsing/CMakeLists.txt index c705d41c22..62ce881a9d 100644 --- a/modules/solarbrowsing/CMakeLists.txt +++ b/modules/solarbrowsing/CMakeLists.txt @@ -38,6 +38,8 @@ set(HEADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/rendering/renderablesolarimagery.h ${CMAKE_CURRENT_SOURCE_DIR}/rendering/renderablesolarimageryprojection.h ${CMAKE_CURRENT_SOURCE_DIR}/rendering/spacecraftcameraplane.h + ${CMAKE_CURRENT_SOURCE_DIR}/tasks/helioviewerdownloadtask.h + ${CMAKE_CURRENT_SOURCE_DIR}/util/pixelbufferobject.h ) source_group("Header Files" FILES ${HEADER_FILES}) @@ -48,6 +50,7 @@ set(SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/rendering/renderablesolarimagery.cpp ${CMAKE_CURRENT_SOURCE_DIR}/rendering/renderablesolarimageryprojection.cpp ${CMAKE_CURRENT_SOURCE_DIR}/rendering/spacecraftcameraplane.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tasks/helioviewerdownloadtask.cpp ${CMAKE_CURRENT_SOURCE_DIR}/util/pixelbufferobject.cpp ) source_group("Source Files" FILES ${SOURCE_FILES}) diff --git a/modules/solarbrowsing/shaders/spacecraftimageprojection_fs.glsl b/modules/solarbrowsing/shaders/spacecraftimageprojection_fs.glsl index 32bb462f80..9bbb8dc568 100644 --- a/modules/solarbrowsing/shaders/spacecraftimageprojection_fs.glsl +++ b/modules/solarbrowsing/shaders/spacecraftimageprojection_fs.glsl @@ -26,6 +26,7 @@ const int MAX_SPACECRAFT_OBSERVATORY = 7; in vec4 vs_positionScreenSpace; in vec4 clipSpace; in vec3 vUv[MAX_SPACECRAFT_OBSERVATORY]; +in vec3 vs_positionModelSpace; uniform int numSpacecraftCameraPlanes; uniform dvec3 planePositionSpacecraft[MAX_SPACECRAFT_OBSERVATORY]; @@ -95,8 +96,9 @@ Fragment getFragment() { } if (renderSurface) { - // Yellow-ish. Could discard to get the standard sun texture - outColor = vec4(0.2, 0.2, 0.2, 1.0); + // Arbitrary default shading + vec3 diffuse = vec3((vs_positionModelSpace.y) / SUN_RADIUS) * 0.18; + outColor = vec4(clamp(diffuse, vec3(-1.0), vec3(1.0)) + vec3(0.2, 0.21, 0.22), 1.0); } Fragment frag; diff --git a/modules/solarbrowsing/shaders/spacecraftimageprojection_vs.glsl b/modules/solarbrowsing/shaders/spacecraftimageprojection_vs.glsl index aeabdb9905..882d53a660 100644 --- a/modules/solarbrowsing/shaders/spacecraftimageprojection_vs.glsl +++ b/modules/solarbrowsing/shaders/spacecraftimageprojection_vs.glsl @@ -34,8 +34,8 @@ uniform int numSpacecraftCameraPlanes; layout(location = 0) in vec4 in_position; layout(location = 1) in vec2 in_st; -out vec2 vs_st; out vec4 vs_positionScreenSpace; +out vec3 vs_positionModelSpace; out vec4 clipSpace; out vec3 vUv[MAX_SPACECRAFT_OBSERVATORY]; @@ -43,7 +43,8 @@ out vec3 vUv[MAX_SPACECRAFT_OBSERVATORY]; void main() { // Transform the damn psc to homogenous coordinate - vec4 position = vec4(in_position.xyz * pow(10, in_position.w), 1); + vec4 position = vec4(in_position.xyz, 1); + vs_positionModelSpace = position.xyz; // Transform the positions to the reference frame of the spacecraft to get tex coords for (int i = 0; i < numSpacecraftCameraPlanes; i++) { diff --git a/modules/solarbrowsing/solarbrowsingmodule.cpp b/modules/solarbrowsing/solarbrowsingmodule.cpp index e74328bc62..10d772bee1 100644 --- a/modules/solarbrowsing/solarbrowsingmodule.cpp +++ b/modules/solarbrowsing/solarbrowsingmodule.cpp @@ -25,12 +25,14 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include @@ -53,6 +55,12 @@ void SolarBrowsingModule::internalInitialize(const ghoul::Dictionary&) { fRenderable->registerClass( "RenderableSolarImageryProjection" ); + + auto fTask = FactoryManager::ref().factory(); + ghoul_assert(fTask, "No task factory existed"); + fTask->registerClass("HelioviewerDownloadTask"); + + } } // namespace openspace diff --git a/modules/solarbrowsing/tasks/helioviewerdownloadtask.cpp b/modules/solarbrowsing/tasks/helioviewerdownloadtask.cpp new file mode 100644 index 0000000000..baa18f74d8 --- /dev/null +++ b/modules/solarbrowsing/tasks/helioviewerdownloadtask.cpp @@ -0,0 +1,173 @@ +/***************************************************************************************** + * * + * OpenSpace * + * * + * Copyright (c) 2014-2019 * + * * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this * + * software and associated documentation files (the "Software"), to deal in the Software * + * without restriction, including without limitation the rights to use, copy, modify, * + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to * + * permit persons to whom the Software is furnished to do so, subject to the following * + * conditions: * + * * + * The above copyright notice and this permission notice shall be included in all copies * + * or substantial portions of the Software. * + * * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, * + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A * + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF * + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE * + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * + ****************************************************************************************/ +#include +#include +#include +#include +#include + +namespace { + constexpr const char* _loggerCat = "HelioviewerDownloadTask"; + + constexpr const char* KeyDownloadUrl = "DownloadUrl"; + constexpr const char* KeyFilenames = "Filenames"; + constexpr const char* KeyStartTime = "StartTime"; + constexpr const char* KeyTimeStep = "TimeStep"; + constexpr const char* KeyEndTime = "EndTime"; + constexpr const char* KeyOutputFolder = "OutputFolder"; +} + +namespace openspace { + +documentation::Documentation HelioviewerDownloadTask::documentation() { + using namespace documentation; + return { + "HelioviewerDownloadTask", + "helioviewer_download_task", + { + { + "Type", + new StringEqualVerifier("HelioviewerDownloadTask"), + Optional::No, + "The type of this task" + }, + { + KeyOutputFolder, + new StringAnnotationVerifier("A folder on the local machine"), + Optional::No, + "The folder where to output the downloaded jp2 files" + }, + { + KeyTimeStep, + new DoubleAnnotationVerifier("A positive number"), + Optional::No, + "The preferred number of seconds between each timestep. " + "The actual timestep will be determined by the availability of data + "products but will never be smaller than this number. Use this for " + "temporal downsampling." + }, + { + KeyStartTime, + new StringAnnotationVerifier("A date with format YYYY-MM-DDTHH:MM:SS"), + Optional::No, + "The beginning of the time interval to exteract data from" + }, + { + KeyEndTime, + new StringAnnotationVerifier("A date with format YYYY-MM-DDTHH:MM:SS"), + Optional::No, + "The end of the time interval to exteract data from" + }, + { + KeyDownloadUrl, + new StringAnnotationVerifier( + "A string specifying the directory listings of files to download " + "`https://helioviewer.org/jp2/EUVI-A/${year}/${month}/${day}/195`" + ), + Optional::No, + "The URL to download data from." + "Use ${year}, ${month}, ${day}, ${hour}, ${minute}, ${second} " + "and ${millisecond} as placeholders." + }, + { + KeyFilenames, + new StringAnnotationVerifier( + "A valid filename string such as " + "${year}_${month}_${day}__${hour}_${minute}_${second}__" + "${millisecond}_STEREO-A-SECCHI_EUVI_195.jp2" + ), + Optional::No, + "A string specifying the expected format of filenames on the server " + "Use ${year}, ${month}, ${day}, ${hour}, ${minute}, ${second} " + "and ${millisecond} as placeholders." + } + } + }; +} + +HelioviewerDownloadTask::HelioviewerDownloadTask(const ghoul::Dictionary& dictionary) { + _startTime = Time(Time::convertTime(dictionary.value(KeyStartTime))); + _endTime = Time(Time::convertTime(dictionary.value(KeyEndTime))); + _timeStep = dictionary.value(KeyTimeStep); + _downloadUrl = dictionary.value(KeyDownloadUrl); + _filenames = dictionary.value(KeyFilenames); + _outputFolder = dictionary.value(KeyOutputFolder); +} + +std::string HelioviewerDownloadTask::description() { + return "Download data from helioviewer."; +} + +std::vector HelioviewerDownloadTask::relevantDirectoryListings() const { + std::vector placeholders = { + "${year}", "${month}", "${day}", + "${hour}", "${minute}", "${second}", "${millisecond}" + }; + + std::vector usedPlaceholders; + for (const std::string& placeholder : placeholders) { + const auto it = + std::search(_downloadUrl.begin(), _downloadUrl.end(), placeholder); + if (it != _downloadUrl.end()) { + usedPlaceholders.push_back(placeholder); + } + } + // @TODO: emiax, use + // https://api.helioviewer.org/docs/v2/ + // for example: + // https://api.helioviewer.org/v2/getJP2Image/ + // ?date=2014-01-01T23:59:59Z&sourceId=14&jpip=true +} + +void HelioviewerDownloadTask::perform(const Task::ProgressCallback& progressCallback) { + + // iterate through times, + // for each time, find the expected page. + // Be able to fallback on previous pages, but know when to give up. + // In the page, find the best file to download. + + // get best filename from + + std::unordered_set availableFiles; + std::vector directoryListings = relevantDirectoryListings(); + + + + + SyncHttpMemoryDownload fileListing( + "http://helioviewer.org/jp2/EUVI-A/2006/11/07/195/" + ); + HttpRequest::RequestOptions opt = { 0 }; + fileListing.download(opt); + if (fileListing.hasSucceeded()) { + const std::vector& data = fileListing.downloadedData(); + std::string s(data.begin(), data.end()); + std::cout << s; + } +} + +} + + + diff --git a/modules/solarbrowsing/tasks/helioviewerdownloadtask.h b/modules/solarbrowsing/tasks/helioviewerdownloadtask.h new file mode 100644 index 0000000000..8aba0a8a0f --- /dev/null +++ b/modules/solarbrowsing/tasks/helioviewerdownloadtask.h @@ -0,0 +1,58 @@ +/***************************************************************************************** + * * + * OpenSpace * + * * + * Copyright (c) 2014-2019 * + * * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this * + * software and associated documentation files (the "Software"), to deal in the Software * + * without restriction, including without limitation the rights to use, copy, modify, * + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to * + * permit persons to whom the Software is furnished to do so, subject to the following * + * conditions: * + * * + * The above copyright notice and this permission notice shall be included in all copies * + * or substantial portions of the Software. * + * * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, * + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A * + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF * + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE * + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * + ****************************************************************************************/ +#ifndef __OPENSPACE_MODULE_SPACE___HELIOVIEWERDOWNLOADTASK___H__ +#define __OPENSPACE_MODULE_SPACE___HELIOVIEWERDOWNLOADTASK___H__ + +#include +#include +#include + +#include + +#include +#include + +namespace openspace { + +class HelioviewerDownloadTask : public Task { +public: + HelioviewerDownloadTask(const ghoul::Dictionary& dictionary); + std::string description() override; + void perform(const Task::ProgressCallback& progressCallback) override; + static documentation::Documentation documentation(); + +private: + std::vector relevantDirectoryListings() const; + + std::string _outputFolder; + double _timeStep; + std::string _downloadUrl; + std::string _filenames; + Time _startTime; + Time _endTime; +}; + +} // namespace openspace + +#endif // __OPENSPACE_MODULE_SPACE___HELIOVIEWERDOWNLOADTASK___H__