From 8764ff9c0ca85c48189d024fe0739ead4b533efb Mon Sep 17 00:00:00 2001 From: Sovanny Huy Date: Mon, 1 Jul 2019 16:25:10 -0400 Subject: [PATCH 1/3] downloads initial dataset --- modules/base/rendering/suntexturemanager.cpp | 2 +- .../util/webfieldlinesmanager.cpp | 81 +++++++++++-------- .../util/webfieldlinesmanager.h | 17 ++-- 3 files changed, 56 insertions(+), 44 deletions(-) diff --git a/modules/base/rendering/suntexturemanager.cpp b/modules/base/rendering/suntexturemanager.cpp index 1db8c63b76..52a35715d0 100644 --- a/modules/base/rendering/suntexturemanager.cpp +++ b/modules/base/rendering/suntexturemanager.cpp @@ -44,7 +44,7 @@ namespace openspace SunTextureManager::SunTextureManager() { - _syncDir = FileSys.relativePath("./sync/magnetograms"); + _syncDir = absPath("${BASE}/sync/magnetograms") + ghoul::filesystem::FileSystem::PathSeparator; } void SunTextureManager::update(std::unique_ptr &texture) { diff --git a/modules/fieldlinessequence/util/webfieldlinesmanager.cpp b/modules/fieldlinessequence/util/webfieldlinesmanager.cpp index 738f09a157..7b3b3bbe67 100644 --- a/modules/fieldlinessequence/util/webfieldlinesmanager.cpp +++ b/modules/fieldlinessequence/util/webfieldlinesmanager.cpp @@ -46,45 +46,37 @@ namespace openspace{ // ghoul::filesystem::File tempFile = ghoul::filesystem::File(syncDir); //_syncDir = tempFile.directoryName(); _syncDir = "/Users/shuy/Offline-dokument/OpenSpace/Spaceweather/OpenSpace/data/assets/testwsa/fl_pfss_io_25"; - _flsType = PFSSIO; - _flsTypeString = "PFSSIO"; - - std::string testTime; - triggerTimeInt2String(610056000, testTime); - - - int testInt; - triggerTimeString2Int(testTime, testInt); - + _flsTypeString = "PfssIo"; + _downloadMargin = 3; + getAvailableTriggertimes(); + setInitialSet(global::timeManager.time().j2000Seconds()); + LERROR("WebFieldlinesManager initialized"); - - + } // For testing purposes void WebFieldlinesManager::downloadFieldlines(std::vector& _sourceFile, std::vector& _startTimes, size_t& _nStates){ LERROR("starting download"); - for (auto& tt : _availableTriggertimes){ + for (int index : _filesToDownload){ - downloadOsfls(_flsType, tt); - - //add the timetrigger at the right place in the list - std::string sub = tt.substr(6, 23); - int temp; - triggerTimeString2Int(sub,temp); - double timetriggernumber = temp; - - int i = 0; - while(timetriggernumber > _startTimes[i]){ + std::string filename = _availableTriggertimes[index].second; + double timetrigger = _availableTriggertimes[index].first; + + // download fieldlines file + std::string destPath = downloadOsfls(filename); + + //add the timetrigger at the right place in the list + int i = 0; + while(timetrigger > _startTimes[i]){ if( i == _startTimes.size()) break; else i++; - } - _sourceFile.insert(_sourceFile.begin() + i, _syncDir + '/' + tt.substr(6)); - _startTimes.insert(_startTimes.begin() + i, timetriggernumber); + _sourceFile.insert(_sourceFile.begin() + i, destPath); + _startTimes.insert(_startTimes.begin() + i, timetrigger); _nStates += 1; } } @@ -93,7 +85,7 @@ namespace openspace{ } - void WebFieldlinesManager::downloadOsfls(FlsType type, std::string triggertime){ + std::string WebFieldlinesManager::downloadOsfls(std::string triggertime){ std::string url = "http://localhost:3000/WSA/" + triggertime; std::string destinationpath = absPath(_syncDir + '/' + triggertime.substr(6)); AsyncHttpFileDownload ashd = AsyncHttpFileDownload(url, destinationpath, HttpFileDownload::Overwrite::Yes); @@ -107,6 +99,7 @@ namespace openspace{ if(ashd.hasFailed() == true ){ LERROR("failed: " + destinationpath); } + return destinationpath; } @@ -117,17 +110,33 @@ namespace openspace{ opt.requestTimeoutSeconds = 0; mmryDld.download(opt); - // Put the results in a string and remove [ ] + // Put the results in a string std::string s; std::transform(mmryDld.downloadedData().begin(), mmryDld.downloadedData().end(), std::back_inserter(s), [](char c) { return c; }); parseTriggerTimesList(s); + + // sort ascending by trigger time + std::sort(_availableTriggertimes.begin(), _availableTriggertimes.end()); } void WebFieldlinesManager::setInitialSet(double openSpaceTime){ + int openspaceindex = -1; + do openspaceindex++; + while (openSpaceTime > _availableTriggertimes[openspaceindex].first); + + int startInd = openspaceindex - _downloadMargin; + int endInd = openspaceindex + _downloadMargin; + + if(startInd < 0) startInd = 0; + if(endInd >= _availableTriggertimes.size()) + endInd = _availableTriggertimes.size()-1; + + for(int i = startInd; i <= endInd; i++) + _filesToDownload.push_back(i); } void WebFieldlinesManager::downloadInitialSequence(std::vector triggertimes){ @@ -142,25 +151,27 @@ namespace openspace{ // Turn into stringstream to parse the comma-delimited string into vector std::stringstream ss(s); char c; - std::string substr; + std::string sub; while(ss >> c) { if (c == '[' || c == ']' || c == '"' ) continue; else if (c == ','){ - _availableTriggertimes.push_back(substr); - substr.clear(); + double tt = triggerTimeString2Int(sub.substr(6, 23)); + _availableTriggertimes.push_back(std::make_pair(tt, sub)); + sub.clear(); } - else substr += c; + else sub += c; } - _availableTriggertimes.push_back(substr); + double tt = triggerTimeString2Int(sub.substr(6, 23)); + _availableTriggertimes.push_back(std::make_pair(tt, sub)); } - void WebFieldlinesManager::triggerTimeString2Int(std::string s, int& i){ + int WebFieldlinesManager::triggerTimeString2Int(std::string s){ s.replace(13, 1, ":"); s.replace(16, 1, ":"); Time time = Time(); time.setTime(s); - i = static_cast (time.j2000Seconds() - 69.185013294); + return static_cast (time.j2000Seconds() - 69.185013294); } void WebFieldlinesManager::triggerTimeInt2String(int i, std::string& s){ diff --git a/modules/fieldlinessequence/util/webfieldlinesmanager.h b/modules/fieldlinessequence/util/webfieldlinesmanager.h index 2a537f08a2..cf59fb2e13 100644 --- a/modules/fieldlinessequence/util/webfieldlinesmanager.h +++ b/modules/fieldlinessequence/util/webfieldlinesmanager.h @@ -34,9 +34,7 @@ class WebFieldlinesManager{ public: WebFieldlinesManager(); WebFieldlinesManager(std::string syncDir); - - // What model of FieldlinesSequence is this? - enum FlsType { PFSSIO, PFSSOI, SCSOI }; + // For testing purposes void downloadFieldlines(std::vector& _sourceFile, std::vector& _startTimes, size_t& _nStates); @@ -47,19 +45,22 @@ private: // for now, it's the same as user entered i asset-file std::string _syncDir; - FlsType _flsType; std::string _flsTypeString; + int _downloadMargin; + // List of all triggertimes(field lines states) available for download - // ***turn into ints later*** - std::vector _availableTriggertimes; + std::vector> _availableTriggertimes; + + // Indices for what fieldlines to download + std::vector _filesToDownload; // Function to run in FieldLinesSequence's update loop void update(); // Download one file, given what model type and triggertime in J2000 // ***turn into ints later*** - void downloadOsfls(FlsType type, std::string triggertime); + std::string downloadOsfls(std::string triggertime); // Get list of all triggertimes(field lines states) available form the server void getAvailableTriggertimes(); @@ -79,7 +80,7 @@ private: void parseTriggerTimesList(std::string s); // some temporary functions to translate the filenames to ints - void triggerTimeString2Int(std::string s, int& d); + int triggerTimeString2Int(std::string s); void triggerTimeInt2String(int d, std::string& s); }; From 31a99186d5d1bd99ebb410510e2d55e57f236a20 Mon Sep 17 00:00:00 2001 From: Sovanny Huy Date: Tue, 2 Jul 2019 11:09:01 -0400 Subject: [PATCH 2/3] some cleanup, and now downloads initial list --- .../util/webfieldlinesmanager.cpp | 33 ++++++++++++++----- .../util/webfieldlinesmanager.h | 13 ++++---- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/modules/fieldlinessequence/util/webfieldlinesmanager.cpp b/modules/fieldlinessequence/util/webfieldlinesmanager.cpp index 7b3b3bbe67..8f01c2f018 100644 --- a/modules/fieldlinessequence/util/webfieldlinesmanager.cpp +++ b/modules/fieldlinessequence/util/webfieldlinesmanager.cpp @@ -42,12 +42,11 @@ namespace openspace{ WebFieldlinesManager::WebFieldlinesManager(std::string syncDir){ - // Using constructor for some testing -// ghoul::filesystem::File tempFile = ghoul::filesystem::File(syncDir); - //_syncDir = tempFile.directoryName(); + // change to parameter _syncDir = "/Users/shuy/Offline-dokument/OpenSpace/Spaceweather/OpenSpace/data/assets/testwsa/fl_pfss_io_25"; - _flsTypeString = "PfssIo"; + _flsType = "PfssIo"; _downloadMargin = 3; + _timeTriggerDelta = 7200; getAvailableTriggertimes(); @@ -57,7 +56,9 @@ namespace openspace{ } - // For testing purposes + // dowload files specified in _filestodownload + // I'm thinking we can replace the parameters with pointers to the lists that will be + // initialized in the constuctor instead void WebFieldlinesManager::downloadFieldlines(std::vector& _sourceFile, std::vector& _startTimes, size_t& _nStates){ LERROR("starting download"); for (int index : _filesToDownload){ @@ -81,8 +82,22 @@ namespace openspace{ } } - void WebFieldlinesManager::update(){ - + // this function aint done + void WebFieldlinesManager::update(std::vector startTimes, int activeTriggerTimeIndex){ + // check how many are left until fieldlinessequence runs out - add direction information later + double nextTheroticalTimeTrigger; + double eps = 100; + if(activeTriggerTimeIndex == startTimes.size()-1){ + // if it's at the last index, definetily start some downloading + return; + } + for (int i = activeTriggerTimeIndex; i < startTimes.size(); i++){ + nextTheroticalTimeTrigger = startTimes[i] +_timeTriggerDelta; + if(startTimes[i + 1] > (nextTheroticalTimeTrigger + eps)){ + // do some downloading + } + + } } std::string WebFieldlinesManager::downloadOsfls(std::string triggertime){ @@ -104,7 +119,7 @@ namespace openspace{ void WebFieldlinesManager::getAvailableTriggertimes(){ - std::string url = "http://localhost:3000/WSA/available/" + _flsTypeString; + std::string url = "http://localhost:3000/WSA/available/" + _flsType; SyncHttpMemoryDownload mmryDld = SyncHttpMemoryDownload(url); HttpRequest::RequestOptions opt = {}; opt.requestTimeoutSeconds = 0; @@ -139,10 +154,12 @@ namespace openspace{ _filesToDownload.push_back(i); } + // TODO void WebFieldlinesManager::downloadInitialSequence(std::vector triggertimes){ } + // TODO void WebFieldlinesManager::updateStartTimes(){ } diff --git a/modules/fieldlinessequence/util/webfieldlinesmanager.h b/modules/fieldlinessequence/util/webfieldlinesmanager.h index cf59fb2e13..45cbbf4a2d 100644 --- a/modules/fieldlinessequence/util/webfieldlinesmanager.h +++ b/modules/fieldlinessequence/util/webfieldlinesmanager.h @@ -35,20 +35,21 @@ public: WebFieldlinesManager(); WebFieldlinesManager(std::string syncDir); - - // For testing purposes + // download files specified in _filestodownload void downloadFieldlines(std::vector& _sourceFile, std::vector& _startTimes, size_t& _nStates); private: - // for now, it's the same as user entered i asset-file std::string _syncDir; - std::string _flsTypeString; + std::string _flsType; int _downloadMargin; + // How long between the timesteps? + double _timeTriggerDelta; + // List of all triggertimes(field lines states) available for download std::vector> _availableTriggertimes; @@ -56,7 +57,7 @@ private: std::vector _filesToDownload; // Function to run in FieldLinesSequence's update loop - void update(); + void update(std::vector startTimes, int activeTriggerTimeIndex); // Download one file, given what model type and triggertime in J2000 // ***turn into ints later*** @@ -79,7 +80,7 @@ private: // Parse the data from http request void parseTriggerTimesList(std::string s); - // some temporary functions to translate the filenames to ints + // some temporary functions to translate the filenames to ints (doubles?) int triggerTimeString2Int(std::string s); void triggerTimeInt2String(int d, std::string& s); From 4551631bd56721783eb0c2cb9d645cf240e419ab Mon Sep 17 00:00:00 2001 From: Sovanny Huy Date: Tue, 2 Jul 2019 14:46:15 -0400 Subject: [PATCH 3/3] fix bug (set LoadAtRunime to true) --- data/assets/testwsa/pfss_io.asset | 2 +- data/assets/testwsa/pfss_io_25.asset | 2 +- data/assets/testwsa/pfss_oi.asset | 4 ++-- data/assets/testwsa/pfss_oi_25.asset | 2 +- data/assets/testwsa/pfss_oi_540.asset | 2 +- data/assets/testwsa/pfss_oi_boundary.asset | 2 +- data/assets/testwsa/scs_oi.asset | 2 +- data/assets/testwsa/scs_oi_25.asset | 2 +- data/assets/testwsa/scs_oi_540.asset | 2 +- data/assets/testwsa/scs_oi_boundary.asset | 2 +- .../rendering/renderablefieldlinessequence.cpp | 5 +---- modules/fieldlinessequence/util/webfieldlinesmanager.cpp | 6 +++--- modules/fieldlinessequence/util/webfieldlinesmanager.h | 2 +- 13 files changed, 16 insertions(+), 19 deletions(-) diff --git a/data/assets/testwsa/pfss_io.asset b/data/assets/testwsa/pfss_io.asset index c982364b76..3d03c3b3c1 100755 --- a/data/assets/testwsa/pfss_io.asset +++ b/data/assets/testwsa/pfss_io.asset @@ -23,7 +23,7 @@ local fieldlines = { ColorTableRanges = { { -0.6, 0.6 }, }, - LoadAtRuntime = false, + LoadAtRuntime = true, ScaleToMeters = 1.0, SecondsBefore = 24*60*60, SecondsAfter = 24*60*60, diff --git a/data/assets/testwsa/pfss_io_25.asset b/data/assets/testwsa/pfss_io_25.asset index ad1f7f1a81..6ce0b01c87 100755 --- a/data/assets/testwsa/pfss_io_25.asset +++ b/data/assets/testwsa/pfss_io_25.asset @@ -23,7 +23,7 @@ local fieldlines = { ColorTableRanges = { { -0.6, 0.6 }, }, - LoadAtRuntime = false, + LoadAtRuntime = true, ScaleToMeters = 1.0, SecondsBefore = 24*60*60, SecondsAfter = 24*60*60, diff --git a/data/assets/testwsa/pfss_oi.asset b/data/assets/testwsa/pfss_oi.asset index 4e7ddf956a..56de4ee9e2 100755 --- a/data/assets/testwsa/pfss_oi.asset +++ b/data/assets/testwsa/pfss_oi.asset @@ -23,7 +23,7 @@ local fieldlines = { ColorTableRanges = { { -0.6, 0.6 }, }, - LoadAtRuntime = false, + LoadAtRuntime = true, ScaleToMeters = 1.0, SecondsBefore = 24*60*60, SecondsAfter = 24*60*60, @@ -31,7 +31,7 @@ local fieldlines = { Color = { Uniform = {0.725, 0.1, 0, 1} }, - ColoringMethod = "Quantity", + ColoringMethod = "Quantity" }, GUI = { Path = "/Solar System/Heliosphere", diff --git a/data/assets/testwsa/pfss_oi_25.asset b/data/assets/testwsa/pfss_oi_25.asset index 1437a203de..af07aaed7a 100755 --- a/data/assets/testwsa/pfss_oi_25.asset +++ b/data/assets/testwsa/pfss_oi_25.asset @@ -23,7 +23,7 @@ local fieldlines = { ColorTableRanges = { { -0.6, 0.6 }, }, - LoadAtRuntime = false, + LoadAtRuntime = true, ScaleToMeters = 1.0, SecondsBefore = 24*60*60, SecondsAfter = 24*60*60, diff --git a/data/assets/testwsa/pfss_oi_540.asset b/data/assets/testwsa/pfss_oi_540.asset index e35ffa9ceb..7974bf1a21 100644 --- a/data/assets/testwsa/pfss_oi_540.asset +++ b/data/assets/testwsa/pfss_oi_540.asset @@ -19,7 +19,7 @@ local fieldlines = { { 0, 1000000 }, { 100, 2000 } }, - LoadAtRuntime = false, + LoadAtRuntime = true, ScaleToMeters = 1.0, SecondsBefore = 24*60*60, SecondsAfter = 24*60*60, diff --git a/data/assets/testwsa/pfss_oi_boundary.asset b/data/assets/testwsa/pfss_oi_boundary.asset index 9999cb6422..ecf430b3e7 100644 --- a/data/assets/testwsa/pfss_oi_boundary.asset +++ b/data/assets/testwsa/pfss_oi_boundary.asset @@ -23,7 +23,7 @@ local fieldlines = { ColorTableRanges = { { -0.6, 0.6 }, }, - LoadAtRuntime = false, + LoadAtRuntime = true, ScaleToMeters = 1.0, SecondsBefore = 24*60*60, SecondsAfter = 24*60*60, diff --git a/data/assets/testwsa/scs_oi.asset b/data/assets/testwsa/scs_oi.asset index 62b5d7d42f..a75a5711e4 100755 --- a/data/assets/testwsa/scs_oi.asset +++ b/data/assets/testwsa/scs_oi.asset @@ -23,7 +23,7 @@ local fieldlines = { ColorTableRanges = { { -0.02, 0.02 }, }, - LoadAtRuntime = false, + LoadAtRuntime = true, ScaleToMeters = 1.0, SecondsBefore = 24*60*60, SecondsAfter = 24*60*60, diff --git a/data/assets/testwsa/scs_oi_25.asset b/data/assets/testwsa/scs_oi_25.asset index ce6c635520..29546d8d57 100755 --- a/data/assets/testwsa/scs_oi_25.asset +++ b/data/assets/testwsa/scs_oi_25.asset @@ -23,7 +23,7 @@ local fieldlines = { ColorTableRanges = { { -0.02, 0.02 }, }, - LoadAtRuntime = false, + LoadAtRuntime = true, ScaleToMeters = 1.0, SecondsBefore = 24*60*60, SecondsAfter = 24*60*60, diff --git a/data/assets/testwsa/scs_oi_540.asset b/data/assets/testwsa/scs_oi_540.asset index af1af3c225..acc77e0741 100644 --- a/data/assets/testwsa/scs_oi_540.asset +++ b/data/assets/testwsa/scs_oi_540.asset @@ -19,7 +19,7 @@ local fieldlines = { { 0, 1000000 }, { 100, 2000 } }, - LoadAtRuntime = false, + LoadAtRuntime = true, ScaleToMeters = 1.0, SecondsBefore = 24*60*60, SecondsAfter = 24*60*60, diff --git a/data/assets/testwsa/scs_oi_boundary.asset b/data/assets/testwsa/scs_oi_boundary.asset index 2a6a6de903..e1e76d3fae 100644 --- a/data/assets/testwsa/scs_oi_boundary.asset +++ b/data/assets/testwsa/scs_oi_boundary.asset @@ -23,7 +23,7 @@ local fieldlines = { ColorTableRanges = { { -0.02, 0.02 }, }, - LoadAtRuntime = false, + LoadAtRuntime = true, ScaleToMeters = 1.0, SecondsBefore = 24*60*60, SecondsAfter = 24*60*60, diff --git a/modules/fieldlinessequence/rendering/renderablefieldlinessequence.cpp b/modules/fieldlinessequence/rendering/renderablefieldlinessequence.cpp index 342042a58b..6727ebb123 100644 --- a/modules/fieldlinessequence/rendering/renderablefieldlinessequence.cpp +++ b/modules/fieldlinessequence/rendering/renderablefieldlinessequence.cpp @@ -1171,17 +1171,14 @@ void RenderableFieldlinesSequence::update(const UpdateData& data) { // en liten fuling för att testa att trigga nedladdning if(currentTime > 610056120.0 && currentTime < 610056120.2){ - LERROR("downloading is starting"); WebFieldlinesManager webFieldlinesManager(_sourceFiles[0]); + LERROR("downloading is starting"); webFieldlinesManager.downloadFieldlines(_sourceFiles, _startTimes, _nStates); computeSequenceEndTime(); } - const bool isInInterval = (currentTime >= _startTimes[0]) && (currentTime < _sequenceEndTime); - - // Check if current time in OpenSpace is within sequence interval if (isInInterval) { diff --git a/modules/fieldlinessequence/util/webfieldlinesmanager.cpp b/modules/fieldlinessequence/util/webfieldlinesmanager.cpp index 8f01c2f018..19df3eca5e 100644 --- a/modules/fieldlinessequence/util/webfieldlinesmanager.cpp +++ b/modules/fieldlinessequence/util/webfieldlinesmanager.cpp @@ -59,7 +59,7 @@ namespace openspace{ // dowload files specified in _filestodownload // I'm thinking we can replace the parameters with pointers to the lists that will be // initialized in the constuctor instead - void WebFieldlinesManager::downloadFieldlines(std::vector& _sourceFile, std::vector& _startTimes, size_t& _nStates){ + void WebFieldlinesManager::downloadFieldlines(std::vector& _sourceFiles, std::vector& _startTimes, size_t& _nStates){ LERROR("starting download"); for (int index : _filesToDownload){ @@ -73,10 +73,10 @@ namespace openspace{ //add the timetrigger at the right place in the list int i = 0; while(timetrigger > _startTimes[i]){ - if( i == _startTimes.size()) break; + if( i == static_cast(_nStates)) break; else i++; } - _sourceFile.insert(_sourceFile.begin() + i, destPath); + _sourceFiles.insert(_sourceFiles.begin() + i, destPath); _startTimes.insert(_startTimes.begin() + i, timetrigger); _nStates += 1; } diff --git a/modules/fieldlinessequence/util/webfieldlinesmanager.h b/modules/fieldlinessequence/util/webfieldlinesmanager.h index 45cbbf4a2d..23fffdd576 100644 --- a/modules/fieldlinessequence/util/webfieldlinesmanager.h +++ b/modules/fieldlinessequence/util/webfieldlinesmanager.h @@ -36,7 +36,7 @@ public: WebFieldlinesManager(std::string syncDir); // download files specified in _filestodownload - void downloadFieldlines(std::vector& _sourceFile, std::vector& _startTimes, size_t& _nStates); + void downloadFieldlines(std::vector& _sourceFiles, std::vector& _startTimes, size_t& _nStates);