diff --git a/data/assets/test.scene b/data/assets/test.scene index 16f1b2419d..ddf0e3d8be 100644 --- a/data/assets/test.scene +++ b/data/assets/test.scene @@ -6,11 +6,11 @@ local renderableHelper = asset.require('util/renderable_helper') -- Specifying which other assets should be loaded in this scene asset.require('spice/base') assetHelper.requestAll(asset, 'scene/solarsystem/sun') --- asset.require('scene/solarsystem/planets') +asset.require('scene/solarsystem/planets') -- asset.require('scene/solarsystem/planets/mars/moons/phobos') -- asset.require('scene/solarsystem/planets/mars/moons/deimos') -- asset.require('scene/solarsystem/dwarf_planets/pluto/system') --- assetHelper.requestAll(asset, 'scene/digitaluniverse') +assetHelper.requestAll(asset, 'scene/digitaluniverse') -- Load default key bindings applicable to most scenes asset.require('util/default_keybindings') @@ -29,15 +29,15 @@ local sunRadius = 695508000 local earthRadius = 6371 -- Selected field lines + some additional field lines --- asset.require('./testwsa/scs_oi') --- asset.require('./testwsa/pfss_oi') +asset.require('./testwsa/scs_oi') +asset.require('./testwsa/pfss_oi') asset.require('./testwsa/pfss_io') -- Parker Solar Probe -- asset.require('scene/solarsystem/missions/parkersolarprobe/parker_solar_probe_trail') -- Sun earth connection --- asset.require('./testwsa/sun_earth') +asset.require('./testwsa/sub_earth') -- Every N:th WSA field line -- asset.require('./testwsa/pfss_io_25') diff --git a/data/assets/testwsa/pfss_io.asset b/data/assets/testwsa/pfss_io.asset index bf048b1700..1ad79ab861 100755 --- a/data/assets/testwsa/pfss_io.asset +++ b/data/assets/testwsa/pfss_io.asset @@ -3,7 +3,7 @@ local transforms = asset.require('./transforms') local fieldlinesDirectory = asset.localResource("fl_pfss_io") local transferFunctions = asset.localResource("transferfunctions") -local polarityColorTable = transferFunctions .. "/openness_spec.txt" +local opennessColorTable = transferFunctions .. "/openness_spec.txt" local sunRadius = 695508000 @@ -13,16 +13,16 @@ local fieldlines = { Parent = transforms.HNMReferenceFrame.Identifier, Renderable = { Type = "RenderableFieldlinesSequence", - -- SourceFolder = fieldlinesDirectory, + SourceFolder = fieldlinesDirectory, WebFieldlines = true, FlowEnabled = false, AlphaBlendlingEnabled = false, InputFileType = "osfls", ColorTablePaths = { - polarityColorTable + opennessColorTable }, ColorTableRanges = { - { -1.0, 1.0 }, + { 0.0, 2.0 } }, LoadAtRuntime = true, ScaleToMeters = 1.0, diff --git a/data/assets/testwsa/pfss_io_25.asset b/data/assets/testwsa/pfss_io_25.asset deleted file mode 100755 index e378ab50b5..0000000000 --- a/data/assets/testwsa/pfss_io_25.asset +++ /dev/null @@ -1,44 +0,0 @@ -local assetHelper = asset.require('util/asset_helper') -local transforms = asset.require('./transforms') - -local transferFunctions = asset.localResource("transferfunctions") -local polarityColorTable = transferFunctions .. "/pfss_spec.txt" -local fieldlinesDirectory = asset.localResource("fl_pfss_io_25") - -local sunRadius = 695508000 - --- Fieldlies from json -local fieldlines = { - Identifier = "WSA_Fieldlines_PFSS_IO_25", - Parent = transforms.HNMReferenceFrame.Identifier, - Renderable = { - Type = "RenderableFieldlinesSequence", - WebFieldlines = true, - FlowEnabled = false, - AlphaBlendlingEnabled = false, - InputFileType = "osfls", - ColorTablePaths = { - polarityColorTable - }, - ColorTableRanges = { - { -0.6, 0.6 }, - }, - LoadAtRuntime = true, - ScaleToMeters = 1.0, - SecondsBefore = 24*60*60, - SecondsAfter = 24*60*60, - SimulationModel = "wsa", - }, - GUI = { - Path = "/Solar System/Heliosphere", - Name = "WSA PFSS In to Out Step 25" - }, - Transform = { - Scale = { - Type = "StaticScale", - Scale = sunRadius - }, - } -} - -assetHelper.registerSceneGraphNodesAndExport(asset, { fieldlines }) diff --git a/data/assets/testwsa/pfss_oi.asset b/data/assets/testwsa/pfss_oi.asset index 56de4ee9e2..dae2a93bd9 100755 --- a/data/assets/testwsa/pfss_oi.asset +++ b/data/assets/testwsa/pfss_oi.asset @@ -3,7 +3,8 @@ local transforms = asset.require('./transforms') local fieldlinesDirectory = asset.localResource("fl_pfss_oi") local transferFunctions = asset.localResource("transferfunctions") -local polarityColorTable = transferFunctions .. "/pfss_spec.txt" +local windSpeedPolarityColorTable = transferFunctions .. "/polarity_spec.txt" +local currentSheetColorTable = transferFunctions .. "/currentsheet_spec.txt" local sunRadius = 695508000 @@ -14,14 +15,17 @@ local fieldlines = { Renderable = { Type = "RenderableFieldlinesSequence", SourceFolder = fieldlinesDirectory, + WebFieldlines = true, FlowEnabled = false, AlphaBlendlingEnabled = false, InputFileType = "osfls", ColorTablePaths = { - polarityColorTable + windSpeedPolarityColorTable, + currentSheetColorTable }, ColorTableRanges = { - { -0.6, 0.6 }, + { -1.0, 1.0 }, + { 0.0, 1.0 }, }, LoadAtRuntime = true, ScaleToMeters = 1.0, diff --git a/data/assets/testwsa/pfss_oi_boundary.asset b/data/assets/testwsa/pfss_oi_boundary.asset deleted file mode 100644 index ecf430b3e7..0000000000 --- a/data/assets/testwsa/pfss_oi_boundary.asset +++ /dev/null @@ -1,47 +0,0 @@ -local assetHelper = asset.require('util/asset_helper') -local transforms = asset.require('./transforms') - -local fieldlinesDirectory = asset.localResource("fl_pfss_oi_boundary") -local transferFunctions = asset.localResource("transferfunctions") -local polarityColorTable = transferFunctions .. "/pfss_spec.txt" - -local sunRadius = 695508000 - --- Fieldlies from json -local fieldlines = { - Identifier = "WSA_Fieldlines_PFSS_OI_BOUNDARY", - Parent = transforms.HNMReferenceFrame.Identifier, - Renderable = { - Type = "RenderableFieldlinesSequence", - SourceFolder = fieldlinesDirectory, - FlowEnabled = false, - AlphaBlendlingEnabled = false, - InputFileType = "osfls", - ColorTablePaths = { - polarityColorTable - }, - ColorTableRanges = { - { -0.6, 0.6 }, - }, - LoadAtRuntime = true, - ScaleToMeters = 1.0, - SecondsBefore = 24*60*60, - SecondsAfter = 24*60*60, - SimulationModel = "wsa", - Color = { - Uniform = {0.725, 0.1, 0, 1} - } - }, - GUI = { - Path = "/Solar System/Heliosphere", - Name = "WSA PFSS Out to In Only Boundary Lines" - }, - Transform = { - Scale = { - Type = "StaticScale", - Scale = sunRadius - }, - } -} - -assetHelper.registerSceneGraphNodesAndExport(asset, { fieldlines }) diff --git a/data/assets/testwsa/scs_oi.asset b/data/assets/testwsa/scs_oi.asset index a75a5711e4..314d4fabf9 100755 --- a/data/assets/testwsa/scs_oi.asset +++ b/data/assets/testwsa/scs_oi.asset @@ -3,7 +3,8 @@ local transforms = asset.require('./transforms') local fieldlinesDirectory = asset.localResource("fl_scs_oi") local transferFunctions = asset.localResource("transferfunctions") -local polarityColorTable = transferFunctions .. "/scs_spec.txt" +local windSpeedPolarityColorTable = transferFunctions .. "/polarity_spec.txt" +local currentSheetColorTable = transferFunctions .. "/currentsheet_spec.txt" local sunRadius = 695508000 @@ -14,14 +15,17 @@ local fieldlines = { Renderable = { Type = "RenderableFieldlinesSequence", SourceFolder = fieldlinesDirectory, + WebFieldlines = true, FlowEnabled = false, AlphaBlendlingEnabled = false, InputFileType = "osfls", ColorTablePaths = { - polarityColorTable + windSpeedPolarityColorTable, + currentSheetColorTable }, ColorTableRanges = { - { -0.02, 0.02 }, + { -1.0, 1.0 }, + { 0.0, 1.0 }, }, LoadAtRuntime = true, ScaleToMeters = 1.0, diff --git a/data/assets/testwsa/scs_oi_boundary.asset b/data/assets/testwsa/scs_oi_boundary.asset deleted file mode 100644 index e1e76d3fae..0000000000 --- a/data/assets/testwsa/scs_oi_boundary.asset +++ /dev/null @@ -1,47 +0,0 @@ -local assetHelper = asset.require('util/asset_helper') -local transforms = asset.require('./transforms') - -local fieldlinesDirectory = asset.localResource("fl_scs_oi_boundary") -local transferFunctions = asset.localResource("transferfunctions") -local polarityColorTable = transferFunctions .. "/scs_spec.txt" - -local sunRadius = 695508000 - --- Fieldlies from json -local fieldlines = { - Identifier = "WSA_Fieldlines_SCS_OI_BOUNDARY", - Parent = transforms.HNMReferenceFrame.Identifier, - Renderable = { - Type = "RenderableFieldlinesSequence", - SourceFolder = fieldlinesDirectory, - FlowEnabled = false, - AlphaBlendlingEnabled = false, - InputFileType = "osfls", - ColorTablePaths = { - polarityColorTable - }, - ColorTableRanges = { - { -0.02, 0.02 }, - }, - LoadAtRuntime = true, - ScaleToMeters = 1.0, - SecondsBefore = 24*60*60, - SecondsAfter = 24*60*60, - SimulationModel = "wsa", - Color = { - Uniform = {0.725, 0.1, 0, 1} - } - }, - GUI = { - Path = "/Solar System/Heliosphere", - Name = "WSA SCS Out to In Only Boundary Lines" - }, - Transform = { - Scale = { - Type = "StaticScale", - Scale = sunRadius - }, - } -} - -assetHelper.registerSceneGraphNodesAndExport(asset, { fieldlines }) diff --git a/data/assets/testwsa/sun_earth.asset b/data/assets/testwsa/sun_earth.asset deleted file mode 100644 index 83aa4d91e1..0000000000 --- a/data/assets/testwsa/sun_earth.asset +++ /dev/null @@ -1,51 +0,0 @@ -local assetHelper = asset.require('util/asset_helper') -local transforms = asset.require('./transforms') - -local transferFunctions = asset.localResource("transferfunctions") -local polarityColorTable = transferFunctions .. "/polarity_spec.txt" -local sunearthColorTable = transferFunctions .. "/sun_earth_spec.txt" -local fieldlinesDirectory = asset.localResource("fl_sun_earth") - -local sunRadius = 695508000 - --- Fieldlies from json -local fieldlines = { - Identifier = "WSA_Fieldlines_Sun_Earth_Connection", - Parent = transforms.HNMReferenceFrame.Identifier, - Renderable = { - Type = "RenderableFieldlinesSequence", - SourceFolder = fieldlinesDirectory, - FlowEnabled = false, - AlphaBlendlingEnabled = false, - InputFileType = "osfls", - ColorTablePaths = { - polarityColorTable, - sunearthColorTable - }, - ColorTableRanges = { - { 0.0, 1.0 }, - { 0.0, 2.0 } - }, - LoadAtRuntime = true, - ScaleToMeters = 1.0, - SecondsBefore = 24*60*60, - SecondsAfter = 24*60*60, - SimulationModel = "wsa", - Color = { - Uniform = {0.725, 0.1, 0, 1} - }, - ColoringMethod = "Quantity", - }, - GUI = { - Path = "/Solar System/Heliosphere", - Name = "WSA Sun-Earth Connection" - }, - Transform = { - Scale = { - Type = "StaticScale", - Scale = sunRadius - }, - } -} - -assetHelper.registerSceneGraphNodesAndExport(asset, { fieldlines }) diff --git a/data/assets/testwsa/transferfunctions/openness_spec.txt b/data/assets/testwsa/transferfunctions/openness_spec.txt index 8685af8d23..e60ef94919 100644 --- a/data/assets/testwsa/transferfunctions/openness_spec.txt +++ b/data/assets/testwsa/transferfunctions/openness_spec.txt @@ -1,7 +1,7 @@ width 3.0 -lower -1.0 +lower 0.0 upper 1.0 -mappingkey 0.0 0 0 255 255 -mappingkey 0.5 255 255 0 255 -mappingkey 1.0 255 0 0 255 \ No newline at end of file +mappingkey 0.0 255 255 50 225 +mappingkey 0.5 20 50 255 225 +mappingkey 1.0 255 50 20 225 \ No newline at end of file diff --git a/data/assets/testwsa/transferfunctions/pfss_spec.txt b/data/assets/testwsa/transferfunctions/pfss_spec.txt deleted file mode 100644 index b91e18e08f..0000000000 --- a/data/assets/testwsa/transferfunctions/pfss_spec.txt +++ /dev/null @@ -1,7 +0,0 @@ -width 4 -lower 0.0 -upper 1.0 - -mappingkey 0.0 0 0 255 255 -mappingkey 1.0 255 0 0 255 - diff --git a/data/assets/testwsa/transferfunctions/polarity_spec.txt b/data/assets/testwsa/transferfunctions/polarity_spec.txt index 964be8ad27..86c724ef5b 100644 --- a/data/assets/testwsa/transferfunctions/polarity_spec.txt +++ b/data/assets/testwsa/transferfunctions/polarity_spec.txt @@ -2,5 +2,5 @@ width 2.0 lower 0.0 upper 1.0 -mappingkey 0.0 0 0 255 255 -mappingkey 1.0 255 0 0 255 +mappingkey 0.0 20 50 255 205 +mappingkey 1.0 255 50 20 205 diff --git a/data/assets/testwsa/transferfunctions/scs_spec.txt b/data/assets/testwsa/transferfunctions/scs_spec.txt deleted file mode 100644 index 925fb96819..0000000000 --- a/data/assets/testwsa/transferfunctions/scs_spec.txt +++ /dev/null @@ -1,9 +0,0 @@ -width 4 -lower 0.0 -upper 1.0 - -mappingkey 0.0 0 0 255 255 -mappingkey 0.49 0 0 255 255 -mappingkey 0.51 255 0 0 255 -mappingkey 1.0 255 0 0 255 - diff --git a/data/assets/testwsa/transferfunctions/sun_earth_spec.txt b/data/assets/testwsa/transferfunctions/sun_earth_spec.txt deleted file mode 100644 index 151c3133b6..0000000000 --- a/data/assets/testwsa/transferfunctions/sun_earth_spec.txt +++ /dev/null @@ -1,8 +0,0 @@ -width 3.0 -lower 0.0 -upper 2.0 - -mappingkey 0.0 0 255 255 255 -mappingkey 0.5 255 255 0 255 -mappingkey 1.0 255 0 255 255 - diff --git a/include/openspace/util/httprequest.h b/include/openspace/util/httprequest.h index eb3e345a24..785b4335f5 100644 --- a/include/openspace/util/httprequest.h +++ b/include/openspace/util/httprequest.h @@ -141,6 +141,7 @@ public: HttpDownload(); HttpDownload(HttpDownload&& d) = default; + HttpDownload& operator=(HttpDownload&&) = default; virtual ~HttpDownload() = default; void onProgress(ProgressCallback progressCallback); void onHeader(HeaderCallback headerCallback); @@ -169,6 +170,7 @@ class SyncHttpDownload : public virtual HttpDownload { public: SyncHttpDownload(std::string url); SyncHttpDownload(SyncHttpDownload&& d) = default; + SyncHttpDownload& operator=(SyncHttpDownload&&) = default; virtual ~SyncHttpDownload() = default; void download(HttpRequest::RequestOptions opt); @@ -182,6 +184,7 @@ class AsyncHttpDownload : public virtual HttpDownload { public: AsyncHttpDownload(std::string url); AsyncHttpDownload(AsyncHttpDownload&& d); + AsyncHttpDownload& operator=(AsyncHttpDownload&&) = default; virtual ~AsyncHttpDownload() = default; void start(HttpRequest::RequestOptions opt); void cancel(); @@ -208,6 +211,7 @@ public: HttpFileDownload() = default; HttpFileDownload(std::string destination, Overwrite = Overwrite::No); HttpFileDownload(HttpFileDownload&& d) = default; + HttpFileDownload& operator=(HttpFileDownload&&) = default; virtual ~HttpFileDownload() = default; const std::string& destination() const; @@ -233,6 +237,8 @@ class HttpMemoryDownload : public virtual HttpDownload { public: HttpMemoryDownload() = default; HttpMemoryDownload(HttpMemoryDownload&& d) = default; + HttpMemoryDownload& operator=(HttpMemoryDownload&&) = default; + virtual ~HttpMemoryDownload() = default; const std::vector& downloadedData() const; @@ -278,6 +284,8 @@ public: std::string destinationPath, HttpFileDownload::Overwrite = Overwrite::No ); + AsyncHttpFileDownload(AsyncHttpFileDownload&& d) = default; + AsyncHttpFileDownload& operator=(AsyncHttpFileDownload&&) = default; virtual ~AsyncHttpFileDownload() = default; }; @@ -288,4 +296,4 @@ public: #endif -#endif // __OPENSPACE_CORE___HTTPREQUEST___H__ +#endif // __OPENSPACE_CORE___HTTPREQUEST___H__ \ No newline at end of file diff --git a/modules/fieldlinessequence/rendering/renderablefieldlinessequence.cpp b/modules/fieldlinessequence/rendering/renderablefieldlinessequence.cpp index a7be45c377..165e7f51a2 100644 --- a/modules/fieldlinessequence/rendering/renderablefieldlinessequence.cpp +++ b/modules/fieldlinessequence/rendering/renderablefieldlinessequence.cpp @@ -433,15 +433,10 @@ namespace openspace { } } else { - - if (_webFieldlinesManager.checkConnectionToServer()) { - LERROR("initializing sync-dir and downloading startset webFLman"); - sourceFolderPath = _webFieldlinesManager.initializeSyncDirectory(_identifier); - _webFieldlinesManager.preDownload(); - } - else { - LERROR("Unable to establish connection with fieldline provider"); - } + LERROR("initializing sync-dir and downloading startset webFLman"); + sourceFolderPath = _webFieldlinesManager.initializeSyncDirectory(_identifier); + _webFieldlinesManager.preDownload(); + } // Ensure that the source folder exists and then extract @@ -1187,7 +1182,7 @@ namespace openspace { } void RenderableFieldlinesSequence::initializeWebManager() { - _webFieldlinesManager.initializeWebFieldlinesManager(_identifier, "PfssIo", _nStates, _sourceFiles, _startTimes); + _webFieldlinesManager.initializeWebFieldlinesManager(_identifier, _nStates, _sourceFiles, _startTimes); } void RenderableFieldlinesSequence::update(const UpdateData& data) { @@ -1198,7 +1193,7 @@ namespace openspace { const double currentTime = data.time.j2000Seconds(); - if (_webFieldlines && _webFieldlinesManager.isConnected()) { + if (_webFieldlines) { if (!_webFieldlinesManager.hasUpdated && _webFieldlinesManager.checkIfWindowIsReadyToLoad()) { _startTimes.clear(); @@ -1208,17 +1203,6 @@ namespace openspace { std::sort(_startTimes.begin(), _startTimes.end()); _nStates = _startTimes.size(); - - /*for (int i = 0; i < _sourceFiles.size(); ++i) { - LERROR(_sourceFiles[i]); - } - - for (int i = 0; i < _startTimes.size(); ++i) { - LERROR(std::to_string(_startTimes[i])); - } */ - - //updateActiveTriggerTimeIndex(currentTime); - _webFieldlinesManager.hasUpdated = true; _webFieldlinesManager.notifyUpdate = true; _webFieldlinesManager.resetWorker(); @@ -1263,7 +1247,7 @@ namespace openspace { _needsUpdate = false; } - if (_webFieldlines && _webFieldlinesManager.isConnected() && _webFieldlinesManager.notifyUpdate) { + if (_webFieldlines && _webFieldlinesManager.notifyUpdate) { updateActiveTriggerTimeIndex(currentTime); computeSequenceEndTime(); _webFieldlinesManager.notifyUpdate = false; diff --git a/modules/fieldlinessequence/util/webfieldlinesmanager.cpp b/modules/fieldlinessequence/util/webfieldlinesmanager.cpp index 1486b84e33..b56126518d 100644 --- a/modules/fieldlinessequence/util/webfieldlinesmanager.cpp +++ b/modules/fieldlinessequence/util/webfieldlinesmanager.cpp @@ -45,18 +45,14 @@ namespace openspace{ // --------------------------- PUBLIC FUNCTIONS --------------------------- // - void WebFieldlinesManager::initializeWebFieldlinesManager(std::string identifier, std::string fieldLineModelType, size_t& _nStates, std::vector& _sourceFiles, + void WebFieldlinesManager::initializeWebFieldlinesManager(std::string identifier, size_t& _nStates, std::vector& _sourceFiles, std::vector& _startTimes) { - if (_connected) { - _flsType = fieldLineModelType; + // Initialize the sliding window + _webFieldlinesWindow = WebFieldlinesWindow(_syncDir, "http://localhost:3000/", _sourceFiles, _startTimes, _nStates, convertIdentifierToID(identifier)); - // Initialize the sliding window - _webFieldlinesWindow = WebFieldlinesWindow(_syncDir, "http://localhost:3000/", _sourceFiles, _startTimes, _nStates, convertIdentifierToID(identifier)); + LINFO("WebFieldlinesManager initialized " + identifier); - - LERROR("WebFieldlinesManager initialized " + identifier); - } } bool WebFieldlinesManager::checkConnectionToServer() { @@ -102,11 +98,10 @@ namespace openspace{ // the start latest line // this could be used to test the connection too void WebFieldlinesManager::preDownload(){ - // for some reason the fieldlines looks f-ed up when downloaded and read fromt his endpoint???? so weird - // could it have something to do with endianness? - //std::string url = "http://localhost:3000/WSA/fieldline/WSA_Fieldlines_PFSS_IO/2019-05-02T20-00-00.000.osfls"; - std::string url = "http://localhost:3000/WSA/PfssIo2019-05-02T20-00-00.000.osfls"; // temp thing, should be the latest one - std::string destinationpath = absPath(_syncDir + ghoul::filesystem::FileSystem::PathSeparator + "2019-05-02T20-00-00.000.osfls"); // what the downloaded filename is to be + // Download a dataset that we know exists, hopefully the API could send the latest entry of that dataset + // TODO(Axel): Change this endpoint to be dynamic for each dataset + std::string url = "https://iswa.ccmc.gsfc.nasa.gov/iswa_data_tree/model/solar/WSA4.5/WSA4.5_fieldlines/trace_pfss_intoout/2017/09/2017-09-28T00-23-22.000.osfls"; + std::string destinationpath = absPath(_syncDir + ghoul::filesystem::FileSystem::PathSeparator + "2017-09-28T00-23-22.000.osfls"); // what the downloaded filename is to be AsyncHttpFileDownload ashd = AsyncHttpFileDownload(url, destinationpath, HttpFileDownload::Overwrite::Yes); HttpRequest::RequestOptions opt = {}; opt.requestTimeoutSeconds = 0; @@ -175,7 +170,7 @@ namespace openspace{ FieldLineType WebFieldlinesManager::convertIdentifierToID(std::string identifier) { std::map string2ID{ - {"WSA_Fieldlines_Sun_Earth_Connection",WSA_Fieldlines_Sun_Earth_Connection}, + {"WSA_Fieldlines_Sub_Earth_Track",WSA_Fieldlines_Sub_Earth_Track}, {"WSA_Fieldlines_SCS_OI",WSA_Fieldlines_SCS_OI}, {"WSA_Fieldlines_PFSS_IO",WSA_Fieldlines_PFSS_IO}, {"WSA_Fieldlines_PFSS_OI",WSA_Fieldlines_PFSS_OI} diff --git a/modules/fieldlinessequence/util/webfieldlinesmanager.h b/modules/fieldlinessequence/util/webfieldlinesmanager.h index 2b9d3c7a19..b2295096da 100644 --- a/modules/fieldlinessequence/util/webfieldlinesmanager.h +++ b/modules/fieldlinessequence/util/webfieldlinesmanager.h @@ -30,7 +30,7 @@ #include enum FieldLineType { - WSA_Fieldlines_Sun_Earth_Connection = 1176, + WSA_Fieldlines_Sub_Earth_Track = 1176, WSA_Fieldlines_SCS_OI = 1177, WSA_Fieldlines_PFSS_IO = 1178, WSA_Fieldlines_PFSS_OI = 1179, @@ -45,14 +45,10 @@ public: // Constructor WebFieldlinesManager() = default; - // download files specified in _filestodownload - //void downloadFieldlines(); - - // To replace the constructor, takes the identifier of the field line, is used for storing the field lines mainly // Also takes a second parameter containing the name of the field line model used. // These may in the future be the same. - void initializeWebFieldlinesManager(std::string identifier, std::string fieldLineModelType, size_t& _nStates, std::vector& _sourceFiles, std::vector& _startTimes); + void initializeWebFieldlinesManager(std::string identifier, size_t& _nStates, std::vector& _sourceFiles, std::vector& _startTimes); bool checkConnectionToServer(); @@ -61,7 +57,6 @@ public: std::string initializeSyncDirectory(std::string identifier); - // Temporary function - this should be moved to the worker. It's to download // the start lines if the directory is empty or launching for the first time void preDownload(); @@ -87,16 +82,12 @@ private: bool _connected = false; std::string _syncDir; - - // What model is this field line derived from, may come to be the same as the identifier - std::string _flsType; - + // The datastructure for managing the interval of fieldline sets to be downloaded WebFieldlinesWindow _webFieldlinesWindow; // Converts a string to id used in api request. FieldLineType convertIdentifierToID(std::string identifier); - }; diff --git a/modules/fieldlinessequence/util/webfieldlineswindow.cpp b/modules/fieldlinessequence/util/webfieldlineswindow.cpp index f0addbd61b..ab3723dc51 100644 --- a/modules/fieldlinessequence/util/webfieldlineswindow.cpp +++ b/modules/fieldlinessequence/util/webfieldlineswindow.cpp @@ -42,17 +42,14 @@ namespace openspace{ // --------------------------- CONSTRUCTORS ---------------------------------------// WebFieldlinesWindow::WebFieldlinesWindow(std::string syncDir, std::string serverUrl, std::vector& _sourceFiles, - std::vector& _startTimes, size_t& _nStates, size_t apiID): + std::vector& _startTimes, size_t& _nStates, int apiID): _apiID(apiID){ _window.backWidth = 3; _window.forwardWidth = 3; _window.nTriggerTimes = static_cast(_nStates); - _triggerTimesOnDisk = std::make_shared>>(); - for(int i = 0; i < _window.nTriggerTimes ; i++){ - _triggerTimesOnDisk->push_back(std::make_pair(_startTimes[i], _sourceFiles[i])); _window.triggerTimes.push_back(std::make_pair(_startTimes[i], _sourceFiles[i])); } @@ -62,7 +59,7 @@ namespace openspace{ _nAvailableWeb = 0; // haven't downloaded that list yet - _worker = WebFieldlinesWorker(syncDir, serverUrl, _triggerTimesOnDisk); + _worker = WebFieldlinesWorker(syncDir, serverUrl); } @@ -116,16 +113,22 @@ namespace openspace{ // Find where in the list we are int index = _nAvailableWeb-1; while(true){ - if(time > std::get<0>(_triggerTimesWeb[index])) break; + if(time > (_triggerTimesWeb[index].first)) break; index--; } + + // Should do the same as above? + //std::find_if(_triggerTimesWeb.rbegin(), _triggerTimesWeb.rend(), [time](auto it) { + // if (time > it.first) + // return true; + //}) // Make a window around that area _window.triggerTimes.clear(); _window.nTriggerTimes = 0; for(int i = index - _window.backWidth; i <= index + _window.forwardWidth; i++){ if(i < 0) i = 0; - _window.triggerTimes.push_back(std::make_pair(std::get<0>(_triggerTimesWeb[i]), std::get<1>(_triggerTimesWeb[i]))); + _window.triggerTimes.push_back(std::make_pair(_triggerTimesWeb[i].first, _triggerTimesWeb[i].second)); _window.nTriggerTimes++; } LERROR("new window"); @@ -135,7 +138,7 @@ namespace openspace{ bool WebFieldlinesWindow::timeIsInTriggerTimesWebList(double time){ if(_nAvailableWeb == 0) return false; - if(time >= std::get<0>(_triggerTimesWeb.front()) && time <= std::get<0>(_triggerTimesWeb.back())) + if(time >= (_triggerTimesWeb.front().first) && time <= (_triggerTimesWeb.back().first)) return true; else return false; @@ -154,12 +157,12 @@ namespace openspace{ } bool WebFieldlinesWindow::expectedWindowIsOutOfBounds(double time) { - auto resultForwards = std::find_if(_triggerTimesWeb.rbegin(), _triggerTimesWeb.rbegin() + _window.forwardWidth, [time](auto tuple) { - return time > std::get<0>(tuple); + auto resultForwards = std::find_if(_triggerTimesWeb.rbegin(), _triggerTimesWeb.rbegin() + _window.forwardWidth, [time](auto pair) { + return time > pair.first; }); - auto resultBackwards = std::find_if(_triggerTimesWeb.begin(), _triggerTimesWeb.begin() + _window.backWidth, [time](auto tuple) { - return time < std::get<0>(tuple); + auto resultBackwards = std::find_if(_triggerTimesWeb.begin(), _triggerTimesWeb.begin() + _window.backWidth, [time](auto pair) { + return time < pair.first; }); return resultForwards != _triggerTimesWeb.rbegin() + _window.forwardWidth || resultBackwards != _triggerTimesWeb.begin() + _window.backWidth; diff --git a/modules/fieldlinessequence/util/webfieldlineswindow.h b/modules/fieldlinessequence/util/webfieldlineswindow.h index 0e584da53a..eb9af64aea 100644 --- a/modules/fieldlinessequence/util/webfieldlineswindow.h +++ b/modules/fieldlinessequence/util/webfieldlineswindow.h @@ -71,8 +71,11 @@ public: // If files existed on disk already WebFieldlinesWindow(std::string syncDir, std::string serverUrl, std::vector& _sourceFiles, - std::vector& _startTimes, size_t& _nStates, size_t apiID); + std::vector& _startTimes, size_t& _nStates, int apiID); + // Copy constructor + WebFieldlinesWindow& operator=(WebFieldlinesWindow&&) = default; + // PUBLIC MEMBER FUNCTIONS bool timeIsInTriggerTimesWebList(double time); @@ -113,17 +116,12 @@ private: // The worker that handles downloads WebFieldlinesWorker _worker; - - // Keeps track of the files available on disk - // Shared ptr because WebfieldWorker has the same copy - // (Still unsure if this is the best way) - std::shared_ptr>> _triggerTimesOnDisk; - + // This is a long list of everything available online, // they are tuples of 3; double is timetrigger, string is donwload key, int is index to where it is on disk // and indices to the respective files on disk if they exist/are downloaded // -1 means not downloaded - std::vector> _triggerTimesWeb; + std::vector> _triggerTimesWeb; int _nAvailableWeb; diff --git a/modules/fieldlinessequence/util/webfieldlinesworker.cpp b/modules/fieldlinessequence/util/webfieldlinesworker.cpp index 304a854311..ba46c1a905 100644 --- a/modules/fieldlinessequence/util/webfieldlinesworker.cpp +++ b/modules/fieldlinessequence/util/webfieldlinesworker.cpp @@ -25,7 +25,6 @@ #include #include -#include #include #include #include @@ -41,73 +40,83 @@ namespace { namespace openspace{ // CONSTRUCTOR - WebFieldlinesWorker::WebFieldlinesWorker(std::string syncDir, std::string serverUrl,std::shared_ptr>> triggerTimesOnDisk) + WebFieldlinesWorker::WebFieldlinesWorker(std::string syncDir, std::string serverUrl) : _serverUrl(serverUrl) , _syncDir(syncDir) - , _triggerTimesOnDisk(triggerTimesOnDisk) - , _fileEnding(".osfls") - , _FLType("WSA_Fieldlines_PFSS_IO"){ - _endpointSingleDownload = _serverUrl + "WSA/fieldline/" + _FLType + "/"; // should be set by argument to be more general + , _fileEnding(".osfls"){ + _endpointSingleDownload = _serverUrl + "WSA/fieldline/"; // should be set by argument to be more general [DEPRICATED FOR NOW, DO WE NEED THIS?] } // Destructor, deleting all files that were downloaded during the run. WebFieldlinesWorker::~WebFieldlinesWorker() { + // Cancel any potential download + _downloading->cancel(); + // Remova all files downloaded std::for_each(_downloadedTriggerTimes.begin(), _downloadedTriggerTimes.end(), [&](auto it) { - FileSys.deleteFile(_syncDir + FileSys.PathSeparator + it.second + ".osfls"); + FileSys.deleteFile(_syncDir + FileSys.PathSeparator + it.second); }); } // PUBLIC FUNCTIONS - void WebFieldlinesWorker::getRangeOfAvailableTriggerTimes(double start, double end, std::vector> &_triggerTimesWeb, size_t id){ - const std::string oSpaceStartTime = "2000-01-01"; - // temporary, should use parameters for real endpoint later on - auto time = global::timeManager.time().ISO8601(); - Time maxTime; - Time minTime; + void WebFieldlinesWorker::getRangeOfAvailableTriggerTimes(double start, double end, std::vector> &_triggerTimesWeb, int id){ - maxTime.setTime(time); - maxTime.advanceTime(48000); - - std::string url = "https://iswa.gsfc.nasa.gov/IswaSystemWebApp/FilesInRangeServlet?dataID=" + std::to_string(id) + "&time.min=" + time + "&time.max=" + maxTime.ISO8601(); - // This is done quite early in the startup, requesting based on Open Space start time, will result - // In fetching filed lines that doesnt exist, and is too far away back in time. - //if (time.substr(0, 10) != oSpaceStartTime) - // url = "http://localhost:3000/WSA/times/" + time + "50"; - SyncHttpMemoryDownload mmryDld = SyncHttpMemoryDownload(url); - HttpRequest::RequestOptions opt = {}; - opt.requestTimeoutSeconds = 0; - mmryDld.download(opt); - - // Put the results in a string - std::string s; - - std::transform(mmryDld.downloadedData().begin(), mmryDld.downloadedData().end(), std::back_inserter(s), - [](char c) { - return c; - }); - - - auto res = json::parse(s); - - const std::string dataID = "dataID"; - const std::string files = "files"; - const std::string timeMax = "time.max"; - const std::string timeMin = "time.min"; - std::vector urlList; - std::vector timeList; - - - for (auto& elem : res[files]) { - timeList.push_back(elem["timestamp"]); - urlList.push_back(elem["url"]); + if (global::timeManager.time().j2000Seconds() < acceptableToStartRequestingAgain.first || global::timeManager.time().j2000Seconds() > acceptableToStartRequestingAgain.second) { + _noEmptyResponses = true; } - - // NEXT: Combine lists - _triggerTimesWeb.clear(); // we don't want to save too much in memory at the same time + // We don't want to keep sending request, if we just get empty responses. + if (_noEmptyResponses) { + auto time = global::timeManager.time().ISO8601(); + Time maxTime; + Time minTime; + const int timeSpann = 86400; // The timespann we would like to request (in seconds) [ 1 day = 86400, 1 week = 604800 ] + const std::string dataID = "dataID"; + const std::string files = "files"; + std::string stringResult; + std::vector urlList; + std::vector timeList; - std::sort(_triggerTimesWeb.begin(), _triggerTimesWeb.end()); + maxTime.setTime(time); + minTime.setTime(time); + maxTime.advanceTime(timeSpann); + minTime.advanceTime(-timeSpann); + std::string url = "https://iswa.gsfc.nasa.gov/IswaSystemWebApp/FilesInRangeServlet?dataID=" + std::to_string(id) + "&time.min=" + minTime.ISO8601() + "&time.max=" + maxTime.ISO8601(); + SyncHttpMemoryDownload mmryDld = SyncHttpMemoryDownload(url); + HttpRequest::RequestOptions opt = {}; + opt.requestTimeoutSeconds = 0; + mmryDld.download(opt); + + std::transform(mmryDld.downloadedData().begin(), mmryDld.downloadedData().end(), std::back_inserter(stringResult), + [](char c) { + return c; + }); + + auto res = json::parse(stringResult); + + for (auto& elem : res[files]) { + timeList.push_back(elem["timestamp"]); + urlList.push_back(elem["url"]); + } + + _triggerTimesWeb.clear(); // Clear old big window + + // Just want to make sure there is no error in the parsing, so taking the smallest dimension, but should be the same + for (int i = 0; i < std::min(timeList.size(), urlList.size()); i++) { + _triggerTimesWeb.push_back(std::make_pair(triggerTimeString2Double(timeList[i]), urlList[i])); + } + + std::sort(_triggerTimesWeb.begin(), _triggerTimesWeb.end()); // If by any chance it would not sort in properly + + if (_triggerTimesWeb.size() == 0) // We got an empty response + _strikes++; + + if (_strikes % 3 == 0){ // We have got 3 strikes, no more requests for you, Mr.Sir. + _noEmptyResponses = false; + acceptableToStartRequestingAgain = std::make_pair(minTime.j2000Seconds(), maxTime.j2000Seconds()); + } + + } } // Download all files in the current window @@ -115,49 +124,72 @@ namespace openspace{ // then steps backwards from the middle //TODO(Axel): Different behaviour depending on direction the user is moving in. void WebFieldlinesWorker::downloadWindow(std::vector> triggerTimes) { + + // Limit the number of concurrent downloads // Want to use std::find_if to break the for_each int middle = triggerTimes.size() / 2; bool downloaded = false; + bool oneUpdate = false; - // Forwards - std::vector>::iterator forwardIt = triggerTimes.begin(); - std::advance(forwardIt, middle); - std::for_each(forwardIt, triggerTimes.end(), [this, &downloaded](auto it) { - if (!downloaded && !fileIsOnDisk(it.first)) { - downloadOsfls(it.second); - addToDownloadedList(it); - downloaded = true; - _doneUpdating = false; - } - }); - - // Backwards - if (!downloaded) { - std::for_each(triggerTimes.rbegin(), triggerTimes.rend(), [this, &downloaded](auto it) { - if (!downloaded && !fileIsOnDisk(it.first)) { - downloadOsfls(it.second); - addToDownloadedList(it); - downloaded = true; - _doneUpdating = false; - } - }); + /*if (auto index = std::find_if(_downloadList.begin(), _downloadList.end(), [](auto element) { + return element.first->hasSucceeded(); + }); index != _downloadList.end()) { + // There is a successful download + index->first->wait(); + addToDownloadedList(index->second); + _downloadList.erase(index); + } + else { */ + if (_downloading && _downloading->hasSucceeded() && _newWindow) { + _downloading->wait(); + addToDownloadedList(_latestDownload); + _readyToDownload = true; + // This is to trigger one update of the fieldline timestamp that the user is currently on, + // while the rest of them will be downloaded in the background, and updated once ready + if (_latestDownload.second == triggerTimes[middle].second) + oneUpdate = true; } - if (!downloaded && !_doneUpdating && _newWindow) { + if (_readyToDownload) { + // Forwards + std::vector>::iterator forwardIt = triggerTimes.begin(); + std::advance(forwardIt, middle); + std::for_each(forwardIt, triggerTimes.end(), [this, &downloaded](auto it) { + if (!downloaded && !fileIsOnDisk(it.first)) { + downloadOsfls(it); + downloaded = true; + _doneUpdating = false; + } + }); + + // Backwards + if (!downloaded) { + std::for_each(triggerTimes.rbegin(), triggerTimes.rend(), [this, &downloaded](auto it) { + if (!downloaded && !fileIsOnDisk(it.first)) { + downloadOsfls(it); + downloaded = true; + _doneUpdating = false; + } + }); + } + + } + + if (!downloaded && !_doneUpdating && _newWindow && _readyToDownload || oneUpdate) { // If reach this point, we now know that we have downloaded all the sets _readyToUpdateSourceFiles = true; _newWindow = false; + oneUpdate = false; } - - + } // Updates the list of available sourcefiles, owned by renderablefieldlinessequence. void WebFieldlinesWorker::updateRFSSourceFiles(std::vector& _sourceFiles) { if (_readyToUpdateSourceFiles) { std::vector toInsert; - std::transform(_downloadedTriggerTimes.begin(), _downloadedTriggerTimes.end(), std::back_inserter(toInsert), [this](auto const& pair) { return _syncDir + FileSys.PathSeparator + pair.second + ".osfls"; }); + std::transform(_downloadedTriggerTimes.begin(), _downloadedTriggerTimes.end(), std::back_inserter(toInsert), [this](auto const& pair) { return _syncDir + FileSys.PathSeparator + pair.second; }); auto sourcePtr = _sourceFiles.begin(); std::for_each(toInsert.begin(), toInsert.end(), [&sourcePtr, &_sourceFiles](auto insertableElement) { @@ -203,21 +235,19 @@ namespace openspace{ _newWindow = true; } - std::string WebFieldlinesWorker::downloadOsfls(std::string downloadkey){ - //std::string url = _endpointSingleDownload + downloadkey + _fileEnding; // adjust this to match final endpoint - std::string url = "http://localhost:3000/WSA/PfssIo2019-09-16T13-21-20.000.osfls/" + downloadkey; // temp thing, should be the latest one - std::string destinationpath = absPath(_syncDir + ghoul::filesystem::FileSystem::PathSeparator + downloadkey + _fileEnding); // what the downloaded filename is to be - AsyncHttpFileDownload ashd = AsyncHttpFileDownload(url, destinationpath, HttpFileDownload::Overwrite::Yes); + std::string WebFieldlinesWorker::downloadOsfls(std::pair downloadKey){ + + _latestDownload = downloadKey; + // YYYY-MM-DDTHH-MM-SS.sss.osfls - Might change + const int fileNameLength = 29; + const std::string fileName = downloadKey.second.substr(downloadKey.second.size()-fileNameLength); + std::string url = downloadKey.second; + std::string destinationpath = absPath(_syncDir + ghoul::filesystem::FileSystem::PathSeparator + fileName); // what the downloaded filename is to be + _downloading = std::make_unique(url, destinationpath, HttpFileDownload::Overwrite::Yes); HttpRequest::RequestOptions opt = {}; opt.requestTimeoutSeconds = 0; - ashd.start(opt); - ashd.wait(); - if(ashd.hasSucceeded() == true ){ - //LERROR("succeeeded: " + destinationpath); - } - if(ashd.hasFailed() == true ){ - //LERROR("failed: " + destinationpath); - } + _downloading->start(opt); + _readyToDownload = false; return destinationpath; } @@ -283,7 +313,8 @@ namespace openspace{ // Inserts the pair in sorted order void WebFieldlinesWorker::addToDownloadedList(std::pair pair){ - _downloadedTriggerTimes.insert(std::upper_bound(_downloadedTriggerTimes.begin(),_downloadedTriggerTimes.end(),pair),pair); + const std::string fileName = pair.second.substr(pair.second.size() - 29); + _downloadedTriggerTimes.insert(std::upper_bound(_downloadedTriggerTimes.begin(),_downloadedTriggerTimes.end(),pair),std::make_pair(pair.first, fileName)); } bool WebFieldlinesWorker::compareTimetriggersEqual(double first, double second){ diff --git a/modules/fieldlinessequence/util/webfieldlinesworker.h b/modules/fieldlinessequence/util/webfieldlinesworker.h index 0a5c77b551..0e5bfce1dd 100644 --- a/modules/fieldlinessequence/util/webfieldlinesworker.h +++ b/modules/fieldlinessequence/util/webfieldlinesworker.h @@ -27,6 +27,7 @@ #include #include +#include namespace openspace { @@ -35,11 +36,13 @@ public: // Constructor WebFieldlinesWorker() = default; + WebFieldlinesWorker& operator=(WebFieldlinesWorker&&) = default; + ~WebFieldlinesWorker(); - WebFieldlinesWorker(std::string syncDir, std::string serverUrl, std::shared_ptr>> _triggerTimesOnDisk); + WebFieldlinesWorker(std::string syncDir, std::string serverUrl); - void getRangeOfAvailableTriggerTimes(double start, double end, std::vector>& _triggerTimesWeb, size_t id); + void getRangeOfAvailableTriggerTimes(double start, double end, std::vector>& _triggerTimesWeb, int id); // Sets the third index in _triggerTimesWeb to whether it is on disk already or not // If it's on disk, get the index of the corresponding triggertime in _triggerTimesOnDisk @@ -62,9 +65,15 @@ public: void newWindowToDownload(); private: - // POINTERS - std::shared_ptr>> _triggerTimesOnDisk; - // Pointers end + + // This list is the keep all the started downloads alive between frames, the second argument is a pair, used for identifying which download it is + std::vector _downloadList; + std::vector> _downloadListIdentifier; + + std::unique_ptr _downloading; + + // Might need this l8r + std::pair _latestDownload; // Contains a list of all the trigger times that has been downloaded already, std::vector> _downloadedTriggerTimes; @@ -80,27 +89,23 @@ private: // File format, only .oslfs for now std::string _fileEnding; - - std::string _FLType; - - /******************************************** - | USED FOR THREADS | - | | - ********************************************/ - //std::condition_variable _workerConditional; - //std::mutex _workerMutex; // TODO(Axel): Hmm, can we get around using these bools somehow? bool _readyToUpdateSourceFiles = false; bool _doneUpdating = false; bool _newWindow = false; + + + bool _readyToDownload = true; + bool _noEmptyResponses = true; + unsigned int _strikes = 0; + std::pair acceptableToStartRequestingAgain = std::make_pair(0.0, 0.0); // Download one file to sync directory - std::string downloadOsfls(std::string downloadkey); + std::string downloadOsfls(std::pair downloadKey); bool fileIsOnDisk(double triggerTime); - /******************************************** | Helper Functions | | |