From f3c4fc54dbce0a848b3fa445058eb10b51e78fee Mon Sep 17 00:00:00 2001 From: Andreas Engberg <48772850+engbergandreas@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:08:38 +0100 Subject: [PATCH] Update UrlSynchronization to limit amount of downloads (#2586) (#2933) Limits the number of URL synchronization downloads by time stamping files for X seconds fixes (#2586). Also, show the amount of downloaded data as soon as the download is started fix (#2460) * Adds time validity to UrlSynchronized files (#2586) Updated affected asset files. Satellite asset files lasts for 24h by default now. Other assets are updated to work as before. Obs timestamp is computed using UTC time. * Code cleanup * Update httpsynchronization.cpp * Applied suggestions from code review * Update multi-line comment to `//` * Remove accidental commas in .asset files Co-authored-by: Emma Broman * Fixed PR comments + bug in .asset file * Refactor code * Reports downloaded data as early as possible, fixes (#2460) * More files --------- Co-authored-by: Alexander Bock Co-authored-by: Emma Broman --- .../educational/scale/burj_khalifa.asset | 3 +- data/assets/examples/urlsynchronization.asset | 30 +- data/assets/global/openspacebookmarks.asset | 2 +- .../satellites/communications/amateur.asset | 2 +- .../communications/experimental.asset | 2 +- .../communications/geostationary.asset | 2 +- .../communications/globalstar.asset | 2 +- .../satellites/communications/gorizont.asset | 2 +- .../satellites/communications/intelsat.asset | 2 +- .../satellites/communications/iridium.asset | 2 +- .../communications/iridium_next.asset | 2 +- .../satellites/communications/molniya.asset | 2 +- .../satellites/communications/orbcomm.asset | 2 +- .../communications/other_comm.asset | 2 +- .../satellites/communications/raduga.asset | 2 +- .../earth/satellites/communications/ses.asset | 2 +- .../satellites/communications/starlink.asset | 2 +- .../earth/satellites/debris/debris_asat.asset | 2 +- .../satellites/debris/debris_breezem.asset | 2 +- .../satellites/debris/debris_fengyun.asset | 2 +- .../satellites/debris/debris_iridium33.asset | 2 +- .../satellites/debris/debris_kosmos2251.asset | 2 +- .../earth/satellites/misc/active.asset | 2 +- .../earth/satellites/misc/brightest.asset | 2 +- .../earth/satellites/misc/cubesats.asset | 2 +- .../earth/satellites/misc/hubble_trail.asset | 2 +- .../planets/earth/satellites/misc/iss.asset | 2 +- .../earth/satellites/misc/military.asset | 2 +- .../planets/earth/satellites/misc/other.asset | 2 +- .../planets/earth/satellites/misc/radar.asset | 2 +- .../earth/satellites/misc/spacestations.asset | 2 +- .../earth/satellites/misc/tle-new.asset | 2 +- .../earth/satellites/navigation/beidou.asset | 2 +- .../earth/satellites/navigation/galileo.asset | 2 +- .../satellites/navigation/glosnass.asset | 2 +- .../earth/satellites/navigation/gps.asset | 2 +- .../earth/satellites/navigation/musson.asset | 2 +- .../earth/satellites/navigation/nnss.asset | 2 +- .../earth/satellites/navigation/sbas.asset | 2 +- .../earth/satellites/science/education.asset | 2 +- .../satellites/science/engineering.asset | 2 +- .../earth/satellites/science/geodetic.asset | 2 +- .../earth/satellites/science/spaceearth.asset | 2 +- .../earth/satellites/weather/aqua.asset | 2 +- .../earth/satellites/weather/argos.asset | 2 +- .../earth/satellites/weather/dmc.asset | 2 +- .../satellites/weather/earth_resources.asset | 2 +- .../earth/satellites/weather/goes.asset | 2 +- .../earth/satellites/weather/noaa.asset | 2 +- .../earth/satellites/weather/planet.asset | 2 +- .../earth/satellites/weather/sarsat.asset | 2 +- .../earth/satellites/weather/snpp.asset | 2 +- .../earth/satellites/weather/spire.asset | 2 +- .../earth/satellites/weather/tdrss.asset | 2 +- .../earth/satellites/weather/terra.asset | 2 +- .../earth/satellites/weather/weather.asset | 2 +- data/assets/spice/base.asset | 10 - include/openspace/util/spicemanager.h | 3 + modules/sync/syncs/httpsynchronization.cpp | 10 +- modules/sync/syncs/urlsynchronization.cpp | 344 ++++++++++++------ modules/sync/syncs/urlsynchronization.h | 17 + src/engine/openspaceengine.cpp | 2 +- src/rendering/loadingscreen.cpp | 15 +- src/util/spicemanager.cpp | 167 +++++++++ 64 files changed, 526 insertions(+), 183 deletions(-) diff --git a/data/assets/educational/scale/burj_khalifa.asset b/data/assets/educational/scale/burj_khalifa.asset index ff46cd6976..5215a971d7 100644 --- a/data/assets/educational/scale/burj_khalifa.asset +++ b/data/assets/educational/scale/burj_khalifa.asset @@ -8,8 +8,7 @@ local modelFolder = asset.resource({ Type = "UrlSynchronization", Identifier = "scale_model_burj_khalifa", Url = "https://wms.openspace.amnh.org/static/sync/url/scalemodels/Burj_Khalifa.osmodel", - Filename = "Burj_Khalifa.osmodel", - Override = false + Filename = "Burj_Khalifa.osmodel" }) diff --git a/data/assets/examples/urlsynchronization.asset b/data/assets/examples/urlsynchronization.asset index e45895f791..98bf9554dd 100644 --- a/data/assets/examples/urlsynchronization.asset +++ b/data/assets/examples/urlsynchronization.asset @@ -19,8 +19,32 @@ asset.resource({ Name = "Example Large", Type = "UrlSynchronization", Identifier = "example_large", - Url = "http://ipv4.download.thinkbroadband.com/100MB.zip", - Override = true + Url = { + "http://speedtest.tele2.net/1000GB.zip", + "https://speedtest.tele2.net/100GB.zip", + "https://speedtest.tele2.net/50GB.zip", + "http://speedtest.tele2.net/10GB.zip", + "http://speedtest.tele2.net/1GB.zip", + "http://speedtest.tele2.net/100MB.zip", + + "https://proof.ovh.net/files/10Gb.dat", + "https://proof.ovh.net/files/1Gb.dat", + "https://proof.ovh.net/files/100Mb.dat", + + "https://speed.hetzner.de/10GB.bin", + "https://speed.hetzner.de/1GB.bin", + "https://speed.hetzner.de/100MB.bin", + + "http://ipv4.download.thinkbroadband.com/1GB.zip", + "http://ipv4.download.thinkbroadband.com/512MB.zip", + "http://ipv4.download.thinkbroadband.com/200MB.zip", + "http://ipv4.download.thinkbroadband.com/100MB.zip", + "http://ipv4.download.thinkbroadband.com/50MB.zip", + "http://ipv4.download.thinkbroadband.com/20MB.zip", + "http://ipv4.download.thinkbroadband.com/10MB.zip", + "http://ipv4.download.thinkbroadband.com/5MB.zip", + }, + SecondsUntilResync = 0 }) asset.resource({ @@ -28,7 +52,7 @@ asset.resource({ Type = "UrlSynchronization", Identifier = "example_medium", Url = "http://ipv4.download.thinkbroadband.com/5MB.zip", - Override = true + SecondsUntilResync = 0 }) -- Load a resource without hashing, meaning that the bare directory name will be diff --git a/data/assets/global/openspacebookmarks.asset b/data/assets/global/openspacebookmarks.asset index d8770c2e20..bf554226c1 100644 --- a/data/assets/global/openspacebookmarks.asset +++ b/data/assets/global/openspacebookmarks.asset @@ -7,7 +7,7 @@ local bookmarksCSV = asset.resource({ Name = "OpenSpace Bookmarks", Type = "UrlSynchronization", UseHash = false, - Override = true, + SecondsUntilResync = 0 Url = dataProvider }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/amateur.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/amateur.asset index 4e4f755a3f..ee6c80271d 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/amateur.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/amateur.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_amateur_radio", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=amateur&FORMAT=kvn", Filename = "amateur.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/experimental.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/experimental.asset index 654c64564a..7541fbbca9 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/experimental.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/experimental.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_x-comm", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=x-comm&FORMAT=kvn", Filename = "x-comm.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/geostationary.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/geostationary.asset index b6668e7ef2..0d4097fecf 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/geostationary.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/geostationary.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_geo", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=geo&FORMAT=kvn", Filename = "geo.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/globalstar.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/globalstar.asset index e3d54ba07c..2203bd564c 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/globalstar.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/globalstar.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_globalstar", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=globalstar&FORMAT=kvn", Filename = "globalstar.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/gorizont.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/gorizont.asset index 0ab12c2d39..13811ce9d8 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/gorizont.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/gorizont.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_gorizont", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=gorizont&FORMAT=kvn", Filename = "gorizont.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/intelsat.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/intelsat.asset index b54b81e4f4..7198b46408 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/intelsat.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/intelsat.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_intelsat", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=intelsat&FORMAT=kvn", Filename = "intelsat.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/iridium.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/iridium.asset index c5a623371a..132919987f 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/iridium.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/iridium.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_iridium", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=iridium&FORMAT=kvn", Filename = "iridium.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/iridium_next.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/iridium_next.asset index b56bb45a39..5019ebd3e6 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/iridium_next.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/iridium_next.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_iridium-NEXT", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=iridium-NEXT&FORMAT=kvn", Filename = "iridium-NEXT.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/molniya.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/molniya.asset index 5be8ee8886..31632e7ad5 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/molniya.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/molniya.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_molniya", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=molniya&FORMAT=kvn", Filename = "molniya.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/orbcomm.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/orbcomm.asset index 8f00be9cac..c1a759df97 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/orbcomm.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/orbcomm.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_orbcomm", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=orbcomm&FORMAT=kvn", Filename = "orbcomm.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/other_comm.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/other_comm.asset index 99e98cc528..5fda8ec905 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/other_comm.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/other_comm.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_other-comm", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=other-comm&FORMAT=kvn", Filename = "other-comm.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/raduga.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/raduga.asset index 589e70850f..cecf990b27 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/raduga.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/raduga.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_raduga", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=raduga&FORMAT=kvn", Filename = "raduga.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/ses.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/ses.asset index 68ff8c2343..532977a477 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/ses.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/ses.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_ses", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=ses&FORMAT=kvn", Filename = "ses.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/communications/starlink.asset b/data/assets/scene/solarsystem/planets/earth/satellites/communications/starlink.asset index d9ae0f3fa4..a0ad96956b 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/communications/starlink.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/communications/starlink.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_starlink", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=starlink&FORMAT=kvn", Filename = "starlink.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_asat.asset b/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_asat.asset index 197d0817b1..d220142513 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_asat.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_asat.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_2019-006", Url = "https://www.celestrak.com/NORAD/elements/gp.php?INTDES=2019-006&FORMAT=kvn", Filename = "2019-006.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_breezem.asset b/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_breezem.asset index 429d75054b..6820d20e93 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_breezem.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_breezem.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_2012-044", Url = "https://www.celestrak.com/NORAD/elements/gp.php?INTDES=2012-044&FORMAT=kvn", Filename = "2012-044.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_fengyun.asset b/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_fengyun.asset index 6de5152b47..78ae414dfa 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_fengyun.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_fengyun.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_1999-025", Url = "https://www.celestrak.com/NORAD/elements/gp.php?INTDES=1999-025&FORMAT=kvn", Filename = "1999-025.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_iridium33.asset b/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_iridium33.asset index 2745a39fbf..24176b22e9 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_iridium33.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_iridium33.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_iridium-33-debris", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=iridium-33-debris&FORMAT=kvn", Filename = "iridium-33-debris.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_kosmos2251.asset b/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_kosmos2251.asset index 8681bf7bc7..d91337ce68 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_kosmos2251.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/debris/debris_kosmos2251.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_cosmos-2251-debris", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=cosmos-2251-debris&FORMAT=kvn", Filename = "cosmos-2251-debris.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/misc/active.asset b/data/assets/scene/solarsystem/planets/earth/satellites/misc/active.asset index d146a2a599..a01b2a68fb 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/misc/active.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/misc/active.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_active", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=active&FORMAT=kvn", Filename = "active.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/misc/brightest.asset b/data/assets/scene/solarsystem/planets/earth/satellites/misc/brightest.asset index 95870e8de3..a9fc480c3a 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/misc/brightest.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/misc/brightest.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_visual", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=visual&FORMAT=kvn", Filename = "visual.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/misc/cubesats.asset b/data/assets/scene/solarsystem/planets/earth/satellites/misc/cubesats.asset index 7f7a66fc55..e95fbb3df7 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/misc/cubesats.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/misc/cubesats.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_cubesat", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=cubesat&FORMAT=kvn", Filename = "cubesat.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/misc/hubble_trail.asset b/data/assets/scene/solarsystem/planets/earth/satellites/misc/hubble_trail.asset index 4166915558..bfe34c64f3 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/misc/hubble_trail.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/misc/hubble_trail.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_hst", Url = "https://www.celestrak.com/NORAD/elements/gp.php?CATNR=20580&FORMAT=kvn", Filename = "hst.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/misc/iss.asset b/data/assets/scene/solarsystem/planets/earth/satellites/misc/iss.asset index d8a069c31a..b95f913208 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/misc/iss.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/misc/iss.asset @@ -16,7 +16,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_iss", Url = "https://www.celestrak.com/NORAD/elements/gp.php?CATNR=25544&FORMAT=kvn", Filename = "ISS.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/misc/military.asset b/data/assets/scene/solarsystem/planets/earth/satellites/misc/military.asset index ac741d8d11..cadb37ead3 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/misc/military.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/misc/military.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_military", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=military&FORMAT=kvn", Filename = "military.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/misc/other.asset b/data/assets/scene/solarsystem/planets/earth/satellites/misc/other.asset index 786a10b15f..ad63db30b2 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/misc/other.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/misc/other.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_other", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=other&FORMAT=kvn", Filename = "other.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/misc/radar.asset b/data/assets/scene/solarsystem/planets/earth/satellites/misc/radar.asset index 83143280f9..74b5cdcc6f 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/misc/radar.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/misc/radar.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_radar", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=radar&FORMAT=kvn", Filename = "radar.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/misc/spacestations.asset b/data/assets/scene/solarsystem/planets/earth/satellites/misc/spacestations.asset index 167d2b2cd9..c1c3286224 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/misc/spacestations.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/misc/spacestations.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_stations", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=stations&FORMAT=kvn", Filename = "stations.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/misc/tle-new.asset b/data/assets/scene/solarsystem/planets/earth/satellites/misc/tle-new.asset index 71336d51c1..d616a0de3e 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/misc/tle-new.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/misc/tle-new.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_tle-new", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=last-30-days&FORMAT=kvn", Filename = "tle-new.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/beidou.asset b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/beidou.asset index 94429c0294..f2b5412ba3 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/beidou.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/beidou.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_beidou", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=beidou&FORMAT=kvn", Filename = "beidou.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/galileo.asset b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/galileo.asset index 36cc8d0572..8d81d92302 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/galileo.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/galileo.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_galileo", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=galileo&FORMAT=kvn", Filename = "galileo.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/glosnass.asset b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/glosnass.asset index 3e29b0f1c9..fc6933875b 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/glosnass.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/glosnass.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_glo-ops", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=glo-ops&FORMAT=kvn", Filename = "glo-ops.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/gps.asset b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/gps.asset index 0e1cf3c5c5..2d147c8e04 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/gps.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/gps.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_gps-ops", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=gps-ops&FORMAT=kvn", Filename = "gps-ops.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/musson.asset b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/musson.asset index 89df6c73c3..35664b2186 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/musson.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/musson.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_musson", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=musson&FORMAT=kvn", Filename = "musson.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/nnss.asset b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/nnss.asset index a478d319c9..665d5a24cf 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/nnss.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/nnss.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_nnss", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=nnss&FORMAT=kvn", Filename = "nnss.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/sbas.asset b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/sbas.asset index 181722c5c1..10d5dffa5c 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/navigation/sbas.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/navigation/sbas.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_sbas", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=sbas&FORMAT=kvn", Filename = "sbas.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/science/education.asset b/data/assets/scene/solarsystem/planets/earth/satellites/science/education.asset index 84d4d1ad16..f7e003db03 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/science/education.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/science/education.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_education", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=education&FORMAT=kvn", Filename = "education.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/science/engineering.asset b/data/assets/scene/solarsystem/planets/earth/satellites/science/engineering.asset index 1277fa0e99..3ba32d83f4 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/science/engineering.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/science/engineering.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_engineering", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=engineering&FORMAT=kvn", Filename = "engineering.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/science/geodetic.asset b/data/assets/scene/solarsystem/planets/earth/satellites/science/geodetic.asset index 6b5bc0468b..d9bb14f901 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/science/geodetic.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/science/geodetic.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_geodetic", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=geodetic&FORMAT=kvn", Filename = "geodetic.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/science/spaceearth.asset b/data/assets/scene/solarsystem/planets/earth/satellites/science/spaceearth.asset index 1f222f6c5a..a314217808 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/science/spaceearth.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/science/spaceearth.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_science", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=science&FORMAT=kvn", Filename = "science.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/aqua.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/aqua.asset index abe707e3c8..38c187bc96 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/aqua.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/aqua.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_aqua", Url = "https://www.celestrak.com/NORAD/elements/gp.php?CATNR=27424&FORMAT=kvn", Filename = "Aqua.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/argos.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/argos.asset index f4632ee3e3..071d3d616d 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/argos.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/argos.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_argos", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=argos&FORMAT=kvn", Filename = "argos.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/dmc.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/dmc.asset index 7ecdebfb9e..3c1fbd90c8 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/dmc.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/dmc.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_dmc", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=dmc&FORMAT=kvn", Filename = "dmc.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/earth_resources.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/earth_resources.asset index 6451aace49..835560ac3a 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/earth_resources.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/earth_resources.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_resource", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=resource&FORMAT=kvn", Filename = "resource.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/goes.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/goes.asset index fc24bdd839..42a26fad92 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/goes.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/goes.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_goes", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=goes&FORMAT=kvn", Filename = "goes.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/noaa.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/noaa.asset index 65b1a6cf81..4d8324c9a5 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/noaa.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/noaa.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_noaa", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=noaa&FORMAT=kvn", Filename = "noaa.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/planet.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/planet.asset index b56b882db8..a8a9df324b 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/planet.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/planet.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_planet", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=planet&FORMAT=kvn", Filename = "planet.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/sarsat.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/sarsat.asset index 4b3b9dc29e..151c23c4dc 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/sarsat.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/sarsat.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_sarsat", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=sarsat&FORMAT=kvn", Filename = "sarsat.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/snpp.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/snpp.asset index b1a4f7cd7b..89e19db0fd 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/snpp.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/snpp.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_snpp", Url = "https://www.celestrak.com/NORAD/elements/gp.php?CATNR=37849&FORMAT=kvn", Filename = "SNPP.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/spire.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/spire.asset index ee3267a8e1..e991a56c90 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/spire.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/spire.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_spire", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=spire&FORMAT=kvn", Filename = "spire.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/tdrss.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/tdrss.asset index c8cfc79075..71da56bcda 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/tdrss.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/tdrss.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_tdrss", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=tdrss&FORMAT=kvn", Filename = "tdrss.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/terra.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/terra.asset index b4d4791ab1..d8a3693079 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/terra.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/terra.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_terra", Url = "https://www.celestrak.com/NORAD/elements/gp.php?CATNR=25994&FORMAT=kvn", Filename = "Terra.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/scene/solarsystem/planets/earth/satellites/weather/weather.asset b/data/assets/scene/solarsystem/planets/earth/satellites/weather/weather.asset index 3993910bfa..23ff2d1e37 100644 --- a/data/assets/scene/solarsystem/planets/earth/satellites/weather/weather.asset +++ b/data/assets/scene/solarsystem/planets/earth/satellites/weather/weather.asset @@ -8,7 +8,7 @@ local omm = asset.resource({ Identifier = "satellite_omm_data_weather", Url = "https://www.celestrak.com/NORAD/elements/gp.php?GROUP=weather&FORMAT=kvn", Filename = "weather.txt", - Override = true + SecondsUntilResync = 24 * 60 * 60 }) diff --git a/data/assets/spice/base.asset b/data/assets/spice/base.asset index 24408b5898..7f651af46f 100644 --- a/data/assets/spice/base.asset +++ b/data/assets/spice/base.asset @@ -1,10 +1,3 @@ -local lsk = asset.resource({ - Name = "General LSK Kernels", - Type = "HttpSynchronization", - Identifier = "general_lsk", - Version = 1 -}) - local pck = asset.resource({ Name = "General PCK Kernels", Type = "HttpSynchronization", @@ -21,14 +14,12 @@ local spk = asset.resource({ local kernels = { - lsk .. "naif0012.tls", pck .. "pck00011.tpc", spk .. "de430.bsp" } asset.onInitialize(function() - openspace.spice.loadKernel(lsk .. "naif0012.tls") openspace.spice.loadKernel(pck .. "pck00011.tpc") openspace.spice.loadKernel(spk .. "de430.bsp") end) @@ -36,7 +27,6 @@ end) asset.onDeinitialize(function() openspace.spice.unloadKernel(spk .. "de430.bsp") openspace.spice.unloadKernel(pck .. "pck00011.tpc") - openspace.spice.unloadKernel(lsk .. "naif0012.tls") end) diff --git a/include/openspace/util/spicemanager.h b/include/openspace/util/spicemanager.h index 17bacf348d..eec285c09f 100644 --- a/include/openspace/util/spicemanager.h +++ b/include/openspace/util/spicemanager.h @@ -1085,6 +1085,9 @@ private: glm::dmat3 getEstimatedTransformMatrix(const std::string& fromFrame, const std::string& toFrame, double time) const; + /// Loads pre defined leap seconds time kernel (naif00012.tls). + void loadLeapSecondsSpiceKernel(); + /// A list of all loaded kernels std::vector _loadedKernels; diff --git a/modules/sync/syncs/httpsynchronization.cpp b/modules/sync/syncs/httpsynchronization.cpp index 199e78c7f9..7b71f7dea3 100644 --- a/modules/sync/syncs/httpsynchronization.cpp +++ b/modules/sync/syncs/httpsynchronization.cpp @@ -206,12 +206,10 @@ bool HttpSynchronization::isEachFileDownloaded() { // Otherwise first line is the version number. std::string ossyncVersion = line; - /* - Format of 1.0 ossync: - Version number: E.g., 1.0 - Synchronization status: Synchronized or Partial Synchronized - Optionally list of already synched files - */ + //Format of 1.0 ossync: + //Version number: E.g., 1.0 + //Synchronization status: Synchronized or Partial Synchronized + //Optionally list of already synched files if (ossyncVersion == OssyncVersionNumber) { std::getline(file >> std::ws, line); // Read synchronization status diff --git a/modules/sync/syncs/urlsynchronization.cpp b/modules/sync/syncs/urlsynchronization.cpp index 855b814504..ca0fda73f5 100644 --- a/modules/sync/syncs/urlsynchronization.cpp +++ b/modules/sync/syncs/urlsynchronization.cpp @@ -27,6 +27,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -35,6 +38,10 @@ #include namespace { + constexpr std::string_view _loggerCat = "UrlSynchronization"; + constexpr std::string_view OssyncVersionNumber = "1.0"; + constexpr std::string_view SynchronizationToken = "Synchronized"; + struct [[codegen::Dictionary(UrlSynchronization)]] Parameters { // The URL or urls from where the files are downloaded. If multiple URLs are // provided, all files will be downloaded to the same directory and the filename @@ -45,10 +52,7 @@ namespace { // manually find the downloaded folder in the synchronization folder std::string identifier [[codegen::identifier()]]; - // If this value is set to 'true' and it is not overwritten by the global - // settings, the file(s) pointed to by this URLSynchronization will always be - // downloaded, thus overwriting the local files. This is useful for files that are - // updated regularly remotely and should be fetch at every startup + // Deprecated, use SecondsUntilResync instead std::optional forceOverride [[codegen::key("Override")]]; // If this value is set to 'true' (the default), the hash of the URL is appended @@ -62,6 +66,11 @@ namespace { // automatically created from the url. If this value is specified, the url // parameter only only contain exactly one URL std::optional filename; + + // This variable determines the validity period of a file(s) in seconds before it + // needs to be re-downloaded. The default value keeps the file permanently cached, + // while a value of 0 forces the file to be downloaded on every startup. + std::optional secondsUntilResync [[codegen::greaterequal(0.0)]]; }; #include "urlsynchronization_codegen.cpp" } // namespace @@ -99,6 +108,14 @@ UrlSynchronization::UrlSynchronization(const ghoul::Dictionary& dictionary, _filename = p.filename.value_or(_filename); _forceOverride = p.forceOverride.value_or(_forceOverride); + // (anden88 2023-11-03) TODO: When we decide to remove override variable this should + // be cleaned up. + // Mimic behavior of time to live if override is specified (true => force download, + // false keeps the file permanently (default behavior)). + _secondsUntilResync = _forceOverride ? 0 : MaxDateAsJ2000; + // Disregard override variable if user specified a specific time to live. + _secondsUntilResync = p.secondsUntilResync.value_or(_secondsUntilResync); + const bool useHash = p.useHash.value_or(true); _identifier = p.identifier; @@ -113,6 +130,14 @@ UrlSynchronization::UrlSynchronization(const ghoul::Dictionary& dictionary, ); _identifier += fmt::format("({})", hash); } + + if (p.forceOverride.has_value()) { + LWARNING(fmt::format( + "{}: The variable ForceOverride has been deprecated." + "Optionally, use SecondsUntilResync instead to specify file validity date.", + p.identifier + )); + } } UrlSynchronization::~UrlSynchronization() { @@ -132,115 +157,25 @@ void UrlSynchronization::start() { } _state = State::Syncing; - if (hasSyncFile() && !_forceOverride) { + if (isEachFileValid()) { _state = State::Resolved; return; } + if (isRejected()) { + return; + } + _syncThread = std::thread([this]() { - std::unordered_map fileSizes; - std::mutex fileSizeMutex; - size_t nDownloads = 0; - std::atomic_bool startedAllDownloads = false; - std::vector> downloads; + const bool success = trySyncUrls(); - for (const std::string& url : _urls) { - if (_filename.empty() || _urls.size() > 1) { - std::filesystem::path fn = std::filesystem::path(url).filename(); - if (fn.empty() && url.back() == '/') { - // If the user provided a path that ends in / the `filename` will - // result in an empty path with causes the downloading to fail - fn = std::filesystem::path(url).parent_path().filename(); - } - std::string name = fn.string(); - - // We can not create filenames with question marks - name.erase(std::remove(name.begin(), name.end(), '?'), name.end()); - _filename = name; - } - std::filesystem::path destination = directory() / (_filename + ".tmp"); - - auto download = std::make_unique( - url, - destination, - HttpFileDownload::Overwrite::Yes - ); - HttpFileDownload* dl = download.get(); - - downloads.push_back(std::move(download)); - - ++nDownloads; - - dl->onProgress( - [this, url, &fileSizes, &fileSizeMutex, - &startedAllDownloads, &nDownloads](int64_t, - std::optional totalBytes) - { - if (!totalBytes.has_value()) { - return !_shouldCancel; - } - - std::lock_guard guard(fileSizeMutex); - fileSizes[url] = *totalBytes; - - if (!_nTotalBytesKnown && startedAllDownloads && - fileSizes.size() == nDownloads) - { - _nTotalBytesKnown = true; - _nTotalBytes = 0; - for (const std::pair& fs : fileSizes) { - _nTotalBytes += fs.second; - } - } - return !_shouldCancel; - }); - - dl->start(); - } - - startedAllDownloads = true; - - bool failed = false; - for (const std::unique_ptr& d : downloads) { - d->wait(); - if (!d->hasSucceeded()) { - failed = true; - continue; - } - - // If we are forcing the override, we download to a temporary file first, so - // when we are done here, we need to rename the file to the original name - - std::filesystem::path tempName = d->destination(); - std::filesystem::path originalName = tempName; - // Remove the .tmp extension - originalName.replace_extension(""); - - if (std::filesystem::is_regular_file(originalName)) { - std::filesystem::remove(originalName); - } - - std::error_code ec; - std::filesystem::rename(tempName, originalName, ec); - if (ec) { - LERRORC( - "URLSynchronization", - fmt::format("Error renaming file {} to {}", tempName, originalName) - ); - - failed = true; - } - } - - if (!failed) { + if (success) { createSyncFile(); + _state = State::Resolved; } else { - for (const std::unique_ptr& d : downloads) { - d->cancel(); - } + _state = State::Rejected; } - _state = State::Resolved; }); } @@ -253,4 +188,207 @@ std::string UrlSynchronization::generateUid() { return _identifier; } +bool UrlSynchronization::isEachFileValid() { + std::filesystem::path path = directory(); + path.replace_extension("ossync"); + // Check if file exists at all + if (!std::filesystem::is_regular_file(path)) { + return false; + } + + std::ifstream file(path); + std::string line; + + file >> line; + // Update ossync files that does not have a version number to new format + if (line == SynchronizationToken) { + if (_secondsUntilResync == 0) { + return false; // Force download new file + } + + // We must close file early because createSyncFile writes to it + file.close(); + createSyncFile(); + // File is valid until some date + return true; + } + // Otherwise first line is the version number. + std::string ossyncVersion = line; + + // Format of 1.0 ossync: + // Version number: e.g., 1.0 + // Date that specifies how long the files are valid for in ISO8601 format + // Valid to: yyyy-mm-ddThr:mn:sc.xxx + + if (ossyncVersion == "1.0") { + std::getline(file >> std::ws, line); + std::string& fileIsValidToDate = line; + double fileValidAsJ2000 = Time::convertTime(fileIsValidToDate); + + std::string todaysDate = Time::currentWallTime(); + double todaysDateAsJ2000 = Time::convertTime(todaysDate); + + // Issue warning if file is kept but user changed setting to download on startup. + if ((fileValidAsJ2000 > todaysDateAsJ2000) && _secondsUntilResync == 0) { + LWARNING(fmt::format( + "{}: File is valid to {} but asset specifies SecondsUntilResync = {} " + "Did you mean to re-download the file? If so, remove file from sync " + "folder to resync", + _identifier, + fileIsValidToDate, + _secondsUntilResync + )); + } + + // Returns true if date in ossync file is 'larger' than todays date, + // i.e. no sync needed. + return fileValidAsJ2000 > todaysDateAsJ2000; + } + else { + LERROR(fmt::format( + "{}: Unknown ossync version number read. Got {} while {} and below are valid", + _identifier, ossyncVersion, OssyncVersionNumber + )); + _state = State::Rejected; + } + + return false; +} + +void UrlSynchronization::createSyncFile(bool) const { + std::filesystem::path dir = directory(); + std::filesystem::create_directories(dir); + + dir.replace_extension("ossync"); + std::ofstream syncFile(dir, std::ofstream::out); + + std::string currentTimeAsISO8601 = Time::currentWallTime(); + double currentTimeAsJ2000 = Time::convertTime(currentTimeAsISO8601); + + // With the format YYYY-MM... any year thats larger than 4 digits throws an error + // Limit the future date to year 9999 + double futureTimeAsJ2000 = std::min( + currentTimeAsJ2000 + _secondsUntilResync, + MaxDateAsJ2000 + ); + + std::string fileIsValidTo = SpiceManager::ref().dateFromEphemerisTime( + futureTimeAsJ2000, + "YYYY-MM-DDTHR:MN:SC.###" + ); + + const std::string msg = fmt::format("{}\n{}\n", OssyncVersionNumber, fileIsValidTo); + syncFile << msg; +} + +bool UrlSynchronization::trySyncUrls() { + struct SizeData { + int64_t downloadedBytes = 0; + std::optional totalBytes; + }; + + std::unordered_map sizeData; + std::mutex fileSizeMutex; + size_t nDownloads = 0; + std::atomic_bool startedAllDownloads = false; + std::vector> downloads; + + for (const std::string& url : _urls) { + if (_filename.empty() || _urls.size() > 1) { + std::filesystem::path fn = std::filesystem::path(url).filename(); + if (fn.empty() && url.back() == '/') { + // If the user provided a path that ends in / the `filename` will + // result in an empty path with causes the downloading to fail + fn = std::filesystem::path(url).parent_path().filename(); + } + std::string name = fn.string(); + + // We can not create filenames with question marks + name.erase(std::remove(name.begin(), name.end(), '?'), name.end()); + _filename = name; + } + std::filesystem::path destination = directory() / (_filename + ".tmp"); + + if (sizeData.find(url) != sizeData.end()) { + LWARNING(fmt::format("{}: Duplicate entry for {}", _identifier, url)); + continue; + } + + auto download = std::make_unique( + url, + destination, + HttpFileDownload::Overwrite::Yes + ); + HttpFileDownload* dl = download.get(); + + downloads.push_back(std::move(download)); + + ++nDownloads; + sizeData[url] = SizeData(); + + dl->onProgress( + [this, url, &sizeData, &fileSizeMutex](int64_t downloadedBytes, + std::optional totalBytes) + { + if (!totalBytes.has_value()) { + return !_shouldCancel; + } + + std::lock_guard guard(fileSizeMutex); + sizeData[url] = { downloadedBytes, totalBytes }; + + _nTotalBytesKnown = true; + _nTotalBytes = 0; + _nSynchronizedBytes = 0; + for (const std::pair& sd : sizeData) { + _nTotalBytesKnown = _nTotalBytesKnown && + sd.second.totalBytes.has_value(); + _nTotalBytes += sd.second.totalBytes.value_or(0); + _nSynchronizedBytes += sd.second.downloadedBytes; + } + + return !_shouldCancel; + }); + + dl->start(); + } + + startedAllDownloads = true; + + bool failed = false; + for (const std::unique_ptr& d : downloads) { + d->wait(); + if (!d->hasSucceeded()) { + failed = true; + LERROR(fmt::format("Error downloading file from URL {}", d->url())); + continue; + } + + // If we are forcing the override, we download to a temporary file first, so + // when we are done here, we need to rename the file to the original name + + std::filesystem::path tempName = d->destination(); + std::filesystem::path originalName = tempName; + // Remove the .tmp extension + originalName.replace_extension(""); + + if (std::filesystem::is_regular_file(originalName)) { + std::filesystem::remove(originalName); + } + + std::error_code ec; + std::filesystem::rename(tempName, originalName, ec); + if (ec) { + LERRORC( + "URLSynchronization", + fmt::format("Error renaming file {} to {}", tempName, originalName) + ); + + failed = true; + } + } + + return !failed; +} + } // namespace openspace diff --git a/modules/sync/syncs/urlsynchronization.h b/modules/sync/syncs/urlsynchronization.h index 0b0f85463c..0545d2c6a1 100644 --- a/modules/sync/syncs/urlsynchronization.h +++ b/modules/sync/syncs/urlsynchronization.h @@ -77,7 +77,21 @@ public: static documentation::Documentation Documentation(); +protected: + /// Read the `ossync` file and check if the downloaded files can be used. Returns + /// `true` if they are valid and `false` if we should download them again + bool isEachFileValid(); + private: + static constexpr double MaxDateAsJ2000 = 252424036869.18289; + + /// Creates a file next to the directory that indicates that this + /// ResourceSynchronization has successfully synchronized its contents + void createSyncFile(bool isFullySynchronized = true) const override; + + /// Tries to get a reply from the asset URLs and returns that success to the caller + bool trySyncUrls(); + /// The list of URLs that will be downloaded std::vector _urls; @@ -93,6 +107,9 @@ private: // The thread that will be doing the synchronization std::thread _syncThread; + + /// Determines how long the file is valid before it should be downloaded again + double _secondsUntilResync = MaxDateAsJ2000; }; } // namespace openspace diff --git a/src/engine/openspaceengine.cpp b/src/engine/openspaceengine.cpp index 0d79e58b38..84cf70c4d3 100644 --- a/src/engine/openspaceengine.cpp +++ b/src/engine/openspaceengine.cpp @@ -780,8 +780,8 @@ void OpenSpaceEngine::loadAssets() { static_cast(sync->nTotalBytes()); }(*it); + progressInfo.currentSize = (*it)->nSynchronizedBytes(); if ((*it)->nTotalBytesIsKnown()) { - progressInfo.currentSize = (*it)->nSynchronizedBytes(); progressInfo.totalSize = (*it)->nTotalBytes(); } diff --git a/src/rendering/loadingscreen.cpp b/src/rendering/loadingscreen.cpp index a50aed0e92..c01e39e9cd 100644 --- a/src/rendering/loadingscreen.cpp +++ b/src/rendering/loadingscreen.cpp @@ -392,12 +392,12 @@ void LoadingScreen::render() { #endif // LOADINGSCREEN_DEBUGGING std::string text = item.name; - if (item.status == ItemStatus::Started && item.progress.progress > 0) { + if (item.status == ItemStatus::Started && item.progress.currentSize > 0) { ProgressInfo& info = item.progress; - bool hasSecondLine = (info.totalSize != -1 && info.currentSize != -1); + const bool isTotalSizeKnown = info.totalSize != -1; int p = static_cast(std::round(info.progress * 100)); - if (hasSecondLine) { + if (isTotalSizeKnown) { if (info.totalSize < 1024 * 1024) { // 1MB text = fmt::format( "{} ({}%)\n{}/{} {}", @@ -415,7 +415,14 @@ void LoadingScreen::render() { } } else { - text = fmt::format("{} ({}%)", text, p); + // We don't know the total size but we have started downloading data + if (info.currentSize < 1024 * 1024) { + text = fmt::format("{}\n{} {}", text, info.currentSize, "bytes"); + } + else { + float curr = info.currentSize / (1024.f * 1024.f); + text = fmt::format("{}\n{:.3f} {}", text, curr, "MB"); + } } } diff --git a/src/util/spicemanager.cpp b/src/util/spicemanager.cpp index 23fc6c29be..01cd152654 100644 --- a/src/util/spicemanager.cpp +++ b/src/util/spicemanager.cpp @@ -154,6 +154,8 @@ SpiceManager::SpiceManager() { erract_c("SET", 0, const_cast("REPORT")); // But we do not want SPICE to print the errors, we will fetch them ourselves errprt_c("SET", 0, const_cast("NONE")); + + loadLeapSecondsSpiceKernel(); } SpiceManager::~SpiceManager() { @@ -1349,6 +1351,171 @@ glm::dmat3 SpiceManager::getEstimatedTransformMatrix(const std::string& fromFram return result; } +void SpiceManager::loadLeapSecondsSpiceKernel() { + constexpr std::string_view Naif00012tlsSource = R"( +KPL/LSK + + +LEAPSECONDS KERNEL FILE +=========================================================================== + +Modifications: +-------------- + +2016, Jul. 14 NJB Modified file to account for the leapsecond that + will occur on December 31, 2016. + +2015, Jan. 5 NJB Modified file to account for the leapsecond that + will occur on June 30, 2015. + +2012, Jan. 5 NJB Modified file to account for the leapsecond that + will occur on June 30, 2012. + +2008, Jul. 7 NJB Modified file to account for the leapsecond that + will occur on December 31, 2008. + +2005, Aug. 3 NJB Modified file to account for the leapsecond that + will occur on December 31, 2005. + +1998, Jul 17 WLT Modified file to account for the leapsecond that + will occur on December 31, 1998. + +1997, Feb 22 WLT Modified file to account for the leapsecond that + will occur on June 30, 1997. + +1995, Dec 14 KSZ Corrected date of last leapsecond from 1-1-95 + to 1-1-96. + +1995, Oct 25 WLT Modified file to account for the leapsecond that + will occur on Dec 31, 1995. + +1994, Jun 16 WLT Modified file to account for the leapsecond on + June 30, 1994. + +1993, Feb. 22 CHA Modified file to account for the leapsecond on + June 30, 1993. + +1992, Mar. 6 HAN Modified file to account for the leapsecond on + June 30, 1992. + +1990, Oct. 8 HAN Modified file to account for the leapsecond on + Dec. 31, 1990. + + +Explanation: +------------ + +The contents of this file are used by the routine DELTET to compute the +time difference + +[1] DELTA_ET = ET - UTC + +the increment to be applied to UTC to give ET. + +The difference between UTC and TAI, + +[2] DELTA_AT = TAI - UTC + +is always an integral number of seconds. The value of DELTA_AT was 10 +seconds in January 1972, and increases by one each time a leap second +is declared. Combining [1] and [2] gives + +[3] DELTA_ET = ET - (TAI - DELTA_AT) + + = (ET - TAI) + DELTA_AT + +The difference (ET - TAI) is periodic, and is given by + +[4] ET - TAI = DELTA_T_A + K sin E + +where DELTA_T_A and K are constant, and E is the eccentric anomaly of the +heliocentric orbit of the Earth-Moon barycenter. Equation [4], which ignores +small-period fluctuations, is accurate to about 0.000030 seconds. + +The eccentric anomaly E is given by + +[5] E = M + EB sin M + +where M is the mean anomaly, which in turn is given by + +[6] M = M + M t + 0 1 + +where t is the number of ephemeris seconds past J2000. + +Thus, in order to compute DELTA_ET, the following items are necessary. + + DELTA_TA + K + EB + M0 + M1 + DELTA_AT after each leap second. + +The numbers, and the formulation, are taken from the following sources. + + 1) Moyer, T.D., Transformation from Proper Time on Earth to + Coordinate Time in Solar System Barycentric Space-Time Frame + of Reference, Parts 1 and 2, Celestial Mechanics 23 (1981), + 33-56 and 57-68. + + 2) Moyer, T.D., Effects of Conversion to the J2000 Astronomical + Reference System on Algorithms for Computing Time Differences + and Clock Rates, JPL IOM 314.5--942, 1 October 1985. + +The variable names used above are consistent with those used in the +Astronomical Almanac. + +\begindata + +DELTET/DELTA_T_A = 32.184 +DELTET/K = 1.657D-3 +DELTET/EB = 1.671D-2 +DELTET/M = ( 6.239996D0 1.99096871D-7 ) + +DELTET/DELTA_AT = ( 10, @1972-JAN-1 + 11, @1972-JUL-1 + 12, @1973-JAN-1 + 13, @1974-JAN-1 + 14, @1975-JAN-1 + 15, @1976-JAN-1 + 16, @1977-JAN-1 + 17, @1978-JAN-1 + 18, @1979-JAN-1 + 19, @1980-JAN-1 + 20, @1981-JUL-1 + 21, @1982-JUL-1 + 22, @1983-JUL-1 + 23, @1985-JUL-1 + 24, @1988-JAN-1 + 25, @1990-JAN-1 + 26, @1991-JAN-1 + 27, @1992-JUL-1 + 28, @1993-JUL-1 + 29, @1994-JUL-1 + 30, @1996-JAN-1 + 31, @1997-JUL-1 + 32, @1999-JAN-1 + 33, @2006-JAN-1 + 34, @2009-JAN-1 + 35, @2012-JUL-1 + 36, @2015-JUL-1 + 37, @2017-JAN-1 ) + +\begintext + + +)"; + std::filesystem::path path = std::filesystem::temp_directory_path(); + std::filesystem::path file = path / "naif0012.tls"; + { + std::ofstream f(file); + f << Naif00012tlsSource; + } + loadKernel(file.string()); + std::filesystem::remove(file); +} + void SpiceManager::setExceptionHandling(UseException useException) { _useExceptions = useException; }