Move temporal information from the XML file format into the asset file

This commit is contained in:
Alexander Bock
2021-12-31 12:23:28 +01:00
parent 0b496c32a9
commit d81644ecc5
14 changed files with 230 additions and 223 deletions

View File

@@ -11,16 +11,15 @@ local layer = {
Identifier = "ERA5_Land_HighRes_Monthly_2M_Temperature_Temporal",
Name = "ERA5 Land HighRes Monthly 2M Temperature (Temporal)",
Type = "TemporalTileLayer",
FilePath =
"<OpenSpaceTemporalGDALDataset>" ..
"<OpenSpaceTimeStart>1981-01-01</OpenSpaceTimeStart>" ..
"<OpenSpaceTimeEnd>1990-10-01</OpenSpaceTimeEnd>" ..
"<OpenSpaceTimeResolution>1M</OpenSpaceTimeResolution>" ..
"<OpenSpaceTimeIdFormat>YYYY-MM-DD</OpenSpaceTimeIdFormat>" ..
"<OpenSpaceTransferFunction>" .. path .. "rainbow.png</OpenSpaceTransferFunction>" ..
"<OpenSpaceTimeInterpolation>linear</OpenSpaceTimeInterpolation>" ..
"<FilePath>" .. path .. "${OpenSpaceTimeId}-land.png</FilePath>" ..
"</OpenSpaceTemporalGDALDataset>",
Time = {
Start = "1981-01-01",
End = "1990-10-01"
},
TemporalResolution = "1M",
TimeFormat = "YYYY-MM-DD",
Interpolation = true,
Colormap = path .. "rainbow.png"
FilePath = path .. "${OpenSpaceTimeId}-land.png",
Description = [[ Temporal coverage: 01 Jan 1981 - 31 Dec 2020.]]
}

View File

@@ -4,11 +4,16 @@ local layer = {
Identifier = "AMSR2_GCOM_W1_Sea_Ice_Concentration_Temporal",
Name = "AMSR2 GCOM W1 Sea Ice Concentration (Temporal)",
Type = "TemporalTileLayer",
Generative = {
Time = {
Start = "2012-05-08",
End = "Yesterday"
},
TemporalResolution = "1d",
TimeFormat = "YYYY-MM-DD",
},
FilePath = openspace.globebrowsing.createTemporalGibsGdalXml(
"AMSRU2_Sea_Ice_Concentration_12km",
"2012-05-08",
"Yesterday",
"1d",
"2km",
"png"
),

View File

@@ -4,11 +4,16 @@ local layer = {
Identifier = "Aqua_Modis_Temporal",
Name = "Aqua Modis (Temporal)",
Type = "TemporalTileLayer",
Generative = {
Time = {
Start = "2002-07-04",
End = "Yesterday"
},
TemporalResolution = "1d",
TimeFormat = "YYYY-MM-DD",
},
FilePath = openspace.globebrowsing.createTemporalGibsGdalXml(
"MODIS_Aqua_CorrectedReflectance_TrueColor",
"2002-07-04",
"Yesterday",
"1d",
"250m",
"jpg"
),

View File

@@ -13,11 +13,16 @@ local layer = {
Identifier = "Temporal_VIIRS_SNPP",
Name = "Temporal VIIRS SNPP",
Type = "TemporalTileLayer",
Generative = {
Time = {
Start = "2015-11-24",
End = "Today"
},
TemporalResolution = "1d",
TimeFormat = "YYYY-MM-DD",
},
FilePath = openspace.globebrowsing.createTemporalGibsGdalXml(
"VIIRS_SNPP_CorrectedReflectance_TrueColor",
"2015-11-24",
"Today",
"1d",
"250m",
"jpg"
),

View File

@@ -4,11 +4,16 @@ local layer = {
Identifier = "GHRSST_L4_G1SST_Sea_Surface_Temperature_Temporal",
Name = "GHRSST L4 G1SST Sea Surface Temperature (Temporal)",
Type = "TemporalTileLayer",
Generative = {
Time = {
Start = "2010-06-21",
End = "2019-12-08"
},
TemporalResolution = "1d",
TimeFormat = "YYYY-MM-DD",
},
FilePath = openspace.globebrowsing.createTemporalGibsGdalXml(
"GHRSST_L4_G1SST_Sea_Surface_Temperature",
"2010-06-21",
"2019-12-08",
"1d",
"1km",
"png"
),

View File

@@ -4,11 +4,16 @@ local layer = {
Identifier = "GHRSST_L4_MUR_Sea_Surface_Temperature_Temporal",
Name = "GHRSST L4 MUR Sea Surface Temperature (Temporal)",
Type = "TemporalTileLayer",
Generative = {
Time = {
Start = "2002-06-01",
End = "Yesterday"
},
TemporalResolution = "1d",
TimeFormat = "YYYY-MM-DD",
},
FilePath = openspace.globebrowsing.createTemporalGibsGdalXml(
"GHRSST_L4_MUR_Sea_Surface_Temperature",
"2002-06-01",
"Yesterday",
"1d",
"1km",
"png"
),

View File

@@ -4,13 +4,18 @@ local layer = {
Identifier = "MODIS_Terra_Chlorophyll_A_Temporal",
Name = "MODIS Terra Chlorophyll A (Temporal)",
Type = "TemporalTileLayer",
Generative = {
Time = {
Start = "2013-07-02",
End = "Yesterday"
},
TemporalResolution = "1d",
TimeFormat = "YYYY-MM-DD",
},
FilePath = openspace.globebrowsing.createTemporalGibsGdalXml(
"MODIS_Terra_Chlorophyll_A",
"2013-07-02",
"Yesterday",
"1d",
"1km",
"png"
"MODIS_Terra_Chlorophyll_A",
"1km",
"png"
),
Description = [[ Temporal coverage: 02 July 2013 - Present. The imagery resolution
is 1 km, and the temporal resolution is daily.]]

View File

@@ -4,11 +4,16 @@ local layer = {
Identifier = "Terra_Modis_Temporal",
Name = "Terra Modis (Temporal)",
Type = "TemporalTileLayer",
Generative = {
Time = {
Start = "2000-02-24",
End = "Yesterday"
},
TemporalResolution = "1d",
TimeFormat = "YYYY-MM-DD",
},
FilePath = openspace.globebrowsing.createTemporalGibsGdalXml(
"MODIS_Terra_CorrectedReflectance_TrueColor",
"2000-02-24",
"Yesterday",
"1d",
"250m",
"jpg"
),

View File

@@ -4,11 +4,16 @@ local layer = {
Identifier = "VIIRS_SNPP_Temporal",
Name = "VIIRS SNPP (Temporal)",
Type = "TemporalTileLayer",
Generative = {
Time = {
Start = "2015-11-24",
End = "Yesterday"
},
TemporalResolution = "1d",
TimeFormat = "YYYY-MM-DD",
},
FilePath = openspace.globebrowsing.createTemporalGibsGdalXml(
"VIIRS_SNPP_CorrectedReflectance_TrueColor",
"2015-11-24",
"Yesterday",
"1d",
"250m",
"jpg"
),

View File

@@ -4,11 +4,16 @@ local layer = {
Identifier = "Earth_at_Night_Temporal",
Name = "Earth at Night (Temporal)",
Type = "TemporalTileLayer",
Generative = {
Time = {
Start = "2012-05-08",
End = "Yesterday"
},
TemporalResolution = "1d",
TimeFormat = "YYYY-MM-DD",
},
FilePath = openspace.globebrowsing.createTemporalGibsGdalXml(
"VIIRS_SNPP_DayNightBand_ENCC",
"2012-05-08",
"Yesterday",
"1d",
"500m",
"png"
),

View File

@@ -1,30 +1,11 @@
openspace.globebrowsing.documentation = {
{
Name = "createTemporalGibsGdalXml",
Arguments = "string, string, string, string, string, string, [string]",
Documentation =
"Creates an XML configuration for a temporal GIBS dataset." ..
"Arguments are: Name, Start date, end date, time resolution, time format," ..
"resolution, file format. The last parameter is the temporal format and " ..
"defaults to YYYY-MM-DD. For all specifications, see " ..
"https://wiki.earthdata.nasa.gov/display/GIBS/GIBS+Available+Imagery+Products" ..
"Usage:" ..
"openspace.globebrowsing.addLayer(" ..
"\"Earth\"," ..
"\"ColorLayers\"," ..
"{" ..
"Type = \"TemporalTileLayer\"," ..
"Name = \"MODIS_Terra_Chlorophyll_A\"," ..
"FilePath = openspace.globebrowsing.createTemporalGibsGdalXml(" ..
"\"MODIS_Terra_Chlorophyll_A\"," ..
"\"2013-07-02\"," ..
"\"Yesterday\"," ..
"\"1d\"," ..
"\"1km\"," ..
"\"png\"" ..
")" ..
"}" ..
")"
Arguments = "string, string, string",
Documentation = [[
Creates an XML configuration for a temporal GIBS dataset to be used in
a TemporalTileprovider
]]
},
{
Name = "createGibsGdalXml",
@@ -40,7 +21,7 @@ openspace.globebrowsing.documentation = {
"\"ColorLayers\"," ..
"{" ..
"Name = \"MODIS_Terra_Chlorophyll_A\"," ..
"FilePath = openspace.globebrowsing.createTemporalGibsGdalXml(" ..
"FilePath = openspace.globebrowsing.createGibsGdalXml(" ..
"\"MODIS_Terra_Chlorophyll_A\"," ..
"\"2013-07-02\"," ..
"\"1km\"," ..
@@ -111,21 +92,26 @@ openspace.globebrowsing.addGibsLayer = function(layer, resolution, format, start
if endDate == 'Present' then
endDate = ''
end
local xml = openspace.globebrowsing.createTemporalGibsGdalXml(layer, startDate, endDate, '1d', resolution, format)
openspace.globebrowsing.addLayer('Earth', 'ColorLayers', { Identifier = layer, Type = "TemporalTileLayer", FilePath = xml })
end
openspace.globebrowsing.createTemporalGibsGdalXml = function (layerName, startDate, endDate, timeResolution, resolution, format, temporalFormat)
temporalFormat = temporalFormat or 'YYYY-MM-DD'
local temporalTemplate =
"<OpenSpaceTemporalGDALDataset>" ..
"<OpenSpaceTimeStart>" .. startDate .. "</OpenSpaceTimeStart>" ..
"<OpenSpaceTimeEnd>" .. endDate .. "</OpenSpaceTimeEnd>" ..
"<OpenSpaceTimeResolution>" .. timeResolution .. "</OpenSpaceTimeResolution>" ..
"<OpenSpaceTimeIdFormat>" .. temporalFormat .. "</OpenSpaceTimeIdFormat>" ..
openspace.globebrowsing.createGibsGdalXml(layerName, "${OpenSpaceTimeId}", resolution, format) ..
"</OpenSpaceTemporalGDALDataset>"
return temporalTemplate
local layer = {
Identifier = layerName,
Type = "TemporalTileLayer",
Generative = {
Time = {
Start = startDate,
End = endDate
},
TemporalResolution = "1d",
TimeFormat = "YYYY-MM-DD",
FilePath = openspace.globebrowsing.createTemporalGibsGdalXml(layerName, resolution, format)
}
}
openspace.globebrowsing.addLayer(
'Earth',
'ColorLayers',
layer
)
end
openspace.globebrowsing.createGibsGdalXml = function (layerName, date, resolution, format)
@@ -196,6 +182,10 @@ openspace.globebrowsing.createGibsGdalXml = function (layerName, date, resolutio
return gdalWmsTemplate
end
openspace.globebrowsing.createTemporalGibsGdalXml = function (layerName, resolution, format)
return openspace.globebrowsing.createGibsGdalXml(layerName, "${OpenSpaceTimeId}", resolution, format)
end
openspace.globebrowsing.parseInfoFile = function (file)
-- We are loading these values from an external info file and since we are switching
-- to a strict Lua, we need to predefine these global variables

View File

@@ -44,12 +44,12 @@ namespace {
constexpr const char* KeyBasePath = "BasePath";
constexpr const char* UrlTimePlaceholder = "${OpenSpaceTimeId}";
constexpr const char* TimeStart = "OpenSpaceTimeStart";
constexpr const char* TimeEnd = "OpenSpaceTimeEnd";
constexpr const char* TimeResolution = "OpenSpaceTimeResolution";
constexpr const char* TimeFormat = "OpenSpaceTimeIdFormat";
constexpr const char* TimeInterpolation = "OpenSpaceTimeInterpolation";
constexpr const char* TransferFunction = "OpenSpaceTransferFunction";
//constexpr const char* TimeStart = "OpenSpaceTimeStart";
//constexpr const char* TimeEnd = "OpenSpaceTimeEnd";
//constexpr const char* TimeResolution = "OpenSpaceTimeResolution";
//constexpr const char* TimeFormat = "OpenSpaceTimeIdFormat";
//constexpr const char* TimeInterpolation = "OpenSpaceTimeInterpolation";
//constexpr const char* TransferFunction = "OpenSpaceTransferFunction";
constexpr openspace::properties::Property::PropertyInfo FilePathInfo = {
"FilePath",
@@ -82,8 +82,86 @@ namespace {
// [[codegen::verbatim(FixedTimeInfo.description)]]
std::optional<std::string> fixedTime;
struct Generative {
struct Time {
// The (inclusive) starting time of the temporal image range
std::string start;
// The (inclusive) ending time of the temporal image range
std::string end;
};
// The starting and ending times for the range of values
Time time;
// The temporal resolution between each image
std::string temporalResolution;
// The specification of the date format that is used in the tile provider
std::string timeFormat;
};
std::optional<Generative> generative;
std::optional<bool> interpolation;
std::optional<std::string> colormap;
};
#include "temporaltileprovider_codegen.cpp"
// Buffer needs at least 22 characters space
std::string_view timeStringify(
openspace::globebrowsing::TemporalTileProvider::TimeFormatType type,
const openspace::Time& t)
{
ZoneScoped
using namespace openspace;
using namespace openspace::globebrowsing;
char* buffer = reinterpret_cast<char*>(
global::memoryManager->TemporaryMemory.allocate(22)
);
std::memset(buffer, 0, 22);
const double time = t.j2000Seconds();
switch (type) {
case TemporalTileProvider::TimeFormatType::YYYY_MM_DD:
{
constexpr const char Format[] = "YYYY-MM-DD";
constexpr const int Size = sizeof(Format);
SpiceManager::ref().dateFromEphemerisTime(time, buffer, Size, Format);
return std::string_view(buffer, Size - 1);
}
case TemporalTileProvider::TimeFormatType::YYYYMMDD_hhmmss: {
constexpr const char Format[] = "YYYYMMDD_HRMNSC";
constexpr const int Size = sizeof(Format);
SpiceManager::ref().dateFromEphemerisTime(time, buffer, Size, Format);
return std::string_view(buffer, Size - 1);
}
case TemporalTileProvider::TimeFormatType::YYYYMMDD_hhmm: {
constexpr const char Format[] = "YYYYMMDD_HRMN";
constexpr const int Size = sizeof(Format);
SpiceManager::ref().dateFromEphemerisTime(time, buffer, Size, Format);
return std::string_view(buffer, Size - 1);
}
case TemporalTileProvider::TimeFormatType::YYYY_MM_DDThhColonmmColonssZ:
{
constexpr const char Format[] = "YYYY-MM-DDTHR:MN:SCZ";
constexpr const int Size = sizeof(Format);
SpiceManager::ref().dateFromEphemerisTime(time, buffer, Size, Format);
return std::string_view(buffer, Size - 1);
}
case TemporalTileProvider::TimeFormatType::YYYY_MM_DDThh_mm_ssZ: {
constexpr const char Format[] = "YYYY-MM-DDTHR_MN_SCZ";
constexpr const int Size = sizeof(Format);
SpiceManager::ref().dateFromEphemerisTime(time, buffer, Size, Format);
return std::string_view(buffer, Size - 1);
}
default:
throw ghoul::MissingCaseException();
}
}
} // namespace
namespace ghoul {
@@ -135,83 +213,46 @@ TemporalTileProvider::TemporalTileProvider(const ghoul::Dictionary& dictionary)
addProperty(_fixedTime);
std::ifstream in(_filePath.value());
std::string xml;
if (in.is_open()) {
// read file
xml = std::string(
std::istreambuf_iterator<char>(in),
(std::istreambuf_iterator<char>())
);
}
else {
// Assume that it is already an xml
xml = _filePath;
}
// File path was not a path to a file but a GDAL config or empty
std::filesystem::path f(_filePath.value());
if (std::filesystem::is_regular_file(f)) {
_initDict.setValue(KeyBasePath, f.parent_path().string());
}
CPLXMLNode* node = CPLParseXMLString(xml.c_str());
_colormap = p.colormap.value_or(_colormap);
std::string timeStart = xmlValue(node, TimeStart, "2000 Jan 1");
std::string timeResolution = xmlValue(node, TimeResolution, "2d");
std::string timeEnd = xmlValue(node, TimeEnd, "Today");
std::string timeIdFormat = xmlValue(
node,
TimeFormat,
"YYYY-MM-DDThh:mm:ssZ"
);
std::string timeInterpolation = xmlValue(
node,
TimeInterpolation,
"none",
true
);
_colormap = xmlValue(node, TransferFunction, "none", true);
Time start = Time(timeStart);
Time end = Time::now();
Time endOfInterval = Time(timeEnd);
_startTimeJ2000 = start.j2000Seconds();
_endTimeJ2000 = endOfInterval.j2000Seconds();
if (timeEnd == "Yesterday") {
end.advanceTime(-60.0 * 60.0 * 24.0); // Go back one day
}
else if (timeEnd != "Today") {
end.setTime(std::move(timeEnd));
}
try {
_timeQuantizer.setStartEndRange(
std::string(start.ISO8601()),
std::string(end.ISO8601())
);
_timeQuantizer.setResolution(timeResolution);
_myResolution = timeResolution;
}
catch (const ghoul::RuntimeError& e) {
throw ghoul::RuntimeError(fmt::format(
"Could not create time quantizer for Temporal GDAL dataset '{}'. {}",
_filePath.value(), e.message
));
}
_timeFormat = ghoul::from_string<TemporalTileProvider::TimeFormatType>(timeIdFormat);
_interpolation = (timeInterpolation == "linear");
CPLXMLNode* gdalNode = CPLSearchXMLNode(node, "GDAL_WMS");
if (gdalNode) {
_gdalXmlTemplate = CPLSerializeXMLTree(gdalNode);
}
else {
gdalNode = CPLSearchXMLNode(node, "FilePath");
if (gdalNode) {
_gdalXmlTemplate = std::string(gdalNode->psChild->pszValue);
if (p.generative.has_value()) {
Time start = Time(p.generative->time.start);
Time end = Time::now();
Time endOfInterval = Time(p.generative->time.end);
_startTimeJ2000 = start.j2000Seconds();
_endTimeJ2000 = endOfInterval.j2000Seconds();
if (p.generative->time.end == "Yesterday") {
end.advanceTime(-60.0 * 60.0 * 24.0); // Go back one day
}
else if (p.generative->time.end != "Today") {
end.setTime(p.generative->time.end);
}
try {
_timeQuantizer.setStartEndRange(
std::string(start.ISO8601()),
std::string(end.ISO8601())
);
_timeQuantizer.setResolution(p.generative->temporalResolution);
_myResolution = p.generative->temporalResolution;
}
catch (const ghoul::RuntimeError& e) {
throw ghoul::RuntimeError(fmt::format(
"Could not create time quantizer for Temporal GDAL dataset '{}'. {}",
_filePath.value(), e.message
));
}
_timeFormat = ghoul::from_string<TimeFormatType>(p.generative->timeFormat);
}
_interpolation = p.interpolation.value_or(_interpolation);
_gdalXmlTemplate = p.filePath;
if (_interpolation) {
_interpolateTileProvider = std::make_unique<InterpolateTileProvider>(dictionary);
@@ -259,9 +300,7 @@ void TemporalTileProvider::update() {
}
void TemporalTileProvider::reset() {
using K = TemporalTileProvider::TimeKey;
using V = std::unique_ptr<TileProvider>;
for (std::pair<const K, V>& it : _tileProviderMap) {
for (std::pair<const TimeKey, std::unique_ptr<TileProvider>>& it : _tileProviderMap) {
it.second->reset();
}
}
@@ -275,69 +314,6 @@ float TemporalTileProvider::noDataValueAsFloat() {
return std::numeric_limits<float>::min();
}
// Buffer needs at least 22 characters space
std::string_view TemporalTileProvider::timeStringify(TimeFormatType type, const Time& t) {
ZoneScoped
char* buffer = reinterpret_cast<char*>(
global::memoryManager->TemporaryMemory.allocate(22)
);
std::memset(buffer, 0, 22);
const double time = t.j2000Seconds();
switch (type) {
case TemporalTileProvider::TimeFormatType::YYYY_MM_DD:
{
constexpr const char Format[] = "YYYY-MM-DD";
constexpr const int Size = sizeof(Format);
SpiceManager::ref().dateFromEphemerisTime(time, buffer, Size, Format);
return std::string_view(buffer, Size - 1);
}
case TemporalTileProvider::TimeFormatType::YYYYMMDD_hhmmss: {
constexpr const char Format[] = "YYYYMMDD_HRMNSC";
constexpr const int Size = sizeof(Format);
SpiceManager::ref().dateFromEphemerisTime(time, buffer, Size, Format);
return std::string_view(buffer, Size - 1);
}
case TemporalTileProvider::TimeFormatType::YYYYMMDD_hhmm: {
constexpr const char Format[] = "YYYYMMDD_HRMN";
constexpr const int Size = sizeof(Format);
SpiceManager::ref().dateFromEphemerisTime(time, buffer, Size, Format);
return std::string_view(buffer, Size - 1);
}
case TemporalTileProvider::TimeFormatType::YYYY_MM_DDThhColonmmColonssZ:
{
constexpr const char Format[] = "YYYY-MM-DDTHR:MN:SCZ";
constexpr const int Size = sizeof(Format);
SpiceManager::ref().dateFromEphemerisTime(time, buffer, Size, Format);
return std::string_view(buffer, Size - 1);
}
case TemporalTileProvider::TimeFormatType::YYYY_MM_DDThh_mm_ssZ: {
constexpr const char Format[] = "YYYY-MM-DDTHR_MN_SCZ";
constexpr const int Size = sizeof(Format);
SpiceManager::ref().dateFromEphemerisTime(time, buffer, Size, Format);
return std::string_view(buffer, Size - 1);
}
default:
throw ghoul::MissingCaseException();
}
}
std::string TemporalTileProvider::xmlValue(CPLXMLNode* node, const std::string& key,
const std::string& defaultVal, bool isOptional)
{
CPLXMLNode* n = CPLSearchXMLNode(node, key.c_str());
if (!n && !isOptional) {
throw ghoul::RuntimeError(
fmt::format("Unable to parse file {}. {} missing", _filePath.value(), key)
);
}
const bool hasValue = n && n->psChild && n->psChild->pszValue;
return hasValue ? n->psChild->pszValue : defaultVal;
}
std::unique_ptr<TileProvider> TemporalTileProvider::initTileProvider(
std::string_view timekey)
{

View File

@@ -111,12 +111,9 @@ private:
std::unique_ptr<InterpolateTileProvider> _interpolateTileProvider;
void ensureUpdated();
std::string_view timeStringify(TimeFormatType type, const Time& t);
std::unique_ptr<TileProvider> initTileProvider(std::string_view timekey);
TileProvider* tileProvider(std::string_view timekey);
TileProvider* tileProvider(const Time& time);
std::string xmlValue(CPLXMLNode* node, const std::string& key,
const std::string& defaultVal, bool isOptional = false);
};
} // namespace openspace::globebrowsing