diff --git a/modules/globebrowsing/CMakeLists.txt b/modules/globebrowsing/CMakeLists.txt index 7f080eb79e..1c032afb0b 100644 --- a/modules/globebrowsing/CMakeLists.txt +++ b/modules/globebrowsing/CMakeLists.txt @@ -51,6 +51,7 @@ set(HEADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/tile/temporaltileprovider.h ${CMAKE_CURRENT_SOURCE_DIR}/tile/tileprovider.h ${CMAKE_CURRENT_SOURCE_DIR}/tile/tileselector.h + ${CMAKE_CURRENT_SOURCE_DIR}/tile/tilediskcache.h ${CMAKE_CURRENT_SOURCE_DIR}/tile/tiledataset.h ${CMAKE_CURRENT_SOURCE_DIR}/tile/asynctilereader.h ${CMAKE_CURRENT_SOURCE_DIR}/tile/tileprovidermanager.h @@ -92,6 +93,7 @@ set(SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/tile/temporaltileprovider.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tile/tileprovider.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tile/tileselector.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tile/tilediskcache.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tile/tiledataset.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tile/asynctilereader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tile/tileprovidermanager.cpp diff --git a/modules/globebrowsing/globes/renderableglobe.cpp b/modules/globebrowsing/globes/renderableglobe.cpp index 292983f3a7..da7713d07b 100644 --- a/modules/globebrowsing/globes/renderableglobe.cpp +++ b/modules/globebrowsing/globes/renderableglobe.cpp @@ -59,7 +59,7 @@ namespace openspace { , doFrustumCulling(properties::BoolProperty("doFrustumCulling", "doFrustumCulling")) , doHorizonCulling(properties::BoolProperty("doHorizonCulling", "doHorizonCulling")) , mergeInvisible(properties::BoolProperty("mergeInvisible", "mergeInvisible", true)) - , lodScaleFactor(properties::FloatProperty("lodScaleFactor", "lodScaleFactor", 5.0f, 1.0f, 20.0f)) + , lodScaleFactor(properties::FloatProperty("lodScaleFactor", "lodScaleFactor", 5.0f, 1.0f, 50.0f)) , initChunkVisible(properties::BoolProperty("initChunkVisible", "initChunkVisible", true)) , renderSmallChunksFirst(properties::BoolProperty("renderSmallChunksFirst", "renderSmallChunksFirst", true)) , chunkHeight(properties::FloatProperty("chunkHeight", "chunkHeight", 8700.0f, 0.0f, 8700.0f)) diff --git a/modules/globebrowsing/tile/asynctilereader.cpp b/modules/globebrowsing/tile/asynctilereader.cpp index fcce97d5b9..8408197579 100644 --- a/modules/globebrowsing/tile/asynctilereader.cpp +++ b/modules/globebrowsing/tile/asynctilereader.cpp @@ -29,6 +29,8 @@ #include #include +#include + #include @@ -40,6 +42,77 @@ namespace { namespace openspace { + void TileLoadJob::execute() { + _tileIOResult = _tileDataset->readTileData(_chunkIndex); + } + + + + + DiskCachedTileLoadJob::DiskCachedTileLoadJob(std::shared_ptr textureDataProvider, + const ChunkIndex& chunkIndex, std::shared_ptr tdc, CacheMode m) + : TileLoadJob(textureDataProvider, chunkIndex) + , _tileDiskCache(tdc) + , _mode(m) + { + + } + + DiskCachedTileLoadJob::DiskCachedTileLoadJob(std::shared_ptr textureDataProvider, + const ChunkIndex& chunkIndex, std::shared_ptr tdc, const std::string cacheMode) + : TileLoadJob(textureDataProvider, chunkIndex) + , _tileDiskCache(tdc) + { + if (cacheMode == "Disabled") _mode = CacheMode::Disabled; + else if (cacheMode == "ReadOnly") _mode = CacheMode::ReadOnly; + else if (cacheMode == "ReadAndWrite") _mode = CacheMode::ReadAndWrite; + else if (cacheMode == "WriteOnly") _mode = CacheMode::WriteOnly; + else if (cacheMode == "CacheHitsOnly") _mode = CacheMode::CacheHitsOnly; + } + + + + void DiskCachedTileLoadJob::execute() { + _tileIOResult = nullptr; + + switch (_mode) { + case CacheMode::Disabled: + _tileIOResult = _tileDataset->readTileData(_chunkIndex); + break; + + case CacheMode::ReadOnly: + _tileIOResult = _tileDiskCache->get(_chunkIndex); + if (_tileIOResult == nullptr) { + _tileIOResult = _tileDataset->readTileData(_chunkIndex); + } + break; + + case CacheMode::ReadAndWrite: + _tileIOResult = _tileDiskCache->get(_chunkIndex); + if (_tileIOResult == nullptr) { + _tileIOResult = _tileDataset->readTileData(_chunkIndex); + _tileDiskCache->put(_chunkIndex, _tileIOResult); + } + break; + + case CacheMode::WriteOnly: + _tileIOResult = _tileDataset->readTileData(_chunkIndex); + _tileDiskCache->put(_chunkIndex, _tileIOResult); + break; + + case CacheMode::CacheHitsOnly: + _tileIOResult = _tileDiskCache->get(_chunkIndex); + if (_tileIOResult == nullptr) { + TileIOResult res = TileIOResult::createDefaultRes(); + res.chunkIndex = _chunkIndex; + _tileIOResult = std::make_shared(res); + } + break; + } + + + } + AsyncTileDataProvider::AsyncTileDataProvider( @@ -61,10 +134,10 @@ namespace openspace { } bool AsyncTileDataProvider::enqueueTextureData(const ChunkIndex& chunkIndex) { + //auto tileDiskCache = std::make_shared("test"); if (satisfiesEnqueueCriteria(chunkIndex)) { - std::shared_ptr job = std::shared_ptr( - new TileLoadJob(_tileDataset, chunkIndex)); - + auto job = std::make_shared(_tileDataset, chunkIndex); + //auto job = std::make_shared(_tileDataset, chunkIndex, tileDiskCache, "ReadAndWrite"); _concurrentJobManager.enqueueJob(job); _enqueuedTileRequests[chunkIndex.hashKey()] = chunkIndex; return true; diff --git a/modules/globebrowsing/tile/asynctilereader.h b/modules/globebrowsing/tile/asynctilereader.h index acf69e8e5e..c44f51f7f2 100644 --- a/modules/globebrowsing/tile/asynctilereader.h +++ b/modules/globebrowsing/tile/asynctilereader.h @@ -37,17 +37,25 @@ #include + #include #include #include +#include namespace openspace { + + struct LoadJob : public Job { + virtual void execute() = 0; + virtual std::shared_ptr product() = 0; + }; + struct TileLoadJob : LoadJob { - struct TileLoadJob : public Job { + TileLoadJob(std::shared_ptr textureDataProvider, const ChunkIndex& chunkIndex) : _tileDataset(textureDataProvider) @@ -58,26 +66,53 @@ namespace openspace { virtual ~TileLoadJob() { } - virtual void execute() { - _uninitedTexture = _tileDataset->readTileData(_chunkIndex); - } - - + virtual void execute(); virtual std::shared_ptr product() { - return _uninitedTexture; + return _tileIOResult; } - private: + protected: + ChunkIndex _chunkIndex; std::shared_ptr _tileDataset; - std::shared_ptr _uninitedTexture; + std::shared_ptr _tileIOResult; }; + class TileDiskCache; + + struct DiskCachedTileLoadJob : public TileLoadJob { + enum CacheMode { + Disabled, + ReadOnly, + ReadAndWrite, + WriteOnly, + CacheHitsOnly, + }; + + DiskCachedTileLoadJob(std::shared_ptr textureDataProvider, + const ChunkIndex& chunkIndex, std::shared_ptr tdc, + const std::string cacheMode); + + DiskCachedTileLoadJob(std::shared_ptr textureDataProvider, + const ChunkIndex& chunkIndex, std::shared_ptr tdc, + CacheMode cacheMode = CacheMode::ReadOnly); + + virtual void execute(); + + protected: + + std::shared_ptr _tileDiskCache; + CacheMode _mode; + + }; + + + class AsyncTileDataProvider { public: @@ -107,7 +142,6 @@ namespace openspace { ConcurrentJobManager _concurrentJobManager; std::unordered_map _enqueuedTileRequests; - }; diff --git a/modules/globebrowsing/tile/tiledataset.cpp b/modules/globebrowsing/tile/tiledataset.cpp index 69315043d6..4323b85f56 100644 --- a/modules/globebrowsing/tile/tiledataset.cpp +++ b/modules/globebrowsing/tile/tiledataset.cpp @@ -35,6 +35,11 @@ #include +#include +#include + + + namespace { @@ -44,6 +49,99 @@ namespace { namespace openspace { + void TilePreprocessData::serialize(std::ostream& os) { + os << maxValues.size() << std::endl; + for (float f : maxValues) { + os << f << " "; + } + os << std::endl; + for (float f : minValues) { + os << f << " "; + } + os << std::endl; + } + + + TilePreprocessData TilePreprocessData::deserialize(std::istream& is) { + TilePreprocessData res; + int n; is >> n; + res.maxValues.resize(n); + for (int i = 0; i < n; i++) { + is >> res.maxValues[i]; + } + res.minValues.resize(n); + for (int i = 0; i < n; i++) { + is >> res.minValues[i]; + } + + return std::move(res); + } + + TileIOResult::TileIOResult() + : imageData(nullptr) + , dimensions(0, 0, 0) + , preprocessData(nullptr) + , chunkIndex(0, 0, 0) + , error(CE_None) + , nBytesImageData(0) + { + + } + + + + TileIOResult TileIOResult::createDefaultRes() { + TileIOResult defaultRes; + int w = 8; + int h = 8; + defaultRes.dimensions = glm::uvec3(w, h, 1); + defaultRes.nBytesImageData = w * h * 1 * 3 * 4; // assume max 3 channels, max 4 bytes per pixel + defaultRes.imageData = new char[defaultRes.nBytesImageData]; + std::fill_n((char*)defaultRes.imageData, defaultRes.nBytesImageData, 0); + return std::move(defaultRes); + } + + + + void TileIOResult::serializeMetaData(std::ostream& os) { + os << dimensions.x << " " << dimensions.y << " " << dimensions.z << std::endl; + os << chunkIndex.x << " " << chunkIndex.y << " " << chunkIndex.level << std::endl; + os << error << std::endl; + + // preprocess data + os << (preprocessData != nullptr) << std::endl; + if (preprocessData != nullptr) { + preprocessData->serialize(os); + } + + os << nBytesImageData << std::endl; + } + + + TileIOResult TileIOResult::deserializeMetaData(std::istream& is) { + TileIOResult res; + is >> res.dimensions.x >> res.dimensions.y >> res.dimensions.z; + is >> res.chunkIndex.x >> res.chunkIndex.y >> res.chunkIndex.level; + int err; is >> err; res.error = (CPLErr) err; + + res.preprocessData = nullptr; + bool hasPreprocessData; + is >> hasPreprocessData; + if (hasPreprocessData) { + TilePreprocessData preprocessData = TilePreprocessData::deserialize(is); + res.preprocessData = std::make_shared(preprocessData); + } + + is >> res.nBytesImageData; + + char binaryDataSeparator; + is >> binaryDataSeparator; // not used + + char* buffer = new char[res.nBytesImageData](); + return std::move(res); + } + + // INIT THIS TO FALSE AFTER REMOVED FROM TILEPROVIDER bool TileDataset::GdalHasBeenInitialized = false; @@ -147,6 +245,7 @@ namespace openspace { result->chunkIndex = chunkIndex; result->imageData = getImageDataFlippedY(region, _dataLayout, imageData); result->dimensions = glm::uvec3(region.numPixels, 1); + result->nBytesImageData = _dataLayout.bytesPerPixel * region.numPixels.x * region.numPixels.y; if (_doPreprocessing) { result->preprocessData = preprocess(imageData, region, _dataLayout); } @@ -219,7 +318,7 @@ namespace openspace { } } - return std::shared_ptr < TilePreprocessData>(preprocessData); + return std::shared_ptr(preprocessData); } diff --git a/modules/globebrowsing/tile/tiledataset.h b/modules/globebrowsing/tile/tiledataset.h index f56b8ad34a..d29db929b4 100644 --- a/modules/globebrowsing/tile/tiledataset.h +++ b/modules/globebrowsing/tile/tiledataset.h @@ -32,21 +32,29 @@ #include #include +#include + + #include "gdal_priv.h" #include #include #include +#include namespace openspace { using namespace ghoul::opengl; + using namespace ghoul::filesystem; struct TilePreprocessData { std::vector maxValues; std::vector minValues; + + void serialize(std::ostream& s); + static TilePreprocessData deserialize(std::istream& s); }; struct TextureFormat { @@ -54,16 +62,27 @@ namespace openspace { GLuint glFormat; }; + struct TileIOResult { + TileIOResult(); + void* imageData; glm::uvec3 dimensions; std::shared_ptr preprocessData; ChunkIndex chunkIndex; CPLErr error; + size_t nBytesImageData; + + void serializeMetaData(std::ostream& s); + static TileIOResult deserializeMetaData(std::istream& s); + + static TileIOResult createDefaultRes(); + }; + struct TileDepthTransform { float depthScale; float depthOffset; diff --git a/modules/globebrowsing/tile/tilediskcache.cpp b/modules/globebrowsing/tile/tilediskcache.cpp new file mode 100644 index 0000000000..a8eb6effbb --- /dev/null +++ b/modules/globebrowsing/tile/tilediskcache.cpp @@ -0,0 +1,123 @@ +/***************************************************************************************** +* * +* OpenSpace * +* * +* Copyright (c) 2014-2016 * +* * +* Permission is hereby granted, free of charge, to any person obtaining a copy of this * +* software and associated documentation files (the "Software"), to deal in the Software * +* without restriction, including without limitation the rights to use, copy, modify, * +* merge, publish, distribute, sublicense, and/or sell copies of the Software, and to * +* permit persons to whom the Software is furnished to do so, subject to the following * +* conditions: * +* * +* The above copyright notice and this permission notice shall be included in all copies * +* or substantial portions of the Software. * +* * +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, * +* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A * +* PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF * +* CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE * +* OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * +****************************************************************************************/ + +#include + +#include + +#include +#include +#include + +namespace { + const std::string _loggerCat = "TileDiskCache"; +} + + +namespace openspace { + const std::string TileDiskCache::CACHE_ROOT = "tilecache"; + + + TileDiskCache::TileDiskCache(const std::string& name) + : _name(name) + { + if (!FileSystem::isInitialized()) { + FileSystem::initialize(); + } + + std::string pathToCacheDir = FileSys.pathByAppendingComponent(CACHE_ROOT, name); + Directory cacheDir(pathToCacheDir, Directory::RawPath::No); + if (!FileSys.directoryExists(cacheDir)) { + FileSys.createDirectory(pathToCacheDir, FileSystem::Recursive::Yes); + } + _cacheDir = cacheDir; + } + + bool TileDiskCache::has(const ChunkIndex& chunkIndex) const { + File metaFile = getMetaDataFile(chunkIndex); + return FileSys.fileExists(metaFile); + } + + + std::shared_ptr TileDiskCache::get(const ChunkIndex& chunkIndex) { + File metaDataFile = getMetaDataFile(chunkIndex); + File dataFile = getDataFile(chunkIndex); + if (FileSys.fileExists(metaDataFile) && FileSys.fileExists(dataFile)) { + // read meta + std::ifstream ifsMeta; + ifsMeta.open(metaDataFile.path(), std::ifstream::in); + TileIOResult res = TileIOResult::deserializeMetaData(ifsMeta); + ifsMeta.close(); + + // read data + std::ifstream ifsData; + ifsData.open(dataFile.path(), std::ifstream::binary); + char * buffer = new char[res.nBytesImageData]; + ifsData.read(buffer, res.nBytesImageData); + res.imageData = buffer; + + return std::make_shared(res); + } + return nullptr; + } + + bool TileDiskCache::put(const ChunkIndex& chunkIndex, std::shared_ptr tileIOResult) { + File metaDataFile = getMetaDataFile(chunkIndex); + if (!FileSys.fileExists(metaDataFile)) { + std::ofstream ofsMeta; + ofsMeta.open(metaDataFile.path()); + tileIOResult->serializeMetaData(ofsMeta); + ofsMeta.close(); + + std::ofstream ofsData; + File dataFile = getDataFile(chunkIndex); + ofsData.open(dataFile.path(), std::ofstream::binary); + char * data = (char*)tileIOResult->imageData; + ofsData.write(data, tileIOResult->nBytesImageData); + ofsData.close(); + return true; + } + return false; + } + + std::string TileDiskCache::getFilePath(const ChunkIndex& chunkIndex) const { + std::stringstream ss; + ss << chunkIndex.level; + ss << "_" << chunkIndex.x; + ss << "_" << chunkIndex.y; + std::string filePath = FileSys.pathByAppendingComponent(_cacheDir.path(), ss.str()); + return filePath; + } + + File TileDiskCache::getMetaDataFile(const ChunkIndex& chunkIndex) const { + return File(getFilePath(chunkIndex) + ".meta"); + } + + File TileDiskCache::getDataFile(const ChunkIndex& chunkIndex) const { + return File(getFilePath(chunkIndex) + ".data"); + } + + + +} // namespace openspace diff --git a/modules/globebrowsing/tile/tilediskcache.h b/modules/globebrowsing/tile/tilediskcache.h new file mode 100644 index 0000000000..87ceb3906e --- /dev/null +++ b/modules/globebrowsing/tile/tilediskcache.h @@ -0,0 +1,64 @@ +/***************************************************************************************** +* * +* OpenSpace * +* * +* Copyright (c) 2014-2016 * +* * +* Permission is hereby granted, free of charge, to any person obtaining a copy of this * +* software and associated documentation files (the "Software"), to deal in the Software * +* without restriction, including without limitation the rights to use, copy, modify, * +* merge, publish, distribute, sublicense, and/or sell copies of the Software, and to * +* permit persons to whom the Software is furnished to do so, subject to the following * +* conditions: * +* * +* The above copyright notice and this permission notice shall be included in all copies * +* or substantial portions of the Software. * +* * +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, * +* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A * +* PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF * +* CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE * +* OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * +****************************************************************************************/ + +#ifndef __TILE_DISK_CACHE_H__ +#define __TILE_DISK_CACHE_H__ + + +#include +#include + +#include + + +namespace openspace { + + using namespace ghoul::filesystem; + + class TileDiskCache { + public: + TileDiskCache(const std::string& name); + + std::shared_ptr get(const ChunkIndex& chunkIndex); + bool has(const ChunkIndex& chunkIndex) const; + bool TileDiskCache::put(const ChunkIndex& chunkIndex, std::shared_ptr tileIOResult); + + + static const std::string CACHE_ROOT; + + private: + const std::string _name; + + Directory _cacheDir; + + std::string getFilePath(const ChunkIndex& chunkIndex) const; + File getMetaDataFile(const ChunkIndex& chunkIndex) const; + File getDataFile(const ChunkIndex& chunkIndex) const; + + }; + +} // namespace openspace + + +#endif // __TILE_DISK_CACHE_H__ \ No newline at end of file diff --git a/modules/globebrowsing/tile/tileprovider.cpp b/modules/globebrowsing/tile/tileprovider.cpp index a98a4e21c9..7089a660d7 100644 --- a/modules/globebrowsing/tile/tileprovider.cpp +++ b/modules/globebrowsing/tile/tileprovider.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include @@ -35,6 +36,7 @@ #include #include + #include @@ -56,6 +58,7 @@ namespace openspace { int framesUntilFlushRequestQueue) : _asyncTextureDataProvider(tileReader) , _tileCache(tileCache) + , _framesUntilRequestFlush(framesUntilFlushRequestQueue) , _framesSinceLastRequestFlush(0) {