From 6107e245bc6a1e7e718d5222f7cea80ae390c329 Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Thu, 16 Jun 2016 21:04:39 -0400 Subject: [PATCH 1/6] Implement serialization of TileIOResults --- .../globebrowsing/tile/asynctilereader.cpp | 10 ++ modules/globebrowsing/tile/asynctilereader.h | 23 +++-- modules/globebrowsing/tile/tiledataset.cpp | 96 ++++++++++++++++++- modules/globebrowsing/tile/tiledataset.h | 17 ++++ modules/globebrowsing/tile/tileprovider.cpp | 2 +- 5 files changed, 137 insertions(+), 11 deletions(-) diff --git a/modules/globebrowsing/tile/asynctilereader.cpp b/modules/globebrowsing/tile/asynctilereader.cpp index fcce97d5b9..d2ab2f09a4 100644 --- a/modules/globebrowsing/tile/asynctilereader.cpp +++ b/modules/globebrowsing/tile/asynctilereader.cpp @@ -42,6 +42,16 @@ namespace openspace { + void TileLoadJob::execute() { + _uninitedTexture = _tileDataset->readTileData(_chunkIndex); + } + + void DiskCachedTileLoadJob::execute() { + + } + + + AsyncTileDataProvider::AsyncTileDataProvider( std::shared_ptr tileDataset, std::shared_ptr pool) diff --git a/modules/globebrowsing/tile/asynctilereader.h b/modules/globebrowsing/tile/asynctilereader.h index acf69e8e5e..ed5f059fc2 100644 --- a/modules/globebrowsing/tile/asynctilereader.h +++ b/modules/globebrowsing/tile/asynctilereader.h @@ -40,14 +40,20 @@ #include #include #include +#include namespace openspace { + + struct LoadJob : public Job { + virtual void execute() = 0; + virtual std::shared_ptr product() = 0; + }; + struct TileLoadJob : LoadJob { - struct TileLoadJob : public Job { TileLoadJob(std::shared_ptr textureDataProvider, const ChunkIndex& chunkIndex) : _tileDataset(textureDataProvider) @@ -58,25 +64,25 @@ namespace openspace { virtual ~TileLoadJob() { } - virtual void execute() { - _uninitedTexture = _tileDataset->readTileData(_chunkIndex); - } - - + virtual void execute(); virtual std::shared_ptr product() { return _uninitedTexture; } - private: + protected: + + ChunkIndex _chunkIndex; std::shared_ptr _tileDataset; std::shared_ptr _uninitedTexture; }; - + struct DiskCachedTileLoadJob : public TileLoadJob { + virtual void execute(); + }; class AsyncTileDataProvider { @@ -107,7 +113,6 @@ namespace openspace { ConcurrentJobManager _concurrentJobManager; std::unordered_map _enqueuedTileRequests; - }; diff --git a/modules/globebrowsing/tile/tiledataset.cpp b/modules/globebrowsing/tile/tiledataset.cpp index 64a2389bfb..25f1e14def 100644 --- a/modules/globebrowsing/tile/tiledataset.cpp +++ b/modules/globebrowsing/tile/tiledataset.cpp @@ -34,6 +34,10 @@ #include +#include + + + namespace { @@ -43,6 +47,95 @@ namespace { namespace openspace { + void TilePreprocessData::serialize(std::stringstream& s) { + s << maxValues.size() << std::endl; + for (float f : maxValues) { + s << f << " "; + } + s << std::endl; + for (float f : minValues) { + s << f << " "; + } + s << std::endl; + } + + + TilePreprocessData TilePreprocessData::deserialize(std::stringstream& s) { + TilePreprocessData res; + int n; s >> n; + res.maxValues.resize(n); + for (int i = 0; i < n; i++) { + s >> res.maxValues[i]; + } + res.minValues.resize(n); + for (int i = 0; i < n; i++) { + s >> res.minValues[i]; + } + + return std::move(res); + } + + TileIOResult::TileIOResult() + : imageData(nullptr) + , dimensions(0, 0, 0) + , preprocessData(nullptr) + , chunkIndex(0, 0, 0) + , error(CE_None) + , nBytesImageData(0) + { + + } + + + void TileIOResult::serialize(std::stringstream& s) { + s << dimensions.x << " " << dimensions.y << " " << dimensions.z << std::endl; + s << chunkIndex.x << " " << chunkIndex.y << " " << chunkIndex.level << std::endl; + s << error << std::endl; + + // preprocess data + s << (preprocessData != nullptr) << std::endl; + if (preprocessData != nullptr) { + preprocessData->serialize(s); + } + + s << nBytesImageData << std::endl; + + char binaryDataSeparator = 'Ö'; // sweden represent! + s << binaryDataSeparator; + if (nBytesImageData) { + char* buffer = reinterpret_cast(imageData); + s.write(buffer, nBytesImageData); + } + } + + + TileIOResult TileIOResult::deserialize(std::stringstream& s) { + TileIOResult res; + s >> res.dimensions.x >> res.dimensions.y >> res.dimensions.z; + s >> res.chunkIndex.x >> res.chunkIndex.y >> res.chunkIndex.level; + int err; s >> err; res.error = (CPLErr) err; + + res.preprocessData = nullptr; + bool hasPreprocessData; + s >> hasPreprocessData; + if (hasPreprocessData) { + TilePreprocessData preprocessData = TilePreprocessData::deserialize(s); + res.preprocessData = std::make_shared(preprocessData); + } + + s >> res.nBytesImageData; + + char binaryDataSeparator; + s >> binaryDataSeparator; // not used + + char* buffer = new char[res.nBytesImageData](); + s.read(buffer, res.nBytesImageData); + res.imageData = reinterpret_cast(buffer); + + return std::move(res); + } + + // INIT THIS TO FALSE AFTER REMOVED FROM TILEPROVIDER bool TileDataset::GdalHasBeenInitialized = false; @@ -142,6 +235,7 @@ namespace openspace { result->chunkIndex = chunkIndex; result->imageData = getImageDataFlippedY(region, _dataLayout, imageData); result->dimensions = glm::uvec3(region.numPixels, 1); + result->nBytesImageData = _dataLayout.bytesPerPixel * region.numPixels.x * region.numPixels.y; if (_doPreprocessing) { result->preprocessData = preprocess(imageData, region, _dataLayout); } @@ -214,7 +308,7 @@ namespace openspace { } } - return std::shared_ptr < TilePreprocessData>(preprocessData); + return std::shared_ptr(preprocessData); } diff --git a/modules/globebrowsing/tile/tiledataset.h b/modules/globebrowsing/tile/tiledataset.h index f56b8ad34a..a6a974070a 100644 --- a/modules/globebrowsing/tile/tiledataset.h +++ b/modules/globebrowsing/tile/tiledataset.h @@ -32,21 +32,29 @@ #include #include +#include + + #include "gdal_priv.h" #include #include #include +#include namespace openspace { using namespace ghoul::opengl; + using namespace ghoul::filesystem; struct TilePreprocessData { std::vector maxValues; std::vector minValues; + + void serialize(std::stringstream& s); + static TilePreprocessData deserialize(std::stringstream& s); }; struct TextureFormat { @@ -54,16 +62,25 @@ namespace openspace { GLuint glFormat; }; + struct TileIOResult { + TileIOResult(); + void* imageData; glm::uvec3 dimensions; std::shared_ptr preprocessData; ChunkIndex chunkIndex; CPLErr error; + size_t nBytesImageData; + + void serialize(std::stringstream& s); + static TileIOResult deserialize(std::stringstream& s); + }; + struct TileDepthTransform { float depthScale; float depthOffset; diff --git a/modules/globebrowsing/tile/tileprovider.cpp b/modules/globebrowsing/tile/tileprovider.cpp index a98a4e21c9..4a97517136 100644 --- a/modules/globebrowsing/tile/tileprovider.cpp +++ b/modules/globebrowsing/tile/tileprovider.cpp @@ -144,7 +144,7 @@ namespace openspace { } - void CachingTileProvider::initializeAndAddToCache(std::shared_ptr tileIOResult) { + void CachingTileProvider::initializeAndAddToCache(std::shared_ptr tileIOResult) { HashKey key = tileIOResult->chunkIndex.hashKey(); TileDataset::DataLayout dataLayout = _asyncTextureDataProvider->getTextureDataProvider()->getDataLayout(); Texture* texturePtr = new Texture( From ef228c7a5ef135235b1710321b1bbe451a8f5f75 Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Fri, 17 Jun 2016 00:50:20 -0400 Subject: [PATCH 2/6] Add basic class for disc caching of tile data --- modules/globebrowsing/CMakeLists.txt | 2 + modules/globebrowsing/tile/tiledataset.cpp | 65 +++++----- modules/globebrowsing/tile/tiledataset.h | 8 +- modules/globebrowsing/tile/tilediskcache.cpp | 123 +++++++++++++++++++ modules/globebrowsing/tile/tilediskcache.h | 64 ++++++++++ modules/globebrowsing/tile/tileprovider.cpp | 4 +- 6 files changed, 231 insertions(+), 35 deletions(-) create mode 100644 modules/globebrowsing/tile/tilediskcache.cpp create mode 100644 modules/globebrowsing/tile/tilediskcache.h diff --git a/modules/globebrowsing/CMakeLists.txt b/modules/globebrowsing/CMakeLists.txt index 7f080eb79e..1c032afb0b 100644 --- a/modules/globebrowsing/CMakeLists.txt +++ b/modules/globebrowsing/CMakeLists.txt @@ -51,6 +51,7 @@ set(HEADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/tile/temporaltileprovider.h ${CMAKE_CURRENT_SOURCE_DIR}/tile/tileprovider.h ${CMAKE_CURRENT_SOURCE_DIR}/tile/tileselector.h + ${CMAKE_CURRENT_SOURCE_DIR}/tile/tilediskcache.h ${CMAKE_CURRENT_SOURCE_DIR}/tile/tiledataset.h ${CMAKE_CURRENT_SOURCE_DIR}/tile/asynctilereader.h ${CMAKE_CURRENT_SOURCE_DIR}/tile/tileprovidermanager.h @@ -92,6 +93,7 @@ set(SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/tile/temporaltileprovider.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tile/tileprovider.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tile/tileselector.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tile/tilediskcache.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tile/tiledataset.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tile/asynctilereader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tile/tileprovidermanager.cpp diff --git a/modules/globebrowsing/tile/tiledataset.cpp b/modules/globebrowsing/tile/tiledataset.cpp index 25f1e14def..bea96c9c46 100644 --- a/modules/globebrowsing/tile/tiledataset.cpp +++ b/modules/globebrowsing/tile/tiledataset.cpp @@ -47,29 +47,29 @@ namespace { namespace openspace { - void TilePreprocessData::serialize(std::stringstream& s) { - s << maxValues.size() << std::endl; + void TilePreprocessData::serialize(std::ostream& os) { + os << maxValues.size() << std::endl; for (float f : maxValues) { - s << f << " "; + os << f << " "; } - s << std::endl; + os << std::endl; for (float f : minValues) { - s << f << " "; + os << f << " "; } - s << std::endl; + os << std::endl; } - TilePreprocessData TilePreprocessData::deserialize(std::stringstream& s) { + TilePreprocessData TilePreprocessData::deserialize(std::istream& is) { TilePreprocessData res; - int n; s >> n; + int n; is >> n; res.maxValues.resize(n); for (int i = 0; i < n; i++) { - s >> res.maxValues[i]; + is >> res.maxValues[i]; } res.minValues.resize(n); for (int i = 0; i < n; i++) { - s >> res.minValues[i]; + is >> res.minValues[i]; } return std::move(res); @@ -87,51 +87,56 @@ namespace openspace { } - void TileIOResult::serialize(std::stringstream& s) { - s << dimensions.x << " " << dimensions.y << " " << dimensions.z << std::endl; - s << chunkIndex.x << " " << chunkIndex.y << " " << chunkIndex.level << std::endl; - s << error << std::endl; + void TileIOResult::serialize(std::ostream& os) { + os << dimensions.x << " " << dimensions.y << " " << dimensions.z << std::endl; + os << chunkIndex.x << " " << chunkIndex.y << " " << chunkIndex.level << std::endl; + os << error << std::endl; // preprocess data - s << (preprocessData != nullptr) << std::endl; + os << (preprocessData != nullptr) << std::endl; if (preprocessData != nullptr) { - preprocessData->serialize(s); + preprocessData->serialize(os); } - s << nBytesImageData << std::endl; - + os << nBytesImageData << std::endl; + /* char binaryDataSeparator = 'Ö'; // sweden represent! - s << binaryDataSeparator; + os << binaryDataSeparator; + if (nBytesImageData) { char* buffer = reinterpret_cast(imageData); - s.write(buffer, nBytesImageData); + os.write(buffer, nBytesImageData); } + */ } - TileIOResult TileIOResult::deserialize(std::stringstream& s) { + TileIOResult TileIOResult::deserialize(std::istream& is) { TileIOResult res; - s >> res.dimensions.x >> res.dimensions.y >> res.dimensions.z; - s >> res.chunkIndex.x >> res.chunkIndex.y >> res.chunkIndex.level; - int err; s >> err; res.error = (CPLErr) err; + is >> res.dimensions.x >> res.dimensions.y >> res.dimensions.z; + is >> res.chunkIndex.x >> res.chunkIndex.y >> res.chunkIndex.level; + int err; is >> err; res.error = (CPLErr) err; res.preprocessData = nullptr; bool hasPreprocessData; - s >> hasPreprocessData; + is >> hasPreprocessData; if (hasPreprocessData) { - TilePreprocessData preprocessData = TilePreprocessData::deserialize(s); + TilePreprocessData preprocessData = TilePreprocessData::deserialize(is); res.preprocessData = std::make_shared(preprocessData); } - s >> res.nBytesImageData; + is >> res.nBytesImageData; char binaryDataSeparator; - s >> binaryDataSeparator; // not used + is >> binaryDataSeparator; // not used char* buffer = new char[res.nBytesImageData](); - s.read(buffer, res.nBytesImageData); + /*is.read(buffer, res.nBytesImageData); + for (size_t i = 0; i < res.nBytesImageData; i++){ + is.get(buffer[i]); + } res.imageData = reinterpret_cast(buffer); - + */ return std::move(res); } diff --git a/modules/globebrowsing/tile/tiledataset.h b/modules/globebrowsing/tile/tiledataset.h index a6a974070a..187785bb6d 100644 --- a/modules/globebrowsing/tile/tiledataset.h +++ b/modules/globebrowsing/tile/tiledataset.h @@ -53,8 +53,8 @@ namespace openspace { std::vector maxValues; std::vector minValues; - void serialize(std::stringstream& s); - static TilePreprocessData deserialize(std::stringstream& s); + void serialize(std::ostream& s); + static TilePreprocessData deserialize(std::istream& s); }; struct TextureFormat { @@ -73,8 +73,8 @@ namespace openspace { CPLErr error; size_t nBytesImageData; - void serialize(std::stringstream& s); - static TileIOResult deserialize(std::stringstream& s); + void serialize(std::ostream& s); + static TileIOResult deserialize(std::istream& s); }; diff --git a/modules/globebrowsing/tile/tilediskcache.cpp b/modules/globebrowsing/tile/tilediskcache.cpp new file mode 100644 index 0000000000..df75d2884b --- /dev/null +++ b/modules/globebrowsing/tile/tilediskcache.cpp @@ -0,0 +1,123 @@ +/***************************************************************************************** +* * +* OpenSpace * +* * +* Copyright (c) 2014-2016 * +* * +* Permission is hereby granted, free of charge, to any person obtaining a copy of this * +* software and associated documentation files (the "Software"), to deal in the Software * +* without restriction, including without limitation the rights to use, copy, modify, * +* merge, publish, distribute, sublicense, and/or sell copies of the Software, and to * +* permit persons to whom the Software is furnished to do so, subject to the following * +* conditions: * +* * +* The above copyright notice and this permission notice shall be included in all copies * +* or substantial portions of the Software. * +* * +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, * +* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A * +* PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF * +* CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE * +* OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * +****************************************************************************************/ + +#include + +#include + +#include +#include +#include + +namespace { + const std::string _loggerCat = "TileDiskCache"; +} + + +namespace openspace { + const std::string TileDiskCache::CACHE_ROOT = "tilecache"; + + + TileDiskCache::TileDiskCache(const std::string& name) + : _name(name) + { + if (!FileSystem::isInitialized()) { + FileSystem::initialize(); + } + + std::string pathToCacheDir = FileSys.pathByAppendingComponent(CACHE_ROOT, name); + Directory cacheDir(pathToCacheDir, Directory::RawPath::No); + if (!FileSys.directoryExists(cacheDir)) { + FileSys.createDirectory(pathToCacheDir, FileSystem::Recursive::Yes); + } + _cacheDir = cacheDir; + } + + bool TileDiskCache::has(const ChunkIndex& chunkIndex) const { + File metaFile = getMetaFile(chunkIndex); + return FileSys.fileExists(metaFile); + } + + + std::shared_ptr TileDiskCache::get(const ChunkIndex& chunkIndex) { + File metaFile = getMetaFile(chunkIndex); + File dataFile = getDataFile(chunkIndex); + if (FileSys.fileExists(metaFile) && FileSys.fileExists(dataFile)) { + // read meta + std::ifstream ifsMeta; + ifsMeta.open(metaFile.path(), std::ifstream::in); + TileIOResult res = TileIOResult::deserialize(ifsMeta); + ifsMeta.close(); + + // read data + std::ifstream ifsData; + ifsData.open(dataFile.path(), std::ifstream::binary); + char * buffer = new char[res.nBytesImageData]; + ifsData.read(buffer, res.nBytesImageData); + res.imageData = buffer; + + return std::make_shared(res); + } + return nullptr; + } + + bool TileDiskCache::put(const ChunkIndex& chunkIndex, std::shared_ptr tileIOResult) { + File metaFile = getMetaFile(chunkIndex); + if (!FileSys.fileExists(metaFile)) { + std::ofstream ofsMeta; + ofsMeta.open(metaFile.path()); + tileIOResult->serialize(ofsMeta); + ofsMeta.close(); + + std::ofstream ofsData; + File dataFile = getDataFile(chunkIndex); + ofsData.open(dataFile.path(), std::ofstream::binary); + char * data = (char*)tileIOResult->imageData; + ofsData.write(data, tileIOResult->nBytesImageData); + ofsData.close(); + return true; + } + return false; + } + + std::string TileDiskCache::getFilePath(const ChunkIndex& chunkIndex) const { + std::stringstream ss; + ss << chunkIndex.level; + ss << "_" << chunkIndex.x; + ss << "_" << chunkIndex.y; + std::string filePath = FileSys.pathByAppendingComponent(_cacheDir.path(), ss.str()); + return filePath; + } + + File TileDiskCache::getMetaFile(const ChunkIndex& chunkIndex) const { + return File(getFilePath(chunkIndex) + ".meta"); + } + + File TileDiskCache::getDataFile(const ChunkIndex& chunkIndex) const { + return File(getFilePath(chunkIndex) + ".data"); + } + + + +} // namespace openspace diff --git a/modules/globebrowsing/tile/tilediskcache.h b/modules/globebrowsing/tile/tilediskcache.h new file mode 100644 index 0000000000..30cf72124f --- /dev/null +++ b/modules/globebrowsing/tile/tilediskcache.h @@ -0,0 +1,64 @@ +/***************************************************************************************** +* * +* OpenSpace * +* * +* Copyright (c) 2014-2016 * +* * +* Permission is hereby granted, free of charge, to any person obtaining a copy of this * +* software and associated documentation files (the "Software"), to deal in the Software * +* without restriction, including without limitation the rights to use, copy, modify, * +* merge, publish, distribute, sublicense, and/or sell copies of the Software, and to * +* permit persons to whom the Software is furnished to do so, subject to the following * +* conditions: * +* * +* The above copyright notice and this permission notice shall be included in all copies * +* or substantial portions of the Software. * +* * +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, * +* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A * +* PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF * +* CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE * +* OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * +****************************************************************************************/ + +#ifndef __TILE_DISK_CACHE_H__ +#define __TILE_DISK_CACHE_H__ + + +#include +#include + +#include + + +namespace openspace { + + using namespace ghoul::filesystem; + + class TileDiskCache { + public: + TileDiskCache(const std::string& name); + + std::shared_ptr get(const ChunkIndex& chunkIndex); + bool has(const ChunkIndex& chunkIndex) const; + bool TileDiskCache::put(const ChunkIndex& chunkIndex, std::shared_ptr tileIOResult); + + + static const std::string CACHE_ROOT; + + private: + const std::string _name; + + Directory _cacheDir; + + std::string getFilePath(const ChunkIndex& chunkIndex) const; + File getMetaFile(const ChunkIndex& chunkIndex) const; + File getDataFile(const ChunkIndex& chunkIndex) const; + + }; + +} // namespace openspace + + +#endif // __TILE_DISK_CACHE_H__ \ No newline at end of file diff --git a/modules/globebrowsing/tile/tileprovider.cpp b/modules/globebrowsing/tile/tileprovider.cpp index 4a97517136..a877b97ceb 100644 --- a/modules/globebrowsing/tile/tileprovider.cpp +++ b/modules/globebrowsing/tile/tileprovider.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include @@ -35,6 +36,7 @@ #include #include + #include @@ -144,7 +146,7 @@ namespace openspace { } - void CachingTileProvider::initializeAndAddToCache(std::shared_ptr tileIOResult) { + void CachingTileProvider::initializeAndAddToCache(std::shared_ptr tileIOResult) { HashKey key = tileIOResult->chunkIndex.hashKey(); TileDataset::DataLayout dataLayout = _asyncTextureDataProvider->getTextureDataProvider()->getDataLayout(); Texture* texturePtr = new Texture( From 553131c40788524369fb11a81e8e52e4598c208d Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Fri, 17 Jun 2016 15:55:00 -0400 Subject: [PATCH 3/6] Enabling a simple black default tileIOResult for debugging --- .../globebrowsing/tile/asynctilereader.cpp | 68 +++++++++++++++++-- modules/globebrowsing/tile/asynctilereader.h | 40 +++++++++-- modules/globebrowsing/tile/tiledataset.cpp | 15 ++++ modules/globebrowsing/tile/tiledataset.h | 2 + 4 files changed, 117 insertions(+), 8 deletions(-) diff --git a/modules/globebrowsing/tile/asynctilereader.cpp b/modules/globebrowsing/tile/asynctilereader.cpp index d2ab2f09a4..a352618238 100644 --- a/modules/globebrowsing/tile/asynctilereader.cpp +++ b/modules/globebrowsing/tile/asynctilereader.cpp @@ -29,6 +29,8 @@ #include #include +#include + #include @@ -41,13 +43,71 @@ namespace { namespace openspace { + TileLoadJob::TileLoadJob(const TileLoadJob& o) + : _chunkIndex(o._chunkIndex) + , _tileDataset(o._tileDataset) + , _tileIOResult(o._tileIOResult) + { + + } void TileLoadJob::execute() { - _uninitedTexture = _tileDataset->readTileData(_chunkIndex); + _tileIOResult = _tileDataset->readTileData(_chunkIndex); + } + + + + + DiskCachedTileLoadJob::DiskCachedTileLoadJob(std::shared_ptr textureDataProvider, + const ChunkIndex& chunkIndex, std::shared_ptr tdc, CacheMode m) + : _tileDataset(textureDataProvider) + , _chunkIndex(chunkIndex) + , _tileDiskCache(tdc) + , _mode(m) + { + + } + + DiskCachedTileLoadJob::DiskCachedTileLoadJob(std::shared_ptr textureDataProvider, + const ChunkIndex& chunkIndex, std::shared_ptr tdc, const std::string cacheMode) + : _tileDataset(textureDataProvider) + , _chunkIndex(chunkIndex) + , _tileDiskCache(tdc) + { + if (cacheMode == "Disabled") _mode = CacheMode::Disabled; + else if (cacheMode == "ReadOnly") _mode = CacheMode::ReadOnly; + else if (cacheMode == "ReadAndWrite") _mode = CacheMode::ReadAndWrite; + else if (cacheMode == "WriteOnly") _mode = CacheMode::WriteOnly; } void DiskCachedTileLoadJob::execute() { + _tileIOResult = nullptr; + switch (_mode) { + case CacheMode::Disabled: + _tileIOResult = _tileDataset->readTileData(_chunkIndex); + break; + + case CacheMode::ReadOnly: + _tileIOResult = _tileDiskCache->get(_chunkIndex); + if (_tileIOResult == nullptr) { + _tileIOResult = _tileDataset->readTileData(_chunkIndex); + } + break; + + case CacheMode::ReadAndWrite: + _tileIOResult = _tileDiskCache->get(_chunkIndex); + if (_tileIOResult == nullptr) { + _tileIOResult = _tileDataset->readTileData(_chunkIndex); + _tileDiskCache->put(_chunkIndex, _tileIOResult); + } + break; + + case CacheMode::WriteOnly: + _tileIOResult = _tileDataset->readTileData(_chunkIndex); + _tileDiskCache->put(_chunkIndex, _tileIOResult); + break; + } } @@ -71,10 +131,10 @@ namespace openspace { } bool AsyncTileDataProvider::enqueueTextureData(const ChunkIndex& chunkIndex) { + auto tileDiskCache = std::make_shared("test"); if (satisfiesEnqueueCriteria(chunkIndex)) { - std::shared_ptr job = std::shared_ptr( - new TileLoadJob(_tileDataset, chunkIndex)); - + auto job = std::make_shared(_tileDataset, chunkIndex); + //auto job = std::make_shared(_tileDataset, chunkIndex, tileDiskCache, "ReadOnly"); _concurrentJobManager.enqueueJob(job); _enqueuedTileRequests[chunkIndex.hashKey()] = chunkIndex; return true; diff --git a/modules/globebrowsing/tile/asynctilereader.h b/modules/globebrowsing/tile/asynctilereader.h index ed5f059fc2..23fe99059f 100644 --- a/modules/globebrowsing/tile/asynctilereader.h +++ b/modules/globebrowsing/tile/asynctilereader.h @@ -37,6 +37,7 @@ #include + #include #include #include @@ -54,6 +55,8 @@ namespace openspace { struct TileLoadJob : LoadJob { + TileLoadJob(const TileLoadJob&); + TileLoadJob(std::shared_ptr textureDataProvider, const ChunkIndex& chunkIndex) : _tileDataset(textureDataProvider) @@ -67,21 +70,50 @@ namespace openspace { virtual void execute(); virtual std::shared_ptr product() { - return _uninitedTexture; + return _tileIOResult; } protected: - ChunkIndex _chunkIndex; std::shared_ptr _tileDataset; - std::shared_ptr _uninitedTexture; + std::shared_ptr _tileIOResult; }; - struct DiskCachedTileLoadJob : public TileLoadJob { + class TileDiskCache; + + struct DiskCachedTileLoadJob : public LoadJob { + enum CacheMode { + Disabled, + ReadOnly, + ReadAndWrite, + WriteOnly + }; + + DiskCachedTileLoadJob(std::shared_ptr textureDataProvider, + const ChunkIndex& chunkIndex, std::shared_ptr tdc, + const std::string cacheMode); + + DiskCachedTileLoadJob(std::shared_ptr textureDataProvider, + const ChunkIndex& chunkIndex, std::shared_ptr tdc, + CacheMode cacheMode = CacheMode::ReadOnly); + virtual void execute(); + + virtual std::shared_ptr product() { + return _tileIOResult; + } + + protected: + + ChunkIndex _chunkIndex; + std::shared_ptr _tileDataset; + std::shared_ptr _tileIOResult; + std::shared_ptr _tileDiskCache; + CacheMode _mode; + }; diff --git a/modules/globebrowsing/tile/tiledataset.cpp b/modules/globebrowsing/tile/tiledataset.cpp index bea96c9c46..808abe0dd4 100644 --- a/modules/globebrowsing/tile/tiledataset.cpp +++ b/modules/globebrowsing/tile/tiledataset.cpp @@ -35,6 +35,7 @@ #include #include +#include @@ -87,6 +88,20 @@ namespace openspace { } + + TileIOResult TileIOResult::createDefaultRes() { + TileIOResult defaultRes; + int w = 8; + int h = 8; + defaultRes.dimensions = glm::uvec3(w, h, 1); + defaultRes.nBytesImageData = w * h * 1 * 3 * 4; // assume max 3 channels, max 4 bytes per pixel + defaultRes.imageData = new char[defaultRes.nBytesImageData]; + std::fill_n((char*)defaultRes.imageData, defaultRes.nBytesImageData, 0); + return std::move(defaultRes); + } + + + void TileIOResult::serialize(std::ostream& os) { os << dimensions.x << " " << dimensions.y << " " << dimensions.z << std::endl; os << chunkIndex.x << " " << chunkIndex.y << " " << chunkIndex.level << std::endl; diff --git a/modules/globebrowsing/tile/tiledataset.h b/modules/globebrowsing/tile/tiledataset.h index 187785bb6d..8db37f67c4 100644 --- a/modules/globebrowsing/tile/tiledataset.h +++ b/modules/globebrowsing/tile/tiledataset.h @@ -75,6 +75,8 @@ namespace openspace { void serialize(std::ostream& s); static TileIOResult deserialize(std::istream& s); + + static TileIOResult createDefaultRes(); }; From a5aa5ec1a8b75456b2f904324ad2735b0e639a6d Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Fri, 17 Jun 2016 15:59:08 -0400 Subject: [PATCH 4/6] Add mode CacheHitOnly for DiskCachedLoadJob and go back to use old TileLoadJob --- .../globebrowsing/globes/renderableglobe.cpp | 2 +- .../globebrowsing/tile/asynctilereader.cpp | 33 ++++++++++--------- modules/globebrowsing/tile/asynctilereader.h | 19 +++++------ 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/modules/globebrowsing/globes/renderableglobe.cpp b/modules/globebrowsing/globes/renderableglobe.cpp index 5ce09278b3..159f9828c3 100644 --- a/modules/globebrowsing/globes/renderableglobe.cpp +++ b/modules/globebrowsing/globes/renderableglobe.cpp @@ -59,7 +59,7 @@ namespace openspace { , doFrustumCulling(properties::BoolProperty("doFrustumCulling", "doFrustumCulling")) , doHorizonCulling(properties::BoolProperty("doHorizonCulling", "doHorizonCulling")) , mergeInvisible(properties::BoolProperty("mergeInvisible", "mergeInvisible", true)) - , lodScaleFactor(properties::FloatProperty("lodScaleFactor", "lodScaleFactor", 5.0f, 1.0f, 20.0f)) + , lodScaleFactor(properties::FloatProperty("lodScaleFactor", "lodScaleFactor", 5.0f, 1.0f, 50.0f)) , initChunkVisible(properties::BoolProperty("initChunkVisible", "initChunkVisible", true)) , renderSmallChunksFirst(properties::BoolProperty("renderSmallChunksFirst", "renderSmallChunksFirst", true)) , chunkHeight(properties::FloatProperty("chunkHeight", "chunkHeight", 8700.0f, 0.0f, 8700.0f)) diff --git a/modules/globebrowsing/tile/asynctilereader.cpp b/modules/globebrowsing/tile/asynctilereader.cpp index a352618238..8408197579 100644 --- a/modules/globebrowsing/tile/asynctilereader.cpp +++ b/modules/globebrowsing/tile/asynctilereader.cpp @@ -42,15 +42,6 @@ namespace { namespace openspace { - - TileLoadJob::TileLoadJob(const TileLoadJob& o) - : _chunkIndex(o._chunkIndex) - , _tileDataset(o._tileDataset) - , _tileIOResult(o._tileIOResult) - { - - } - void TileLoadJob::execute() { _tileIOResult = _tileDataset->readTileData(_chunkIndex); } @@ -60,8 +51,7 @@ namespace openspace { DiskCachedTileLoadJob::DiskCachedTileLoadJob(std::shared_ptr textureDataProvider, const ChunkIndex& chunkIndex, std::shared_ptr tdc, CacheMode m) - : _tileDataset(textureDataProvider) - , _chunkIndex(chunkIndex) + : TileLoadJob(textureDataProvider, chunkIndex) , _tileDiskCache(tdc) , _mode(m) { @@ -70,16 +60,18 @@ namespace openspace { DiskCachedTileLoadJob::DiskCachedTileLoadJob(std::shared_ptr textureDataProvider, const ChunkIndex& chunkIndex, std::shared_ptr tdc, const std::string cacheMode) - : _tileDataset(textureDataProvider) - , _chunkIndex(chunkIndex) + : TileLoadJob(textureDataProvider, chunkIndex) , _tileDiskCache(tdc) { if (cacheMode == "Disabled") _mode = CacheMode::Disabled; else if (cacheMode == "ReadOnly") _mode = CacheMode::ReadOnly; else if (cacheMode == "ReadAndWrite") _mode = CacheMode::ReadAndWrite; else if (cacheMode == "WriteOnly") _mode = CacheMode::WriteOnly; + else if (cacheMode == "CacheHitsOnly") _mode = CacheMode::CacheHitsOnly; } + + void DiskCachedTileLoadJob::execute() { _tileIOResult = nullptr; @@ -107,7 +99,18 @@ namespace openspace { _tileIOResult = _tileDataset->readTileData(_chunkIndex); _tileDiskCache->put(_chunkIndex, _tileIOResult); break; + + case CacheMode::CacheHitsOnly: + _tileIOResult = _tileDiskCache->get(_chunkIndex); + if (_tileIOResult == nullptr) { + TileIOResult res = TileIOResult::createDefaultRes(); + res.chunkIndex = _chunkIndex; + _tileIOResult = std::make_shared(res); + } + break; } + + } @@ -131,10 +134,10 @@ namespace openspace { } bool AsyncTileDataProvider::enqueueTextureData(const ChunkIndex& chunkIndex) { - auto tileDiskCache = std::make_shared("test"); + //auto tileDiskCache = std::make_shared("test"); if (satisfiesEnqueueCriteria(chunkIndex)) { auto job = std::make_shared(_tileDataset, chunkIndex); - //auto job = std::make_shared(_tileDataset, chunkIndex, tileDiskCache, "ReadOnly"); + //auto job = std::make_shared(_tileDataset, chunkIndex, tileDiskCache, "ReadAndWrite"); _concurrentJobManager.enqueueJob(job); _enqueuedTileRequests[chunkIndex.hashKey()] = chunkIndex; return true; diff --git a/modules/globebrowsing/tile/asynctilereader.h b/modules/globebrowsing/tile/asynctilereader.h index 23fe99059f..c44f51f7f2 100644 --- a/modules/globebrowsing/tile/asynctilereader.h +++ b/modules/globebrowsing/tile/asynctilereader.h @@ -55,8 +55,7 @@ namespace openspace { struct TileLoadJob : LoadJob { - TileLoadJob(const TileLoadJob&); - + TileLoadJob(std::shared_ptr textureDataProvider, const ChunkIndex& chunkIndex) : _tileDataset(textureDataProvider) @@ -82,14 +81,17 @@ namespace openspace { }; + + class TileDiskCache; - struct DiskCachedTileLoadJob : public LoadJob { + struct DiskCachedTileLoadJob : public TileLoadJob { enum CacheMode { Disabled, ReadOnly, ReadAndWrite, - WriteOnly + WriteOnly, + CacheHitsOnly, }; DiskCachedTileLoadJob(std::shared_ptr textureDataProvider, @@ -102,21 +104,16 @@ namespace openspace { virtual void execute(); - virtual std::shared_ptr product() { - return _tileIOResult; - } - protected: - ChunkIndex _chunkIndex; - std::shared_ptr _tileDataset; - std::shared_ptr _tileIOResult; std::shared_ptr _tileDiskCache; CacheMode _mode; }; + + class AsyncTileDataProvider { public: From df2b68042e4feaba9f0715cce6fb7fbf409bda1e Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Mon, 20 Jun 2016 11:13:47 -0400 Subject: [PATCH 5/6] Better variable names in DiskCache --- modules/globebrowsing/tile/tiledataset.cpp | 19 ++----------------- modules/globebrowsing/tile/tiledataset.h | 4 ++-- modules/globebrowsing/tile/tilediskcache.cpp | 20 ++++++++++---------- modules/globebrowsing/tile/tilediskcache.h | 2 +- 4 files changed, 15 insertions(+), 30 deletions(-) diff --git a/modules/globebrowsing/tile/tiledataset.cpp b/modules/globebrowsing/tile/tiledataset.cpp index 5c3a75699d..d473df92c2 100644 --- a/modules/globebrowsing/tile/tiledataset.cpp +++ b/modules/globebrowsing/tile/tiledataset.cpp @@ -102,7 +102,7 @@ namespace openspace { - void TileIOResult::serialize(std::ostream& os) { + void TileIOResult::serializeMetaData(std::ostream& os) { os << dimensions.x << " " << dimensions.y << " " << dimensions.z << std::endl; os << chunkIndex.x << " " << chunkIndex.y << " " << chunkIndex.level << std::endl; os << error << std::endl; @@ -114,19 +114,10 @@ namespace openspace { } os << nBytesImageData << std::endl; - /* - char binaryDataSeparator = 'Ö'; // sweden represent! - os << binaryDataSeparator; - - if (nBytesImageData) { - char* buffer = reinterpret_cast(imageData); - os.write(buffer, nBytesImageData); - } - */ } - TileIOResult TileIOResult::deserialize(std::istream& is) { + TileIOResult TileIOResult::deserializeMetaData(std::istream& is) { TileIOResult res; is >> res.dimensions.x >> res.dimensions.y >> res.dimensions.z; is >> res.chunkIndex.x >> res.chunkIndex.y >> res.chunkIndex.level; @@ -146,12 +137,6 @@ namespace openspace { is >> binaryDataSeparator; // not used char* buffer = new char[res.nBytesImageData](); - /*is.read(buffer, res.nBytesImageData); - for (size_t i = 0; i < res.nBytesImageData; i++){ - is.get(buffer[i]); - } - res.imageData = reinterpret_cast(buffer); - */ return std::move(res); } diff --git a/modules/globebrowsing/tile/tiledataset.h b/modules/globebrowsing/tile/tiledataset.h index 8db37f67c4..d29db929b4 100644 --- a/modules/globebrowsing/tile/tiledataset.h +++ b/modules/globebrowsing/tile/tiledataset.h @@ -73,8 +73,8 @@ namespace openspace { CPLErr error; size_t nBytesImageData; - void serialize(std::ostream& s); - static TileIOResult deserialize(std::istream& s); + void serializeMetaData(std::ostream& s); + static TileIOResult deserializeMetaData(std::istream& s); static TileIOResult createDefaultRes(); diff --git a/modules/globebrowsing/tile/tilediskcache.cpp b/modules/globebrowsing/tile/tilediskcache.cpp index df75d2884b..a8eb6effbb 100644 --- a/modules/globebrowsing/tile/tilediskcache.cpp +++ b/modules/globebrowsing/tile/tilediskcache.cpp @@ -55,19 +55,19 @@ namespace openspace { } bool TileDiskCache::has(const ChunkIndex& chunkIndex) const { - File metaFile = getMetaFile(chunkIndex); + File metaFile = getMetaDataFile(chunkIndex); return FileSys.fileExists(metaFile); } std::shared_ptr TileDiskCache::get(const ChunkIndex& chunkIndex) { - File metaFile = getMetaFile(chunkIndex); + File metaDataFile = getMetaDataFile(chunkIndex); File dataFile = getDataFile(chunkIndex); - if (FileSys.fileExists(metaFile) && FileSys.fileExists(dataFile)) { + if (FileSys.fileExists(metaDataFile) && FileSys.fileExists(dataFile)) { // read meta std::ifstream ifsMeta; - ifsMeta.open(metaFile.path(), std::ifstream::in); - TileIOResult res = TileIOResult::deserialize(ifsMeta); + ifsMeta.open(metaDataFile.path(), std::ifstream::in); + TileIOResult res = TileIOResult::deserializeMetaData(ifsMeta); ifsMeta.close(); // read data @@ -83,11 +83,11 @@ namespace openspace { } bool TileDiskCache::put(const ChunkIndex& chunkIndex, std::shared_ptr tileIOResult) { - File metaFile = getMetaFile(chunkIndex); - if (!FileSys.fileExists(metaFile)) { + File metaDataFile = getMetaDataFile(chunkIndex); + if (!FileSys.fileExists(metaDataFile)) { std::ofstream ofsMeta; - ofsMeta.open(metaFile.path()); - tileIOResult->serialize(ofsMeta); + ofsMeta.open(metaDataFile.path()); + tileIOResult->serializeMetaData(ofsMeta); ofsMeta.close(); std::ofstream ofsData; @@ -110,7 +110,7 @@ namespace openspace { return filePath; } - File TileDiskCache::getMetaFile(const ChunkIndex& chunkIndex) const { + File TileDiskCache::getMetaDataFile(const ChunkIndex& chunkIndex) const { return File(getFilePath(chunkIndex) + ".meta"); } diff --git a/modules/globebrowsing/tile/tilediskcache.h b/modules/globebrowsing/tile/tilediskcache.h index 30cf72124f..87ceb3906e 100644 --- a/modules/globebrowsing/tile/tilediskcache.h +++ b/modules/globebrowsing/tile/tilediskcache.h @@ -53,7 +53,7 @@ namespace openspace { Directory _cacheDir; std::string getFilePath(const ChunkIndex& chunkIndex) const; - File getMetaFile(const ChunkIndex& chunkIndex) const; + File getMetaDataFile(const ChunkIndex& chunkIndex) const; File getDataFile(const ChunkIndex& chunkIndex) const; }; From 8075c6e449a74d4a152c9aa6f5f76c7beeab3dbf Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Mon, 20 Jun 2016 12:06:28 -0400 Subject: [PATCH 6/6] Fix bug that caused tileRequestQueue to not get flushed correctly --- modules/globebrowsing/tile/tileprovider.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/globebrowsing/tile/tileprovider.cpp b/modules/globebrowsing/tile/tileprovider.cpp index a877b97ceb..7089a660d7 100644 --- a/modules/globebrowsing/tile/tileprovider.cpp +++ b/modules/globebrowsing/tile/tileprovider.cpp @@ -58,6 +58,7 @@ namespace openspace { int framesUntilFlushRequestQueue) : _asyncTextureDataProvider(tileReader) , _tileCache(tileCache) + , _framesUntilRequestFlush(framesUntilFlushRequestQueue) , _framesSinceLastRequestFlush(0) {