From 1615b921623e3972d1b96fdd4f6348ec997e09f4 Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Wed, 18 May 2016 10:53:56 -0400 Subject: [PATCH 1/9] Changed gdaldataconverter to TextureDataProvider. Clean up public interface and added skeleton interface for async data fetching --- modules/globebrowsing/CMakeLists.txt | 4 +- ...ldataconverter.h => texturedataprovider.h} | 76 ++++++++++++++----- ...aconverter.inl => texturedataprovider.inl} | 63 +++++++++++---- .../globebrowsing/other/texturetileset.cpp | 4 +- modules/globebrowsing/other/tileprovider.cpp | 24 +++--- modules/globebrowsing/other/tileprovider.h | 38 ++++++---- .../shaders/globalchunkedlodpatch_fs.glsl | 4 +- .../shaders/localchunkedlodpatch_fs.glsl | 2 +- 8 files changed, 148 insertions(+), 67 deletions(-) rename modules/globebrowsing/other/{gdaldataconverter.h => texturedataprovider.h} (64%) rename modules/globebrowsing/other/{gdaldataconverter.inl => texturedataprovider.inl} (88%) diff --git a/modules/globebrowsing/CMakeLists.txt b/modules/globebrowsing/CMakeLists.txt index 3c93f41d47..d2e9626990 100644 --- a/modules/globebrowsing/CMakeLists.txt +++ b/modules/globebrowsing/CMakeLists.txt @@ -52,7 +52,7 @@ set(HEADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/other/texturetileset.h ${CMAKE_CURRENT_SOURCE_DIR}/other/patchcoverageprovider.h ${CMAKE_CURRENT_SOURCE_DIR}/other/tileprovider.h - ${CMAKE_CURRENT_SOURCE_DIR}/other/gdaldataconverter.h + ${CMAKE_CURRENT_SOURCE_DIR}/other/texturedataprovider.h ${CMAKE_CURRENT_SOURCE_DIR}/other/lrucache.h ${CMAKE_CURRENT_SOURCE_DIR}/other/concurrentjobmanager.h ${CMAKE_CURRENT_SOURCE_DIR}/other/threadpool.h @@ -89,7 +89,7 @@ set(SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/other/texturetileset.cpp ${CMAKE_CURRENT_SOURCE_DIR}/other/patchcoverageprovider.cpp ${CMAKE_CURRENT_SOURCE_DIR}/other/tileprovider.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/other/gdaldataconverter.inl + ${CMAKE_CURRENT_SOURCE_DIR}/other/texturedataprovider.inl ${CMAKE_CURRENT_SOURCE_DIR}/other/lrucache.inl ${CMAKE_CURRENT_SOURCE_DIR}/other/concurrentjobmanager.inl ${CMAKE_CURRENT_SOURCE_DIR}/other/threadpool.cpp diff --git a/modules/globebrowsing/other/gdaldataconverter.h b/modules/globebrowsing/other/texturedataprovider.h similarity index 64% rename from modules/globebrowsing/other/gdaldataconverter.h rename to modules/globebrowsing/other/texturedataprovider.h index 93a86dafc3..d41c5367c4 100644 --- a/modules/globebrowsing/other/gdaldataconverter.h +++ b/modules/globebrowsing/other/texturedataprovider.h @@ -31,6 +31,7 @@ #include #include +#include #include "gdal_priv.h" @@ -40,7 +41,7 @@ namespace openspace { using namespace ghoul::opengl; - struct UninitializedTextureTile { + struct TextureData { struct TextureFormat { @@ -48,12 +49,8 @@ namespace openspace { GLuint glFormat; }; - UninitializedTextureTile( - void* data, - glm::uvec3 dims, - TextureFormat format, - GLuint glType, - const ChunkIndex& chunkIndex) + TextureData(void* data, glm::uvec3 dims, TextureFormat format, + GLuint glType, const ChunkIndex& chunkIndex) : imageData(data) , dimensions(dims) , texFormat(format) @@ -70,32 +67,71 @@ namespace openspace { const ChunkIndex chunkIndex; }; + template - class GdalDataConverter + class TextureDataProvider { public: - GdalDataConverter(); - ~GdalDataConverter(); + TextureDataProvider(); + ~TextureDataProvider(); - std::shared_ptr getUninitializedTextureTile( - GDALDataset * dataSet, - ChunkIndex chunkIndex, - int tileLevelDifference); - UninitializedTextureTile::TextureFormat getTextureFormat( - int rasterCount, - GDALDataType gdalType); - GLuint getGlDataTypeFromGdalDataType(GDALDataType gdalType); + std::shared_ptr getTextureData( + GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference); + + + + struct async { + public: + void request(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference); + void updateRequests(); + bool hasTextureTileData() const; + std::shared_ptr nextTextureTile(); + + private: + std::queue> loadedTextureTiles; + std::set asyncReaders; + }; + + + + private: + + ////////////////////////////////////////////////////////////////////////////////// + // HELPER STRUCTS // + ////////////////////////////////////////////////////////////////////////////////// + + + struct GdalRequestParams { + glm::uvec2 pixelStart; + glm::uvec2 numPixels; + GDALDataType dataType; + int pixelSpacing; + int lineSpacing; + }; + + + ////////////////////////////////////////////////////////////////////////////////// + // HELPER FUNCTIONS // + ////////////////////////////////////////////////////////////////////////////////// + glm::uvec2 geodeticToPixel(GDALDataset* dataSet, const Geodetic2& geo); - }; + GLuint getGlDataTypeFromGdalDataType(GDALDataType gdalType); + + TextureData::TextureFormat getTextureFormat( + int rasterCount, + GDALDataType gdalType); + + + }; } // namespace openspace +#include -#include #endif // __GDALDATACONVERTER_H__ \ No newline at end of file diff --git a/modules/globebrowsing/other/gdaldataconverter.inl b/modules/globebrowsing/other/texturedataprovider.inl similarity index 88% rename from modules/globebrowsing/other/gdaldataconverter.inl rename to modules/globebrowsing/other/texturedataprovider.inl index 7e01541e35..8f2538283c 100644 --- a/modules/globebrowsing/other/gdaldataconverter.inl +++ b/modules/globebrowsing/other/texturedataprovider.inl @@ -23,7 +23,7 @@ ****************************************************************************************/ -#include +#include #include #include @@ -31,20 +31,22 @@ namespace openspace { template - GdalDataConverter::GdalDataConverter() + TextureDataProvider::TextureDataProvider() { } template - GdalDataConverter::~GdalDataConverter() + TextureDataProvider::~TextureDataProvider() { } + + template - std::shared_ptr GdalDataConverter::getUninitializedTextureTile( + std::shared_ptr TextureDataProvider::getTextureData( GDALDataset* dataSet, ChunkIndex chunkIndex, int tileLevelDifference) @@ -94,6 +96,9 @@ namespace openspace { // GDAL reads image data top to bottom T* imageData = new T[numPixels.x * numPixels.y * nRasters]; + int pixelSpacing = sizeof(T) * nRasters; + int lineSpacing = pixelSpacing * numPixels.x; + // Read the data (each rasterband is a separate channel) for (size_t i = 0; i < nRasters; i++) { GDALRasterBand* rasterBand = dataSet->GetRasterBand(i + 1)->GetOverview(ov); @@ -101,6 +106,7 @@ namespace openspace { int xSize = rasterBand->GetXSize(); int ySize = rasterBand->GetYSize(); + CPLErr err = rasterBand->RasterIO( GF_Read, pixelStart.x, // Begin read x @@ -111,8 +117,8 @@ namespace openspace { numPixels.x, // width to write x in destination numPixels.y, // width to write y in destination gdalType, // Type - sizeof(T) * nRasters, // Pixel spacing - 0); // Line spacing + pixelSpacing, // Pixel spacing + lineSpacing); // Line spacing if (err != CE_None) { ;//LERROR("There was a IO error (" << err << ") for: " << dataSet->GetDescription()); @@ -130,22 +136,22 @@ namespace openspace { delete[] imageData; glm::uvec3 dims(numPixels.x, numPixels.y, 1); - UninitializedTextureTile::TextureFormat textureFormat = + TextureData::TextureFormat textureFormat = getTextureFormat(nRasters, gdalType); GLuint glType = getGlDataTypeFromGdalDataType(gdalType); - UninitializedTextureTile* uninitedTexPtr = new UninitializedTextureTile( + TextureData* uninitedTexPtr = new TextureData( imageDataYflipped, dims, textureFormat, glType, chunkIndex); - std::shared_ptr uninitedTex = - std::shared_ptr(uninitedTexPtr); + std::shared_ptr uninitedTex = + std::shared_ptr(uninitedTexPtr); return uninitedTex; } template - glm::uvec2 GdalDataConverter::geodeticToPixel(GDALDataset* dataSet, const Geodetic2& geo) { + glm::uvec2 TextureDataProvider::geodeticToPixel(GDALDataset* dataSet, const Geodetic2& geo) { double padfTransform[6]; CPLErr err = dataSet->GetGeoTransform(padfTransform); @@ -183,11 +189,11 @@ namespace openspace { } template - UninitializedTextureTile::TextureFormat GdalDataConverter::getTextureFormat( + TextureData::TextureFormat TextureDataProvider::getTextureFormat( int rasterCount, GDALDataType gdalType) { - UninitializedTextureTile::TextureFormat format; + TextureData::TextureFormat format; switch (rasterCount) { @@ -323,7 +329,7 @@ namespace openspace { } template - GLuint GdalDataConverter::getGlDataTypeFromGdalDataType(GDALDataType gdalType) + GLuint TextureDataProvider::getGlDataTypeFromGdalDataType(GDALDataType gdalType) { switch (gdalType) { @@ -354,4 +360,33 @@ namespace openspace { } } + + template + void TextureDataProvider::async::request(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference) { + + } + + template + void TextureDataProvider::async::updateRequests() { + + } + + template + bool TextureDataProvider::async::hasTextureTileData() const { + + } + + template + std::shared_ptr TextureDataProvider::async::nextTextureTile() { + return nullptr; + } + + + /* + void asyncRequest(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference); + void updateAsyncRequests(); + std::shared_ptr nextTextureTile(); + + */ + } // namespace openspace diff --git a/modules/globebrowsing/other/texturetileset.cpp b/modules/globebrowsing/other/texturetileset.cpp index 7c456f33d6..0ae0475f19 100644 --- a/modules/globebrowsing/other/texturetileset.cpp +++ b/modules/globebrowsing/other/texturetileset.cpp @@ -25,7 +25,7 @@ #include #include -#include +#include #include #include @@ -67,7 +67,7 @@ namespace openspace { poDataset = (GDALDataset *)GDALOpen(testFile.c_str(), GA_ReadOnly); assert(poDataset != nullptr, "Unable to read dataset" << testFile); - GdalDataConverter conv; + TextureDataProvider conv; GeodeticTileIndex ti(0, 0, 0); diff --git a/modules/globebrowsing/other/tileprovider.cpp b/modules/globebrowsing/other/tileprovider.cpp index 601c9ee660..c55fd12631 100644 --- a/modules/globebrowsing/other/tileprovider.cpp +++ b/modules/globebrowsing/other/tileprovider.cpp @@ -101,6 +101,9 @@ namespace openspace { _depthTransform.depthOffset = firstBand->GetOffset(); _depthTransform.depthScale = firstBand->GetScale() * maximumValue; + + + } TileProvider::~TileProvider(){ @@ -120,7 +123,7 @@ namespace openspace { void TileProvider::initTexturesFromLoadedData() { while (_tileLoadManager.numFinishedJobs() > 0) { auto finishedJob = _tileLoadManager.popFinishedJob(); - std::shared_ptr uninitedTex = + std::shared_ptr uninitedTex = finishedJob->product(); HashKey key = uninitedTex->chunkIndex.hashKey(); std::shared_ptr texture = initializeTexture(uninitedTex); @@ -228,7 +231,7 @@ namespace openspace { } - std::shared_ptr TileProvider::getUninitializedTextureTile( + std::shared_ptr TileProvider::getTextureData( const ChunkIndex& chunkIndex) { // We assume here that all rasterbands have the same data type @@ -237,44 +240,44 @@ namespace openspace { switch (gdalType) { case GDT_Byte: - return _uByteConverter.getUninitializedTextureTile( + return _uByteTextureTileDataProvider.getTextureData( _gdalDataSet, chunkIndex, _tileLevelDifference); break; case GDT_UInt16: - return _uShortConverter.getUninitializedTextureTile( + return _uShortTextureDataProvider.getTextureData( _gdalDataSet, chunkIndex, _tileLevelDifference); break; case GDT_Int16: - return _shortConverter.getUninitializedTextureTile( + return _shortTextureDataProvider.getTextureData( _gdalDataSet, chunkIndex, _tileLevelDifference); break; case GDT_UInt32: - return _uIntConverter.getUninitializedTextureTile( + return _uIntTextureDataProvider.getTextureData( _gdalDataSet, chunkIndex, _tileLevelDifference); break; case GDT_Int32: - return _intConverter.getUninitializedTextureTile( + return _intTextureDataProvider.getTextureData( _gdalDataSet, chunkIndex, _tileLevelDifference); break; case GDT_Float32: - return _floatConverter.getUninitializedTextureTile( + return _floatTextureDataProvider.getTextureData( _gdalDataSet, chunkIndex, _tileLevelDifference); break; case GDT_Float64: - return _doubleConverter.getUninitializedTextureTile( + return _doubleTextureDataProvider.getTextureData( _gdalDataSet, chunkIndex, _tileLevelDifference); @@ -287,7 +290,7 @@ namespace openspace { } std::shared_ptr TileProvider::initializeTexture( - std::shared_ptr uninitedTexture) { + std::shared_ptr uninitedTexture) { Texture* tex = new Texture( uninitedTexture->imageData, uninitedTexture->dimensions, @@ -299,7 +302,6 @@ namespace openspace { // The texture should take ownership of the data std::shared_ptr texture = std::shared_ptr(tex); - texture->uploadTexture(); return texture; } diff --git a/modules/globebrowsing/other/tileprovider.h b/modules/globebrowsing/other/tileprovider.h index fda5226f3b..3001e62129 100644 --- a/modules/globebrowsing/other/tileprovider.h +++ b/modules/globebrowsing/other/tileprovider.h @@ -38,7 +38,7 @@ #include #include #include -#include +#include ////////////////////////////////////////////////////////////////////////////////////////// // TILE PROVIDER // @@ -99,14 +99,14 @@ namespace openspace { /** Fetches all the needeed texture data from the GDAL dataset. */ - std::shared_ptr getUninitializedTextureTile( + std::shared_ptr getTextureData( const ChunkIndex& chunkIndex); /** Creates an OpenGL texture and pushes the data to the GPU. */ std::shared_ptr initializeTexture( - std::shared_ptr uninitedTexture); + std::shared_ptr uninitedTexture); bool enqueueTileRequest(const ChunkIndex& ci); @@ -133,15 +133,15 @@ namespace openspace { GDALDataset* _gdalDataSet; // Converters are needed for all different data types since they are templated. - GdalDataConverter _uByteConverter; - GdalDataConverter _uShortConverter; - GdalDataConverter _shortConverter; - GdalDataConverter _uIntConverter; - GdalDataConverter _intConverter; - GdalDataConverter _floatConverter; - GdalDataConverter _doubleConverter; + TextureDataProvider _uByteTextureTileDataProvider; + TextureDataProvider _uShortTextureDataProvider; + TextureDataProvider _shortTextureDataProvider; + TextureDataProvider _uIntTextureDataProvider; + TextureDataProvider _intTextureDataProvider; + TextureDataProvider _floatTextureDataProvider; + TextureDataProvider _doubleTextureDataProvider; - ConcurrentJobManager _tileLoadManager; + ConcurrentJobManager _tileLoadManager; std::shared_ptr _defaultTexture; int _tileLevelDifference; @@ -150,11 +150,19 @@ namespace openspace { } // namespace openspace + + + + + + + + namespace openspace { using namespace ghoul::opengl; - struct TextureTileLoadJob : public Job { + struct TextureTileLoadJob : public Job { TextureTileLoadJob(TileProvider * tileProvider, const ChunkIndex& chunkIndex) : _tileProvider(tileProvider) , _chunkIndex(chunkIndex) { @@ -164,11 +172,11 @@ namespace openspace { virtual ~TextureTileLoadJob() { } virtual void execute() { - _uninitedTexture = _tileProvider->getUninitializedTextureTile(_chunkIndex); + _uninitedTexture = _tileProvider->getTextureData(_chunkIndex); } - virtual std::shared_ptr product() { + virtual std::shared_ptr product() { return _uninitedTexture; } @@ -176,7 +184,7 @@ namespace openspace { private: ChunkIndex _chunkIndex; TileProvider * _tileProvider; - std::shared_ptr _uninitedTexture; + std::shared_ptr _uninitedTexture; }; } diff --git a/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl b/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl index 1665862311..b6e30e9b4d 100644 --- a/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl +++ b/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl @@ -44,7 +44,7 @@ Fragment getFragment() { colorTiles[#{i}].uvTransform.uvScale * fs_uv + colorTiles[#{i}].uvTransform.uvOffset; vec4 colorSample = texture(colorTiles[#{i}].textureSampler, samplePos); - frag.color = blendOver(frag.color, colorSample); + frag.color = blendAdd(frag.color, colorSample); } #endfor @@ -54,7 +54,7 @@ Fragment getFragment() { //frag.color = frag.color * 0.9 + 0.2*vec4(samplePos, 0, 1); // Border overlay - //frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); + frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); frag.depth = fs_position.w; diff --git a/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl b/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl index f0cb8f2ed0..4cfa122486 100644 --- a/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl +++ b/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl @@ -59,7 +59,7 @@ Fragment getFragment() { //frag.color = frag.color * 0.9 + 0.2*vec4(samplePos, 0, 1); // Border overlay - //frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); + frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); frag.depth = fs_position.w; From efa78e67561f4ddaf7c7c389538a9b87c52c2afc Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Wed, 18 May 2016 16:52:58 -0400 Subject: [PATCH 2/9] TextureDataProvider is no longer templated --- modules/globebrowsing/CMakeLists.txt | 2 +- ...taprovider.inl => texturedataprovider.cpp} | 99 +++++++++++-------- .../globebrowsing/other/texturedataprovider.h | 59 +++++------ modules/globebrowsing/other/tileprovider.cpp | 53 +--------- modules/globebrowsing/other/tileprovider.h | 9 +- 5 files changed, 95 insertions(+), 127 deletions(-) rename modules/globebrowsing/other/{texturedataprovider.inl => texturedataprovider.cpp} (83%) diff --git a/modules/globebrowsing/CMakeLists.txt b/modules/globebrowsing/CMakeLists.txt index d2e9626990..4e1b191bce 100644 --- a/modules/globebrowsing/CMakeLists.txt +++ b/modules/globebrowsing/CMakeLists.txt @@ -89,7 +89,7 @@ set(SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/other/texturetileset.cpp ${CMAKE_CURRENT_SOURCE_DIR}/other/patchcoverageprovider.cpp ${CMAKE_CURRENT_SOURCE_DIR}/other/tileprovider.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/other/texturedataprovider.inl + ${CMAKE_CURRENT_SOURCE_DIR}/other/texturedataprovider.cpp ${CMAKE_CURRENT_SOURCE_DIR}/other/lrucache.inl ${CMAKE_CURRENT_SOURCE_DIR}/other/concurrentjobmanager.inl ${CMAKE_CURRENT_SOURCE_DIR}/other/threadpool.cpp diff --git a/modules/globebrowsing/other/texturedataprovider.inl b/modules/globebrowsing/other/texturedataprovider.cpp similarity index 83% rename from modules/globebrowsing/other/texturedataprovider.inl rename to modules/globebrowsing/other/texturedataprovider.cpp index 8f2538283c..4366bfa63d 100644 --- a/modules/globebrowsing/other/texturedataprovider.inl +++ b/modules/globebrowsing/other/texturedataprovider.cpp @@ -23,30 +23,26 @@ ****************************************************************************************/ +#include + #include #include #include + namespace openspace { - template - TextureDataProvider::TextureDataProvider() - { + TextureDataProvider::TextureDataProvider() { } - template - TextureDataProvider::~TextureDataProvider() - { + TextureDataProvider::~TextureDataProvider() { } - - - template - std::shared_ptr TextureDataProvider::getTextureData( + std::shared_ptr TextureDataProvider::getTextureData( GDALDataset* dataSet, ChunkIndex chunkIndex, int tileLevelDifference) @@ -70,6 +66,7 @@ namespace openspace { // pixelStart0 and pixelEnd0 defines the interval in the pixel space // at overview 0 GeodeticPatch patch = GeodeticPatch(chunkIndex); + glm::uvec2 pixelStart0 = geodeticToPixel(dataSet, patch.northWestCorner()); glm::uvec2 pixelEnd0 = geodeticToPixel(dataSet, patch.southEastCorner()); glm::uvec2 numPixels0 = pixelEnd0 - pixelStart0; @@ -77,7 +74,7 @@ namespace openspace { // Calculate a suitable overview to choose from the GDAL dataset int minNumPixels0 = glm::min(numPixels0.x, numPixels0.y); int sizeLevel0 = firstBand->GetOverview(numOverviews - 1)->GetXSize(); - int ov = log2(minNumPixels0) - log2(sizeLevel0 + 1) - tileLevelDifference; + int ov = std::log2(minNumPixels0) - std::log2(sizeLevel0 + 1) - tileLevelDifference; ov = glm::clamp(ov, 0, numOverviews - 1); // Convert the interval [pixelStart0, pixelEnd0] to pixel space at @@ -93,10 +90,13 @@ namespace openspace { numPixels = glm::uvec2(32, 32); } - // GDAL reads image data top to bottom - T* imageData = new T[numPixels.x * numPixels.y * nRasters]; + int bytesPerPixel = numberOfBytes(gdalType); - int pixelSpacing = sizeof(T) * nRasters; + // GDAL reads image data top to bottom + //T* imageData = new T[numPixels.x * numPixels.y * nRasters]; + char* imageData = new char[numPixels.x * numPixels.y * nRasters * bytesPerPixel]; + + int pixelSpacing = bytesPerPixel * nRasters; int lineSpacing = pixelSpacing * numPixels.x; // Read the data (each rasterband is a separate channel) @@ -106,6 +106,7 @@ namespace openspace { int xSize = rasterBand->GetXSize(); int ySize = rasterBand->GetYSize(); + char* dataDestination = imageData + (i * bytesPerPixel); CPLErr err = rasterBand->RasterIO( GF_Read, @@ -113,7 +114,7 @@ namespace openspace { pixelStart.y, // Begin read y numPixels.x, // width to read x numPixels.y, // width to read y - imageData + i, // Where to put data + dataDestination, // Where to put data numPixels.x, // width to write x in destination numPixels.y, // width to write y in destination gdalType, // Type @@ -125,19 +126,21 @@ namespace openspace { } } // GDAL reads image data top to bottom. We want the opposite. - T* imageDataYflipped = new T[numPixels.x * numPixels.y * nRasters]; + //T* imageDataYflipped = new T[numPixels.x * numPixels.y * nRasters]; + + size_t bytesPerLine = numPixels.x * nRasters * bytesPerPixel; + char* imageDataYflipped = new char[numPixels.y * bytesPerLine]; for (size_t y = 0; y < numPixels.y; y++) { - for (size_t x = 0; x < numPixels.x * nRasters; x++) { - imageDataYflipped[x + y * numPixels.x * nRasters] = - imageData[x + (numPixels.y - 1 - y) * numPixels.x * nRasters]; + for (size_t x = 0; x < bytesPerLine; x++) { + imageDataYflipped[x + y * bytesPerLine] = + imageData[x + (numPixels.y - 1 - y) * bytesPerLine]; } } delete[] imageData; glm::uvec3 dims(numPixels.x, numPixels.y, 1); - TextureData::TextureFormat textureFormat = - getTextureFormat(nRasters, gdalType); + TextureData::TextureFormat textureFormat = getTextureFormat(nRasters, gdalType); GLuint glType = getGlDataTypeFromGdalDataType(gdalType); TextureData* uninitedTexPtr = new TextureData( imageDataYflipped, @@ -150,8 +153,25 @@ namespace openspace { return uninitedTex; } - template - glm::uvec2 TextureDataProvider::geodeticToPixel(GDALDataset* dataSet, const Geodetic2& geo) { + + + size_t TextureDataProvider::numberOfBytes(GDALDataType gdalType) const { + switch (gdalType) { + case GDT_Byte: return sizeof(GLubyte); + case GDT_UInt16: return sizeof(GLushort); + case GDT_Int16: return sizeof(GLshort); + case GDT_UInt32: return sizeof(GLuint); + case GDT_Int32: return sizeof(GLint); + case GDT_Float32: return sizeof(GLfloat); + case GDT_Float64: return sizeof(GLdouble); + default: + //LERROR("Unknown data type"); + return -1; + } + } + + + glm::uvec2 TextureDataProvider::geodeticToPixel(GDALDataset* dataSet, const Geodetic2& geo) { double padfTransform[6]; CPLErr err = dataSet->GetGeoTransform(padfTransform); @@ -188,10 +208,9 @@ namespace openspace { return glm::uvec2(glm::round(P), glm::round(L)); } - template - TextureData::TextureFormat TextureDataProvider::getTextureFormat( - int rasterCount, - GDALDataType gdalType) + + TextureData::TextureFormat TextureDataProvider::getTextureFormat( + int rasterCount, GDALDataType gdalType) { TextureData::TextureFormat format; @@ -328,8 +347,8 @@ namespace openspace { return format; } - template - GLuint TextureDataProvider::getGlDataTypeFromGdalDataType(GDALDataType gdalType) + + GLuint TextureDataProvider::getGlDataTypeFromGdalDataType(GDALDataType gdalType) { switch (gdalType) { @@ -361,32 +380,26 @@ namespace openspace { } - template - void TextureDataProvider::async::request(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference) { + + void TextureDataProvider::asyncRequest(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference) { } - template - void TextureDataProvider::async::updateRequests() { + + void TextureDataProvider::updateAsyncRequests() { } - template - bool TextureDataProvider::async::hasTextureTileData() const { + bool TextureDataProvider::hasTextureTileData() const { + return false; } - template - std::shared_ptr TextureDataProvider::async::nextTextureTile() { + + std::shared_ptr TextureDataProvider::nextTextureTile() { return nullptr; } - /* - void asyncRequest(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference); - void updateAsyncRequests(); - std::shared_ptr nextTextureTile(); - - */ } // namespace openspace diff --git a/modules/globebrowsing/other/texturedataprovider.h b/modules/globebrowsing/other/texturedataprovider.h index d41c5367c4..b0b2240894 100644 --- a/modules/globebrowsing/other/texturedataprovider.h +++ b/modules/globebrowsing/other/texturedataprovider.h @@ -22,12 +22,11 @@ * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * ****************************************************************************************/ -#ifndef __GDALDATACONVERTER_H__ -#define __GDALDATACONVERTER_H__ +#ifndef __TEXTURE_DATA_PROVIDER_H__ +#define __TEXTURE_DATA_PROVIDER_H__ //#include -#include #include #include @@ -36,6 +35,9 @@ #include "gdal_priv.h" #include +#include +#include + namespace openspace { @@ -43,12 +45,13 @@ namespace openspace { struct TextureData { - struct TextureFormat - { + struct TextureFormat { Texture::Format ghoulFormat; GLuint glFormat; }; + + TextureData(void* data, glm::uvec3 dims, TextureFormat format, GLuint glType, const ChunkIndex& chunkIndex) : imageData(data) @@ -68,9 +71,12 @@ namespace openspace { }; - template - class TextureDataProvider - { + + + + + + class TextureDataProvider { public: TextureDataProvider(); @@ -81,23 +87,18 @@ namespace openspace { GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference); - - struct async { - public: - void request(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference); - void updateRequests(); - bool hasTextureTileData() const; - std::shared_ptr nextTextureTile(); - - private: - std::queue> loadedTextureTiles; - std::set asyncReaders; - }; - - + void asyncRequest(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference); + void updateAsyncRequests(); + bool hasTextureTileData() const; + std::shared_ptr nextTextureTile(); private: + std::queue> loadedTextureTiles; + std::set asyncReaders; + + + ////////////////////////////////////////////////////////////////////////////////// // HELPER STRUCTS // ////////////////////////////////////////////////////////////////////////////////// @@ -107,6 +108,7 @@ namespace openspace { glm::uvec2 pixelStart; glm::uvec2 numPixels; GDALDataType dataType; + int numRasters; int pixelSpacing; int lineSpacing; }; @@ -121,17 +123,18 @@ namespace openspace { GLuint getGlDataTypeFromGdalDataType(GDALDataType gdalType); + TextureData::TextureFormat getTextureFormat(int rasterCount, GDALDataType gdalType); - TextureData::TextureFormat getTextureFormat( - int rasterCount, - GDALDataType gdalType); - + size_t numberOfBytes(GDALDataType gdalType) const; }; + + + } // namespace openspace -#include -#endif // __GDALDATACONVERTER_H__ \ No newline at end of file + +#endif // __TEXTURE_DATA_PROVIDER_H__ \ No newline at end of file diff --git a/modules/globebrowsing/other/tileprovider.cpp b/modules/globebrowsing/other/tileprovider.cpp index c55fd12631..f123422a2b 100644 --- a/modules/globebrowsing/other/tileprovider.cpp +++ b/modules/globebrowsing/other/tileprovider.cpp @@ -237,56 +237,13 @@ namespace openspace { // We assume here that all rasterbands have the same data type GDALDataType gdalType = _gdalDataSet->GetRasterBand(1)->GetRasterDataType(); - switch (gdalType) - { - case GDT_Byte: - return _uByteTextureTileDataProvider.getTextureData( - _gdalDataSet, - chunkIndex, - _tileLevelDifference); - break; - case GDT_UInt16: - return _uShortTextureDataProvider.getTextureData( - _gdalDataSet, - chunkIndex, - _tileLevelDifference); - break; - case GDT_Int16: - return _shortTextureDataProvider.getTextureData( - _gdalDataSet, - chunkIndex, - _tileLevelDifference); - break; - case GDT_UInt32: - return _uIntTextureDataProvider.getTextureData( - _gdalDataSet, - chunkIndex, - _tileLevelDifference); - break; - case GDT_Int32: - return _intTextureDataProvider.getTextureData( - _gdalDataSet, - chunkIndex, - _tileLevelDifference); - break; - case GDT_Float32: - return _floatTextureDataProvider.getTextureData( - _gdalDataSet, - chunkIndex, - _tileLevelDifference); - break; - case GDT_Float64: - return _doubleTextureDataProvider.getTextureData( - _gdalDataSet, - chunkIndex, - _tileLevelDifference); - break; - default: - LERROR("GDAL data type unknown to OpenGL: " << gdalType); - } - return nullptr; + //auto provider = TextureDataProviderFactory::get(gdalType); + //return provider->getTextureData(_gdalDataSet, chunkIndex, _tileLevelDifference); + + return _uByteTextureTileDataProvider.getTextureData( + _gdalDataSet, chunkIndex, _tileLevelDifference); } std::shared_ptr TileProvider::initializeTexture( diff --git a/modules/globebrowsing/other/tileprovider.h b/modules/globebrowsing/other/tileprovider.h index 3001e62129..4790ca3b68 100644 --- a/modules/globebrowsing/other/tileprovider.h +++ b/modules/globebrowsing/other/tileprovider.h @@ -133,13 +133,8 @@ namespace openspace { GDALDataset* _gdalDataSet; // Converters are needed for all different data types since they are templated. - TextureDataProvider _uByteTextureTileDataProvider; - TextureDataProvider _uShortTextureDataProvider; - TextureDataProvider _shortTextureDataProvider; - TextureDataProvider _uIntTextureDataProvider; - TextureDataProvider _intTextureDataProvider; - TextureDataProvider _floatTextureDataProvider; - TextureDataProvider _doubleTextureDataProvider; + TextureDataProvider _uByteTextureTileDataProvider; + ConcurrentJobManager _tileLoadManager; From bcb107890f7f009c71b4dd48e9b34f1b41f744b4 Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Thu, 19 May 2016 12:18:31 -0400 Subject: [PATCH 3/9] Breaking down GDAL data fetching into sub tasks which can be reused later in async data fetching --- .../other/texturedataprovider.cpp | 401 +++++++----------- .../globebrowsing/other/texturedataprovider.h | 53 ++- modules/globebrowsing/other/tileprovider.cpp | 6 +- modules/globebrowsing/other/tileprovider.h | 12 +- .../shaders/globalchunkedlodpatch_fs.glsl | 2 +- .../shaders/localchunkedlodpatch_fs.glsl | 2 +- 6 files changed, 213 insertions(+), 263 deletions(-) diff --git a/modules/globebrowsing/other/texturedataprovider.cpp b/modules/globebrowsing/other/texturedataprovider.cpp index 4366bfa63d..a28c7d2d00 100644 --- a/modules/globebrowsing/other/texturedataprovider.cpp +++ b/modules/globebrowsing/other/texturedataprovider.cpp @@ -42,120 +42,73 @@ namespace openspace { } - std::shared_ptr TextureDataProvider::getTextureData( - GDALDataset* dataSet, - ChunkIndex chunkIndex, - int tileLevelDifference) + std::shared_ptr TextureDataProvider::getTextureData(GDALDataset* dataSet, + ChunkIndex chunkIndex, int tileLevelDifference) { - int nRasters = dataSet->GetRasterCount(); - - ghoul_assert(nRasters > 0, "Bad dataset. Contains no rasterband."); ghoul_assert(chunkIndex.level > 0, "Level of chunk index must be bigger than 0."); - - GDALRasterBand* firstBand = dataSet->GetRasterBand(1); - - // Assume all raster bands have the same data type - GDALDataType gdalType = firstBand->GetRasterDataType(); - - // Level = overviewCount - overview (default, levels may be overridden) - int numOverviews = firstBand->GetOverviewCount(); - - - // Generate a patch from the chunkIndex, extract the bounds which - // are used to calculated where in the GDAL data set to read data. - // pixelStart0 and pixelEnd0 defines the interval in the pixel space - // at overview 0 - GeodeticPatch patch = GeodeticPatch(chunkIndex); - - glm::uvec2 pixelStart0 = geodeticToPixel(dataSet, patch.northWestCorner()); - glm::uvec2 pixelEnd0 = geodeticToPixel(dataSet, patch.southEastCorner()); - glm::uvec2 numPixels0 = pixelEnd0 - pixelStart0; - - // Calculate a suitable overview to choose from the GDAL dataset - int minNumPixels0 = glm::min(numPixels0.x, numPixels0.y); - int sizeLevel0 = firstBand->GetOverview(numOverviews - 1)->GetXSize(); - int ov = std::log2(minNumPixels0) - std::log2(sizeLevel0 + 1) - tileLevelDifference; - ov = glm::clamp(ov, 0, numOverviews - 1); - - // Convert the interval [pixelStart0, pixelEnd0] to pixel space at - // the calculated suitable overview, ov. using a >> b = a / 2^b - int toShift = ov + 1; - glm::uvec2 pixelStart(pixelStart0.x >> toShift, pixelStart0.y >> toShift); - glm::uvec2 pixelEnd(pixelEnd0.x >> toShift, pixelEnd0.y >> toShift); - glm::uvec2 numPixels = pixelEnd - pixelStart; - - // When GDAL reads rasterbands of small size, the image data gets screwed up. - // This is not a solution to the problem, but makes it look a little better - if (numPixels.x < 32 || numPixels.y < 32) { - numPixels = glm::uvec2(32, 32); - } - - int bytesPerPixel = numberOfBytes(gdalType); - - // GDAL reads image data top to bottom - //T* imageData = new T[numPixels.x * numPixels.y * nRasters]; - char* imageData = new char[numPixels.x * numPixels.y * nRasters * bytesPerPixel]; - - int pixelSpacing = bytesPerPixel * nRasters; - int lineSpacing = pixelSpacing * numPixels.x; + + GdalDataRegion region(dataSet, chunkIndex, tileLevelDifference); + DataLayout dataLayout(dataSet, region); + char* imageData = new char[dataLayout.totalNumBytes]; // Read the data (each rasterband is a separate channel) - for (size_t i = 0; i < nRasters; i++) { - GDALRasterBand* rasterBand = dataSet->GetRasterBand(i + 1)->GetOverview(ov); + for (size_t i = 0; i < region.numRasters; i++) { + GDALRasterBand* rasterBand = dataSet->GetRasterBand(i + 1)->GetOverview(region.overview); - int xSize = rasterBand->GetXSize(); - int ySize = rasterBand->GetYSize(); - - char* dataDestination = imageData + (i * bytesPerPixel); + char* dataDestination = imageData + (i * dataLayout.bytesPerPixel); CPLErr err = rasterBand->RasterIO( GF_Read, - pixelStart.x, // Begin read x - pixelStart.y, // Begin read y - numPixels.x, // width to read x - numPixels.y, // width to read y - dataDestination, // Where to put data - numPixels.x, // width to write x in destination - numPixels.y, // width to write y in destination - gdalType, // Type - pixelSpacing, // Pixel spacing - lineSpacing); // Line spacing + region.pixelStart.x, // Begin read x + region.pixelStart.y, // Begin read y + region.numPixels.x, // width to read x + region.numPixels.y, // width to read y + dataDestination, // Where to put data + region.numPixels.x, // width to write x in destination + region.numPixels.y, // width to write y in destination + dataLayout.gdalType, // Type + dataLayout.pixelSpacing, // Pixel spacing + dataLayout.lineSpacing); // Line spacing if (err != CE_None) { ;//LERROR("There was a IO error (" << err << ") for: " << dataSet->GetDescription()); } } - // GDAL reads image data top to bottom. We want the opposite. - //T* imageDataYflipped = new T[numPixels.x * numPixels.y * nRasters]; - size_t bytesPerLine = numPixels.x * nRasters * bytesPerPixel; - char* imageDataYflipped = new char[numPixels.y * bytesPerLine]; - for (size_t y = 0; y < numPixels.y; y++) { - for (size_t x = 0; x < bytesPerLine; x++) { - imageDataYflipped[x + y * bytesPerLine] = - imageData[x + (numPixels.y - 1 - y) * bytesPerLine]; + return createRawTileData(region, dataLayout, imageData); + } + + std::shared_ptr TextureDataProvider::createRawTileData(const GdalDataRegion& region, + const DataLayout& dataLayout, char* imageData) + { + // GDAL reads image data top to bottom. We want the opposite. + char* imageDataYflipped = new char[dataLayout.totalNumBytes]; + for (size_t y = 0; y < region.numPixels.y; y++) { + for (size_t x = 0; x < dataLayout.lineSpacing; x++) { + imageDataYflipped[x + y * dataLayout.lineSpacing] = + imageData[x + (region.numPixels.y - 1 - y) * dataLayout.lineSpacing]; } } delete[] imageData; - - glm::uvec3 dims(numPixels.x, numPixels.y, 1); - TextureData::TextureFormat textureFormat = getTextureFormat(nRasters, gdalType); - GLuint glType = getGlDataTypeFromGdalDataType(gdalType); - TextureData* uninitedTexPtr = new TextureData( - imageDataYflipped, - dims, - textureFormat, - glType, - chunkIndex); - std::shared_ptr uninitedTex = - std::shared_ptr(uninitedTexPtr); - return uninitedTex; + + glm::uvec3 dims(region.numPixels.x, region.numPixels.y, 1); + RawTileData::TextureFormat textureFormat = getTextureFormat(region.numRasters, dataLayout.gdalType); + GLuint glType = getOpenGLDataType(dataLayout.gdalType); + RawTileData* textureDataPtr = new RawTileData(imageDataYflipped, dims, + textureFormat, glType, region.chunkIndex); + + std::shared_ptr textureData = + std::shared_ptr(textureDataPtr); + + return textureData; } - size_t TextureDataProvider::numberOfBytes(GDALDataType gdalType) const { + + + size_t TextureDataProvider::numberOfBytes(GDALDataType gdalType) { switch (gdalType) { case GDT_Byte: return sizeof(GLubyte); case GDT_UInt16: return sizeof(GLushort); @@ -209,177 +162,72 @@ namespace openspace { } - TextureData::TextureFormat TextureDataProvider::getTextureFormat( + RawTileData::TextureFormat TextureDataProvider::getTextureFormat( int rasterCount, GDALDataType gdalType) { - TextureData::TextureFormat format; + RawTileData::TextureFormat format; - switch (rasterCount) - { + switch (rasterCount) { case 1: // Red format.ghoulFormat = Texture::Format::Red; - switch (gdalType) - { - case GDT_Byte: - format.glFormat = GL_R8; - break; - case GDT_UInt16: - format.glFormat = GL_R16; - break; - case GDT_Int16: - format.glFormat = GL_R16; - break; - case GDT_UInt32: - format.glFormat = GL_R32UI; - break; - case GDT_Int32: - format.glFormat = GL_R32I; - break; - case GDT_Float32: - format.glFormat = GL_R32F; - break; - /* - case GDT_Float64: - format.glFormat = GL_RED; // No representation of 64 bit float? - break; - */ - default: - ;//LERROR("GDAL data type unknown to OpenGL: " << gdalType); + switch (gdalType) { + case GDT_Byte: format.glFormat = GL_R8; break; + case GDT_UInt16: format.glFormat = GL_R16; break; + case GDT_Int16: format.glFormat = GL_R16; break; + case GDT_UInt32: format.glFormat = GL_R32UI; break; + case GDT_Int32: format.glFormat = GL_R32I; break; + case GDT_Float32: format.glFormat = GL_R32F; break; + //case GDT_Float64: format.glFormat = GL_RED; break; // No representation of 64 bit float? + default: ;//LERROR("GDAL data type unknown to OpenGL: " << gdalType); } break; case 2: format.ghoulFormat = Texture::Format::RG; - switch (gdalType) - { - case GDT_Byte: - format.glFormat = GL_RG8; - break; - case GDT_UInt16: - format.glFormat = GL_RG16; - break; - case GDT_Int16: - format.glFormat = GL_RG16; - break; - case GDT_UInt32: - format.glFormat = GL_RG32UI; - break; - case GDT_Int32: - format.glFormat = GL_RG32I; - break; - case GDT_Float32: - format.glFormat = GL_RG32F; - break; - /* - case GDT_Float64: - format.glFormat = GL_RED; // No representation of 64 bit float? - break; - */ - default: - ;//LERROR("GDAL data type unknown to OpenGL: " << gdalType); + switch (gdalType) { + case GDT_Byte: format.glFormat = GL_RG8; break; + case GDT_UInt16: format.glFormat = GL_RG16; break; + case GDT_Int16: format.glFormat = GL_RG16; break; + case GDT_UInt32: format.glFormat = GL_RG32UI; break; + case GDT_Int32: format.glFormat = GL_RG32I; break; + case GDT_Float32: format.glFormat = GL_RG32F; break; + case GDT_Float64: format.glFormat = GL_RED; break; // No representation of 64 bit float? + default: ;//LERROR("GDAL data type unknown to OpenGL: " << gdalType); } break; case 3: format.ghoulFormat = Texture::Format::RGB; - switch (gdalType) - { - case GDT_Byte: - format.glFormat = GL_RGB8; - break; - case GDT_UInt16: - format.glFormat = GL_RGB16; - break; - case GDT_Int16: - format.glFormat = GL_RGB16; - break; - case GDT_UInt32: - format.glFormat = GL_RGB32UI; - break; - case GDT_Int32: - format.glFormat = GL_RGB32I; - break; - case GDT_Float32: - format.glFormat = GL_RGB32F; - break; - /* - case GDT_Float64: - format.glFormat = GL_RED; // No representation of 64 bit float? - break; - */ - default: - ;//LERROR("GDAL data type unknown to OpenGL: " << gdalType); + switch (gdalType) { + case GDT_Byte: format.glFormat = GL_RGB8; break; + case GDT_UInt16: format.glFormat = GL_RGB16; break; + case GDT_Int16: format.glFormat = GL_RGB16; break; + case GDT_UInt32: format.glFormat = GL_RGB32UI; break; + case GDT_Int32: format.glFormat = GL_RGB32I; break; + case GDT_Float32: format.glFormat = GL_RGB32F; break; + // case GDT_Float64: format.glFormat = GL_RED; break;// No representation of 64 bit float? + default: ;//LERROR("GDAL data type unknown to OpenGL: " << gdalType); } break; case 4: format.ghoulFormat = Texture::Format::RGBA; - switch (gdalType) - { - case GDT_Byte: - format.glFormat = GL_RGBA8; - break; - case GDT_UInt16: - format.glFormat = GL_RGBA16; - break; - case GDT_Int16: - format.glFormat = GL_RGBA16; - break; - case GDT_UInt32: - format.glFormat = GL_RGBA32UI; - break; - case GDT_Int32: - format.glFormat = GL_RGBA32I; - break; - case GDT_Float32: - format.glFormat = GL_RGBA32F; - break; - /* - case GDT_Float64: - format.glFormat = GL_RED; // No representation of 64 bit float? - break; - */ - default: - ;//LERROR("GDAL data type unknown to OpenGL: " << gdalType); + switch (gdalType) { + case GDT_Byte: format.glFormat = GL_RGBA8; break; + case GDT_UInt16: format.glFormat = GL_RGBA16; break; + case GDT_Int16: format.glFormat = GL_RGBA16; break; + case GDT_UInt32: format.glFormat = GL_RGBA32UI; break; + case GDT_Int32: format.glFormat = GL_RGBA32I; break; + case GDT_Float32: format.glFormat = GL_RGBA32F; break; + case GDT_Float64: format.glFormat = GL_RED; break; // No representation of 64 bit float? + default: ;//LERROR("GDAL data type unknown to OpenGL: " << gdalType); } break; default: - ;//LERROR("Unknown number of channels for OpenGL texture: " << rasterCount); + //LERROR("Unknown number of channels for OpenGL texture: " << rasterCount); break; } return format; } - GLuint TextureDataProvider::getGlDataTypeFromGdalDataType(GDALDataType gdalType) - { - switch (gdalType) - { - case GDT_Byte: - return GL_UNSIGNED_BYTE; - break; - case GDT_UInt16: - return GL_UNSIGNED_SHORT; - break; - case GDT_Int16: - return GL_SHORT; - break; - case GDT_UInt32: - return GL_UNSIGNED_INT; - break; - case GDT_Int32: - return GL_INT; - break; - case GDT_Float32: - return GL_FLOAT; - break; - case GDT_Float64: - return GL_DOUBLE; - break; - default: - //LERROR("GDAL data type unknown to OpenGL: " << gdalType); - return GL_UNSIGNED_BYTE; - } - } - - void TextureDataProvider::asyncRequest(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference) { @@ -396,10 +244,87 @@ namespace openspace { } - std::shared_ptr TextureDataProvider::nextTextureTile() { + std::shared_ptr TextureDataProvider::nextTextureTile() { return nullptr; } + + + + GLuint TextureDataProvider::getOpenGLDataType(GDALDataType gdalType) { + switch (gdalType) { + case GDT_Byte: return GL_UNSIGNED_BYTE; + case GDT_UInt16: return GL_UNSIGNED_SHORT; + case GDT_Int16: return GL_SHORT; + case GDT_UInt32: return GL_UNSIGNED_INT; + case GDT_Int32: return GL_INT; + case GDT_Float32: return GL_FLOAT; + case GDT_Float64: return GL_DOUBLE; + default: + //LERROR("GDAL data type unknown to OpenGL: " << gdalType); + return GL_UNSIGNED_BYTE; + } + } + + + TextureDataProvider::GdalDataRegion::GdalDataRegion(GDALDataset * dataSet, + const ChunkIndex& chunkIndex, int tileLevelDifference) + : chunkIndex(chunkIndex) + { + + GDALRasterBand* firstBand = dataSet->GetRasterBand(1); + + // Assume all raster bands have the same data type + + // Level = overviewCount - overview (default, levels may be overridden) + int numOverviews = firstBand->GetOverviewCount(); + + + // Generate a patch from the chunkIndex, extract the bounds which + // are used to calculated where in the GDAL data set to read data. + // pixelStart0 and pixelEnd0 defines the interval in the pixel space + // at overview 0 + GeodeticPatch patch = GeodeticPatch(chunkIndex); + + glm::uvec2 pixelStart0 = geodeticToPixel(dataSet, patch.northWestCorner()); + glm::uvec2 pixelEnd0 = geodeticToPixel(dataSet, patch.southEastCorner()); + glm::uvec2 numPixels0 = pixelEnd0 - pixelStart0; + + // Calculate a suitable overview to choose from the GDAL dataset + int minNumPixels0 = glm::min(numPixels0.x, numPixels0.y); + int sizeLevel0 = firstBand->GetOverview(numOverviews - 1)->GetXSize(); + int ov = std::log2(minNumPixels0) - std::log2(sizeLevel0 + 1) - tileLevelDifference; + ov = glm::clamp(ov, 0, numOverviews - 1); + + // Convert the interval [pixelStart0, pixelEnd0] to pixel space at + // the calculated suitable overview, ov. using a >> b = a / 2^b + int toShift = ov + 1; + + // Set member variables + overview = ov; + numRasters = dataSet->GetRasterCount(); + ghoul_assert(numRasters > 0, "Bad dataset. Contains no rasterband."); + + pixelStart = glm::uvec2(pixelStart0.x >> toShift, pixelStart0.y >> toShift); + pixelEnd = glm::uvec2(pixelEnd0.x >> toShift, pixelEnd0.y >> toShift); + numPixels = pixelEnd - pixelStart; + } + + + TextureDataProvider::DataLayout::DataLayout(GDALDataset* dataSet, const GdalDataRegion& region) { + // Assume all raster bands have the same data type + gdalType = dataSet->GetRasterBand(1)->GetRasterDataType(); + bytesPerPixel = numberOfBytes(gdalType); + pixelSpacing = bytesPerPixel * region.numRasters; + lineSpacing = pixelSpacing * region.numPixels.x; + totalNumBytes = lineSpacing * region.numPixels.y; + } + + + + + + } // namespace openspace diff --git a/modules/globebrowsing/other/texturedataprovider.h b/modules/globebrowsing/other/texturedataprovider.h index b0b2240894..b6b33e69f1 100644 --- a/modules/globebrowsing/other/texturedataprovider.h +++ b/modules/globebrowsing/other/texturedataprovider.h @@ -43,7 +43,7 @@ namespace openspace { using namespace ghoul::opengl; - struct TextureData { + struct RawTileData { struct TextureFormat { Texture::Format ghoulFormat; @@ -52,7 +52,7 @@ namespace openspace { - TextureData(void* data, glm::uvec3 dims, TextureFormat format, + RawTileData(void* data, glm::uvec3 dims, TextureFormat format, GLuint glType, const ChunkIndex& chunkIndex) : imageData(data) , dimensions(dims) @@ -83,18 +83,18 @@ namespace openspace { ~TextureDataProvider(); - std::shared_ptr getTextureData( + std::shared_ptr getTextureData( GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference); void asyncRequest(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference); void updateAsyncRequests(); bool hasTextureTileData() const; - std::shared_ptr nextTextureTile(); + std::shared_ptr nextTextureTile(); private: - std::queue> loadedTextureTiles; + std::queue> loadedTextureTiles; std::set asyncReaders; @@ -104,28 +104,53 @@ namespace openspace { ////////////////////////////////////////////////////////////////////////////////// - struct GdalRequestParams { + struct GdalDataRegion { + + GdalDataRegion(GDALDataset* dataSet, const ChunkIndex& chunkIndex, + int tileLevelDifference); + + const ChunkIndex& chunkIndex; + glm::uvec2 pixelStart; + glm::uvec2 pixelEnd; glm::uvec2 numPixels; - GDALDataType dataType; - int numRasters; - int pixelSpacing; - int lineSpacing; + + size_t numRasters; + + int overview; + }; + struct DataLayout { + DataLayout(GDALDataset* dataSet, const GdalDataRegion& region); + + GDALDataType gdalType; + size_t bytesPerPixel; + size_t pixelSpacing; + size_t lineSpacing; + size_t totalNumBytes; + + }; + + ////////////////////////////////////////////////////////////////////////////////// // HELPER FUNCTIONS // ////////////////////////////////////////////////////////////////////////////////// - glm::uvec2 geodeticToPixel(GDALDataset* dataSet, const Geodetic2& geo); + - GLuint getGlDataTypeFromGdalDataType(GDALDataType gdalType); + static glm::uvec2 geodeticToPixel(GDALDataset* dataSet, const Geodetic2& geo); - TextureData::TextureFormat getTextureFormat(int rasterCount, GDALDataType gdalType); + static GLuint getOpenGLDataType(GDALDataType gdalType); - size_t numberOfBytes(GDALDataType gdalType) const; + static RawTileData::TextureFormat getTextureFormat(int rasterCount, GDALDataType gdalType); + + static size_t numberOfBytes(GDALDataType gdalType); + + static std::shared_ptr createRawTileData(const GdalDataRegion& region, + const DataLayout& dataLayout, char* imageData); }; diff --git a/modules/globebrowsing/other/tileprovider.cpp b/modules/globebrowsing/other/tileprovider.cpp index f123422a2b..4f3a42c0da 100644 --- a/modules/globebrowsing/other/tileprovider.cpp +++ b/modules/globebrowsing/other/tileprovider.cpp @@ -123,7 +123,7 @@ namespace openspace { void TileProvider::initTexturesFromLoadedData() { while (_tileLoadManager.numFinishedJobs() > 0) { auto finishedJob = _tileLoadManager.popFinishedJob(); - std::shared_ptr uninitedTex = + std::shared_ptr uninitedTex = finishedJob->product(); HashKey key = uninitedTex->chunkIndex.hashKey(); std::shared_ptr texture = initializeTexture(uninitedTex); @@ -231,7 +231,7 @@ namespace openspace { } - std::shared_ptr TileProvider::getTextureData( + std::shared_ptr TileProvider::getTextureData( const ChunkIndex& chunkIndex) { // We assume here that all rasterbands have the same data type @@ -247,7 +247,7 @@ namespace openspace { } std::shared_ptr TileProvider::initializeTexture( - std::shared_ptr uninitedTexture) { + std::shared_ptr uninitedTexture) { Texture* tex = new Texture( uninitedTexture->imageData, uninitedTexture->dimensions, diff --git a/modules/globebrowsing/other/tileprovider.h b/modules/globebrowsing/other/tileprovider.h index 4790ca3b68..99589f7e1d 100644 --- a/modules/globebrowsing/other/tileprovider.h +++ b/modules/globebrowsing/other/tileprovider.h @@ -99,14 +99,14 @@ namespace openspace { /** Fetches all the needeed texture data from the GDAL dataset. */ - std::shared_ptr getTextureData( + std::shared_ptr getTextureData( const ChunkIndex& chunkIndex); /** Creates an OpenGL texture and pushes the data to the GPU. */ std::shared_ptr initializeTexture( - std::shared_ptr uninitedTexture); + std::shared_ptr uninitedTexture); bool enqueueTileRequest(const ChunkIndex& ci); @@ -136,7 +136,7 @@ namespace openspace { TextureDataProvider _uByteTextureTileDataProvider; - ConcurrentJobManager _tileLoadManager; + ConcurrentJobManager _tileLoadManager; std::shared_ptr _defaultTexture; int _tileLevelDifference; @@ -157,7 +157,7 @@ namespace openspace { using namespace ghoul::opengl; - struct TextureTileLoadJob : public Job { + struct TextureTileLoadJob : public Job { TextureTileLoadJob(TileProvider * tileProvider, const ChunkIndex& chunkIndex) : _tileProvider(tileProvider) , _chunkIndex(chunkIndex) { @@ -171,7 +171,7 @@ namespace openspace { } - virtual std::shared_ptr product() { + virtual std::shared_ptr product() { return _uninitedTexture; } @@ -179,7 +179,7 @@ namespace openspace { private: ChunkIndex _chunkIndex; TileProvider * _tileProvider; - std::shared_ptr _uninitedTexture; + std::shared_ptr _uninitedTexture; }; } diff --git a/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl b/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl index b6e30e9b4d..3495644fea 100644 --- a/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl +++ b/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl @@ -54,7 +54,7 @@ Fragment getFragment() { //frag.color = frag.color * 0.9 + 0.2*vec4(samplePos, 0, 1); // Border overlay - frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); + //frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); frag.depth = fs_position.w; diff --git a/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl b/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl index 4cfa122486..f0cb8f2ed0 100644 --- a/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl +++ b/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl @@ -59,7 +59,7 @@ Fragment getFragment() { //frag.color = frag.color * 0.9 + 0.2*vec4(samplePos, 0, 1); // Border overlay - frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); + //frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); frag.depth = fs_position.w; From bfb8e1a92dc69b407938da8313ab9f565085e448 Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Thu, 19 May 2016 21:23:46 -0400 Subject: [PATCH 4/9] Added functionality for fetching tile using GDALAsyncReader. Not working yet, so requests are still serial --- data/scene/debugglobe/debugglobe.mod | 23 +++- ...RS_SNPP_CorrectedReflectance_TrueColor.xml | 2 +- .../other/TileProviderManager.cpp | 1 + .../globebrowsing/other/TileProviderManager.h | 6 +- .../other/texturedataprovider.cpp | 128 +++++++++++++----- .../globebrowsing/other/texturedataprovider.h | 40 +++++- modules/globebrowsing/other/tileprovider.cpp | 52 ++++--- modules/globebrowsing/other/tileprovider.h | 15 +- .../shaders/globalchunkedlodpatch_fs.glsl | 2 +- 9 files changed, 196 insertions(+), 73 deletions(-) diff --git a/data/scene/debugglobe/debugglobe.mod b/data/scene/debugglobe/debugglobe.mod index 2c0249d74d..fb0ddd01b5 100644 --- a/data/scene/debugglobe/debugglobe.mod +++ b/data/scene/debugglobe/debugglobe.mod @@ -8,20 +8,31 @@ return { Radii = {6378137.0, 6378137.0, 6356752.314245}, -- Earth's radii Textures = { ColorTextures = { - --[[ { Name = "ESRI Imagery World 2D", FilePath = "map_service_configs/ESRI_Imagery_World_2D.wms", }, - --]] - { - Name = "MODIS_Terra_CorrectedReflectance_TrueColor", - FilePath = "map_service_configs/MODIS_Terra_CorrectedReflectance_TrueColor.xml", - }, + { Name = "Coastlines", FilePath = "map_service_configs/Coastlines.xml", }, + --[[ + { + Name = "VIIRS_SNPP_CorrectedReflectance_TrueColor", + FilePath = "map_service_configs/VIIRS_SNPP_CorrectedReflectance_TrueColor.xml" + }, + { + Name = "ESRI Imagery World 2D", + FilePath = "map_service_configs/ESRI_Imagery_World_2D.wms", + }, + + { + Name = "MODIS_Terra_CorrectedReflectance_TrueColor", + FilePath = "map_service_configs/MODIS_Terra_CorrectedReflectance_TrueColor.xml", + }, + + --]] }, HeightMaps = { { diff --git a/data/scene/debugglobe/map_service_configs/VIIRS_SNPP_CorrectedReflectance_TrueColor.xml b/data/scene/debugglobe/map_service_configs/VIIRS_SNPP_CorrectedReflectance_TrueColor.xml index b1a5674684..881dce3e58 100644 --- a/data/scene/debugglobe/map_service_configs/VIIRS_SNPP_CorrectedReflectance_TrueColor.xml +++ b/data/scene/debugglobe/map_service_configs/VIIRS_SNPP_CorrectedReflectance_TrueColor.xml @@ -7,7 +7,7 @@ 90 396.0 -198 - 5 + 8 2 1 top diff --git a/modules/globebrowsing/other/TileProviderManager.cpp b/modules/globebrowsing/other/TileProviderManager.cpp index 74a67fac2d..db34e502d6 100644 --- a/modules/globebrowsing/other/TileProviderManager.cpp +++ b/modules/globebrowsing/other/TileProviderManager.cpp @@ -31,6 +31,7 @@ namespace { namespace openspace { + TileProviderManager::TileProviderManager() { } diff --git a/modules/globebrowsing/other/TileProviderManager.h b/modules/globebrowsing/other/TileProviderManager.h index aa3bb705c1..ac52d56fc1 100644 --- a/modules/globebrowsing/other/TileProviderManager.h +++ b/modules/globebrowsing/other/TileProviderManager.h @@ -26,15 +26,16 @@ #define __TILE_PROVIDER_MANAGER_H__ #include "modules/globebrowsing/other/tileprovider.h" +#include #include #include #include + namespace openspace { - class TileProviderManager - { + class TileProviderManager { public: TileProviderManager(); ~TileProviderManager(); @@ -47,6 +48,7 @@ namespace openspace { */ const std::vector >& heightMapProviders(); const std::vector >& colorTextureProviders(); + private: std::vector > _heightMapProviders; std::vector > _colorTextureProviders; diff --git a/modules/globebrowsing/other/texturedataprovider.cpp b/modules/globebrowsing/other/texturedataprovider.cpp index a28c7d2d00..700abf3136 100644 --- a/modules/globebrowsing/other/texturedataprovider.cpp +++ b/modules/globebrowsing/other/texturedataprovider.cpp @@ -29,6 +29,10 @@ #include #include +namespace { + const std::string _loggerCat = "TextureDataProvider"; +} + namespace openspace { @@ -44,9 +48,7 @@ namespace openspace { std::shared_ptr TextureDataProvider::getTextureData(GDALDataset* dataSet, ChunkIndex chunkIndex, int tileLevelDifference) - { - ghoul_assert(chunkIndex.level > 0, "Level of chunk index must be bigger than 0."); - + { GdalDataRegion region(dataSet, chunkIndex, tileLevelDifference); DataLayout dataLayout(dataSet, region); char* imageData = new char[dataLayout.totalNumBytes]; @@ -55,7 +57,7 @@ namespace openspace { for (size_t i = 0; i < region.numRasters; i++) { GDALRasterBand* rasterBand = dataSet->GetRasterBand(i + 1)->GetOverview(region.overview); - char* dataDestination = imageData + (i * dataLayout.bytesPerPixel); + char* dataDestination = imageData + (i * dataLayout.bytesPerDatum); CPLErr err = rasterBand->RasterIO( GF_Read, @@ -67,8 +69,8 @@ namespace openspace { region.numPixels.x, // width to write x in destination region.numPixels.y, // width to write y in destination dataLayout.gdalType, // Type - dataLayout.pixelSpacing, // Pixel spacing - dataLayout.lineSpacing); // Line spacing + dataLayout.bytesPerPixel, // Pixel spacing + dataLayout.bytesPerLine); // Line spacing if (err != CE_None) { ;//LERROR("There was a IO error (" << err << ") for: " << dataSet->GetDescription()); @@ -78,15 +80,94 @@ namespace openspace { return createRawTileData(region, dataLayout, imageData); } + + void TextureDataProvider::asyncRequest(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference) { + GdalDataRegion region(dataSet, chunkIndex, tileLevelDifference); + DataLayout dataLayout(dataSet, region); + + char* imageData = new char[dataLayout.totalNumBytes]; + + int* rasterBandSelection = nullptr; // default to { 1, 2, ... , region.numRasters } + GDALAsyncReader * asyncReader = dataSet->BeginAsyncReader( + region.pixelStart.x, // nXOff + region.pixelStart.y, // nYOff + region.numPixels.x, // nXSize + region.numPixels.y, // nYSize + imageData, // pBuf + region.numPixels.x, // nBufXSize + region.numPixels.y, // nBufYSize + dataLayout.gdalType, // eBufType + region.numRasters, // nBandCount + rasterBandSelection, // panBandMap + dataLayout.bytesPerDatum, // nPixelSpace + dataLayout.bytesPerLine, // nLineSpace + dataLayout.bytesPerPixel, // nBandSpace + nullptr // papszOptions + ); + + ghoul_assert(asyncReader != nullptr, "Async reader is null"); + + GdalAsyncRequest request = { dataSet, asyncReader, region, dataLayout, imageData }; + + asyncRequests.insert(request); + + } + + + void TextureDataProvider::updateAsyncRequests() { + double updateWaitTime = -1.0; + int nBufXOff, nBufYOff, nBufXSize, nBufYSize; + + auto it = asyncRequests.begin(); + auto end = asyncRequests.end(); + + while (it != end) { + GDALAsyncStatusType status = it->asyncReader->GetNextUpdatedRegion( + updateWaitTime, &nBufXOff, &nBufYOff, &nBufXSize, &nBufYSize); + + + if (status == GDALAsyncStatusType::GARIO_ERROR) { + LERROR("Async IO error for chunk " << it->region.chunkIndex); + it->dataSet->EndAsyncReader(it->asyncReader); + it = asyncRequests.erase(it); + } + else if (status == GDALAsyncStatusType::GARIO_COMPLETE) { + auto rawTileData = createRawTileData(it->region, it->dataLayout, it->imageData); + loadedTextureTiles.push(rawTileData); + it->dataSet->EndAsyncReader(it->asyncReader); + it = asyncRequests.erase(it); + } + else { + it++; + } + } + } + + + bool TextureDataProvider::hasTextureTileData() const { + return loadedTextureTiles.size() > 0; + } + + + std::shared_ptr TextureDataProvider::nextTextureTile() { + auto tile = loadedTextureTiles.front(); + loadedTextureTiles.pop(); + return tile; + } + + + + + std::shared_ptr TextureDataProvider::createRawTileData(const GdalDataRegion& region, - const DataLayout& dataLayout, char* imageData) + const DataLayout& dataLayout, const char* imageData) { // GDAL reads image data top to bottom. We want the opposite. char* imageDataYflipped = new char[dataLayout.totalNumBytes]; for (size_t y = 0; y < region.numPixels.y; y++) { - for (size_t x = 0; x < dataLayout.lineSpacing; x++) { - imageDataYflipped[x + y * dataLayout.lineSpacing] = - imageData[x + (region.numPixels.y - 1 - y) * dataLayout.lineSpacing]; + for (size_t x = 0; x < dataLayout.bytesPerLine; x++) { + imageDataYflipped[x + y * dataLayout.bytesPerLine] = + imageData[x + (region.numPixels.y - 1 - y) * dataLayout.bytesPerLine]; } } @@ -229,25 +310,6 @@ namespace openspace { - void TextureDataProvider::asyncRequest(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference) { - - } - - - void TextureDataProvider::updateAsyncRequests() { - - } - - - bool TextureDataProvider::hasTextureTileData() const { - return false; - } - - - std::shared_ptr TextureDataProvider::nextTextureTile() { - return nullptr; - } - @@ -314,10 +376,10 @@ namespace openspace { TextureDataProvider::DataLayout::DataLayout(GDALDataset* dataSet, const GdalDataRegion& region) { // Assume all raster bands have the same data type gdalType = dataSet->GetRasterBand(1)->GetRasterDataType(); - bytesPerPixel = numberOfBytes(gdalType); - pixelSpacing = bytesPerPixel * region.numRasters; - lineSpacing = pixelSpacing * region.numPixels.x; - totalNumBytes = lineSpacing * region.numPixels.y; + bytesPerDatum = numberOfBytes(gdalType); + bytesPerPixel = bytesPerDatum * region.numRasters; + bytesPerLine = bytesPerPixel * region.numPixels.x; + totalNumBytes = bytesPerLine * region.numPixels.y; } diff --git a/modules/globebrowsing/other/texturedataprovider.h b/modules/globebrowsing/other/texturedataprovider.h index b6b33e69f1..76666eb4da 100644 --- a/modules/globebrowsing/other/texturedataprovider.h +++ b/modules/globebrowsing/other/texturedataprovider.h @@ -94,8 +94,8 @@ namespace openspace { private: - std::queue> loadedTextureTiles; - std::set asyncReaders; + + @@ -104,12 +104,14 @@ namespace openspace { ////////////////////////////////////////////////////////////////////////////////// + + struct GdalDataRegion { GdalDataRegion(GDALDataset* dataSet, const ChunkIndex& chunkIndex, int tileLevelDifference); - const ChunkIndex& chunkIndex; + const ChunkIndex chunkIndex; glm::uvec2 pixelStart; glm::uvec2 pixelEnd; @@ -125,17 +127,32 @@ namespace openspace { DataLayout(GDALDataset* dataSet, const GdalDataRegion& region); GDALDataType gdalType; + size_t bytesPerDatum; size_t bytesPerPixel; - size_t pixelSpacing; - size_t lineSpacing; + size_t bytesPerLine; size_t totalNumBytes; }; + struct GdalAsyncRequest { + GDALDataset* dataSet; + GDALAsyncReader* asyncReader; + const GdalDataRegion region; + const DataLayout dataLayout; + const char* imageData; + }; + + struct GdalAsyncRequestCompare { + bool operator() (const GdalAsyncRequest& lhs, const GdalAsyncRequest& rhs) const { + return lhs.region.chunkIndex.hashKey() < rhs.region.chunkIndex.hashKey(); + } + }; + + ////////////////////////////////////////////////////////////////////////////////// - // HELPER FUNCTIONS // + // HELPER FUNCTIONS // ////////////////////////////////////////////////////////////////////////////////// @@ -150,7 +167,16 @@ namespace openspace { static size_t numberOfBytes(GDALDataType gdalType); static std::shared_ptr createRawTileData(const GdalDataRegion& region, - const DataLayout& dataLayout, char* imageData); + const DataLayout& dataLayout, const char* imageData); + + + ////////////////////////////////////////////////////////////////////////////////// + // MEMBER VARIABLES // + ////////////////////////////////////////////////////////////////////////////////// + + + std::queue> loadedTextureTiles; + std::set asyncRequests; }; diff --git a/modules/globebrowsing/other/tileprovider.cpp b/modules/globebrowsing/other/tileprovider.cpp index 4f3a42c0da..6ff9d420fa 100644 --- a/modules/globebrowsing/other/tileprovider.cpp +++ b/modules/globebrowsing/other/tileprovider.cpp @@ -45,6 +45,8 @@ namespace openspace { bool TileProvider::hasInitializedGDAL = false; + + TileProvider::TileProvider( const std::string& filePath, int tileCacheSize, @@ -113,6 +115,7 @@ namespace openspace { void TileProvider::prerender() { + _rawTextureTileDataProvider.updateAsyncRequests(); initTexturesFromLoadedData(); if (_framesSinceLastRequestFlush++ > _framesUntilRequestFlush) { @@ -122,12 +125,13 @@ namespace openspace { void TileProvider::initTexturesFromLoadedData() { while (_tileLoadManager.numFinishedJobs() > 0) { - auto finishedJob = _tileLoadManager.popFinishedJob(); - std::shared_ptr uninitedTex = - finishedJob->product(); - HashKey key = uninitedTex->chunkIndex.hashKey(); - std::shared_ptr texture = initializeTexture(uninitedTex); - _tileCache.put(key, texture); + auto rawTextureTile = _tileLoadManager.popFinishedJob()->product(); + initializeAndAddToCache(rawTextureTile); + } + + while (_rawTextureTileDataProvider.hasTextureTileData()) { + auto rawTextureTile = _rawTextureTileDataProvider.nextTextureTile(); + initializeAndAddToCache(rawTextureTile); } } @@ -209,11 +213,19 @@ namespace openspace { HashKey key = chunkIndex.hashKey(); bool tileHasBeenQueued = _queuedTileRequests.find(key) != _queuedTileRequests.end(); if (!tileHasBeenQueued) { - // enque load job - std::shared_ptr job = std::shared_ptr( - new TextureTileLoadJob(this, chunkIndex)); + + bool requestDataAsync = false; + if (requestDataAsync) { + _rawTextureTileDataProvider.asyncRequest(_gdalDataSet, chunkIndex, _tileLevelDifference); + } + else { - _tileLoadManager.enqueueJob(job); + // enque load job + std::shared_ptr job = std::shared_ptr( + new TextureTileLoadJob(this, chunkIndex)); + + _tileLoadManager.enqueueJob(job); + } _queuedTileRequests.insert(key); } @@ -242,25 +254,27 @@ namespace openspace { //auto provider = TextureDataProviderFactory::get(gdalType); //return provider->getTextureData(_gdalDataSet, chunkIndex, _tileLevelDifference); - return _uByteTextureTileDataProvider.getTextureData( + return _rawTextureTileDataProvider.getTextureData( _gdalDataSet, chunkIndex, _tileLevelDifference); } - std::shared_ptr TileProvider::initializeTexture( - std::shared_ptr uninitedTexture) { + void TileProvider::initializeAndAddToCache( + std::shared_ptr rawTileData) { + HashKey key = rawTileData->chunkIndex.hashKey(); Texture* tex = new Texture( - uninitedTexture->imageData, - uninitedTexture->dimensions, - uninitedTexture->texFormat.ghoulFormat, - uninitedTexture->texFormat.glFormat, - uninitedTexture->glType, + rawTileData->imageData, + rawTileData->dimensions, + rawTileData->texFormat.ghoulFormat, + rawTileData->texFormat.glFormat, + rawTileData->glType, Texture::FilterMode::Linear, Texture::WrappingMode::ClampToEdge); // The texture should take ownership of the data std::shared_ptr texture = std::shared_ptr(tex); texture->uploadTexture(); - return texture; + + _tileCache.put(key, texture); } diff --git a/modules/globebrowsing/other/tileprovider.h b/modules/globebrowsing/other/tileprovider.h index 99589f7e1d..02b70c585b 100644 --- a/modules/globebrowsing/other/tileprovider.h +++ b/modules/globebrowsing/other/tileprovider.h @@ -82,6 +82,9 @@ namespace openspace { void prerender(); + + + private: friend class TextureTileLoadJob; @@ -105,7 +108,7 @@ namespace openspace { /** Creates an OpenGL texture and pushes the data to the GPU. */ - std::shared_ptr initializeTexture( + void initializeAndAddToCache( std::shared_ptr uninitedTexture); bool enqueueTileRequest(const ChunkIndex& ci); @@ -117,6 +120,8 @@ namespace openspace { + + ////////////////////////////////////////////////////////////////////////////////// // Member variables // ////////////////////////////////////////////////////////////////////////////////// @@ -132,17 +137,19 @@ namespace openspace { static bool hasInitializedGDAL; GDALDataset* _gdalDataSet; - // Converters are needed for all different data types since they are templated. - TextureDataProvider _uByteTextureTileDataProvider; - + TextureDataProvider _rawTextureTileDataProvider; ConcurrentJobManager _tileLoadManager; std::shared_ptr _defaultTexture; int _tileLevelDifference; TileDepthTransform _depthTransform; + }; + + + } // namespace openspace diff --git a/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl b/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl index 3495644fea..1665862311 100644 --- a/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl +++ b/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl @@ -44,7 +44,7 @@ Fragment getFragment() { colorTiles[#{i}].uvTransform.uvScale * fs_uv + colorTiles[#{i}].uvTransform.uvOffset; vec4 colorSample = texture(colorTiles[#{i}].textureSampler, samplePos); - frag.color = blendAdd(frag.color, colorSample); + frag.color = blendOver(frag.color, colorSample); } #endfor From bb551432e3ffd39cae5cb4e342eabd79ee38296f Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Fri, 20 May 2016 12:55:37 -0400 Subject: [PATCH 5/9] ConcurrentJobManager uses ThreadPool to do parallel Gdal RasterIO calls. Some tile gets corrupted... --- .../other/concurrentjobmanager.h | 39 +++---------------- modules/globebrowsing/other/threadpool.cpp | 9 ++++- modules/globebrowsing/other/threadpool.h | 1 + modules/globebrowsing/other/tileprovider.cpp | 1 + modules/globebrowsing/other/tileprovider.h | 2 +- 5 files changed, 16 insertions(+), 36 deletions(-) diff --git a/modules/globebrowsing/other/concurrentjobmanager.h b/modules/globebrowsing/other/concurrentjobmanager.h index 33cd150b5a..04df7c5f90 100644 --- a/modules/globebrowsing/other/concurrentjobmanager.h +++ b/modules/globebrowsing/other/concurrentjobmanager.h @@ -65,7 +65,6 @@ namespace openspace { class ConcurrentJobManager{ public: ConcurrentJobManager() - : _hasWorkingThread(false) { } @@ -76,17 +75,14 @@ namespace openspace { void enqueueJob(std::shared_ptr> job) { - _incomingJobs.push(job); - if (!_hasWorkingThread) { - _hasWorkingThread = true; // Can only be set to true by the main thread - executeJobsInSeparateThread(); - } + TileProvider::threadPool.enqueue([this, job]() { + job->execute(); + _finishedJobs.push(job); + }); } void clearEnqueuedJobs() { - while (_incomingJobs.size()) { - _incomingJobs.pop(); - } + TileProvider::threadPool.clearTasks(); } std::shared_ptr> popFinishedJob() { @@ -102,33 +98,8 @@ namespace openspace { private: - - void executeJobsInSeparateThread() { - // Create new thread and run workerThreadMainTask on that thread - std::thread t(&ConcurrentJobManager::workerThreadMainTask, this); - t.detach(); - } - - void workerThreadMainTask() { - while (_incomingJobs.size() > 0) { - auto job = _incomingJobs.pop(); - - job->execute(); - - _finishedJobs.push(job); - } - - _hasWorkingThread = false; // Can only be set to false by worker thread - } - - ConcurrentQueue>> _incomingJobs; ConcurrentQueue>> _finishedJobs; - // Using this atomic bool is probably not optimal - Should probably - // use a conditional variable instead - std::atomic _hasWorkingThread; - std::atomic _numActiveThreads; - }; diff --git a/modules/globebrowsing/other/threadpool.cpp b/modules/globebrowsing/other/threadpool.cpp index 8712ed65b6..daf3c80559 100644 --- a/modules/globebrowsing/other/threadpool.cpp +++ b/modules/globebrowsing/other/threadpool.cpp @@ -113,8 +113,15 @@ namespace openspace { } // release lock // wake up one thread - std::cout << "Notify one thread" << std::endl; condition.notify_one(); } + void ThreadPool::clearTasks() { + { // acquire lock + std::unique_lock lock(queue_mutex); + tasks.clear(); + } // release lock + } + + } // namespace openspace \ No newline at end of file diff --git a/modules/globebrowsing/other/threadpool.h b/modules/globebrowsing/other/threadpool.h index 73ae6a2d6d..d792ee01d0 100644 --- a/modules/globebrowsing/other/threadpool.h +++ b/modules/globebrowsing/other/threadpool.h @@ -59,6 +59,7 @@ namespace openspace { ~ThreadPool(); void enqueue(std::function f); + void clearTasks(); private: friend class Worker; diff --git a/modules/globebrowsing/other/tileprovider.cpp b/modules/globebrowsing/other/tileprovider.cpp index 6ff9d420fa..36c7c85378 100644 --- a/modules/globebrowsing/other/tileprovider.cpp +++ b/modules/globebrowsing/other/tileprovider.cpp @@ -45,6 +45,7 @@ namespace openspace { bool TileProvider::hasInitializedGDAL = false; + ThreadPool TileProvider::threadPool(10); TileProvider::TileProvider( diff --git a/modules/globebrowsing/other/tileprovider.h b/modules/globebrowsing/other/tileprovider.h index 02b70c585b..28b43f44f9 100644 --- a/modules/globebrowsing/other/tileprovider.h +++ b/modules/globebrowsing/other/tileprovider.h @@ -82,7 +82,7 @@ namespace openspace { void prerender(); - + static ThreadPool threadPool; private: From 8844775a1fbfe5bc8ccd85e112415768f834b7c5 Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Fri, 20 May 2016 13:20:46 -0400 Subject: [PATCH 6/9] TileProviderManager owns a common thread pool for all TileProviders --- modules/globebrowsing/globes/renderableglobe.h | 1 + modules/globebrowsing/other/TileProviderManager.cpp | 1 + modules/globebrowsing/other/TileProviderManager.h | 2 ++ modules/globebrowsing/other/concurrentjobmanager.h | 9 +++++---- modules/globebrowsing/other/tileprovider.cpp | 2 ++ modules/globebrowsing/other/tileprovider.h | 1 - 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/modules/globebrowsing/globes/renderableglobe.h b/modules/globebrowsing/globes/renderableglobe.h index 7e2ce29cb5..da676550c9 100644 --- a/modules/globebrowsing/globes/renderableglobe.h +++ b/modules/globebrowsing/globes/renderableglobe.h @@ -41,6 +41,7 @@ #include #include +#include namespace ghoul { namespace opengl { diff --git a/modules/globebrowsing/other/TileProviderManager.cpp b/modules/globebrowsing/other/TileProviderManager.cpp index db34e502d6..9a9ca49957 100644 --- a/modules/globebrowsing/other/TileProviderManager.cpp +++ b/modules/globebrowsing/other/TileProviderManager.cpp @@ -31,6 +31,7 @@ namespace { namespace openspace { + ThreadPool TileProviderManager::tileRequestThreadPool(1); TileProviderManager::TileProviderManager() { diff --git a/modules/globebrowsing/other/TileProviderManager.h b/modules/globebrowsing/other/TileProviderManager.h index ac52d56fc1..1142c1e26d 100644 --- a/modules/globebrowsing/other/TileProviderManager.h +++ b/modules/globebrowsing/other/TileProviderManager.h @@ -40,6 +40,8 @@ namespace openspace { TileProviderManager(); ~TileProviderManager(); + static ThreadPool tileRequestThreadPool; + void addHeightMap(std::string name, std::shared_ptr tileProvider); void addColorTexture(std::string name, std::shared_ptr tileProvider); /* diff --git a/modules/globebrowsing/other/concurrentjobmanager.h b/modules/globebrowsing/other/concurrentjobmanager.h index 04df7c5f90..9843ea02de 100644 --- a/modules/globebrowsing/other/concurrentjobmanager.h +++ b/modules/globebrowsing/other/concurrentjobmanager.h @@ -33,6 +33,7 @@ #include #include +#include #include @@ -64,7 +65,7 @@ namespace openspace { template class ConcurrentJobManager{ public: - ConcurrentJobManager() + ConcurrentJobManager(ThreadPool& pool) : threadPool(pool) { } @@ -75,14 +76,14 @@ namespace openspace { void enqueueJob(std::shared_ptr> job) { - TileProvider::threadPool.enqueue([this, job]() { + threadPool.enqueue([this, job]() { job->execute(); _finishedJobs.push(job); }); } void clearEnqueuedJobs() { - TileProvider::threadPool.clearTasks(); + threadPool.clearTasks(); } std::shared_ptr> popFinishedJob() { @@ -99,7 +100,7 @@ namespace openspace { private: ConcurrentQueue>> _finishedJobs; - + ThreadPool& threadPool; }; diff --git a/modules/globebrowsing/other/tileprovider.cpp b/modules/globebrowsing/other/tileprovider.cpp index 36c7c85378..4bc4dc4126 100644 --- a/modules/globebrowsing/other/tileprovider.cpp +++ b/modules/globebrowsing/other/tileprovider.cpp @@ -24,6 +24,7 @@ #include #include +#include #include @@ -57,6 +58,7 @@ namespace openspace { , _tileCache(tileCacheSize) // setting cache size , _framesSinceLastRequestFlush(0) , _framesUntilRequestFlush(framesUntilRequestFlush) + , _tileLoadManager(TileProviderManager::tileRequestThreadPool) { // Set a temporary texture std::string fileName = "textures/earth_bluemarble.jpg"; diff --git a/modules/globebrowsing/other/tileprovider.h b/modules/globebrowsing/other/tileprovider.h index 28b43f44f9..32ff275e83 100644 --- a/modules/globebrowsing/other/tileprovider.h +++ b/modules/globebrowsing/other/tileprovider.h @@ -121,7 +121,6 @@ namespace openspace { - ////////////////////////////////////////////////////////////////////////////////// // Member variables // ////////////////////////////////////////////////////////////////////////////////// From b74e3bbb5ba46f85d7efd7eaefd2e8d46a0c1bc1 Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Fri, 20 May 2016 17:03:25 -0400 Subject: [PATCH 7/9] Removed unused include of atomic - avoids annoying comiler warnings --- modules/globebrowsing/other/concurrentjobmanager.h | 1 - modules/globebrowsing/other/threadpool.cpp | 1 - modules/globebrowsing/other/threadpool.h | 1 - 3 files changed, 3 deletions(-) diff --git a/modules/globebrowsing/other/concurrentjobmanager.h b/modules/globebrowsing/other/concurrentjobmanager.h index 9843ea02de..785ebfca30 100644 --- a/modules/globebrowsing/other/concurrentjobmanager.h +++ b/modules/globebrowsing/other/concurrentjobmanager.h @@ -30,7 +30,6 @@ #include #include #include -#include #include #include diff --git a/modules/globebrowsing/other/threadpool.cpp b/modules/globebrowsing/other/threadpool.cpp index daf3c80559..4463b7df8c 100644 --- a/modules/globebrowsing/other/threadpool.cpp +++ b/modules/globebrowsing/other/threadpool.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include diff --git a/modules/globebrowsing/other/threadpool.h b/modules/globebrowsing/other/threadpool.h index d792ee01d0..0b61d93df7 100644 --- a/modules/globebrowsing/other/threadpool.h +++ b/modules/globebrowsing/other/threadpool.h @@ -30,7 +30,6 @@ #include #include #include -#include #include From 50a0141cce3e00e3170a1668f3b2e60ed64befdc Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Mon, 23 May 2016 16:37:31 -0400 Subject: [PATCH 8/9] Keeping track of tiles with IO errors, and display parent instead --- .../other/TileProviderManager.cpp | 2 +- .../other/texturedataprovider.cpp | 27 +++- .../globebrowsing/other/texturedataprovider.h | 13 +- modules/globebrowsing/other/tileprovider.cpp | 116 ++++++++---------- modules/globebrowsing/other/tileprovider.h | 35 ++++-- .../shaders/globalchunkedlodpatch_fs.glsl | 2 +- .../shaders/localchunkedlodpatch_fs.glsl | 2 +- 7 files changed, 110 insertions(+), 87 deletions(-) diff --git a/modules/globebrowsing/other/TileProviderManager.cpp b/modules/globebrowsing/other/TileProviderManager.cpp index 9a9ca49957..695150ccfd 100644 --- a/modules/globebrowsing/other/TileProviderManager.cpp +++ b/modules/globebrowsing/other/TileProviderManager.cpp @@ -31,7 +31,7 @@ namespace { namespace openspace { - ThreadPool TileProviderManager::tileRequestThreadPool(1); + ThreadPool TileProviderManager::tileRequestThreadPool(5); TileProviderManager::TileProviderManager() { diff --git a/modules/globebrowsing/other/texturedataprovider.cpp b/modules/globebrowsing/other/texturedataprovider.cpp index 700abf3136..96c2d64b17 100644 --- a/modules/globebrowsing/other/texturedataprovider.cpp +++ b/modules/globebrowsing/other/texturedataprovider.cpp @@ -46,13 +46,15 @@ namespace openspace { } - std::shared_ptr TextureDataProvider::getTextureData(GDALDataset* dataSet, + std::shared_ptr TextureDataProvider::getTextureData(GDALDataset* dataSet, ChunkIndex chunkIndex, int tileLevelDifference) { GdalDataRegion region(dataSet, chunkIndex, tileLevelDifference); DataLayout dataLayout(dataSet, region); char* imageData = new char[dataLayout.totalNumBytes]; + CPLErr worstError = CPLErr::CE_None; + // Read the data (each rasterband is a separate channel) for (size_t i = 0; i < region.numRasters; i++) { GDALRasterBand* rasterBand = dataSet->GetRasterBand(i + 1)->GetOverview(region.overview); @@ -72,15 +74,24 @@ namespace openspace { dataLayout.bytesPerPixel, // Pixel spacing dataLayout.bytesPerLine); // Line spacing - if (err != CE_None) { - ;//LERROR("There was a IO error (" << err << ") for: " << dataSet->GetDescription()); - } + // CE_None = 0, CE_Debug = 1, CE_Warning = 2, CE_Failure = 3, CE_Fatal = 4 + worstError = std::max(worstError, err); } - return createRawTileData(region, dataLayout, imageData); + std::shared_ptr tileData = nullptr; + //if (worstError <= CE_Warning) { + tileData = createRawTileData(region, dataLayout, imageData); + //} + + std::shared_ptr result(new TileIOResult); + result->error = worstError; + result->rawTileData = tileData; + + return result; + //return tileData; } - + /* void TextureDataProvider::asyncRequest(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference) { GdalDataRegion region(dataSet, chunkIndex, tileLevelDifference); DataLayout dataLayout(dataSet, region); @@ -154,6 +165,7 @@ namespace openspace { loadedTextureTiles.pop(); return tile; } + */ @@ -162,6 +174,9 @@ namespace openspace { std::shared_ptr TextureDataProvider::createRawTileData(const GdalDataRegion& region, const DataLayout& dataLayout, const char* imageData) { + + //if(cplError == CPLErr::CE_Fatal) + // GDAL reads image data top to bottom. We want the opposite. char* imageDataYflipped = new char[dataLayout.totalNumBytes]; for (size_t y = 0; y < region.numPixels.y; y++) { diff --git a/modules/globebrowsing/other/texturedataprovider.h b/modules/globebrowsing/other/texturedataprovider.h index 76666eb4da..2d6f064373 100644 --- a/modules/globebrowsing/other/texturedataprovider.h +++ b/modules/globebrowsing/other/texturedataprovider.h @@ -43,6 +43,7 @@ namespace openspace { using namespace ghoul::opengl; + struct RawTileData { struct TextureFormat { @@ -67,11 +68,16 @@ namespace openspace { glm::uvec3 dimensions; TextureFormat texFormat; GLuint glType; - const ChunkIndex chunkIndex; + ChunkIndex chunkIndex; }; + struct TileIOResult { + CPLErr error; + std::shared_ptr rawTileData; + }; + @@ -83,14 +89,15 @@ namespace openspace { ~TextureDataProvider(); - std::shared_ptr getTextureData( + std::shared_ptr getTextureData( GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference); - + /* void asyncRequest(GDALDataset * dataSet, ChunkIndex chunkIndex, int tileLevelDifference); void updateAsyncRequests(); bool hasTextureTileData() const; std::shared_ptr nextTextureTile(); + */ private: diff --git a/modules/globebrowsing/other/tileprovider.cpp b/modules/globebrowsing/other/tileprovider.cpp index 4bc4dc4126..ec7a504324 100644 --- a/modules/globebrowsing/other/tileprovider.cpp +++ b/modules/globebrowsing/other/tileprovider.cpp @@ -44,9 +44,27 @@ namespace { namespace openspace { + + + TileDepthTransform::TileDepthTransform() { } + + TileDepthTransform::TileDepthTransform(GDALDataset* dataset) { + GDALRasterBand* firstBand = dataset->GetRasterBand(1); + GDALDataType gdalType = firstBand->GetRasterDataType(); + + double maximumValue = (gdalType == GDT_Float32 || gdalType == GDT_Float64) ? + 1.0 : firstBand->GetMaximum(); + + depthOffset = firstBand->GetOffset(); + depthScale = firstBand->GetScale() * maximumValue; + } + + + + bool TileProvider::hasInitializedGDAL = false; - ThreadPool TileProvider::threadPool(10); + ThreadPool TileProvider::threadPool(1); TileProvider::TileProvider( @@ -74,24 +92,14 @@ namespace openspace { _defaultTexture->setFilter(ghoul::opengl::Texture::FilterMode::Linear); _defaultTexture->setWrapping(ghoul::opengl::Texture::WrappingMode::ClampToBorder); } - - - int downloadApplicationVersion = 1; - if (!DownloadManager::isInitialized()) { - DownloadManager::initialize( - "../tmp_openspace_downloads/", - downloadApplicationVersion); - } if (!hasInitializedGDAL) { GDALAllRegister(); + //CPLSetConfigOption("GDAL_CACHEMAX", "0"); hasInitializedGDAL = true; } - std::string absFilePath = absPath(filePath); - _gdalDataSet = (GDALDataset *)GDALOpen(absFilePath.c_str(), GA_ReadOnly); - //auto desc = _gdalDataSet->GetDescription(); - + _gdalDataSet = (GDALDataset *)GDALOpen(absPath(filePath).c_str(), GA_ReadOnly); ghoul_assert(_gdalDataSet != nullptr, "Failed to load dataset: " << filePath); GDALRasterBand* firstBand = _gdalDataSet->GetRasterBand(1); @@ -100,15 +108,7 @@ namespace openspace { _tileLevelDifference = log2(minimumPixelSize) - log2(sizeLevel0); - GDALDataType gdalType = firstBand->GetRasterDataType(); - double maximumValue = (gdalType == GDT_Float32 || gdalType == GDT_Float64) ? - 1.0 : firstBand->GetMaximum(); - - _depthTransform.depthOffset = firstBand->GetOffset(); - _depthTransform.depthScale = firstBand->GetScale() * maximumValue; - - - + _depthTransform = TileDepthTransform(_gdalDataSet); } TileProvider::~TileProvider(){ @@ -118,7 +118,7 @@ namespace openspace { void TileProvider::prerender() { - _rawTextureTileDataProvider.updateAsyncRequests(); + //_rawTextureTileDataProvider.updateAsyncRequests(); initTexturesFromLoadedData(); if (_framesSinceLastRequestFlush++ > _framesUntilRequestFlush) { @@ -128,14 +128,15 @@ namespace openspace { void TileProvider::initTexturesFromLoadedData() { while (_tileLoadManager.numFinishedJobs() > 0) { - auto rawTextureTile = _tileLoadManager.popFinishedJob()->product(); - initializeAndAddToCache(rawTextureTile); + std::shared_ptr tileIOResult= _tileLoadManager.popFinishedJob()->product(); + initializeAndAddToCache(tileIOResult); } - + /* while (_rawTextureTileDataProvider.hasTextureTileData()) { auto rawTextureTile = _rawTextureTileDataProvider.nextTextureTile(); initializeAndAddToCache(rawTextureTile); } + */ } void TileProvider::clearRequestQueue() { @@ -164,8 +165,9 @@ namespace openspace { TileUvTransform& uvTransform) { HashKey key = chunkIndex.hashKey(); - if (_tileCache.exist(key)) { - return { _tileCache.get(key), uvTransform }; + if (_tileCache.exist(key) && _tileCache.get(key).ioError == CPLErr::CE_None) { + std::shared_ptr texture = _tileCache.get(key).texture; + return { texture, uvTransform }; } else if (chunkIndex.level <= 1) { return { getDefaultTexture(), uvTransform }; @@ -204,7 +206,7 @@ namespace openspace { std::shared_ptr TileProvider::getOrStartFetchingTile(ChunkIndex chunkIndex) { HashKey hashkey = chunkIndex.hashKey(); if (_tileCache.exist(hashkey)) { - return _tileCache.get(hashkey); + return _tileCache.get(hashkey).texture; } else { enqueueTileRequest(chunkIndex); @@ -216,20 +218,9 @@ namespace openspace { HashKey key = chunkIndex.hashKey(); bool tileHasBeenQueued = _queuedTileRequests.find(key) != _queuedTileRequests.end(); if (!tileHasBeenQueued) { - - bool requestDataAsync = false; - if (requestDataAsync) { - _rawTextureTileDataProvider.asyncRequest(_gdalDataSet, chunkIndex, _tileLevelDifference); - } - else { - - // enque load job - std::shared_ptr job = std::shared_ptr( - new TextureTileLoadJob(this, chunkIndex)); - - _tileLoadManager.enqueueJob(job); - } - + std::shared_ptr job = std::shared_ptr( + new TextureTileLoadJob(this, chunkIndex)); + _tileLoadManager.enqueueJob(job); _queuedTileRequests.insert(key); } return !tileHasBeenQueued; @@ -246,38 +237,35 @@ namespace openspace { } - std::shared_ptr TileProvider::getTextureData( + std::shared_ptr TileProvider::syncDownloadData( const ChunkIndex& chunkIndex) { - - // We assume here that all rasterbands have the same data type - GDALDataType gdalType = _gdalDataSet->GetRasterBand(1)->GetRasterDataType(); - - - //auto provider = TextureDataProviderFactory::get(gdalType); - //return provider->getTextureData(_gdalDataSet, chunkIndex, _tileLevelDifference); - - return _rawTextureTileDataProvider.getTextureData( + std::shared_ptr res = _rawTextureTileDataProvider.getTextureData( _gdalDataSet, chunkIndex, _tileLevelDifference); + + return res; } - void TileProvider::initializeAndAddToCache( - std::shared_ptr rawTileData) { - HashKey key = rawTileData->chunkIndex.hashKey(); - Texture* tex = new Texture( - rawTileData->imageData, - rawTileData->dimensions, - rawTileData->texFormat.ghoulFormat, - rawTileData->texFormat.glFormat, - rawTileData->glType, + void TileProvider::initializeAndAddToCache(std::shared_ptr tileIOResult) { + + std::shared_ptr tileData = tileIOResult->rawTileData; + HashKey key = tileData->chunkIndex.hashKey(); + Texture* texturePtr = new Texture( + tileData->imageData, + tileData->dimensions, + tileData->texFormat.ghoulFormat, + tileData->texFormat.glFormat, + tileData->glType, Texture::FilterMode::Linear, Texture::WrappingMode::ClampToEdge); // The texture should take ownership of the data - std::shared_ptr texture = std::shared_ptr(tex); + std::shared_ptr texture = std::shared_ptr(texturePtr); texture->uploadTexture(); - _tileCache.put(key, texture); + MetaTexture metaTexture = { texture, tileIOResult->error }; + + _tileCache.put(key, metaTexture); } diff --git a/modules/globebrowsing/other/tileprovider.h b/modules/globebrowsing/other/tileprovider.h index 32ff275e83..b81b2b9ecf 100644 --- a/modules/globebrowsing/other/tileprovider.h +++ b/modules/globebrowsing/other/tileprovider.h @@ -46,24 +46,38 @@ namespace openspace { using namespace ghoul::opengl; + - struct TileDepthTransform - { + struct TileDepthTransform { + TileDepthTransform(); + TileDepthTransform(GDALDataset* dataset); + float depthScale; float depthOffset; }; + + struct TileUvTransform { glm::vec2 uvOffset; glm::vec2 uvScale; }; + + struct Tile { std::shared_ptr texture; TileUvTransform uvTransform; }; + + struct MetaTexture { + std::shared_ptr texture; + CPLErr ioError; + }; + + /** Provides tiles through GDAL datasets which can be defined with xml files for example for wms. @@ -102,14 +116,14 @@ namespace openspace { /** Fetches all the needeed texture data from the GDAL dataset. */ - std::shared_ptr getTextureData( + std::shared_ptr syncDownloadData( const ChunkIndex& chunkIndex); /** Creates an OpenGL texture and pushes the data to the GPU. */ void initializeAndAddToCache( - std::shared_ptr uninitedTexture); + std::shared_ptr uninitedTexture); bool enqueueTileRequest(const ChunkIndex& ci); @@ -125,7 +139,7 @@ namespace openspace { // Member variables // ////////////////////////////////////////////////////////////////////////////////// - LRUCache> _tileCache; + LRUCache _tileCache; std::set _queuedTileRequests; int _framesSinceLastRequestFlush; @@ -138,7 +152,7 @@ namespace openspace { TextureDataProvider _rawTextureTileDataProvider; - ConcurrentJobManager _tileLoadManager; + ConcurrentJobManager _tileLoadManager; std::shared_ptr _defaultTexture; int _tileLevelDifference; @@ -163,7 +177,7 @@ namespace openspace { using namespace ghoul::opengl; - struct TextureTileLoadJob : public Job { + struct TextureTileLoadJob : public Job { TextureTileLoadJob(TileProvider * tileProvider, const ChunkIndex& chunkIndex) : _tileProvider(tileProvider) , _chunkIndex(chunkIndex) { @@ -173,11 +187,10 @@ namespace openspace { virtual ~TextureTileLoadJob() { } virtual void execute() { - _uninitedTexture = _tileProvider->getTextureData(_chunkIndex); - + _uninitedTexture = _tileProvider->syncDownloadData(_chunkIndex); } - virtual std::shared_ptr product() { + virtual std::shared_ptr product() { return _uninitedTexture; } @@ -185,7 +198,7 @@ namespace openspace { private: ChunkIndex _chunkIndex; TileProvider * _tileProvider; - std::shared_ptr _uninitedTexture; + std::shared_ptr _uninitedTexture; }; } diff --git a/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl b/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl index 1665862311..11ca813371 100644 --- a/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl +++ b/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl @@ -54,7 +54,7 @@ Fragment getFragment() { //frag.color = frag.color * 0.9 + 0.2*vec4(samplePos, 0, 1); // Border overlay - //frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); + frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); frag.depth = fs_position.w; diff --git a/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl b/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl index f0cb8f2ed0..4cfa122486 100644 --- a/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl +++ b/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl @@ -59,7 +59,7 @@ Fragment getFragment() { //frag.color = frag.color * 0.9 + 0.2*vec4(samplePos, 0, 1); // Border overlay - //frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); + frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); frag.depth = fs_position.w; From 3bb2161a09eb71798fa663ce7db55d311038d9bd Mon Sep 17 00:00:00 2001 From: Erik Broberg Date: Mon, 23 May 2016 18:26:25 -0400 Subject: [PATCH 9/9] 1 Thread for Tile loading, allowing MaxConnections concurrent connections per RasterIO --- data/scene/debugglobe/debugglobe.mod | 3 ++- .../ESRI_Imagery_World_2D.wms | 1 + .../frmt_wms_virtualearth.xml | 7 ++++++ .../debugglobe/map_service_configs/test.wms | 17 +++++++++++++ modules/globebrowsing/globes/chunkindex.cpp | 5 ++++ modules/globebrowsing/globes/chunkindex.h | 2 ++ .../other/TileProviderManager.cpp | 3 ++- .../other/texturedataprovider.cpp | 3 --- modules/globebrowsing/other/tileprovider.cpp | 24 ++++++++++++------- modules/globebrowsing/other/tileprovider.h | 2 +- .../shaders/globalchunkedlodpatch_fs.glsl | 2 +- .../shaders/localchunkedlodpatch_fs.glsl | 2 +- 12 files changed, 55 insertions(+), 16 deletions(-) create mode 100644 data/scene/debugglobe/map_service_configs/frmt_wms_virtualearth.xml create mode 100644 data/scene/debugglobe/map_service_configs/test.wms diff --git a/data/scene/debugglobe/debugglobe.mod b/data/scene/debugglobe/debugglobe.mod index fb0ddd01b5..391ca4ffc9 100644 --- a/data/scene/debugglobe/debugglobe.mod +++ b/data/scene/debugglobe/debugglobe.mod @@ -13,11 +13,12 @@ return { FilePath = "map_service_configs/ESRI_Imagery_World_2D.wms", }, + + --[[ { Name = "Coastlines", FilePath = "map_service_configs/Coastlines.xml", }, - --[[ { Name = "VIIRS_SNPP_CorrectedReflectance_TrueColor", FilePath = "map_service_configs/VIIRS_SNPP_CorrectedReflectance_TrueColor.xml" diff --git a/data/scene/debugglobe/map_service_configs/ESRI_Imagery_World_2D.wms b/data/scene/debugglobe/map_service_configs/ESRI_Imagery_World_2D.wms index 23eebe2657..bc36b79e7e 100644 --- a/data/scene/debugglobe/map_service_configs/ESRI_Imagery_World_2D.wms +++ b/data/scene/debugglobe/map_service_configs/ESRI_Imagery_World_2D.wms @@ -12,4 +12,5 @@ 512 512 3 + 5 \ No newline at end of file diff --git a/data/scene/debugglobe/map_service_configs/frmt_wms_virtualearth.xml b/data/scene/debugglobe/map_service_configs/frmt_wms_virtualearth.xml new file mode 100644 index 0000000000..3b628e9a89 --- /dev/null +++ b/data/scene/debugglobe/map_service_configs/frmt_wms_virtualearth.xml @@ -0,0 +1,7 @@ + + + http://a${server_num}.ortho.tiles.virtualearth.net/tiles/a${quadkey}.jpeg?g=90 + + 4 + + diff --git a/data/scene/debugglobe/map_service_configs/test.wms b/data/scene/debugglobe/map_service_configs/test.wms new file mode 100644 index 0000000000..835e83473d --- /dev/null +++ b/data/scene/debugglobe/map_service_configs/test.wms @@ -0,0 +1,17 @@ + + + http://services.arcgisonline.com/ArcGIS/rest/services/ESRI_Imagery_World_2D/MapServer/tile/${z}/${y}/${x} + + + + 15 + 2 + 1 + + top + + EPSG:4326 + 512 + 512 + 3 + \ No newline at end of file diff --git a/modules/globebrowsing/globes/chunkindex.cpp b/modules/globebrowsing/globes/chunkindex.cpp index 9b0230140d..06af21d7df 100644 --- a/modules/globebrowsing/globes/chunkindex.cpp +++ b/modules/globebrowsing/globes/chunkindex.cpp @@ -69,6 +69,11 @@ namespace openspace { return ChunkIndex(newX, newY, level); } + int ChunkIndex::manhattan(const ChunkIndex& other) const { + ghoul_assert(level == other.level, "makes no sense if not on same level"); + return std::abs(x - other.x) + std::abs(y - other.y); + } + HashKey ChunkIndex::hashKey() const { return x ^ (y << 16) ^ (level << 24); } diff --git a/modules/globebrowsing/globes/chunkindex.h b/modules/globebrowsing/globes/chunkindex.h index 0f70e6f690..231d7c95f8 100644 --- a/modules/globebrowsing/globes/chunkindex.h +++ b/modules/globebrowsing/globes/chunkindex.h @@ -89,6 +89,8 @@ struct ChunkIndex { */ ChunkIndex getRelatedTile(int deltaX, int deltaY) const; + int manhattan(const ChunkIndex& other) const; + HashKey hashKey() const; bool operator==(const ChunkIndex& other) const; diff --git a/modules/globebrowsing/other/TileProviderManager.cpp b/modules/globebrowsing/other/TileProviderManager.cpp index 695150ccfd..8b5cf75895 100644 --- a/modules/globebrowsing/other/TileProviderManager.cpp +++ b/modules/globebrowsing/other/TileProviderManager.cpp @@ -31,7 +31,8 @@ namespace { namespace openspace { - ThreadPool TileProviderManager::tileRequestThreadPool(5); + ThreadPool TileProviderManager::tileRequestThreadPool(1); + TileProviderManager::TileProviderManager() { diff --git a/modules/globebrowsing/other/texturedataprovider.cpp b/modules/globebrowsing/other/texturedataprovider.cpp index 96c2d64b17..889ffb52d1 100644 --- a/modules/globebrowsing/other/texturedataprovider.cpp +++ b/modules/globebrowsing/other/texturedataprovider.cpp @@ -79,16 +79,13 @@ namespace openspace { } std::shared_ptr tileData = nullptr; - //if (worstError <= CE_Warning) { tileData = createRawTileData(region, dataLayout, imageData); - //} std::shared_ptr result(new TileIOResult); result->error = worstError; result->rawTileData = tileData; return result; - //return tileData; } /* diff --git a/modules/globebrowsing/other/tileprovider.cpp b/modules/globebrowsing/other/tileprovider.cpp index ec7a504324..28e9d319c6 100644 --- a/modules/globebrowsing/other/tileprovider.cpp +++ b/modules/globebrowsing/other/tileprovider.cpp @@ -64,7 +64,6 @@ namespace openspace { bool TileProvider::hasInitializedGDAL = false; - ThreadPool TileProvider::threadPool(1); TileProvider::TileProvider( @@ -216,14 +215,23 @@ namespace openspace { bool TileProvider::enqueueTileRequest(const ChunkIndex& chunkIndex) { HashKey key = chunkIndex.hashKey(); - bool tileHasBeenQueued = _queuedTileRequests.find(key) != _queuedTileRequests.end(); - if (!tileHasBeenQueued) { - std::shared_ptr job = std::shared_ptr( - new TextureTileLoadJob(this, chunkIndex)); - _tileLoadManager.enqueueJob(job); - _queuedTileRequests.insert(key); + auto it = _queuedTileRequests.begin(); + auto end = _queuedTileRequests.end(); + for (; it != end; it++) { + const ChunkIndex& otherChunk = it->second; + if (chunkIndex.level == otherChunk.level && + chunkIndex.manhattan(otherChunk) < 1) { + return false; + } } - return !tileHasBeenQueued; + + + std::shared_ptr job = std::shared_ptr( + new TextureTileLoadJob(this, chunkIndex)); + _tileLoadManager.enqueueJob(job); + _queuedTileRequests[key] = chunkIndex; + + return true; } diff --git a/modules/globebrowsing/other/tileprovider.h b/modules/globebrowsing/other/tileprovider.h index b81b2b9ecf..ed58947267 100644 --- a/modules/globebrowsing/other/tileprovider.h +++ b/modules/globebrowsing/other/tileprovider.h @@ -140,7 +140,7 @@ namespace openspace { ////////////////////////////////////////////////////////////////////////////////// LRUCache _tileCache; - std::set _queuedTileRequests; + std::unordered_map _queuedTileRequests; int _framesSinceLastRequestFlush; int _framesUntilRequestFlush; diff --git a/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl b/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl index 11ca813371..1665862311 100644 --- a/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl +++ b/modules/globebrowsing/shaders/globalchunkedlodpatch_fs.glsl @@ -54,7 +54,7 @@ Fragment getFragment() { //frag.color = frag.color * 0.9 + 0.2*vec4(samplePos, 0, 1); // Border overlay - frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); + //frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); frag.depth = fs_position.w; diff --git a/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl b/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl index 4cfa122486..f0cb8f2ed0 100644 --- a/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl +++ b/modules/globebrowsing/shaders/localchunkedlodpatch_fs.glsl @@ -59,7 +59,7 @@ Fragment getFragment() { //frag.color = frag.color * 0.9 + 0.2*vec4(samplePos, 0, 1); // Border overlay - frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); + //frag.color = frag.color + patchBorderOverlay(fs_uv, vec3(0.5, 0.5, 0.5), 0.02); frag.depth = fs_position.w;