/***************************************************************************************** * * * OpenSpace * * * * Copyright (c) 2014-2018 * * * * Permission is hereby granted, free of charge, to any person obtaining a copy of this * * software and associated documentation files (the "Software"), to deal in the Software * * without restriction, including without limitation the rights to use, copy, modify, * * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to * * permit persons to whom the Software is furnished to do so, subject to the following * * conditions: * * * * The above copyright notice and this permission notice shall be included in all copies * * or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, * * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A * * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE * * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * ****************************************************************************************/ #include #include #include #include #include // nfd #include // nfd #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using Directory = ghoul::filesystem::Directory; using File = ghoul::filesystem::File; using RawPath = ghoul::filesystem::Directory::RawPath; using Recursive = ghoul::filesystem::Directory::Recursive; using TestResult = openspace::documentation::TestResult; using json = nlohmann::json; namespace { constexpr const char *_loggerCat = "Loader"; constexpr const char *KeyTime = "Time"; const std::string KeyRenderableType = "RenderableTimeVaryingVolume"; const std::string scaleTypeKey = "StaticScale"; const std::string translationTypeKey = "SpiceTranslation"; const std::string volumesGuiPathKey = "/Solar System/Volumes"; const std::string KeyVolumeToRawTask = "KameleonVolumeToRawTask"; constexpr const char *KeyVariable = "Variable"; const std::string KeyStepSize = "StepSize"; const std::string KeyGridType = "GridType"; const std::string KeySecondsAfter = "SecondsAfter"; const std::string KeySecondsBefore = "SecondsBefore"; constexpr const char *KeyTask = "Task"; constexpr const char *KeyItemName = "ItemName"; constexpr const char *KeyTransform = "Transform"; constexpr const char *KeyTranslation = "Translation"; constexpr const char *KeyPosition = "Position"; constexpr const char *KeyType = "Type"; constexpr const char *KeyObserver = "Observer"; constexpr const char *KeyTarget = "Target"; constexpr const char *KeyStaticTranslation = "StaticTranslation"; constexpr const char *KeyStaticScale = "StaticScale"; constexpr const char *KeyScale = "Scale"; constexpr const char *KeyTransferFunctionPath = "TransferFunctionPath"; constexpr const char *KeyTranslationValues = "TranslationValues"; const double DefaultStepSize = 0.02; const std::string DefaultGridType = "Spherical"; const double DefaultSecondsAfter = 24 * 60; const double DefaultSecondsBefore = 24 * 60; const float sunRadiusScale = 695508000; const bool guiHidden = false; } // namespace namespace { static const openspace::properties::Property::PropertyInfo SelectedFilesInfo = { "SelectedFiles", "List of selected files and ready to load", "This list contains names of selected files in char format"}; static const openspace::properties::Property::PropertyInfo CurrentVolumesConvertedCountInfo = { "CurrentVolumesConvertedCount", "The amount of volumes currently converted", "This number indicates how many of the volumes currently selected " "for conversion have been converted."}; static const openspace::properties::Property::PropertyInfo CurrentVolumesToConvertCount = { "CurrentVolumesToConvertCount", "The amount of volumes to be converted", "This number indicates how many volumes are currently selected to be converted."}; static const openspace::properties::Property::PropertyInfo UploadDataTriggerInfo = { "UploadDataTrigger", "Trigger load data files", "If this property is triggered it will call the function to load data"}; static const openspace::properties::Property::PropertyInfo VolumeConversionProgressInfo = { "VolumeConversionProgress", "Progress value for volume conversion", "This float value between 0 and 1 corresponds to the progress of volume conversion"}; static const openspace::properties::Property::PropertyInfo VolumeMetaDataInfo = { "VolumeMetaData", "The metadata from the selected files", "This dictionary contains the metadata for the first of the selected volume files"}; static const openspace::properties::Property::PropertyInfo VolumeMetaDataJSONInfo = { "VolumeMetaDataJSON", "JSON formatted string of volume meta data", ""}; } // namespace namespace openspace::dataloader { Loader::Loader() : PropertyOwner({"Loader"}), _selectedFilePaths(SelectedFilesInfo), _currentVolumesConvertedCount(CurrentVolumesConvertedCountInfo), _currentVolumesToConvertCount(CurrentVolumesToConvertCount), _uploadDataTrigger(UploadDataTriggerInfo), _volumeConversionProgress(VolumeConversionProgressInfo), _volumeMetaDataJSON(VolumeMetaDataJSONInfo) { _uploadDataTrigger.onChange([this]() { selectData(); }); addProperty(_selectedFilePaths); addProperty(_currentVolumesConvertedCount); addProperty(_currentVolumesToConvertCount); addProperty(_uploadDataTrigger); addProperty(_volumeConversionProgress); addProperty(_volumeMetaDataJSON); _volumeConversionThreadCanRun = false; } void Loader::selectData() { { std::thread t([&]() { nfdpathset_t outPathSet; std::vector paths; nfdresult_t result = NFD_OpenDialogMultiple("cdf", NULL, &outPathSet); size_t count = NFD_PathSet_GetCount(&outPathSet); if (count > 0 && result == NFD_OKAY) { for (size_t i = 0; i < count; ++i) { nfdchar_t *path = NFD_PathSet_GetPath(&outPathSet, i); paths.push_back(static_cast(path)); } // TODO: reset function _volumeConversionProgress = FLT_MIN; _currentVolumesConvertedCount = 0; _volumeConversionThreadCanRun = false; _currentVolumesToConvertCount = count; _selectedFilePaths = paths; NFD_PathSet_Free(&outPathSet); getVolumeMetaData(_volumeMetaDataDictionary); } else if (result == NFD_CANCEL) { LINFO("User pressed cancel."); } else { std::string error = NFD_GetError(); LINFO("Error: \n" + error); } }); t.detach(); } } //void Loader::createInternalDataItemProperties() { // module()->validateDataDirectory(); // std::vector volumeItems = module()->volumeDataItems(); // // // LDEBUG("volume items vec size " + std::to_string(volumeItems.size())); // // for (auto item : volumeItems) { // const char* dirLeaf = openspace::dataloader::helpers::getDirLeaf(item).c_str(); // const char* ItemTrigger = "ItemTrigger_"; // const openspace::properties::Property::PropertyInfo info = { // ItemTrigger + dirLeaf, // dirLeaf, // "" // }; // // // Initialize trigger property with data item name (are they unique?) // auto volumeItemTrigger = properties::TriggerProperty(info); // // // Set onChange method to call loadDataItem with the path as argument // volumeItemTrigger.onChange([this](){ // // loadDataItem(item); // }); // // // addProperty(volumeItemTrigger); // // LDEBUG("Added property " + dirLeaf); // } //} // addDataItemProperty(); // removeDataItemProperties(); // void Loader::createVolumeDataItem(std::string absPath) {} void Loader::loadDataItem(const std::string &absPathToItem) { std::string sourceDir = absPathToItem; const std::string dirLeaf = openspace::dataloader::helpers::getDirLeaf(absPathToItem); if (scene()->sceneGraphNode(dirLeaf)) { LINFO(fmt::format("The sceneGraphNode {} has already been created!", std::string(dirLeaf))); goToFirstTimeStep(absPathToItem); return; } // TODO: Variables to let user initialize in GUI std::string sunKey = "SUN"; const float sunRadiusScale = 695508000; const float auScale = 149600000000; const std::string parent = "SolarSystemBarycenter"; // valid for all volume data? std::string stateFile = openspace::dataloader::helpers::getFileWithExtensionFromItemFolder(absPathToItem, "state"); ghoul::Dictionary renderableDictionary = ghoul::lua::loadDictionaryFromFile(stateFile); if (!renderableDictionary.hasKey(KeyTransform)) { LERROR("Transform was not declared in the .state file!"); } ghoul::Dictionary transformDictionary; renderableDictionary.getValue(KeyTransform, transformDictionary); std::string tfFilePath = absPathToItem + ghoul::filesystem::FileSystem::PathSeparator + "transferfunction.txt"; renderableDictionary.setValue("Type", KeyRenderableType); renderableDictionary.setValue("SourceDirectory", sourceDir); renderableDictionary.setValue("TransferFunction", tfFilePath); std::initializer_list> gui = { std::make_pair("Path", volumesGuiPathKey), std::make_pair("Hidden", guiHidden), }; ghoul::Dictionary guiDictionary(gui); std::initializer_list> completeList = { std::make_pair("Identifier", dirLeaf), std::make_pair("Parent", parent), std::make_pair("Renderable", renderableDictionary), std::make_pair("GUI", guiDictionary)}; LINFO("Before setting transformDictionary to completeDictionary"); ghoul::Dictionary completeDictionary(completeList); completeDictionary.setValue(KeyTransform, transformDictionary); LINFO("Before setting transformDictionary to completeDictionary"); initializeNode(completeDictionary); goToFirstTimeStep(absPathToItem); } void Loader::initializeNode(ghoul::Dictionary dict) { SceneGraphNode *node = scene()->loadNode(dict); scene()->initializeNode(node); } void Loader::goToFirstTimeStep(const std::string &absPathToItem) { std::string firstDictionaryFilePath = openspace::dataloader::helpers::getFileWithExtensionFromItemFolder(absPathToItem, "dictionary"); ghoul::Dictionary dict = ghoul::lua::loadDictionaryFromFile(firstDictionaryFilePath); std::string firstTimeStep = dict.value(KeyTime); setTime(firstTimeStep); } void Loader::getVolumeMetaData(ghoul::Dictionary &metaDataDictionary) { std::vector selectedFileVector = _selectedFilePaths; openspace::kameleonvolume::KameleonVolumeReader reader(selectedFileVector[0]); _volumeMetaDataDictionary = reader.readMetaData(); ghoul::Dictionary gAttribDictionary; ghoul::Dictionary vAttribDictionary; const std::string KeyGlobalAttributes = "globalAttributes"; const std::string KeyVariableAttributes = "variableAttributes"; _volumeMetaDataDictionary.getValue(KeyGlobalAttributes, gAttribDictionary); _volumeMetaDataDictionary.getValue(KeyVariableAttributes, vAttribDictionary); constexpr const char *MetaDataGridSystemKey = "grid_system_1"; constexpr const char *MetaDataGridSystemDimension1 = "grid_system_1_dimension_1_size"; constexpr const char *MetaDataGridSystemDimension2 = "grid_system_1_dimension_2_size"; constexpr const char *MetaDataGridSystemDimension3 = "grid_system_1_dimension_3_size"; const std::vector vKeys = vAttribDictionary.keys(); const std::string gridSystem1 = gAttribDictionary.value(MetaDataGridSystemKey); const int gridSystem1DimensionSize = gAttribDictionary.value(MetaDataGridSystemDimension1); const int gridSystem2DimensionSize = gAttribDictionary.value(MetaDataGridSystemDimension2); const int gridSystem3DimensionSize = gAttribDictionary.value(MetaDataGridSystemDimension3); json j; j["gridSystem"] = gridSystem1; j["gridSystem1DimensionSize"] = gridSystem1DimensionSize; j["gridSystem2DimensionSize"] = gridSystem2DimensionSize; j["gridSystem3DimensionSize"] = gridSystem3DimensionSize; j["variableAttributes"] = vKeys; _volumeMetaDataJSON = j.dump(); } void Loader::processCurrentlySelectedUploadData(const std::string &dictionaryString) { LINFO(dictionaryString); _currentVolumeConversionDictionary = ghoul::lua::loadDictionaryFromString(dictionaryString); TestResult result; result.success = false; result = openspace::documentation::testSpecification( volumeConversionDocumentation(), _currentVolumeConversionDictionary); if (!result.success) { throw "Error in specification for volume conversion dictionary: " + dictionaryString; } else { _volumeConversionThreadCanRun = true; } { std::thread t([&]() { // ??? won't work multithreaded // std::string testPath = "${DATA}" + // ghoul::filesystem::FileSystem::PathSeparator; // testPath += ".internal"; // LINFO(testPath); // Hold up thread until dictionary is loaded and valid while (!_volumeConversionThreadCanRun) { } std::string itemName = _currentVolumeConversionDictionary.value(KeyItemName); std::string itemPathBase = "${DATA}/.internal/volumes_from_cdf/" + itemName; Directory d(itemPathBase, RawPath::No); FileSys.createDirectory(d); ghoul::Dictionary translationDictionary; if (!_currentVolumeConversionDictionary.getValue(KeyTranslation, translationDictionary)) { throw ghoul::RuntimeError("Must provide Translation table for volume conversion."); } // Quick fix to make formatter handle nestled dvec3 type correctly glm::dvec3 position = translationDictionary.value(KeyPosition); translationDictionary.setValue(KeyPosition, position); // Read and create Scale Dictionary float scale = _currentVolumeConversionDictionary.value(KeyScale) * sunRadiusScale; ghoul::Dictionary scaleDictionary; scaleDictionary.setValue(KeyType, KeyStaticScale); scaleDictionary.setValue(KeyScale, scale); // Create Transform Dictionary ghoul::Dictionary transformDictionary; transformDictionary.setValue(KeyTranslation, translationDictionary); transformDictionary.setValue(KeyScale, scaleDictionary); /*** create state file ***/ // Check if file exists // If it exists, clear contents? delete and create new? // Create file, write dictionary to string contents const std::string gridType = _currentVolumeConversionDictionary.value(KeyGridType); std::initializer_list> stateList = { std::make_pair(KeyStepSize, DefaultStepSize), std::make_pair(KeyGridType, gridType), std::make_pair(KeySecondsAfter, DefaultSecondsAfter), std::make_pair(KeySecondsBefore, DefaultSecondsBefore), std::make_pair(KeyTransform, transformDictionary)}; ghoul::Dictionary stateDict(stateList); ghoul::DictionaryLuaFormatter formatter; std::string stateString = formatter.format(stateDict); LINFO(stateString); std::fstream stateStream(absPath(itemPathBase + "/" + itemName + ".state"), std::ios::out); if (!stateStream) { LERROR("Could not create state file"); } stateStream << "return " << stateString; stateStream.close(); /*** copy over tf file ***/ const std::string tfFileToCopyPath = _currentVolumeConversionDictionary.value(KeyTransferFunctionPath); LDEBUG("tf path as it is: " + tfFileToCopyPath); LDEBUG("tf path abs path: " + absPath(tfFileToCopyPath)); std::ifstream tfSource(absPath(tfFileToCopyPath)); std::ofstream tfDest(absPath(itemPathBase + "/transferfunction.txt")); if (!tfSource) { LERROR("Could not open source transferfunction file."); } if (!tfDest) { LERROR("Could not open destination transferfunction file."); } tfDest << tfSource.rdbuf(); tfSource.close(); tfDest.close(); ghoul::Dictionary taskDictionary; if (!_currentVolumeConversionDictionary.getValue(KeyTask, taskDictionary)) { throw ghoul::RuntimeError("Must provide Task dictionary for volume conversion."); } std::mutex m; std::function cb = [&](float progress) { std::lock_guard g(m); _volumeConversionProgress = progress; }; std::vector selectedFiles = _selectedFilePaths; unsigned int counter = 0; for (const std::string &file : selectedFiles) { _volumeConversionProgress = FLT_MIN; auto selectedFile = File(file); LDEBUG("raw path = " + file); LDEBUG("abs path = " + absPath(file)); LDEBUG("file handle path = " + selectedFile.path()); const std::string outputBasePath = d.path() + "/" + selectedFile.filename(); const std::string rawVolumeOutput = outputBasePath + ".rawvolume"; const std::string dictionaryOutput = outputBasePath + ".dictionary"; taskDictionary.setValue("Type", KeyVolumeToRawTask); taskDictionary.setValue("Input", file); taskDictionary.setValue("RawVolumeOutput", rawVolumeOutput); taskDictionary.setValue("DictionaryOutput", dictionaryOutput); auto volumeToRawTask = Task::createFromDictionary(taskDictionary); volumeToRawTask->perform(cb); counter++; _currentVolumesConvertedCount = counter; } LINFO("Created files in " + d.path()); _volumeConversionThreadCanRun = false; loadDataItem(absPath(itemPathBase)); }); t.detach(); } // Create state file, transferfunction file in volume item directory } // void Loader::createVolumeDataItem(std::string absPath) {} // ghoul::Dictionary Loader::createTaskDictionaryForOneVolumeItem(std::string inputPath, std::string outputBasePath) { // defaults // const int dimensions[3] = {100, 100, 128}; // const int lowerDomainBound[3] = {1, -90, 0}; // const int upperDomainBound[3] = {15, 90, 360}; // Set item dirLeaf as namelowerDomainBound // const std::string itemName = openspace::dataloader::helpers::getDirLeaf(inputPath); // const std::string itemOutputFilePath = outputBasePath + // ghoul::filesystem::FileSystem::PathSeparator + // itemName; // const std::string RawVolumeOutput = itemOutputFilePath + ".rawvolume"; // const std::string DictionaryOutput = itemOutputFilePath + ".dictionary"; // std::initializer_list> list = { // std::make_pair( "Type", "KameleonVolumeToRawTask" ), // std::make_pair( "Input", inputPath ), // std::make_pair( "Dimensions", _uploadedDataDimensions ), // std::make_pair( "Variable", _uploadedDataVariable), // std::make_pair( "FactorRSquared", _uploadedDataFactorRSquared ), // std::make_pair( "LowerDomainBound", _uploadedDataLowerDomainBounds ), // std::make_pair( "UpperDomainBound", _uploadedDataHigherDomainBounds ), // std::make_pair( "RawVolumeOutput", RawVolumeOutput ), // std::make_pair( "DictionaryOutput", DictionaryOutput) // }; // return ghoul::Dictionary(list); // } // ghoul::Dictionary Loader::createVolumeItemDictionary(std::string dataDictionaryPath, std::string dataStatePath) { // } documentation::Documentation Loader::volumeConversionDocumentation() { using namespace documentation; return { "LoaderVolumeConversion", "", {{ KeyItemName, new StringAnnotationVerifier("A name for the volume item"), Optional::No, "The volume item name", }, { KeyGridType, new StringAnnotationVerifier("What type of grid for ray caster"), Optional::No, "What type of grid for ray caster", }, { KeyTranslation, new TableAnnotationVerifier("A table with values for translation"), Optional::No, "The table with values for translation transform", }, { KeyTask, new TableAnnotationVerifier("A table with values for volume conversion task"), Optional::No, "The table with values for volume conversion task", }, {KeyTransferFunctionPath, new StringAnnotationVerifier("An absolute path to a transfer function file"), Optional::No, "An absolute path to a transfer function file to copy to the item directory"}}}; } } // namespace openspace::dataloader