From 774fcbe49c84ffdaff68295d8b2244251d7a93eb Mon Sep 17 00:00:00 2001 From: Daniel Pfeifer Date: Sat, 5 Oct 2024 14:45:05 +0200 Subject: [PATCH] CTest: Base command line mode on top of scripting commands Make sure that all CMake variables that are translated into CTest options in `cmCTest*Command` implementations are translated from CTest options into CMake variables before the functions are called. This back-and-forth translation should be temporary. It is a necessary prerequisite for refactoring `cmCTest*Handler` implementations to operate on CMake variables directly rather than CTest options. --- Source/CTest/cmCTestScriptHandler.h | 1 + Source/CTest/cmCTestTestHandler.cxx | 27 +++ Source/CTest/cmCTestTestHandler.h | 5 + Source/cmCTest.cxx | 178 +++++++++++++++--- Source/cmCTest.h | 5 + .../EmptyDirCoverage-ctest-result.txt | 1 + .../EmptyDirCoverage-ctest-stderr.txt | 2 +- 7 files changed, 191 insertions(+), 28 deletions(-) create mode 100644 Tests/RunCMake/CTestCommandLine/EmptyDirCoverage-ctest-result.txt diff --git a/Source/CTest/cmCTestScriptHandler.h b/Source/CTest/cmCTestScriptHandler.h index 8aa07e7342..a3f1339e82 100644 --- a/Source/CTest/cmCTestScriptHandler.h +++ b/Source/CTest/cmCTestScriptHandler.h @@ -112,6 +112,7 @@ public: void CreateCMake(); cmake* GetCMake() { return this->CMake.get(); } + cmMakefile* GetMakefile() { return this->Makefile.get(); } void SetRunCurrentScript(bool value); diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index c7875cd99e..d108271391 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -380,6 +380,33 @@ void cmCTestTestHandler::PopulateCustomVectors(cmMakefile* mf) } } +void cmCTestTestHandler::SetCMakeVariables(cmMakefile& mf) +{ + mf.AddDefinition("CTEST_CUSTOM_PRE_TEST", + cmList(this->CustomPreTest).to_string()); + mf.AddDefinition("CTEST_CUSTOM_POST_TEST", + cmList(this->CustomPostTest).to_string()); + mf.AddDefinition("CTEST_CUSTOM_TESTS_IGNORE", + cmList(this->CustomTestsIgnore).to_string()); + mf.AddDefinition("CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE", + std::to_string(this->CustomMaximumPassedTestOutputSize)); + mf.AddDefinition("CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE", + std::to_string(this->CustomMaximumFailedTestOutputSize)); + mf.AddDefinition("CTEST_CUSTOM_TEST_OUTPUT_TRUNCATION", + [this]() -> cm::string_view { + switch (this->TestOutputTruncation) { + case cmCTestTypes::TruncationMode::Tail: + return "tail"_s; + case cmCTestTypes::TruncationMode::Middle: + return "middle"_s; + case cmCTestTypes::TruncationMode::Head: + return "head"_s; + default: + return ""_s; + } + }()); +} + int cmCTestTestHandler::PreProcessHandler() { if (!this->ExecuteCommands(this->CustomPreTest)) { diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h index dd1bc598b6..024b544eaa 100644 --- a/Source/CTest/cmCTestTestHandler.h +++ b/Source/CTest/cmCTestTestHandler.h @@ -228,6 +228,11 @@ public: // Support for writing test results in JUnit XML format. void SetJUnitXMLFileName(const std::string& id); + /** + * Set CMake variables from CTest Options + */ + void SetCMakeVariables(cmMakefile& mf); + protected: using SetOfTests = std::set; diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 829fbf9238..852682589f 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -54,10 +54,12 @@ #include "cmCTestUpdateHandler.h" #include "cmCTestUploadHandler.h" #include "cmDynamicLoader.h" +#include "cmExecutionStatus.h" #include "cmGeneratedFileStream.h" #include "cmGlobalGenerator.h" #include "cmJSONState.h" #include "cmList.h" +#include "cmListFileCache.h" #include "cmMakefile.h" #include "cmProcessOutput.h" #include "cmState.h" @@ -943,56 +945,101 @@ int cmCTest::ProcessSteps() { int res = 0; bool notest = true; - int update_count = 0; for (Part p = PartStart; notest && p != PartCount; p = static_cast(p + 1)) { notest = !this->Impl->Parts[p]; } + + if (notest) { + if (this->GetTestHandler()->ProcessHandler() < 0) { + cmCTestLog(this, ERROR_MESSAGE, "Errors while running CTest\n"); + if (!this->Impl->OutputTestOutputOnTestFailure) { + const std::string lastTestLog = + this->GetBinaryDir() + "/Testing/Temporary/LastTest.log"; + cmCTestLog(this, ERROR_MESSAGE, + "Output from these tests are in: " << lastTestLog << '\n'); + cmCTestLog(this, ERROR_MESSAGE, + "Use \"--rerun-failed --output-on-failure\" to re-run the " + "failed cases verbosely.\n"); + } + return cmCTest::TEST_ERRORS; + } + return 0; + } + + cmCTestScriptHandler script; + script.SetCTestInstance(this); + script.CreateCMake(); + cmMakefile& mf = *script.GetMakefile(); + this->SetCMakeVariables(mf); + std::vector args{ + cmListFileArgument("RETURN_VALUE", cmListFileArgument::Unquoted, 0), + cmListFileArgument("return_value", cmListFileArgument::Unquoted, 0), + }; + if (this->Impl->Parts[PartUpdate] && (this->GetRemainingTimeAllowed() > std::chrono::minutes(2))) { - cmCTestUpdateHandler* uphandler = this->GetUpdateHandler(); - uphandler->SetPersistentOption( - "SourceDirectory", this->GetCTestConfiguration("SourceDirectory")); - update_count = uphandler->ProcessHandler(); - if (update_count < 0) { + auto const func = cmListFileFunction("ctest_update", 0, 0, args); + auto status = cmExecutionStatus(mf); + if (!mf.ExecuteCommand(func, status)) { res |= cmCTest::UPDATE_ERRORS; } } - if (this->Impl->TestModel == cmCTest::CONTINUOUS && !update_count) { + if (this->Impl->TestModel == cmCTest::CONTINUOUS && + mf.GetDefinition("return_value").IsOff()) { return 0; } if (this->Impl->Parts[PartConfigure] && (this->GetRemainingTimeAllowed() > std::chrono::minutes(2))) { - if (this->GetConfigureHandler()->ProcessHandler() < 0) { + auto const func = cmListFileFunction("ctest_configure", 0, 0, args); + auto status = cmExecutionStatus(mf); + if (!mf.ExecuteCommand(func, status) || + std::stoi(mf.GetDefinition("return_value")) < 0) { res |= cmCTest::CONFIGURE_ERRORS; } } if (this->Impl->Parts[PartBuild] && (this->GetRemainingTimeAllowed() > std::chrono::minutes(2))) { this->UpdateCTestConfiguration(); - if (this->GetBuildHandler()->ProcessHandler() < 0) { + this->SetCMakeVariables(mf); + auto const func = cmListFileFunction("ctest_build", 0, 0, args); + auto status = cmExecutionStatus(mf); + if (!mf.ExecuteCommand(func, status) || + std::stoi(mf.GetDefinition("return_value")) < 0) { res |= cmCTest::BUILD_ERRORS; } } if ((this->Impl->Parts[PartTest] || notest) && (this->GetRemainingTimeAllowed() > std::chrono::minutes(2))) { this->UpdateCTestConfiguration(); - if (this->GetTestHandler()->ProcessHandler() < 0) { + this->SetCMakeVariables(mf); + auto const func = cmListFileFunction("ctest_test", 0, 0, args); + auto status = cmExecutionStatus(mf); + if (!mf.ExecuteCommand(func, status) || + std::stoi(mf.GetDefinition("return_value")) < 0) { res |= cmCTest::TEST_ERRORS; } } if (this->Impl->Parts[PartCoverage] && (this->GetRemainingTimeAllowed() > std::chrono::minutes(2))) { this->UpdateCTestConfiguration(); - if (this->GetCoverageHandler()->ProcessHandler() < 0) { + this->SetCMakeVariables(mf); + auto const func = cmListFileFunction("ctest_coverage", 0, 0, args); + auto status = cmExecutionStatus(mf); + if (!mf.ExecuteCommand(func, status) || + std::stoi(mf.GetDefinition("return_value")) < 0) { res |= cmCTest::COVERAGE_ERRORS; } } if (this->Impl->Parts[PartMemCheck] && (this->GetRemainingTimeAllowed() > std::chrono::minutes(2))) { this->UpdateCTestConfiguration(); - if (this->GetMemCheckHandler()->ProcessHandler() < 0) { + this->SetCMakeVariables(mf); + auto const func = cmListFileFunction("ctest_memcheck", 0, 0, args); + auto status = cmExecutionStatus(mf); + if (!mf.ExecuteCommand(func, status) || + std::stoi(mf.GetDefinition("return_value")) < 0) { res |= cmCTest::MEMORY_ERRORS; } } @@ -1023,24 +1070,26 @@ int cmCTest::ProcessSteps() } if (this->Impl->Parts[PartSubmit]) { this->UpdateCTestConfiguration(); - if (this->GetSubmitHandler()->ProcessHandler() < 0) { + this->SetCMakeVariables(mf); + + std::string count = this->GetCTestConfiguration("CTestSubmitRetryCount"); + std::string delay = this->GetCTestConfiguration("CTestSubmitRetryDelay"); + auto const func = cmListFileFunction( + "ctest_submit", 0, 0, + { + cmListFileArgument("RETRY_COUNT", cmListFileArgument::Unquoted, 0), + cmListFileArgument(count, cmListFileArgument::Quoted, 0), + cmListFileArgument("RETRY_DELAY", cmListFileArgument::Unquoted, 0), + cmListFileArgument(delay, cmListFileArgument::Quoted, 0), + cmListFileArgument("RETURN_VALUE", cmListFileArgument::Unquoted, 0), + cmListFileArgument("return_value", cmListFileArgument::Unquoted, 0), + }); + auto status = cmExecutionStatus(mf); + if (!mf.ExecuteCommand(func, status) || + std::stoi(mf.GetDefinition("return_value")) < 0) { res |= cmCTest::SUBMIT_ERRORS; } } - if (res != 0) { - cmCTestLog(this, ERROR_MESSAGE, "Errors while running CTest" << std::endl); - if (!this->Impl->OutputTestOutputOnTestFailure) { - const std::string lastTestLog = - this->GetBinaryDir() + "/Testing/Temporary/LastTest.log"; - cmCTestLog(this, ERROR_MESSAGE, - "Output from these tests are in: " << lastTestLog - << std::endl); - cmCTestLog(this, ERROR_MESSAGE, - "Use \"--rerun-failed --output-on-failure\" to re-run the " - "failed cases verbosely." - << std::endl); - } - } return res; } @@ -3520,6 +3569,81 @@ bool cmCTest::SetCTestConfigurationFromCMakeVariable( return true; } +void cmCTest::SetCMakeVariables(cmMakefile& mf) +{ + auto set = [&](char const* cmake_var, char const* ctest_opt) { + std::string val = this->GetCTestConfiguration(ctest_opt); + if (!val.empty()) { + cmCTestOptionalLog( + this, HANDLER_VERBOSE_OUTPUT, + "SetCMakeVariable:" << cmake_var << ":" << val << std::endl, false); + mf.AddDefinition(cmake_var, val); + } + }; + + set("CTEST_SITE", "Site"); + set("CTEST_BUILD_NAME", "BuildName"); + set("CTEST_NIGHTLY_START_TIME", "NightlyStartTime"); + set("CTEST_SOURCE_DIRECTORY", "SourceDirectory"); + set("CTEST_BINARY_DIRECTORY", "BuildDirectory"); + + // CTest Update Step + set("CTEST_UPDATE_COMMAND", "UpdateCommand"); + set("CTEST_UPDATE_OPTIONS", "UpdateOptions"); + set("CTEST_CVS_COMMAND", "CVSCommand"); + set("CTEST_CVS_UPDATE_OPTIONS", "CVSUpdateOptions"); + set("CTEST_SVN_COMMAND", "SVNCommand"); + set("CTEST_SVN_UPDATE_OPTIONS", "SVNUpdateOptions"); + set("CTEST_SVN_OPTIONS", "SVNOptions"); + set("CTEST_BZR_COMMAND", "BZRCommand"); + set("CTEST_BZR_UPDATE_OPTIONS", "BZRUpdateOptions"); + set("CTEST_GIT_COMMAND", "GITCommand"); + set("CTEST_GIT_UPDATE_OPTIONS", "GITUpdateOptions"); + set("CTEST_GIT_INIT_SUBMODULES", "GITInitSubmodules"); + set("CTEST_GIT_UPDATE_CUSTOM", "GITUpdateCustom"); + set("CTEST_UPDATE_VERSION_ONLY", "UpdateVersionOnly"); + set("CTEST_UPDATE_VERSION_OVERRIDE", "UpdateVersionOverride"); + set("CTEST_HG_COMMAND", "HGCommand"); + set("CTEST_HG_UPDATE_OPTIONS", "HGUpdateOptions"); + set("CTEST_P4_COMMAND", "P4Command"); + set("CTEST_P4_UPDATE_OPTIONS", "P4UpdateOptions"); + set("CTEST_P4_CLIENT", "P4Client"); + set("CTEST_P4_OPTIONS", "P4Options"); + + // CTest Configure Step + set("CTEST_CONFIGURE_COMMAND", "ConfigureCommand"); + set("CTEST_LABELS_FOR_SUBPROJECTS", "LabelsForSubprojects"); + + // CTest Build Step + set("CTEST_BUILD_COMMAND", "MakeCommand"); + set("CTEST_USE_LAUNCHERS", "UseLaunchers"); + + // CTest Coverage Step + set("CTEST_COVERAGE_COMMAND", "CoverageCommand"); + set("CTEST_COVERAGE_EXTRA_FLAGS", "CoverageExtraFlags"); + + // CTest MemCheck Step + set("CTEST_MEMORYCHECK_TYPE", "MemoryCheckType"); + set("CTEST_MEMORYCHECK_SANITIZER_OPTIONS", "MemoryCheckSanitizerOptions"); + set("CTEST_MEMORYCHECK_COMMAND", "MemoryCheckCommand"); + set("CTEST_MEMORYCHECK_COMMAND_OPTIONS", "MemoryCheckCommandOptions"); + set("CTEST_MEMORYCHECK_SUPPRESSIONS_FILE", "MemoryCheckSuppressionFile"); + + // CTest Submit Step + set("CTEST_SUBMIT_URL", "SubmitURL"); + set("CTEST_DROP_METHOD", "DropMethod"); + set("CTEST_DROP_SITE_USER", "DropSiteUser"); + set("CTEST_DROP_SITE_PASSWORD", "DropSitePassword"); + set("CTEST_DROP_SITE", "DropSite"); + set("CTEST_DROP_LOCATION", "DropLocation"); + set("CTEST_TLS_VERIFY", "TLSVerify"); + set("CTEST_TLS_VERSION", "TLSVersion"); + set("CTEST_CURL_OPTIONS", "CurlOptions"); + set("CTEST_SUBMIT_INACTIVITY_TIMEOUT", "SubmitInactivityTimeout"); + + this->GetTestHandler()->SetCMakeVariables(mf); +} + bool cmCTest::RunCommand(std::vector const& args, std::string* stdOut, std::string* stdErr, int* retVal, const char* dir, cmDuration timeout, diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 85f75fd766..02a57f23bd 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -331,6 +331,11 @@ public: const std::string& cmake_var, bool suppress = false); + /** + * Set CMake variables from CTest Options + */ + void SetCMakeVariables(cmMakefile& mf); + /** Decode a URL to the original string. */ static std::string DecodeURL(const std::string&); diff --git a/Tests/RunCMake/CTestCommandLine/EmptyDirCoverage-ctest-result.txt b/Tests/RunCMake/CTestCommandLine/EmptyDirCoverage-ctest-result.txt new file mode 100644 index 0000000000..f5c89552bd --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/EmptyDirCoverage-ctest-result.txt @@ -0,0 +1 @@ +32 diff --git a/Tests/RunCMake/CTestCommandLine/EmptyDirCoverage-ctest-stderr.txt b/Tests/RunCMake/CTestCommandLine/EmptyDirCoverage-ctest-stderr.txt index e6f93256b0..f6d28a17f3 100644 --- a/Tests/RunCMake/CTestCommandLine/EmptyDirCoverage-ctest-stderr.txt +++ b/Tests/RunCMake/CTestCommandLine/EmptyDirCoverage-ctest-stderr.txt @@ -1,3 +1,3 @@ Cannot find file: [^ ]*/Tests/RunCMake/CTestCommandLine/EmptyDirCoverage-ctest-build/DartConfiguration.tcl -Binary directory is not set. No coverage checking will be performed.$ +CTEST_BINARY_DIRECTORY not set