Merge topic 'ctest_test-repeat'

28994115e8 ctest_test: Add option to REPEAT tests
42d5d8f425 cmCTestMultiProcessHandler: Hold repeat mode as a member
ed65b3e984 CTest: Rename internal APIs for --repeat options

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !4011

Help/command/ctest_test.rst

@@ -23,6 +23,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`.
              [STOP_TIME <time-of-day>]
              [RETURN_VALUE <result-var>]
              [CAPTURE_CMAKE_ERROR <result-var>]
+             [REPEAT <mode>:<n>]
              [QUIET]
              )
 
@@ -95,6 +96,25 @@ The options are:
   and then the ``--test-load`` command-line argument to :manual:`ctest(1)`.
   See also the ``TestLoad`` setting in the :ref:`CTest Test Step`.
 
+``REPEAT <mode>:<n>``
+  Run tests repeatedly based on the given ``<mode>`` up to ``<n>`` times.
+  The modes are:
+
+  ``UNTIL_FAIL``
+    Require each test to run ``<n>`` times without failing in order to pass.
+    This is useful in finding sporadic failures in test cases.
+
+  ``UNTIL_PASS``
+    Allow each test to run up to ``<n>`` times in order to pass.
+    Repeats tests if they fail for any reason.
+    This is useful in tolerating sporadic failures in test cases.
+
+  ``AFTER_TIMEOUT``
+    Allow each test to run up to ``<n>`` times in order to pass.
+    Repeats tests only if they timeout.
+    This is useful in tolerating sporadic timeouts in test cases
+    on busy machines.
+
 ``SCHEDULE_RANDOM <ON|OFF>``
   Launch tests in a random order.  This may be useful for detecting
   implicit test dependencies.

Help/release/dev/ctest-repeat-until-pass.rst

@@ -4,3 +4,6 @@ ctest-repeat-until-pass
 * The :manual:`ctest(1)` tool learned new ``--repeat-until-pass <n>``
   and ``--repeat-after-timeout <n>`` options to help tolerate sporadic
   test failures.
+
+* The :command:`ctest_test` command gained a ``REPEAT <mode>:<n>`` option
+  to specify conditions in which to repeat tests.

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -171,9 +171,9 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
   this->RunningCount += GetProcessorsUsed(test);
 
   cmCTestRunTest* testRun = new cmCTestRunTest(*this);
-  if (this->CTest->GetRerunMode() != cmCTest::Rerun::Never) {
-    testRun->SetRerunMode(this->CTest->GetRerunMode());
-    testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
+  if (this->RepeatMode != cmCTest::Repeat::Never) {
+    testRun->SetRepeatMode(this->RepeatMode);
+    testRun->SetNumberOfRuns(this->RepeatCount);
   }
   testRun->SetIndex(test);
   testRun->SetTestProperties(this->Properties[test]);

Source/CTest/cmCTestMultiProcessHandler.h

@@ -14,11 +14,11 @@
 
 #include "cm_uv.h"
 
+#include "cmCTest.h"
 #include "cmCTestResourceAllocator.h"
 #include "cmCTestTestHandler.h"
 #include "cmUVHandlePtr.h"
 
-class cmCTest;
 struct cmCTestBinPackerAllocation;
 class cmCTestResourceSpec;
 class cmCTestRunTest;
@@ -85,6 +85,12 @@ public:
 
   cmCTestTestHandler* GetTestHandler() { return this->TestHandler; }
 
+  void SetRepeatMode(cmCTest::Repeat mode, int count)
+  {
+    this->RepeatMode = mode;
+    this->RepeatCount = count;
+  }
+
   void SetQuiet(bool b) { this->Quiet = b; }
 
   void InitResourceAllocator(const cmCTestResourceSpec& spec)
@@ -179,6 +185,8 @@ protected:
   cmCTestTestHandler* TestHandler;
   cmCTest* CTest;
   bool HasCycles;
+  cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never;
+  int RepeatCount = 1;
   bool Quiet;
   bool SerialTestRunning;
 };

Source/CTest/cmCTestRunTest.cxx

@@ -307,7 +307,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
   }
   // If the test does not need to rerun push the current TestResult onto the
   // TestHandler vector
-  if (!this->NeedsToRerun()) {
+  if (!this->NeedsToRepeat()) {
     this->TestHandler->TestResults.push_back(this->TestResult);
   }
   this->TestProcess.reset();
@@ -333,7 +333,7 @@ bool cmCTestRunTest::StartAgain(size_t completed)
   return true;
 }
 
-bool cmCTestRunTest::NeedsToRerun()
+bool cmCTestRunTest::NeedsToRepeat()
 {
   this->NumberOfRunsLeft--;
   if (this->NumberOfRunsLeft == 0) {
@@ -342,11 +342,11 @@ bool cmCTestRunTest::NeedsToRerun()
   // if number of runs left is not 0, and we are running until
   // we find a failed (or passed) test, then return true so the test can be
   // restarted
-  if ((this->RerunMode == cmCTest::Rerun::UntilFail &&
+  if ((this->RepeatMode == cmCTest::Repeat::UntilFail &&
        this->TestResult.Status == cmCTestTestHandler::COMPLETED) ||
-      (this->RerunMode == cmCTest::Rerun::UntilPass &&
+      (this->RepeatMode == cmCTest::Repeat::UntilPass &&
        this->TestResult.Status != cmCTestTestHandler::COMPLETED) ||
-      (this->RerunMode == cmCTest::Rerun::AfterTimeout &&
+      (this->RepeatMode == cmCTest::Repeat::AfterTimeout &&
        this->TestResult.Status == cmCTestTestHandler::TIMEOUT)) {
     this->RunAgain = true;
     return true;
@@ -748,8 +748,8 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
   // got for run until pass.  Trick is when this is called we don't
   // yet know if we are passing or failing.
   bool const progressOnLast =
-    (this->RerunMode != cmCTest::Rerun::UntilPass &&
-     this->RerunMode != cmCTest::Rerun::AfterTimeout);
+    (this->RepeatMode != cmCTest::Repeat::UntilPass &&
+     this->RepeatMode != cmCTest::Repeat::AfterTimeout);
   if ((progressOnLast && this->NumberOfRunsLeft == 1) ||
       (!progressOnLast && this->NumberOfRunsLeft == this->NumberOfRunsTotal) ||
       this->CTest->GetTestProgressOutput()) {

Source/CTest/cmCTestRunTest.h

@@ -35,7 +35,7 @@ public:
     this->NumberOfRunsTotal = n;
   }
 
-  void SetRerunMode(cmCTest::Rerun r) { this->RerunMode = r; }
+  void SetRepeatMode(cmCTest::Repeat r) { this->RepeatMode = r; }
   void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties* prop)
   {
     this->TestProperties = prop;
@@ -102,7 +102,7 @@ public:
   }
 
 private:
-  bool NeedsToRerun();
+  bool NeedsToRepeat();
   void DartProcessing();
   void ExeNotFound(std::string exe);
   bool ForkProcess(cmDuration testTimeOut, bool explicitTimeout,
@@ -136,7 +136,7 @@ private:
   std::vector<std::map<
     std::string, std::vector<cmCTestMultiProcessHandler::ResourceAllocation>>>
     AllocatedResources;
-  cmCTest::Rerun RerunMode = cmCTest::Rerun::Never;
+  cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never;
   int NumberOfRunsLeft = 1;  // default to 1 run of the test
   int NumberOfRunsTotal = 1; // default to 1 run of the test
   bool RunAgain = false;     // default to not having to run again

Source/CTest/cmCTestTestCommand.cxx

@@ -29,6 +29,7 @@ void cmCTestTestCommand::BindArguments()
   this->Bind("EXCLUDE_FIXTURE_SETUP"_s, this->ExcludeFixtureSetup);
   this->Bind("EXCLUDE_FIXTURE_CLEANUP"_s, this->ExcludeFixtureCleanup);
   this->Bind("PARALLEL_LEVEL"_s, this->ParallelLevel);
+  this->Bind("REPEAT"_s, this->Repeat);
   this->Bind("SCHEDULE_RANDOM"_s, this->ScheduleRandom);
   this->Bind("STOP_TIME"_s, this->StopTime);
   this->Bind("TEST_LOAD"_s, this->TestLoad);
@@ -85,6 +86,9 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
   if (!this->ParallelLevel.empty()) {
     handler->SetOption("ParallelLevel", this->ParallelLevel.c_str());
   }
+  if (!this->Repeat.empty()) {
+    handler->SetOption("Repeat", this->Repeat.c_str());
+  }
   if (!this->ScheduleRandom.empty()) {
     handler->SetOption("ScheduleRandom", this->ScheduleRandom.c_str());
   }

Source/CTest/cmCTestTestCommand.h

@@ -55,6 +55,7 @@ protected:
   std::string ExcludeFixtureSetup;
   std::string ExcludeFixtureCleanup;
   std::string ParallelLevel;
+  std::string Repeat;
   std::string ScheduleRandom;
   std::string StopTime;
   std::string TestLoad;

Source/CTest/cmCTestTestHandler.cxx

@@ -471,6 +471,30 @@ bool cmCTestTestHandler::ProcessOptions()
   if (cmIsOn(this->GetOption("ScheduleRandom"))) {
     this->CTest->SetScheduleType("Random");
   }
+  if (const char* repeat = this->GetOption("Repeat")) {
+    cmsys::RegularExpression repeatRegex(
+      "^(UNTIL_FAIL|UNTIL_PASS|AFTER_TIMEOUT):([0-9]+)$");
+    if (repeatRegex.find(repeat)) {
+      std::string const& count = repeatRegex.match(2);
+      unsigned long n = 1;
+      cmStrToULong(count, &n); // regex guarantees success
+      this->RepeatCount = static_cast<int>(n);
+      if (this->RepeatCount > 1) {
+        std::string const& mode = repeatRegex.match(1);
+        if (mode == "UNTIL_FAIL") {
+          this->RepeatMode = cmCTest::Repeat::UntilFail;
+        } else if (mode == "UNTIL_PASS") {
+          this->RepeatMode = cmCTest::Repeat::UntilPass;
+        } else if (mode == "AFTER_TIMEOUT") {
+          this->RepeatMode = cmCTest::Repeat::AfterTimeout;
+        }
+      }
+    } else {
+      cmCTestLog(this->CTest, ERROR_MESSAGE,
+                 "Repeat option invalid value: " << repeat << std::endl);
+      return false;
+    }
+  }
   if (this->GetOption("ParallelLevel")) {
     this->CTest->SetParallelLevel(atoi(this->GetOption("ParallelLevel")));
   }
@@ -1231,6 +1255,12 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
   parallel->SetCTest(this->CTest);
   parallel->SetParallelLevel(this->CTest->GetParallelLevel());
   parallel->SetTestHandler(this);
+  if (this->RepeatMode != cmCTest::Repeat::Never) {
+    parallel->SetRepeatMode(this->RepeatMode, this->RepeatCount);
+  } else {
+    parallel->SetRepeatMode(this->CTest->GetRepeatMode(),
+                            this->CTest->GetRepeatCount());
+  }
   parallel->SetQuiet(this->Quiet);
   if (this->TestLoad > 0) {
     parallel->SetTestLoad(this->TestLoad);

Source/CTest/cmCTestTestHandler.h

@@ -18,12 +18,12 @@
 
 #include "cmsys/RegularExpression.hxx"
 
+#include "cmCTest.h"
 #include "cmCTestGenericHandler.h"
 #include "cmCTestResourceSpec.h"
 #include "cmDuration.h"
 #include "cmListFileCache.h"
 
-class cmCTest;
 class cmMakefile;
 class cmXMLWriter;
 
@@ -353,6 +353,8 @@ private:
 
   std::ostream* LogFile;
 
+  cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never;
+  int RepeatCount = 1;
   bool RerunFailed;
 };
 

Source/cmCTest.cxx

@@ -83,8 +83,8 @@ struct cmCTest::Private
     std::string Name;
   };
 
-  int RepeatTests = 1; // default to run each test once
-  cmCTest::Rerun RerunMode = cmCTest::Rerun::Never;
+  int RepeatCount = 1; // default to run each test once
+  cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never;
   std::string ConfigType;
   std::string ScheduleType;
   std::chrono::system_clock::time_point StopTime;
@@ -1845,7 +1845,7 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
       errormsg = "'--repeat-until-fail' requires an argument";
       return false;
     }
-    if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
+    if (this->Impl->RepeatMode != cmCTest::Repeat::Never) {
       errormsg = "At most one '--repeat-*' option may be used.";
       return false;
     }
@@ -1856,9 +1856,9 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
         "'--repeat-until-fail' given non-integer value '" + args[i] + "'";
       return false;
     }
-    this->Impl->RepeatTests = static_cast<int>(repeat);
+    this->Impl->RepeatCount = static_cast<int>(repeat);
     if (repeat > 1) {
-      this->Impl->RerunMode = cmCTest::Rerun::UntilFail;
+      this->Impl->RepeatMode = cmCTest::Repeat::UntilFail;
     }
   }
 
@@ -1867,7 +1867,7 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
       errormsg = "'--repeat-until-pass' requires an argument";
       return false;
     }
-    if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
+    if (this->Impl->RepeatMode != cmCTest::Repeat::Never) {
       errormsg = "At most one '--repeat-*' option may be used.";
       return false;
     }
@@ -1878,9 +1878,9 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
         "'--repeat-until-pass' given non-integer value '" + args[i] + "'";
       return false;
     }
-    this->Impl->RepeatTests = static_cast<int>(repeat);
+    this->Impl->RepeatCount = static_cast<int>(repeat);
     if (repeat > 1) {
-      this->Impl->RerunMode = cmCTest::Rerun::UntilPass;
+      this->Impl->RepeatMode = cmCTest::Repeat::UntilPass;
     }
   }
 
@@ -1889,7 +1889,7 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
       errormsg = "'--repeat-after-timeout' requires an argument";
       return false;
     }
-    if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
+    if (this->Impl->RepeatMode != cmCTest::Repeat::Never) {
       errormsg = "At most one '--repeat-*' option may be used.";
       return false;
     }
@@ -1900,9 +1900,9 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
         "'--repeat-after-timeout' given non-integer value '" + args[i] + "'";
       return false;
     }
-    this->Impl->RepeatTests = static_cast<int>(repeat);
+    this->Impl->RepeatCount = static_cast<int>(repeat);
     if (repeat > 1) {
-      this->Impl->RerunMode = cmCTest::Rerun::AfterTimeout;
+      this->Impl->RepeatMode = cmCTest::Repeat::AfterTimeout;
     }
   }
 
@@ -2896,14 +2896,14 @@ const std::map<std::string, std::string>& cmCTest::GetDefinitions() const
   return this->Impl->Definitions;
 }
 
-int cmCTest::GetTestRepeat() const
+int cmCTest::GetRepeatCount() const
 {
-  return this->Impl->RepeatTests;
+  return this->Impl->RepeatCount;
 }
 
-cmCTest::Rerun cmCTest::GetRerunMode() const
+cmCTest::Repeat cmCTest::GetRepeatMode() const
 {
-  return this->Impl->RerunMode;
+  return this->Impl->RepeatMode;
 }
 
 void cmCTest::SetBuildID(const std::string& id)

Source/cmCTest.h

@@ -431,16 +431,16 @@ public:
   const std::map<std::string, std::string>& GetDefinitions() const;
 
   /** Return the number of times a test should be run */
-  int GetTestRepeat() const;
+  int GetRepeatCount() const;
 
-  enum class Rerun
+  enum class Repeat
   {
     Never,
     UntilFail,
     UntilPass,
     AfterTimeout,
   };
-  Rerun GetRerunMode() const;
+  Repeat GetRepeatMode() const;
 
   void GenerateSubprojectsOutput(cmXMLWriter& xml);
   std::vector<std::string> GetLabelsForSubprojects();

Tests/RunCMake/ctest_test/RunCMakeTest.cmake

@@ -1,6 +1,9 @@
 include(RunCTest)
 set(RunCMake_TEST_TIMEOUT 60)
 
+unset(ENV{CTEST_PARALLEL_LEVEL})
+unset(ENV{CTEST_OUTPUT_ON_FAILURE})
+
 set(CASE_CTEST_TEST_ARGS "")
 set(CASE_CTEST_TEST_LOAD "")
 
@@ -71,7 +74,24 @@ add_test(NAME PassingTest COMMAND ${CMAKE_COMMAND} -E echo PassingTestOutput)
 add_test(NAME FailingTest COMMAND ${CMAKE_COMMAND} -E no_such_command)
   ]])
 
-  unset(ENV{CTEST_PARALLEL_LEVEL})
   run_ctest(TestOutputSize)
 endfunction()
 run_TestOutputSize()
+
+run_ctest_test(TestRepeatBad1 REPEAT UNKNOWN:3)
+run_ctest_test(TestRepeatBad2 REPEAT UNTIL_FAIL:-1)
+
+function(run_TestRepeat case)
+  set(CASE_CTEST_TEST_ARGS EXCLUDE RunCMakeVersion ${ARGN})
+  string(CONCAT CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME testRepeat
+  COMMAND ${CMAKE_COMMAND} -D COUNT_FILE=${CMAKE_CURRENT_BINARY_DIR}/count.cmake
+                           -P "]] "${RunCMake_SOURCE_DIR}/TestRepeat${case}" [[.cmake")
+set_property(TEST testRepeat PROPERTY TIMEOUT 5)
+  ]])
+
+  run_ctest(TestRepeat${case})
+endfunction()
+run_TestRepeat(UntilFail REPEAT UNTIL_FAIL:3)
+run_TestRepeat(UntilPass REPEAT UNTIL_PASS:3)
+run_TestRepeat(AfterTimeout REPEAT AFTER_TIMEOUT:3)

Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-stdout.txt

@@ -0,0 +1,10 @@
+Test project [^
+]*/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-build
+    Start 1: testRepeat
+1/1 Test #1: testRepeat .......................\*\*\*Timeout +[0-9.]+ sec
+    Start 1: testRepeat
+    Test #1: testRepeat .......................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$

Tests/RunCMake/ctest_test/TestRepeatAfterTimeout.cmake

@@ -0,0 +1,10 @@
+include("${COUNT_FILE}" OPTIONAL)
+if(NOT COUNT)
+  set(COUNT 0)
+endif()
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
+if(NOT COUNT EQUAL 2)
+  message("this test times out except on the 2nd run")
+  execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10)
+endif()

Tests/RunCMake/ctest_test/TestRepeatBad1-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_test/TestRepeatBad1-stderr.txt

@@ -0,0 +1 @@
+Repeat option invalid value: UNKNOWN:3

Tests/RunCMake/ctest_test/TestRepeatBad2-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_test/TestRepeatBad2-stderr.txt

@@ -0,0 +1 @@
+Repeat option invalid value: UNTIL_FAIL:-1

Tests/RunCMake/ctest_test/TestRepeatUntilFail-stdout.txt

@@ -0,0 +1,13 @@
+Test project [^
+]*/Tests/RunCMake/ctest_test/TestRepeatUntilFail-build
+    Start 1: testRepeat
+    Test #1: testRepeat .......................   Passed +[0-9.]+ sec
+    Start 1: testRepeat
+    Test #1: testRepeat .......................\*\*\*Failed +[0-9.]+ sec
++
+0% tests passed, 1 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests FAILED:
+[	 ]+1 - testRepeat \(Failed\)$

Tests/RunCMake/ctest_test/TestRepeatUntilFail.cmake

@@ -0,0 +1,9 @@
+include("${COUNT_FILE}" OPTIONAL)
+if(NOT COUNT)
+  set(COUNT 0)
+endif()
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
+if(COUNT EQUAL 2)
+  message(FATAL_ERROR "this test fails on the 2nd run")
+endif()

Tests/RunCMake/ctest_test/TestRepeatUntilPass-stdout.txt

@@ -0,0 +1,10 @@
+Test project [^
+]*/Tests/RunCMake/ctest_test/TestRepeatUntilPass-build
+    Start 1: testRepeat
+1/1 Test #1: testRepeat .......................\*\*\*Failed +[0-9.]+ sec
+    Start 1: testRepeat
+    Test #1: testRepeat .......................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$

Tests/RunCMake/ctest_test/TestRepeatUntilPass.cmake

@@ -0,0 +1,9 @@
+include("${COUNT_FILE}" OPTIONAL)
+if(NOT COUNT)
+  set(COUNT 0)
+endif()
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
+if(NOT COUNT EQUAL 2)
+  message(FATAL_ERROR "this test passes only on the 2nd run")
+endif()