ctest: add command line option to exclude tests listed in a given file

Add `--exclude-from-file <filename>` to exclude the tests listed in the
given file.

Issue: #25455

Help/manual/ctest.1.rst

@@ -246,6 +246,18 @@ Run Tests
  This option can be combined with the other options like
  ``-R``, ``-E``, ``-L`` or ``-LE``.
 
+.. option:: --exclude-from-file <filename>
+
+ .. versionadded:: 3.29
+
+ Exclude tests listed in the given file.
+
+ This option tells CTest to NOT run the tests which are listed in the given
+ file. The file must contain one exact test name per line.
+ Lines can be commented out using a ``#``.
+ This option can be combined with the other options like
+ ``-R``, ``-E``, ``-L`` or ``-LE``.
+
 .. option:: -FA <regex>, --fixture-exclude-any <regex>
 
  Exclude fixtures matching ``<regex>`` from automatically adding any tests to

Help/release/dev/ctest-tests-from-file.rst

@@ -2,4 +2,5 @@ ctest-tests-from-file
 ---------------------
 
 * :manual:`ctest(1)` gained the :option:`--tests-from-file <ctest
-  --tests-from-file>` option to run tests named in a file.
+  --tests-from-file>` and :option:`--exclude-from-file <ctest
+  --exclude-from-file>` options to run or exclude tests named in a file.

Source/CTest/cmCTestTestHandler.cxx

@@ -346,6 +346,7 @@ void cmCTestTestHandler::Initialize()
   this->ExcludeFixtureSetupRegExp.clear();
   this->ExcludeFixtureCleanupRegExp.clear();
   this->TestListFile.clear();
+  this->ExcludeTestListFile.clear();
 
   this->TestsToRunString.clear();
   this->UseUnion = false;
@@ -590,6 +591,10 @@ bool cmCTestTestHandler::ProcessOptions()
   if (val) {
     this->TestListFile = val;
   }
+  val = this->GetOption("ExcludeTestListFile");
+  if (val) {
+    this->ExcludeTestListFile = val;
+  }
   this->SetRerunFailed(cmIsOn(this->GetOption("RerunFailed")));
 
   return true;
@@ -946,6 +951,13 @@ bool cmCTestTestHandler::ComputeTestList()
       }
     }
 
+    if (!this->TestsToExcludeByName.empty()) {
+      if (this->TestsToExcludeByName.find(tp.Name) !=
+          this->TestsToExcludeByName.end()) {
+        continue;
+      }
+    }
+
     tp.Index = cnt; // save the index into the test list for this test
     finalList.push_back(tp);
   }
@@ -1833,7 +1845,11 @@ bool cmCTestTestHandler::GetListOfTests()
   }
 
   if (!this->TestListFile.empty()) {
-    this->ReadTestListFile(this->TestListFile);
+    this->TestsToRunByName = this->ReadTestListFile(this->TestListFile);
+  }
+  if (!this->ExcludeTestListFile.empty()) {
+    this->TestsToExcludeByName =
+      this->ReadTestListFile(this->ExcludeTestListFile);
   }
 
   cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
@@ -2004,8 +2020,11 @@ void cmCTestTestHandler::ExpandTestsToRunInformationForRerunFailed()
   }
 }
 
-void cmCTestTestHandler::ReadTestListFile(const std::string& testListFileName)
+std::set<std::string> cmCTestTestHandler::ReadTestListFile(
+  const std::string& testListFileName) const
 {
+  std::set<std::string> testNames;
+
   cmsys::ifstream ifs(testListFileName.c_str());
   if (ifs) {
     std::string line;
@@ -2015,7 +2034,7 @@ void cmCTestTestHandler::ReadTestListFile(const std::string& testListFileName)
         continue;
       }
 
-      this->TestsToRunByName.insert(trimmed);
+      testNames.insert(trimmed);
     }
     ifs.close();
   } else if (!this->CTest->GetShowOnly() &&
@@ -2025,6 +2044,8 @@ void cmCTestTestHandler::ReadTestListFile(const std::string& testListFileName)
                  << testListFileName
                  << " while generating list of tests to run." << std::endl);
   }
+
+  return testNames;
 }
 
 void cmCTestTestHandler::RecordCustomTestMeasurements(cmXMLWriter& xml,

Source/CTest/cmCTestTestHandler.h

@@ -341,7 +341,8 @@ private:
   std::string GetTestStatus(cmCTestTestResult const&);
   void ExpandTestsToRunInformation(size_t numPossibleTests);
   void ExpandTestsToRunInformationForRerunFailed();
-  void ReadTestListFile(const std::string& testListFileName);
+  std::set<std::string> ReadTestListFile(
+    const std::string& testListFileName) const;
 
   std::vector<std::string> CustomPreTest;
   std::vector<std::string> CustomPostTest;
@@ -361,7 +362,9 @@ private:
   cmsys::RegularExpression IncludeTestsRegularExpression;
   cmsys::RegularExpression ExcludeTestsRegularExpression;
   std::string TestListFile;
+  std::string ExcludeTestListFile;
   std::set<std::string> TestsToRunByName;
+  std::set<std::string> TestsToExcludeByName;
 
   std::string ResourceSpecFile;
 

Source/cmCTest.cxx

@@ -2231,6 +2231,15 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
     this->GetMemCheckHandler()->SetPersistentOption("TestListFile", args[i]);
   }
 
+  else if (this->CheckArgument(arg, "--exclude-from-file"_s) &&
+           i < args.size() - 1) {
+    i++;
+    this->GetTestHandler()->SetPersistentOption("ExcludeTestListFile",
+                                                args[i]);
+    this->GetMemCheckHandler()->SetPersistentOption("ExcludeTestListFile",
+                                                    args[i]);
+  }
+
   else if (this->CheckArgument(arg, "--rerun-failed"_s)) {
     this->GetTestHandler()->SetPersistentOption("RerunFailed", "true");
     this->GetMemCheckHandler()->SetPersistentOption("RerunFailed", "true");

Source/ctest.cxx

@@ -108,6 +108,8 @@ const cmDocumentationEntry cmDocumentationOptions[] = {
   { "-U, --union", "Take the Union of -I and -R" },
   { "--rerun-failed", "Run only the tests that failed previously" },
   { "--tests-from-file <file>", "Run the tests listed in the given file" },
+  { "--exclude-from-file <file>",
+    "Run tests except those listed in the given file" },
   { "--repeat until-fail:<n>, --repeat-until-fail <n>",
     "Require each test to run <n> times without failing in order to pass" },
   { "--repeat until-pass:<n>",

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -222,6 +222,7 @@ add_test(Test11 \"${CMAKE_COMMAND}\" -E echo \"test11\")
   run_cmake_command(TestsFromFile-${arg} ${CMAKE_CTEST_COMMAND} --${arg} ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt )
 endfunction()
 run_TestsFromFileTest(tests-from-file)
+run_TestsFromFileTest(exclude-from-file)
 
 
 function(run_SerialFailed)

Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-from-file-stdout.txt

@@ -0,0 +1,7 @@
+ +Start 2: Test2
+1/2 Test #2: Test2 ............................   Passed +[0-9.]+ sec
+ +Start 3: Test11
+2/2 Test #3: Test11 ...........................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 2
++