ctest: Add option for live progress summary in terminal

Help/release/dev/ctest-progress-output.rst

@@ -0,0 +1,5 @@
+ctest-progress-output
+---------------------
+
+* :manual:`ctest(1)` gained a ``--progress`` option to enable a live
+  test progress summary when output goes to a terminal.

Source/CTest/cmCTestRunTest.cxx

@@ -246,7 +246,30 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
   sprintf(buf, "%6.2f sec", this->TestProcess->GetTotalTime().count());
   outputStream << buf << "\n";
 
-  cmCTestLog(this->CTest, HANDLER_OUTPUT, outputStream.str());
+  if (this->CTest->GetTestProgressOutput()) {
+    if (!passed) {
+      // If the test did not pass, reprint test name and error
+      std::string output = GetTestPrefix(completed, total);
+      std::string testName = this->TestProperties->Name;
+      const int maxTestNameWidth = this->CTest->GetMaxTestNameWidth();
+      testName.resize(maxTestNameWidth + 4, '.');
+
+      output += testName;
+      output += outputStream.str();
+      outputStream.str("");
+      outputStream.clear();
+      outputStream << output;
+      cmCTestLog(this->CTest, HANDLER_TEST_PROGRESS_OUTPUT, "\n"); // flush
+    }
+    if (completed == total) {
+      std::string testName =
+        GetTestPrefix(completed, total) + this->TestProperties->Name + "\n";
+      cmCTestLog(this->CTest, HANDLER_TEST_PROGRESS_OUTPUT, testName);
+    }
+  }
+  if (!this->CTest->GetTestProgressOutput() || !passed) {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, outputStream.str());
+  }
 
   if (outputTestErrorsToConsole) {
     cmCTestLog(this->CTest, HANDLER_OUTPUT, this->ProcessOutput << std::endl);
@@ -401,12 +424,14 @@ void cmCTestRunTest::StartFailure(std::string const& output)
 {
   // Still need to log the Start message so the test summary records our
   // attempt to start this test
-  cmCTestLog(this->CTest, HANDLER_OUTPUT,
-             std::setw(2 * getNumWidth(this->TotalNumberOfTests) + 8)
-               << "Start "
-               << std::setw(getNumWidth(this->TestHandler->GetMaxIndex()))
-               << this->TestProperties->Index << ": "
-               << this->TestProperties->Name << std::endl);
+  if (!this->CTest->GetTestProgressOutput()) {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT,
+               std::setw(2 * getNumWidth(this->TotalNumberOfTests) + 8)
+                 << "Start "
+                 << std::setw(getNumWidth(this->TestHandler->GetMaxIndex()))
+                 << this->TestProperties->Index << ": "
+                 << this->TestProperties->Name << std::endl);
+  }
 
   this->ProcessOutput.clear();
   if (!output.empty()) {
@@ -428,17 +453,44 @@ void cmCTestRunTest::StartFailure(std::string const& output)
   this->TestProcess = cm::make_unique<cmProcess>(*this);
 }
 
+std::string cmCTestRunTest::GetTestPrefix(size_t completed, size_t total) const
+{
+  std::ostringstream outputStream;
+  outputStream << std::setw(getNumWidth(total)) << completed << "/";
+  outputStream << std::setw(getNumWidth(total)) << total << " ";
+
+  if (this->TestHandler->MemCheck) {
+    outputStream << "MemCheck";
+  } else {
+    outputStream << "Test";
+  }
+
+  std::ostringstream indexStr;
+  indexStr << " #" << this->Index << ":";
+  outputStream << std::setw(3 + getNumWidth(this->TestHandler->GetMaxIndex()))
+               << indexStr.str();
+  outputStream << " ";
+
+  return outputStream.str();
+}
+
 // Starts the execution of a test.  Returns once it has started
 bool cmCTestRunTest::StartTest(size_t completed, size_t total)
 {
   this->TotalNumberOfTests = total; // save for rerun case
-  static_cast<void>(completed);
-  cmCTestLog(this->CTest, HANDLER_OUTPUT,
-             std::setw(2 * getNumWidth(total) + 8)
-               << "Start "
-               << std::setw(getNumWidth(this->TestHandler->GetMaxIndex()))
-               << this->TestProperties->Index << ": "
-               << this->TestProperties->Name << std::endl);
+  if (!this->CTest->GetTestProgressOutput()) {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT,
+               std::setw(2 * getNumWidth(total) + 8)
+                 << "Start "
+                 << std::setw(getNumWidth(this->TestHandler->GetMaxIndex()))
+                 << this->TestProperties->Index << ": "
+                 << this->TestProperties->Name << std::endl);
+  } else {
+    std::string testName =
+      GetTestPrefix(completed, total) + this->TestProperties->Name + "\n";
+    cmCTestLog(this->CTest, HANDLER_TEST_PROGRESS_OUTPUT, testName);
+  }
+
   this->ProcessOutput.clear();
 
   // Return immediately if test is disabled
@@ -695,13 +747,13 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
 {
   std::ostringstream outputStream;
 
-  // if this is the last or only run of this test
-  // then print out completed / total
+  // If this is the last or only run of this test, or progress output is
+  // requested, then print out completed / total.
   // Only issue is if a test fails and we are running until fail
   // then it will never print out the completed / total, same would
   // got for run until pass.  Trick is when this is called we don't
   // yet know if we are passing or failing.
-  if (this->NumberOfRunsLeft == 1) {
+  if (this->NumberOfRunsLeft == 1 || this->CTest->GetTestProgressOutput()) {
     outputStream << std::setw(getNumWidth(total)) << completed << "/";
     outputStream << std::setw(getNumWidth(total)) << total << " ";
   }
@@ -755,7 +807,9 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
   *this->TestHandler->LogFile << this->ProcessOutput << "<end of output>"
                               << std::endl;
 
-  cmCTestLog(this->CTest, HANDLER_OUTPUT, outputStream.str());
+  if (!this->CTest->GetTestProgressOutput()) {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, outputStream.str());
+  }
 
   cmCTestLog(this->CTest, DEBUG,
              "Testing " << this->TestProperties->Name << " ... ");

Source/CTest/cmCTestRunTest.h

@@ -94,6 +94,9 @@ private:
   // Run post processing of the process output for MemCheck
   void MemCheckPostProcess();
 
+  // Returns "completed/total Test #Index: "
+  std::string GetTestPrefix(size_t completed, size_t total) const;
+
   cmCTestTestHandler::cmCTestTestProperties* TestProperties;
   bool TimeoutIsForStopTime = false;
   // Pointer back to the "parent"; the handler that invoked this test run

Source/cmCTest.cxx

@@ -24,6 +24,11 @@
 #include <time.h>
 #include <utility>
 #include <vector>
+#if defined(_WIN32)
+#  include <windows.h> // IWYU pragma: keep
+#else
+#  include <unistd.h> // IWYU pragma: keep
+#endif
 
 #include "cmAlgorithms.h"
 #include "cmCTestBuildAndTestHandler.h"
@@ -263,6 +268,8 @@ cmCTest::cmCTest()
   this->Failover = false;
   this->ForceNewCTestProcess = false;
   this->TomorrowTag = false;
+  this->TestProgressOutput = false;
+  this->FlushTestProgressLine = false;
   this->Verbose = false;
 
   this->Debug = false;
@@ -290,10 +297,16 @@ cmCTest::cmCTest()
   this->OutputTestOutputOnTestFailure = false;
   this->RepeatTests = 1; // default to run each test once
   this->RepeatUntilFail = false;
-  std::string outOnFail;
-  if (cmSystemTools::GetEnv("CTEST_OUTPUT_ON_FAILURE", outOnFail)) {
-    this->OutputTestOutputOnTestFailure = !cmSystemTools::IsOff(outOnFail);
+
+  std::string envValue;
+  if (cmSystemTools::GetEnv("CTEST_OUTPUT_ON_FAILURE", envValue)) {
+    this->OutputTestOutputOnTestFailure = !cmSystemTools::IsOff(envValue);
   }
+  envValue.clear();
+  if (cmSystemTools::GetEnv("CTEST_PROGRESS_OUTPUT", envValue)) {
+    this->TestProgressOutput = !cmSystemTools::IsOff(envValue);
+  }
+
   this->InitStreams();
 
   this->Parts[PartStart].SetName("Start");
@@ -1875,6 +1888,9 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
   if (this->CheckArgument(arg, "-Q", "--quiet")) {
     this->Quiet = true;
   }
+  if (this->CheckArgument(arg, "--progress")) {
+    this->TestProgressOutput = true;
+  }
   if (this->CheckArgument(arg, "-V", "--verbose")) {
     this->Verbose = true;
   }
@@ -2038,6 +2054,23 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
   return true;
 }
 
+bool cmCTest::ProgressOutputSupportedByConsole() const
+{
+#if defined(_WIN32)
+  // On Windows we need a console buffer.
+  void* console = GetStdHandle(STD_OUTPUT_HANDLE);
+  CONSOLE_SCREEN_BUFFER_INFO csbi;
+  return GetConsoleScreenBufferInfo(console, &csbi);
+#else
+  // On UNIX we need a non-dumb tty.
+  std::string term_env_variable;
+  if (cmSystemTools::GetEnv("TERM", term_env_variable)) {
+    return isatty(1) && term_env_variable != "dumb";
+  }
+#endif
+  return false;
+}
+
 // handle the -S -SR and -SP arguments
 void cmCTest::HandleScriptArguments(size_t& i, std::vector<std::string>& args,
                                     bool& SRArgumentSpecified)
@@ -2192,6 +2225,18 @@ int cmCTest::Run(std::vector<std::string>& args, std::string* output)
     }
   }
 
+  // TestProgressOutput only supported if console supports it and not logging
+  // to a file
+  this->TestProgressOutput = this->TestProgressOutput &&
+    !this->OutputLogFile && this->ProgressOutputSupportedByConsole();
+#ifdef _WIN32
+  if (this->TestProgressOutput) {
+    // Disable output line buffering so we can print content without
+    // a newline.
+    std::setvbuf(stdout, nullptr, _IONBF, 0);
+  }
+#endif
+
   // now what should cmake do? if --build-and-test was specified then
   // we run the build and test handler and return
   if (cmakeAndTest) {
@@ -2761,6 +2806,7 @@ static const char* cmCTestStringLogType[] = { "DEBUG",
                                               "OUTPUT",
                                               "HANDLER_OUTPUT",
                                               "HANDLER_PROGRESS_OUTPUT",
+                                              "HANDLER_TEST_PROGRESS_OUTPUT",
                                               "HANDLER_VERBOSE_OUTPUT",
                                               "WARNING",
                                               "ERROR_MESSAGE",
@@ -2821,6 +2867,34 @@ void cmCTest::Log(int logType, const char* file, int line, const char* msg,
   if (!this->Quiet) {
     std::ostream& out = *this->StreamOut;
     std::ostream& err = *this->StreamErr;
+
+    if (logType == HANDLER_TEST_PROGRESS_OUTPUT) {
+      if (this->TestProgressOutput) {
+        cmCTestLogOutputFileLine(out);
+        if (this->FlushTestProgressLine) {
+          printf("\r");
+          this->FlushTestProgressLine = false;
+          out.flush();
+        }
+
+        std::string msg_str{ msg };
+        auto const lineBreakIt = msg_str.find('\n');
+        if (lineBreakIt != std::string::npos) {
+          this->FlushTestProgressLine = true;
+          msg_str.erase(std::remove(msg_str.begin(), msg_str.end(), '\n'),
+                        msg_str.end());
+        }
+
+        out << msg_str;
+#ifndef _WIN32
+        printf("\x1B[K"); // move caret to end
+#endif
+        out.flush();
+        return;
+      }
+      logType = HANDLER_OUTPUT;
+    }
+
     switch (logType) {
       case DEBUG:
         if (this->Debug) {

Source/cmCTest.h

@@ -390,6 +390,7 @@ public:
     OUTPUT,
     HANDLER_OUTPUT,
     HANDLER_PROGRESS_OUTPUT,
+    HANDLER_TEST_PROGRESS_OUTPUT,
     HANDLER_VERBOSE_OUTPUT,
     WARNING,
     ERROR_MESSAGE,
@@ -429,6 +430,8 @@ public:
   void SetFailover(bool failover) { this->Failover = failover; }
   bool GetFailover() { return this->Failover; }
 
+  bool GetTestProgressOutput() const { return this->TestProgressOutput; }
+
   bool GetVerbose() { return this->Verbose; }
   bool GetExtraVerbose() { return this->ExtraVerbose; }
 
@@ -467,6 +470,7 @@ private:
   std::string ConfigType;
   std::string ScheduleType;
   std::chrono::system_clock::time_point StopTime;
+  bool TestProgressOutput;
   bool Verbose;
   bool ExtraVerbose;
   bool ProduceXML;
@@ -476,6 +480,8 @@ private:
   bool PrintLabels;
   bool Failover;
 
+  bool FlushTestProgressLine;
+
   bool ForceNewCTestProcess;
 
   bool RunConfigurationScript;
@@ -561,6 +567,9 @@ private:
   bool HandleCommandLineArguments(size_t& i, std::vector<std::string>& args,
                                   std::string& errormsg);
 
+  /** returns true iff the console supports progress output */
+  bool ProgressOutputSupportedByConsole() const;
+
   /** handle the -S -SP and -SR arguments */
   void HandleScriptArguments(size_t& i, std::vector<std::string>& args,
                              bool& SRArgumentSpecified);

Source/ctest.cxx

@@ -27,6 +27,7 @@ static const char* cmDocumentationUsage[][2] = { { nullptr,
 
 static const char* cmDocumentationOptions[][2] = {
   { "-C <cfg>, --build-config <cfg>", "Choose configuration to test." },
+  { "--progress", "Enable short progress output from tests." },
   { "-V,--verbose", "Enable verbose output from tests." },
   { "-VV,--extra-verbose", "Enable more verbose output from tests." },
   { "--debug", "Displaying more verbose internals of CTest." },