Merge topic 'ctest-done'

a6e0158712 ctest_submit: Add support for a "Done" part
f460bbd4c8 ctest_submit: Refactor file list to use a vector instead of a set

Acked-by: Kitware Robot <kwrobot@kitware.com>
Acked-by: Zack Galbreath <zack.galbreath@kitware.com>
Merge-request: !2405

Help/command/ctest_submit.rst

@@ -33,6 +33,7 @@ The options are:
     ExtraFiles = Files listed by CTEST_EXTRA_SUBMIT_FILES
     Upload     = Files prepared for upload by ctest_upload(), in Upload.xml
     Submit     = nothing
+    Done       = Build is complete, in Done.xml
 
 ``FILES <file>...``
   Specify an explicit list of specific files to be submitted.

Help/release/dev/ctest-done.rst

@@ -0,0 +1,5 @@
+ctest-done
+----------
+
+* The :command:`ctest_submit` command learned a new ``Done`` part that can be used
+  to inform CDash that a build is complete and that no more parts will be uploaded.

Source/CTest/cmCTestSubmitHandler.cxx

@@ -12,6 +12,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
+#include "cmAlgorithms.h"
 #include "cmCTest.h"
 #include "cmCTestCurl.h"
 #include "cmCTestScriptHandler.h"
@@ -55,6 +56,7 @@ public:
   std::string Filename;
   std::string MD5;
   std::string Message;
+  std::string BuildID;
 
 private:
   std::vector<char> CurrentValue;
@@ -96,6 +98,8 @@ private:
       this->MD5 = this->GetCurrentValue();
     } else if (name == "message") {
       this->Message = this->GetCurrentValue();
+    } else if (name == "buildId") {
+      this->BuildID = this->GetCurrentValue();
     }
   }
 };
@@ -152,10 +156,9 @@ void cmCTestSubmitHandler::Initialize()
   this->Files.clear();
 }
 
-bool cmCTestSubmitHandler::SubmitUsingFTP(const std::string& localprefix,
-                                          const std::set<std::string>& files,
-                                          const std::string& remoteprefix,
-                                          const std::string& url)
+bool cmCTestSubmitHandler::SubmitUsingFTP(
+  const std::string& localprefix, const std::vector<std::string>& files,
+  const std::string& remoteprefix, const std::string& url)
 {
   CURL* curl;
   CURLcode res;
@@ -299,10 +302,9 @@ bool cmCTestSubmitHandler::SubmitUsingFTP(const std::string& localprefix,
 }
 
 // Uploading files is simpler
-bool cmCTestSubmitHandler::SubmitUsingHTTP(const std::string& localprefix,
-                                           const std::set<std::string>& files,
-                                           const std::string& remoteprefix,
-                                           const std::string& url)
+bool cmCTestSubmitHandler::SubmitUsingHTTP(
+  const std::string& localprefix, const std::vector<std::string>& files,
+  const std::string& remoteprefix, const std::string& url)
 {
   CURL* curl;
   CURLcode res;
@@ -646,6 +648,7 @@ void cmCTestSubmitHandler::ParseResponse(
                  "   Submission failed: " << parser.Message << std::endl);
       return;
     }
+    this->CTest->SetBuildID(parser.BuildID);
   }
   output = cmSystemTools::UpperCase(output);
   if (output.find("WARNING") != std::string::npos) {
@@ -662,9 +665,9 @@ void cmCTestSubmitHandler::ParseResponse(
   }
 }
 
-bool cmCTestSubmitHandler::TriggerUsingHTTP(const std::set<std::string>& files,
-                                            const std::string& remoteprefix,
-                                            const std::string& url)
+bool cmCTestSubmitHandler::TriggerUsingHTTP(
+  const std::vector<std::string>& files, const std::string& remoteprefix,
+  const std::string& url)
 {
   CURL* curl;
   char error_buffer[1024];
@@ -792,11 +795,10 @@ bool cmCTestSubmitHandler::TriggerUsingHTTP(const std::set<std::string>& files,
   return true;
 }
 
-bool cmCTestSubmitHandler::SubmitUsingSCP(const std::string& scp_command,
-                                          const std::string& localprefix,
-                                          const std::set<std::string>& files,
-                                          const std::string& remoteprefix,
-                                          const std::string& url)
+bool cmCTestSubmitHandler::SubmitUsingSCP(
+  const std::string& scp_command, const std::string& localprefix,
+  const std::vector<std::string>& files, const std::string& remoteprefix,
+  const std::string& url)
 {
   if (scp_command.empty() || localprefix.empty() || files.empty() ||
       remoteprefix.empty() || url.empty()) {
@@ -890,7 +892,7 @@ bool cmCTestSubmitHandler::SubmitUsingSCP(const std::string& scp_command,
 }
 
 bool cmCTestSubmitHandler::SubmitUsingCP(const std::string& localprefix,
-                                         const std::set<std::string>& files,
+                                         const std::vector<std::string>& files,
                                          const std::string& remoteprefix,
                                          const std::string& destination)
 {
@@ -925,7 +927,7 @@ bool cmCTestSubmitHandler::SubmitUsingCP(const std::string& localprefix,
 
 #if defined(CTEST_USE_XMLRPC)
 bool cmCTestSubmitHandler::SubmitUsingXMLRPC(
-  const std::string& localprefix, const std::set<std::string>& files,
+  const std::string& localprefix, const std::vector<std::string>& files,
   const std::string& remoteprefix, const std::string& url)
 {
   xmlrpc_env env;
@@ -1020,7 +1022,7 @@ bool cmCTestSubmitHandler::SubmitUsingXMLRPC(
 }
 #else
 bool cmCTestSubmitHandler::SubmitUsingXMLRPC(
-  std::string const& /*unused*/, std::set<std::string> const& /*unused*/,
+  std::string const& /*unused*/, std::vector<std::string> const& /*unused*/,
   std::string const& /*unused*/, std::string const& /*unused*/)
 {
   return false;
@@ -1351,13 +1353,13 @@ int cmCTestSubmitHandler::ProcessHandler()
   cmGeneratedFileStream ofs;
   this->StartLogFile("Submit", ofs);
 
-  cmCTest::SetOfStrings files;
+  std::vector<std::string> files;
   std::string prefix = this->GetSubmitResultsPrefix();
 
   if (!this->Files.empty()) {
     // Submit the explicitly selected files:
     //
-    files.insert(this->Files.begin(), this->Files.end());
+    files.insert(files.end(), this->Files.begin(), this->Files.end());
   }
 
   // Add to the list of files to submit from any selected, existing parts:
@@ -1404,7 +1406,22 @@ int cmCTestSubmitHandler::ProcessHandler()
 
     // Submit files from this part.
     std::vector<std::string> const& pfiles = this->CTest->GetSubmitFiles(p);
-    files.insert(pfiles.begin(), pfiles.end());
+    files.insert(files.end(), pfiles.begin(), pfiles.end());
+  }
+
+  // Make sure files are unique, but preserve order.
+  {
+    // This endPos intermediate is needed to work around non-conformant C++11
+    // standard libraries that have erase(iterator,iterator) instead of
+    // erase(const_iterator,const_iterator).
+    size_t endPos = cmRemoveDuplicates(files) - files.cbegin();
+    files.erase(files.begin() + endPos, files.end());
+  }
+
+  // Submit Done.xml last
+  if (this->SubmitPart[cmCTest::PartDone]) {
+    this->CTest->GenerateDoneFile();
+    files.push_back("Done.xml");
   }
 
   if (ofs) {

Source/CTest/cmCTestSubmitHandler.h

@@ -57,27 +57,27 @@ private:
    * Submit file using various ways
    */
   bool SubmitUsingFTP(const std::string& localprefix,
-                      const std::set<std::string>& files,
+                      const std::vector<std::string>& files,
                       const std::string& remoteprefix, const std::string& url);
   bool SubmitUsingHTTP(const std::string& localprefix,
-                       const std::set<std::string>& files,
+                       const std::vector<std::string>& files,
                        const std::string& remoteprefix,
                        const std::string& url);
   bool SubmitUsingSCP(const std::string& scp_command,
                       const std::string& localprefix,
-                      const std::set<std::string>& files,
+                      const std::vector<std::string>& files,
                       const std::string& remoteprefix, const std::string& url);
 
   bool SubmitUsingCP(const std::string& localprefix,
-                     const std::set<std::string>& files,
+                     const std::vector<std::string>& files,
                      const std::string& remoteprefix, const std::string& url);
 
-  bool TriggerUsingHTTP(const std::set<std::string>& files,
+  bool TriggerUsingHTTP(const std::vector<std::string>& files,
                         const std::string& remoteprefix,
                         const std::string& url);
 
   bool SubmitUsingXMLRPC(const std::string& localprefix,
-                         const std::set<std::string>& files,
+                         const std::vector<std::string>& files,
                          const std::string& remoteprefix,
                          const std::string& url);
 

Source/cmCTest.cxx

@@ -294,6 +294,7 @@ cmCTest::cmCTest()
   this->SuppressUpdatingCTestConfiguration = false;
   this->DartVersion = 1;
   this->DropSiteCDash = false;
+  this->BuildID = "";
   this->OutputTestOutputOnTestFailure = false;
   this->RepeatTests = 1; // default to run each test once
   this->RepeatUntilFail = false;
@@ -320,6 +321,7 @@ cmCTest::cmCTest()
   this->Parts[PartNotes].SetName("Notes");
   this->Parts[PartExtraFiles].SetName("ExtraFiles");
   this->Parts[PartUpload].SetName("Upload");
+  this->Parts[PartDone].SetName("Done");
 
   // Fill the part name-to-id map.
   for (Part p = PartStart; p != PartCount; p = Part(p + 1)) {
@@ -612,6 +614,7 @@ bool cmCTest::InitializeFromCommand(cmCTestStartCommand* command)
   std::string bld_dir = this->GetCTestConfiguration("BuildDirectory");
   this->DartVersion = 1;
   this->DropSiteCDash = false;
+  this->BuildID = "";
   for (Part p = PartStart; p != PartCount; p = Part(p + 1)) {
     this->Parts[p].SubmitFiles.clear();
   }
@@ -1565,6 +1568,24 @@ int cmCTest::GenerateNotesFile(const char* cfiles)
   return this->GenerateNotesFile(files);
 }
 
+int cmCTest::GenerateDoneFile()
+{
+  cmGeneratedFileStream ofs;
+  if (!this->OpenOutputFile(this->CurrentTag, "Done.xml", ofs)) {
+    cmCTestLog(this, ERROR_MESSAGE, "Cannot open done file" << std::endl);
+    return 1;
+  }
+  cmXMLWriter xml(ofs);
+  xml.StartDocument();
+  xml.StartElement("Done");
+  xml.Element("buildId", this->BuildID);
+  xml.Element("time", std::chrono::system_clock::now());
+  xml.EndElement(); // Done
+  xml.EndDocument();
+
+  return 0;
+}
+
 std::string cmCTest::Base64GzipEncodeFile(std::string const& file)
 {
   std::string tarFile = file + "_temp.tar.gz";

Source/cmCTest.h

@@ -50,6 +50,7 @@ public:
     PartNotes,
     PartExtraFiles,
     PartUpload,
+    PartDone,
     PartCount // Update names in constructor when adding a part
   };
 
@@ -373,6 +374,9 @@ public:
   /** Create XML file that contains all the notes specified */
   int GenerateNotesFile(const VectorOfStrings& files);
 
+  /** Create XML file to indicate that build is complete */
+  int GenerateDoneFile();
+
   /** Submit extra files to the server */
   bool SubmitExtraFiles(const char* files);
   bool SubmitExtraFiles(const VectorOfStrings& files);
@@ -405,6 +409,10 @@ public:
   int GetDartVersion() { return this->DartVersion; }
   int GetDropSiteCDash() { return this->DropSiteCDash; }
 
+  /** The Build ID is assigned by CDash */
+  void SetBuildID(const std::string& id) { this->BuildID = id; }
+  std::string GetBuildID() { return this->BuildID; }
+
   /** Add file to be submitted */
   void AddSubmitFile(Part part, const char* name);
   std::vector<std::string> const& GetSubmitFiles(Part part)
@@ -607,6 +615,8 @@ private:
   int DartVersion;
   bool DropSiteCDash;
 
+  std::string BuildID;
+
   std::vector<std::string> InitialCommandLineArguments;
 
   int SubmitIndex;

Tests/RunCMake/ctest_submit/PARTSDone-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_submit/PARTSDone-stderr.txt

@@ -0,0 +1,3 @@
+ *Error when uploading file: .*/Done.xml
+ *Error message was: ([Cc]ould *n.t resolve host:? '?-no-site-'?.*|The requested URL returned error:.*)
+ *Problems when submitting via HTTP

Tests/RunCMake/ctest_submit/RunCMakeTest.cmake

@@ -24,6 +24,7 @@ run_ctest_submit(BadFILES FILES bad-file)
 run_ctest_submit(RepeatRETURN_VALUE RETURN_VALUE res RETURN_VALUE res)
 run_ctest_submit(PARTSCDashUpload PARTS Configure CDASH_UPLOAD)
 run_ctest_submit(PARTSCDashUploadType PARTS Configure CDASH_UPLOAD_TYPE)
+run_ctest_submit(PARTSDone PARTS Done)
 run_ctest_submit(CDashUploadPARTS CDASH_UPLOAD bad-upload PARTS)
 run_ctest_submit(CDashUploadFILES CDASH_UPLOAD bad-upload FILES)
 run_ctest_submit(CDashUploadNone CDASH_UPLOAD)