mirror/CMake
1
0
Fork 0
mirror of https://github.com/Kitware/CMake.git synced 2026-02-15 11:46:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

ctest_submit: Allow RETRY_COUNT for CDASH_UPLOAD

Browse Source 
Teach the CDASH_UPLOAD signature of ctest_submit() to honor the
RETRY_COUNT and RETRY_DELAY options.

Also teach HttpRequest() to honor the default 120 second timeout
for curl connections.
This commit is contained in:
Zack Galbreath
2017-01-04 10:27:37 -05:00
committed by Brad King
parent c628fd2fab
commit 5614a5cd1f
11 changed files with 132 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
Source/CTest/cmCTestCurl.cxx
View File

@@ -96,6 +96,13 @@ bool cmCTestCurl::InitCurl()
}
// enable HTTP ERROR parsing
curl_easy_setopt(this->Curl, CURLOPT_FAILONERROR, 1);
// if there is little to no activity for too long stop submitting
if (this->TimeOutSeconds) {
curl_easy_setopt(this->Curl, CURLOPT_LOW_SPEED_LIMIT, 1);
curl_easy_setopt(this->Curl, CURLOPT_LOW_SPEED_TIME, this->TimeOutSeconds);
}
return true;
}
@@ -110,12 +117,7 @@ bool cmCTestCurl::UploadFile(std::string const& local_file,
}
/* enable uploading */
curl_easy_setopt(this->Curl, CURLOPT_UPLOAD, 1);
// if there is little to no activity for too long stop submitting
if (this->TimeOutSeconds) {
::curl_easy_setopt(this->Curl, CURLOPT_LOW_SPEED_LIMIT, 1);
::curl_easy_setopt(this->Curl, CURLOPT_LOW_SPEED_TIME,
this->TimeOutSeconds);
}
/* HTTP PUT please */
::curl_easy_setopt(this->Curl, CURLOPT_PUT, 1);
::curl_easy_setopt(this->Curl, CURLOPT_VERBOSE, 1);

29
Source/CTest/cmCTestSubmitCommand.cxx
View File

@@ -157,6 +157,7 @@ bool cmCTestSubmitCommand::InitialPass(std::vector<std::string> const& args,
bool cmCTestSubmitCommand::CheckArgumentKeyword(std::string const& arg)
{
if (this->CDashUpload) {
// Arguments specific to the CDASH_UPLOAD signature.
if (arg == "CDASH_UPLOAD") {
this->ArgumentDoing = ArgumentDoingCDashUpload;
return true;
@@ -167,7 +168,7 @@ bool cmCTestSubmitCommand::CheckArgumentKeyword(std::string const& arg)
return true;
}
} else {
// Look for arguments specific to this command.
// Arguments that cannot be used with CDASH_UPLOAD.
if (arg == "PARTS") {
this->ArgumentDoing = ArgumentDoingParts;
this->PartsMentioned = true;
@@ -179,21 +180,21 @@ bool cmCTestSubmitCommand::CheckArgumentKeyword(std::string const& arg)
this->FilesMentioned = true;
return true;
}
}
// Arguments used by both modes.
if (arg == "RETRY_COUNT") {
this->ArgumentDoing = ArgumentDoingRetryCount;
return true;
}
if (arg == "RETRY_COUNT") {
this->ArgumentDoing = ArgumentDoingRetryCount;
return true;
}
if (arg == "RETRY_DELAY") {
this->ArgumentDoing = ArgumentDoingRetryDelay;
return true;
}
if (arg == "RETRY_DELAY") {
this->ArgumentDoing = ArgumentDoingRetryDelay;
return true;
}
if (arg == "INTERNAL_TEST_CHECKSUM") {
this->InternalTest = true;
return true;
}
if (arg == "INTERNAL_TEST_CHECKSUM") {
this->InternalTest = true;
return true;
}
// Look for other arguments.

117
Source/CTest/cmCTestSubmitHandler.cxx
View File

@@ -1022,6 +1022,30 @@ int cmCTestSubmitHandler::HandleCDashUploadFile(std::string const& file,
"Only http and https are supported for CDASH_UPLOAD\n");
return -1;
}
bool internalTest = cmSystemTools::IsOn(this->GetOption("InternalTest"));
// Get RETRY_COUNT and RETRY_DELAY values if they were set.
std::string retryDelayString = this->GetOption("RetryDelay") == CM_NULLPTR
? ""
: this->GetOption("RetryDelay");
std::string retryCountString = this->GetOption("RetryCount") == CM_NULLPTR
? ""
: this->GetOption("RetryCount");
unsigned long retryDelay = 0;
if (retryDelayString != "") {
if (!cmSystemTools::StringToULong(retryDelayString.c_str(), &retryDelay)) {
cmCTestLog(this->CTest, WARNING, "Invalid value for 'RETRY_DELAY' : "
<< retryDelayString << std::endl);
}
}
unsigned long retryCount = 0;
if (retryCountString != "") {
if (!cmSystemTools::StringToULong(retryCountString.c_str(), &retryCount)) {
cmCTestLog(this->CTest, WARNING, "Invalid value for 'RETRY_DELAY' : "
<< retryCountString << std::endl);
}
}
char md5sum[33];
md5sum[32] = 0;
cmSystemTools::ComputeFileMD5(file, md5sum);
@@ -1058,7 +1082,33 @@ int cmCTestSubmitHandler::HandleCDashUploadFile(std::string const& file,
<< "\nfile: " << file << "\n",
this->Quiet);
std::string response;
if (!curl.HttpRequest(url, fields, response)) {
bool requestSucceeded = curl.HttpRequest(url, fields, response);
if (!internalTest && !requestSucceeded) {
// If request failed, wait and retry.
for (unsigned long i = 0; i < retryCount; i++) {
cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
" Request failed, waiting " << retryDelay
<< " seconds...\n",
this->Quiet);
double stop = cmSystemTools::GetTime() + static_cast<double>(retryDelay);
while (cmSystemTools::GetTime() < stop) {
cmSystemTools::Delay(100);
}
cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
" Retry request: Attempt "
<< (i + 1) << " of " << retryCount << std::endl,
this->Quiet);
requestSucceeded = curl.HttpRequest(url, fields, response);
if (requestSucceeded) {
break;
}
}
}
if (!internalTest && !requestSucceeded) {
cmCTestLog(this->CTest, ERROR_MESSAGE, "Error in HttpRequest\n"
<< response);
return -1;
@@ -1068,30 +1118,32 @@ int cmCTestSubmitHandler::HandleCDashUploadFile(std::string const& file,
this->Quiet);
Json::Value json;
Json::Reader reader;
if (!reader.parse(response, json)) {
if (!internalTest && !reader.parse(response, json)) {
cmCTestLog(this->CTest, ERROR_MESSAGE, "error parsing json string ["
<< response << "]\n"
<< reader.getFormattedErrorMessages() << "\n");
return -1;
}
if (json["status"].asInt() != 0) {
if (!internalTest && json["status"].asInt() != 0) {
cmCTestLog(this->CTest, ERROR_MESSAGE,
"Bad status returned from CDash: " << json["status"].asInt());
return -1;
}
if (json["datafilesmd5"].isArray()) {
int datares = json["datafilesmd5"][0].asInt();
if (datares == 1) {
cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
"File already exists on CDash, skip upload " << file
<< "\n",
this->Quiet);
return 0;
if (!internalTest) {
if (json["datafilesmd5"].isArray()) {
int datares = json["datafilesmd5"][0].asInt();
if (datares == 1) {
cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
"File already exists on CDash, skip upload "
<< file << "\n",
this->Quiet);
return 0;
}
} else {
cmCTestLog(this->CTest, ERROR_MESSAGE,
"bad datafilesmd5 value in response " << response << "\n");
return -1;
}
} else {
cmCTestLog(this->CTest, ERROR_MESSAGE,
"bad datafilesmd5 value in response " << response << "\n");
return -1;
}
std::string upload_as = cmSystemTools::GetFilenameName(file);
@@ -1100,7 +1152,40 @@ int cmCTestSubmitHandler::HandleCDashUploadFile(std::string const& file,
<< "md5=" << md5sum << "&"
<< "filename=" << curl.Escape(upload_as) << "&"
<< "buildid=" << json["buildid"].asString();
if (!curl.UploadFile(file, url, fstr.str(), response)) {
bool uploadSucceeded = false;
if (!internalTest) {
uploadSucceeded = curl.UploadFile(file, url, fstr.str(), response);
}
if (!uploadSucceeded) {
// If upload failed, wait and retry.
for (unsigned long i = 0; i < retryCount; i++) {
cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
" Upload failed, waiting " << retryDelay
<< " seconds...\n",
this->Quiet);
double stop = cmSystemTools::GetTime() + static_cast<double>(retryDelay);
while (cmSystemTools::GetTime() < stop) {
cmSystemTools::Delay(100);
}
cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
" Retry upload: Attempt "
<< (i + 1) << " of " << retryCount << std::endl,
this->Quiet);
if (!internalTest) {
uploadSucceeded = curl.UploadFile(file, url, fstr.str(), response);
}
if (uploadSucceeded) {
break;
}
}
}
if (!uploadSucceeded) {
cmCTestLog(this->CTest, ERROR_MESSAGE, "error uploading to CDash. "
<< file << " " << url << " " << fstr.str());
return -1;

0
Tests/RunCMake/ctest_submit/CDashUploadRETRY_COUNT-result.txt → Tests/RunCMake/ctest_submit/CDashUploadMissingFile-result.txt
View File

1
Tests/RunCMake/ctest_submit/CDashUploadMissingFile-stderr.txt Normal file
View File

@@ -0,0 +1 @@
Upload file not found: 'bad-upload'

2
Tests/RunCMake/ctest_submit/CDashUploadRETRY_COUNT-stderr.txt
View File

@@ -1,2 +0,0 @@
CMake Error at .*/Tests/RunCMake/ctest_submit/CDashUploadRETRY_COUNT/test.cmake:[0-9]+ \(ctest_submit\):
ctest_submit called with unknown argument "RETRY_COUNT".

2
Tests/RunCMake/ctest_submit/CDashUploadRETRY_DELAY-stderr.txt
View File

@@ -1,2 +0,0 @@
CMake Error at .*/Tests/RunCMake/ctest_submit/CDashUploadRETRY_DELAY/test.cmake:[0-9]+ \(ctest_submit\):
ctest_submit called with unknown argument "RETRY_DELAY".

0
Tests/RunCMake/ctest_submit/CDashUploadRETRY_DELAY-result.txt → Tests/RunCMake/ctest_submit/CDashUploadRetry-result.txt
View File

1
Tests/RunCMake/ctest_submit/CDashUploadRetry-stderr.txt Normal file
View File

@@ -0,0 +1 @@
error uploading to CDash.

4
Tests/RunCMake/ctest_submit/CDashUploadRetry-stdout.txt Normal file
View File

@@ -0,0 +1,4 @@
Upload failed, waiting 1 seconds...
Retry upload: Attempt 1 of 2
Upload failed, waiting 1 seconds...
Retry upload: Attempt 2 of 2

4
Tests/RunCMake/ctest_submit/RunCMakeTest.cmake
View File

@@ -21,9 +21,9 @@ run_ctest_submit(PARTSCDashUpload PARTS Configure CDASH_UPLOAD)
run_ctest_submit(PARTSCDashUploadType PARTS Configure CDASH_UPLOAD_TYPE)
run_ctest_submit(CDashUploadPARTS CDASH_UPLOAD bad-upload PARTS)
run_ctest_submit(CDashUploadFILES CDASH_UPLOAD bad-upload FILES)
run_ctest_submit(CDashUploadRETRY_COUNT CDASH_UPLOAD bad-upload RETRY_COUNT)
run_ctest_submit(CDashUploadRETRY_DELAY CDASH_UPLOAD bad-upload RETRY_DELAY)
run_ctest_submit(CDashUploadNone CDASH_UPLOAD)
run_ctest_submit(CDashUploadMissingFile CDASH_UPLOAD bad-upload)
run_ctest_submit(CDashUploadRetry CDASH_UPLOAD ${CMAKE_CURRENT_LIST_FILE} CDASH_UPLOAD_TYPE foo RETRY_COUNT 2 RETRY_DELAY 1 INTERNAL_TEST_CHECKSUM)
run_ctest_submit(CDashSubmitQuiet QUIET)
function(run_ctest_CDashUploadFTP)