mirror/CMake
1
0
Fork 0
mirror of https://github.com/Kitware/CMake.git synced 2025-12-31 10:50:16 -06:00
Code Issues Packages Projects Releases Wiki Activity

ctest: Include cmake instrumentation data in XML files

Browse Source
This commit is contained in:
Zack Galbreath
2025-02-11 10:49:19 -05:00
committed by Brad King
parent 85a63143ed
commit a6d4a9a2ae
25 changed files with 652 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Help/dev/experimental.rst
View File

@@ -129,7 +129,13 @@ set
* variable ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` to
* value ``a37d1069-1972-4901-b9c9-f194aaf2b6e0``.
To enable instrumentation at the user-level, files should be blaced under
To enable instrumentation at the user-level, files should be placed under
either
``<CMAKE_CONFIG_DIR>/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0`` or
``<CMAKE_BINARY_DIR>/.cmake/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0``.
To include instrumentation data in CTest XML files (for submission to CDash),
you need to set the following environment variables:
* ``CTEST_USE_INSTRUMENTATION=1``
* ``CTEST_EXPERIMENTAL_INSTRUMENTATION=a37d1069-1972-4901-b9c9-f194aaf2b6e0``

15
Help/envvar/CTEST_USE_INSTRUMENTATION.rst Normal file
View File

@@ -0,0 +1,15 @@
CTEST_USE_INSTRUMENTATION
-------------------------
.. versionadded:: 4.0
.. include:: ENV_VAR.txt
.. note::
This feature is only available when experimental support for instrumentation
has been enabled by the ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` gate.
Setting this environment variable enables
:manual:`instrumentation <cmake-instrumentation(7)>` for CTest in
:ref:`Dashboard Client` mode.

17
Help/envvar/CTEST_USE_VERBOSE_INSTRUMENTATION.rst Normal file
View File

@@ -0,0 +1,17 @@
CTEST_USE_VERBOSE_INSTRUMENTATION
---------------------------------
.. versionadded:: 4.0
.. include:: ENV_VAR.txt
.. note::
This feature is only available when experimental support for instrumentation
has been enabled by the ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` gate.
Setting this environment variable causes CTest to report the full
command line (including arguments) to CDash for each instrumented command.
By default, CTest truncates the command line at the first space.
See also :envvar:`CTEST_USE_INSTRUMENTATION`

2
Help/manual/cmake-env-variables.7.rst
View File

@@ -116,7 +116,9 @@ Environment Variables for CTest
/envvar/CTEST_OUTPUT_ON_FAILURE
/envvar/CTEST_PARALLEL_LEVEL
/envvar/CTEST_PROGRESS_OUTPUT
/envvar/CTEST_USE_INSTRUMENTATION
/envvar/CTEST_USE_LAUNCHERS_DEFAULT
/envvar/CTEST_USE_VERBOSE_INSTRUMENTATION
/envvar/DASHBOARD_TEST_FROM_CTEST
Environment Variables for the CMake curses interface

35
Help/manual/cmake-instrumentation.7.rst
View File

@@ -94,6 +94,37 @@ Instrumentation can be configured at the user-level by placing query files in
the :envvar:`CMAKE_CONFIG_DIR` under
``<config_dir>/instrumentation/<version>/query/``.
Enabling Instrumentation for CDash Submissions
----------------------------------------------
You can enable instrumentation when using CTest in :ref:`Dashboard Client`
mode by setting the :envvar:`CTEST_USE_INSTRUMENTATION` environment variable
to the current UUID for the ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` feature.
Doing so automatically enables the ``dynamicSystemInformation`` query.
The following table shows how each type of instrumented command gets mapped
to a corresponding type of CTest XML file.
=================================================== ==================
:ref:`Snippet Role <cmake-instrumentation Data v1>` CTest XML File
=================================================== ==================
``configure`` ``Configure.xml``
``generate`` ``Configure.xml``
``compile`` ``Build.xml``
``link`` ``Build.xml``
``custom`` ``Build.xml``
``build`` unused!
``cmakeBuild`` ``Build.xml``
``cmakeInstall`` ``Build.xml``
``install`` ``Build.xml``
``ctest`` ``Build.xml``
``test`` ``Test.xml``
=================================================== ==================
By default the command line reported to CDash is truncated at the first space.
You can instead choose to report the full command line (including arguments)
by setting :envvar:`CTEST_USE_VERBOSE_INSTRUMENTATION` to 1.
.. _`cmake-instrumentation API v1`:
API v1
@@ -123,6 +154,10 @@ subdirectories:
files, they should never be removed by other processes. Data collected here
remains until after `Indexing`_ occurs and all `Callbacks`_ are executed.
``cdash/``
Holds temporary files used internally to generate XML content to be submitted
to CDash.
.. _`cmake-instrumentation v1 Query Files`:
v1 Query Files

91
Source/CTest/cmCTestBuildHandler.cxx
View File

@@ -11,6 +11,7 @@
#include <cm/string_view>
#include <cmext/algorithm>
#include <cm3p/json/value.h>
#include <cm3p/uv.h>
#include "cmsys/Directory.hxx"
@@ -21,6 +22,9 @@
#include "cmDuration.h"
#include "cmFileTimeCache.h"
#include "cmGeneratedFileStream.h"
#include "cmInstrumentation.h"
#include "cmInstrumentationQuery.h"
#include "cmJSONState.h"
#include "cmList.h"
#include "cmMakefile.h"
#include "cmProcessOutput.h"
@@ -429,6 +433,11 @@ int cmCTestBuildHandler::ProcessHandler()
} else {
this->GenerateXMLLogScraped(xml);
}
this->CTest->GetInstrumentation().CollectTimingData(
cmInstrumentationQuery::Hook::PrepareForCDash);
this->GenerateInstrumentationXML(xml);
this->GenerateXMLFooter(xml, elapsed_build_time);
if (!res || retVal || this->TotalErrors > 0) {
@@ -595,6 +604,88 @@ void cmCTestBuildHandler::GenerateXMLLogScraped(cmXMLWriter& xml)
}
}
void cmCTestBuildHandler::GenerateInstrumentationXML(cmXMLWriter& xml)
{
// Record instrumentation data on a per-target basis.
cmsys::Directory targets_dir;
std::string targets_snippet_dir = cmStrCat(
this->CTest->GetInstrumentation().GetCDashDir(), "/build/targets");
if (targets_dir.Load(targets_snippet_dir) &&
targets_dir.GetNumberOfFiles() > 0) {
xml.StartElement("Targets");
for (unsigned int i = 0; i < targets_dir.GetNumberOfFiles(); i++) {
if (!targets_dir.FileIsDirectory(i)) {
continue;
}
std::string target_name = targets_dir.GetFile(i);
if (target_name == "." || target_name == "..") {
continue;
}
std::string target_type = "UNKNOWN";
xml.StartElement("Target");
xml.Attribute("name", target_name);
// Check if we have a link snippet for this target.
cmsys::Directory target_dir;
if (!target_dir.Load(targets_dir.GetFilePath(i))) {
cmSystemTools::Error(
cmStrCat("Error loading directory ", targets_dir.GetFilePath(i)));
}
Json::Value link_item;
for (unsigned int j = 0; j < target_dir.GetNumberOfFiles(); j++) {
std::string fname = target_dir.GetFile(j);
if (fname.rfind("link-", 0) == 0) {
std::string fpath = target_dir.GetFilePath(j);
cmJSONState parseState = cmJSONState(fpath, &link_item);
if (!parseState.errors.empty()) {
cmSystemTools::Error(parseState.GetErrorMessage(true));
break;
}
if (!link_item.isObject()) {
std::string error_msg =
cmStrCat("Expected snippet ", fpath, " to contain an object");
cmSystemTools::Error(error_msg);
break;
}
break;
}
}
// If so, parse targetType and targetLabels (optional) from it.
if (link_item.isMember("targetType")) {
target_type = link_item["targetType"].asString();
}
xml.Attribute("type", target_type);
if (link_item.isMember("targetLabels") &&
!link_item["targetLabels"].empty()) {
xml.StartElement("Labels");
for (auto const& json_label_item : link_item["targetLabels"]) {
xml.Element("Label", json_label_item.asString());
}
xml.EndElement(); // Labels
}
// Write instrumendation data for this target.
std::string target_subdir = cmStrCat("build/targets/", target_name);
this->CTest->ConvertInstrumentationSnippetsToXML(xml, target_subdir);
std::string target_dir_fullpath = cmStrCat(
this->CTest->GetInstrumentation().GetCDashDir(), '/', target_subdir);
if (cmSystemTools::FileIsDirectory(target_dir_fullpath)) {
cmSystemTools::RemoveADirectory(target_dir_fullpath);
}
xml.EndElement(); // Target
}
xml.EndElement(); // Targets
}
// Also record instrumentation data for custom commands (no target).
this->CTest->ConvertInstrumentationSnippetsToXML(xml, "build/commands");
}
void cmCTestBuildHandler::GenerateXMLFooter(cmXMLWriter& xml,
cmDuration elapsed_build_time)
{

1
Source/CTest/cmCTestBuildHandler.h
View File

@@ -85,6 +85,7 @@ private:
void GenerateXMLHeader(cmXMLWriter& xml);
void GenerateXMLLaunched(cmXMLWriter& xml);
void GenerateXMLLogScraped(cmXMLWriter& xml);
void GenerateInstrumentationXML(cmXMLWriter& xml);
void GenerateXMLFooter(cmXMLWriter& xml, cmDuration elapsed_build_time);
bool IsLaunchedErrorFile(char const* fname);
bool IsLaunchedWarningFile(char const* fname);

7
Source/CTest/cmCTestConfigureCommand.cxx
View File

@@ -17,6 +17,8 @@
#include "cmExecutionStatus.h"
#include "cmGeneratedFileStream.h"
#include "cmGlobalGenerator.h"
#include "cmInstrumentation.h"
#include "cmInstrumentationQuery.h"
#include "cmList.h"
#include "cmMakefile.h"
#include "cmStringAlgorithms.h"
@@ -203,6 +205,11 @@ bool cmCTestConfigureCommand::ExecuteConfigure(ConfigureArguments const& args,
xml.Element("EndDateTime", endDateTime);
xml.Element("EndConfigureTime", endTime);
xml.Element("ElapsedMinutes", elapsedMinutes.count());
this->CTest->GetInstrumentation().CollectTimingData(
cmInstrumentationQuery::Hook::PrepareForCDash);
this->CTest->ConvertInstrumentationSnippetsToXML(xml, "configure");
xml.EndElement(); // Configure
this->CTest->EndXML(xml);

11
Source/CTest/cmCTestRunTest.cxx
View File

@@ -21,6 +21,7 @@
#include "cmCTestMemCheckHandler.h"
#include "cmCTestMultiProcessHandler.h"
#include "cmDuration.h"
#include "cmInstrumentation.h"
#include "cmProcess.h"
#include "cmStringAlgorithms.h"
#include "cmSystemTools.h"
@@ -34,7 +35,6 @@ cmCTestRunTest::cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler,
, CTest(MultiTestHandler.CTest)
, TestHandler(MultiTestHandler.TestHandler)
, TestProperties(MultiTestHandler.Properties[Index])
, Instrumentation(cmSystemTools::GetLogicalWorkingDirectory())
{
}
@@ -664,8 +664,8 @@ bool cmCTestRunTest::StartTest(size_t completed, size_t total)
return false;
}
this->StartTime = this->CTest->CurrentTime();
if (this->Instrumentation.HasQuery()) {
this->Instrumentation.GetPreTestStats();
if (this->CTest->GetInstrumentation().HasQuery()) {
this->CTest->GetInstrumentation().GetPreTestStats();
}
return this->ForkProcess();
@@ -1016,12 +1016,13 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
void cmCTestRunTest::FinalizeTest(bool started)
{
if (this->Instrumentation.HasQuery()) {
this->Instrumentation.InstrumentTest(
if (this->CTest->GetInstrumentation().HasQuery()) {
std::string data_file = this->CTest->GetInstrumentation().InstrumentTest(
this->TestProperties->Name, this->ActualCommand, this->Arguments,
this->TestProcess->GetExitValue(), this->TestProcess->GetStartTime(),
this->TestProcess->GetSystemStartTime(),
this->GetCTest()->GetConfigType());
this->TestResult.InstrumentationFile = data_file;
}
this->MultiTestHandler.FinishTestProcess(this->TestProcess->GetRunner(),
started);

2
Source/CTest/cmCTestRunTest.h
View File

@@ -14,7 +14,6 @@
#include "cmCTest.h"
#include "cmCTestMultiProcessHandler.h"
#include "cmCTestTestHandler.h"
#include "cmInstrumentation.h"
#include "cmProcess.h"
/** \class cmRunTest
@@ -141,7 +140,6 @@ private:
int NumberOfRunsTotal = 1; // default to 1 run of the test
bool RunAgain = false; // default to not having to run again
size_t TotalNumberOfTests;
cmInstrumentation Instrumentation;
};
inline int getNumWidth(size_t n)

15
Source/CTest/cmCTestTestHandler.cxx
View File

@@ -41,6 +41,8 @@
#include "cmExecutionStatus.h"
#include "cmGeneratedFileStream.h"
#include "cmGlobalGenerator.h"
#include "cmInstrumentation.h"
#include "cmInstrumentationQuery.h"
#include "cmList.h"
#include "cmMakefile.h"
#include "cmState.h"
@@ -1381,6 +1383,9 @@ void cmCTestTestHandler::GenerateCTestXML(cmXMLWriter& xml)
return;
}
this->CTest->GetInstrumentation().CollectTimingData(
cmInstrumentationQuery::Hook::PrepareForCDash);
this->CTest->StartXML(xml, this->CMake, this->AppendXML);
this->CTest->GenerateSubprojectsOutput(xml);
xml.StartElement("Testing");
@@ -1395,7 +1400,6 @@ void cmCTestTestHandler::GenerateCTestXML(cmXMLWriter& xml)
for (cmCTestTestResult& result : this->TestResults) {
this->WriteTestResultHeader(xml, result);
xml.StartElement("Results");
if (result.Status != cmCTestTestHandler::NOT_RUN) {
if (result.Status != cmCTestTestHandler::COMPLETED ||
result.ReturnValue) {
@@ -1473,6 +1477,15 @@ void cmCTestTestHandler::GenerateCTestXML(cmXMLWriter& xml)
xml.Content(result.Output);
xml.EndElement(); // Value
xml.EndElement(); // Measurement
if (!result.InstrumentationFile.empty()) {
std::string instrument_file_path =
cmStrCat(this->CTest->GetInstrumentation().GetCDashDir(), "/test/",
result.InstrumentationFile);
this->CTest->ConvertInstrumentationJSONFileToXML(instrument_file_path,
xml);
}
xml.EndElement(); // Results
this->AttachFiles(xml, result);

2
Source/CTest/cmCTestTestHandler.h
View File

@@ -192,6 +192,7 @@ public:
std::string CustomCompletionStatus;
std::string Output;
std::string TestMeasurementsOutput;
std::string InstrumentationFile;
int TestCount = 0;
cmCTestTestProperties* Properties = nullptr;
};
@@ -250,6 +251,7 @@ protected:
cmCTestTestResult const& result);
void WriteTestResultFooter(cmXMLWriter& xml,
cmCTestTestResult const& result);
// Write attached test files into the xml
void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result);
void AttachFile(cmXMLWriter& xml, std::string const& file,

139
Source/cmCTest.cxx
View File

@@ -27,6 +27,7 @@
#include <cmext/string_view>
#include <cm3p/curl/curl.h>
#include <cm3p/json/value.h>
#include <cm3p/uv.h>
#include <cm3p/zlib.h>
@@ -115,6 +116,8 @@ struct cmCTest::Private
bool UseHTTP10 = false;
bool PrintLabels = false;
bool Failover = false;
bool UseVerboseInstrumentation = false;
cmJSONState parseState;
bool FlushTestProgressLine = false;
@@ -195,6 +198,8 @@ struct cmCTest::Private
cmCTestTestOptions TestOptions;
std::vector<std::string> CommandLineHttpHeaders;
std::unique_ptr<cmInstrumentation> Instrumentation;
};
struct tm* cmCTest::GetNightlyTime(std::string const& str, bool tomorrowtag)
@@ -320,6 +325,11 @@ cmCTest::cmCTest()
if (cmSystemTools::GetEnv("CTEST_PROGRESS_OUTPUT", envValue)) {
this->Impl->TestProgressOutput = !cmIsOff(envValue);
}
envValue.clear();
if (cmSystemTools::GetEnv("CTEST_USE_VERBOSE_INSTRUMENTATION", envValue)) {
this->Impl->UseVerboseInstrumentation = !cmIsOff(envValue);
}
envValue.clear();
this->Impl->Parts[PartStart].SetName("Start");
this->Impl->Parts[PartUpdate].SetName("Update");
@@ -2628,8 +2638,6 @@ int cmCTest::Run(std::vector<std::string> const& args)
}
#endif
cmInstrumentation instrumentation(
cmSystemTools::GetCurrentWorkingDirectory());
std::function<int()> doTest = [this, &cmakeAndTest, &runScripts,
&processSteps]() -> int {
// now what should cmake do? if --build-and-test was specified then
@@ -2650,6 +2658,8 @@ int cmCTest::Run(std::vector<std::string> const& args)
return this->ExecuteTests();
};
cmInstrumentation instrumentation(
cmSystemTools::GetCurrentWorkingDirectory());
int ret = instrumentation.InstrumentCommand("ctest", args,
[doTest]() { return doTest(); });
instrumentation.CollectTimingData(cmInstrumentationQuery::Hook::PostTest);
@@ -3673,3 +3683,128 @@ bool cmCTest::StartLogFile(char const* name, int submitIndex,
}
return true;
}
cmInstrumentation& cmCTest::GetInstrumentation()
{
if (!this->Impl->Instrumentation) {
this->Impl->Instrumentation =
cm::make_unique<cmInstrumentation>(this->GetBinaryDir());
}
return *this->Impl->Instrumentation;
}
bool cmCTest::GetUseVerboseInstrumentation() const
{
return this->Impl->UseVerboseInstrumentation;
}
void cmCTest::ConvertInstrumentationSnippetsToXML(cmXMLWriter& xml,
std::string const& subdir)
{
std::string data_dir =
cmStrCat(this->GetInstrumentation().GetCDashDir(), '/', subdir);
cmsys::Directory d;
if (!d.Load(data_dir) || d.GetNumberOfFiles() == 0) {
return;
}
xml.StartElement("Commands");
for (unsigned int i = 0; i < d.GetNumberOfFiles(); i++) {
std::string fpath = d.GetFilePath(i);
std::string fname = d.GetFile(i);
if (fname.rfind('.', 0) == 0) {
continue;
}
this->ConvertInstrumentationJSONFileToXML(fpath, xml);
}
xml.EndElement(); // Commands
}
bool cmCTest::ConvertInstrumentationJSONFileToXML(std::string const& fpath,
cmXMLWriter& xml)
{
Json::Value root;
this->Impl->parseState = cmJSONState(fpath, &root);
if (!this->Impl->parseState.errors.empty()) {
cmCTestLog(this, ERROR_MESSAGE,
this->Impl->parseState.GetErrorMessage(true) << std::endl);
return false;
}
if (root.type() != Json::objectValue) {
cmCTestLog(this, ERROR_MESSAGE,
"Expected object, found " << root.type() << " for "
<< root.asString() << std::endl);
return false;
}
std::vector<std::string> required_members = {
"command",
"role",
"dynamicSystemInformation",
};
for (std::string const& required_member : required_members) {
if (!root.isMember(required_member)) {
cmCTestLog(this, ERROR_MESSAGE,
fpath << " is missing the '" << required_member << "' key"
<< std::endl);
return false;
}
}
// Do not record command-level data for Test.xml files because
// it is redundant with information actually captured by CTest.
bool generating_test_xml = root["role"] == "test";
if (!generating_test_xml) {
std::string element_name = root["role"].asString();
element_name[0] = static_cast<char>(std::toupper(element_name[0]));
xml.StartElement(element_name);
std::vector<std::string> keys = root.getMemberNames();
for (auto const& key : keys) {
auto key_type = root[key].type();
if (key_type == Json::objectValue || key_type == Json::arrayValue) {
continue;
}
if (key == "role" || key == "target" || key == "targetType" ||
key == "targetLabels") {
continue;
}
// Truncate the full command line if verbose instrumentation
// was not requested.
if (key == "command" && !this->GetUseVerboseInstrumentation()) {
std::string command_str = root[key].asString();
std::string truncated = command_str.substr(0, command_str.find(' '));
if (command_str != truncated) {
truncated = cmStrCat(truncated, " (truncated)");
}
xml.Attribute(key.c_str(), truncated);
continue;
}
xml.Attribute(key.c_str(), root[key].asString());
}
}
// Record dynamicSystemInformation section as XML.
auto dynamic_information = root["dynamicSystemInformation"];
std::vector<std::string> keys = dynamic_information.getMemberNames();
for (auto const& key : keys) {
std::string measurement_name = key;
measurement_name[0] = static_cast<char>(std::toupper(measurement_name[0]));
xml.StartElement("NamedMeasurement");
xml.Attribute("type", "numeric/double");
xml.Attribute("name", measurement_name);
xml.Element("Value", dynamic_information[key].asString());
xml.EndElement(); // NamedMeasurement
}
if (!generating_test_xml) {
xml.EndElement(); // role
}
cmSystemTools::RemoveFile(fpath);
return true;
}

9
Source/cmCTest.h
View File

@@ -21,6 +21,7 @@
class cmake;
class cmGeneratedFileStream;
class cmInstrumentation;
class cmMakefile;
class cmValue;
class cmXMLWriter;
@@ -391,6 +392,11 @@ public:
bool StartLogFile(char const* name, int submitIndex,
cmGeneratedFileStream& xofs);
void ConvertInstrumentationSnippetsToXML(cmXMLWriter& xml,
std::string const& subdir);
bool ConvertInstrumentationJSONFileToXML(std::string const& fpath,
cmXMLWriter& xml);
void AddSiteProperties(cmXMLWriter& xml, cmake* cm);
bool GetInteractiveDebugMode() const;
@@ -433,6 +439,9 @@ public:
cmCTestTestOptions const& GetTestOptions() const;
std::vector<std::string> GetCommandLineHttpHeaders() const;
cmInstrumentation& GetInstrumentation();
bool GetUseVerboseInstrumentation() const;
private:
int GenerateNotesFile(cmake* cm, std::string const& files);

186
Source/cmInstrumentation.cxx
View File

@@ -20,10 +20,12 @@
#include "cmCryptoHash.h"
#include "cmExperimental.h"
#include "cmInstrumentationQuery.h"
#include "cmJSONState.h"
#include "cmStringAlgorithms.h"
#include "cmSystemTools.h"
#include "cmTimestamp.h"
#include "cmUVProcessChain.h"
#include "cmValue.h"
cmInstrumentation::cmInstrumentation(std::string const& binary_dir)
{
@@ -53,6 +55,75 @@ void cmInstrumentation::LoadQueries()
this->hasQuery = this->hasQuery ||
this->ReadJSONQueries(cmStrCat(this->userTimingDirv1, "/query"));
}
std::string envVal;
if (cmSystemTools::GetEnv("CTEST_USE_INSTRUMENTATION", envVal) &&
!cmIsOff(envVal)) {
if (cmSystemTools::GetEnv("CTEST_EXPERIMENTAL_INSTRUMENTATION", envVal)) {
std::string const uuid = cmExperimental::DataForFeature(
cmExperimental::Feature::Instrumentation)
.Uuid;
if (envVal == uuid) {
this->AddHook(cmInstrumentationQuery::Hook::PrepareForCDash);
this->AddQuery(
cmInstrumentationQuery::Query::DynamicSystemInformation);
this->cdashDir = cmStrCat(this->timingDirv1, "/cdash");
cmSystemTools::MakeDirectory(this->cdashDir);
cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/configure"));
cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/build"));
cmSystemTools::MakeDirectory(
cmStrCat(this->cdashDir, "/build/commands"));
cmSystemTools::MakeDirectory(
cmStrCat(this->cdashDir, "/build/targets"));
cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/test"));
this->cdashSnippetsMap = { {
"configure",
"configure",
},
{
"generate",
"configure",
},
{
"compile",
"build",
},
{
"link",
"build",
},
{
"custom",
"build",
},
{
"build",
"skip",
},
{
"cmakeBuild",
"build",
},
{
"cmakeInstall",
"build",
},
{
"install",
"build",
},
{
"ctest",
"build",
},
{
"test",
"test",
} };
this->hasQuery = true;
}
}
}
}
bool cmInstrumentation::ReadJSONQueries(std::string const& directory)
@@ -211,6 +282,11 @@ int cmInstrumentation::CollectTimingData(cmInstrumentationQuery::Hook hook)
cmSystemTools::OUTPUT_PASSTHROUGH);
}
// Special case for CDash collation
if (this->HasHook(cmInstrumentationQuery::Hook::PrepareForCDash)) {
this->PrepareDataForCDash(directory, index_path);
}
// Delete files
for (auto const& f : index["snippets"]) {
cmSystemTools::RemoveFile(cmStrCat(directory, "/", f.asString()));
@@ -308,7 +384,7 @@ void cmInstrumentation::WriteInstrumentationJson(Json::Value& root,
ftmp.close();
}
int cmInstrumentation::InstrumentTest(
std::string cmInstrumentation::InstrumentTest(
std::string const& name, std::string const& command,
std::vector<std::string> const& args, int64_t result,
std::chrono::steady_clock::time_point steadyStart,
@@ -331,11 +407,11 @@ int cmInstrumentation::InstrumentTest(
this->InsertDynamicSystemInformation(root, "after");
}
std::string const& file_name =
std::string file_name =
cmStrCat("test-", this->ComputeSuffixHash(command_str),
this->ComputeSuffixTime(), ".json");
this->WriteInstrumentationJson(root, "data", file_name);
return 1;
return file_name;
}
void cmInstrumentation::GetPreTestStats()
@@ -547,3 +623,107 @@ int cmInstrumentation::CollectTimingAfterBuild(int ppid)
this->CollectTimingData(cmInstrumentationQuery::Hook::PostBuild);
return ret;
}
void cmInstrumentation::AddHook(cmInstrumentationQuery::Hook hook)
{
this->hooks.insert(hook);
}
void cmInstrumentation::AddQuery(cmInstrumentationQuery::Query query)
{
this->queries.insert(query);
}
std::string const& cmInstrumentation::GetCDashDir()
{
return this->cdashDir;
}
/** Copy the snippets referred to by an index file to a separate
* directory where they will be parsed for submission to CDash.
**/
void cmInstrumentation::PrepareDataForCDash(std::string const& data_dir,
std::string const& index_path)
{
Json::Value root;
std::string error_msg;
cmJSONState parseState = cmJSONState(index_path, &root);
if (!parseState.errors.empty()) {
cmSystemTools::Error(parseState.GetErrorMessage(true));
return;
}
if (!root.isObject()) {
error_msg =
cmStrCat("Expected index file ", index_path, " to contain an object");
cmSystemTools::Error(error_msg);
return;
}
if (!root.isMember("snippets")) {
error_msg = cmStrCat("Expected index file ", index_path,
" to have a key 'snippets'");
cmSystemTools::Error(error_msg);
return;
}
std::string dst_dir;
Json::Value snippets = root["snippets"];
for (auto const& snippet : snippets) {
// Parse the role of this snippet.
std::string snippet_str = snippet.asString();
std::string snippet_path = cmStrCat(data_dir, '/', snippet_str);
Json::Value snippet_root;
parseState = cmJSONState(snippet_path, &snippet_root);
if (!parseState.errors.empty()) {
cmSystemTools::Error(parseState.GetErrorMessage(true));
continue;
}
if (!snippet_root.isObject()) {
error_msg = cmStrCat("Expected snippet file ", snippet_path,
" to contain an object");
cmSystemTools::Error(error_msg);
continue;
}
if (!snippet_root.isMember("role")) {
error_msg = cmStrCat("Expected snippet file ", snippet_path,
" to have a key 'role'");
cmSystemTools::Error(error_msg);
continue;
}
std::string snippet_role = snippet_root["role"].asString();
auto map_element = this->cdashSnippetsMap.find(snippet_role);
if (map_element == this->cdashSnippetsMap.end()) {
std::string message =
"Unexpected snippet type encountered: " + snippet_role;
cmSystemTools::Message(message, "Warning");
continue;
}
if (map_element->second == "skip") {
continue;
}
if (map_element->second == "build") {
// We organize snippets on a per-target basis (when possible)
// for Build.xml.
if (snippet_root.isMember("target")) {
dst_dir = cmStrCat(this->cdashDir, "/build/targets/",
snippet_root["target"].asString());
cmSystemTools::MakeDirectory(dst_dir);
} else {
dst_dir = cmStrCat(this->cdashDir, "/build/commands");
}
} else {
dst_dir = cmStrCat(this->cdashDir, '/', map_element->second);
}
std::string dst = cmStrCat(dst_dir, '/', snippet_str);
cmsys::Status copied = cmSystemTools::CopyFileAlways(snippet_path, dst);
if (!copied) {
error_msg = cmStrCat("Failed to copy ", snippet_path, " to ", dst);
cmSystemTools::Error(error_msg);
}
}
}

19
Source/cmInstrumentation.h
View File

@@ -29,11 +29,13 @@ public:
cm::optional<std::map<std::string, std::string>> arrayOptions =
cm::nullopt,
bool reloadQueriesAfterCommand = false);
int InstrumentTest(std::string const& name, std::string const& command,
std::vector<std::string> const& args, int64_t result,
std::chrono::steady_clock::time_point steadyStart,
std::chrono::system_clock::time_point systemStart,
std::string config);
std::string InstrumentTest(std::string const& name,
std::string const& command,
std::vector<std::string> const& args,
int64_t result,
std::chrono::steady_clock::time_point steadyStart,
std::chrono::system_clock::time_point systemStart,
std::string config);
void GetPreTestStats();
void LoadQueries();
bool HasQuery() const;
@@ -49,7 +51,10 @@ public:
int CollectTimingData(cmInstrumentationQuery::Hook hook);
int SpawnBuildDaemon();
int CollectTimingAfterBuild(int ppid);
void AddHook(cmInstrumentationQuery::Hook hook);
void AddQuery(cmInstrumentationQuery::Query query);
std::string errorMsg;
std::string const& GetCDashDir();
private:
void WriteInstrumentationJson(Json::Value& index,
@@ -66,13 +71,17 @@ private:
static std::string GetCommandStr(std::vector<std::string> const& args);
static std::string ComputeSuffixHash(std::string const& command_str);
static std::string ComputeSuffixTime();
void PrepareDataForCDash(std::string const& data_dir,
std::string const& index_path);
std::string binaryDir;
std::string timingDirv1;
std::string userTimingDirv1;
std::string cdashDir;
std::set<cmInstrumentationQuery::Query> queries;
std::set<cmInstrumentationQuery::Hook> hooks;
std::vector<std::string> callbacks;
std::vector<std::string> queryFiles;
std::map<std::string, std::string> cdashSnippetsMap;
Json::Value preTestStats;
bool hasQuery = false;
};

5
Source/cmInstrumentationQuery.cxx
View File

@@ -19,8 +19,9 @@ std::vector<std::string> const cmInstrumentationQuery::QueryString{
"staticSystemInformation", "dynamicSystemInformation"
};
std::vector<std::string> const cmInstrumentationQuery::HookString{
"postGenerate", "preBuild", "postBuild", "preCMakeBuild",
"postCMakeBuild", "postTest", "postInstall", "manual"
"postGenerate", "preBuild", "postBuild",
"preCMakeBuild", "postCMakeBuild", "postTest",
"postInstall", "prepareForCDash", "manual"
};
namespace ErrorMessages {

1
Source/cmInstrumentationQuery.h
View File

@@ -28,6 +28,7 @@ public:
PostCMakeBuild,
PostTest,
PostInstall,
PrepareForCDash,
Manual
};
static std::vector<std::string> const HookString;

3
Tests/RunCMake/CMakeLists.txt
View File

@@ -605,6 +605,9 @@ add_RunCMake_test(ctest_upload)
add_RunCMake_test(ctest_environment)
add_RunCMake_test(ctest_empty_binary_directory)
add_RunCMake_test(ctest_fixtures)
if(CMAKE_GENERATOR MATCHES "Make|Ninja")
add_RunCMake_test(ctest_instrumentation)
endif()
add_RunCMake_test(define_property)
add_RunCMake_test(file -DCYGWIN=${CYGWIN} -DMSYS=${MSYS})
add_RunCMake_test(file-CHMOD -DMSYS=${MSYS})

10
Tests/RunCMake/ctest_instrumentation/CMakeLists.txt.in Normal file
View File

@@ -0,0 +1,10 @@
cmake_minimum_required(VERSION 3.10)
@CASE_CMAKELISTS_PREFIX_CODE@
project(CTestInstrumentation@CASE_NAME@)
if(USE_INSTRUMENTATION)
set(CMAKE_EXPERIMENTAL_INSTRUMENTATION "a37d1069-1972-4901-b9c9-f194aaf2b6e0")
endif()
include(CTest)
add_executable(main main.c)
add_test(NAME main COMMAND main)
@CASE_CMAKELISTS_SUFFIX_CODE@

41
Tests/RunCMake/ctest_instrumentation/InstrumentationInCTestXML-check.cmake Normal file
View File

@@ -0,0 +1,41 @@
foreach(xml_type Configure Build Test)
file(GLOB xml_file "${RunCMake_TEST_BINARY_DIR}/Testing/*/${xml_type}.xml")
if(xml_file)
file(READ "${xml_file}" xml_content)
if(NOT xml_content MATCHES "AfterHostMemoryUsed")
set(RunCMake_TEST_FAILED "'AfterHostMemoryUsed' not found in ${xml_type}.xml")
endif()
if(NOT xml_type STREQUAL "Test")
if(NOT xml_content MATCHES "<Commands>")
set(RunCMake_TEST_FAILED "<Commands> element not found in ${xml_type}.xml")
endif()
endif()
if (xml_type STREQUAL "Build")
if(NOT xml_content MATCHES "<Targets>")
set(RunCMake_TEST_FAILED "<Targets> element not found in Build.xml")
endif()
if(NOT xml_content MATCHES "<Target name=\"main\" type=\"EXECUTABLE\">")
set(RunCMake_TEST_FAILED "<Target> element for 'main' not found in Build.xml")
endif()
if(NOT xml_content MATCHES "<Compile")
set(RunCMake_TEST_FAILED "<Compile> element not found in Build.xml")
endif()
if(NOT xml_content MATCHES "<Link")
set(RunCMake_TEST_FAILED "<Link> element not found in Build.xml")
endif()
if(NOT xml_content MATCHES "<CmakeBuild")
set(RunCMake_TEST_FAILED "<CmakeBuild> element not found in Build.xml")
endif()
endif()
else()
set(RunCMake_TEST_FAILED "${xml_type}.xml not found")
endif()
endforeach()
foreach(dir_to_check "configure" "test" "build/targets" "build/commands")
file(GLOB leftover_cdash_snippets
"${RunCMake_TEST_BINARY_DIR}/.cmake/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0/v1/cdash/${dir_to_check}/*")
if(leftover_cdash_snippets)
set(RunCMake_TEST_FAILED "Leftover snippets found in cdash dir: ${leftover_cdash_snippets}")
endif()
endforeach()

11
Tests/RunCMake/ctest_instrumentation/NoInstrumentationInCTestXML-check.cmake Normal file
View File

@@ -0,0 +1,11 @@
foreach(xml_type Configure Build Test)
file(GLOB xml_file "${RunCMake_TEST_BINARY_DIR}/Testing/*/${xml_type}.xml")
if(xml_file)
file(READ "${xml_file}" xml_content)
if(xml_content MATCHES "AfterHostMemoryUsed")
set(RunCMake_TEST_FAILED "'AfterHostMemoryUsed' found in ${xml_type}.xml")
endif()
else()
set(RunCMake_TEST_FAILED "${xml_type}.xml not found")
endif()
endforeach()

22
Tests/RunCMake/ctest_instrumentation/RunCMakeTest.cmake Normal file
View File

@@ -0,0 +1,22 @@
include(RunCTest)
function(run_InstrumentationInCTestXML USE_INSTRUMENTATION)
if(USE_INSTRUMENTATION)
set(ENV{CTEST_USE_INSTRUMENTATION} "1")
set(ENV{CTEST_EXPERIMENTAL_INSTRUMENTATION} "a37d1069-1972-4901-b9c9-f194aaf2b6e0")
set(RunCMake_USE_INSTRUMENTATION TRUE)
set(CASE_NAME InstrumentationInCTestXML)
else()
set(ENV{CTEST_USE_INSTRUMENTATION} "0")
set(ENV{CTEST_EXPERIMENTAL_INSTRUMENTATION} "0")
set(RunCMake_USE_INSTRUMENTATION FALSE)
set(CASE_NAME NoInstrumentationInCTestXML)
endif()
configure_file(${RunCMake_SOURCE_DIR}/main.c
${RunCMake_BINARY_DIR}/${CASE_NAME}/main.c COPYONLY)
run_ctest("${CASE_NAME}")
unset(RunCMake_USE_LAUNCHERS)
unset(RunCMake_USE_INSTRUMENTATION)
endfunction()
run_InstrumentationInCTestXML(ON)
run_InstrumentationInCTestXML(OFF)

4
Tests/RunCMake/ctest_instrumentation/main.c Normal file
View File

@@ -0,0 +1,4 @@
int main(void)
{
return 0;
}

17
Tests/RunCMake/ctest_instrumentation/test.cmake.in Normal file
View File

@@ -0,0 +1,17 @@
cmake_minimum_required(VERSION 3.10)
set(CTEST_SITE "test-site")
set(CTEST_BUILD_NAME "test-build-name")
set(CTEST_SOURCE_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@")
set(CTEST_BINARY_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@-build")
set(CTEST_CMAKE_GENERATOR "@RunCMake_GENERATOR@")
set(CTEST_CMAKE_GENERATOR_PLATFORM "@RunCMake_GENERATOR_PLATFORM@")
set(CTEST_CMAKE_GENERATOR_TOOLSET "@RunCMake_GENERATOR_TOOLSET@")
set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_CONFIG_TYPE}")
set(CTEST_USE_LAUNCHERS TRUE)
set(CTEST_USE_INSTRUMENTATION "@RunCMake_USE_INSTRUMENTATION@")
ctest_start(Experimental)
ctest_configure(OPTIONS "-DUSE_INSTRUMENTATION=${CTEST_USE_INSTRUMENTATION}")
ctest_build()
ctest_test()