diff --git a/Help/command/cmake_instrumentation.rst b/Help/command/cmake_instrumentation.rst new file mode 100644 index 0000000000..48b288a3aa --- /dev/null +++ b/Help/command/cmake_instrumentation.rst @@ -0,0 +1,62 @@ +cmake_instrumentation +--------------------- + +.. versionadded:: 3.32 + +Enables interacting with the +:manual:`CMake Instrumentation API `. + +This allows for configuring instrumentation at the project-level. + +.. code-block:: cmake + + cmake_instrumentation( + API_VERSION + DATA_VERSION + [HOOKS ...] + [QUERIES ...] + [CALLBACK ] + ) + +The ``API_VERSION`` and ``DATA_VERSION`` must always be given. Currently, the +only supported value for both fields is 1. See :ref:`cmake-instrumentation v1` +for details of the data output content and location. + +Each of the optional keywords ``HOOKS``, ``QUERIES``, and ``CALLBACK`` +correspond to one of the parameters to the :ref:`cmake-instrumentation v1 Query Files`. Note that the +``CALLBACK`` keyword only accepts a single callback. + +Whenever ``cmake_instrumentation`` is invoked, a query file is generated in +``/.cmake/timing/v1/query/generated`` to enable instrumentation +with the provided arguments. + +Example +^^^^^^^ + +The following example shows an invocation of the command and its +equivalent JSON query file. + +.. code-block:: cmake + + cmake_instrumentation( + API_VERSION 1 + DATA_VERSION 1 + HOOKS postGenerate preCMakeBuild postCMakeBuild + QUERIES staticSystemInformation dynamicSystemInformation + CALLBACK "${CMAKE_COMMAND} -P /path/to/handle_data.cmake" + ) + +.. code-block:: json + + { + "version": 1, + "hooks": [ + "postGenerate", "preCMakeBuild", "postCMakeBuild" + ], + "queries": [ + "staticSystemInformation", "dynamicSystemInformation" + ], + "callbacks": [ + "/path/to/cmake -P /path/to/handle_data.cmake" + ] + } diff --git a/Help/dev/experimental.rst b/Help/dev/experimental.rst index 6f02744086..fc03cdf0fa 100644 --- a/Help/dev/experimental.rst +++ b/Help/dev/experimental.rst @@ -119,3 +119,17 @@ When activated, this experimental feature provides the following: * Targets with the property set to a true value will have their C++ build information exported to the build database. + +Instrumentation +=============== + +In order to activate support for the :command:`cmake_instrumentation` command, +set + +* variable ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` to +* value ``a37d1069-1972-4901-b9c9-f194aaf2b6e0``. + +To enable instrumentation at the user-level, files should be blaced under +either +``/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0`` or +``/.cmake/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0``. diff --git a/Help/index.rst b/Help/index.rst index ca03213184..f9ffcbaf72 100644 --- a/Help/index.rst +++ b/Help/index.rst @@ -64,6 +64,7 @@ Reference Manuals /manual/cmake-file-api.7 /manual/cmake-generator-expressions.7 /manual/cmake-generators.7 + /manual/cmake-instrumentation.7 /manual/cmake-language.7 /manual/cmake-modules.7 /manual/cmake-packages.7 diff --git a/Help/manual/cmake-commands.7.rst b/Help/manual/cmake-commands.7.rst index 00f46aaed5..26b41d8cfe 100644 --- a/Help/manual/cmake-commands.7.rst +++ b/Help/manual/cmake-commands.7.rst @@ -89,6 +89,7 @@ These commands are available only in CMake projects. /command/aux_source_directory /command/build_command /command/cmake_file_api + /command/cmake_instrumentation /command/create_test_sourcelist /command/define_property /command/enable_language diff --git a/Help/manual/cmake-instrumentation.7.rst b/Help/manual/cmake-instrumentation.7.rst new file mode 100644 index 0000000000..b806d517eb --- /dev/null +++ b/Help/manual/cmake-instrumentation.7.rst @@ -0,0 +1,374 @@ +.. cmake-manual-description: CMake Instrumentation + +cmake-instrumentation(7) +************************ + +.. versionadded:: 3.32 + +.. only:: html + + .. contents:: + +Introduction +============ + +The CMake Instrumentation API allows for the collection of timing data, target +information and system diagnostic information during the configure, generate, +build, test and install steps for a CMake project. + +This feature is only available for projects using the :ref:`Makefile Generators` +or the :ref:`Ninja Generators`. + +All interactions with the CMake instrumentation API must specify both an API +version and a Data version. At this time, there is only one version for each of +these: the `API v1`_ and `Data v1`_. + +When instrumentation is enabled, CMake sets the :prop_gbl:`RULE_LAUNCH_COMPILE`, +:prop_gbl:`RULE_LAUNCH_LINK` and :prop_gbl:`RULE_LAUNCH_CUSTOM` global properties +to use the ``ctest --instrument`` launcher. Whenever a command is executed with +instrumentation enabled, a `v1 Snippet File`_ is created in the project build +tree. If the project has been configured with :module:`CTestUseLaunchers`, +``ctest --instrument`` will also include the behavior usually performed by +``ctest --launch``. + +Hooks are specific intervals, configured as part of the `v1 Query Files`_, +during which snippet data files are coallated. Whenever a hook executes, an +index file is generated containing a list of snippet files newer than the +previous indexing, and a sequence of custom callbacks are executed using +the index file as an argument. + +Indexing and callbacks can also be performed by manually invoking +``ctest --collect-instrumentation``. + +These callbacks, defined either at the user-level or project-level should read +the instrumentation data and perform any desired handling of it. The index file +and its listed snippets are automatically deleted by CMake once all callbacks +have completed. + +Configuring Instrumentation at the User-Level +--------------------------------------------- + +Instrumentation can be configured at the user-level by placing query files in the +:envvar:`CMAKE_CONFIG_DIR` under +``/instrumentation//query/``. This version of CMake +supports only one version schema, `API v1`_. + +Configuring Instrumentation at the Project-Level +------------------------------------------------ + +Configuring Instrumentation at the project level can be done by placing query +files under ``/.cmake/instrumentation/query/`` at the top of a build +tree. + +Additionally, project code can contain instrumentation queries with the +:command:`cmake_instrumentation` command. + +.. _`cmake-instrumentation v1`: + +API v1 +====== + +The API version specifies both the subdirectory layout of the instrumentation data, +and the format of the query files. + +The Instrumentation API v1 is housed in the ``instrumentation/v1/`` directory +under either ``/.cmake/`` for output data and project-level queries, or +``/`` for user-level queries. The ``v1`` component of this +directory is what signifies the API version. It has the following +subdirectories: + +``query/`` + Holds query files written by users or clients. Any file with the ``.json`` + file extension will be recognized as a query file. These files are owned by + whichever client or user creates them. + +``query/generated/`` + Holds query files generated by a CMake project with the + :command:`cmake_instrumentation` command. These files are owned by CMake and + are deleted and regenerated automatically during the CMake configure step. + +``data/`` + Holds instrumentation data collected on the project. CMake owns all data + files, they should never be removed by other processes. + +.. _`cmake-instrumentation v1 Query Files`: + +v1 Query Files +-------------- + +Any file with the ``.json`` extension under the ``instrumentation/v1/query/`` +directory is recognized as a query for instrumentation data. + +These files must contain a JSON object with the following keys which are all +optional. + +``version`` + The Data version of snippet file to generate, an integer. Currently the only + supported version is `1`. + +``callbacks`` + A list of command-line strings for callbacks to handle collected timing + data. Whenever these callbacks are executed, the full path to a + `v1 Index File`_ is appended to the arguments included in the string. + +``hooks`` + A list of strings specifying when instrumentation data should be collated + and user callbacks should be invoked on the data. Elements in this list + should be one of the following: + + * ``postGenerate`` + * ``preCMakeBuild`` + * ``postCMakeBuild`` + * ``postInstall`` + * ``postTest`` + +``queries`` + A list of strings specifying additional optional data to collect during + instrumentation. Elements in this list should be one of the following: + + ``staticSystemInformation`` + Enables collection of the static information about the host machine + CMake is being run from. This data is collected once at each hook and + included in the generated ``index-.json`` file. + + ``dynamicSystemInformation`` + Enables collection of the dynamic information about the host machine + CMake is being run from. Data is collected for every snippet file + generated by CMake, with data immediately before and after the command is + executed. + +The ``callbacks`` listed will be invoked during the specified hooks +*at a minimum*. When there are multiple queries, the ``callbacks``, ``hooks`` +and ``queries`` between them will be merged. Therefore, if any query file +includes any ``hooks``, every ``callback`` across all query files will be +executed at every ``hook`` across all query files. Additionally, if any query +file includes any optional ``queries``, the optional query data will be present +in all data files. + +Example: + +.. code-block:: json + + { + "version": 1, + "callbacks": [ + "/usr/bin/python callback.py", + "/usr/bin/cmake -P callback.cmake arg", + ], + "hooks": [ + "postCMakeBuild", + "postInstall" + ], + "queries": [ + "staticSystemInformation", + "dynamicSystemInformation" + ] + } + +In this example, after every ``cmake --build`` or ``cmake --install`` +invocation, an index file ``index-.json`` will be generated in +``/.cmake/instrumentation/v1/data`` containing a list of data snippet +files created since the previous indexing. The commands +``/usr/bin/python callback.py index-.json`` and +``/usr/bin/cmake -P callback.cmake arg index-.json`` will be executed in +that order. The index file will contain the ``staticSystemInformation`` data and +each snippet file listed in the index will contain the +``dynamicSystemInformation`` data. Once both callbacks have completed, the index +file and all snippet files listed by it will be deleted from the project build +tree. + +Data v1 +======= + +Data version specifies the contents of the output files generated by the CMake +instrumentation API. There are two types of data files generated. When using +the `API v1`_, these files live in ``/.cmake/instrumentation/v1/data/`` +under the project build tree. These are the `v1 Snippet File`_ and +`v1 Index File`_. + +v1 Snippet File +--------------- + +Snippet files are generated for every compile, link and custom command invoked +as part of the CMake build or install step and contain instrumentation data about +the command executed. Additionally, snippet files are created for the following: + +* The CMake configure step +* The CMake generate step +* Entire build step (executed with ``cmake --build``) +* Entire install step (executed with ``cmake --install``) +* Each ``ctest`` invocation +* Each individual test executed by ``ctest``. + +Snippet files have a filename with the syntax ``--.json`` +and contain the following data: + + ``version`` + The Data version of the snippet file, an integer. Currently the version is + always `1`. + + ``command`` + The full command executed. + + ``result`` + The exit-value of the command, an integer. + + ``role`` + The type of command executed, which will be one of the following values: + + * ``compile`` + * ``link`` + * ``custom`` + * ``cmakeBuild`` + * ``install`` + * ``ctest`` + * ``test`` + + ``target`` + The CMake target associated with the command. Only included when ``role`` is + one of ``compile``, ``link``, ``custom``. + + ``targetType`` + The :prop_tgt:`TYPE` of the target. Only included when ``role`` is + ``link``. + + ``timeStart`` + Time at which the command started, expressed as the number of milliseconds + since the system epoch. + + ``duration`` + The duration that the command ran for, expressed in milliseconds. + + ``outputs`` + The command's output file(s), an array. Only included when ``role`` is one + of: ``compile``, ``link``, ``custom``. + + ``outputSizes`` + The size(s) in bytes of the ``outputs``, an array. For files which do not + exist, the size is 0. + + ``source`` + The source file being compiled. Only included when ``role`` is ``compile``. + + ``language`` + The language of the source file being compiled. Only included when ``role`` is + ``compile``. + + ``testName`` + The name of the test being executed. Only included when ``role`` is ``test``. + + ``dynamicSystemInformation`` + Specifies the dynamic information collected about the host machine + CMake is being run from. Data is collected for every snippet file + generated by CMake, with data immediately before and after the command is + executed. + + ``beforeHostMemoryUsed`` + The Host Memory Used in KiB at ``timeStart``. + + ``afterHostMemoryUsed`` + The Host Memory Used in KiB at ``timeStop``. + + ``beforeCPULoadAverage`` + The Average CPU Load at ``timeStart``. + + ``afterCPULoadAverage`` + The Average CPU Load at ``timeStop``. + +Example: + +.. code-block:: json + + { + "version": 1, + "command" : "/usr/bin/c++ -MD -MT CMakeFiles/main.dir/main.cxx.o -MF CMakeFiles/main.dir/main.cxx.o.d -o CMakeFiles/main.dir/main.cxx.o -c /main.cxx", + "role" : "compile", + "return" : 1, + "target": "main", + "language" : "C++", + "outputs" : [ "CMakeFiles/main.dir/main.cxx.o" ], + "outputSizes" : [ 0 ], + "source" : "/main.cxx" + "dynamicSystemInformation" : + { + "afterCPULoadAverage" : 2.3500000000000001, + "afterHostMemoryUsed" : 6635680.0 + "beforeCPULoadAverage" : 2.3500000000000001, + "beforeHostMemoryUsed" : 6635832.0 + }, + "timeStart" : 31997009, + "timeStop" : 31997056 + } + +v1 Index File +------------- + +Index files contain a list of `v1 Snippet File`_. It serves as an entry point +for navigating the instrumentation data. + +``version`` + The Data version of the index file, an integer. Currently the version is + always `1`. + +``buildDir`` + The build directory of the CMake project. + +``dataDir`` + The full path to the ``/.cmake/instrumentation/v1/data/`` directory. + +``hook`` + The name of the hook responsible for generating the index file. In addition + to the hooks that can be specified by one of the `v1 Query Files`_, this value may + be set to ``manual`` if indexing is performed by invoking + ``ctest --collect-instrumentation``. + +``snippets`` + Contains a list of `v1 Snippet File`_. This includes all snippet files + generated since the previous index file was created. The file paths are + relative to ``dataDir``. + +``staticSystemInformation`` + Specifies the static information collected about the host machine + CMake is being run from. This data is collected once at each hook and + included in the generated ``index-.json`` file. + + * ``OSName`` + * ``OSPlatform`` + * ``OSRelease`` + * ``OSVersion`` + * ``familyId`` + * ``hostname`` + * ``is64Bits`` + * ``modelId`` + * ``numberOfLogicalCPU`` + * ``numberOfPhysicalCPU`` + * ``processorAPICID`` + * ``processorCacheSize`` + * ``processorClockFrequency`` + * ``processorName`` + * ``totalPhysicalMemory`` + * ``totalVirtualMemory`` + * ``vendorID`` + * ``vendorString`` + +Example: + +.. code-block:: json + + { + "version": 1, + "hook": "manual", + "buildDir": "", + "dataDir": "/.cmake/instrumentation/v1/data", + "snippets": [ + "configure--.json", + "generate--.json", + "compile--.json", + "compile--.json", + "link--.json", + "install--.json", + "ctest--.json", + "test--.json", + "test--.json", + ] + } diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index 31bc8054e9..047105c2eb 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -359,6 +359,12 @@ add_library( cmInstallDirectoryGenerator.cxx cmInstallScriptHandler.h cmInstallScriptHandler.cxx + cmInstrumentation.h + cmInstrumentation.cxx + cmInstrumentationCommand.h + cmInstrumentationCommand.cxx + cmInstrumentationQuery.h + cmInstrumentationQuery.cxx cmJSONHelpers.cxx cmJSONHelpers.h cmJSONState.cxx diff --git a/Source/CTest/cmCTestLaunch.cxx b/Source/CTest/cmCTestLaunch.cxx index 9669d76a32..3a2db59cab 100644 --- a/Source/CTest/cmCTestLaunch.cxx +++ b/Source/CTest/cmCTestLaunch.cxx @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,7 @@ #include "cmCTestLaunchReporter.h" #include "cmGlobalGenerator.h" +#include "cmInstrumentation.h" #include "cmMakefile.h" #include "cmProcessOutput.h" #include "cmState.h" @@ -33,7 +35,7 @@ # include // for _setmode #endif -cmCTestLaunch::cmCTestLaunch(int argc, const char* const* argv) +cmCTestLaunch::cmCTestLaunch(int argc, const char* const* argv, Op operation) { if (!this->ParseArguments(argc, argv)) { return; @@ -45,6 +47,7 @@ cmCTestLaunch::cmCTestLaunch(int argc, const char* const* argv) this->ScrapeRulesLoaded = false; this->HaveOut = false; this->HaveErr = false; + this->Operation = operation; } cmCTestLaunch::~cmCTestLaunch() = default; @@ -61,6 +64,8 @@ bool cmCTestLaunch::ParseArguments(int argc, const char* const* argv) DoingLanguage, DoingTargetName, DoingTargetType, + DoingCommandType, + DoingRole, DoingBuildDir, DoingCount, DoingFilterPrefix @@ -71,6 +76,8 @@ bool cmCTestLaunch::ParseArguments(int argc, const char* const* argv) const char* arg = argv[i]; if (strcmp(arg, "--") == 0) { arg0 = i + 1; + } else if (strcmp(arg, "--command-type") == 0) { + doing = DoingCommandType; } else if (strcmp(arg, "--output") == 0) { doing = DoingOutput; } else if (strcmp(arg, "--source") == 0) { @@ -81,6 +88,8 @@ bool cmCTestLaunch::ParseArguments(int argc, const char* const* argv) doing = DoingTargetName; } else if (strcmp(arg, "--target-type") == 0) { doing = DoingTargetType; + } else if (strcmp(arg, "--role") == 0) { + doing = DoingRole; } else if (strcmp(arg, "--build-dir") == 0) { doing = DoingBuildDir; } else if (strcmp(arg, "--filter-prefix") == 0) { @@ -109,6 +118,12 @@ bool cmCTestLaunch::ParseArguments(int argc, const char* const* argv) } else if (doing == DoingFilterPrefix) { this->Reporter.OptionFilterPrefix = arg; doing = DoingNone; + } else if (doing == DoingCommandType) { + this->Reporter.OptionCommandType = arg; + doing = DoingNone; + } else if (doing == DoingRole) { + this->Reporter.OptionRole = arg; + doing = DoingNone; } } @@ -233,15 +248,33 @@ void cmCTestLaunch::RunChild() int cmCTestLaunch::Run() { - this->RunChild(); + auto instrumenter = cmInstrumentation(this->Reporter.OptionBuildDir); + std::map options; + options["target"] = this->Reporter.OptionTargetName; + options["source"] = this->Reporter.OptionSource; + options["language"] = this->Reporter.OptionLanguage; + options["targetType"] = this->Reporter.OptionTargetType; + options["role"] = this->Reporter.OptionRole; + std::map arrayOptions; + arrayOptions["outputs"] = this->Reporter.OptionOutput; + instrumenter.InstrumentCommand( + this->Reporter.OptionCommandType, this->RealArgV, + [this]() -> int { + this->RunChild(); + return 0; + }, + options, arrayOptions); - if (this->CheckResults()) { - return this->Reporter.ExitCode; + if (this->Operation == Op::Normal) { + + if (this->CheckResults()) { + return this->Reporter.ExitCode; + } + + this->LoadConfig(); + this->Reporter.WriteXML(); } - this->LoadConfig(); - this->Reporter.WriteXML(); - return this->Reporter.ExitCode; } @@ -314,14 +347,14 @@ bool cmCTestLaunch::ScrapeLog(std::string const& fname) return false; } -int cmCTestLaunch::Main(int argc, const char* const argv[]) +int cmCTestLaunch::Main(int argc, const char* const argv[], Op operation) { if (argc == 2) { std::cerr << "ctest --launch: this mode is for internal CTest use only" << std::endl; return 1; } - cmCTestLaunch self(argc, argv); + cmCTestLaunch self(argc, argv, operation); return self.Run(); } diff --git a/Source/CTest/cmCTestLaunch.h b/Source/CTest/cmCTestLaunch.h index ef21a26ccd..3e3519c306 100644 --- a/Source/CTest/cmCTestLaunch.h +++ b/Source/CTest/cmCTestLaunch.h @@ -20,16 +20,23 @@ class RegularExpression; */ class cmCTestLaunch { + public: + enum class Op + { + Normal, + Instrument, + }; + /** Entry point from ctest executable main(). */ - static int Main(int argc, const char* const argv[]); + static int Main(int argc, const char* const argv[], Op operation); cmCTestLaunch(const cmCTestLaunch&) = delete; cmCTestLaunch& operator=(const cmCTestLaunch&) = delete; private: // Initialize the launcher from its command line. - cmCTestLaunch(int argc, const char* const* argv); + cmCTestLaunch(int argc, const char* const* argv, Op operation); ~cmCTestLaunch(); // Run the real command. @@ -65,4 +72,7 @@ private: // Configuration void LoadConfig(); + + // Mode + Op Operation; }; diff --git a/Source/CTest/cmCTestLaunchReporter.h b/Source/CTest/cmCTestLaunchReporter.h index 2bb78f8a51..16f14617c2 100644 --- a/Source/CTest/cmCTestLaunchReporter.h +++ b/Source/CTest/cmCTestLaunchReporter.h @@ -38,6 +38,8 @@ public: std::string OptionTargetType; std::string OptionBuildDir; std::string OptionFilterPrefix; + std::string OptionCommandType; + std::string OptionRole; // The real command line appearing after launcher arguments. std::string CWD; diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index 1ca9807ebf..6c8d3d2b8c 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -34,6 +34,7 @@ cmCTestRunTest::cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler, , CTest(MultiTestHandler.CTest) , TestHandler(MultiTestHandler.TestHandler) , TestProperties(MultiTestHandler.Properties[Index]) + , Instrumentation(cmSystemTools::GetLogicalWorkingDirectory()) { } @@ -663,6 +664,9 @@ bool cmCTestRunTest::StartTest(size_t completed, size_t total) return false; } this->StartTime = this->CTest->CurrentTime(); + if (this->Instrumentation.HasQuery()) { + this->Instrumentation.GetPreTestStats(); + } return this->ForkProcess(); } @@ -1012,6 +1016,12 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total) void cmCTestRunTest::FinalizeTest(bool started) { + if (this->Instrumentation.HasQuery()) { + this->Instrumentation.InstrumentTest( + this->TestProperties->Name, this->ActualCommand, this->Arguments, + this->TestProcess->GetExitValue(), this->TestProcess->GetStartTime(), + this->TestProcess->GetSystemStartTime()); + } this->MultiTestHandler.FinishTestProcess(this->TestProcess->GetRunner(), started); } diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h index 71d0865417..b5e2a96d20 100644 --- a/Source/CTest/cmCTestRunTest.h +++ b/Source/CTest/cmCTestRunTest.h @@ -14,6 +14,7 @@ #include "cmCTest.h" #include "cmCTestMultiProcessHandler.h" #include "cmCTestTestHandler.h" +#include "cmInstrumentation.h" #include "cmProcess.h" /** \class cmRunTest @@ -140,6 +141,7 @@ private: int NumberOfRunsTotal = 1; // default to 1 run of the test bool RunAgain = false; // default to not having to run again size_t TotalNumberOfTests; + cmInstrumentation Instrumentation; }; inline int getNumWidth(size_t n) diff --git a/Source/CTest/cmProcess.h b/Source/CTest/cmProcess.h index dc755eb645..0131de654c 100644 --- a/Source/CTest/cmProcess.h +++ b/Source/CTest/cmProcess.h @@ -66,6 +66,14 @@ public: void SetId(int id) { this->Id = id; } int64_t GetExitValue() const { return this->ExitValue; } cmDuration GetTotalTime() { return this->TotalTime; } + std::chrono::steady_clock::time_point GetStartTime() + { + return this->StartTime; + } + std::chrono::system_clock::time_point GetSystemStartTime() + { + return this->SystemStartTime; + } enum class Exception { @@ -97,6 +105,7 @@ private: cm::optional Timeout; TimeoutReason TimeoutReason_ = TimeoutReason::Normal; std::chrono::steady_clock::time_point StartTime; + std::chrono::system_clock::time_point SystemStartTime; cmDuration TotalTime; bool ReadHandleClosed = false; bool ProcessHandleClosed = false; diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 0793a867c3..0aeffc1524 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -50,6 +51,8 @@ #include "cmExecutionStatus.h" #include "cmGeneratedFileStream.h" #include "cmGlobalGenerator.h" +#include "cmInstrumentation.h" +#include "cmInstrumentationQuery.h" #include "cmJSONState.h" #include "cmList.h" #include "cmListFileCache.h" @@ -2623,23 +2626,32 @@ int cmCTest::Run(std::vector const& args) } #endif - // now what should cmake do? if --build-and-test was specified then - // we run the build and test handler and return - if (cmakeAndTest) { - return this->RunCMakeAndTest(); - } + cmInstrumentation instrumentation( + cmSystemTools::GetCurrentWorkingDirectory()); + std::function doTest = [this, &cmakeAndTest, &runScripts, + &processSteps]() -> int { + // now what should cmake do? if --build-and-test was specified then + // we run the build and test handler and return + if (cmakeAndTest) { + return this->RunCMakeAndTest(); + } - // -S, -SP, and/or -SP was specified - if (!runScripts.empty()) { - return this->RunScripts(runScripts); - } + // -S, -SP, and/or -SP was specified + if (!runScripts.empty()) { + return this->RunScripts(runScripts); + } - // -D, -T, and/or -M was specified - if (processSteps) { - return this->ProcessSteps(); - } + // -D, -T, and/or -M was specified + if (processSteps) { + return this->ProcessSteps(); + } - return this->ExecuteTests(); + return this->ExecuteTests(); + }; + int ret = instrumentation.InstrumentCommand("ctest", args, + [doTest]() { return doTest(); }); + instrumentation.CollectTimingData(cmInstrumentationQuery::Hook::PostTest); + return ret; } int cmCTest::RunScripts( diff --git a/Source/cmCommands.cxx b/Source/cmCommands.cxx index 089bafc713..ce690bd3b2 100644 --- a/Source/cmCommands.cxx +++ b/Source/cmCommands.cxx @@ -52,6 +52,7 @@ #include "cmInstallCommand.h" #include "cmInstallFilesCommand.h" #include "cmInstallTargetsCommand.h" +#include "cmInstrumentationCommand.h" #include "cmLinkDirectoriesCommand.h" #include "cmListCommand.h" #include "cmMacroCommand.h" @@ -301,6 +302,7 @@ void GetProjectCommands(cmState* state) state->AddBuiltinCommand("remove_definitions", cmRemoveDefinitionsCommand); state->AddBuiltinCommand("source_group", cmSourceGroupCommand); state->AddBuiltinCommand("cmake_file_api", cmFileAPICommand); + state->AddBuiltinCommand("cmake_instrumentation", cmInstrumentationCommand); state->AddDisallowedCommand( "export_library_dependencies", cmExportLibraryDependenciesCommand, diff --git a/Source/cmCustomCommand.cxx b/Source/cmCustomCommand.cxx index 9958e4d25d..afb9517619 100644 --- a/Source/cmCustomCommand.cxx +++ b/Source/cmCustomCommand.cxx @@ -154,6 +154,16 @@ void cmCustomCommand::SetUsesTerminal(bool b) this->UsesTerminal = b; } +void cmCustomCommand::SetRole(const std::string& role) +{ + this->Role = role; +} + +const std::string& cmCustomCommand::GetRole() const +{ + return this->Role; +} + bool cmCustomCommand::GetCommandExpandLists() const { return this->CommandExpandLists; diff --git a/Source/cmCustomCommand.h b/Source/cmCustomCommand.h index 6f63d0a169..9fc4dfb095 100644 --- a/Source/cmCustomCommand.h +++ b/Source/cmCustomCommand.h @@ -132,6 +132,10 @@ public: const std::string& GetTarget() const; void SetTarget(const std::string& target); + /** Set/Get the custom command rolee */ + const std::string& GetRole() const; + void SetRole(const std::string& role); + /** Record if the custom command can be used for code generation. */ bool GetCodegen() const { return Codegen; } void SetCodegen(bool b) { Codegen = b; } @@ -148,6 +152,7 @@ private: std::string WorkingDirectory; std::string Depfile; std::string JobPool; + std::string Role; bool JobserverAware = false; bool HaveComment = false; bool EscapeAllowMakeVars = false; diff --git a/Source/cmExperimental.cxx b/Source/cmExperimental.cxx index f6afc945eb..3e701bff44 100644 --- a/Source/cmExperimental.cxx +++ b/Source/cmExperimental.cxx @@ -75,6 +75,15 @@ cmExperimental::FeatureData LookupTable[] = { {}, cmExperimental::TryCompileCondition::Never, false }, + // Instrumentation + { "Instrumentation", + "a37d1069-1972-4901-b9c9-f194aaf2b6e0", + "CMAKE_EXPERIMENTAL_INSTRUMENTATION", + "CMake's support for collecting instrumentation data is experimental. It " + "is meant only for experimentation and feedback to CMake developers.", + {}, + cmExperimental::TryCompileCondition::Never, + false }, }; static_assert(sizeof(LookupTable) / sizeof(LookupTable[0]) == static_cast(cmExperimental::Feature::Sentinel), diff --git a/Source/cmExperimental.h b/Source/cmExperimental.h index 6410918142..b9fa1f2649 100644 --- a/Source/cmExperimental.h +++ b/Source/cmExperimental.h @@ -23,6 +23,7 @@ public: ImportPackageInfo, ExportPackageInfo, ExportBuildDatabase, + Instrumentation, Sentinel, }; diff --git a/Source/cmGeneratorTarget.cxx b/Source/cmGeneratorTarget.cxx index 7eb23bd94d..805e07837d 100644 --- a/Source/cmGeneratorTarget.cxx +++ b/Source/cmGeneratorTarget.cxx @@ -2306,6 +2306,7 @@ cmGeneratorTarget::GetClassifiedFlagsForSource(cmSourceFile const* sf, vars.CMTargetName = this->GetName().c_str(); vars.CMTargetType = cmState::GetTargetTypeName(this->GetType()).c_str(); vars.Language = lang.c_str(); + auto const sfPath = this->LocalGenerator->ConvertToOutputFormat( sf->GetFullPath(), cmOutputConverter::SHELL); diff --git a/Source/cmGlobalGenerator.cxx b/Source/cmGlobalGenerator.cxx index 1d560c8895..2e6430edce 100644 --- a/Source/cmGlobalGenerator.cxx +++ b/Source/cmGlobalGenerator.cxx @@ -3119,6 +3119,7 @@ void cmGlobalGenerator::AddGlobalTarget_Install( gti.Message = "Install the project..."; gti.UsesTerminal = true; gti.StdPipesUTF8 = true; + gti.Role = "install"; cmCustomCommandLine singleLine; if (this->GetPreinstallTargetName()) { gti.Depends.emplace_back(this->GetPreinstallTargetName()); @@ -3157,6 +3158,7 @@ void cmGlobalGenerator::AddGlobalTarget_Install( if (const char* install_local = this->GetInstallLocalTargetName()) { gti.Name = install_local; gti.Message = "Installing only the local directory..."; + gti.Role = "install"; gti.UsesTerminal = !this->GetCMakeInstance()->GetState()->GetGlobalPropertyAsBool( "INSTALL_PARALLEL"); @@ -3177,6 +3179,7 @@ void cmGlobalGenerator::AddGlobalTarget_Install( gti.Name = install_strip; gti.Message = "Installing the project stripped..."; gti.UsesTerminal = true; + gti.Role = "install"; gti.CommandLines.clear(); cmCustomCommandLine stripCmdLine = singleLine; @@ -3437,6 +3440,7 @@ void cmGlobalGenerator::CreateGlobalTarget(GlobalTargetInfo const& gti, cc.SetWorkingDirectory(gti.WorkingDir.c_str()); cc.SetStdPipesUTF8(gti.StdPipesUTF8); cc.SetUsesTerminal(gti.UsesTerminal); + cc.SetRole(gti.Role); target.AddPostBuildCommand(std::move(cc)); if (!gti.Message.empty()) { target.SetProperty("EchoString", gti.Message); diff --git a/Source/cmGlobalGenerator.h b/Source/cmGlobalGenerator.h index 7f07c1cfd2..c0f0fdf998 100644 --- a/Source/cmGlobalGenerator.h +++ b/Source/cmGlobalGenerator.h @@ -745,6 +745,7 @@ protected: bool UsesTerminal = false; cmTarget::PerConfig PerConfig = cmTarget::PerConfig::Yes; bool StdPipesUTF8 = false; + std::string Role; }; void CreateDefaultGlobalTargets(std::vector& targets); diff --git a/Source/cmInstallScriptHandler.cxx b/Source/cmInstallScriptHandler.cxx index e4620d1e3d..eb189987d3 100644 --- a/Source/cmInstallScriptHandler.cxx +++ b/Source/cmInstallScriptHandler.cxx @@ -21,6 +21,7 @@ #include "cmCryptoHash.h" #include "cmGeneratedFileStream.h" +#include "cmInstrumentation.h" #include "cmJSONState.h" #include "cmProcessOutput.h" #include "cmStringAlgorithms.h" @@ -102,13 +103,27 @@ std::vector> cmInstallScriptHandler::GetCommands() return this->commands; } -int cmInstallScriptHandler::Install(unsigned int j) +int cmInstallScriptHandler::Install(unsigned int j, + cmInstrumentation& instrumentation) { cm::uv_loop_ptr loop; loop.init(); std::vector scripts; scripts.reserve(this->commands.size()); - for (auto const& cmd : this->commands) { + + std::vector instrument_arg; + if (instrumentation.HasQuery()) { + instrument_arg = { cmSystemTools::GetCTestCommand(), + "--instrument", + "--command-type", + "install", + "--build-dir", + this->binaryDir, + "--" }; + } + + for (auto& cmd : this->commands) { + cmd.insert(cmd.begin(), instrument_arg.begin(), instrument_arg.end()); scripts.emplace_back(cmd); } std::size_t working = 0; diff --git a/Source/cmInstallScriptHandler.h b/Source/cmInstallScriptHandler.h index a904dc9cc6..aa489014f1 100644 --- a/Source/cmInstallScriptHandler.h +++ b/Source/cmInstallScriptHandler.h @@ -12,6 +12,8 @@ #include "cmUVProcessChain.h" #include "cmUVStream.h" +class cmInstrumentation; + class cmInstallScriptHandler { public: @@ -19,7 +21,7 @@ public: cmInstallScriptHandler(std::string, std::string, std::string, std::vector&); bool IsParallel(); - int Install(unsigned int j); + int Install(unsigned int j, cmInstrumentation& instrumentation); std::vector> GetCommands() const; class InstallScript { diff --git a/Source/cmInstrumentation.cxx b/Source/cmInstrumentation.cxx new file mode 100644 index 0000000000..c7147c6ae8 --- /dev/null +++ b/Source/cmInstrumentation.cxx @@ -0,0 +1,493 @@ +#include "cmInstrumentation.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "cmsys/Directory.hxx" +#include "cmsys/FStream.hxx" +#include + +#include "cmCryptoHash.h" +#include "cmExperimental.h" +#include "cmInstrumentationQuery.h" +#include "cmStringAlgorithms.h" +#include "cmSystemTools.h" +#include "cmTimestamp.h" + +cmInstrumentation::cmInstrumentation(const std::string& binary_dir, + bool clear_generated) +{ + const std::string uuid = + cmExperimental::DataForFeature(cmExperimental::Feature::Instrumentation) + .Uuid; + this->binaryDir = binary_dir; + this->timingDirv1 = + cmStrCat(this->binaryDir, "/.cmake/instrumentation-", uuid, "/v1"); + if (clear_generated) { + this->ClearGeneratedQueries(); + } + if (cm::optional configDir = + cmSystemTools::GetCMakeConfigDirectory()) { + this->userTimingDirv1 = + cmStrCat(configDir.value(), "/instrumentation-", uuid, "/v1"); + } + this->LoadQueries(); +} + +void cmInstrumentation::LoadQueries() +{ + if (cmSystemTools::FileExists(cmStrCat(this->timingDirv1, "/query"))) { + this->hasQuery = + this->ReadJSONQueries(cmStrCat(this->timingDirv1, "/query")) || + this->ReadJSONQueries(cmStrCat(this->timingDirv1, "/query/generated")); + } + if (!this->userTimingDirv1.empty() && + cmSystemTools::FileExists(cmStrCat(this->userTimingDirv1, "/query"))) { + this->hasQuery = this->hasQuery || + this->ReadJSONQueries(cmStrCat(this->userTimingDirv1, "/query")); + } +} + +cmInstrumentation::cmInstrumentation( + const std::string& binary_dir, + std::set& queries_, + std::set& hooks_, std::string& callback) +{ + this->binaryDir = binary_dir; + this->timingDirv1 = cmStrCat( + this->binaryDir, "/.cmake/instrumentation-", + cmExperimental::DataForFeature(cmExperimental::Feature::Instrumentation) + .Uuid, + "/v1"); + this->queries = queries_; + this->hooks = hooks_; + if (!callback.empty()) { + this->callbacks.push_back(callback); + } +} + +bool cmInstrumentation::ReadJSONQueries(const std::string& directory) +{ + cmsys::Directory d; + std::string json = ".json"; + bool result = false; + if (d.Load(directory)) { + for (unsigned int i = 0; i < d.GetNumberOfFiles(); i++) { + std::string fpath = d.GetFilePath(i); + if (fpath.rfind(json) == (fpath.size() - json.size())) { + result = true; + this->ReadJSONQuery(fpath); + } + } + } + return result; +} + +void cmInstrumentation::ReadJSONQuery(const std::string& file) +{ + auto query = cmInstrumentationQuery(); + query.ReadJSON(file, this->errorMsg, this->queries, this->hooks, + this->callbacks); +} + +void cmInstrumentation::WriteJSONQuery() +{ + Json::Value root; + root["version"] = 1; + root["queries"] = Json::arrayValue; + for (auto const& query : this->queries) { + root["queries"].append(cmInstrumentationQuery::QueryString[query]); + } + root["hooks"] = Json::arrayValue; + for (auto const& hook : this->hooks) { + root["hooks"].append(cmInstrumentationQuery::HookString[hook]); + } + root["callbacks"] = Json::arrayValue; + for (auto const& callback : this->callbacks) { + root["callbacks"].append(callback); + } + cmsys::Directory d; + int n = 0; + if (d.Load(cmStrCat(this->timingDirv1, "/query/generated"))) { + n = (int)d.GetNumberOfFiles() - 2; // Don't count '.' or '..' + } + this->WriteInstrumentationJson(root, "query/generated", + cmStrCat("query-", n, ".json")); +} + +void cmInstrumentation::ClearGeneratedQueries() +{ + std::string dir = cmStrCat(this->timingDirv1, "/query/generated"); + if (cmSystemTools::FileIsDirectory(dir)) { + cmSystemTools::RemoveADirectory(dir); + } +} + +bool cmInstrumentation::HasQuery() +{ + return this->hasQuery; +} + +bool cmInstrumentation::HasQuery(cmInstrumentationQuery::Query query) +{ + return (this->queries.find(query) != this->queries.end()); +} + +int cmInstrumentation::CollectTimingData(cmInstrumentationQuery::Hook hook) +{ + // Don't run collection if hook is disabled + if (hook != cmInstrumentationQuery::Hook::Manual && + this->hooks.find(hook) == this->hooks.end()) { + return 0; + } + + // Touch index file immediately to claim snippets + const std::string& directory = cmStrCat(this->timingDirv1, "/data"); + std::string const& file_name = + cmStrCat("index-", ComputeSuffixTime(), ".json"); + std::string index_path = cmStrCat(directory, "/", file_name); + cmSystemTools::Touch(index_path, true); + + // Gather Snippets + using snippet = std::pair; + std::vector files; + cmsys::Directory d; + std::string last_index; + if (d.Load(directory)) { + for (unsigned int i = 0; i < d.GetNumberOfFiles(); i++) { + std::string fpath = d.GetFilePath(i); + std::string fname = d.GetFile(i); + if (fname.rfind('.', 0) == 0) { + continue; + } + if (fname == file_name) { + continue; + } + if (fname.rfind("index-", 0) == 0) { + if (last_index.empty()) { + last_index = fpath; + } else { + int compare; + cmSystemTools::FileTimeCompare(fpath, last_index, &compare); + if (compare == 1) { + last_index = fpath; + } + } + } + files.push_back(snippet(std::move(fname), std::move(fpath))); + } + } + + // Build Json Object + Json::Value index(Json::objectValue); + index["snippets"] = Json::arrayValue; + index["hook"] = cmInstrumentationQuery::HookString[hook]; + index["dataDir"] = directory; + index["buildDir"] = this->binaryDir; + index["version"] = 1; + if (this->HasQuery(cmInstrumentationQuery::Query::StaticSystemInformation)) { + this->InsertStaticSystemInformation(index); + } + for (auto const& file : files) { + if (last_index.empty()) { + index["snippets"].append(file.first); + } else { + int compare; + cmSystemTools::FileTimeCompare(file.second, last_index, &compare); + if (compare == 1) { + index["snippets"].append(file.first); + } + } + } + this->WriteInstrumentationJson(index, "data", file_name); + + // Execute callbacks + for (auto& cb : this->callbacks) { + cmSystemTools::RunSingleCommand(cmStrCat(cb, " \"", index_path, "\""), + nullptr, nullptr, nullptr, nullptr, + cmSystemTools::OUTPUT_PASSTHROUGH); + } + + // Delete files + for (auto const& f : index["snippets"]) { + cmSystemTools::RemoveFile(cmStrCat(directory, "/", f.asString())); + } + cmSystemTools::RemoveFile(index_path); + + return 0; +} + +void cmInstrumentation::InsertDynamicSystemInformation( + Json::Value& root, const std::string& prefix) +{ + cmsys::SystemInformation info; + Json::Value data; + info.RunCPUCheck(); + info.RunMemoryCheck(); + if (!root.isMember("dynamicSystemInformation")) { + root["dynamicSystemInformation"] = Json::objectValue; + } + root["dynamicSystemInformation"][cmStrCat(prefix, "HostMemoryUsed")] = + (double)info.GetHostMemoryUsed(); + root["dynamicSystemInformation"][cmStrCat(prefix, "CPULoadAverage")] = + info.GetLoadAverage(); +} + +void cmInstrumentation::GetDynamicSystemInformation(double& memory, + double& load) +{ + cmsys::SystemInformation info; + Json::Value data; + info.RunCPUCheck(); + info.RunMemoryCheck(); + memory = (double)info.GetHostMemoryUsed(); + load = info.GetLoadAverage(); +} + +void cmInstrumentation::InsertStaticSystemInformation(Json::Value& root) +{ + cmsys::SystemInformation info; + info.RunCPUCheck(); + info.RunOSCheck(); + info.RunMemoryCheck(); + Json::Value infoRoot; + infoRoot["familyId"] = info.GetFamilyID(); + infoRoot["hostname"] = info.GetHostname(); + infoRoot["is64Bits"] = info.Is64Bits(); + infoRoot["modelId"] = info.GetModelID(); + infoRoot["numberOfLogicalCPU"] = info.GetNumberOfLogicalCPU(); + infoRoot["numberOfPhysicalCPU"] = info.GetNumberOfPhysicalCPU(); + infoRoot["OSName"] = info.GetOSName(); + infoRoot["OSPlatform"] = info.GetOSPlatform(); + infoRoot["OSRelease"] = info.GetOSRelease(); + infoRoot["OSVersion"] = info.GetOSVersion(); + infoRoot["processorAPICID"] = info.GetProcessorAPICID(); + infoRoot["processorCacheSize"] = info.GetProcessorCacheSize(); + infoRoot["processorClockFrequency"] = + (double)info.GetProcessorClockFrequency(); + infoRoot["processorName"] = info.GetExtendedProcessorName(); + infoRoot["totalPhysicalMemory"] = + static_cast(info.GetTotalPhysicalMemory()); + infoRoot["totalVirtualMemory"] = + static_cast(info.GetTotalVirtualMemory()); + infoRoot["vendorID"] = info.GetVendorID(); + infoRoot["vendorString"] = info.GetVendorString(); + root["staticSystemInformation"] = infoRoot; +} + +void cmInstrumentation::InsertTimingData( + Json::Value& root, std::chrono::steady_clock::time_point steadyStart, + std::chrono::system_clock::time_point systemStart) +{ + uint64_t timeStart = std::chrono::duration_cast( + systemStart.time_since_epoch()) + .count(); + uint64_t duration = std::chrono::duration_cast( + std::chrono::steady_clock::now() - steadyStart) + .count(); + root["timeStart"] = static_cast(timeStart); + root["duration"] = static_cast(duration); +} + +void cmInstrumentation::WriteInstrumentationJson(Json::Value& root, + const std::string& subdir, + const std::string& file_name) +{ + Json::StreamWriterBuilder wbuilder; + wbuilder["indentation"] = "\t"; + std::unique_ptr JsonWriter = + std::unique_ptr(wbuilder.newStreamWriter()); + const std::string& directory = cmStrCat(this->timingDirv1, "/", subdir); + cmSystemTools::MakeDirectory(directory); + cmsys::ofstream ftmp(cmStrCat(directory, "/", file_name).c_str()); + JsonWriter->write(root, &ftmp); + ftmp << "\n"; + ftmp.close(); +} + +int cmInstrumentation::InstrumentTest( + const std::string& name, const std::string& command, + const std::vector& args, int64_t result, + std::chrono::steady_clock::time_point steadyStart, + std::chrono::system_clock::time_point systemStart) +{ + // Store command info + Json::Value root(this->preTestStats); + std::string command_str = cmStrCat(command, ' ', GetCommandStr(args)); + root["version"] = 1; + root["command"] = command_str; + root["role"] = "test"; + root["testName"] = name; + root["binaryDir"] = this->binaryDir; + root["result"] = static_cast(result); + + // Post-Command + this->InsertTimingData(root, steadyStart, systemStart); + if (this->HasQuery( + cmInstrumentationQuery::Query::DynamicSystemInformation)) { + this->InsertDynamicSystemInformation(root, "after"); + } + + std::string const& file_name = + cmStrCat("test-", this->ComputeSuffixHash(command_str), + this->ComputeSuffixTime(), ".json"); + this->WriteInstrumentationJson(root, "data", file_name); + return 1; +} + +void cmInstrumentation::GetPreTestStats() +{ + if (this->HasQuery( + cmInstrumentationQuery::Query::DynamicSystemInformation)) { + this->InsertDynamicSystemInformation(this->preTestStats, "before"); + } +} + +int cmInstrumentation::InstrumentCommand( + std::string command_type, const std::vector& command, + const std::function& callback, + cm::optional> options, + cm::optional> arrayOptions, + bool reloadQueriesAfterCommand) +{ + + // Always begin gathering data for configure in case cmake_instrumentation + // command creates a query + if (!this->hasQuery && !reloadQueriesAfterCommand) { + return callback(); + } + + // Store command info + Json::Value root(Json::objectValue); + Json::Value commandInfo(Json::objectValue); + std::string command_str = GetCommandStr(command); + + root["command"] = command_str; + root["version"] = 1; + + // Pre-Command + auto steady_start = std::chrono::steady_clock::now(); + auto system_start = std::chrono::system_clock::now(); + double preConfigureMemory = 0; + double preConfigureLoad = 0; + if (this->HasQuery( + cmInstrumentationQuery::Query::DynamicSystemInformation)) { + this->InsertDynamicSystemInformation(root, "before"); + } else if (reloadQueriesAfterCommand) { + this->GetDynamicSystemInformation(preConfigureMemory, preConfigureLoad); + } + + // Execute Command + int ret = callback(); + root["result"] = ret; + + // Exit early if configure didn't generate a query + if (reloadQueriesAfterCommand) { + this->LoadQueries(); + if (!this->hasQuery) { + return ret; + } + if (this->HasQuery( + cmInstrumentationQuery::Query::DynamicSystemInformation)) { + root["dynamicSystemInformation"] = Json::objectValue; + root["dynamicSystemInformation"]["beforeHostMemoryUsed"] = + preConfigureMemory; + root["dynamicSystemInformation"]["beforeCPULoadAverage"] = + preConfigureLoad; + } + } + + // Post-Command + this->InsertTimingData(root, steady_start, system_start); + if (this->HasQuery( + cmInstrumentationQuery::Query::DynamicSystemInformation)) { + this->InsertDynamicSystemInformation(root, "after"); + } + + // Gather additional data + if (options.has_value()) { + for (auto const& item : options.value()) { + if (item.first == "role" && !item.second.empty()) { + command_type = item.second; + } else if (!item.second.empty()) { + root[item.first] = item.second; + } + } + } + if (arrayOptions.has_value()) { + for (auto const& item : arrayOptions.value()) { + root[item.first] = Json::arrayValue; + std::stringstream ss(item.second); + std::string element; + while (getline(ss, element, ',')) { + root[item.first].append(element); + } + if (item.first == "outputs") { + root["outputSizes"] = Json::arrayValue; + for (auto const& output : root["outputs"]) { + root["outputSizes"].append( + static_cast(cmSystemTools::FileLength( + cmStrCat(this->binaryDir, "/", output.asCString())))); + } + } + } + } + root["role"] = command_type; + root["binaryDir"] = this->binaryDir; + + // Write Json + std::string const& file_name = + cmStrCat(command_type, "-", this->ComputeSuffixHash(command_str), + this->ComputeSuffixTime(), ".json"); + this->WriteInstrumentationJson(root, "data", file_name); + return ret; +} + +std::string cmInstrumentation::GetCommandStr( + const std::vector& args) +{ + std::string command_str; + for (size_t i = 0; i < args.size(); ++i) { + command_str = cmStrCat(command_str, args[i]); + if (i < args.size() - 1) { + command_str = cmStrCat(command_str, " "); + } + } + return command_str; +} + +std::string cmInstrumentation::ComputeSuffixHash( + std::string const& command_str) +{ + cmCryptoHash hasher(cmCryptoHash::AlgoSHA3_256); + std::string hash = hasher.HashString(command_str); + hash.resize(20, '0'); + return hash; +} + +std::string cmInstrumentation::ComputeSuffixTime() +{ + std::chrono::milliseconds ms = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()); + std::chrono::seconds s = + std::chrono::duration_cast(ms); + + std::time_t ts = s.count(); + std::size_t tms = ms.count() % 1000; + + cmTimestamp cmts; + std::ostringstream ss; + ss << cmts.CreateTimestampFromTimeT(ts, "%Y-%m-%dT%H-%M-%S", true) << '-' + << std::setfill('0') << std::setw(4) << tms; + return ss.str(); +} diff --git a/Source/cmInstrumentation.h b/Source/cmInstrumentation.h new file mode 100644 index 0000000000..cefd9e1fda --- /dev/null +++ b/Source/cmInstrumentation.h @@ -0,0 +1,78 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying + file Copyright.txt or https://cmake.org/licensing for details. */ +#pragma once + +#include "cmConfigure.h" // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "cmInstrumentationQuery.h" + +class cmInstrumentation +{ +public: + // Read Queries + cmInstrumentation(const std::string& binary_dir, + bool clear_generated = false); + // Create Query + cmInstrumentation(const std::string& binary_dir, + std::set& queries, + std::set& hooks, + std::string& callback); + int InstrumentCommand( + std::string command_type, const std::vector& command, + const std::function& callback, + cm::optional> options = cm::nullopt, + cm::optional> arrayOptions = + cm::nullopt, + bool reloadQueriesAfterCommand = false); + int InstrumentTest(const std::string& name, const std::string& command, + const std::vector& args, int64_t result, + std::chrono::steady_clock::time_point steadyStart, + std::chrono::system_clock::time_point systemStart); + void GetPreTestStats(); + void LoadQueries(); + bool HasQuery(); + bool HasQuery(cmInstrumentationQuery::Query); + bool ReadJSONQueries(const std::string& directory); + void ReadJSONQuery(const std::string& file); + void WriteJSONQuery(); + int CollectTimingData(cmInstrumentationQuery::Hook hook); + std::string errorMsg; + +private: + void WriteInstrumentationJson(Json::Value& index, + const std::string& directory, + const std::string& file_name); + static void InsertStaticSystemInformation(Json::Value& index); + static void GetDynamicSystemInformation(double& memory, double& load); + static void InsertDynamicSystemInformation(Json::Value& index, + const std::string& instant); + static void InsertTimingData( + Json::Value& root, std::chrono::steady_clock::time_point steadyStart, + std::chrono::system_clock::time_point systemStart); + void ClearGeneratedQueries(); + bool HasQueryFile(const std::string& file); + static std::string GetCommandStr(const std::vector& args); + static std::string ComputeSuffixHash(std::string const& command_str); + static std::string ComputeSuffixTime(); + std::string binaryDir; + std::string timingDirv1; + std::string userTimingDirv1; + std::set queries; + std::set hooks; + std::vector callbacks; + std::vector queryFiles; + Json::Value preTestStats; + bool hasQuery = false; +}; diff --git a/Source/cmInstrumentationCommand.cxx b/Source/cmInstrumentationCommand.cxx new file mode 100644 index 0000000000..fdf56c40dd --- /dev/null +++ b/Source/cmInstrumentationCommand.cxx @@ -0,0 +1,149 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying +file Copyright.txt or https://cmake.org/licensing for details. */ +#include "cmInstrumentationCommand.h" + +#include +#include +#include +#include +#include + +#include + +#include "cmArgumentParser.h" +#include "cmArgumentParserTypes.h" +#include "cmExecutionStatus.h" +#include "cmExperimental.h" +#include "cmInstrumentation.h" +#include "cmInstrumentationQuery.h" +#include "cmMakefile.h" +#include "cmStringAlgorithms.h" + +namespace { + +bool isCharDigit(char ch) +{ + return std::isdigit(static_cast(ch)); +} +bool validateVersion(const std::string& key, const std::string& versionString, + int& version, cmExecutionStatus& status) +{ + if (!std::all_of(versionString.begin(), versionString.end(), isCharDigit)) { + status.SetError(cmStrCat("given a non-integer ", key, ".")); + return false; + } + version = std::atoi(versionString.c_str()); + if (version != 1) { + status.SetError(cmStrCat( + "QUERY subcommand given an unsupported ", key, " \"", versionString, + "\" (the only currently supported version is 1).")); + return false; + } + return true; +} + +template +std::function EnumParser( + const std::vector toString) +{ + return [toString](const std::string& value, E& out) -> bool { + for (size_t i = 0; i < toString.size(); ++i) { + if (value == toString[i]) { + out = (E)i; + return true; + } + } + return false; + }; +} +} + +bool cmInstrumentationCommand(std::vector const& args, + cmExecutionStatus& status) +{ + // if (status->GetMakefile().GetPropertyKeys) { + if (!cmExperimental::HasSupportEnabled( + status.GetMakefile(), cmExperimental::Feature::Instrumentation)) { + status.SetError( + "requires the experimental Instrumentation flag to be enabled"); + return false; + } + + if (args.empty()) { + status.SetError("must be called with arguments."); + return false; + } + + struct Arguments : public ArgumentParser::ParseResult + { + ArgumentParser::NonEmpty ApiVersion; + ArgumentParser::NonEmpty DataVersion; + ArgumentParser::NonEmpty> Queries; + ArgumentParser::NonEmpty> Hooks; + ArgumentParser::NonEmpty> Callback; + }; + + static auto const parser = cmArgumentParser{} + .Bind("API_VERSION"_s, &Arguments::ApiVersion) + .Bind("DATA_VERSION"_s, &Arguments::DataVersion) + .Bind("QUERIES"_s, &Arguments::Queries) + .Bind("HOOKS"_s, &Arguments::Hooks) + .Bind("CALLBACK"_s, &Arguments::Callback); + + std::vector unparsedArguments; + Arguments const arguments = parser.Parse(args, &unparsedArguments); + + if (arguments.MaybeReportError(status.GetMakefile())) { + return true; + } + if (!unparsedArguments.empty()) { + status.SetError("given unknown argument \"" + unparsedArguments.front() + + "\"."); + return false; + } + int apiVersion; + int dataVersion; + if (!validateVersion("API_VERSION", arguments.ApiVersion, apiVersion, + status) || + !validateVersion("DATA_VERSION", arguments.DataVersion, dataVersion, + status)) { + return false; + } + + std::set queries; + auto queryParser = EnumParser( + cmInstrumentationQuery::QueryString); + for (auto const& arg : arguments.Queries) { + cmInstrumentationQuery::Query query; + if (!queryParser(arg, query)) { + status.SetError( + cmStrCat("given invalid argument to QUERIES \"", arg, "\"")); + return false; + } + queries.insert(query); + } + + std::set hooks; + auto hookParser = EnumParser( + cmInstrumentationQuery::HookString); + for (auto const& arg : arguments.Hooks) { + cmInstrumentationQuery::Hook hook; + if (!hookParser(arg, hook)) { + status.SetError( + cmStrCat("given invalid argument to HOOKS \"", arg, "\"")); + return false; + } + hooks.insert(hook); + } + + std::string callback; + for (auto const& arg : arguments.Callback) { + callback = cmStrCat(callback, arg); + } + + auto instrument = cmInstrumentation( + status.GetMakefile().GetHomeOutputDirectory(), queries, hooks, callback); + instrument.WriteJSONQuery(); + + return true; +} diff --git a/Source/cmInstrumentationCommand.h b/Source/cmInstrumentationCommand.h new file mode 100644 index 0000000000..0a9c10deb1 --- /dev/null +++ b/Source/cmInstrumentationCommand.h @@ -0,0 +1,13 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying +file Copyright.txt or https://cmake.org/licensing for details. */ +#pragma once + +#include "cmConfigure.h" // IWYU pragma: keep + +#include +#include + +class cmExecutionStatus; + +bool cmInstrumentationCommand(std::vector const& args, + cmExecutionStatus& status); diff --git a/Source/cmInstrumentationQuery.cxx b/Source/cmInstrumentationQuery.cxx new file mode 100644 index 0000000000..355751936e --- /dev/null +++ b/Source/cmInstrumentationQuery.cxx @@ -0,0 +1,114 @@ +#include "cmInstrumentationQuery.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "cmJSONHelpers.h" +#include "cmStringAlgorithms.h" + +const std::vector cmInstrumentationQuery::QueryString{ + "staticSystemInformation", "dynamicSystemInformation" +}; +const std::vector cmInstrumentationQuery::HookString{ + "postGenerate", "preBuild", "postBuild", "preCMakeBuild", + "postCMakeBuild", "postTest", "postInstall", "manual" +}; + +namespace ErrorMessages { +using ErrorGenerator = + std::function; +ErrorGenerator ErrorGeneratorBuilder(const std::string& errorMessage) +{ + return [errorMessage](const Json::Value* value, cmJSONState* state) -> void { + state->AddErrorAtValue(errorMessage, value); + }; +}; + +static ErrorGenerator InvalidArray = ErrorGeneratorBuilder("Invalid Array"); +JsonErrors::ErrorGenerator InvalidRootQueryObject( + JsonErrors::ObjectError errorType, const Json::Value::Members& extraFields) +{ + return JsonErrors::INVALID_NAMED_OBJECT( + [](const Json::Value*, cmJSONState*) -> std::string { + return "root object"; + })(errorType, extraFields); +} +}; + +using JSONHelperBuilder = cmJSONHelperBuilder; + +template +static std::function EnumHelper( + const std::vector toString, const std::string& type) +{ + return [toString, type](E& out, const Json::Value* value, + cmJSONState* state) -> bool { + for (size_t i = 0; i < toString.size(); ++i) { + if (value->asString() == toString[i]) { + out = (E)i; + return true; + } + } + state->AddErrorAtValue( + cmStrCat("Not a valid ", type, ": \"", value->asString(), "\""), value); + return false; + }; +} +static auto const QueryHelper = EnumHelper( + cmInstrumentationQuery::QueryString, "query"); +static auto const QueryListHelper = + JSONHelperBuilder::Vector( + ErrorMessages::InvalidArray, QueryHelper); +static auto const HookHelper = EnumHelper( + cmInstrumentationQuery::HookString, "hook"); +static auto const HookListHelper = + JSONHelperBuilder::Vector( + ErrorMessages::InvalidArray, HookHelper); +static auto const CallbackHelper = JSONHelperBuilder::String(); +static auto const CallbackListHelper = JSONHelperBuilder::Vector( + ErrorMessages::InvalidArray, CallbackHelper); +static auto const VersionHelper = JSONHelperBuilder::Int(); + +using QueryRoot = cmInstrumentationQuery::QueryJSONRoot; + +static auto const QueryRootHelper = + JSONHelperBuilder::Object(ErrorMessages::InvalidRootQueryObject, + false) + .Bind("version"_s, &QueryRoot::version, VersionHelper, true) + .Bind("queries"_s, &QueryRoot::queries, QueryListHelper, false) + .Bind("hooks"_s, &QueryRoot::hooks, HookListHelper, false) + .Bind("callbacks"_s, &QueryRoot::callbacks, CallbackListHelper, false); + +bool cmInstrumentationQuery::ReadJSON(const std::string& filename, + std::string& errorMessage, + std::set& queries, + std::set& hooks, + std::vector& callbacks) +{ + Json::Value root; + this->parseState = cmJSONState(filename, &root); + if (!this->parseState.errors.empty()) { + std::cerr << this->parseState.GetErrorMessage(true) << std::endl; + return false; + } + if (!QueryRootHelper(this->queryRoot, &root, &this->parseState)) { + errorMessage = this->parseState.GetErrorMessage(true); + return false; + } + std::move(this->queryRoot.queries.begin(), this->queryRoot.queries.end(), + std::inserter(queries, queries.end())); + std::move(this->queryRoot.hooks.begin(), this->queryRoot.hooks.end(), + std::inserter(hooks, hooks.end())); + std::move(this->queryRoot.callbacks.begin(), this->queryRoot.callbacks.end(), + std::back_inserter(callbacks)); + return true; +} diff --git a/Source/cmInstrumentationQuery.h b/Source/cmInstrumentationQuery.h new file mode 100644 index 0000000000..bfbd73a11b --- /dev/null +++ b/Source/cmInstrumentationQuery.h @@ -0,0 +1,49 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying + file Copyright.txt or https://cmake.org/licensing for details. */ +#pragma once + +#include +#include +#include + +#include "cmJSONState.h" + +class cmInstrumentationQuery +{ + +public: + enum Query + { + StaticSystemInformation, + DynamicSystemInformation + }; + static const std::vector QueryString; + + enum Hook + { + PostGenerate, + PreBuild, + PostBuild, + PreCMakeBuild, + PostCMakeBuild, + PostTest, + PostInstall, + Manual + }; + static const std::vector HookString; + + struct QueryJSONRoot + { + std::vector queries; + std::vector hooks; + std::vector callbacks; + int version; + }; + + cmInstrumentationQuery() = default; + bool ReadJSON(const std::string& file, std::string& errorMessage, + std::set& queries, std::set& hooks, + std::vector& callbacks); + QueryJSONRoot queryRoot; + cmJSONState parseState; +}; diff --git a/Source/cmLocalNinjaGenerator.cxx b/Source/cmLocalNinjaGenerator.cxx index 6217d5dc60..923bbcb59b 100644 --- a/Source/cmLocalNinjaGenerator.cxx +++ b/Source/cmLocalNinjaGenerator.cxx @@ -912,14 +912,19 @@ std::string cmLocalNinjaGenerator::MakeCustomLauncher( std::string output; const std::vector& outputs = ccg.GetOutputs(); - if (!outputs.empty()) { - output = outputs[0]; - if (ccg.GetWorkingDirectory().empty()) { - output = this->MaybeRelativeToCurBinDir(output); + for (size_t i = 0; i < outputs.size(); ++i) { + output = cmStrCat(output, + this->ConvertToOutputFormat( + ccg.GetWorkingDirectory().empty() + ? this->MaybeRelativeToCurBinDir(outputs[i]) + : outputs[i], + cmOutputConverter::SHELL)); + if (i != outputs.size() - 1) { + output = cmStrCat(output, ","); } - output = this->ConvertToOutputFormat(output, cmOutputConverter::SHELL); } vars.Output = output.c_str(); + vars.Role = ccg.GetCC().GetRole().c_str(); auto rulePlaceholderExpander = this->CreateRulePlaceholderExpander(); diff --git a/Source/cmLocalUnixMakefileGenerator3.cxx b/Source/cmLocalUnixMakefileGenerator3.cxx index afd2af9cca..7c0731926d 100644 --- a/Source/cmLocalUnixMakefileGenerator3.cxx +++ b/Source/cmLocalUnixMakefileGenerator3.cxx @@ -996,15 +996,19 @@ void cmLocalUnixMakefileGenerator3::AppendCustomCommand( cmState::GetTargetTypeName(target->GetType()).c_str(); std::string output; const std::vector& outputs = ccg.GetOutputs(); - if (!outputs.empty()) { - output = outputs[0]; - if (workingDir.empty()) { - output = this->MaybeRelativeToCurBinDir(output); + for (size_t i = 0; i < outputs.size(); ++i) { + output = cmStrCat(output, + this->ConvertToOutputFormat( + ccg.GetWorkingDirectory().empty() + ? this->MaybeRelativeToCurBinDir(outputs[i]) + : outputs[i], + cmOutputConverter::SHELL)); + if (i != outputs.size() - 1) { + output = cmStrCat(output, ","); } - output = - this->ConvertToOutputFormat(output, cmOutputConverter::SHELL); } vars.Output = output.c_str(); + vars.Role = ccg.GetCC().GetRole().c_str(); launcher = val; rulePlaceholderExpander->ExpandRuleVariables(this, launcher, vars); diff --git a/Source/cmRulePlaceholderExpander.cxx b/Source/cmRulePlaceholderExpander.cxx index a2435a383f..ce41c8ae5b 100644 --- a/Source/cmRulePlaceholderExpander.cxx +++ b/Source/cmRulePlaceholderExpander.cxx @@ -270,6 +270,12 @@ std::string cmRulePlaceholderExpander::ExpandVariable( return this->OutputConverter->ConvertToOutputFormat( cmSystemTools::GetCMakeCommand(), cmOutputConverter::SHELL); } + if (variable == "ROLE") { + if (this->ReplaceValues->Role) { + return this->ReplaceValues->Role; + } + return ""; + } auto compIt = this->Compilers.find(variable); diff --git a/Source/cmRulePlaceholderExpander.h b/Source/cmRulePlaceholderExpander.h index 4858b0838c..d2a88e0b47 100644 --- a/Source/cmRulePlaceholderExpander.h +++ b/Source/cmRulePlaceholderExpander.h @@ -73,6 +73,7 @@ public: const char* Fatbinary = nullptr; const char* RegisterFile = nullptr; const char* Launcher = nullptr; + const char* Role = nullptr; }; // Expand rule variables in CMake of the type found in language rules diff --git a/Source/cmake.cxx b/Source/cmake.cxx index 5b4d58f72c..7634021f27 100644 --- a/Source/cmake.cxx +++ b/Source/cmake.cxx @@ -81,6 +81,8 @@ # include "cmConfigureLog.h" # include "cmFileAPI.h" # include "cmGraphVizWriter.h" +# include "cmInstrumentation.h" +# include "cmInstrumentationQuery.h" # include "cmVariableWatch.h" #endif @@ -929,6 +931,7 @@ enum class ListPresets // Parse the args void cmake::SetArgs(const std::vector& args) { + this->cmdArgs = args; bool haveToolset = false; bool havePlatform = false; bool haveBArg = false; @@ -2604,9 +2607,28 @@ int cmake::ActualConfigure() // actually do the configure auto startTime = std::chrono::steady_clock::now(); +#if !defined(CMAKE_BOOTSTRAP) + cmInstrumentation instrumentation(this->State->GetBinaryDirectory(), true); + if (!instrumentation.errorMsg.empty()) { + cmSystemTools::Error(instrumentation.errorMsg); + return 1; + } + std::function doConfigure = [this]() -> int { + this->GlobalGenerator->Configure(); + return 0; + }; + int ret = instrumentation.InstrumentCommand( + "configure", this->cmdArgs, [doConfigure]() { return doConfigure(); }, + cm::nullopt, cm::nullopt, true); + if (ret != 0) { + return ret; + } +#else this->GlobalGenerator->Configure(); +#endif auto endTime = std::chrono::steady_clock::now(); + // configure result if (this->GetWorkingMode() == cmake::NORMAL_MODE) { std::ostringstream msg; if (cmSystemTools::GetErrorOccurredFlag()) { @@ -2650,6 +2672,7 @@ int cmake::ActualConfigure() } const auto& mf = this->GlobalGenerator->GetMakefiles()[0]; + if (mf->IsOn("CTEST_USE_LAUNCHERS") && !this->State->GetGlobalProperty("RULE_LAUNCH_COMPILE")) { cmSystemTools::Error( @@ -2658,6 +2681,37 @@ int cmake::ActualConfigure() "Did you forget to include(CTest) in the toplevel " "CMakeLists.txt ?"); } + // Setup launchers for instrumentation +#if !defined(CMAKE_BOOTSTRAP) + instrumentation.LoadQueries(); + if (instrumentation.HasQuery()) { + std::string launcher; + if (mf->IsOn("CTEST_USE_LAUNCHERS")) { + launcher = + cmStrCat("\"", cmSystemTools::GetCTestCommand(), "\" --launch "); + } else { + launcher = + cmStrCat("\"", cmSystemTools::GetCTestCommand(), "\" --instrument "); + } + std::string common_args = + cmStrCat(" --target-name ", "--build-dir \"", + this->State->GetBinaryDirectory(), "\" "); + this->State->SetGlobalProperty( + "RULE_LAUNCH_COMPILE", + cmStrCat( + launcher, "--command-type compile", common_args, + "--output --source --language -- ")); + this->State->SetGlobalProperty( + "RULE_LAUNCH_LINK", + cmStrCat(launcher, "--command-type link", common_args, + "--output --target-type ", + "--language -- ")); + this->State->SetGlobalProperty( + "RULE_LAUNCH_CUSTOM", + cmStrCat(launcher, "--command-type custom", common_args, + "--output \"\" --role -- ")); + } +#endif this->State->SaveVerificationScript(this->GetHomeOutputDirectory(), this->Messenger.get()); @@ -2945,15 +2999,29 @@ int cmake::Generate() return -1; } + auto startTime = std::chrono::steady_clock::now(); #if !defined(CMAKE_BOOTSTRAP) auto profilingRAII = this->CreateProfilingEntry("project", "generate"); -#endif + cmInstrumentation instrumentation(this->State->GetBinaryDirectory()); + std::function doGenerate = [this]() -> int { + if (!this->GlobalGenerator->Compute()) { + return -1; + } + this->GlobalGenerator->Generate(); + return 0; + }; - auto startTime = std::chrono::steady_clock::now(); + int ret = instrumentation.InstrumentCommand( + "generate", this->cmdArgs, [doGenerate]() { return doGenerate(); }); + if (ret != 0) { + return ret; + } +#else if (!this->GlobalGenerator->Compute()) { return -1; } this->GlobalGenerator->Generate(); +#endif auto endTime = std::chrono::steady_clock::now(); { auto ms = std::chrono::duration_cast(endTime - @@ -2963,6 +3031,10 @@ int cmake::Generate() << ms.count() / 1000.0L << "s)"; this->UpdateProgress(msg.str(), -1); } +#if !defined(CMAKE_BOOTSTRAP) + instrumentation.CollectTimingData( + cmInstrumentationQuery::Hook::PostGenerate); +#endif if (!this->GraphVizFile.empty()) { std::cout << "Generate graphviz: " << this->GraphVizFile << '\n'; this->GenerateGraphViz(this->GraphVizFile); diff --git a/Source/cmake.h b/Source/cmake.h index d79629a03c..572adc834b 100644 --- a/Source/cmake.h +++ b/Source/cmake.h @@ -754,6 +754,7 @@ protected: void GenerateGraphViz(const std::string& fileName) const; private: + std::vector cmdArgs; std::string CMakeWorkingDirectory; ProgressCallbackType ProgressCallback; WorkingMode CurrentWorkingMode = NORMAL_MODE; diff --git a/Source/cmakemain.cxx b/Source/cmakemain.cxx index d8072374eb..cc63ba02e6 100644 --- a/Source/cmakemain.cxx +++ b/Source/cmakemain.cxx @@ -28,6 +28,8 @@ #include "cmDocumentationEntry.h" #include "cmGlobalGenerator.h" #include "cmInstallScriptHandler.h" +#include "cmInstrumentation.h" +#include "cmInstrumentationQuery.h" #include "cmList.h" #include "cmMakefile.h" #include "cmMessageMetadata.h" @@ -703,11 +705,27 @@ int do_build(int ac, char const* const* av) cmakemainProgressCallback(msg, prog, &cm); }); + cmInstrumentation instrumentation(dir); + if (!instrumentation.errorMsg.empty()) { + cmSystemTools::Error(instrumentation.errorMsg); + return 1; + } cmBuildOptions buildOptions(cleanFirst, false, resolveMode); - - return cm.Build(jobs, std::move(dir), std::move(targets), std::move(config), - std::move(nativeOptions), buildOptions, verbose, presetName, - listPresets); + std::function doBuild = [&cm, &jobs, &dir, &targets, &config, + &nativeOptions, &buildOptions, &verbose, + &presetName, &listPresets]() { + return cm.Build(jobs, dir, std::move(targets), std::move(config), + std::move(nativeOptions), buildOptions, verbose, + presetName, listPresets); + }; + instrumentation.CollectTimingData( + cmInstrumentationQuery::Hook::PreCMakeBuild); + std::vector cmd; + cm::append(cmd, av, av + ac); + int ret = instrumentation.InstrumentCommand("cmakeBuild", cmd, doBuild); + instrumentation.CollectTimingData( + cmInstrumentationQuery::Hook::PostCMakeBuild); + return ret; #endif } @@ -952,6 +970,7 @@ int do_install(int ac, char const* const* av) args.emplace_back("-P"); + cmInstrumentation instrumentation(dir); auto handler = cmInstallScriptHandler(dir, component, config, args); int ret = 0; if (!jobs && handler.IsParallel()) { @@ -966,27 +985,38 @@ int do_install(int ac, char const* const* av) } } } - if (handler.IsParallel()) { - ret = handler.Install(jobs); - } else { - for (auto const& cmd : handler.GetCommands()) { - cmake cm(cmake::RoleScript, cmState::Script); - cmSystemTools::SetMessageCallback( - [&cm](const std::string& msg, const cmMessageMetadata& md) { - cmakemainMessageCallback(msg, md, &cm); - }); - cm.SetProgressCallback([&cm](const std::string& msg, float prog) { - cmakemainProgressCallback(msg, prog, &cm); - }); - cm.SetHomeDirectory(""); - cm.SetHomeOutputDirectory(""); - cm.SetDebugOutputOn(verbose); - cm.SetWorkingMode(cmake::SCRIPT_MODE); - ret = int(bool(cm.Run(cmd))); - } - } - return int(ret > 0); + std::function doInstall = [&handler, &verbose, &jobs, + &instrumentation]() -> int { + int ret_ = 0; + if (handler.IsParallel()) { + ret_ = handler.Install(jobs, instrumentation); + } else { + for (auto const& cmd : handler.GetCommands()) { + cmake cm(cmake::RoleScript, cmState::Script); + cmSystemTools::SetMessageCallback( + [&cm](const std::string& msg, const cmMessageMetadata& md) { + cmakemainMessageCallback(msg, md, &cm); + }); + cm.SetProgressCallback([&cm](const std::string& msg, float prog) { + cmakemainProgressCallback(msg, prog, &cm); + }); + cm.SetHomeDirectory(""); + cm.SetHomeOutputDirectory(""); + cm.SetDebugOutputOn(verbose); + cm.SetWorkingMode(cmake::SCRIPT_MODE); + ret_ = int(bool(cm.Run(cmd))); + } + } + return int(ret_ > 0); + }; + + std::vector cmd; + cm::append(cmd, av, av + ac); + ret = instrumentation.InstrumentCommand( + "cmakeInstall", cmd, [doInstall]() { return doInstall(); }); + instrumentation.CollectTimingData(cmInstrumentationQuery::Hook::PostInstall); + return ret; #endif } diff --git a/Source/ctest.cxx b/Source/ctest.cxx index 388e96b253..c20602a301 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -12,6 +12,8 @@ #include "cmConsoleBuf.h" #include "cmDocumentation.h" #include "cmDocumentationEntry.h" +#include "cmInstrumentation.h" +#include "cmInstrumentationQuery.h" #include "cmSystemTools.h" #include "CTest/cmCTestLaunch.h" @@ -179,7 +181,18 @@ int main(int argc, char const* const* argv) // Dispatch 'ctest --launch' mode directly. if (argc >= 2 && strcmp(argv[1], "--launch") == 0) { - return cmCTestLaunch::Main(argc, argv); + return cmCTestLaunch::Main(argc, argv, cmCTestLaunch::Op::Normal); + } + + // Dispatch 'ctest --instrument' mode directly. + if (argc >= 2 && strcmp(argv[1], "--instrument") == 0) { + return cmCTestLaunch::Main(argc, argv, cmCTestLaunch::Op::Instrument); + } + + // Dispatch 'ctest --collect-instrumentation' mode directly. + if (argc == 3 && strcmp(argv[1], "--collect-instrumentation") == 0) { + return cmInstrumentation(argv[2]).CollectTimingData( + cmInstrumentationQuery::Hook::Manual); } if (cmSystemTools::GetLogicalWorkingDirectory().empty()) { diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt index 92d258de1b..c4564b1ac9 100644 --- a/Tests/RunCMake/CMakeLists.txt +++ b/Tests/RunCMake/CMakeLists.txt @@ -413,6 +413,9 @@ if(CMAKE_USE_SYSTEM_JSONCPP) endif() add_RunCMake_test(FileAPI -DPython_EXECUTABLE=${Python_EXECUTABLE} -DCMAKE_CXX_COMPILER_ID=${CMAKE_CXX_COMPILER_ID}) +if("${CMAKE_GENERATOR}" MATCHES "Unix Makefiles|Ninja") + add_RunCMake_test(Instrumentation) +endif() add_RunCMake_test(ConfigDir) add_RunCMake_test(FindBoost) add_RunCMake_test(FindLua) diff --git a/Tests/RunCMake/ConfigDir/check-reply.cmake b/Tests/RunCMake/ConfigDir/check-reply.cmake index 6e0ecf9ddf..eb534c01a2 100644 --- a/Tests/RunCMake/ConfigDir/check-reply.cmake +++ b/Tests/RunCMake/ConfigDir/check-reply.cmake @@ -1,3 +1,6 @@ if (NOT EXISTS ${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/reply) set(RunCMake_TEST_FAILED "Failed to read FileAPI query from user config directory") endif() +if (NOT EXISTS ${RunCMake_TEST_BINARY_DIR}/.cmake/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0/v1/data) + set(RunCMake_TEST_FAILED "Failed to read Instrumentation query from user config directory") +endif() diff --git a/Tests/RunCMake/ConfigDir/config/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0/v1/query/query.json b/Tests/RunCMake/ConfigDir/config/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0/v1/query/query.json new file mode 100644 index 0000000000..61a2092b1b --- /dev/null +++ b/Tests/RunCMake/ConfigDir/config/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0/v1/query/query.json @@ -0,0 +1,3 @@ +{ + "version": 1 +} diff --git a/Tests/RunCMake/Instrumentation/CMakeLists.txt b/Tests/RunCMake/Instrumentation/CMakeLists.txt new file mode 100644 index 0000000000..dda37d8bc5 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/CMakeLists.txt @@ -0,0 +1,3 @@ +cmake_minimum_required(VERSION 3.30) +project(${RunCMake_TEST} NONE) +include(${RunCMake_TEST}.cmake) diff --git a/Tests/RunCMake/Instrumentation/RunCMakeTest.cmake b/Tests/RunCMake/Instrumentation/RunCMakeTest.cmake new file mode 100644 index 0000000000..f7dc4eb36d --- /dev/null +++ b/Tests/RunCMake/Instrumentation/RunCMakeTest.cmake @@ -0,0 +1,114 @@ +cmake_minimum_required(VERSION 3.30) +include(RunCMake) + +function(instrument test) + # Set Paths Variables + set(config "${CMAKE_CURRENT_LIST_DIR}/config") + set(ENV{CMAKE_CONFIG_DIR} ${config}) + cmake_parse_arguments(ARGS + "BUILD;INSTALL;TEST;COPY_QUERIES;NO_WARN;STATIC_QUERY;DYNAMIC_QUERY;INSTALL_PARALLEL;MANUAL_HOOK" + "CHECK_SCRIPT;CONFIGURE_ARG" "" ${ARGN}) + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/${test}) + set(uuid "a37d1069-1972-4901-b9c9-f194aaf2b6e0") + set(v1 ${RunCMake_TEST_BINARY_DIR}/.cmake/instrumentation-${uuid}/v1) + set(query_dir ${CMAKE_CURRENT_LIST_DIR}/query) + + # Clear previous instrumentation data + # We can't use RunCMake_TEST_NO_CLEAN 0 because we preserve queries placed in the build tree after + file(REMOVE_RECURSE ${RunCMake_TEST_BINARY_DIR}) + + # Set hook command + set(static_query_hook_arg 0) + if (ARGS_STATIC_QUERY) + set(static_query_hook_arg 1) + endif() + set(GET_HOOK "\\\"${CMAKE_COMMAND}\\\" -P \\\"${RunCMake_SOURCE_DIR}/hook.cmake\\\" ${static_query_hook_arg}") + + # Load query JSON and cmake (with cmake_instrumentation(...)) files + set(query ${query_dir}/${test}.json.in) + set(cmake_file ${query_dir}/${test}.cmake) + if (EXISTS ${query}) + file(MAKE_DIRECTORY ${v1}/query) + configure_file(${query} ${v1}/query/${test}.json) + elseif (EXISTS ${cmake_file}) + list(APPEND ARGS_CONFIGURE_ARG "-DINSTRUMENT_COMMAND_FILE=${cmake_file}") + endif() + + # Configure generated query files to compare CMake output + if (ARGS_COPY_QUERIES) + file(MAKE_DIRECTORY ${RunCMake_TEST_BINARY_DIR}/query) + set(generated_queries "0;1;2") + foreach(n ${generated_queries}) + configure_file( + "${query_dir}/generated/query-${n}.json.in" + "${RunCMake_TEST_BINARY_DIR}/query/query-${n}.json" + ) + endforeach() + endif() + + # Configure Test Case + set(RunCMake_TEST_NO_CLEAN 1) + if (ARGS_NO_WARN) + list(APPEND ARGS_CONFIGURE_ARG "-Wno-dev") + endif() + set(RunCMake_TEST_SOURCE_DIR ${RunCMake_SOURCE_DIR}/project) + run_cmake_with_options(${test} ${ARGS_CONFIGURE_ARG}) + + # Follow-up Commands + if (ARGS_BUILD) + run_cmake_command(${test}-build ${CMAKE_COMMAND} --build . --config Debug) + endif() + if (ARGS_INSTALL) + run_cmake_command(${test}-install ${CMAKE_COMMAND} --install . --prefix install --config Debug) + endif() + if (ARGS_TEST) + run_cmake_command(${test}-test ${CMAKE_CTEST_COMMAND} . -C Debug) + endif() + if (ARGS_MANUAL_HOOK) + run_cmake_command(${test}-index ${CMAKE_CTEST_COMMAND} --collect-instrumentation .) + endif() + + # Run Post-Test Checks + # Check scripts need to run after ALL run_cmake_command have finished + if (ARGS_CHECK_SCRIPT) + set(RunCMake-check-file ${ARGS_CHECK_SCRIPT}) + set(RunCMake_CHECK_ONLY 1) + run_cmake(${test}-verify) + unset(RunCMake-check-file) + unset(RunCMake_CHECK_ONLY) + endif() +endfunction() + +# Bad Queries +instrument(bad-query) +instrument(bad-hook) +instrument(empty) +instrument(bad-version) + +# Verify Hooks Run and Index File +instrument(hooks-1 BUILD INSTALL TEST STATIC_QUERY) +instrument(hooks-2 BUILD INSTALL TEST) +instrument(hooks-no-callbacks MANUAL_HOOK) + +# Check data file contents +instrument(no-query BUILD INSTALL TEST + CHECK_SCRIPT check-data-dir.cmake) +instrument(dynamic-query BUILD INSTALL TEST DYNAMIC_QUERY + CHECK_SCRIPT check-data-dir.cmake) +instrument(both-query BUILD INSTALL TEST DYNAMIC_QUERY + CHECK_SCRIPT check-data-dir.cmake) + +# cmake_instrumentation command +instrument(cmake-command + COPY_QUERIES NO_WARN DYNAMIC_QUERY + CHECK_SCRIPT check-generated-queries.cmake) +instrument(cmake-command-data + COPY_QUERIES NO_WARN BUILD INSTALL TEST DYNAMIC_QUERY + CHECK_SCRIPT check-data-dir.cmake) +instrument(cmake-command-bad-api-version NO_WARN) +instrument(cmake-command-bad-data-version NO_WARN) +instrument(cmake-command-missing-version NO_WARN) +instrument(cmake-command-bad-arg NO_WARN) +instrument(cmake-command-parallel-install + BUILD INSTALL TEST NO_WARN INSTALL_PARALLEL DYNAMIC_QUERY + CHECK_SCRIPT check-data-dir.cmake) diff --git a/Tests/RunCMake/Instrumentation/bad-hook-result.txt b/Tests/RunCMake/Instrumentation/bad-hook-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/Instrumentation/bad-hook-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/Instrumentation/bad-hook-stderr.txt b/Tests/RunCMake/Instrumentation/bad-hook-stderr.txt new file mode 100644 index 0000000000..651de1dd38 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/bad-hook-stderr.txt @@ -0,0 +1,4 @@ +^CMake Error: + +Error: @3,13: Not a valid hook: "bad hook" + "hooks": \["bad hook", "postGenerate", "preCMakeBuild", "postCMakeBuild", "postInstall"\] + \^$ diff --git a/Tests/RunCMake/Instrumentation/bad-query-result.txt b/Tests/RunCMake/Instrumentation/bad-query-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/Instrumentation/bad-query-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/Instrumentation/bad-query-stderr.txt b/Tests/RunCMake/Instrumentation/bad-query-stderr.txt new file mode 100644 index 0000000000..caa0eecba8 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/bad-query-stderr.txt @@ -0,0 +1,4 @@ +^CMake Error: + +Error: @3,42: Not a valid query: "bad query" + "queries": \["staticSystemInformation", "bad query"\] + \^$ diff --git a/Tests/RunCMake/Instrumentation/check-data-dir.cmake b/Tests/RunCMake/Instrumentation/check-data-dir.cmake new file mode 100644 index 0000000000..5776ce0e28 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/check-data-dir.cmake @@ -0,0 +1,115 @@ +include(${CMAKE_CURRENT_LIST_DIR}/verify-snippet.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/json.cmake) + +file(GLOB snippets ${v1}/data/*) +if (NOT snippets) + add_error("No snippet files generated") +endif() + +set(FOUND_SNIPPETS "") +foreach(snippet ${snippets}) + read_json(${snippet} contents) + + # Verify snippet file is valid + verify_snippet(${snippet} ${contents}) + + # Append to list of collected snippet roles + if (NOT role IN_LIST FOUND_SNIPPETS) + list(APPEND FOUND_SNIPPETS ${role}) + endif() + + # Verify target + string(JSON target ERROR_VARIABLE noTarget GET ${contents} target) + if (NOT target MATCHES NOTFOUND) + set(targets "main;lib;customTarget;TARGET_NAME") + if (NOT ${target} IN_LIST targets) + snippet_error(${snippet} "Unexpected target: ${target}") + endif() + endif() + + # Verify output + string(JSON result GET ${contents} result) + if (NOT ${result} EQUAL 0) + snippet_error(${snippet} "Compile command had non-0 result") + endif() + + # Verify contents of compile-* Snippets + if (snippet MATCHES ^compile-) + string(JSON target GET ${contents} target) + string(JSON source GET ${contents} source) + string(JSON language GET ${contents} language) + if (NOT language MATCHES "C\\+\\+") + snippet_error(${snippet} "Expected C++ compile language") + endif() + if (NOT source MATCHES "${target}.cxx$") + snippet_error(${snippet} "Unexpected source file") + endif() + endif() + + # Verify contents of link-* Snippets + if (snippet MATCHES ^link-) + string(JSON target GET ${contents} target) + string(JSON targetType GET ${contents} targetType) + if (target MATCHES main) + if (NOT targetType MATCHES "EXECUTABLE") + snippet_error(${snippet} "Expected EXECUTABLE, target type was ${targetType}") + endif() + endif() + if (target MATCHES lib) + if (NOT targetType MATCHES "STATIC_LIBRARY") + snippet_error(${snippet} "Expected STATIC_LIBRARY, target type was ${targetType}") + endif() + endif() + endif() + + # Verify contents of custom-* Snippets + if (snippet MATCHES ^custom-) + string(JSON outputs GET ${contents} outputs) + if (NOT output1 MATCHES "output1" OR NOT output2 MATCHES "output2") + snippet_error(${snippet} "Custom command missing outputs") + endif() + endif() + + # Verify contents of test-* Snippets + if (snippet MATCHES ^test-) + string(JSON testName GET ${contents} testName) + if (NOT testName EQUAL "test") + snippet_error(${snippet} "Unexpected testName: ${testName}") + endif() + endif() +endforeach() + +# Verify that listed snippets match expected roles +set(EXPECTED_SNIPPETS configure generate) +if (ARGS_BUILD) + list(APPEND EXPECTED_SNIPPETS compile link custom cmakeBuild) +endif() +if (ARGS_TEST) + list(APPEND EXPECTED_SNIPPETS ctest test) +endif() +if (ARGS_INSTALL) + list(APPEND EXPECTED_SNIPPETS cmakeInstall) + if (ARGS_INSTALL_PARALLEL) + list(APPEND EXPECTED_SNIPPETS install) + endif() +endif() +foreach(role ${EXPECTED_SNIPPETS}) + list(FIND FOUND_SNIPPETS ${role} found) + if (${found} EQUAL -1) + add_error("No snippet files of role \"${role}\" were found in ${v1}") + endif() +endforeach() +foreach(role ${FOUND_SNIPPETS}) + list(FIND EXPECTED_SNIPPETS ${role} found) + if (${found} EQUAL -1) + add_error("Found unexpected snippet file of role \"${role}\" in ${v1}") + endif() +endforeach() + +# Verify test/install artifacts +if (ARGS_INSTALL AND NOT EXISTS ${RunCMake_TEST_BINARY_DIR}/install) + add_error("ctest --instrument launcher failed to install the project") +endif() +if (ARGS_TEST AND NOT EXISTS ${RunCMake_TEST_BINARY_DIR}/Testing) + add_error("ctest --instrument launcher failed to test the project") +endif() diff --git a/Tests/RunCMake/Instrumentation/check-generated-queries.cmake b/Tests/RunCMake/Instrumentation/check-generated-queries.cmake new file mode 100644 index 0000000000..ba3fd6b98a --- /dev/null +++ b/Tests/RunCMake/Instrumentation/check-generated-queries.cmake @@ -0,0 +1,17 @@ +include(${CMAKE_CURRENT_LIST_DIR}/json.cmake) +macro(check_generated_json n) + set(expected_file ${RunCMake_TEST_BINARY_DIR}/query/query-${n}.json) + set(generated_file ${v1}/query/generated/query-${n}.json) + read_json(${expected_file} expected) + read_json(${generated_file} generated) + string(JSON equal EQUAL ${expected} ${generated}) + if (NOT equal) + set(RunCMake_TEST_FAILED + "Generated JSON ${generated}\nNot equal to expected ${expected}" + ) + endif() +endmacro() + +foreach(n ${generated_queries}) + check_generated_json(${n}) +endforeach() diff --git a/Tests/RunCMake/Instrumentation/cmake-command-bad-api-version-result.txt b/Tests/RunCMake/Instrumentation/cmake-command-bad-api-version-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-bad-api-version-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/Instrumentation/cmake-command-bad-api-version-stderr.txt b/Tests/RunCMake/Instrumentation/cmake-command-bad-api-version-stderr.txt new file mode 100644 index 0000000000..ab172e0829 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-bad-api-version-stderr.txt @@ -0,0 +1,6 @@ +CMake Error at [^ +]*\(cmake_instrumentation\): + cmake_instrumentation QUERY subcommand given an unsupported API_VERSION "0" + \(the only currently supported version is 1\). +Call Stack \(most recent call first\): + CMakeLists.txt:6 \(include\) diff --git a/Tests/RunCMake/Instrumentation/cmake-command-bad-arg-result.txt b/Tests/RunCMake/Instrumentation/cmake-command-bad-arg-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-bad-arg-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/Instrumentation/cmake-command-bad-arg-stderr.txt b/Tests/RunCMake/Instrumentation/cmake-command-bad-arg-stderr.txt new file mode 100644 index 0000000000..4d0f5320f7 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-bad-arg-stderr.txt @@ -0,0 +1,5 @@ +CMake Error at [^ +]* \(cmake_instrumentation\): + cmake_instrumentation given unknown argument "UNKNOWN_ARG". +Call Stack \(most recent call first\): + CMakeLists.txt:6 \(include\) diff --git a/Tests/RunCMake/Instrumentation/cmake-command-bad-data-version-result.txt b/Tests/RunCMake/Instrumentation/cmake-command-bad-data-version-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-bad-data-version-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/Instrumentation/cmake-command-bad-data-version-stderr.txt b/Tests/RunCMake/Instrumentation/cmake-command-bad-data-version-stderr.txt new file mode 100644 index 0000000000..512d1048f7 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-bad-data-version-stderr.txt @@ -0,0 +1,5 @@ +CMake Error at [^ +]*\(cmake_instrumentation\): + cmake_instrumentation given a non-integer DATA_VERSION. +Call Stack \(most recent call first\): + CMakeLists.txt:6 \(include\) diff --git a/Tests/RunCMake/Instrumentation/cmake-command-missing-version-result.txt b/Tests/RunCMake/Instrumentation/cmake-command-missing-version-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-missing-version-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/Instrumentation/cmake-command-missing-version-stderr.txt b/Tests/RunCMake/Instrumentation/cmake-command-missing-version-stderr.txt new file mode 100644 index 0000000000..0d1d8c350d --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-missing-version-stderr.txt @@ -0,0 +1,6 @@ +CMake Error at [^ +]*\(cmake_instrumentation\): + cmake_instrumentation QUERY subcommand given an unsupported DATA_VERSION "" + \(the only currently supported version is 1\). +Call Stack \(most recent call first\): + CMakeLists.txt:6 \(include\) diff --git a/Tests/RunCMake/Instrumentation/cmake-command-non-int-version-result.txt b/Tests/RunCMake/Instrumentation/cmake-command-non-int-version-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-non-int-version-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/Instrumentation/cmake-command-non-int-version-stderr.txt b/Tests/RunCMake/Instrumentation/cmake-command-non-int-version-stderr.txt new file mode 100644 index 0000000000..b6615ffa60 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-non-int-version-stderr.txt @@ -0,0 +1,2 @@ +CMake Error at CMakeLists\.txt:37 \(cmake_instrumentation\): + cmake_instrumentation given a non-integer DATA_VERSION\. diff --git a/Tests/RunCMake/Instrumentation/cmake-command-unsupported-version-result.txt b/Tests/RunCMake/Instrumentation/cmake-command-unsupported-version-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-unsupported-version-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/Instrumentation/cmake-command-unsupported-version-stderr.txt b/Tests/RunCMake/Instrumentation/cmake-command-unsupported-version-stderr.txt new file mode 100644 index 0000000000..2eab2247d2 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-unsupported-version-stderr.txt @@ -0,0 +1,3 @@ +CMake Error at CMakeLists\.txt:44 \(cmake_instrumentation\): + cmake_instrumentation given an unsupported API_VERSION "0" \(the only + currently supported version is 1\)\. diff --git a/Tests/RunCMake/Instrumentation/empty-result.txt b/Tests/RunCMake/Instrumentation/empty-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/Instrumentation/empty-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/Instrumentation/empty-stderr.txt b/Tests/RunCMake/Instrumentation/empty-stderr.txt new file mode 100644 index 0000000000..c8a539723d --- /dev/null +++ b/Tests/RunCMake/Instrumentation/empty-stderr.txt @@ -0,0 +1,4 @@ +^CMake Error: + +Error: @1,1: Missing required field "version" in root object +{ +\^$ diff --git a/Tests/RunCMake/Instrumentation/hook.cmake b/Tests/RunCMake/Instrumentation/hook.cmake new file mode 100644 index 0000000000..d3be3a41ba --- /dev/null +++ b/Tests/RunCMake/Instrumentation/hook.cmake @@ -0,0 +1,70 @@ +include(${CMAKE_CURRENT_LIST_DIR}/json.cmake) +# Test CALLBACK script. Prints output information and verifies index file +# Called as: cmake -P hook.cmake [CheckForStaticQuery?] [index.json] +set(index ${CMAKE_ARGV4}) +if (NOT ${CMAKE_ARGV3}) + set(hasStaticInfo "UNEXPECTED") +endif() +read_json(${index} contents) +string(JSON hook GET ${contents} hook) + +# Output is verified by *-stdout.txt files that the HOOK is run +message(STATUS ${hook}) +# Not a check-*.cmake script, this is called as an instrumentation CALLBACK +set(ERROR_MESSAGE "") +macro(add_error error) + string(APPEND ERROR_MESSAGE "${error}\n") +endmacro() + +macro(has_key key json) + cmake_parse_arguments(ARG "UNEXPECTED" "" "" ${ARGN}) + unset(missingKey) + string(JSON ${key} ERROR_VARIABLE missingKey GET ${json} ${key}) + if (NOT ARG_UNEXPECTED AND NOT "${missingKey}" MATCHES NOTFOUND) + add_error("\nKey \"${key}\" not in index:\n${json}") + elseif(ARG_UNEXPECTED AND "${missingKey}" MATCHES NOTFOUND) + add_error("\nUnexpected key \"${key}\" in index:\n${json}") + endif() +endmacro() + +has_key(version ${contents}) +has_key(buildDir ${contents}) +has_key(dataDir ${contents}) +has_key(snippets ${contents}) + +if (NOT ${version} EQUAL 1) + add_error("Version must be 1, got: ${version}") +endif() + +string(JSON length LENGTH ${snippets}) +math(EXPR length ${length}-1) +foreach(i RANGE ${length}) + string(JSON filename GET ${snippets} ${i}) + if (NOT EXISTS ${dataDir}/${filename}) + add_error("Listed snippet: ${dataDir}/${filename} does not exist") + endif() +endforeach() + +has_key(staticSystemInformation ${contents} ${hasStaticInfo}) +has_key(OSName ${staticSystemInformation} ${hasStaticInfo}) +has_key(OSPlatform ${staticSystemInformation} ${hasStaticInfo}) +has_key(OSRelease ${staticSystemInformation} ${hasStaticInfo}) +has_key(OSVersion ${staticSystemInformation} ${hasStaticInfo}) +has_key(familyId ${staticSystemInformation} ${hasStaticInfo}) +has_key(hostname ${staticSystemInformation} ${hasStaticInfo}) +has_key(is64Bits ${staticSystemInformation} ${hasStaticInfo}) +has_key(modelId ${staticSystemInformation} ${hasStaticInfo}) +has_key(numberOfLogicalCPU ${staticSystemInformation} ${hasStaticInfo}) +has_key(numberOfPhysicalCPU ${staticSystemInformation} ${hasStaticInfo}) +has_key(processorAPICID ${staticSystemInformation} ${hasStaticInfo}) +has_key(processorCacheSize ${staticSystemInformation} ${hasStaticInfo}) +has_key(processorClockFrequency ${staticSystemInformation} ${hasStaticInfo}) +has_key(processorName ${staticSystemInformation} ${hasStaticInfo}) +has_key(totalPhysicalMemory ${staticSystemInformation} ${hasStaticInfo}) +has_key(totalVirtualMemory ${staticSystemInformation} ${hasStaticInfo}) +has_key(vendorID ${staticSystemInformation} ${hasStaticInfo}) +has_key(vendorString ${staticSystemInformation} ${hasStaticInfo}) + +if (NOT ERROR_MESSAGE MATCHES "^$") + message(FATAL_ERROR ${ERROR_MESSAGE}) +endif() diff --git a/Tests/RunCMake/Instrumentation/hooks-1-build-stdout.txt b/Tests/RunCMake/Instrumentation/hooks-1-build-stdout.txt new file mode 100644 index 0000000000..8062d6a1ea --- /dev/null +++ b/Tests/RunCMake/Instrumentation/hooks-1-build-stdout.txt @@ -0,0 +1 @@ +^\-\- preCMakeBuild diff --git a/Tests/RunCMake/Instrumentation/hooks-1-install-stdout.txt b/Tests/RunCMake/Instrumentation/hooks-1-install-stdout.txt new file mode 100644 index 0000000000..da0da3b7eb --- /dev/null +++ b/Tests/RunCMake/Instrumentation/hooks-1-install-stdout.txt @@ -0,0 +1 @@ +.*\-\- postInstall$ diff --git a/Tests/RunCMake/Instrumentation/hooks-2-build-stdout.txt b/Tests/RunCMake/Instrumentation/hooks-2-build-stdout.txt new file mode 100644 index 0000000000..655b134276 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/hooks-2-build-stdout.txt @@ -0,0 +1 @@ +.*\-\- postCMakeBuild$ diff --git a/Tests/RunCMake/Instrumentation/hooks-2-stdout.txt b/Tests/RunCMake/Instrumentation/hooks-2-stdout.txt new file mode 100644 index 0000000000..e2458f777d --- /dev/null +++ b/Tests/RunCMake/Instrumentation/hooks-2-stdout.txt @@ -0,0 +1,5 @@ +.*\-\- Configuring done[^ +]* +\-\- Generating done[^ +]* +\-\- postGenerate.* diff --git a/Tests/RunCMake/Instrumentation/hooks-2-test-stdout.txt b/Tests/RunCMake/Instrumentation/hooks-2-test-stdout.txt new file mode 100644 index 0000000000..442a48c534 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/hooks-2-test-stdout.txt @@ -0,0 +1 @@ +.*\-\- postTest$ diff --git a/Tests/RunCMake/Instrumentation/hooks-no-callbacks-index-stdout.txt b/Tests/RunCMake/Instrumentation/hooks-no-callbacks-index-stdout.txt new file mode 100644 index 0000000000..8ed4f5e005 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/hooks-no-callbacks-index-stdout.txt @@ -0,0 +1 @@ +^\-\- manual$ diff --git a/Tests/RunCMake/Instrumentation/json.cmake b/Tests/RunCMake/Instrumentation/json.cmake new file mode 100644 index 0000000000..1b5b097117 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/json.cmake @@ -0,0 +1,8 @@ +macro(read_json filename outvar) + file(READ ${filename} contents) + # string(JSON *) will fail if JSON file contains any forward-slash paths + string(REGEX REPLACE "[\\]([a-zA-Z0-9 ])" "/\\1" contents ${contents}) + # string(JSON *) will fail if JSON file contains any escaped quotes \" + string(REPLACE "\\\"" "'" contents ${contents}) + set(${outvar} ${contents}) +endmacro() diff --git a/Tests/RunCMake/Instrumentation/project/CMakeLists.txt b/Tests/RunCMake/Instrumentation/project/CMakeLists.txt new file mode 100644 index 0000000000..30b28c6b2b --- /dev/null +++ b/Tests/RunCMake/Instrumentation/project/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.30) +project(instrumentation) +enable_testing() +if (EXISTS ${INSTRUMENT_COMMAND_FILE}) + set(CMAKE_EXPERIMENTAL_INSTRUMENTATION "a37d1069-1972-4901-b9c9-f194aaf2b6e0") + include(${INSTRUMENT_COMMAND_FILE}) +endif() + +add_executable(main main.cxx) +add_library(lib lib.cxx) +target_link_libraries(main lib) +add_custom_command( + COMMAND ${CMAKE_COMMAND} -E true + OUTPUT output1 output2 +) +add_custom_target(customTarget ALL + DEPENDS output1 +) +add_test(NAME test COMMAND $) +install(TARGETS main) +set_target_properties(main PROPERTIES LABELS "label1;label2") +set_target_properties(lib PROPERTIES LABELS "label3") diff --git a/Tests/RunCMake/Instrumentation/project/lib.cxx b/Tests/RunCMake/Instrumentation/project/lib.cxx new file mode 100644 index 0000000000..c91239731b --- /dev/null +++ b/Tests/RunCMake/Instrumentation/project/lib.cxx @@ -0,0 +1,4 @@ +int lib() +{ + return 0; +} diff --git a/Tests/RunCMake/Instrumentation/project/lib.h b/Tests/RunCMake/Instrumentation/project/lib.h new file mode 100644 index 0000000000..9f73b0d3bb --- /dev/null +++ b/Tests/RunCMake/Instrumentation/project/lib.h @@ -0,0 +1 @@ +int lib(); diff --git a/Tests/RunCMake/Instrumentation/project/main.cxx b/Tests/RunCMake/Instrumentation/project/main.cxx new file mode 100644 index 0000000000..0362522752 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/project/main.cxx @@ -0,0 +1,5 @@ +#include "lib.h" +int main() +{ + return lib(); +} diff --git a/Tests/RunCMake/Instrumentation/query/bad-hook.json.in b/Tests/RunCMake/Instrumentation/query/bad-hook.json.in new file mode 100644 index 0000000000..5f92ba0347 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/bad-hook.json.in @@ -0,0 +1,4 @@ +{ + "version": 1, + "hooks": ["bad hook", "postGenerate", "preCMakeBuild", "postCMakeBuild", "postInstall"] +} diff --git a/Tests/RunCMake/Instrumentation/query/bad-query.json.in b/Tests/RunCMake/Instrumentation/query/bad-query.json.in new file mode 100644 index 0000000000..5bd40541d0 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/bad-query.json.in @@ -0,0 +1,4 @@ +{ + "version": 1, + "queries": ["staticSystemInformation", "bad query"] +} diff --git a/Tests/RunCMake/Instrumentation/query/bad-version.json.in b/Tests/RunCMake/Instrumentation/query/bad-version.json.in new file mode 100644 index 0000000000..5dfe44db27 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/bad-version.json.in @@ -0,0 +1,3 @@ +{ + "version": 0 +} diff --git a/Tests/RunCMake/Instrumentation/query/both-query.json.in b/Tests/RunCMake/Instrumentation/query/both-query.json.in new file mode 100644 index 0000000000..839361fc7b --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/both-query.json.in @@ -0,0 +1,7 @@ +{ + "version": 1, + "queries": [ + "staticSystemInformation", + "dynamicSystemInformation" + ] +} diff --git a/Tests/RunCMake/Instrumentation/query/cmake-command-bad-api-version.cmake b/Tests/RunCMake/Instrumentation/query/cmake-command-bad-api-version.cmake new file mode 100644 index 0000000000..2d4d746fa8 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/cmake-command-bad-api-version.cmake @@ -0,0 +1,3 @@ +cmake_instrumentation( + API_VERSION 0 +) diff --git a/Tests/RunCMake/Instrumentation/query/cmake-command-bad-arg.cmake b/Tests/RunCMake/Instrumentation/query/cmake-command-bad-arg.cmake new file mode 100644 index 0000000000..ed88bd1bbf --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/cmake-command-bad-arg.cmake @@ -0,0 +1,5 @@ +cmake_instrumentation( + API_VERSION 1 + DATA_VERSION 1 + UNKNOWN_ARG +) diff --git a/Tests/RunCMake/Instrumentation/query/cmake-command-bad-data-version.cmake b/Tests/RunCMake/Instrumentation/query/cmake-command-bad-data-version.cmake new file mode 100644 index 0000000000..8fe34b6b0a --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/cmake-command-bad-data-version.cmake @@ -0,0 +1,4 @@ +cmake_instrumentation( + API_VERSION 1 + DATA_VERSION NOT_AN_INT +) diff --git a/Tests/RunCMake/Instrumentation/query/cmake-command-data.cmake b/Tests/RunCMake/Instrumentation/query/cmake-command-data.cmake new file mode 100644 index 0000000000..5c238fd71c --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/cmake-command-data.cmake @@ -0,0 +1,5 @@ +cmake_instrumentation( + API_VERSION 1 + DATA_VERSION 1 + QUERIES dynamicSystemInformation +) diff --git a/Tests/RunCMake/Instrumentation/query/cmake-command-missing-version.cmake b/Tests/RunCMake/Instrumentation/query/cmake-command-missing-version.cmake new file mode 100644 index 0000000000..84cffd03a7 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/cmake-command-missing-version.cmake @@ -0,0 +1,3 @@ +cmake_instrumentation( + API_VERSION 1 +) diff --git a/Tests/RunCMake/Instrumentation/query/cmake-command-parallel-install.cmake b/Tests/RunCMake/Instrumentation/query/cmake-command-parallel-install.cmake new file mode 100644 index 0000000000..237f14e904 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/cmake-command-parallel-install.cmake @@ -0,0 +1,6 @@ +set_property(GLOBAL PROPERTY INSTALL_PARALLEL ON) +cmake_instrumentation( + API_VERSION 1 + DATA_VERSION 1 + QUERIES dynamicSystemInformation +) diff --git a/Tests/RunCMake/Instrumentation/query/cmake-command.cmake b/Tests/RunCMake/Instrumentation/query/cmake-command.cmake new file mode 100644 index 0000000000..dbbebb1215 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/cmake-command.cmake @@ -0,0 +1,20 @@ + # Query 0 + cmake_instrumentation( + API_VERSION 1 + DATA_VERSION 1 + ) + # Query 1 + cmake_instrumentation( + API_VERSION 1 + DATA_VERSION 1 + HOOKS postGenerate + CALLBACK "\"${CMAKE_COMMAND}\" -E echo callback1" + ) + # Query 2 + cmake_instrumentation( + API_VERSION 1 + DATA_VERSION 1 + HOOKS postCMakeBuild + QUERIES staticSystemInformation dynamicSystemInformation + CALLBACK "\"${CMAKE_COMMAND}\" -E echo callback2" + ) diff --git a/Tests/RunCMake/Instrumentation/query/dynamic-query.json.in b/Tests/RunCMake/Instrumentation/query/dynamic-query.json.in new file mode 100644 index 0000000000..839361fc7b --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/dynamic-query.json.in @@ -0,0 +1,7 @@ +{ + "version": 1, + "queries": [ + "staticSystemInformation", + "dynamicSystemInformation" + ] +} diff --git a/Tests/RunCMake/Instrumentation/query/empty.json.in b/Tests/RunCMake/Instrumentation/query/empty.json.in new file mode 100644 index 0000000000..2c63c08510 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/empty.json.in @@ -0,0 +1,2 @@ +{ +} diff --git a/Tests/RunCMake/Instrumentation/query/generated/query-0.json.in b/Tests/RunCMake/Instrumentation/query/generated/query-0.json.in new file mode 100644 index 0000000000..f3659f447f --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/generated/query-0.json.in @@ -0,0 +1,6 @@ +{ + "callbacks" : [], + "hooks" : [], + "queries" : [], + "version": 1 +} diff --git a/Tests/RunCMake/Instrumentation/query/generated/query-1.json.in b/Tests/RunCMake/Instrumentation/query/generated/query-1.json.in new file mode 100644 index 0000000000..0a34392cb3 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/generated/query-1.json.in @@ -0,0 +1,12 @@ +{ + "callbacks" : + [ + "\"@CMAKE_COMMAND@\" -E echo callback1" + ], + "hooks" : + [ + "postGenerate" + ], + "queries" : [], + "version" : 1 +} diff --git a/Tests/RunCMake/Instrumentation/query/generated/query-2.json.in b/Tests/RunCMake/Instrumentation/query/generated/query-2.json.in new file mode 100644 index 0000000000..c29b4d4a30 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/generated/query-2.json.in @@ -0,0 +1,16 @@ +{ + "callbacks" : + [ + "\"@CMAKE_COMMAND@\" -E echo callback2" + ], + "hooks" : + [ + "postCMakeBuild" + ], + "queries" : + [ + "staticSystemInformation", + "dynamicSystemInformation" + ], + "version": 1 +} diff --git a/Tests/RunCMake/Instrumentation/query/hooks-1.json.in b/Tests/RunCMake/Instrumentation/query/hooks-1.json.in new file mode 100644 index 0000000000..31bba8681a --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/hooks-1.json.in @@ -0,0 +1,6 @@ +{ + "version": 1, + "hooks": ["preCMakeBuild", "postInstall"], + "callbacks": ["@GET_HOOK@"], + "queries": ["staticSystemInformation"] +} diff --git a/Tests/RunCMake/Instrumentation/query/hooks-2.json.in b/Tests/RunCMake/Instrumentation/query/hooks-2.json.in new file mode 100644 index 0000000000..ae18b09bce --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/hooks-2.json.in @@ -0,0 +1,5 @@ +{ + "version": 1, + "hooks": ["postGenerate", "postCMakeBuild", "postTest"], + "callbacks": ["@GET_HOOK@"] +} diff --git a/Tests/RunCMake/Instrumentation/query/hooks-no-callbacks.json.in b/Tests/RunCMake/Instrumentation/query/hooks-no-callbacks.json.in new file mode 100644 index 0000000000..69885d4d6f --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/hooks-no-callbacks.json.in @@ -0,0 +1,4 @@ +{ + "version": 1, + "callbacks": ["@GET_HOOK@"] +} diff --git a/Tests/RunCMake/Instrumentation/query/no-query.json.in b/Tests/RunCMake/Instrumentation/query/no-query.json.in new file mode 100644 index 0000000000..61a2092b1b --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/no-query.json.in @@ -0,0 +1,3 @@ +{ + "version": 1 +} diff --git a/Tests/RunCMake/Instrumentation/verify-snippet.cmake b/Tests/RunCMake/Instrumentation/verify-snippet.cmake new file mode 100644 index 0000000000..5f23bf2392 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/verify-snippet.cmake @@ -0,0 +1,100 @@ +# Performs generic (non-project specific) validation of v1 Snippet File Contents + +macro(add_error error) + string(APPEND RunCMake_TEST_FAILED "${error}\n") +endmacro() + +macro(snippet_error snippet error) + add_error("Error in snippet file ${snippet}:\n${error}") +endmacro() + +macro(has_key snippet json key) + string(JSON data ERROR_VARIABLE missingKey GET ${json} ${key}) + if (NOT ${missingKey} MATCHES NOTFOUND) + snippet_error(${snippet} "Missing ${key}") + endif() +endmacro() + +macro(has_not_key snippet json key) + string(JSON data ERROR_VARIABLE missingKey GET ${json} ${key}) + if (${missingKey} MATCHES NOTFOUND) + snippet_error(${snippet} "Has unexpected ${key}") + endif() +endmacro() + +macro(snippet_has_fields snippet contents) + has_key(${snippet} ${contents} command) + has_key(${snippet} ${contents} role) + has_key(${snippet} ${contents} result) + if (snippet MATCHES ^link-*) + has_key(${snippet} ${contents} target) + has_key(${snippet} ${contents} outputs) + has_key(${snippet} ${contents} outputSizes) + has_key(${snippet} ${contents} targetType) + elseif (snippet MATCHES ^compile-*) + has_key(${snippet} ${contents} target) + has_key(${snippet} ${contents} outputs) + has_key(${snippet} ${contents} outputSizes) + has_key(${snippet} ${contents} source) + has_key(${snippet} ${contents} language) + elseif (snippet MATCHES ^custom-*) + has_key(${snippet} ${contents} target) + has_key(${snippet} ${contents} outputs) + has_key(${snippet} ${contents} outputSizes) + elseif (snippet MATCHES ^test-*) + has_key(${snippet} ${contents} testName) + endif() + if(ARGS_DYNAMIC_QUERY) + has_key(${snippet} ${contents} dynamicSystemInformation) + string(JSON dynamicSystemInfo ERROR_VARIABLE noInfo GET ${contents} dynamicSystemInformation) + if (noInfo MATCHES NOTFOUND) + has_key(${snippet} ${dynamicSystemInfo} beforeCPULoadAverage) + has_key(${snippet} ${dynamicSystemInfo} beforeHostMemoryUsed) + has_key(${snippet} ${dynamicSystemInfo} beforeCPULoadAverage) + has_key(${snippet} ${dynamicSystemInfo} beforeHostMemoryUsed) + endif() + else() + has_not_key(${snippet} ${contents} dynamicSystemInformation) + string(JSON dynamicSystemInfo ERROR_VARIABLE noInfo GET ${contents} dynamicSystemInformation) + if (noInfo MATCHES NOTFOUND) + has_not_key(${snippet} ${dynamicSystemInfo} beforeCPULoadAverage) + has_not_key(${snippet} ${dynamicSystemInfo} beforeHostMemoryUsed) + has_not_key(${snippet} ${dynamicSystemInfo} beforeCPULoadAverage) + has_not_key(${snippet} ${dynamicSystemInfo} beforeHostMemoryUsed) + endif() + endif() +endmacro() + +macro(snippet_valid_timing contents) + string(JSON start GET ${contents} timeStart) + string(JSON duration GET ${contents} duration) + if (${start} LESS 0) + snippet_error(${snippet} "Negative time start: ${start}") + endif() + if (${duration} LESS 0) + snippet_error(${snippet} "Negative duration: ${end}") + endif() +endmacro() + +macro(verify_snippet snippet contents) + snippet_has_fields(${snippet} ${contents}) + snippet_valid_timing(${contents}) + string(JSON version GET ${contents} version) + if (NOT ${version} EQUAL 1) + snippet_error(${snippet} "Version must be 1, got: ${version}") + endif() + string(JSON role GET ${contents} role) + get_filename_component(filename ${snippet} NAME) + if (NOT ${filename} MATCHES ^${role}-) + snippet_error(${snippet} "Role \"${role}\" doesn't match snippet filename") + endif() + string(JSON outputs ERROR_VARIABLE noOutputs GET ${contents} outputs) + if (NOT outputs MATCHES NOTFOUND) + string(JSON outputSizes ERROR_VARIABLE noOutputSizes GET ${contents} outputSizes) + list(LENGTH outputs outputsLen) + list(LENGTH outputSizes outputSizesLen) + if (outputSizes MATCHES NOTFOUND OR NOT outputsLen EQUAL outputSizesLen) + snippet_error(${snippet} "outputs and outputSizes do not match") + endif() + endif() +endmacro()