diff --git a/Help/manual/cmake-instrumentation.7.rst b/Help/manual/cmake-instrumentation.7.rst index 635a1be5d5..967b5a251b 100644 --- a/Help/manual/cmake-instrumentation.7.rst +++ b/Help/manual/cmake-instrumentation.7.rst @@ -344,7 +344,8 @@ and contain the following data: The working directory in which the ``command`` was executed. ``result`` - The exit-value of the command, an integer. + The exit code of the command, an integer. This will be ``null`` when + ``role`` is ``build``. ``role`` The type of command executed, which will be one of the following values: diff --git a/Source/CTest/cmCTestLaunch.cxx b/Source/CTest/cmCTestLaunch.cxx index 286715f9d1..68d0d8e5a6 100644 --- a/Source/CTest/cmCTestLaunch.cxx +++ b/Source/CTest/cmCTestLaunch.cxx @@ -306,7 +306,7 @@ int cmCTestLaunch::Run() this->Reporter.OptionCommandType, this->RealArgV, [this]() -> int { this->RunChild(); - return 0; + return this->Reporter.ExitCode; }, options, arrayOptions); diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 221e4ef854..8da4cfcbfb 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -2670,14 +2670,14 @@ int cmCTest::ExecuteTests(std::vector const& args) cmInstrumentation instrumentation(this->GetBinaryDir()); auto processHandler = [&handler]() -> int { - return handler.ProcessHandler(); + return handler.ProcessHandler() < 0 ? cmCTest::TEST_ERRORS : 0; }; std::map data; data["showOnly"] = this->GetShowOnly() ? "1" : "0"; int ret = instrumentation.InstrumentCommand("ctest", args, processHandler, data); instrumentation.CollectTimingData(cmInstrumentationQuery::Hook::PostCTest); - if (ret < 0) { + if (ret == cmCTest::TEST_ERRORS) { cmCTestLog(this, ERROR_MESSAGE, "Errors while running CTest\n"); if (!this->Impl->OutputTestOutputOnTestFailure) { std::string const lastTestLog = @@ -2688,10 +2688,8 @@ int cmCTest::ExecuteTests(std::vector const& args) "Use \"--rerun-failed --output-on-failure\" to re-run the " "failed cases verbosely.\n"); } - return cmCTest::TEST_ERRORS; } - - return 0; + return ret; } int cmCTest::RunCMakeAndTest() diff --git a/Source/cmInstrumentation.cxx b/Source/cmInstrumentation.cxx index cfd2e9884d..6ccf01dd48 100644 --- a/Source/cmInstrumentation.cxx +++ b/Source/cmInstrumentation.cxx @@ -633,7 +633,6 @@ int cmInstrumentation::InstrumentCommand( // Execute Command int ret = callback(); - root["result"] = ret; // Exit early if configure didn't generate a query if (reloadQueriesAfterCommand == LoadQueriesAfter::Yes) { @@ -671,6 +670,9 @@ int cmInstrumentation::InstrumentCommand( } } + // See SpawnBuildDaemon(); this data is currently meaningless for build. + root["result"] = command_type == "build" ? Json::nullValue : ret; + // Create empty config entry if config not found if (!root.isMember("config") && (command_type == "compile" || command_type == "link" || @@ -858,6 +860,8 @@ int cmInstrumentation::CollectTimingAfterBuild(int ppid) while (0 == uv_kill(ppid, 0)) { cmSystemTools::Delay(100); }; + // FIXME(#27331): Investigate a cross-platform solution to obtain the exit + // code given the `ppid` above. return 0; }; int ret = this->InstrumentCommand( diff --git a/Tests/RunCMake/Instrumentation/RunCMakeTest.cmake b/Tests/RunCMake/Instrumentation/RunCMakeTest.cmake index 51e08f2159..6fe6581f32 100644 --- a/Tests/RunCMake/Instrumentation/RunCMakeTest.cmake +++ b/Tests/RunCMake/Instrumentation/RunCMakeTest.cmake @@ -20,6 +20,7 @@ function(instrument test) "MANUAL_HOOK" "PRESERVE_DATA" "NO_CONFIGURE" + "FAIL" ) cmake_parse_arguments(ARGS "${OPTIONS}" "CHECK_SCRIPT;CONFIGURE_ARG" "" ${ARGN}) set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/${test}) @@ -82,6 +83,9 @@ function(instrument test) if (ARGS_NO_WARN) list(APPEND ARGS_CONFIGURE_ARG "-Wno-dev") endif() + if (ARGS_FAIL) + list(APPEND ARGS_CONFIGURE_ARG "-DFAIL=ON") + endif() set(RunCMake_TEST_SOURCE_DIR ${RunCMake_SOURCE_DIR}/project) if(NOT RunCMake_GENERATOR_IS_MULTI_CONFIG) set(maybe_CMAKE_BUILD_TYPE -DCMAKE_BUILD_TYPE=Debug) @@ -92,7 +96,29 @@ function(instrument test) # Follow-up Commands if (ARGS_BUILD) - run_cmake_command(${test}-build ${CMAKE_COMMAND} --build . --config Debug) + set(cmake_build_args --config Debug) + set(additional_build_args) + if (ARGS_FAIL) + # Tests with ARGS_FAIL expect all targets to build, including the ones + # which should succeed and those which should fail. + if (RunCMake_GENERATOR MATCHES "Ninja") + set(keep_going_arg -k 0) + elseif (RunCMake_GENERATOR MATCHES "FASTBuild") + set(keep_going_arg -nostoponerror) + else() + set(keep_going_arg -k) + endif() + string(APPEND additional_build_args ${keep_going_arg}) + # Merge stdout and stderr because different compilers will throw their + # errors to different places. + set(RunCMake_TEST_OUTPUT_MERGE 1) + endif() + run_cmake_command(${test}-build + ${CMAKE_COMMAND} --build . ${cmake_build_args} -- ${additional_build_args} + ) + if (ARGS_FAIL) + unset(RunCMake_TEST_OUTPUT_MERGE) + endif() endif() if (ARGS_BUILD_MAKE_PROGRAM) set(RunCMake_TEST_OUTPUT_MERGE 1) @@ -171,6 +197,16 @@ instrument(cmake-command-cmake-build NO_WARN BUILD CHECK_SCRIPT check-no-make-program-hooks.cmake ) +if(RunCMake_GENERATOR STREQUAL "Borland Makefiles") + # Borland 'make' has no '-k' flag. + set(Skip_COMMAND_FAILURES_Case 1) +endif() +if(NOT Skip_COMMAND_FAILURES_Case) + instrument(cmake-command-failures + FAIL NO_WARN BUILD TEST INSTALL + CHECK_SCRIPT check-data-dir.cmake + ) +endif() # Test CUSTOM_CONTENT instrument(cmake-command-custom-content diff --git a/Tests/RunCMake/Instrumentation/check-data-dir.cmake b/Tests/RunCMake/Instrumentation/check-data-dir.cmake index 5d96b39b04..5d76f871d3 100644 --- a/Tests/RunCMake/Instrumentation/check-data-dir.cmake +++ b/Tests/RunCMake/Instrumentation/check-data-dir.cmake @@ -24,28 +24,43 @@ foreach(snippet IN LISTS snippets) string(JSON target ERROR_VARIABLE noTarget GET "${contents}" target) if (NOT target MATCHES NOTFOUND) set(targets "main;lib;customTarget;TARGET_NAME") + if (ARGS_FAIL) + list(APPEND targets "dummy") + endif() if (NOT ${target} IN_LIST targets) json_error("${snippet}" "Unexpected target: ${target}") endif() endif() - # Verify output - string(JSON result GET "${contents}" result) - if (NOT ${result} EQUAL 0) - json_error("${snippet}" "Compile command had non-0 result") - endif() - # Verify contents of compile-* Snippets if (filename MATCHES "^compile-") string(JSON target GET "${contents}" target) string(JSON source GET "${contents}" source) string(JSON language GET "${contents}" language) + string(JSON result GET "${contents}" result) if (NOT language MATCHES "C\\+\\+") json_error("${snippet}" "Expected C++ compile language") endif() if (NOT source MATCHES "${target}.cxx$") json_error("${snippet}" "Unexpected source file") endif() + if (ARGS_FAIL) + if (source MATCHES "dummy.cxx" AND result EQUAL 0) + json_error("${snippet}" + "Expected nonzero exit code for compile command, got: ${result}" + ) + elseif (NOT source MATCHES "dummy.cxx" AND NOT result EQUAL 0) + json_error("${snippet}" + "Expected zero exit code for compile command, got: ${result}" + ) + endif() + else() + if (NOT result EQUAL 0) + json_error("${snippet}" + "Expected zero exit code for compile command, got: ${result}" + ) + endif() + endif() endif() # Verify contents of link-* Snippets @@ -80,8 +95,40 @@ foreach(snippet IN LISTS snippets) # Verify contents of test-* Snippets if (filename MATCHES "^test-") string(JSON testName GET "${contents}" testName) - if (NOT testName STREQUAL "test") - json_error("${snippet}" "Unexpected testName: ${testName}") + string(JSON result GET "${contents}" result) + if (ARGS_FAIL) + if (testName STREQUAL "test" AND NOT result EQUAL 0) + json_error("${snippet}" "Expected zero exit code for test") + elseif (testName STREQUAL "dummy" AND result EQUAL 0) + json_error("${snippet}" + "Expected nonzero exit code for dummy test, got: ${result}" + ) + elseif (NOT testName MATCHES "test|dummy") + json_error("${snippet}" "Unexpected test name: ${testName}") + endif() + else() + if (NOT testName STREQUAL "test") + json_error("${snippet}" "Unexpected test name: ${testName}") + endif() + if (NOT result EQUAL 0) + json_error("${snippet}" + "Expected zero exit code for test, got: ${result}" + ) + endif() + endif() + endif() + + # Verify the overall result, in addition to the sub-commands above. + if (filename MATCHES "^cmakeInstall|^cmakeBuild|^ctest") + string(JSON result GET "${contents}" result) + if (ARGS_FAIL AND result EQUAL 0) + json_error("${snippet}" + "Expected nonzero exit code, got: ${result}" + ) + elseif (NOT ARGS_FAIL AND NOT result EQUAL 0) + json_error("${snippet}" + "Expected zero exit code, got: ${result}" + ) endif() endif() diff --git a/Tests/RunCMake/Instrumentation/cmake-command-failures-build-result.txt b/Tests/RunCMake/Instrumentation/cmake-command-failures-build-result.txt new file mode 100644 index 0000000000..d197c913c2 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-failures-build-result.txt @@ -0,0 +1 @@ +[^0] diff --git a/Tests/RunCMake/Instrumentation/cmake-command-failures-build-stdout.txt b/Tests/RunCMake/Instrumentation/cmake-command-failures-build-stdout.txt new file mode 100644 index 0000000000..2d19ba1478 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-failures-build-stdout.txt @@ -0,0 +1 @@ +.*dummy\.cxx.* diff --git a/Tests/RunCMake/Instrumentation/cmake-command-failures-install-result.txt b/Tests/RunCMake/Instrumentation/cmake-command-failures-install-result.txt new file mode 100644 index 0000000000..d197c913c2 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-failures-install-result.txt @@ -0,0 +1 @@ +[^0] diff --git a/Tests/RunCMake/Instrumentation/cmake-command-failures-install-stderr.txt b/Tests/RunCMake/Instrumentation/cmake-command-failures-install-stderr.txt new file mode 100644 index 0000000000..2693e9204d --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-failures-install-stderr.txt @@ -0,0 +1,2 @@ +CMake Error at cmake_install\.cmake.*: + Failed install\. diff --git a/Tests/RunCMake/Instrumentation/cmake-command-failures-test-result.txt b/Tests/RunCMake/Instrumentation/cmake-command-failures-test-result.txt new file mode 100644 index 0000000000..45a4fb75db --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-failures-test-result.txt @@ -0,0 +1 @@ +8 diff --git a/Tests/RunCMake/Instrumentation/cmake-command-failures-test-stderr.txt b/Tests/RunCMake/Instrumentation/cmake-command-failures-test-stderr.txt new file mode 100644 index 0000000000..ba4235defb --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-failures-test-stderr.txt @@ -0,0 +1 @@ +Errors while running CTest diff --git a/Tests/RunCMake/Instrumentation/cmake-command-failures-test-stdout.txt b/Tests/RunCMake/Instrumentation/cmake-command-failures-test-stdout.txt new file mode 100644 index 0000000000..c69cff24dd --- /dev/null +++ b/Tests/RunCMake/Instrumentation/cmake-command-failures-test-stdout.txt @@ -0,0 +1 @@ +dummy \(Failed\) diff --git a/Tests/RunCMake/Instrumentation/cmake-command-non-int-version-result.txt b/Tests/RunCMake/Instrumentation/cmake-command-non-int-version-result.txt deleted file mode 100644 index d00491fd7e..0000000000 --- a/Tests/RunCMake/Instrumentation/cmake-command-non-int-version-result.txt +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/Tests/RunCMake/Instrumentation/cmake-command-non-int-version-stderr.txt b/Tests/RunCMake/Instrumentation/cmake-command-non-int-version-stderr.txt deleted file mode 100644 index b6615ffa60..0000000000 --- a/Tests/RunCMake/Instrumentation/cmake-command-non-int-version-stderr.txt +++ /dev/null @@ -1,2 +0,0 @@ -CMake Error at CMakeLists\.txt:37 \(cmake_instrumentation\): - cmake_instrumentation given a non-integer DATA_VERSION\. diff --git a/Tests/RunCMake/Instrumentation/cmake-command-unsupported-version-result.txt b/Tests/RunCMake/Instrumentation/cmake-command-unsupported-version-result.txt deleted file mode 100644 index d00491fd7e..0000000000 --- a/Tests/RunCMake/Instrumentation/cmake-command-unsupported-version-result.txt +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/Tests/RunCMake/Instrumentation/cmake-command-unsupported-version-stderr.txt b/Tests/RunCMake/Instrumentation/cmake-command-unsupported-version-stderr.txt deleted file mode 100644 index 2eab2247d2..0000000000 --- a/Tests/RunCMake/Instrumentation/cmake-command-unsupported-version-stderr.txt +++ /dev/null @@ -1,3 +0,0 @@ -CMake Error at CMakeLists\.txt:44 \(cmake_instrumentation\): - cmake_instrumentation given an unsupported API_VERSION "0" \(the only - currently supported version is 1\)\. diff --git a/Tests/RunCMake/Instrumentation/project/CMakeLists.txt b/Tests/RunCMake/Instrumentation/project/CMakeLists.txt index 80fd14bbad..8e9a47ab87 100644 --- a/Tests/RunCMake/Instrumentation/project/CMakeLists.txt +++ b/Tests/RunCMake/Instrumentation/project/CMakeLists.txt @@ -25,3 +25,12 @@ add_test(NAME test COMMAND $) install(TARGETS main) set_target_properties(main PROPERTIES LABELS "label1;label2") set_target_properties(lib PROPERTIES LABELS "label3") + +if (FAIL) + file(WRITE "${CMAKE_BINARY_DIR}/dummy.cxx" + "#error \"something which will not compile\"\n" + ) + add_executable(dummy "${CMAKE_BINARY_DIR}/dummy.cxx") + add_test(NAME dummy COMMAND ${CMAKE_COMMAND} -E false) + install(CODE "message(FATAL_ERROR \"Failed install.\")") +endif() diff --git a/Tests/RunCMake/Instrumentation/query/cmake-command-failures.cmake b/Tests/RunCMake/Instrumentation/query/cmake-command-failures.cmake new file mode 100644 index 0000000000..bdfdacf6f1 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/cmake-command-failures.cmake @@ -0,0 +1,4 @@ +cmake_instrumentation( + API_VERSION 1 + DATA_VERSION 1 +) diff --git a/Tests/RunCMake/Instrumentation/verify-snippet.cmake b/Tests/RunCMake/Instrumentation/verify-snippet.cmake index 92d0f3d959..add1ba979d 100644 --- a/Tests/RunCMake/Instrumentation/verify-snippet.cmake +++ b/Tests/RunCMake/Instrumentation/verify-snippet.cmake @@ -5,10 +5,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/json.cmake) function(snippet_has_fields snippet contents) get_filename_component(filename "${snippet}" NAME) json_has_key("${snippet}" "${contents}" role) - json_has_key("${snippet}" "${contents}" result) json_has_key("${snippet}" "${contents}" workingDir) + json_has_key("${snippet}" "${contents}" result) if (NOT filename MATCHES "^build-*") json_has_key("${snippet}" "${contents}" command) + else() + json_missing_key("${snippet}" "${contents}" command) endif() if (filename MATCHES "^link-*") json_has_key("${snippet}" "${contents}" target) @@ -76,6 +78,13 @@ function(verify_snippet_data snippet contents) if (NOT version EQUAL 1) json_error("${snippet}" "Version must be 1, got: ${version}") endif() + get_filename_component(filename "${snippet}" NAME) + string(JSON result GET "${contents}" result) + if ("${filename}" MATCHES "^build-*" AND result) + json_error("${snippet}" "Result must be null for build snippets, got: ${result}") + elseif (NOT "${filename}" MATCHES "^build-*" AND NOT result MATCHES "^-?[0-9]+$") + json_error("${snippet}" "Result must be integer, got: ${result}") + endif() string(JSON outputs ERROR_VARIABLE noOutputs GET "${contents}" outputs) if (NOT outputs MATCHES NOTFOUND) string(JSON outputSizes ERROR_VARIABLE noOutputSizes GET "${contents}" outputSizes)