diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index 035da3cc01..3a871bb6dc 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -1619,12 +1619,14 @@ model is defined as follows: The string "ctestInfo". ``version`` - A JSON object specifying the version components. Its members are + A JSON object specifying the version components. Its members are: ``major`` - A non-negative integer specifying the major version component. + A positive integer specifying the major version component + of the JSON object model. ``minor`` - A non-negative integer specifying the minor version component. + A non-negative integer specifying the minor version component + of the JSON object model. ``backtraceGraph`` JSON object representing backtrace information with the @@ -1638,32 +1640,54 @@ model is defined as follows: List of node JSON objects with members: ``command`` - Index into the ``commands`` member of the ``backtraceGraph``. + An optional member present when the node represents a command + invocation within the file. The value is an unsigned integer 0-based + index into the ``commands`` member of the ``backtraceGraph``. ``file`` - Index into the ``files`` member of the ``backtraceGraph``. + An unsigned integer 0-based index into the ``files`` member of the + ``backtraceGraph``. ``line`` - Line number in the file where the backtrace was added. + An optional member present when the node represents a line within + the file. The value is an unsigned integer 1-based line number + in the file where the backtrace was added. ``parent`` - Index into the ``nodes`` member of the ``backtraceGraph`` - representing the parent in the graph. + An optional member present when the node is not the bottom of the + call stack. The value is an unsigned integer 0-based index into the + ``nodes`` member of the ``backtraceGraph`` representing the parent + in the graph. ``tests`` A JSON array listing information about each test. Each entry is a JSON object with members: ``name`` - Test name. + Test name. This cannot be empty. ``config`` - Configuration that the test can run on. - Empty string means any config. + Optional field specifying the configuration for which the test will run. + This will always match the :option:`-C ` option specified on the + ``ctest`` command line. If no such option was given, this field will not + be present. ``command`` - List where the first element is the test command and the - remaining elements are the command arguments. + Optional array where the first element is the test command and the + remaining elements are the command arguments. Normally, this field should + be present and non-empty, but in certain corner cases involving generator + expressions, it is possible for a test to have no command and therefore + this field can be missing. ``backtrace`` Index into the ``nodes`` member of the ``backtraceGraph``. ``properties`` - Test properties. - Can contain keys for each of the supported test properties. + Optional array of test properties. + Each array item will be a JSON object with the following members: + + ``name`` + The name of the test property. This cannot be empty. + ``value`` + The property value, which can be a string, a number, a boolean, or an + array of strings. + +.. versionadded:: 4.1 + The JSON output format is described in machine-readable form by + :download:`this JSON schema `. .. _`ctest-resource-allocation`: diff --git a/Help/manual/ctest/show-only-schema.json b/Help/manual/ctest/show-only-schema.json new file mode 100644 index 0000000000..351101e317 --- /dev/null +++ b/Help/manual/ctest/show-only-schema.json @@ -0,0 +1,121 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["kind", "version", "backtraceGraph", "tests"], + "properties": { + "kind": { + "type": "string", + "const": "ctestInfo" + }, + "version": { + "type": "object", + "required": ["major", "minor"], + "properties": { + "major": { + "const": 1, + "description": "A positive integer specifying the major version component of the JSON object model." + }, + "minor": { + "const": 0, + "description": "A non-negative integer specifying the minor version component of the JSON object model." + } + } + }, + "backtraceGraph": { + "type": "object", + "required": ["commands", "files", "nodes"], + "properties": { + "commands": { + "type": "array", + "description": "List of CMake command names.", + "items": { + "type": "string" + } + }, + "files": { + "type": "array", + "description": "List of file paths, which may be relative or absolute. Relative paths are relative to the top-level source directory.", + "items": { + "type": "string" + } + }, + "nodes": { + "type": "array", + "items": { + "type": "object", + "required": ["file"], + "properties": { + "command": { + "type": "integer", + "minimum": 0, + "description": "An optional member present when the node represents a command invocation within the file. The value is an unsigned integer 0-based index into the commands member of the backtraceGraph." + }, + "file": { + "type": "integer", + "minimum": 0, + "description": "An unsigned integer 0-based index into the files member of the backtraceGraph." + }, + "line": { + "type": "integer", + "minimum": 1, + "description": "An optional member present when the node represents a line within the file. The value is an unsigned integer 1-based line number in the file where the backtrace was added." + }, + "parent": { + "type": "integer", + "minimum": 0, + "description": "An optional member present when the node is not the bottom of the call stack. The value is an unsigned integer 0-based index into the nodes member of the backtraceGraph representing the parent in the graph." + } + } + } + } + } + }, + "tests": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "backtrace"], + "properties": { + "name": { + "type": "string", + "description": "Test name", + "minLength": 1 + }, + "config": { + "type": "string", + "description": "Optional field specifying the configuration for which the test will run. This will always match the -C option specified on the ctest command line. If no such option was given, this field will not be present." + }, + "command": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Optional array where the first element is the test command and the remaining elements are the command arguments. Normally, this field should be present and non-empty, but in certain corner cases involving generator expressions, it is possible for a test to have no command and therefore this field can be missing." + }, + "backtrace": { + "type": "integer", + "description": "Index into the nodes member of the backtraceGraph." + }, + "properties": { + "type": "array", + "description": "Optional list of test properties associated with the test.", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Test property name.", + "minLength": 1 + }, + "value": { + "description": "Value of the test property. Any valid JSON type might be present." + } + } + } + } + } + } + } + } +} diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt index 1500a53827..c92aac1484 100644 --- a/Tests/RunCMake/CMakeLists.txt +++ b/Tests/RunCMake/CMakeLists.txt @@ -1084,7 +1084,10 @@ if(CMake_TEST_RunCMake_ExternalProject_RUN_SERIAL) endif() add_RunCMake_test(FetchContent) add_RunCMake_test(FetchContent_find_package) -set(CTestCommandLine_ARGS -DPython_EXECUTABLE=${Python_EXECUTABLE}) +set(CTestCommandLine_ARGS + -DPython_EXECUTABLE=${Python_EXECUTABLE} + -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA} +) if(NOT CMake_TEST_EXTERNAL_CMAKE) list(APPEND CTestCommandLine_ARGS -DTEST_AFFINITY=$) endif() diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index 7048492cca..928eb4693f 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -1,5 +1,6 @@ include(RunCMake) include(RunCTest) +cmake_policy(SET CMP0140 NEW) # Do not use any proxy for lookup of an invalid site. # DNS failure by proxy looks different than DNS failure without proxy. @@ -449,6 +450,20 @@ function(show_only_json_check_python v) set(json_file "${RunCMake_TEST_BINARY_DIR}/ctest.json") file(WRITE "${json_file}" "${actual_stdout}") set(actual_stdout "" PARENT_SCOPE) + + if(CMake_TEST_JSON_SCHEMA) + execute_process( + COMMAND ${Python_EXECUTABLE} "${RunCMake_SOURCE_DIR}/show-only_json_validate_schema.py" "${json_file}" + RESULT_VARIABLE result + OUTPUT_VARIABLE output + ERROR_VARIABLE output + ) + if(NOT result STREQUAL 0) + string(REPLACE "\n" "\n " output "${output}") + string(APPEND RunCMake_TEST_FAILED "Failed to validate version ${v} JSON schema for file: ${file}\nOutput:\n${output}\n") + endif() + endif() + execute_process( COMMAND ${Python_EXECUTABLE} "${RunCMake_SOURCE_DIR}/show-only_json-v${v}_check.py" "${json_file}" RESULT_VARIABLE result @@ -457,8 +472,9 @@ function(show_only_json_check_python v) ) if(NOT result EQUAL 0) string(REPLACE "\n" "\n " output " ${output}") - set(RunCMake_TEST_FAILED "Unexpected output:\n${output}" PARENT_SCOPE) + string(APPEND RunCMake_TEST_FAILED "Unexpected output:\n${output}" PARENT_SCOPE) endif() + return(PROPAGATE RunCMake_TEST_FAILED) endfunction() function(run_ShowOnly) diff --git a/Tests/RunCMake/CTestCommandLine/show-only_json_validate_schema.py b/Tests/RunCMake/CTestCommandLine/show-only_json_validate_schema.py new file mode 100644 index 0000000000..e0a8347273 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/show-only_json_validate_schema.py @@ -0,0 +1,16 @@ +import json +import jsonschema +import os.path +import sys + + +with open(sys.argv[1], "r", encoding="utf-8-sig") as f: + contents = json.load(f) + +schema_file = os.path.join( + os.path.dirname(__file__), + "..", "..", "..", "Help", "manual", "ctest", "show-only-schema.json") +with open(schema_file, "r", encoding="utf-8") as f: + schema = json.load(f) + +jsonschema.validate(contents, schema)