From fe1cd4e1573cc8411628e29d027e1be8cdf7ea99 Mon Sep 17 00:00:00 2001 From: Craig Scott Date: Fri, 6 Jun 2025 17:41:29 +1000 Subject: [PATCH 1/2] Help: Make docs for ctest --show-only=json-v1 reflect the implementation In elements of the backtraceGraph->nodes array, all but the `file` field are optional. The documentation implied all fields were mandatory by not mentioning any of them as optional. Update the docs to make the optional nature explicit. Use the corresponding documentation for the CMake file API as a guide for the updated text. The tests array was also lacking accuracy for which fields were optional and the tests->properties array didn't specify the type of items that the array could hold. The explanation for the "config" item was also wrong, since it always holds the value of the -C option given to ctest. Fix and expand the docs to address these cases as well. Fixes: #26965 Issue: #26986 --- Help/manual/ctest.1.rst | 44 +++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index 035da3cc01..b72336fc45 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -1619,7 +1619,7 @@ model is defined as follows: The string "ctestInfo". ``version`` - A JSON object specifying the version components. Its members are + A JSON object specifying the version components. Its members are: ``major`` A non-negative integer specifying the major version component. @@ -1638,32 +1638,50 @@ model is defined as follows: List of node JSON objects with members: ``command`` - Index into the ``commands`` member of the ``backtraceGraph``. + An optional member present when the node represents a command + invocation within the file. The value is an unsigned integer 0-based + index into the ``commands`` member of the ``backtraceGraph``. ``file`` - Index into the ``files`` member of the ``backtraceGraph``. + An unsigned integer 0-based index into the ``files`` member of the + ``backtraceGraph``. ``line`` - Line number in the file where the backtrace was added. + An optional member present when the node represents a line within + the file. The value is an unsigned integer 1-based line number + in the file where the backtrace was added. ``parent`` - Index into the ``nodes`` member of the ``backtraceGraph`` - representing the parent in the graph. + An optional member present when the node is not the bottom of the + call stack. The value is an unsigned integer 0-based index into the + ``nodes`` member of the ``backtraceGraph`` representing the parent + in the graph. ``tests`` A JSON array listing information about each test. Each entry is a JSON object with members: ``name`` - Test name. + Test name. This cannot be empty. ``config`` - Configuration that the test can run on. - Empty string means any config. + Optional field specifying the configuration for which the test will run. + This will always match the :option:`-C ` option specified on the + ``ctest`` command line. If no such option was given, this field will not + be present. ``command`` - List where the first element is the test command and the - remaining elements are the command arguments. + Optional array where the first element is the test command and the + remaining elements are the command arguments. Normally, this field should + be present and non-empty, but in certain corner cases involving generator + expressions, it is possible for a test to have no command and therefore + this field can be missing. ``backtrace`` Index into the ``nodes`` member of the ``backtraceGraph``. ``properties`` - Test properties. - Can contain keys for each of the supported test properties. + Optional array of test properties. + Each array item will be a JSON object with the following members: + + ``name`` + The name of the test property. This cannot be empty. + ``value`` + The property value, which can be a string, a number, a boolean, or an + array of strings. .. _`ctest-resource-allocation`: From 2e7bca5f052821f8e8063edaeb3b4d19af052f8a Mon Sep 17 00:00:00 2001 From: Craig Scott Date: Sat, 7 Jun 2025 18:36:07 +1000 Subject: [PATCH 2/2] ctest: Add JSON schema for --show-only=json-v1 output Add schema validation to the existing test case for --show-only=json-v1 too. Fixes: #26980 --- Help/manual/ctest.1.rst | 10 +- Help/manual/ctest/show-only-schema.json | 121 ++++++++++++++++++ Tests/RunCMake/CMakeLists.txt | 5 +- .../CTestCommandLine/RunCMakeTest.cmake | 18 ++- .../show-only_json_validate_schema.py | 16 +++ 5 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 Help/manual/ctest/show-only-schema.json create mode 100644 Tests/RunCMake/CTestCommandLine/show-only_json_validate_schema.py diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index b72336fc45..3a871bb6dc 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -1622,9 +1622,11 @@ model is defined as follows: A JSON object specifying the version components. Its members are: ``major`` - A non-negative integer specifying the major version component. + A positive integer specifying the major version component + of the JSON object model. ``minor`` - A non-negative integer specifying the minor version component. + A non-negative integer specifying the minor version component + of the JSON object model. ``backtraceGraph`` JSON object representing backtrace information with the @@ -1683,6 +1685,10 @@ model is defined as follows: The property value, which can be a string, a number, a boolean, or an array of strings. +.. versionadded:: 4.1 + The JSON output format is described in machine-readable form by + :download:`this JSON schema `. + .. _`ctest-resource-allocation`: Resource Allocation diff --git a/Help/manual/ctest/show-only-schema.json b/Help/manual/ctest/show-only-schema.json new file mode 100644 index 0000000000..351101e317 --- /dev/null +++ b/Help/manual/ctest/show-only-schema.json @@ -0,0 +1,121 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["kind", "version", "backtraceGraph", "tests"], + "properties": { + "kind": { + "type": "string", + "const": "ctestInfo" + }, + "version": { + "type": "object", + "required": ["major", "minor"], + "properties": { + "major": { + "const": 1, + "description": "A positive integer specifying the major version component of the JSON object model." + }, + "minor": { + "const": 0, + "description": "A non-negative integer specifying the minor version component of the JSON object model." + } + } + }, + "backtraceGraph": { + "type": "object", + "required": ["commands", "files", "nodes"], + "properties": { + "commands": { + "type": "array", + "description": "List of CMake command names.", + "items": { + "type": "string" + } + }, + "files": { + "type": "array", + "description": "List of file paths, which may be relative or absolute. Relative paths are relative to the top-level source directory.", + "items": { + "type": "string" + } + }, + "nodes": { + "type": "array", + "items": { + "type": "object", + "required": ["file"], + "properties": { + "command": { + "type": "integer", + "minimum": 0, + "description": "An optional member present when the node represents a command invocation within the file. The value is an unsigned integer 0-based index into the commands member of the backtraceGraph." + }, + "file": { + "type": "integer", + "minimum": 0, + "description": "An unsigned integer 0-based index into the files member of the backtraceGraph." + }, + "line": { + "type": "integer", + "minimum": 1, + "description": "An optional member present when the node represents a line within the file. The value is an unsigned integer 1-based line number in the file where the backtrace was added." + }, + "parent": { + "type": "integer", + "minimum": 0, + "description": "An optional member present when the node is not the bottom of the call stack. The value is an unsigned integer 0-based index into the nodes member of the backtraceGraph representing the parent in the graph." + } + } + } + } + } + }, + "tests": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "backtrace"], + "properties": { + "name": { + "type": "string", + "description": "Test name", + "minLength": 1 + }, + "config": { + "type": "string", + "description": "Optional field specifying the configuration for which the test will run. This will always match the -C option specified on the ctest command line. If no such option was given, this field will not be present." + }, + "command": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Optional array where the first element is the test command and the remaining elements are the command arguments. Normally, this field should be present and non-empty, but in certain corner cases involving generator expressions, it is possible for a test to have no command and therefore this field can be missing." + }, + "backtrace": { + "type": "integer", + "description": "Index into the nodes member of the backtraceGraph." + }, + "properties": { + "type": "array", + "description": "Optional list of test properties associated with the test.", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Test property name.", + "minLength": 1 + }, + "value": { + "description": "Value of the test property. Any valid JSON type might be present." + } + } + } + } + } + } + } + } +} diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt index 3fd01594da..e13968ac07 100644 --- a/Tests/RunCMake/CMakeLists.txt +++ b/Tests/RunCMake/CMakeLists.txt @@ -1084,7 +1084,10 @@ if(CMake_TEST_RunCMake_ExternalProject_RUN_SERIAL) endif() add_RunCMake_test(FetchContent) add_RunCMake_test(FetchContent_find_package) -set(CTestCommandLine_ARGS -DPython_EXECUTABLE=${Python_EXECUTABLE}) +set(CTestCommandLine_ARGS + -DPython_EXECUTABLE=${Python_EXECUTABLE} + -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA} +) if(NOT CMake_TEST_EXTERNAL_CMAKE) list(APPEND CTestCommandLine_ARGS -DTEST_AFFINITY=$) endif() diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index 7048492cca..928eb4693f 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -1,5 +1,6 @@ include(RunCMake) include(RunCTest) +cmake_policy(SET CMP0140 NEW) # Do not use any proxy for lookup of an invalid site. # DNS failure by proxy looks different than DNS failure without proxy. @@ -449,6 +450,20 @@ function(show_only_json_check_python v) set(json_file "${RunCMake_TEST_BINARY_DIR}/ctest.json") file(WRITE "${json_file}" "${actual_stdout}") set(actual_stdout "" PARENT_SCOPE) + + if(CMake_TEST_JSON_SCHEMA) + execute_process( + COMMAND ${Python_EXECUTABLE} "${RunCMake_SOURCE_DIR}/show-only_json_validate_schema.py" "${json_file}" + RESULT_VARIABLE result + OUTPUT_VARIABLE output + ERROR_VARIABLE output + ) + if(NOT result STREQUAL 0) + string(REPLACE "\n" "\n " output "${output}") + string(APPEND RunCMake_TEST_FAILED "Failed to validate version ${v} JSON schema for file: ${file}\nOutput:\n${output}\n") + endif() + endif() + execute_process( COMMAND ${Python_EXECUTABLE} "${RunCMake_SOURCE_DIR}/show-only_json-v${v}_check.py" "${json_file}" RESULT_VARIABLE result @@ -457,8 +472,9 @@ function(show_only_json_check_python v) ) if(NOT result EQUAL 0) string(REPLACE "\n" "\n " output " ${output}") - set(RunCMake_TEST_FAILED "Unexpected output:\n${output}" PARENT_SCOPE) + string(APPEND RunCMake_TEST_FAILED "Unexpected output:\n${output}" PARENT_SCOPE) endif() + return(PROPAGATE RunCMake_TEST_FAILED) endfunction() function(run_ShowOnly) diff --git a/Tests/RunCMake/CTestCommandLine/show-only_json_validate_schema.py b/Tests/RunCMake/CTestCommandLine/show-only_json_validate_schema.py new file mode 100644 index 0000000000..e0a8347273 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/show-only_json_validate_schema.py @@ -0,0 +1,16 @@ +import json +import jsonschema +import os.path +import sys + + +with open(sys.argv[1], "r", encoding="utf-8-sig") as f: + contents = json.load(f) + +schema_file = os.path.join( + os.path.dirname(__file__), + "..", "..", "..", "Help", "manual", "ctest", "show-only-schema.json") +with open(schema_file, "r", encoding="utf-8") as f: + schema = json.load(f) + +jsonschema.validate(contents, schema)