Tests: Add JSON schema validation to CMakePresets.json tests

Tests/RunCMake/CMakeLists.txt

@@ -769,7 +769,7 @@ add_RunCMake_test(PrecompileHeaders -DCMAKE_C_COMPILER_ID=${CMAKE_C_COMPILER_ID}
   -DCMAKE_C_COMPILER_VERSION=${CMAKE_C_COMPILER_VERSION})
 
 add_RunCMake_test("UnityBuild")
-add_RunCMake_test(CMakePresets)
+add_RunCMake_test(CMakePresets -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA})
 
 if(WIN32)
   add_RunCMake_test(Win32GenEx)

Tests/RunCMake/CMakePresets/RunCMakeTest.cmake

@@ -7,6 +7,24 @@ if(RunCMake_GENERATOR MATCHES "^(Visual Studio [0-9]+ [0-9]+) ")
   set(RunCMake_GENERATOR "${CMAKE_MATCH_1}")
 endif()
 
+set(RunCMake-check-file check.cmake)
+
+function(validate_schema file expected_result)
+  execute_process(
+    COMMAND "${PYTHON_EXECUTABLE}" "${RunCMake_SOURCE_DIR}/validate_schema.py" "${file}"
+    RESULT_VARIABLE _result
+    OUTPUT_VARIABLE _output
+    ERROR_VARIABLE _error
+    )
+  if(NOT _result STREQUAL expected_result)
+    string(REPLACE "\n" "\n" _output_p "${_output}")
+    string(REPLACE "\n" "\n" _error_p "${_error}")
+    string(APPEND RunCMake_TEST_FAILED "Expected result of validating ${file}: ${expected_result}\nActual result: ${_result}\nOutput:\n${_output_p}\nError:\n${_error_p}")
+  endif()
+
+  set(RunCMake_TEST_FAILED "${RunCMake_TEST_FAILED}" PARENT_SCOPE)
+endfunction()
+
 function(run_cmake_presets name)
   set(RunCMake_TEST_SOURCE_DIR "${RunCMake_BINARY_DIR}/${name}")
   set(_source_arg "${RunCMake_TEST_SOURCE_DIR}")
@@ -57,6 +75,7 @@ function(run_cmake_presets name)
 endfunction()
 
 # Test CMakePresets.json errors
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 1)
 run_cmake_presets(NoCMakePresets)
 run_cmake_presets(JSONParseError)
 run_cmake_presets(InvalidRoot)
@@ -65,15 +84,21 @@ run_cmake_presets(InvalidVersion)
 run_cmake_presets(LowVersion)
 run_cmake_presets(HighVersion)
 run_cmake_presets(InvalidVendor)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
 run_cmake_presets(NoPresets)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 1)
 run_cmake_presets(InvalidPresets)
 run_cmake_presets(PresetNotObject)
 run_cmake_presets(NoPresetName)
 run_cmake_presets(InvalidPresetName)
 run_cmake_presets(EmptyPresetName)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
 run_cmake_presets(NoPresetGenerator)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 1)
 run_cmake_presets(InvalidPresetGenerator)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
 run_cmake_presets(NoPresetBinaryDir)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 1)
 run_cmake_presets(InvalidPresetBinaryDir)
 run_cmake_presets(InvalidVariables)
 run_cmake_presets(VariableNotObject)
@@ -83,6 +108,7 @@ run_cmake_presets(ExtraRootField)
 run_cmake_presets(ExtraPresetField)
 run_cmake_presets(ExtraVariableField)
 run_cmake_presets(InvalidPresetVendor)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
 run_cmake_presets(DuplicatePresets)
 run_cmake_presets(CyclicInheritance0)
 run_cmake_presets(CyclicInheritance1)
@@ -90,6 +116,7 @@ run_cmake_presets(CyclicInheritance2)
 run_cmake_presets(InvalidInheritance)
 run_cmake_presets(ErrorNoWarningDev)
 run_cmake_presets(ErrorNoWarningDeprecated)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 1)
 run_cmake_presets(InvalidCMakeGeneratorConfig)
 run_cmake_presets(UnknownCMakeGeneratorConfig)
 run_cmake_presets(EmptyCacheKey)
@@ -97,6 +124,7 @@ run_cmake_presets(EmptyEnvKey)
 
 # Test cmakeMinimumRequired field
 run_cmake_presets(MinimumRequiredInvalid)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
 run_cmake_presets(MinimumRequiredEmpty)
 run_cmake_presets(MinimumRequiredMajor)
 run_cmake_presets(MinimumRequiredMinor)

Tests/RunCMake/CMakePresets/check.cmake

@@ -0,0 +1,15 @@
+if(PYTHON_EXECUTABLE AND CMake_TEST_JSON_SCHEMA)
+  if(NOT CMakePresets_SCHEMA_EXPECTED_RESULT)
+    set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
+  endif()
+  if(EXISTS "${RunCMake_TEST_SOURCE_DIR}/CMakePresets.json")
+    validate_schema("${RunCMake_TEST_SOURCE_DIR}/CMakePresets.json" "${CMakePresets_SCHEMA_EXPECTED_RESULT}")
+  endif()
+
+  if(NOT CMakeUserPresets_SCHEMA_EXPECTED_RESULT)
+    set(CMakeUserPresets_SCHEMA_EXPECTED_RESULT 0)
+  endif()
+  if(EXISTS "${RunCMake_TEST_SOURCE_DIR}/CMakeUserPresets.json")
+    validate_schema("${RunCMake_TEST_SOURCE_DIR}/CMakeUserPresets.json" "${CMakeUserPresets_SCHEMA_EXPECTED_RESULT}")
+  endif()
+endif()

Tests/RunCMake/CMakePresets/validate_schema.py

@@ -0,0 +1,17 @@
+import jsmin
+import json
+import jsonschema
+import os.path
+import sys
+
+
+with open(sys.argv[1], "rb") as f:
+    contents = json.loads(jsmin.jsmin(f.read().decode("utf-8-sig")))
+
+schema_file = os.path.join(
+        os.path.dirname(__file__),
+        "..", "..", "..", "Help", "manual", "presets", "schema.json")
+with open(schema_file) as f:
+    schema = json.load(f)
+
+jsonschema.validate(contents, schema)