Tests: Add test preset tests

Tests/RunCMake/CMakeLists.txt

@@ -800,6 +800,7 @@ add_RunCMake_test(PrecompileHeaders -DCMAKE_C_COMPILER_ID=${CMAKE_C_COMPILER_ID}
 add_RunCMake_test("UnityBuild")
 add_RunCMake_test(CMakePresets -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA})
 add_RunCMake_test(CMakePresetsBuild -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA})
+add_RunCMake_test(CMakePresetsTest -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA})
 
 if(${CMAKE_GENERATOR} MATCHES "Make|Ninja")
   add_RunCMake_test(TransformDepfile)

Tests/RunCMake/CMakePresetsTest/CMakeLists.txt.in

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.19)
+project("@CASE_NAME@" NONE)
+include("@CASE_SOURCE_DIR@/@CASE_NAME@.cmake")

Tests/RunCMake/CMakePresetsTest/Good-indexFile.txt

@@ -0,0 +1 @@
+2,,2

Tests/RunCMake/CMakePresetsTest/Good-test-config-debug-stdout.txt

@@ -0,0 +1,5 @@
+Test project [^
+]*/Tests/RunCMake/CMakePresetsTest/Good/build/default
+    Start 6: debug-only
+.*
+100% tests passed, 0 tests failed out of 1

Tests/RunCMake/CMakePresetsTest/Good-test-config-release-stdout.txt

@@ -0,0 +1,5 @@
+Test project [^
+]*/Tests/RunCMake/CMakePresetsTest/Good/build/default
+    Start 6: release-only
+.*
+100% tests passed, 0 tests failed out of 1

Tests/RunCMake/CMakePresetsTest/Good-test-exclude-stdout.txt

@@ -0,0 +1,7 @@
+Test project [^
+]*/Tests/RunCMake/CMakePresetsTest/Good/build/default
+    Start 3: testc
+.*
+    Start 4: testd
+.*
+100% tests passed, 0 tests failed out of 2

Tests/RunCMake/CMakePresetsTest/Good-test-index-stdout.txt

@@ -0,0 +1,7 @@
+Test project [^
+]*/Tests/RunCMake/CMakePresetsTest/Good/build/default
+    Start 1: testa
+.*
+    Start 3: testc
+.*
+100% tests passed, 0 tests failed out of 2

Tests/RunCMake/CMakePresetsTest/Good-test-indexFile-stdout.txt

@@ -0,0 +1,7 @@
+Test project [^
+]*/Tests/RunCMake/CMakePresetsTest/Good/build/default
+    Start 2: testb
+.*
+    Start 4: testd
+.*
+100% tests passed, 0 tests failed out of 2

Tests/RunCMake/CMakePresetsTest/Good-test-noEnvironment-stdout.txt

@@ -0,0 +1,8 @@
+Test project [^
+]*/Tests/RunCMake/CMakePresetsTest/Good/build/default
+.*
+    Start 5: test-env
+.*
+5: TEST_ENV_REF=xx
+.*
+100% tests passed, 0 tests failed out of 1

Tests/RunCMake/CMakePresetsTest/Good-test-showOnly-stdout.txt

@@ -0,0 +1,8 @@
+Test project [^
+]*/Tests/RunCMake/CMakePresetsTest/Good/build/default
+  Test #1: testa
+  Test #2: testb
+  Test #3: testc
+  Test #4: testd
+
+Total Tests: 4

Tests/RunCMake/CMakePresetsTest/Good-withEnvironment-check.cmake

@@ -0,0 +1,8 @@
+include("${RunCMake_SOURCE_DIR}/../CMakePresetsBuild/TestVariable.cmake")
+
+test_environment_variable("TEST_ENV" "Environment variable")
+test_environment_variable("TEST_ENV_OVERRIDE" "Override")
+test_environment_variable("TEST_ENV_OVERRIDE_REF" "xOverridex")
+test_environment_variable("TEST_ENV_REF" "xEnvironment variablex")
+
+include("${RunCMake_SOURCE_DIR}/check.cmake")

Tests/RunCMake/CMakePresetsTest/Good.cmake

@@ -0,0 +1,27 @@
+enable_testing()
+add_test(testa ${CMAKE_COMMAND} -E echo testa)
+add_test(testb ${CMAKE_COMMAND} -E echo testb)
+add_test(testc ${CMAKE_COMMAND} -E echo testc)
+add_test(testd ${CMAKE_COMMAND} -E echo testd)
+
+set_tests_properties(testa testb testc testd
+                     PROPERTIES LABELS "echo")
+set_property(TEST testa testb
+             APPEND PROPERTY LABELS ab)
+set_property(TEST testb testc
+             APPEND PROPERTY LABELS bc)
+set_property(TEST testc testd
+             APPEND PROPERTY LABELS cd)
+
+add_test(test-env ${CMAKE_COMMAND} -E environment | sort)
+set_tests_properties(test-env PROPERTIES LABELS "env")
+
+add_test(NAME debug-only
+         COMMAND ${CMAKE_COMMAND} -E echo debug-only
+         CONFIGURATIONS Debug)
+add_test(NAME release-only
+         COMMAND ${CMAKE_COMMAND} -E echo release-only
+         CONFIGURATIONS Release)
+
+set_tests_properties(debug-only release-only
+                     PROPERTIES LABELS "config")

Tests/RunCMake/CMakePresetsTest/Good.json.in

@@ -0,0 +1,184 @@
+{
+    "version": 2,
+    "configurePresets": [
+        {
+            "name": "default",
+            "generator": "@RunCMake_GENERATOR@",
+            "binaryDir": "${sourceDir}/build/${presetName}",
+            "environment": {
+                "TEST_ENV": "Environment variable",
+                "TEST_ENV_OVERRIDE": "Overridden environment variable"
+            }
+        }
+    ],
+    "buildPresets": [
+        {
+            "name": "build-default-debug",
+            "configurePreset": "default",
+            "configuration": "Debug"
+        },
+        {
+            "name": "build-default-release",
+            "inherits": "build-default-debug",
+            "configuration": "Release"
+        }
+    ],
+    "testPresets": [
+        {
+            "name": "minimal",
+            "configurePreset": "default"
+        },
+        {
+            "name": "defaults",
+            "hidden": false,
+            "inherits": [],
+            "vendor": {},
+            "displayName": "",
+            "description": "",
+            "environment": {},
+            "configurePreset": "default",
+            "inheritConfigureEnvironment": true,
+            "configuration": "",
+            "overwriteConfigurationFile": [],
+            "output": {
+                "shortProgress": false,
+                "verbosity": "default",
+                "debug": false,
+                "outputOnFailure": false,
+                "quiet": false,
+                "outputLogFile": "",
+                "labelSummary": true,
+                "subprojectSummary": true
+                // "maxPassedTestOutputSize": 0
+                // "maxTestNameWidth": 0
+            },
+            "filter": {
+                "include": {
+                    "name": "",
+                    "label": "",
+                    "useUnion": false,
+                    "index": ""
+                    // "index": {
+                    //     "start": 0,
+                    //     "end": 0,
+                    //     "stride": 0
+                    // }
+                },
+                "exclude": {
+                    "name": "",
+                    "label": "",
+                    "fixtures": {
+                        "any": "",
+                        "setup": "",
+                        "cleanup": ""
+                    }
+                }
+            },
+            "execution": {
+                "stopOnFailure": false,
+                "enableFailover": false,
+                "jobs": 0,
+                "resourceSpecFile": "",
+                // "testLoad": 0,
+                "showOnly": "human",
+                "repeat": {
+                    "mode": "until-pass",
+                    "count": 1
+                },
+                "interactiveDebugging": false,
+                "scheduleRandom": false,
+                // "timeout": 0,
+                "noTestsAction": "default"
+            }
+        },
+        {
+            "name": "noEnvironment",
+            "configurePreset": "default",
+            "inheritConfigureEnvironment": false,
+            "environment": {
+                "TEST_ENV_REF": "x$env{TEST_ENV}x"
+            },
+            "filter": {
+                "include": {
+                    "name": "test-env"
+                }
+            },
+            "output": {
+                "verbosity": "verbose"
+            }
+        },
+        {
+            "name": "withEnvironment",
+            "inherits": "noEnvironment",
+            "inheritConfigureEnvironment": true,
+            "environment": {
+                "TEST_ENV_OVERRIDE": "Override",
+                "TEST_ENV_OVERRIDE_REF": "x$env{TEST_ENV_OVERRIDE}x",
+                "TEST_ENV_REF": "x$env{TEST_ENV}x"
+            }
+        },
+        {
+            "name": "config-debug",
+            "inherits": "minimal",
+            "configuration": "Debug",
+            "filter": {
+                "include": {
+                    "label": "config"
+                }
+            }
+        },
+        {
+            "name": "config-release",
+            "inherits": "minimal",
+            "configuration": "Release",
+            "filter": {
+                "include": {
+                    "label": "config"
+                }
+            }
+        },
+        {
+            "name": "exclude",
+            "inherits": "minimal",
+            "filter": {
+                "exclude": {
+                    "name": "test-env",
+                    "label": "(ab|config)"
+                }
+            }
+        },
+        {
+            "name": "index",
+            "inherits": "minimal",
+            "filter": {
+                "include": {
+                    "index": {
+                        "end": 4,
+                        "stride": 2
+                    }
+                }
+            }
+        },
+        {
+            "name": "indexFile",
+            "inherits": "minimal",
+            "filter": {
+                "include": {
+                    "index": "${sourceDir}/Good-indexFile.txt"
+                }
+            }
+        },
+        {
+            "name": "showOnly",
+            "inherits": "minimal",
+            "filter": {
+                "include": {
+                    "label": "echo"
+                }
+            },
+            "execution": {
+                "showOnly": "human"
+            }
+        }
+    ]
+}

Tests/RunCMake/CMakePresetsTest/Invalid-test-badConfigurePreset-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CMakePresetsTest/Invalid-test-badConfigurePreset-stderr.txt

@@ -0,0 +1,2 @@
+CMake Error: No such configure preset in [^
+]*/Tests/RunCMake/CMakePresetsTest/Invalid: "dne"

Tests/RunCMake/CMakePresetsTest/Invalid-test-hidden-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CMakePresetsTest/Invalid-test-hidden-stderr.txt

@@ -0,0 +1,2 @@
+CMake Error: Cannot use hidden test preset in [^
+]*/Tests/RunCMake/CMakePresetsTest/Invalid: "hidden"

Tests/RunCMake/CMakePresetsTest/Invalid-test-vendorMacro-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CMakePresetsTest/Invalid-test-vendorMacro-stderr.txt

@@ -0,0 +1 @@
+CMake Error: Could not evaluate test preset "vendorMacro": Invalid macro expansion

Tests/RunCMake/CMakePresetsTest/Invalid.json.in

@@ -0,0 +1,27 @@
+{
+    "version": 2,
+    "configurePresets": [
+        {
+            "name": "default",
+            "generator": "@RunCMake_GENERATOR@",
+            "binaryDir": "${sourceDir}/build/${presetName}"
+        }
+    ],
+    "testPresets": [
+        {
+            "name": "hidden",
+            "hidden": true
+        },
+        {
+            "name": "vendorMacro",
+            "configurePreset": "default",
+            "environment": {
+                "TEST": "$vendor{bad.TEST}"
+            }
+        },
+        {
+            "name": "badConfigurePreset",
+            "configurePreset": "dne"
+        }
+    ]
+}

Tests/RunCMake/CMakePresetsTest/ListPresets-test-x-stdout.txt

@@ -0,0 +1,12 @@
+Available test presets:
+
+  "minimal"
+  "defaults"
+  "noEnvironment"
+  "withEnvironment"
+  "config-debug"
+  "config-release"
+  "exclude"
+  "index"
+  "indexFile"
+  "showOnly"

Tests/RunCMake/CMakePresetsTest/NoConfigurePreset-test-noConfigurePreset-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CMakePresetsTest/NoConfigurePreset-test-noConfigurePreset-stderr.txt

@@ -0,0 +1,2 @@
+CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresetsTest/NoConfigurePreset: Invalid preset

Tests/RunCMake/CMakePresetsTest/NoConfigurePreset.json.in

@@ -0,0 +1,15 @@
+{
+    "version": 2,
+    "configurePresets": [
+        {
+            "name": "default",
+            "generator": "@RunCMake_GENERATOR@",
+            "binaryDir": "${sourceDir}/build/${presetName}"
+        }
+    ],
+    "testPresets": [
+        {
+            "name": "noConfigurePreset"
+        }
+    ]
+}

Tests/RunCMake/CMakePresetsTest/NoTestsAction-test-noTestsAction-result.txt

@@ -0,0 +1 @@
+8

Tests/RunCMake/CMakePresetsTest/NoTestsAction-test-noTestsAction-stderr.txt

@@ -0,0 +1 @@
+No tests were found!!!

Tests/RunCMake/CMakePresetsTest/NoTestsAction.json.in

@@ -0,0 +1,19 @@
+{
+    "version": 2,
+    "configurePresets": [
+        {
+            "name": "default",
+            "generator": "@RunCMake_GENERATOR@",
+            "binaryDir": "${sourceDir}/build/${presetName}"
+        }
+    ],
+    "testPresets": [
+        {
+            "name": "noTestsAction",
+            "configurePreset": "default",
+            "execution": {
+                "noTestsAction": "error"
+            }
+        }
+    ]
+}

Tests/RunCMake/CMakePresetsTest/PresetsUnsupported-test-x-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CMakePresetsTest/PresetsUnsupported-test-x-stderr.txt

@@ -0,0 +1,2 @@
+CMake Error: Could not read presets from [^
+]*Tests/RunCMake/CMakePresetsTest/PresetsUnsupported: File version must be 2 or higher for build and test preset support.

Tests/RunCMake/CMakePresetsTest/PresetsUnsupported.json.in

@@ -0,0 +1,7 @@
+{
+    "version": 1,
+    "configurePresets": [
+    ],
+    "testPresets": [
+    ]
+}

Tests/RunCMake/CMakePresetsTest/RunCMakeTest.cmake

@@ -0,0 +1,94 @@
+include(RunCMake)
+
+# Presets do not support legacy VS generator name architecture suffix.
+if(RunCMake_GENERATOR MATCHES "^(Visual Studio [0-9]+ [0-9]+) ")
+  set(RunCMake_GENERATOR "${CMAKE_MATCH_1}")
+endif()
+
+function(run_cmake_test_presets name CMakePresetsTest_CONFIGURE_PRESETS CMakePresetsTest_BUILD_PRESETS CMakePresetsTest_TEST_PRESETS)
+  set(RunCMake_TEST_SOURCE_DIR "${RunCMake_BINARY_DIR}/${name}")
+  set(RunCMake_TEST_BINARY_DIR "${RunCMake_TEST_SOURCE_DIR}/build")
+  set(RunCMake_TEST_COMMAND_WORKING_DIRECTORY "${RunCMake_TEST_SOURCE_DIR}")
+
+  set(RunCMake_TEST_NO_CLEAN TRUE)
+
+  file(REMOVE_RECURSE "${RunCMake_TEST_SOURCE_DIR}")
+  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+
+  set(CASE_NAME "${name}")
+  set(CASE_SOURCE_DIR "${RunCMake_SOURCE_DIR}")
+  configure_file("${RunCMake_SOURCE_DIR}/CMakeLists.txt.in" "${RunCMake_TEST_SOURCE_DIR}/CMakeLists.txt" @ONLY)
+
+  if(NOT CMakePresetsTest_FILE)
+    set(CMakePresetsTest_FILE "${RunCMake_SOURCE_DIR}/${name}.json.in")
+  endif()
+  if(EXISTS "${CMakePresetsTest_FILE}")
+    configure_file("${CMakePresetsTest_FILE}" "${RunCMake_TEST_SOURCE_DIR}/CMakePresets.json" @ONLY)
+  endif()
+
+  if(NOT CMakeUserPresets_FILE)
+    set(CMakeUserPresets_FILE "${RunCMake_SOURCE_DIR}/${name}User.json.in")
+  endif()
+  if(EXISTS "${CMakeUserPresets_FILE}")
+    configure_file("${CMakeUserPresets_FILE}" "${RunCMake_TEST_SOURCE_DIR}/CMakeUserPresets.json" @ONLY)
+  endif()
+
+  foreach(ASSET ${CMakePresetsTest_ASSETS})
+    configure_file("${RunCMake_SOURCE_DIR}/${ASSET}" "${RunCMake_TEST_SOURCE_DIR}" COPYONLY)
+  endforeach()
+
+  if (NOT CMakePresetsTest_NO_CONFIGURE)
+    foreach(CONFIGURE_PRESET ${CMakePresetsTest_CONFIGURE_PRESETS})
+      run_cmake_command("${name}-configure-${CONFIGURE_PRESET}"
+        "${CMAKE_COMMAND}" "--preset" "${CONFIGURE_PRESET}")
+    endforeach()
+  endif()
+
+  if (NOT CMakePresetsTest_NO_BUILD)
+    foreach(BUILD_PRESET ${CMakePresetsTest_BUILD_PRESETS})
+      run_cmake_command("${name}-build-${BUILD_PRESET}"
+        "${CMAKE_COMMAND}" "--build" "--preset" "${BUILD_PRESET}")
+    endforeach()
+  endif()
+
+  foreach(TEST_PRESET ${CMakePresetsTest_TEST_PRESETS})
+    if (EXISTS "${RunCMake_SOURCE_DIR}/${name}-test-${TEST_PRESET}-check.cmake")
+      set(RunCMake-check-file "${name}-test-${TEST_PRESET}-check.cmake")
+    else()
+      set(RunCMake-check-file "check.cmake")
+    endif()
+
+    run_cmake_command(${name}-test-${TEST_PRESET}
+      ${CMAKE_CTEST_COMMAND} "--preset" "${TEST_PRESET}" ${ARGN})
+  endforeach()
+endfunction()
+
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
+set(CMakePresetsTest_NO_BUILD 1)
+
+set(CMakePresetsTest_ASSETS "Good-indexFile.txt")
+set(GoodTestPresets
+  "minimal;defaults;noEnvironment;withEnvironment"
+  "config-debug;config-release"
+  "exclude;index;indexFile;showOnly")
+run_cmake_test_presets(Good
+                       "default"
+                       ""
+                       "${GoodTestPresets}")
+unset(CMakePresetsTest_ASSETS)
+
+set(CMakePresetsTest_NO_CONFIGURE 1)
+set(CMakePresetsTest_FILE "${RunCMake_SOURCE_DIR}/Good.json.in")
+run_cmake_test_presets(ListPresets "" "" "x" "--list-presets")
+unset(CMakePresetsTest_FILE)
+
+run_cmake_test_presets(NoConfigurePreset "" "" "noConfigurePreset")
+run_cmake_test_presets(NoTestsAction "default" "" "noTestsAction")
+run_cmake_test_presets(Invalid "" "" "hidden;vendorMacro;badConfigurePreset")
+
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 1)
+run_cmake_test_presets(PresetsUnsupported "" "" "x")
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
+set(CMakePresetsTest_NO_CONFIGURE 0)
+
+set(CMakePresetsTest_NO_BUILD 0)

Tests/RunCMake/CMakePresetsTest/check.cmake

@@ -0,0 +1,3 @@
+set(CMakePresets_VALIDATE_SCRIPT_PATH "${RunCMake_SOURCE_DIR}/../CMakePresets/validate_schema.py")
+include("${RunCMake_SOURCE_DIR}/../CMakePresets/validate_schema.cmake")
+include("${RunCMake_SOURCE_DIR}/../CMakePresets/check.cmake")