Autogen: Separate RunCMake.AutogenQtX tests

The RunCMake.AutogenQtX tests occasionally time out so separate
`RunCMake.AutogenQtX` tests.

Example: https://open.cdash.org/tests/1583599782

Tests/RunCMake/Autogen/RunCMakeTest.cmake

@@ -1,487 +0,0 @@
-include(RunCMake)
-
-run_cmake(NoQt)
-if (DEFINED with_qt_version)
-  set(RunCMake_TEST_OPTIONS
-    -Dwith_qt_version=${with_qt_version}
-    "-DQt${with_qt_version}_DIR:PATH=${Qt${with_qt_version}_DIR}"
-    "-DCMAKE_PREFIX_PATH:STRING=${CMAKE_PREFIX_PATH}"
-  )
-
-  run_cmake(QtInFunction)
-  run_cmake(QtInFunctionNested)
-  run_cmake(QtInFunctionProperty)
-
-  run_cmake(CMP0111-imported-target-full)
-  run_cmake(CMP0111-imported-target-libname)
-  run_cmake(CMP0111-imported-target-implib-only)
-
-  block()
-    set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/MocPredefs-build)
-    run_cmake(MocPredefs)
-    set(RunCMake_TEST_NO_CLEAN 1)
-    run_cmake_command(MocPredefs-build ${CMAKE_COMMAND} --build . --config Debug)
-  endblock()
-
-  # Detect information from the toolchain:
-  # - CMAKE_INCLUDE_FLAG_CXX
-  # - CMAKE_INCLUDE_SYSTEM_FLAG_CXX
-  run_cmake(Inspect)
-  include("${RunCMake_BINARY_DIR}/Inspect-build/info.cmake")
-
-  if(CMAKE_INCLUDE_SYSTEM_FLAG_CXX)
-    if(RunCMake_GENERATOR MATCHES "Visual Studio")
-      string(REGEX REPLACE "^-" "/" test_expect_stdout "${CMAKE_INCLUDE_SYSTEM_FLAG_CXX}")
-    else()
-      set(test_expect_stdout "-*${CMAKE_INCLUDE_SYSTEM_FLAG_CXX}")
-    endif()
-    string(APPEND test_expect_stdout " *(\"[^\"]*|([^ ]|\\ )*)[\\/]dummy_autogen[\\/]include")
-    if(RunCMake_GENERATOR_IS_MULTI_CONFIG)
-      string(APPEND test_expect_stdout "_Debug")
-    endif()
-
-    block()
-      set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/CMP0151-new-build)
-      run_cmake_with_options(CMP0151-new ${RunCMake_TEST_OPTIONS} -DCMAKE_POLICY_DEFAULT_CMP0151=NEW)
-      set(RunCMake_TEST_NO_CLEAN 1)
-      set(RunCMake_TEST_EXPECT_stdout "${test_expect_stdout}")
-      message(STATUS "RunCMake_TEST_EXPECT_stdout: ${RunCMake_TEST_EXPECT_stdout}")
-      run_cmake_command(CMP0151-new-build ${CMAKE_COMMAND} --build . --config Debug --verbose)
-    endblock()
-
-    block()
-      set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/AutogenUseSystemIncludeOn-build)
-      run_cmake_with_options(AutogenUseSystemIncludeOn ${RunCMake_TEST_OPTIONS} -DCMAKE_POLICY_DEFAULT_CMP0151=NEW)
-      set(RunCMake_TEST_NO_CLEAN 1)
-      set(RunCMake_TEST_EXPECT_stdout "${test_expect_stdout}")
-      message(STATUS "RunCMake_TEST_EXPECT_stdout: ${RunCMake_TEST_EXPECT_stdout}")
-      run_cmake_command(AutogenUseSystemIncludeOn ${CMAKE_COMMAND} --build . --config Debug --verbose)
-    endblock()
-  endif()
-
-  if(CMAKE_INCLUDE_FLAG_CXX)
-    if(RunCMake_GENERATOR MATCHES "Visual Studio")
-      string(REGEX REPLACE "^-" "/" test_expect_stdout "${CMAKE_INCLUDE_FLAG_CXX}")
-    else()
-      set(test_expect_stdout "-*${CMAKE_INCLUDE_FLAG_CXX}")
-    endif()
-    string(APPEND test_expect_stdout " *(\"[^\"]*|([^ ]|\\ )*)[\\/]dummy_autogen[\\/]include")
-    if(RunCMake_GENERATOR_IS_MULTI_CONFIG)
-      string(APPEND test_expect_stdout "_Debug")
-    endif()
-
-    block()
-      set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/CMP0151-old-build)
-      run_cmake_with_options(CMP0151-old ${RunCMake_TEST_OPTIONS} -DCMAKE_POLICY_DEFAULT_CMP0151=OLD)
-      set(RunCMake_TEST_NO_CLEAN 1)
-      set(RunCMake_TEST_EXPECT_stdout "${test_expect_stdout}")
-      message(STATUS "RunCMake_TEST_EXPECT_stdout: ${RunCMake_TEST_EXPECT_stdout}")
-      run_cmake_command(CMP0151-old-build ${CMAKE_COMMAND} --build . --config Debug --verbose)
-    endblock()
-
-    block()
-      set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/AutogenUseSystemIncludeOff-build)
-      run_cmake_with_options(AutogenUseSystemIncludeOff ${RunCMake_TEST_OPTIONS} -DCMAKE_POLICY_DEFAULT_CMP0151=NEW)
-      set(RunCMake_TEST_NO_CLEAN 1)
-      set(RunCMake_TEST_EXPECT_stdout "${test_expect_stdout}")
-      message(STATUS "RunCMake_TEST_EXPECT_stdout: ${RunCMake_TEST_EXPECT_stdout}")
-      run_cmake_command(AutogenUseSystemIncludeOff ${CMAKE_COMMAND} --build . --config Debug --verbose)
-    endblock()
-
-    if(RunCMake_GENERATOR MATCHES "Make|Ninja")
-      block()
-        set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/AutogenSkipLinting-build)
-        list(APPEND RunCMake_TEST_OPTIONS
-          "-DPSEUDO_CPPCHECK=${PSEUDO_CPPCHECK}"
-          "-DPSEUDO_CPPLINT=${PSEUDO_CPPLINT}"
-          "-DPSEUDO_IWYU=${PSEUDO_IWYU}"
-          "-DPSEUDO_TIDY=${PSEUDO_TIDY}")
-
-        run_cmake(AutogenSkipLinting)
-        set(RunCMake_TEST_NO_CLEAN 1)
-        run_cmake_command(AutogenSkipLinting-build ${CMAKE_COMMAND} --build . --config Debug --verbose)
-      endblock()
-    endif()
-  endif()
-
-  if(RunCMake_GENERATOR_IS_MULTI_CONFIG AND NOT RunCMake_GENERATOR MATCHES "Xcode")
-    block()
-      set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/MocGeneratedFile-build)
-      run_cmake(MocGeneratedFile)
-      set(RunCMake_TEST_NO_CLEAN 1)
-      run_cmake_command(MocGeneratedFile-build ${CMAKE_COMMAND} --build . --config Debug --verbose)
-    endblock()
-    if(RunCMake_GENERATOR MATCHES "Ninja Multi-Config")
-      block()
-        set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/MocGeneratedFile-cross-config-build)
-        list(APPEND RunCMake_TEST_OPTIONS -DCMAKE_CROSS_CONFIGS=all)
-        run_cmake(MocGeneratedFile)
-        set(RunCMake_TEST_NO_CLEAN 1)
-        run_cmake_command(MocGeneratedFile-cross-config-build ${CMAKE_COMMAND} --build . --config Release --target libgen:Debug)
-        run_cmake_command(MocGeneratedFile-cross-config-build ${CMAKE_COMMAND} --build . --config Debug --target libgen:Release)
-      endblock()
-    endif()
-  endif()
-
-  if(RunCMake_GENERATOR MATCHES "Make|Ninja")
-    block()
-      if(QtCore_VERSION VERSION_GREATER_EQUAL 5.15.0)
-        if (RunCMake_GENERATOR MATCHES "Ninja Multi-Config")
-          set(config_list Debug Release RelWithDebInfo)
-          set(use_better_graph_list ON OFF)
-        else()
-          set(config_list single-config)
-          set(use_better_graph_list OFF)
-        endif()
-
-        foreach(use_better_graph IN ITEMS ${use_better_graph_list})
-          foreach(config IN ITEMS ${config_list})
-            block()
-              if (config STREQUAL "single-config")
-                set(config_suffix "")
-              else()
-                set(config_path "_${config}")
-                if (use_better_graph)
-                  set(config_suffix "_${config}")
-                endif()
-              endif()
-
-              set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/QtAutoMocDeps${config_path}-build)
-              run_cmake_with_options(QtAutoMocDeps ${RunCMake_TEST_OPTIONS} -DCMAKE_AUTOGEN_BETTER_GRAPH_MULTI_CONFIG=${use_better_graph})
-              set(RunCMake_TEST_NO_CLEAN 1)
-              # Build the project.
-              if (config STREQUAL "single-config")
-                set(config_param "")
-              else()
-                set(config_param "--config ${config}")
-              endif()
-              run_cmake_command(QtAutoMocDeps-build ${CMAKE_COMMAND} --build . --verbose ${config_param})
-              # Touch just the library source file, which shouldn't cause a rerun of AUTOMOC
-              # for app_with_qt target.
-              file(TOUCH "${RunCMake_SOURCE_DIR}/simple_lib.cpp")
-              set(RunCMake_TEST_NOT_EXPECT_stdout "Automatic MOC for target app_with_qt|\
-Automatic MOC for target sub_exe_1|\
-Automatic MOC for target sub_exe_2")
-              set(RunCMake_TEST_VARIANT_DESCRIPTION "-Don't execute AUTOMOC for 'app_with_qt', 'sub_exe_1' and 'sub_exe_2'")
-              # Build and assert that AUTOMOC was not run for app_with_qt, sub_exe_1 and sub_exe_2.
-              run_cmake_command(QtAutoMocDeps-build ${CMAKE_COMMAND} --build . --verbose ${config_param})
-              unset(RunCMake_TEST_VARIANT_DESCRIPTION)
-              unset(RunCMake_TEST_NOT_EXPECT_stdout)
-
-              macro(check_file_exists file)
-                if (EXISTS "${file}")
-                  set(check_result "PASSED")
-                  set(message_type "STATUS")
-                else()
-                  set(check_result "FAILED")
-                  set(message_type "FATAL_ERROR")
-                endif()
-
-                message(${message_type} "QtAutoMocDeps-build-\"${file}\" was generated - ${check_result}")
-              endmacro()
-
-              check_file_exists("${RunCMake_TEST_BINARY_DIR}/app_with_qt_autogen/deps${config_suffix}")
-              check_file_exists("${RunCMake_TEST_BINARY_DIR}/QtSubDir1/sub_exe_1_autogen/deps${config_suffix}")
-              check_file_exists("${RunCMake_TEST_BINARY_DIR}/QtSubDir2/sub_exe_2_autogen/deps${config_suffix}")
-
-              check_file_exists("${RunCMake_TEST_BINARY_DIR}/app_with_qt_autogen/timestamp${config_suffix}")
-              check_file_exists("${RunCMake_TEST_BINARY_DIR}/QtSubDir1/sub_exe_1_autogen/timestamp${config_suffix}")
-              check_file_exists("${RunCMake_TEST_BINARY_DIR}/QtSubDir2/sub_exe_2_autogen/timestamp${config_suffix}")
-
-              # Touch a header file to make sure an automoc dependency cycle is not introduced.
-              file(TOUCH "${RunCMake_SOURCE_DIR}/MyWindow.h")
-              set(RunCMake_TEST_VARIANT_DESCRIPTION "-First build after touch to detect dependency cycle")
-              run_cmake_command(QtAutoMocDeps-build ${CMAKE_COMMAND} --build . --verbose)
-              # Need to run a second time to hit the dependency cycle.
-              set(RunCMake_TEST_VARIANT_DESCRIPTION "-Don't hit dependency cycle")
-              run_cmake_command(QtAutoMocDeps-build ${CMAKE_COMMAND} --build . --verbose)
-            endblock()
-          endforeach()
-        endforeach()
-      endif()
-    endblock()
-  endif()
-
-  function(run_make_program dir)
-    execute_process(
-      COMMAND "${RunCMake_MAKE_PROGRAM}" ${ARGN}
-      WORKING_DIRECTORY "${dir}"
-      OUTPUT_VARIABLE make_program_stdout
-      ERROR_VARIABLE make_program_stderr
-      RESULT_VARIABLE make_program_result
-      )
-      if (NOT DEFINED RunMakeProgram_expected_result)
-        set(RunMakeProgram_expected_result 0)
-      endif()
-      if(NOT "${make_program_result}" MATCHES "${RunMakeProgram_expected_result}")
-        message(STATUS "
-============ beginning of ${RunCMake_MAKE_PROGRAM}'s stdout ============
-${make_program_stdout}
-=============== end of ${RunCMake_MAKE_PROGRAM}'s stdout ===============
-")
-        message(STATUS "
-============ beginning of ${RunCMake_MAKE_PROGRAM}'s stderr ============
-${make_program_stderr}
-=============== end of ${RunCMake_MAKE_PROGRAM}'s stderr ===============
-")
-        message(FATAL_ERROR
-                "top ${RunCMake_MAKE_PROGRAM} build failed exited with status ${make_program_result}")
-    endif()
-    set(make_program_stdout "${make_program_stdout}" PARENT_SCOPE)
-  endfunction(run_make_program)
-
-  function(count_substring STRING SUBSTRING COUNT_VAR)
-      string(LENGTH "${STRING}" STRING_LENGTH)
-      string(LENGTH "${SUBSTRING}" SUBSTRING_LENGTH)
-      if (SUBSTRING_LENGTH EQUAL 0)
-          message(FATAL_ERROR "SUBSTRING_LENGTH is 0")
-      endif()
-
-      if (STRING_LENGTH EQUAL 0)
-          message(FATAL_ERROR "STRING_LENGTH is 0")
-      endif()
-
-      if (STRING_LENGTH LESS SUBSTRING_LENGTH)
-          message(FATAL_ERROR "STRING_LENGTH is less than SUBSTRING_LENGTH")
-      endif()
-
-      set(COUNT 0)
-      string(FIND "${STRING}" "${SUBSTRING}" SUBSTRING_START)
-      while(SUBSTRING_START GREATER_EQUAL 0)
-          math(EXPR COUNT "${COUNT} + 1")
-          math(EXPR SUBSTRING_START "${SUBSTRING_START} + ${SUBSTRING_LENGTH}")
-          string(SUBSTRING "${STRING}" ${SUBSTRING_START} -1 STRING)
-          string(FIND "${STRING}" "${SUBSTRING}" SUBSTRING_START)
-      endwhile()
-
-      set(${COUNT_VAR} ${COUNT} PARENT_SCOPE)
-  endfunction()
-
-  function(expect_only_once make_program_stdout expected_output test_name)
-    count_substring("${make_program_stdout}" "${expected_output}" count)
-    if(NOT count EQUAL 1)
-      message(STATUS "${test_name}-expect_only_once - FAILED")
-      message(FATAL_ERROR "Expected to find ${expected_output} exactly once in ${make_program_stdout} but found ${count} occurrences of ${expected_output}")
-    else()
-      message(STATUS "${test_name}-expect_only_once - PASSED")
-    endif()
-  endfunction()
-
-  function(expect_n_times string_to_check expected_output expected_count test_name)
-    count_substring("${string_to_check}" "${expected_output}" count)
-    if(NOT count EQUAL ${expected_count})
-      message(STATUS "${test_name}-expect_${expected_count}_times - FAILED")
-      message(FATAL_ERROR "Expected to find ${expected_output} exactly ${expected_count} times in ${string_to_check} but found ${count} occurrences of ${expected_output}")
-    else()
-      message(STATUS "${test_name}-expect_${expected_count}_times - PASSED")
-    endif()
-  endfunction()
-
-  function(not_expect make_program_stdout unexpected_output test_name)
-    count_substring("${make_program_stdout}" "${unexpected_output}" count)
-    if(NOT count EQUAL 0)
-      message(STATUS "${test_name}-not_expect - FAILED")
-      message(FATAL_ERROR "Expected to find ${unexpected_output} exactly 0 times in ${make_program_stdout} but found ${count} occurrences of ${unexpected_output}")
-    else()
-      message(STATUS "${test_name}-not_expect - PASSED")
-    endif()
-  endfunction()
-
-  if (QtCore_VERSION VERSION_GREATER_EQUAL 5.15.0)
-    foreach(exe IN ITEMS Moc Uic Rcc)
-      if(RunCMake_GENERATOR MATCHES "Ninja Multi-Config")
-        block()
-          set(RunCMake_TEST_VARIANT_DESCRIPTION "-CMake-configure")
-          set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/Auto${exe}ExecutableConfig-multi-config-build)
-          run_cmake_with_options(Auto${exe}ExecutableConfig ${RunCMake_TEST_OPTIONS} -DCMAKE_AUTOGEN_VERBOSE=ON -DCMAKE_AUTOGEN_BETTER_GRAPH_MULTI_CONFIG=ON)
-          unset(RunCMake_TEST_VARIANT_DESCRIPTION)
-          set(RunCMake_TEST_NO_CLEAN 1)
-          foreach(config IN ITEMS Debug Release RelWithDebInfo)
-            block()
-              set(RunCMake_TEST_EXPECT_stdout ".*running_exe_${config}*")
-              set(RunCMake_TEST_VARIANT_DESCRIPTION "-${config}-expect_running_exe_${config}")
-              run_cmake_command(Auto${exe}ExecutableConfig-multi-config-build ${CMAKE_COMMAND} --build . --config ${config})
-            endblock()
-          endforeach()
-          set(RunCMake_TEST_EXPECT_stdout "ninja: no work to do")
-          foreach(config IN ITEMS Debug Release RelWithDebInfo)
-            block()
-            set(RunCMake_TEST_VARIANT_DESCRIPTION "-${config}-expect_no_work_to_do")
-              run_cmake_command(Auto${exe}ExecutableConfig-multi-config-build ${CMAKE_COMMAND} --build . --config ${config})
-            endblock()
-          endforeach()
-        endblock()
-        block()
-          set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/Auto${exe}ExecutableConfig-build)
-          run_cmake_with_options(Auto${exe}ExecutableConfig ${RunCMake_TEST_OPTIONS} -DCMAKE_AUTOGEN_VERBOSE=ON -DCMAKE_AUTOGEN_BETTER_GRAPH_MULTI_CONFIG=ON)
-          foreach(config IN ITEMS Debug Release RelWithDebInfo)
-            block()
-              run_make_program(${RunCMake_TEST_BINARY_DIR} --verbose -f build-${config}.ninja)
-
-              set(expected_output "running_exe_${config}")
-              expect_only_once("${make_program_stdout}" "${expected_output}" "Auto${exe}ExecutableConfig-${config}-${expected_output}")
-
-              foreach(sub_config IN ITEMS Debug Release RelWithDebInfo)
-                if(NOT sub_config STREQUAL config)
-                  set(unexpected_output "running_exe_${sub_config}")
-                  not_expect("${make_program_stdout}" "${unexpected_output}" "Auto${exe}ExecutableConfig-${config}-${unexpected_output}")
-                endif()
-              endforeach()
-
-              if (exe STREQUAL "Moc" OR exe STREQUAL "Uic")
-                set(expected_output "cmake_autogen")
-              else()
-                set(expected_output "cmake_autorcc")
-              endif()
-              expect_only_once("${make_program_stdout}" "${expected_output}" "Auto${exe}ExecutableConfig-${config}-${expected_output}")
-            endblock()
-          endforeach()
-        endblock()
-        block()
-          foreach(ninja_config IN ITEMS Debug Release RelWithDebInfo)
-            foreach(target_config IN ITEMS Debug Release RelWithDebInfo)
-              block()
-                set(TEST_SUFFIX "-CrossConfig-${ninja_config}-${target_config}")
-                set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/Auto${exe}ExecutableConfig${TEST_SUFFIX}-build)
-                set(RunCMake_TEST_VARIANT_DESCRIPTION ${TEST_SUFFIX})
-                run_cmake_with_options(Auto${exe}ExecutableConfig ${RunCMake_TEST_OPTIONS} -DCMAKE_CROSS_CONFIGS=all -DCMAKE_DEFAULT_BUILD_TYPE=${ninja_config} -DCMAKE_AUTOGEN_BETTER_GRAPH_MULTI_CONFIG=ON)
-                unset(RunCMake_TEST_VARIANT_DESCRIPTION)
-
-                run_make_program(${RunCMake_TEST_BINARY_DIR} --verbose -f build-${ninja_config}.ninja dummy:${target_config})
-
-                set(expected_output "running_exe_${ninja_config}")
-                expect_only_once("${make_program_stdout}" "${expected_output}" "Auto${exe}ExecutableConfig${TEST_SUFFIX}-${expected_output}")
-
-                foreach(sub_config IN ITEMS Debug Release RelWithDebInfo)
-                  if(NOT sub_config STREQUAL ninja_config)
-                    set(unexpected_output "running_exe_${sub_config}")
-                    not_expect("${make_program_stdout}" "${unexpected_output}" "Auto${exe}ExecutableConfig${TEST_SUFFIX}-${unexpected_output}")
-                  endif()
-                endforeach()
-
-                if (exe STREQUAL "Moc" OR exe STREQUAL "Uic")
-                  set(expected_output "cmake_autogen")
-                else()
-                  set(expected_output "cmake_autorcc")
-                endif()
-                expect_only_once("${make_program_stdout}" "${expected_output}" "Auto${exe}ExecutableConfig${TEST_SUFFIX}-${expected_output}")
-              endblock()
-            endforeach()
-          endforeach()
-        endblock()
-        block()
-          foreach(ninja_config IN ITEMS Debug Release RelWithDebInfo)
-            set(TEST_SUFFIX "-CrossConfig-${ninja_config}-all-all")
-            set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/Auto${exe}ExecutableConfig${TEST_SUFFIX}-build)
-            set(RunCMake_TEST_VARIANT_DESCRIPTION ${TEST_SUFFIX})
-            run_cmake_with_options(Auto${exe}ExecutableConfig ${RunCMake_TEST_OPTIONS} -DCMAKE_CROSS_CONFIGS=all -DCMAKE_AUTOGEN_BETTER_GRAPH_MULTI_CONFIG=ON)
-            unset(RunCMake_TEST_VARIANT_DESCRIPTION)
-            run_make_program(${RunCMake_TEST_BINARY_DIR} --verbose -f build-${ninja_config}.ninja all:all)
-          endforeach()
-        endblock()
-      elseif (RunCMake_GENERATOR MATCHES "Ninja|Make")
-        block()
-          set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/Auto${exe}ExecutableConfig-build)
-          foreach(config IN ITEMS Debug Release RelWithDebInfo)
-            block()
-              set(RunCMake_TEST_VARIANT_DESCRIPTION "-${config}")
-              run_cmake_with_options(Auto${exe}ExecutableConfig ${RunCMake_TEST_OPTIONS} -DCMAKE_BUILD_TYPE=${config} -DCMAKE_AUTOGEN_VERBOSE=ON)
-              unset(RunCMake_TEST_VARIANT_DESCRIPTION)
-              set(RunCMake_TEST_NO_CLEAN 1)
-              set(RunCMake_TEST_EXPECT_stdout ".*running_exe_${config}*")
-              run_cmake_command(Auto${exe}ExecutableConfig-${config}-build ${CMAKE_COMMAND} --build .)
-            endblock()
-          endforeach()
-        endblock()
-      endif()
-    endforeach()
-  endif()
-
-  # Visual Studio specific dependency tests
-  if (RunCMake_GENERATOR MATCHES "Visual Studio")
-      foreach(exe IN ITEMS Moc Uic Rcc)
-          block()
-            set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/${exe}Example-build)
-            set(RunCMake_TEST_VARIANT_DESCRIPTION "-CMake-configure")
-            run_cmake_with_options(${exe}Example ${RunCMake_TEST_OPTIONS} -DCMAKE_AUTOGEN_VERBOSE=ON)
-            unset(RunCMake_TEST_VARIANT_DESCRIPTION)
-            set(RunCMake_TEST_NO_CLEAN 1)
-            foreach(config IN ITEMS Debug Release RelWithDebInfo)
-              block()
-                set(RunCMake_TEST_VARIANT_DESCRIPTION "-${config}-first-build")
-                run_cmake_command(${exe}Example-build ${CMAKE_COMMAND} --build . --config ${config})
-              endblock()
-            endforeach()
-            foreach(config IN ITEMS Debug Release RelWithDebInfo)
-              block()
-                if (exe STREQUAL "Moc" OR exe STREQUAL "Uic")
-                  set(RunCMake_TEST_NOT_EXPECT_stdout "Auto${exe}")
-                  set(not_expect_descripton "Auto${exe}")
-                else ()
-                  set(RunCMake_TEST_NOT_EXPECT_stdout "Auto${exe}")
-                  set(not_expect_descripton "Auto${exe}")
-                endif()
-                set(RunCMake_TEST_VARIANT_DESCRIPTION "-second-build-${config}_expect_no_${not_expect_descripton}")
-                run_cmake_command(${exe}Example-build ${CMAKE_COMMAND} --build . --config ${config})
-              endblock()
-            endforeach()
-          endblock()
-      endforeach()
-  endif()
-
-  if (RunCMake_GENERATOR MATCHES "Xcode")
-    foreach(exe IN ITEMS Moc Uic Rcc)
-      block()
-        set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/${exe}Example-build)
-        set(RunCMake_TEST_VARIANT_DESCRIPTION "-CMake-configure")
-        set(RunCMake_TEST_EXPECT_stderr ".*")
-        run_cmake_with_options(${exe}Example ${RunCMake_TEST_OPTIONS} -DCMAKE_AUTOGEN_VERBOSE=ON)
-        set(RunCMake_TEST_NO_CLEAN 1)
-        set(RunCMake_MAKE_PROGRAM ${CMAKE_COMMAND})
-        run_make_program(${RunCMake_TEST_BINARY_DIR}  --build . --config Debug)
-        if (exe STREQUAL "Moc")
-          set(expected_count 16)
-        elseif (exe STREQUAL "Uic")
-          set(expected_count 4)
-        else()
-          set(expected_count 12)
-        endif()
-        expect_n_times("${make_program_stdout}" "Auto${exe}:" ${expected_count} "${exe}Example-build-Auto${exe}")
-        expect_n_times("${make_program_stdout}" "Auto${exe}:" ${expected_count} "${exe}Example-build-Auto${exe}")
-
-        if (exe STREQUAL "Moc" OR exe STREQUAL "Uic")
-          expect_n_times("${make_program_stdout}" "AutoGen:" 20 "${exe}Example-build-AutoGen:")
-        endif()
-
-        foreach(config IN ITEMS Debug Release RelWithDebInfo)
-          block()
-            run_make_program(${RunCMake_TEST_BINARY_DIR} --build . --config ${config})
-            not_expect("${make_program_stdout}" "Auto${exe}" "${exe}Example-${config}_Auto${exe}")
-            not_expect("${make_program_stdout}" "AutoGen:" "${exe}Example-${config}_AutoGen")
-          endblock()
-        endforeach()
-      endblock()
-    endforeach()
-  endif()
-
-  if (QtCore_VERSION VERSION_GREATER_EQUAL 6)
-    if (RunCMake_GENERATOR MATCHES "Make|Ninja")
-      foreach(value IN ITEMS ON OFF)
-        block()
-          set(RunCMake_TEST_BINARY_DIR
-            ${RunCMake_BINARY_DIR}/RccNoZTSD-${value}-build)
-          run_cmake_with_options(RccExample ${RunCMake_TEST_OPTIONS}
-            -DCMAKE_AUTOGEN_VERBOSE=ON -DZSTD_VALUE=${value})
-          if (value STREQUAL "OFF")
-            set(RunCMake_TEST_EXPECT_stdout "--no-zstd")
-          else()
-            set(RunCMake_TEST_NOT_EXPECT_stdout "--no-zstd")
-          endif()
-          set(RunCMake_TEST_NO_CLEAN 1)
-          run_cmake_command(RccNoZTSD-${value}-build ${CMAKE_COMMAND}
-            --build . --config Debug)
-        endblock()
-      endforeach()
-    endif()
-  endif()
-endif ()

Tests/RunCMake/Autogen/exe.cpp

@@ -1,4 +0,0 @@
-int main()
-{
-  return 0;
-}

Tests/RunCMake/Autogen_1/RunCMakeTest.cmake

@@ -0,0 +1,125 @@
+include(RunCMake)
+
+run_cmake(NoQt)
+if (DEFINED with_qt_version)
+  set(RunCMake_TEST_OPTIONS
+    -Dwith_qt_version=${with_qt_version}
+    "-DQt${with_qt_version}_DIR:PATH=${Qt${with_qt_version}_DIR}"
+    "-DCMAKE_PREFIX_PATH:STRING=${CMAKE_PREFIX_PATH}"
+  )
+
+  run_cmake(QtInFunction)
+  run_cmake(QtInFunctionNested)
+  run_cmake(QtInFunctionProperty)
+
+  run_cmake(CMP0111-imported-target-full)
+  run_cmake(CMP0111-imported-target-libname)
+  run_cmake(CMP0111-imported-target-implib-only)
+
+  block()
+    set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/MocPredefs-build)
+    run_cmake(MocPredefs)
+    set(RunCMake_TEST_NO_CLEAN 1)
+    run_cmake_command(MocPredefs-build ${CMAKE_COMMAND} --build . --config Debug)
+  endblock()
+
+  # Detect information from the toolchain:
+  # - CMAKE_INCLUDE_FLAG_CXX
+  # - CMAKE_INCLUDE_SYSTEM_FLAG_CXX
+  run_cmake(Inspect)
+  include("${RunCMake_BINARY_DIR}/Inspect-build/info.cmake")
+
+  if(CMAKE_INCLUDE_SYSTEM_FLAG_CXX)
+    if(RunCMake_GENERATOR MATCHES "Visual Studio")
+      string(REGEX REPLACE "^-" "/" test_expect_stdout "${CMAKE_INCLUDE_SYSTEM_FLAG_CXX}")
+    else()
+      set(test_expect_stdout "-*${CMAKE_INCLUDE_SYSTEM_FLAG_CXX}")
+    endif()
+    string(APPEND test_expect_stdout " *(\"[^\"]*|([^ ]|\\ )*)[\\/]dummy_autogen[\\/]include")
+    if(RunCMake_GENERATOR_IS_MULTI_CONFIG)
+      string(APPEND test_expect_stdout "_Debug")
+    endif()
+
+    block()
+      set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/CMP0151-new-build)
+      run_cmake_with_options(CMP0151-new ${RunCMake_TEST_OPTIONS} -DCMAKE_POLICY_DEFAULT_CMP0151=NEW)
+      set(RunCMake_TEST_NO_CLEAN 1)
+      set(RunCMake_TEST_EXPECT_stdout "${test_expect_stdout}")
+      run_cmake_command(CMP0151-new-build ${CMAKE_COMMAND} --build . --config Debug --verbose)
+    endblock()
+
+    block()
+      set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/AutogenUseSystemIncludeOn-build)
+      run_cmake_with_options(AutogenUseSystemIncludeOn ${RunCMake_TEST_OPTIONS} -DCMAKE_POLICY_DEFAULT_CMP0151=NEW)
+      set(RunCMake_TEST_NO_CLEAN 1)
+      set(RunCMake_TEST_EXPECT_stdout "${test_expect_stdout}")
+      run_cmake_command(AutogenUseSystemIncludeOn ${CMAKE_COMMAND} --build . --config Debug --verbose)
+    endblock()
+  endif()
+
+  if(CMAKE_INCLUDE_FLAG_CXX)
+    if(RunCMake_GENERATOR MATCHES "Visual Studio")
+      string(REGEX REPLACE "^-" "/" test_expect_stdout "${CMAKE_INCLUDE_FLAG_CXX}")
+    else()
+      set(test_expect_stdout "-*${CMAKE_INCLUDE_FLAG_CXX}")
+    endif()
+    string(APPEND test_expect_stdout " *(\"[^\"]*|([^ ]|\\ )*)[\\/]dummy_autogen[\\/]include")
+    if(RunCMake_GENERATOR_IS_MULTI_CONFIG)
+      string(APPEND test_expect_stdout "_Debug")
+    endif()
+
+    block()
+      set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/CMP0151-old-build)
+      run_cmake_with_options(CMP0151-old ${RunCMake_TEST_OPTIONS} -DCMAKE_POLICY_DEFAULT_CMP0151=OLD)
+      set(RunCMake_TEST_NO_CLEAN 1)
+      set(RunCMake_TEST_EXPECT_stdout "${test_expect_stdout}")
+      run_cmake_command(CMP0151-old-build ${CMAKE_COMMAND} --build . --config Debug --verbose)
+    endblock()
+
+    block()
+      set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/AutogenUseSystemIncludeOff-build)
+      run_cmake_with_options(AutogenUseSystemIncludeOff ${RunCMake_TEST_OPTIONS} -DCMAKE_POLICY_DEFAULT_CMP0151=NEW)
+      set(RunCMake_TEST_NO_CLEAN 1)
+      set(RunCMake_TEST_EXPECT_stdout "${test_expect_stdout}")
+      run_cmake_command(AutogenUseSystemIncludeOff ${CMAKE_COMMAND} --build . --config Debug --verbose)
+    endblock()
+
+    if(RunCMake_GENERATOR MATCHES "Make|Ninja")
+      block()
+        set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/AutogenSkipLinting-build)
+        list(APPEND RunCMake_TEST_OPTIONS
+          "-DPSEUDO_CPPCHECK=${PSEUDO_CPPCHECK}"
+          "-DPSEUDO_CPPLINT=${PSEUDO_CPPLINT}"
+          "-DPSEUDO_IWYU=${PSEUDO_IWYU}"
+          "-DPSEUDO_TIDY=${PSEUDO_TIDY}")
+
+        run_cmake(AutogenSkipLinting)
+        set(RunCMake_TEST_NO_CLEAN 1)
+        run_cmake_command(AutogenSkipLinting-build ${CMAKE_COMMAND} --build . --config Debug --verbose)
+      endblock()
+    endif()
+  endif()
+
+  if(RunCMake_GENERATOR_IS_MULTI_CONFIG AND NOT RunCMake_GENERATOR MATCHES "Xcode")
+    block()
+      set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/MocGeneratedFile-build)
+      run_cmake(MocGeneratedFile)
+      set(RunCMake_TEST_NO_CLEAN 1)
+      run_cmake_command(MocGeneratedFile-build ${CMAKE_COMMAND} --build . --config Debug --verbose)
+    endblock()
+    if(RunCMake_GENERATOR MATCHES "Ninja Multi-Config")
+      block()
+        set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/MocGeneratedFile-cross-config-build)
+        list(APPEND RunCMake_TEST_OPTIONS -DCMAKE_CROSS_CONFIGS=all)
+        set(RunCMake_TEST_VARIANT_DESCRIPTION "-CMAKE_CROSS_CONFIGS-all")
+        run_cmake(MocGeneratedFile)
+        set(RunCMake_TEST_NO_CLEAN 1)
+        set(RunCMake_TEST_VARIANT_DESCRIPTION "-Release")
+        run_cmake_command(MocGeneratedFile-cross-config-build ${CMAKE_COMMAND} --build . --config Release --target libgen:Debug)
+        set(RunCMake_TEST_VARIANT_DESCRIPTION "-Debug")
+        run_cmake_command(MocGeneratedFile-cross-config-build ${CMAKE_COMMAND} --build . --config Debug --target libgen:Release)
+        unset(RunCMake_TEST_VARIANT_DESCRIPTION)
+      endblock()
+    endif()
+  endif()
+endif ()

Tests/RunCMake/Autogen_2/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.16)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)

Tests/RunCMake/Autogen_2/RunCMakeTest.cmake

@@ -0,0 +1,87 @@
+include(RunCMake)
+
+if (DEFINED with_qt_version)
+  set(RunCMake_TEST_OPTIONS
+    -Dwith_qt_version=${with_qt_version}
+    "-DQt${with_qt_version}_DIR:PATH=${Qt${with_qt_version}_DIR}"
+    "-DCMAKE_PREFIX_PATH:STRING=${CMAKE_PREFIX_PATH}"
+  )
+  if(RunCMake_GENERATOR MATCHES "Make|Ninja")
+    block()
+      if(QtCore_VERSION VERSION_GREATER_EQUAL 5.15.0)
+        if (RunCMake_GENERATOR MATCHES "Ninja Multi-Config")
+          set(config_list Debug Release RelWithDebInfo)
+          set(use_better_graph_list ON OFF)
+        else()
+          set(config_list single-config)
+          set(use_better_graph_list OFF)
+        endif()
+
+        foreach(use_better_graph IN ITEMS ${use_better_graph_list})
+          foreach(config IN ITEMS ${config_list})
+            block()
+              if (config STREQUAL "single-config")
+                set(config_suffix "")
+              else()
+                set(config_path "_${config}")
+                if (use_better_graph)
+                  set(config_suffix "_${config}")
+                endif()
+              endif()
+
+              set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/QtAutoMocDeps${config_path}-build)
+              run_cmake_with_options(QtAutoMocDeps ${RunCMake_TEST_OPTIONS} -DCMAKE_AUTOGEN_BETTER_GRAPH_MULTI_CONFIG=${use_better_graph})
+              set(RunCMake_TEST_NO_CLEAN 1)
+              # Build the project.
+              if (config STREQUAL "single-config")
+                set(config_param "")
+              else()
+                set(config_param "--config ${config}")
+              endif()
+              run_cmake_command(QtAutoMocDeps-build ${CMAKE_COMMAND} --build . --verbose ${config_param})
+              # Touch just the library source file, which shouldn't cause a rerun of AUTOMOC
+              # for app_with_qt target.
+              file(TOUCH "${RunCMake_SOURCE_DIR}/simple_lib.cpp")
+              set(RunCMake_TEST_NOT_EXPECT_stdout "Automatic MOC for target app_with_qt|\
+Automatic MOC for target sub_exe_1|\
+Automatic MOC for target sub_exe_2")
+              set(RunCMake_TEST_VARIANT_DESCRIPTION "-Don't execute AUTOMOC for 'app_with_qt', 'sub_exe_1' and 'sub_exe_2'")
+              # Build and assert that AUTOMOC was not run for app_with_qt, sub_exe_1 and sub_exe_2.
+              run_cmake_command(QtAutoMocDeps-build ${CMAKE_COMMAND} --build . --verbose ${config_param})
+              unset(RunCMake_TEST_VARIANT_DESCRIPTION)
+              unset(RunCMake_TEST_NOT_EXPECT_stdout)
+
+              macro(check_file_exists file)
+                if (EXISTS "${file}")
+                  set(check_result "PASSED")
+                  set(message_type "STATUS")
+                else()
+                  set(check_result "FAILED")
+                  set(message_type "FATAL_ERROR")
+                endif()
+
+                message(${message_type} "QtAutoMocDeps-build-\"${file}\" was generated - ${check_result}")
+              endmacro()
+
+              check_file_exists("${RunCMake_TEST_BINARY_DIR}/app_with_qt_autogen/deps${config_suffix}")
+              check_file_exists("${RunCMake_TEST_BINARY_DIR}/QtSubDir1/sub_exe_1_autogen/deps${config_suffix}")
+              check_file_exists("${RunCMake_TEST_BINARY_DIR}/QtSubDir2/sub_exe_2_autogen/deps${config_suffix}")
+
+              check_file_exists("${RunCMake_TEST_BINARY_DIR}/app_with_qt_autogen/timestamp${config_suffix}")
+              check_file_exists("${RunCMake_TEST_BINARY_DIR}/QtSubDir1/sub_exe_1_autogen/timestamp${config_suffix}")
+              check_file_exists("${RunCMake_TEST_BINARY_DIR}/QtSubDir2/sub_exe_2_autogen/timestamp${config_suffix}")
+
+              # Touch a header file to make sure an automoc dependency cycle is not introduced.
+              file(TOUCH "${RunCMake_SOURCE_DIR}/MyWindow.h")
+              set(RunCMake_TEST_VARIANT_DESCRIPTION "-First build after touch to detect dependency cycle")
+              run_cmake_command(QtAutoMocDeps-build ${CMAKE_COMMAND} --build . --verbose)
+              # Need to run a second time to hit the dependency cycle.
+              set(RunCMake_TEST_VARIANT_DESCRIPTION "-Don't hit dependency cycle")
+              run_cmake_command(QtAutoMocDeps-build ${CMAKE_COMMAND} --build . --verbose)
+            endblock()
+          endforeach()
+        endforeach()
+      endif()
+    endblock()
+  endif()
+endif ()

Tests/RunCMake/Autogen_3/AutoMocExecutableConfig.cmake

@@ -7,9 +7,9 @@ endif()
 get_target_property(moc_location Qt${with_qt_version}::moc IMPORTED_LOCATION)
 set_target_properties(dummy PROPERTIES AUTOMOC_MOC_OPTIONS "EXE_PATH=${moc_location}")
 
-add_executable(mymoc $<$<CONFIG:Debug>:exe_debug.cpp>
-                     $<$<CONFIG:Release>:exe_release.cpp>
-                     $<$<CONFIG:RelWithDebInfo>:exe_relwithdebinfo.cpp>
+add_executable(mymoc $<$<CONFIG:Debug>:../Autogen_common/exe_debug.cpp>
+                     $<$<CONFIG:Release>:../Autogen_common/exe_release.cpp>
+                     $<$<CONFIG:RelWithDebInfo>:../Autogen_common/exe_relwithdebinfo.cpp>
 )
 
 set_target_properties(dummy PROPERTIES AUTOMOC_EXECUTABLE $<TARGET_FILE:mymoc>)

Tests/RunCMake/Autogen_3/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.16)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)

Tests/RunCMake/Autogen_3/MocExample.cmake

@@ -3,7 +3,7 @@ enable_language(CXX)
 set(CMAKE_CXX_STANDARD 11)
 find_package(Qt${with_qt_version} REQUIRED COMPONENTS Core Widgets Gui)
 
-add_library(dummy STATIC example.cpp)
+add_library(dummy STATIC ../Autogen_common/example.cpp)
 target_link_libraries(dummy Qt${with_qt_version}::Core
                             Qt${with_qt_version}::Widgets
                             Qt${with_qt_version}::Gui)

Tests/RunCMake/Autogen_3/RunCMakeTest.cmake

@@ -0,0 +1,11 @@
+include(RunCMake)
+include(Autogen_common/utils)
+
+if (DEFINED with_qt_version)
+  set(RunCMake_TEST_OPTIONS
+    -Dwith_qt_version=${with_qt_version}
+    "-DQt${with_qt_version}_DIR:PATH=${Qt${with_qt_version}_DIR}"
+    "-DCMAKE_PREFIX_PATH:STRING=${CMAKE_PREFIX_PATH}"
+  )
+  autogen_executable_test(Moc)
+endif ()

Tests/RunCMake/Autogen_4/AutoUicExecutableConfig.cmake

@@ -7,9 +7,9 @@ endif()
 get_target_property(uic_location Qt${with_qt_version}::uic IMPORTED_LOCATION)
 set_target_properties(dummy PROPERTIES AUTOUIC_OPTIONS "EXE_PATH=${uic_location}")
 
-add_executable(myuic $<$<CONFIG:Debug>:exe_debug.cpp>
-                     $<$<CONFIG:Release>:exe_release.cpp>
-                     $<$<CONFIG:RelWithDebInfo>:exe_relwithdebinfo.cpp>
+add_executable(myuic $<$<CONFIG:Debug>:../Autogen_common/exe_debug.cpp>
+                     $<$<CONFIG:Release>:../Autogen_common/exe_release.cpp>
+                     $<$<CONFIG:RelWithDebInfo>:../Autogen_common/exe_relwithdebinfo.cpp>
 )
 
 set_target_properties(dummy PROPERTIES AUTOUIC_EXECUTABLE $<TARGET_FILE:myuic>)

Tests/RunCMake/Autogen_4/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.16)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)

Tests/RunCMake/Autogen_4/RunCMakeTest.cmake

@@ -0,0 +1,11 @@
+include(RunCMake)
+include(Autogen_common/utils)
+
+if (DEFINED with_qt_version)
+  set(RunCMake_TEST_OPTIONS
+    -Dwith_qt_version=${with_qt_version}
+    "-DQt${with_qt_version}_DIR:PATH=${Qt${with_qt_version}_DIR}"
+    "-DCMAKE_PREFIX_PATH:STRING=${CMAKE_PREFIX_PATH}"
+  )
+  autogen_executable_test(Uic)
+endif ()

Tests/RunCMake/Autogen_5/AutoRccExecutableConfig.cmake

@@ -7,9 +7,9 @@ endif()
 get_target_property(rcc_location Qt${with_qt_version}::rcc IMPORTED_LOCATION)
 set_target_properties(dummy PROPERTIES AUTORCC_OPTIONS "EXE_PATH=${rcc_location}")
 
-add_executable(myrcc $<$<CONFIG:Debug>:exe_debug.cpp>
-                     $<$<CONFIG:Release>:exe_release.cpp>
-                     $<$<CONFIG:RelWithDebInfo>:exe_relwithdebinfo.cpp>
+add_executable(myrcc $<$<CONFIG:Debug>:../Autogen_common/exe_debug.cpp>
+                     $<$<CONFIG:Release>:../Autogen_common/exe_release.cpp>
+                     $<$<CONFIG:RelWithDebInfo>:../Autogen_common/exe_relwithdebinfo.cpp>
 )
 
 set_target_properties(dummy PROPERTIES AUTORCC_EXECUTABLE $<TARGET_FILE:myrcc>)

Tests/RunCMake/Autogen_5/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.16)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)

Tests/RunCMake/Autogen_5/RccExample.cmake

@@ -3,7 +3,7 @@ enable_language(CXX)
 set(CMAKE_CXX_STANDARD 11)
 find_package(Qt${with_qt_version} REQUIRED COMPONENTS Core Widgets Gui)
 
-add_library(dummy STATIC example.cpp data.qrc)
+add_library(dummy STATIC ../Autogen_common/example.cpp data.qrc)
 target_link_libraries(dummy Qt${with_qt_version}::Core
                             Qt${with_qt_version}::Widgets
                             Qt${with_qt_version}::Gui)

Tests/RunCMake/Autogen_5/RunCMakeTest.cmake

@@ -0,0 +1,31 @@
+include(RunCMake)
+include(Autogen_common/utils)
+
+if (DEFINED with_qt_version)
+  set(RunCMake_TEST_OPTIONS
+    -Dwith_qt_version=${with_qt_version}
+    "-DQt${with_qt_version}_DIR:PATH=${Qt${with_qt_version}_DIR}"
+    "-DCMAKE_PREFIX_PATH:STRING=${CMAKE_PREFIX_PATH}"
+  )
+  autogen_executable_test(Rcc)
+  if (QtCore_VERSION VERSION_GREATER_EQUAL 6)
+    if (RunCMake_GENERATOR MATCHES "Make|Ninja")
+      foreach(value IN ITEMS ON OFF)
+        block()
+          set(RunCMake_TEST_BINARY_DIR
+            ${RunCMake_BINARY_DIR}/RccNoZTSD-${value}-build)
+          run_cmake_with_options(RccExample ${RunCMake_TEST_OPTIONS}
+            -DCMAKE_AUTOGEN_VERBOSE=ON -DZSTD_VALUE=${value})
+          if (value STREQUAL "OFF")
+            set(RunCMake_TEST_EXPECT_stdout "--no-zstd")
+          else()
+            set(RunCMake_TEST_NOT_EXPECT_stdout "--no-zstd")
+          endif()
+          set(RunCMake_TEST_NO_CLEAN 1)
+          run_cmake_command(RccNoZTSD-${value}-build ${CMAKE_COMMAND}
+            --build . --config Debug)
+        endblock()
+      endforeach()
+    endif()
+  endif()
+endif ()

Tests/RunCMake/Autogen_common/utils.cmake

@@ -0,0 +1,259 @@
+function(run_make_program dir)
+  execute_process(
+    COMMAND "${RunCMake_MAKE_PROGRAM}" ${ARGN}
+    WORKING_DIRECTORY "${dir}"
+    OUTPUT_VARIABLE make_program_stdout
+    ERROR_VARIABLE make_program_stderr
+    RESULT_VARIABLE make_program_result
+    )
+    if (NOT DEFINED RunMakeProgram_expected_result)
+      set(RunMakeProgram_expected_result 0)
+    endif()
+    if(NOT "${make_program_result}" MATCHES "${RunMakeProgram_expected_result}")
+      message(STATUS "
+============ beginning of ${RunCMake_MAKE_PROGRAM}'s stdout ============
+${make_program_stdout}
+=============== end of ${RunCMake_MAKE_PROGRAM}'s stdout ===============
+")
+    message(STATUS "
+============ beginning of ${RunCMake_MAKE_PROGRAM}'s stderr ============
+${make_program_stderr}
+=============== end of ${RunCMake_MAKE_PROGRAM}'s stderr ===============
+")
+    message(FATAL_ERROR
+            "top ${RunCMake_MAKE_PROGRAM} build failed exited with status ${make_program_result}")
+    endif()
+  set(make_program_stdout "${make_program_stdout}" PARENT_SCOPE)
+endfunction(run_make_program)
+
+function(count_substring STRING SUBSTRING COUNT_VAR)
+  string(LENGTH "${STRING}" STRING_LENGTH)
+  string(LENGTH "${SUBSTRING}" SUBSTRING_LENGTH)
+  if (SUBSTRING_LENGTH EQUAL 0)
+    message(FATAL_ERROR "SUBSTRING_LENGTH is 0")
+  endif()
+
+  if (STRING_LENGTH EQUAL 0)
+      message(FATAL_ERROR "STRING_LENGTH is 0")
+  endif()
+
+  if (STRING_LENGTH LESS SUBSTRING_LENGTH)
+    message(FATAL_ERROR "STRING_LENGTH is less than SUBSTRING_LENGTH")
+  endif()
+
+  set(COUNT 0)
+  string(FIND "${STRING}" "${SUBSTRING}" SUBSTRING_START)
+  while(SUBSTRING_START GREATER_EQUAL 0)
+    math(EXPR COUNT "${COUNT} + 1")
+    math(EXPR SUBSTRING_START "${SUBSTRING_START} + ${SUBSTRING_LENGTH}")
+    string(SUBSTRING "${STRING}" ${SUBSTRING_START} -1 STRING)
+    string(FIND "${STRING}" "${SUBSTRING}" SUBSTRING_START)
+  endwhile()
+
+  set(${COUNT_VAR} ${COUNT} PARENT_SCOPE)
+endfunction()
+
+function(not_expect make_program_stdout unexpected_output test_name)
+  count_substring("${make_program_stdout}" "${unexpected_output}" count)
+  if(NOT count EQUAL 0)
+    message(STATUS "${test_name}-not_expect - FAILED")
+    message(FATAL_ERROR "Expected to find ${unexpected_output} exactly 0 times in ${make_program_stdout} but found ${count} occurrences of ${unexpected_output}")
+  else()
+    message(STATUS "${test_name}-not_expect - PASSED")
+  endif()
+endfunction()
+
+function(expect_only_once make_program_stdout expected_output test_name)
+  count_substring("${make_program_stdout}" "${expected_output}" count)
+  if(NOT count EQUAL 1)
+    message(STATUS "${test_name}-expect_only_once - FAILED")
+    message(FATAL_ERROR "Expected to find ${expected_output} exactly once in ${make_program_stdout} but found ${count} occurrences of ${expected_output}")
+  else()
+    message(STATUS "${test_name}-expect_only_once - PASSED")
+  endif()
+endfunction()
+
+function(expect_n_times string_to_check expected_output expected_count test_name)
+  count_substring("${string_to_check}" "${expected_output}" count)
+  if(NOT count EQUAL ${expected_count})
+    message(STATUS "${test_name}-expect_${expected_count}_times - FAILED")
+    message(FATAL_ERROR "Expected to find ${expected_output} exactly ${expected_count} times in ${string_to_check} but found ${count} occurrences of ${expected_output}")
+  else()
+    message(STATUS "${test_name}-expect_${expected_count}_times - PASSED")
+  endif()
+endfunction()
+
+function(autogen_executable_test exe)
+  if (QtCore_VERSION VERSION_GREATER_EQUAL 5.15.0)
+    if(RunCMake_GENERATOR MATCHES "Ninja Multi-Config")
+      block()
+        set(RunCMake_TEST_VARIANT_DESCRIPTION "-CMake-configure")
+        set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/Auto${exe}ExecutableConfig-multi-config-build)
+        run_cmake_with_options(Auto${exe}ExecutableConfig ${RunCMake_TEST_OPTIONS} -DCMAKE_AUTOGEN_VERBOSE=ON -DCMAKE_AUTOGEN_BETTER_GRAPH_MULTI_CONFIG=ON)
+        unset(RunCMake_TEST_VARIANT_DESCRIPTION)
+        set(RunCMake_TEST_NO_CLEAN 1)
+        foreach(config IN ITEMS Debug Release RelWithDebInfo)
+          block()
+            set(RunCMake_TEST_EXPECT_stdout ".*running_exe_${config}*")
+            set(RunCMake_TEST_VARIANT_DESCRIPTION "-${config}-expect_running_exe_${config}")
+            run_cmake_command(Auto${exe}ExecutableConfig-multi-config-build ${CMAKE_COMMAND} --build . --config ${config})
+          endblock()
+        endforeach()
+        set(RunCMake_TEST_EXPECT_stdout "ninja: no work to do")
+        foreach(config IN ITEMS Debug Release RelWithDebInfo)
+          block()
+          set(RunCMake_TEST_VARIANT_DESCRIPTION "-${config}-expect_no_work_to_do")
+            run_cmake_command(Auto${exe}ExecutableConfig-multi-config-build ${CMAKE_COMMAND} --build . --config ${config})
+          endblock()
+        endforeach()
+      endblock()
+      block()
+        set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/Auto${exe}ExecutableConfig-build)
+        run_cmake_with_options(Auto${exe}ExecutableConfig ${RunCMake_TEST_OPTIONS} -DCMAKE_AUTOGEN_VERBOSE=ON -DCMAKE_AUTOGEN_BETTER_GRAPH_MULTI_CONFIG=ON)
+        foreach(config IN ITEMS Debug Release RelWithDebInfo)
+          block()
+            run_make_program(${RunCMake_TEST_BINARY_DIR} --verbose -f build-${config}.ninja)
+
+            set(expected_output "running_exe_${config}")
+            expect_only_once("${make_program_stdout}" "${expected_output}" "Auto${exe}ExecutableConfig-${config}-${expected_output}")
+
+            foreach(sub_config IN ITEMS Debug Release RelWithDebInfo)
+              if(NOT sub_config STREQUAL config)
+                set(unexpected_output "running_exe_${sub_config}")
+                not_expect("${make_program_stdout}" "${unexpected_output}" "Auto${exe}ExecutableConfig-${config}-${unexpected_output}")
+              endif()
+            endforeach()
+
+            if (exe STREQUAL "Moc" OR exe STREQUAL "Uic")
+              set(expected_output "cmake_autogen")
+            else()
+              set(expected_output "cmake_autorcc")
+            endif()
+            expect_only_once("${make_program_stdout}" "${expected_output}" "Auto${exe}ExecutableConfig-${config}-${expected_output}")
+          endblock()
+        endforeach()
+      endblock()
+      block()
+        foreach(ninja_config IN ITEMS Debug Release RelWithDebInfo)
+          foreach(target_config IN ITEMS Debug Release RelWithDebInfo)
+            block()
+              set(TEST_SUFFIX "-CrossConfig-${ninja_config}-${target_config}")
+              set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/Auto${exe}ExecutableConfig${TEST_SUFFIX}-build)
+              set(RunCMake_TEST_VARIANT_DESCRIPTION ${TEST_SUFFIX})
+              run_cmake_with_options(Auto${exe}ExecutableConfig ${RunCMake_TEST_OPTIONS} -DCMAKE_CROSS_CONFIGS=all -DCMAKE_DEFAULT_BUILD_TYPE=${ninja_config} -DCMAKE_AUTOGEN_BETTER_GRAPH_MULTI_CONFIG=ON)
+              unset(RunCMake_TEST_VARIANT_DESCRIPTION)
+
+              run_make_program(${RunCMake_TEST_BINARY_DIR} --verbose -f build-${ninja_config}.ninja dummy:${target_config})
+
+              set(expected_output "running_exe_${ninja_config}")
+              expect_only_once("${make_program_stdout}" "${expected_output}" "Auto${exe}ExecutableConfig${TEST_SUFFIX}-${expected_output}")
+
+              foreach(sub_config IN ITEMS Debug Release RelWithDebInfo)
+                if(NOT sub_config STREQUAL ninja_config)
+                  set(unexpected_output "running_exe_${sub_config}")
+                  not_expect("${make_program_stdout}" "${unexpected_output}" "Auto${exe}ExecutableConfig${TEST_SUFFIX}-${unexpected_output}")
+                endif()
+              endforeach()
+
+              if (exe STREQUAL "Moc" OR exe STREQUAL "Uic")
+                set(expected_output "cmake_autogen")
+              else()
+                set(expected_output "cmake_autorcc")
+              endif()
+              expect_only_once("${make_program_stdout}" "${expected_output}" "Auto${exe}ExecutableConfig${TEST_SUFFIX}-${expected_output}")
+            endblock()
+          endforeach()
+        endforeach()
+      endblock()
+      block()
+        foreach(ninja_config IN ITEMS Debug Release RelWithDebInfo)
+          set(TEST_SUFFIX "-CrossConfig-${ninja_config}-all-all")
+          set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/Auto${exe}ExecutableConfig${TEST_SUFFIX}-build)
+          set(RunCMake_TEST_VARIANT_DESCRIPTION ${TEST_SUFFIX})
+          run_cmake_with_options(Auto${exe}ExecutableConfig ${RunCMake_TEST_OPTIONS} -DCMAKE_CROSS_CONFIGS=all -DCMAKE_AUTOGEN_BETTER_GRAPH_MULTI_CONFIG=ON)
+          unset(RunCMake_TEST_VARIANT_DESCRIPTION)
+          run_make_program(${RunCMake_TEST_BINARY_DIR} --verbose -f build-${ninja_config}.ninja all:all)
+        endforeach()
+      endblock()
+    elseif (RunCMake_GENERATOR MATCHES "Ninja|Make")
+      block()
+        set(RunCMake_TEST_BINARY_DIR  ${RunCMake_BINARY_DIR}/Auto${exe}ExecutableConfig-build)
+        foreach(config IN ITEMS Debug Release RelWithDebInfo)
+          block()
+            set(RunCMake_TEST_VARIANT_DESCRIPTION "-${config}")
+            run_cmake_with_options(Auto${exe}ExecutableConfig ${RunCMake_TEST_OPTIONS} -DCMAKE_BUILD_TYPE=${config} -DCMAKE_AUTOGEN_VERBOSE=ON)
+            unset(RunCMake_TEST_VARIANT_DESCRIPTION)
+            set(RunCMake_TEST_NO_CLEAN 1)
+            set(RunCMake_TEST_EXPECT_stdout ".*running_exe_${config}*")
+            run_cmake_command(Auto${exe}ExecutableConfig-${config}-build ${CMAKE_COMMAND} --build .)
+          endblock()
+        endforeach()
+      endblock()
+    endif()
+  endif()
+
+  # Visual Studio specific dependency tests
+  if (RunCMake_GENERATOR MATCHES "Visual Studio")
+    block()
+      set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/${exe}Example-build)
+      set(RunCMake_TEST_VARIANT_DESCRIPTION "-CMake-configure")
+      run_cmake_with_options(${exe}Example ${RunCMake_TEST_OPTIONS} -DCMAKE_AUTOGEN_VERBOSE=ON)
+      unset(RunCMake_TEST_VARIANT_DESCRIPTION)
+      set(RunCMake_TEST_NO_CLEAN 1)
+      foreach(config IN ITEMS Debug Release RelWithDebInfo)
+        block()
+          set(RunCMake_TEST_VARIANT_DESCRIPTION "-${config}-first-build")
+          run_cmake_command(${exe}Example-build ${CMAKE_COMMAND} --build . --config ${config})
+        endblock()
+      endforeach()
+      foreach(config IN ITEMS Debug Release RelWithDebInfo)
+        block()
+          if (exe STREQUAL "Moc" OR exe STREQUAL "Uic")
+            set(RunCMake_TEST_NOT_EXPECT_stdout "Auto${exe}")
+            set(not_expect_descripton "Auto${exe}")
+          else ()
+            set(RunCMake_TEST_NOT_EXPECT_stdout "Auto${exe}")
+            set(not_expect_descripton "Auto${exe}")
+          endif()
+          set(RunCMake_TEST_VARIANT_DESCRIPTION "-second-build-${config}_expect_no_${not_expect_descripton}")
+          run_cmake_command(${exe}Example-build ${CMAKE_COMMAND} --build . --config ${config})
+        endblock()
+      endforeach()
+    endblock()
+  endif()
+
+  if (RunCMake_GENERATOR MATCHES "Xcode")
+    foreach(exe IN ITEMS Moc Uic Rcc)
+      block()
+        set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/${exe}Example-build)
+        set(RunCMake_TEST_VARIANT_DESCRIPTION "-CMake-configure")
+        set(RunCMake_TEST_EXPECT_stderr ".*")
+        run_cmake_with_options(${exe}Example ${RunCMake_TEST_OPTIONS} -DCMAKE_AUTOGEN_VERBOSE=ON)
+        set(RunCMake_TEST_NO_CLEAN 1)
+        set(RunCMake_MAKE_PROGRAM ${CMAKE_COMMAND})
+        run_make_program(${RunCMake_TEST_BINARY_DIR}  --build . --config Debug)
+        if (exe STREQUAL "Moc")
+          set(expected_count 16)
+        elseif (exe STREQUAL "Uic")
+          set(expected_count 4)
+        else()
+          set(expected_count 12)
+        endif()
+        expect_n_times("${make_program_stdout}" "Auto${exe}:" ${expected_count} "${exe}Example-build-Auto${exe}")
+        expect_n_times("${make_program_stdout}" "Auto${exe}:" ${expected_count} "${exe}Example-build-Auto${exe}")
+
+        if (exe STREQUAL "Moc" OR exe STREQUAL "Uic")
+          expect_n_times("${make_program_stdout}" "AutoGen:" 20 "${exe}Example-build-AutoGen:")
+        endif()
+
+        foreach(config IN ITEMS Debug Release RelWithDebInfo)
+          block()
+            run_make_program(${RunCMake_TEST_BINARY_DIR} --build . --config ${config})
+            not_expect("${make_program_stdout}" "Auto${exe}" "${exe}Example-${config}_Auto${exe}")
+            not_expect("${make_program_stdout}" "AutoGen:" "${exe}Example-${config}_AutoGen")
+          endblock()
+        endforeach()
+      endblock()
+    endforeach()
+  endif()
+endfunction()

Tests/RunCMake/CMakeLists.txt

@@ -288,28 +288,35 @@ if(CMake_TEST_Qt6 AND Qt6Widgets_FOUND)
   cmake_path(GET Qt6_DIR  PARENT_PATH base_dir)  # <base>/lib/cmake
   cmake_path(GET base_dir PARENT_PATH base_dir)  # <base>/lib
   cmake_path(GET base_dir PARENT_PATH base_dir)  # <base>
-  add_RunCMake_test(AutogenQt6 TEST_DIR Autogen
-    -Dwith_qt_version=6
-    -DQtCore_VERSION=${Qt6Core_VERSION}
-    "-DQt6_DIR:PATH=${Qt6_DIR}"
-    "-DCMAKE_PREFIX_PATH:STRING=${base_dir}"
-    -DPSEUDO_TIDY=$<TARGET_FILE:pseudo_tidy>
-    -DPSEUDO_IWYU=$<TARGET_FILE:pseudo_iwyu>
-    -DPSEUDO_CPPLINT=$<TARGET_FILE:pseudo_cpplint>
-    -DPSEUDO_CPPCHECK=$<TARGET_FILE:pseudo_cppcheck>
-  )
+  # Note: Since RunCMake.Autogen tests cause time out on some CI,
+  # we split the tests.
+  set(autogen_test_number 1 2 3 4 5)
+  foreach(val IN ITEMS ${autogen_test_number})
+    add_RunCMake_test("Autogen_Qt6_${val}" TEST_DIR "Autogen_${val}"
+      -Dwith_qt_version=6
+      -DQtCore_VERSION=${Qt6Core_VERSION}
+      "-DQt6_DIR:PATH=${Qt6_DIR}"
+      "-DCMAKE_PREFIX_PATH:STRING=${base_dir}"
+      -DPSEUDO_TIDY=$<TARGET_FILE:pseudo_tidy>
+      -DPSEUDO_IWYU=$<TARGET_FILE:pseudo_iwyu>
+      -DPSEUDO_CPPLINT=$<TARGET_FILE:pseudo_cpplint>
+      -DPSEUDO_CPPCHECK=$<TARGET_FILE:pseudo_cppcheck>
+    )
+  endforeach()
   set(want_NoQt_test FALSE)
 endif ()
 if(CMake_TEST_Qt5 AND Qt5Widgets_FOUND)
-  add_RunCMake_test(AutogenQt5 TEST_DIR Autogen
-    -Dwith_qt_version=5
-    -DQtCore_VERSION=${Qt5Core_VERSION}
-    "-DQt5_DIR:PATH=${Qt5_DIR}"
-  )
+  foreach(val IN ITEMS ${autogen_test_number})
+    add_RunCMake_test("Autogen_Qt5_${val}" TEST_DIR "Autogen_${val}"
+      -Dwith_qt_version=5
+      -DQtCore_VERSION=${Qt5Core_VERSION}
+      "-DQt5_DIR:PATH=${Qt5_DIR}"
+    )
+  endforeach()
   set(want_NoQt_test FALSE)
 endif ()
 if(want_NoQt_test)
-  add_RunCMake_test(AutogenNoQt TEST_DIR Autogen)
+  add_RunCMake_test(AutogenNoQt TEST_DIR Autogen_1)
 endif()
 
 if(NOT DEFINED CMake_TEST_BuildDepends_GNU_AS