Makefiles Generators: use compiler for dependencies generation

Each source compilation generates a dependencies file. These dependencies
files are consolidated in one file per target. This consolidation is done
as part of command 'cmake -E cmake_depends` launched before evaluation of
makefile dependency graph.

The consolidation uses the same approach as `CMake` dependencies management.

Fixes: #21321
This commit is contained in:
Marc Chevrier
2020-10-18 16:11:27 +02:00
parent afd0f6785d
commit 2c71d051fa
52 changed files with 1311 additions and 269 deletions
+329 -198
View File
@@ -5,6 +5,7 @@
#include <algorithm>
#include <cassert>
#include <cstdio>
#include <functional>
#include <sstream>
#include <utility>
@@ -19,6 +20,7 @@
#include "cmCMakePath.h"
#include "cmCustomCommand.h" // IWYU pragma: keep
#include "cmCustomCommandGenerator.h"
#include "cmDependsCompiler.h"
#include "cmFileTimeCache.h"
#include "cmGeneratedFileStream.h"
#include "cmGeneratorExpression.h"
@@ -52,8 +54,9 @@
# include "cmDependsJava.h"
#endif
namespace {
// Helper function used below.
static std::string cmSplitExtension(std::string const& in, std::string& base)
std::string cmSplitExtension(std::string const& in, std::string& base)
{
std::string ext;
std::string::size_type dot_pos = in.rfind('.');
@@ -67,6 +70,43 @@ static std::string cmSplitExtension(std::string const& in, std::string& base)
return ext;
}
// Helper predicate for removing absolute paths that don't point to the
// source or binary directory. It is used when CMAKE_DEPENDS_IN_PROJECT_ONLY
// is set ON, to only consider in-project dependencies during the build.
class NotInProjectDir
{
public:
// Constructor with the source and binary directory's path
NotInProjectDir(cm::string_view sourceDir, cm::string_view binaryDir)
: SourceDir(sourceDir)
, BinaryDir(binaryDir)
{
}
// Operator evaluating the predicate
bool operator()(const std::string& p) const
{
auto path = cmCMakePath(p).Normal();
// Keep all relative paths:
if (path.IsRelative()) {
return false;
}
// If it's an absolute path, check if it starts with the source
// directory:
return !(cmCMakePath(SourceDir).IsPrefix(path) ||
cmCMakePath(BinaryDir).IsPrefix(path));
}
private:
// The path to the source directory
cm::string_view SourceDir;
// The path to the binary directory
cm::string_view BinaryDir;
};
}
cmLocalUnixMakefileGenerator3::cmLocalUnixMakefileGenerator3(
cmGlobalGenerator* gg, cmMakefile* mf)
: cmLocalCommonGenerator(gg, mf, mf->GetCurrentBinaryDirectory())
@@ -554,8 +594,10 @@ void cmLocalUnixMakefileGenerator3::WriteMakeRule(
}
}
// Write the list of commands.
os << cmWrap("\t", commands, "", "\n") << "\n";
if (!commands.empty()) {
// Write the list of commands.
os << cmWrap("\t", commands, "", "\n") << "\n";
}
if (symbolic && !this->IsWatcomWMake()) {
os << ".PHONY : " << tgt << "\n";
}
@@ -1300,91 +1342,153 @@ bool cmLocalUnixMakefileGenerator3::UpdateDependencies(
cmSystemTools::Error("Target DependInfo.cmake file not found");
}
bool status = true;
// Check if any multiple output pairs have a missing file.
this->CheckMultipleOutputs(verbose);
std::string const targetDir = cmSystemTools::GetFilenamePath(tgtInfo);
std::string const internalDependFile = targetDir + "/depend.internal";
std::string const dependFile = targetDir + "/depend.make";
if (!this->Makefile->GetSafeDefinition("CMAKE_DEPENDS_LANGUAGES").empty()) {
// dependencies are managed by CMake itself
// If the target DependInfo.cmake file has changed since the last
// time dependencies were scanned then force rescanning. This may
// happen when a new source file is added and CMake regenerates the
// project but no other sources were touched.
bool needRescanDependInfo = false;
cmFileTimeCache* ftc =
this->GlobalGenerator->GetCMakeInstance()->GetFileTimeCache();
{
int result;
if (!ftc->Compare(internalDependFile, tgtInfo, &result) || result < 0) {
if (verbose) {
cmSystemTools::Stdout(cmStrCat("Dependee \"", tgtInfo,
"\" is newer than depender \"",
internalDependFile, "\".\n"));
std::string const internalDependFile = targetDir + "/depend.internal";
std::string const dependFile = targetDir + "/depend.make";
// If the target DependInfo.cmake file has changed since the last
// time dependencies were scanned then force rescanning. This may
// happen when a new source file is added and CMake regenerates the
// project but no other sources were touched.
bool needRescanDependInfo = false;
cmFileTimeCache* ftc =
this->GlobalGenerator->GetCMakeInstance()->GetFileTimeCache();
{
int result;
if (!ftc->Compare(internalDependFile, tgtInfo, &result) || result < 0) {
if (verbose) {
cmSystemTools::Stdout(cmStrCat("Dependee \"", tgtInfo,
"\" is newer than depender \"",
internalDependFile, "\".\n"));
}
needRescanDependInfo = true;
}
needRescanDependInfo = true;
}
// If the directory information is newer than depend.internal, include
// dirs may have changed. In this case discard all old dependencies.
bool needRescanDirInfo = false;
{
std::string dirInfoFile =
cmStrCat(this->GetCurrentBinaryDirectory(),
"/CMakeFiles/CMakeDirectoryInformation.cmake");
int result;
if (!ftc->Compare(internalDependFile, dirInfoFile, &result) ||
result < 0) {
if (verbose) {
cmSystemTools::Stdout(cmStrCat("Dependee \"", dirInfoFile,
"\" is newer than depender \"",
internalDependFile, "\".\n"));
}
needRescanDirInfo = true;
}
}
// Check the implicit dependencies to see if they are up to date.
// The build.make file may have explicit dependencies for the object
// files but these will not affect the scanning process so they need
// not be considered.
cmDepends::DependencyMap validDependencies;
bool needRescanDependencies = false;
if (!needRescanDirInfo) {
cmDependsC checker;
checker.SetVerbose(verbose);
checker.SetFileTimeCache(ftc);
// cmDependsC::Check() fills the vector validDependencies() with the
// dependencies for those files where they are still valid, i.e.
// neither the files themselves nor any files they depend on have
// changed. We don't do that if the CMakeDirectoryInformation.cmake
// file has changed, because then potentially all dependencies have
// changed. This information is given later on to cmDependsC, which
// then only rescans the files where it did not get valid dependencies
// via this dependency vector. This means that in the normal case, when
// only few or one file have been edited, then also only this one file
// is actually scanned again, instead of all files for this target.
needRescanDependencies =
!checker.Check(dependFile, internalDependFile, validDependencies);
}
if (needRescanDependInfo || needRescanDirInfo || needRescanDependencies) {
// The dependencies must be regenerated.
std::string targetName = cmSystemTools::GetFilenameName(targetDir);
targetName = targetName.substr(0, targetName.length() - 4);
std::string message =
cmStrCat("Scanning dependencies of target ", targetName);
cmSystemTools::MakefileColorEcho(cmsysTerminal_Color_ForegroundMagenta |
cmsysTerminal_Color_ForegroundBold,
message.c_str(), true, color);
status = this->ScanDependencies(targetDir, dependFile,
internalDependFile, validDependencies);
}
}
// If the directory information is newer than depend.internal, include dirs
// may have changed. In this case discard all old dependencies.
bool needRescanDirInfo = false;
{
std::string dirInfoFile =
cmStrCat(this->GetCurrentBinaryDirectory(),
"/CMakeFiles/CMakeDirectoryInformation.cmake");
int result;
if (!ftc->Compare(internalDependFile, dirInfoFile, &result) ||
result < 0) {
if (verbose) {
cmSystemTools::Stdout(cmStrCat("Dependee \"", dirInfoFile,
"\" is newer than depender \"",
internalDependFile, "\".\n"));
auto depends =
this->Makefile->GetSafeDefinition("CMAKE_DEPENDS_DEPENDENCY_FILES");
if (!depends.empty()) {
// dependencies are managed by compiler
auto depFiles = cmExpandedList(depends);
std::string const internalDepFile =
targetDir + "/compiler_depend.internal";
std::string const depFile = targetDir + "/compiler_depend.make";
cmDepends::DependencyMap dependencies;
cmDependsCompiler depsManager;
bool projectOnly = cmIsOn(
this->Makefile->GetSafeDefinition("CMAKE_DEPENDS_IN_PROJECT_ONLY"));
depsManager.SetVerbose(verbose);
depsManager.SetLocalGenerator(this);
if (!depsManager.CheckDependencies(
internalDepFile, depFiles, dependencies,
projectOnly ? NotInProjectDir(this->GetSourceDirectory(),
this->GetBinaryDirectory())
: std::function<bool(const std::string&)>())) {
// regenerate dependencies files
std::string targetName =
cmCMakePath(targetDir).GetFileName().RemoveExtension().GenericString();
auto message = cmStrCat(
"Consolidate compiler generated dependencies of target ", targetName);
cmSystemTools::MakefileColorEcho(cmsysTerminal_Color_ForegroundMagenta |
cmsysTerminal_Color_ForegroundBold,
message.c_str(), true, color);
// Open the make depends file. This should be copy-if-different
// because the make tool may try to reload it needlessly otherwise.
cmGeneratedFileStream ruleFileStream(
depFile, false, this->GlobalGenerator->GetMakefileEncoding());
ruleFileStream.SetCopyIfDifferent(true);
if (!ruleFileStream) {
return false;
}
needRescanDirInfo = true;
// Open the cmake dependency tracking file. This should not be
// copy-if-different because dependencies are re-scanned when it is
// older than the DependInfo.cmake.
cmGeneratedFileStream internalRuleFileStream(
internalDepFile, false, this->GlobalGenerator->GetMakefileEncoding());
if (!internalRuleFileStream) {
return false;
}
this->WriteDisclaimer(ruleFileStream);
this->WriteDisclaimer(internalRuleFileStream);
depsManager.WriteDependencies(dependencies, ruleFileStream,
internalRuleFileStream);
}
}
// Check the implicit dependencies to see if they are up to date.
// The build.make file may have explicit dependencies for the object
// files but these will not affect the scanning process so they need
// not be considered.
cmDepends::DependencyMap validDependencies;
bool needRescanDependencies = false;
if (!needRescanDirInfo) {
cmDependsC checker;
checker.SetVerbose(verbose);
checker.SetFileTimeCache(ftc);
// cmDependsC::Check() fills the vector validDependencies() with the
// dependencies for those files where they are still valid, i.e. neither
// the files themselves nor any files they depend on have changed.
// We don't do that if the CMakeDirectoryInformation.cmake file has
// changed, because then potentially all dependencies have changed.
// This information is given later on to cmDependsC, which then only
// rescans the files where it did not get valid dependencies via this
// dependency vector. This means that in the normal case, when only
// few or one file have been edited, then also only this one file is
// actually scanned again, instead of all files for this target.
needRescanDependencies =
!checker.Check(dependFile, internalDependFile, validDependencies);
}
if (needRescanDependInfo || needRescanDirInfo || needRescanDependencies) {
// The dependencies must be regenerated.
std::string targetName = cmSystemTools::GetFilenameName(targetDir);
targetName = targetName.substr(0, targetName.length() - 4);
std::string message =
cmStrCat("Scanning dependencies of target ", targetName);
cmSystemTools::MakefileColorEcho(cmsysTerminal_Color_ForegroundMagenta |
cmsysTerminal_Color_ForegroundBold,
message.c_str(), true, color);
return this->ScanDependencies(targetDir, dependFile, internalDependFile,
validDependencies);
}
// The dependencies are already up-to-date.
return true;
return status;
}
bool cmLocalUnixMakefileGenerator3::ScanDependencies(
@@ -1723,166 +1827,193 @@ void cmLocalUnixMakefileGenerator3::ClearDependencies(cmMakefile* mf,
cmDepends clearer;
clearer.SetVerbose(verbose);
for (std::string const& file : files) {
std::string dir = cmSystemTools::GetFilenamePath(file);
auto snapshot = mf->GetState()->CreateBaseSnapshot();
cmMakefile lmf(mf->GetGlobalGenerator(), snapshot);
lmf.ReadListFile(file);
// Clear the implicit dependency makefile.
std::string dependFile = dir + "/depend.make";
clearer.Clear(dependFile);
if (!lmf.GetSafeDefinition("CMAKE_DEPENDS_LANGUAGES").empty()) {
std::string dir = cmSystemTools::GetFilenamePath(file);
// Remove the internal dependency check file to force
// regeneration.
std::string internalDependFile = dir + "/depend.internal";
cmSystemTools::RemoveFile(internalDependFile);
}
}
// Clear the implicit dependency makefile.
std::string dependFile = dir + "/depend.make";
clearer.Clear(dependFile);
namespace {
// Helper predicate for removing absolute paths that don't point to the
// source or binary directory. It is used when CMAKE_DEPENDS_IN_PROJECT_ONLY
// is set ON, to only consider in-project dependencies during the build.
class NotInProjectDir
{
public:
// Constructor with the source and binary directory's path
NotInProjectDir(cm::string_view sourceDir, cm::string_view binaryDir)
: SourceDir(sourceDir)
, BinaryDir(binaryDir)
{
}
// Operator evaluating the predicate
bool operator()(const std::string& p) const
{
auto path = cmCMakePath(p).Normal();
// Keep all relative paths:
if (path.IsRelative()) {
return false;
// Remove the internal dependency check file to force
// regeneration.
std::string internalDependFile = dir + "/depend.internal";
cmSystemTools::RemoveFile(internalDependFile);
}
// If it's an absolute path, check if it starts with the source
// directory:
return !(cmCMakePath(SourceDir).IsPrefix(path) ||
cmCMakePath(BinaryDir).IsPrefix(path));
}
private:
// The path to the source directory
cm::string_view SourceDir;
// The path to the binary directory
cm::string_view BinaryDir;
};
auto depsFiles = lmf.GetSafeDefinition("CMAKE_DEPENDS_DEPENDENCY_FILES");
if (!depsFiles.empty()) {
auto dir = cmCMakePath(file).GetParentPath();
// Clear the implicit dependency makefile.
auto depFile = cmCMakePath(dir).Append("compiler_depend.make");
clearer.Clear(depFile.GenericString());
// Remove the internal dependency check file
auto internalDepFile =
cmCMakePath(dir).Append("compiler_depend.internal");
cmSystemTools::RemoveFile(internalDepFile.GenericString());
// Touch timestamp file to force dependencies regeneration
auto DepTimestamp = cmCMakePath(dir).Append("compiler_depend.ts");
cmSystemTools::Touch(DepTimestamp.GenericString(), true);
// clear the dependencies files generated by the compiler
std::vector<std::string> dependencies = cmExpandedList(depsFiles);
cmDependsCompiler depsManager;
depsManager.SetVerbose(verbose);
depsManager.ClearDependencies(dependencies);
}
}
}
void cmLocalUnixMakefileGenerator3::WriteDependLanguageInfo(
std::ostream& cmakefileStream, cmGeneratorTarget* target)
{
ImplicitDependLanguageMap const& implicitLangs =
this->GetImplicitDepends(target);
// To enable dependencies filtering
cmakefileStream << "\n"
<< "# Consider dependencies only in project.\n"
<< "set(CMAKE_DEPENDS_IN_PROJECT_ONLY "
<< (cmIsOn(this->Makefile->GetSafeDefinition(
"CMAKE_DEPENDS_IN_PROJECT_ONLY"))
? "ON"
: "OFF")
<< ")\n\n";
auto const& implicitLangs =
this->GetImplicitDepends(target, cmDependencyScannerKind::CMake);
// list the languages
cmakefileStream
<< "# The set of languages for which implicit dependencies are needed:\n";
cmakefileStream << "# The set of languages for which implicit "
"dependencies are needed:\n";
cmakefileStream << "set(CMAKE_DEPENDS_LANGUAGES\n";
for (auto const& implicitLang : implicitLangs) {
cmakefileStream << " \"" << implicitLang.first << "\"\n";
}
cmakefileStream << " )\n";
// now list the files for each language
cmakefileStream
<< "# The set of files for implicit dependencies of each language:\n";
for (auto const& implicitLang : implicitLangs) {
cmakefileStream << "set(CMAKE_DEPENDS_CHECK_" << implicitLang.first
<< "\n";
ImplicitDependFileMap const& implicitPairs = implicitLang.second;
if (!implicitLangs.empty()) {
// now list the files for each language
cmakefileStream
<< "# The set of files for implicit dependencies of each language:\n";
for (auto const& implicitLang : implicitLangs) {
const auto& lang = implicitLang.first;
// for each file pair
for (auto const& implicitPair : implicitPairs) {
for (auto const& di : implicitPair.second) {
cmakefileStream << " \"" << di << "\" ";
cmakefileStream << "\"" << implicitPair.first << "\"\n";
cmakefileStream << "set(CMAKE_DEPENDS_CHECK_" << lang << "\n";
auto const& implicitPairs = implicitLang.second;
// for each file pair
for (auto const& implicitPair : implicitPairs) {
for (auto const& di : implicitPair.second) {
cmakefileStream << " \"" << di << "\" ";
cmakefileStream << "\"" << implicitPair.first << "\"\n";
}
}
}
cmakefileStream << " )\n";
cmakefileStream << " )\n";
// Tell the dependency scanner what compiler is used.
std::string cidVar =
cmStrCat("CMAKE_", implicitLang.first, "_COMPILER_ID");
cmProp cid = this->Makefile->GetDefinition(cidVar);
if (cmNonempty(cid)) {
cmakefileStream << "set(CMAKE_" << implicitLang.first
<< "_COMPILER_ID \"" << *cid << "\")\n";
}
// Tell the dependency scanner what compiler is used.
std::string cidVar = cmStrCat("CMAKE_", lang, "_COMPILER_ID");
cmProp cid = this->Makefile->GetDefinition(cidVar);
if (cmNonempty(cid)) {
cmakefileStream << "set(CMAKE_" << lang << "_COMPILER_ID \"" << *cid
<< "\")\n";
}
if (implicitLang.first == "Fortran") {
std::string smodSep =
this->Makefile->GetSafeDefinition("CMAKE_Fortran_SUBMODULE_SEP");
std::string smodExt =
this->Makefile->GetSafeDefinition("CMAKE_Fortran_SUBMODULE_EXT");
cmakefileStream << "set(CMAKE_Fortran_SUBMODULE_SEP \"" << smodSep
<< "\")\n";
cmakefileStream << "set(CMAKE_Fortran_SUBMODULE_EXT \"" << smodExt
<< "\")\n";
}
if (lang == "Fortran") {
std::string smodSep =
this->Makefile->GetSafeDefinition("CMAKE_Fortran_SUBMODULE_SEP");
std::string smodExt =
this->Makefile->GetSafeDefinition("CMAKE_Fortran_SUBMODULE_EXT");
cmakefileStream << "set(CMAKE_Fortran_SUBMODULE_SEP \"" << smodSep
<< "\")\n";
cmakefileStream << "set(CMAKE_Fortran_SUBMODULE_EXT \"" << smodExt
<< "\")\n";
}
// Build a list of preprocessor definitions for the target.
std::set<std::string> defines;
this->GetTargetDefines(target, this->GetConfigName(), implicitLang.first,
defines);
if (!defines.empty()) {
/* clang-format off */
// Build a list of preprocessor definitions for the target.
std::set<std::string> defines;
this->GetTargetDefines(target, this->GetConfigName(), lang, defines);
if (!defines.empty()) {
/* clang-format off */
cmakefileStream
<< "\n"
<< "# Preprocessor definitions for this target.\n"
<< "set(CMAKE_TARGET_DEFINITIONS_" << implicitLang.first << "\n";
/* clang-format on */
for (std::string const& define : defines) {
cmakefileStream << " " << cmOutputConverter::EscapeForCMake(define)
<< "\n";
<< "set(CMAKE_TARGET_DEFINITIONS_" << lang << "\n";
/* clang-format on */
for (std::string const& define : defines) {
cmakefileStream << " " << cmOutputConverter::EscapeForCMake(define)
<< "\n";
}
cmakefileStream << " )\n";
}
// Target-specific include directories:
cmakefileStream << "\n"
<< "# The include file search paths:\n";
cmakefileStream << "set(CMAKE_" << lang << "_TARGET_INCLUDE_PATH\n";
std::vector<std::string> includes;
this->GetIncludeDirectories(includes, target, lang,
this->GetConfigName());
std::string const& binaryDir = this->GetState()->GetBinaryDirectory();
if (this->Makefile->IsOn("CMAKE_DEPENDS_IN_PROJECT_ONLY")) {
std::string const& sourceDir = this->GetState()->GetSourceDirectory();
cm::erase_if(includes, ::NotInProjectDir(sourceDir, binaryDir));
}
for (std::string const& include : includes) {
cmakefileStream << " \""
<< this->MaybeConvertToRelativePath(binaryDir, include)
<< "\"\n";
}
cmakefileStream << " )\n";
}
// Target-specific include directories:
cmakefileStream << "\n"
<< "# The include file search paths:\n";
cmakefileStream << "set(CMAKE_" << implicitLang.first
<< "_TARGET_INCLUDE_PATH\n";
std::vector<std::string> includes;
this->GetIncludeDirectories(includes, target, implicitLang.first,
this->GetConfigName());
std::string const& binaryDir = this->GetState()->GetBinaryDirectory();
if (this->Makefile->IsOn("CMAKE_DEPENDS_IN_PROJECT_ONLY")) {
std::string const& sourceDir = this->GetState()->GetSourceDirectory();
cm::erase_if(includes, ::NotInProjectDir(sourceDir, binaryDir));
// Store include transform rule properties. Write the directory
// rules first because they may be overridden by later target rules.
std::vector<std::string> transformRules;
if (cmProp xform =
this->Makefile->GetProperty("IMPLICIT_DEPENDS_INCLUDE_TRANSFORM")) {
cmExpandList(*xform, transformRules);
}
for (std::string const& include : includes) {
cmakefileStream << " \""
<< this->MaybeConvertToRelativePath(binaryDir, include)
<< "\"\n";
if (cmProp xform =
target->GetProperty("IMPLICIT_DEPENDS_INCLUDE_TRANSFORM")) {
cmExpandList(*xform, transformRules);
}
if (!transformRules.empty()) {
cmakefileStream << "\nset(CMAKE_INCLUDE_TRANSFORMS\n";
for (std::string const& tr : transformRules) {
cmakefileStream << " " << cmOutputConverter::EscapeForCMake(tr)
<< "\n";
}
cmakefileStream << " )\n";
}
cmakefileStream << " )\n";
}
// Store include transform rule properties. Write the directory
// rules first because they may be overridden by later target rules.
std::vector<std::string> transformRules;
if (cmProp xform =
this->Makefile->GetProperty("IMPLICIT_DEPENDS_INCLUDE_TRANSFORM")) {
cmExpandList(*xform, transformRules);
}
if (cmProp xform =
target->GetProperty("IMPLICIT_DEPENDS_INCLUDE_TRANSFORM")) {
cmExpandList(*xform, transformRules);
}
if (!transformRules.empty()) {
cmakefileStream << "set(CMAKE_INCLUDE_TRANSFORMS\n";
for (std::string const& tr : transformRules) {
cmakefileStream << " " << cmOutputConverter::EscapeForCMake(tr) << "\n";
auto const& compilerLangs =
this->GetImplicitDepends(target, cmDependencyScannerKind::Compiler);
// list the dependency files managed by the compiler
cmakefileStream << "\n# The set of dependency files which are needed:\n";
cmakefileStream << "set(CMAKE_DEPENDS_DEPENDENCY_FILES\n";
for (auto const& compilerLang : compilerLangs) {
auto depFormat = this->Makefile->GetSafeDefinition(
cmStrCat("CMAKE_", compilerLang.first, "_DEPFILE_FORMAT"));
auto const& compilerPairs = compilerLang.second;
for (auto const& compilerPair : compilerPairs) {
for (auto const& src : compilerPair.second) {
cmakefileStream << " \"" << src << "\" \""
<< this->MaybeConvertToRelativePath(
this->GetBinaryDirectory(), compilerPair.first)
<< "\" \"" << depFormat << "\" \""
<< this->MaybeConvertToRelativePath(
this->GetBinaryDirectory(), compilerPair.first)
<< ".d\"\n";
}
}
cmakefileStream << " )\n";
}
cmakefileStream << " )\n";
}
void cmLocalUnixMakefileGenerator3::WriteDisclaimer(std::ostream& os)