Merge topic 'fbuild_unity'

23fd5703ad FASTBuild: add support for Unity builds

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !11202
This commit is contained in:
Brad King
2025-09-19 13:26:45 +00:00
committed by Kitware Robot
21 changed files with 421 additions and 19 deletions

View File

@@ -57,6 +57,13 @@ std::string const COMPILE_FLAGS("COMPILE_FLAGS");
std::string const CMAKE_LANGUAGE("CMAKE");
std::string const INCLUDE_DIRECTORIES("INCLUDE_DIRECTORIES");
std::string const CMAKE_UNITY_BUILD("CMAKE_UNITY_BUILD");
std::string const CMAKE_UNITY_BUILD_BATCH_SIZE("CMAKE_UNITY_BUILD_BATCH_SIZE");
std::string const UNITY_BUILD("UNITY_BUILD");
std::string const UNITY_BUILD_BATCH_SIZE("UNITY_BUILD_BATCH_SIZE");
std::string const SKIP_UNITY_BUILD_INCLUSION("SKIP_UNITY_BUILD_INCLUSION");
std::string const UNITY_GROUP("UNITY_GROUP");
} // anonymous namespace
cmFastbuildNormalTargetGenerator::cmFastbuildNormalTargetGenerator(
@@ -841,7 +848,8 @@ void cmFastbuildNormalTargetGenerator::Generate()
fastbuildTarget.PostBuildExecNodes.Nodes.emplace_back(std::move(cc));
}
fastbuildTarget.ObjectListNodes = GenerateObjects();
GenerateObjects(fastbuildTarget);
std::vector<std::string> objectDepends;
AddObjectDependencies(fastbuildTarget, objectDepends);
@@ -1211,8 +1219,7 @@ std::vector<std::string> cmFastbuildNormalTargetGenerator::GetArches() const
return arches;
}
std::vector<FastbuildObjectListNode>
cmFastbuildNormalTargetGenerator::GenerateObjects()
void cmFastbuildNormalTargetGenerator::GenerateObjects(FastbuildTarget& target)
{
this->GetGlobalGenerator()->AllFoldersToClean.insert(ObjectOutDir);
@@ -1225,22 +1232,50 @@ cmFastbuildNormalTargetGenerator::GenerateObjects()
std::set<std::string> createdPCH;
// Directory level.
bool useUnity =
GeneratorTarget->GetLocalGenerator()->GetMakefile()->IsDefinitionSet(
CMAKE_UNITY_BUILD);
// Check if explicitly disabled for this target.
auto const targetProp = GeneratorTarget->GetProperty(UNITY_BUILD);
if (targetProp.IsSet() && targetProp.IsOff()) {
useUnity = false;
}
// List of sources isolated from the unity build if enabled.
std::set<std::string> isolatedFromUnity;
// Mapping from unity group (if any) to sources belonging to that group.
std::map<std::string, std::vector<std::string>> sourcesWithGroups;
for (cmSourceFile const* source : objectSources) {
cmSourceFile const& srcFile = *source;
std::string const pathToFile = srcFile.GetFullPath();
if (useUnity) {
// Check if the source should be added to "UnityInputIsolatedFiles".
if (srcFile.GetPropertyAsBool(SKIP_UNITY_BUILD_INCLUSION)) {
isolatedFromUnity.emplace(pathToFile);
}
std::string const perFileUnityGroup =
srcFile.GetSafeProperty(UNITY_GROUP);
if (!perFileUnityGroup.empty()) {
sourcesWithGroups[perFileUnityGroup].emplace_back(pathToFile);
}
}
this->GetGlobalGenerator()->AddFileToClean(cmStrCat(
ObjectOutDir, '/', this->GeneratorTarget->GetObjectName(source)));
// Do not generate separate node for PCH source file.
if (this->GeneratorTarget->GetPchSource(Config, srcFile.GetLanguage()) ==
srcFile.GetFullPath()) {
pathToFile) {
continue;
}
std::string const language = srcFile.GetLanguage();
LogMessage(cmStrCat("Source file: ",
this->ConvertToFastbuildPath(srcFile.GetFullPath())));
LogMessage(
cmStrCat("Source file: ", this->ConvertToFastbuildPath(pathToFile)));
LogMessage("Language: " + language);
std::string const staticCheckOptions = ComputeCodeCheckOptions(srcFile);
@@ -1278,7 +1313,7 @@ cmFastbuildNormalTargetGenerator::GenerateObjects()
nodesPermutations[objectListHash];
// Absolute path needed in "RunCMake.SymlinkTrees" test.
objectListNode.CompilerInputFiles.push_back(srcFile.GetFullPath());
objectListNode.CompilerInputFiles.push_back(pathToFile);
std::vector<std::string> const outputs =
GetSourceProperty(srcFile, "OBJECT_OUTPUTS");
@@ -1332,8 +1367,7 @@ cmFastbuildNormalTargetGenerator::GenerateObjects()
cmStrCat(objectListNode.Name, "_", std::to_string(++groupNameCount));
LogMessage(cmStrCat("ObjectList name: ", objectListNode.Name));
}
std::vector<FastbuildObjectListNode> objects;
std::vector<FastbuildObjectListNode>& objects = target.ObjectListNodes;
objects.reserve(nodesPermutations.size());
for (auto& val : nodesPermutations) {
auto& node = val.second;
@@ -1344,7 +1378,155 @@ cmFastbuildNormalTargetGenerator::GenerateObjects()
std::swap(*objects.begin(), objects.back());
}
}
return objects;
if (useUnity) {
target.UnityNodes =
GenerateUnity(objects, isolatedFromUnity, sourcesWithGroups);
}
}
FastbuildUnityNode cmFastbuildNormalTargetGenerator::GetOneUnity(
std::set<std::string> const& isolatedFiles, std::vector<std::string>& files,
int unitySize) const
{
FastbuildUnityNode result;
for (auto iter = files.begin(); iter != files.end();) {
std::string pathToFile = std::move(*iter);
iter = files.erase(iter);
// This source must be isolated
if (isolatedFiles.find(pathToFile) != isolatedFiles.end()) {
result.UnityInputFiles.emplace_back(pathToFile);
result.UnityInputIsolatedFiles.emplace_back(std::move(pathToFile));
} else {
result.UnityInputFiles.emplace_back(std::move(pathToFile));
}
if (int(result.UnityInputFiles.size() -
result.UnityInputIsolatedFiles.size()) == unitySize) {
break;
}
}
return result;
}
int cmFastbuildNormalTargetGenerator::GetUnityBatchSize() const
{
int unitySize = 8;
try {
auto const perTargetSize =
GeneratorTarget->GetSafeProperty(UNITY_BUILD_BATCH_SIZE);
if (!perTargetSize.empty()) {
unitySize = std::stoi(perTargetSize);
}
// Per-directory level.
else {
unitySize = std::stoi(
GeneratorTarget->GetLocalGenerator()->GetMakefile()->GetDefinition(
CMAKE_UNITY_BUILD_BATCH_SIZE));
}
} catch (...) {
return unitySize;
}
return unitySize;
}
std::vector<FastbuildUnityNode>
cmFastbuildNormalTargetGenerator::GenerateUnity(
std::vector<FastbuildObjectListNode>& objects,
std::set<std::string> const& isolatedSources,
std::map<std::string, std::vector<std::string>> const& sourcesWithGroups)
{
int const unitySize = GetUnityBatchSize();
// Unity of size less than 2 doesn't make sense.
if (unitySize < 2) {
return {};
}
int unityNumber = 0;
int unityGroupNumber = 0;
std::vector<FastbuildUnityNode> result;
for (FastbuildObjectListNode& obj : objects) {
// Don't use unity for only 1 file.
if (obj.CompilerInputFiles.size() < 2) {
continue;
}
std::string const ext =
cmSystemTools::GetFilenameExtension(obj.CompilerInputFiles[0]);
// Process groups.
auto groupedNode = GenerateGroupedUnityNode(
obj.CompilerInputFiles, sourcesWithGroups, unityGroupNumber);
// We have at least 2 sources in the group.
if (groupedNode.UnityInputFiles.size() > 1) {
groupedNode.UnityOutputPath = obj.CompilerOutputPath;
obj.CompilerInputUnity.emplace_back(groupedNode.Name);
groupedNode.UnityOutputPattern = cmStrCat(groupedNode.Name, ext);
result.emplace_back(std::move(groupedNode));
}
// General unity batching of the remaining (non-grouped) sources.
while (!obj.CompilerInputFiles.empty()) {
FastbuildUnityNode node =
GetOneUnity(isolatedSources, obj.CompilerInputFiles, unitySize);
node.Name =
cmStrCat(this->GetName(), "_Unity_", std::to_string(++unityNumber));
node.UnityOutputPath = obj.CompilerOutputPath;
node.UnityOutputPattern = cmStrCat(node.Name, ext);
// Unity group of size 1 doesn't make sense - just isolate the source.
if (groupedNode.UnityInputFiles.size() == 1) {
node.UnityInputIsolatedFiles.emplace_back(
groupedNode.UnityInputFiles[0]);
node.UnityInputFiles.emplace_back(
std::move(groupedNode.UnityInputFiles[0]));
// Clear so we don't enter here on the next iteration.
groupedNode.UnityInputFiles.clear();
}
// We've got only 1 file left. No need to create a Unity node for it,
// just return it back to the ObjectList and exit.
if (node.UnityInputFiles.size() == 1) {
obj.CompilerInputFiles.emplace_back(
std::move(node.UnityInputFiles[0]));
break;
}
obj.CompilerInputUnity.emplace_back(node.Name);
result.emplace_back(std::move(node));
}
}
return result;
}
FastbuildUnityNode cmFastbuildNormalTargetGenerator::GenerateGroupedUnityNode(
std::vector<std::string>& inputFiles,
std::map<std::string, std::vector<std::string>> const& sourcesWithGroups,
int& groupId)
{
std::vector<FastbuildUnityNode> result;
for (auto const& item : sourcesWithGroups) {
auto const& group = item.first;
auto const& sources = item.second;
FastbuildUnityNode node;
// Check if any of the sources belong to this group.
for (auto const& source : sources) {
auto const iter =
std::find(inputFiles.begin(), inputFiles.end(), source);
if (iter == inputFiles.end()) {
continue;
}
node.Name = cmStrCat(this->GetName(), "_Unity_Group_", group, '_',
std::to_string(++groupId));
node.UnityInputFiles.emplace_back(source);
// Remove from the general batching.
inputFiles.erase(
std::remove(inputFiles.begin(), inputFiles.end(), source),
inputFiles.end());
}
if (!node.UnityInputFiles.empty()) {
// The unity group belongs to the ObjectLists that we're processing.
// We've grouped all the sources we could from the current ObjectList.
return node;
}
}
return {};
}
std::string cmFastbuildNormalTargetGenerator::ResolveIfAlias(

View File

@@ -13,13 +13,9 @@
#include "cmComputeLinkInformation.h"
#include "cmFastbuildTargetGenerator.h"
#include "cmGeneratorTarget.h"
#include "cmGlobalFastbuildGenerator.h"
#include "cmRulePlaceholderExpander.h"
class cmSourceFile;
struct FastbuildExecNode;
struct FastbuildLinkerNode;
struct FastbuildObjectListNode;
struct FastbuildTarget;
struct FastbuildTargetDep;
class cmFastbuildNormalTargetGenerator : public cmFastbuildTargetGenerator
{
@@ -74,7 +70,21 @@ private:
std::vector<std::string> GetArches() const;
std::vector<FastbuildObjectListNode> GenerateObjects();
void GenerateObjects(FastbuildTarget& target);
FastbuildUnityNode GetOneUnity(std::set<std::string> const& isolatedFiles,
std::vector<std::string>& files,
int unitySize) const;
int GetUnityBatchSize() const;
std::vector<FastbuildUnityNode> GenerateUnity(
std::vector<FastbuildObjectListNode>& objects,
std::set<std::string> const& isolatedSources,
std::map<std::string, std::vector<std::string>> const& sourcesWithGroups);
FastbuildUnityNode GenerateGroupedUnityNode(
std::vector<std::string>& inputFiles,
std::map<std::string, std::vector<std::string>> const& sourcesWithGroups,
int& groupId);
// Computes .CompilerOptions for the ObjectList node.
void ComputeCompilerAndOptions(std::string const& compilerOptions,
std::string const& staticCheckOptions,

View File

@@ -616,7 +616,8 @@ void cmGlobalFastbuildGenerator::WriteVariable(std::string const& key,
int indent)
{
Indent(indent);
*this->BuildFileStream << "." << key << " " + op + " " << value << "\n";
*this->BuildFileStream << "." << key << " " + op + (value.empty() ? "" : " ")
<< value << "\n";
}
void cmGlobalFastbuildGenerator::WriteCommand(std::string const& command,
@@ -1164,6 +1165,24 @@ void cmGlobalFastbuildGenerator::WriteExec(FastbuildExecNode const& Exec,
}
}
void cmGlobalFastbuildGenerator::WriteUnity(FastbuildUnityNode const& Unity)
{
WriteCommand("Unity", Quote(Unity.Name), 1);
Indent(1);
*BuildFileStream << "{\n";
{
WriteVariable("UnityOutputPath", Quote(Unity.UnityOutputPath), 2);
WriteVariable("UnityOutputPattern", Quote(Unity.UnityOutputPattern), 2);
WriteArray("UnityInputFiles", Wrap(Unity.UnityInputFiles), 2);
if (!Unity.UnityInputIsolatedFiles.empty()) {
WriteArray("UnityInputIsolatedFiles",
Wrap(Unity.UnityInputIsolatedFiles), 2);
}
}
Indent(1);
*BuildFileStream << "}\n";
}
void cmGlobalFastbuildGenerator::WriteObjectList(
FastbuildObjectListNode const& ObjectList, bool allowDistribution)
{
@@ -1194,7 +1213,12 @@ void cmGlobalFastbuildGenerator::WriteObjectList(
WriteVariable("CompilerOutputExtension",
Quote(ObjectList.CompilerOutputExtension), 2);
WriteVariable("CompilerOutputKeepBaseExtension", "true", 2);
WriteArray("CompilerInputFiles", Wrap(ObjectList.CompilerInputFiles), 2);
if (!ObjectList.CompilerInputUnity.empty()) {
WriteArray("CompilerInputUnity", Wrap(ObjectList.CompilerInputUnity), 2);
}
if (!ObjectList.CompilerInputFiles.empty()) {
WriteArray("CompilerInputFiles", Wrap(ObjectList.CompilerInputFiles), 2);
}
if (!ObjectList.AllowCaching) {
WriteVariable("AllowCaching", "false", 2);
}
@@ -1350,6 +1374,11 @@ void cmGlobalFastbuildGenerator::WriteTarget(FastbuildTarget const& target)
this->WriteCopy(node);
}
// Unity.
for (FastbuildUnityNode const& unity : target.UnityNodes) {
this->WriteUnity(unity);
}
// Objects.
for (FastbuildObjectListNode const& objectList : target.ObjectListNodes) {
this->WriteObjectList(objectList, target.AllowDistribution);

View File

@@ -126,6 +126,7 @@ enum class FastbuildTargetType
EXEC, // Exec node
LINK, // Library, DLL or Executable
OBJECTLIST,
UNITY,
};
struct FastbuildTargetBase
@@ -209,6 +210,7 @@ struct FastbuildObjectListNode : public FastbuildTargetBase
std::string CompilerOptions;
std::string CompilerOutputPath;
std::string CompilerOutputExtension;
std::vector<std::string> CompilerInputUnity;
std::string PCHInputFile;
std::string PCHOutputFile;
std::string PCHOptions;
@@ -228,6 +230,18 @@ struct FastbuildObjectListNode : public FastbuildTargetBase
}
};
struct FastbuildUnityNode : public FastbuildTargetBase
{
std::string UnityOutputPath;
std::vector<std::string> UnityInputFiles;
std::string UnityOutputPattern;
std::vector<std::string> UnityInputIsolatedFiles;
FastbuildUnityNode()
: FastbuildTargetBase(FastbuildTargetType::UNITY)
{
}
};
struct IDEProjectConfig
{
std::string Config;
@@ -305,6 +319,7 @@ struct FastbuildTarget : public FastbuildTargetBase
{
std::map<std::string, std::string> Variables;
std::vector<FastbuildObjectListNode> ObjectListNodes;
std::vector<FastbuildUnityNode> UnityNodes;
// Potentially multiple libs for different archs (apple only);
std::vector<FastbuildLinkerNode> LinkerNode;
std::string RealOutput;
@@ -506,6 +521,7 @@ public:
void WriteTarget(FastbuildTarget const& target);
void WriteExec(FastbuildExecNode const& Exec, int indent = 1);
void WriteUnity(FastbuildUnityNode const& Unity);
void WriteObjectList(FastbuildObjectListNode const& ObjectList,
bool allowDistribution);
void WriteLinker(FastbuildLinkerNode const&, bool);

View File

@@ -3259,7 +3259,9 @@ cmLocalGenerator::AddUnityFilesModeGroup(
void cmLocalGenerator::AddUnityBuild(cmGeneratorTarget* target)
{
if (!target->GetPropertyAsBool("UNITY_BUILD")) {
// cmFastbuildNormalTargetGenerator handles unity build.
if (this->GetGlobalGenerator()->IsFastbuild() ||
!target->GetPropertyAsBool("UNITY_BUILD")) {
return;
}

View File

@@ -1,5 +1,11 @@
include(RunCMake)
# Unity of size 1 doesn't make sense and shouldn't be created.
run_cmake(Unity1)
run_cmake(Unity2)
run_cmake(UnityBatchSize)
run_cmake(UnityGroup)
run_cmake(UnityIsolate)
run_cmake(DisableCaching)
run_cmake(DisableDistribution)
run_cmake(SetCompilerProps)

View File

@@ -0,0 +1,10 @@
set(REGEX_TO_MATCH "
.*ObjectList.*
.*
\.CompilerInputFiles =
{
'.*main.cpp',
'.*some_source_file_1.cpp'
}
")
include(${RunCMake_SOURCE_DIR}/check.cmake)

View File

@@ -0,0 +1,4 @@
set(CMAKE_UNITY_BUILD ON)
set(CMAKE_UNITY_BUILD_BATCH_SIZE 1)
add_executable(main main.cpp some_source_file_1.cpp)

View File

@@ -0,0 +1,20 @@
set(REGEX_TO_MATCH "
Unity\\('main_Unity_1'\\)
{
\.UnityOutputPath = 'CMakeFiles/main.dir'
\.UnityOutputPattern = 'main_Unity_1.cpp'
\.UnityInputFiles =
{
.*main.cpp',
.*some_source_file_1.cpp'
}
}
.*ObjectList.*
.*
.CompilerInputUnity =
{
'main_Unity_1'
}
")
include(${RunCMake_SOURCE_DIR}/check.cmake)

View File

@@ -0,0 +1,3 @@
set(CMAKE_UNITY_BUILD ON)
add_executable(main main.cpp some_source_file_1.cpp)

View File

@@ -0,0 +1,31 @@
set(REGEX_TO_MATCH "
Unity\\('main_Unity_1'\\)
{
.*
.UnityOutputPattern = 'main_Unity_1.cpp'
.UnityInputFiles =
{
'.*main.cpp',
'.*some_source_file_1.cpp'
}
}
Unity\\('main_Unity_2'\\)
{
.*
.UnityOutputPattern = 'main_Unity_2.cpp'
.UnityInputFiles =
{
'.*some_source_file_2.cpp',
'.*some_source_file_3.cpp'
}
}
.*ObjectLis.*
.*
.CompilerInputUnity =
{
'main_Unity_1',
'main_Unity_2'
}
")
include(${RunCMake_SOURCE_DIR}/check.cmake)

View File

@@ -0,0 +1,3 @@
set(CMAKE_UNITY_BUILD ON)
set(CMAKE_UNITY_BUILD_BATCH_SIZE 2)
add_executable(main main.cpp some_source_file_1.cpp some_source_file_2.cpp some_source_file_3.cpp)

View File

@@ -0,0 +1,23 @@
set(REGEX_TO_MATCH "
Unity\\('main_Unity_Group_TestGroup_2'\\)
.*
.UnityOutputPattern = 'main_Unity_Group_TestGroup_2.cpp'
.UnityInputFiles =
{
'.*some_source_file_1.cpp',
'.*some_source_file_2.cpp'
}
.*
ObjectList.*
.*
\.CompilerInputUnity =
{
'main_Unity_Group_TestGroup_2'
}
\.CompilerInputFiles =
{
'.*main.cpp'
}
")
include(${RunCMake_SOURCE_DIR}/check.cmake)

View File

@@ -0,0 +1,12 @@
set(CMAKE_UNITY_BUILD ON)
add_executable(main main.cpp some_source_file_1.cpp some_source_file_2.cpp)
set_source_files_properties(
some_source_file_1.cpp
some_source_file_2.cpp
TARGET_DIRECTORY
main
PROPERTIES
UNITY_GROUP "TestGroup"
)

View File

@@ -0,0 +1,27 @@
set(REGEX_TO_MATCH "
Unity\\('main_Unity_1'\\)
{
.*
.UnityInputFiles =
{
'.*main.cpp',
'.*some_source_file_1.cpp',
'.*some_source_file_2.cpp',
'.*some_source_file_3.cpp',
'.*some_source_file_4.cpp'
}
.UnityInputIsolatedFiles =
{
'.*some_source_file_1.cpp',
'.*some_source_file_4.cpp'
}
}
.*ObjectList.*
.*
.CompilerInputUnity =
{
'main_Unity_1'
}
")
include(${RunCMake_SOURCE_DIR}/check.cmake)

View File

@@ -0,0 +1,18 @@
set(CMAKE_UNITY_BUILD ON)
add_executable(main
main.cpp
some_source_file_1.cpp
some_source_file_2.cpp
some_source_file_3.cpp
some_source_file_4.cpp
)
set_source_files_properties(
some_source_file_1.cpp
some_source_file_4.cpp
TARGET_DIRECTORY
main
PROPERTIES
SKIP_UNITY_BUILD_INCLUSION ON
)

View File

@@ -1,5 +1,11 @@
include(RunCMake)
# FASTBuild generator does Unity build natively
# and has its own tests for it in "RunCMake/FASTBuild" folder.
if(RunCMake_GENERATOR STREQUAL "FASTBuild")
return()
endif()
function(run_build name)
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/${name}-build)
run_cmake(${name})