CPack: add CPACK_THREADS variable to control compression threads

This allows setting how many threads the compressor will use.
Currently only implemented for XZ when using system's lzma library.

Fixes: #21715

Help/cpack_gen/archive.rst

@@ -80,6 +80,8 @@ CPack generators which are essentially archives at their core. These include:
   not all compression modes support threading in all environments. Currently,
   only the XZ compression may support it.
 
+  See also the :variable:`CPACK_THREADS` variable.
+
 .. note::
 
     Official CMake binaries available on ``cmake.org`` ship with a ``liblzma``

Help/release/dev/cpack-compression-threads.rst

@@ -0,0 +1,6 @@
+cpack-compression-threads
+-------------------------
+
+* :module:`CPack` gained the :variable:`CPACK_THREADS` variable to
+  control the number of threads used for parallelized operations,
+  such as compressing the installer package.

Modules/CPack.cmake

@@ -282,6 +282,28 @@ installers.  The most commonly-used variables are:
   received by the cpack program.  Defaults to ``FALSE`` for backwards
   compatibility.
 
+.. variable:: CPACK_THREADS
+
+  .. versionadded:: 3.20
+
+  Number of threads to use when performing parallelized operations, such
+  as compressing the installer package.
+
+  Some compression methods used by CPack generators such as Debian or Archive
+  may take advantage of multiple CPU cores to speed up compression.
+  ``CPACK_THREADS`` can be set to positive integer to specify how many threads
+  will be used for compression. If it is set to 0, CPack will set it so that
+  all available CPU cores are used.
+  By default ``CPACK_THREADS`` is set to ``1``.
+
+  Currently only ``xz`` compression *may* take advantage of multiple cores. Other
+  compression methods ignore this value and use only one thread.
+
+  .. note::
+
+     Official CMake binaries available on ``cmake.org`` ship with a ``liblzma``
+     that does not support parallel compression.
+
 Variables for Source Package Generators
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -746,6 +768,7 @@ _cpack_set_default(CPACK_INSTALL_CMAKE_PROJECTS
   "${CMAKE_BINARY_DIR};${CMAKE_PROJECT_NAME};ALL;/")
 _cpack_set_default(CPACK_CMAKE_GENERATOR "${CMAKE_GENERATOR}")
 _cpack_set_default(CPACK_TOPLEVEL_TAG "${CPACK_SYSTEM_NAME}")
+_cpack_set_default(CPACK_THREADS 1)
 # if the user has set CPACK_NSIS_DISPLAY_NAME remember it
 if(DEFINED CPACK_NSIS_DISPLAY_NAME)
   set(CPACK_NSIS_DISPLAY_NAME_SET TRUE)

Source/CPack/cmCPackArchiveGenerator.cxx

@@ -353,8 +353,12 @@ bool cmCPackArchiveGenerator::SetArchiveOptions(cmArchiveWrite* archive)
   // cause spurious errors to be raised from `strtoull`.
   if (this->Compress == cmArchiveWrite::CompressXZ) {
     const char* threads = "1";
+
+    // CPACK_ARCHIVE_THREADS overrides CPACK_THREADS
     if (this->IsSet("CPACK_ARCHIVE_THREADS")) {
       threads = this->GetOption("CPACK_ARCHIVE_THREADS");
+    } else if (this->IsSet("CPACK_THREADS")) {
+      threads = this->GetOption("CPACK_THREADS");
     }
 
     if (!archive->SetFilterOption("xz", "threads", threads)) {

Source/CPack/cmCPackDebGenerator.cxx

@@ -2,6 +2,7 @@
    file Copyright.txt or https://cmake.org/licensing for details.  */
 #include "cmCPackDebGenerator.h"
 
+#include <cstdlib>
 #include <cstring>
 #include <map>
 #include <ostream>
@@ -28,7 +29,7 @@ class DebGenerator
 public:
   DebGenerator(cmCPackLog* logger, std::string outputName, std::string workDir,
                std::string topLevelDir, std::string temporaryDir,
-               const char* debianCompressionType,
+               const char* debianCompressionType, const char* numThreads,
                const char* debianArchiveType,
                std::map<std::string, std::string> controlValues,
                bool genShLibs, std::string shLibsFilename, bool genPostInst,
@@ -53,6 +54,7 @@ private:
   const std::string TopLevelDir;
   const std::string TemporaryDir;
   const char* DebianArchiveType;
+  int NumThreads;
   const std::map<std::string, std::string> ControlValues;
   const bool GenShLibs;
   const std::string ShLibsFilename;
@@ -69,7 +71,8 @@ private:
 DebGenerator::DebGenerator(
   cmCPackLog* logger, std::string outputName, std::string workDir,
   std::string topLevelDir, std::string temporaryDir,
-  const char* debianCompressionType, const char* debianArchiveType,
+  const char* debianCompressionType, const char* numThreads,
+  const char* debianArchiveType,
   std::map<std::string, std::string> controlValues, bool genShLibs,
   std::string shLibsFilename, bool genPostInst, std::string postInst,
   bool genPostRm, std::string postRm, const char* controlExtra,
@@ -115,6 +118,23 @@ DebGenerator::DebGenerator(
                   "Error unrecognized compression type: "
                     << debianCompressionType << std::endl);
   }
+
+  if (numThreads == nullptr) {
+    numThreads = "1";
+  }
+
+  char* endptr;
+  this->NumThreads = static_cast<int>(strtol(numThreads, &endptr, 10));
+  if (numThreads != endptr && *endptr != '\0') {
+    cmCPackLogger(cmCPackLog::LOG_ERROR,
+                  "Unrecognized number of threads: " << numThreads
+                                                     << std::endl);
+  }
+
+  if (this->NumThreads < 0) {
+    cmCPackLogger(cmCPackLog::LOG_ERROR,
+                  "Number of threads cannot be negative" << std::endl);
+  }
 }
 
 bool DebGenerator::generate() const
@@ -173,7 +193,7 @@ bool DebGenerator::generateDataTar() const
     return false;
   }
   cmArchiveWrite data_tar(fileStream_data_tar, this->TarCompressionType,
-                          this->DebianArchiveType);
+                          this->DebianArchiveType, 0, this->NumThreads);
   data_tar.Open();
 
   // uid/gid should be the one of the root user, and this root user has
@@ -807,6 +827,7 @@ int cmCPackDebGenerator::createDeb()
     this->GetOption("CPACK_TOPLEVEL_DIRECTORY"),
     this->GetOption("CPACK_TEMPORARY_DIRECTORY"),
     this->GetOption("GEN_CPACK_DEBIAN_COMPRESSION_TYPE"),
+    this->GetOption("CPACK_THREADS"),
     this->GetOption("GEN_CPACK_DEBIAN_ARCHIVE_TYPE"), controlValues, gen_shibs,
     shlibsfilename, this->IsOn("GEN_CPACK_DEBIAN_GENERATE_POSTINST"), postinst,
     this->IsOn("GEN_CPACK_DEBIAN_GENERATE_POSTRM"), postrm,
@@ -864,6 +885,7 @@ int cmCPackDebGenerator::createDbgsymDDeb()
     this->GetOption("CPACK_TOPLEVEL_DIRECTORY"),
     this->GetOption("CPACK_TEMPORARY_DIRECTORY"),
     this->GetOption("GEN_CPACK_DEBIAN_COMPRESSION_TYPE"),
+    this->GetOption("CPACK_THREADS"),
     this->GetOption("GEN_CPACK_DEBIAN_ARCHIVE_TYPE"), controlValues, false, "",
     false, "", false, "", nullptr,
     this->IsSet("GEN_CPACK_DEBIAN_PACKAGE_CONTROL_STRICT_PERMISSION"),

Source/cmArchiveWrite.cxx

@@ -2,6 +2,7 @@
    file Copyright.txt or https://cmake.org/licensing for details.  */
 #include "cmArchiveWrite.h"
 
+#include <cstdio>
 #include <cstring>
 #include <ctime>
 #include <iostream>
@@ -81,7 +82,8 @@ struct cmArchiveWrite::Callback
 };
 
 cmArchiveWrite::cmArchiveWrite(std::ostream& os, Compress c,
-                               std::string const& format, int compressionLevel)
+                               std::string const& format, int compressionLevel,
+                               int numThreads)
   : Stream(os)
   , Archive(archive_write_new())
   , Disk(archive_read_disk_new())
@@ -142,6 +144,18 @@ cmArchiveWrite::cmArchiveWrite(std::ostream& os, Compress c,
                                cm_archive_error_string(this->Archive));
         return;
       }
+      {
+        char sNumThreads[8];
+        snprintf(sNumThreads, sizeof(sNumThreads), "%d", numThreads);
+        sNumThreads[7] = '\0'; // for safety
+        if (archive_write_set_filter_option(this->Archive, "xz", "threads",
+                                            sNumThreads) != ARCHIVE_OK) {
+          this->Error = cmStrCat("archive_compressor_xz_options: ",
+                                 cm_archive_error_string(this->Archive));
+          return;
+        }
+      }
+
       break;
     case CompressZstd:
       if (archive_write_add_filter_zstd(this->Archive) != ARCHIVE_OK) {

Source/cmArchiveWrite.h

@@ -54,7 +54,8 @@ public:
 
   /** Construct with output stream to which to write archive.  */
   cmArchiveWrite(std::ostream& os, Compress c = CompressNone,
-                 std::string const& format = "paxr", int compressionLevel = 0);
+                 std::string const& format = "paxr", int compressionLevel = 0,
+                 int numThreads = 1);
 
   ~cmArchiveWrite();
 

Tests/RunCMake/CPack/RunCMakeTest.cmake

@@ -21,8 +21,8 @@ run_cpack_test(LONG_FILENAMES "DEB.LONG_FILENAMES" false "MONOLITHIC")
 run_cpack_test_subtests(MAIN_COMPONENT "invalid;found" "RPM.MAIN_COMPONENT" false "COMPONENT")
 run_cpack_test(MINIMAL "RPM.MINIMAL;DEB.MINIMAL;7Z;TBZ2;TGZ;TXZ;TZ;ZIP;STGZ;External" false "MONOLITHIC;COMPONENT")
 run_cpack_test_package_target(MINIMAL "RPM.MINIMAL;DEB.MINIMAL;7Z;TBZ2;TGZ;TXZ;TZ;ZIP;STGZ;External" false "MONOLITHIC;COMPONENT")
-run_cpack_test_package_target(THREADED_ALL "TXZ" false "MONOLITHIC;COMPONENT")
-run_cpack_test_package_target(THREADED "TXZ" false "MONOLITHIC;COMPONENT")
+run_cpack_test_package_target(THREADED_ALL "TXZ;DEB" false "MONOLITHIC;COMPONENT")
+run_cpack_test_package_target(THREADED "TXZ;DEB" false "MONOLITHIC;COMPONENT")
 run_cpack_test_subtests(PACKAGE_CHECKSUM "invalid;MD5;SHA1;SHA224;SHA256;SHA384;SHA512" "TGZ" false "MONOLITHIC")
 run_cpack_test(PARTIALLY_RELOCATABLE_WARNING "RPM.PARTIALLY_RELOCATABLE_WARNING" false "COMPONENT")
 run_cpack_test(PER_COMPONENT_FIELDS "RPM.PER_COMPONENT_FIELDS;DEB.PER_COMPONENT_FIELDS" false "COMPONENT")

Tests/RunCMake/CPack/tests/THREADED/DEB-Prerequirements.cmake

@@ -0,0 +1 @@
+set(CPACK_DEBIAN_COMPRESSION_TYPE xz)

Tests/RunCMake/CPack/tests/THREADED/test.cmake

@@ -1,6 +1,6 @@
 install(FILES CMakeLists.txt DESTINATION foo COMPONENT test)
 
-set(CPACK_ARCHIVE_THREADS 2)
+set(CPACK_THREADS 2)
 
 if(PACKAGING_TYPE STREQUAL "COMPONENT")
   set(CPACK_COMPONENTS_ALL test)

Tests/RunCMake/CPack/tests/THREADED_ALL/DEB-Prerequirements.cmake

@@ -0,0 +1 @@
+set(CPACK_DEBIAN_COMPRESSION_TYPE xz)

Tests/RunCMake/CPack/tests/THREADED_ALL/test.cmake

@@ -1,6 +1,6 @@
 install(FILES CMakeLists.txt DESTINATION foo COMPONENT test)
 
-set(CPACK_ARCHIVE_THREADS 0)
+set(CPACK_THREADS 0)
 
 if(PACKAGING_TYPE STREQUAL "COMPONENT")
   set(CPACK_COMPONENTS_ALL test)