Merge topic 'fix-27420'

4fdfa0db1a file(ARCHIVE_CREATE): support multithreading compression
cbf71b21b2 cli tar: support multithreading compression

Acked-by: Kitware Robot <kwrobot@kitware.com>
Tested-by: buildbot <buildbot@kitware.com>
Merge-request: !11460

Help/command/file.rst

@@ -915,6 +915,7 @@ Archiving
     [COMPRESSION <compression>
     [COMPRESSION_LEVEL <compression-level>]]
     [MTIME <mtime>]
+    [THREADS <number>]
     [WORKING_DIRECTORY <dir>]
     [VERBOSE])
   :target: ARCHIVE_CREATE
@@ -975,6 +976,14 @@ Archiving
   ``MTIME <mtime>``
     Specify the modification time recorded in tarball entries.
 
+  ``THREADS <number>``
+    .. versionadded:: 4.3
+
+    Use the ``<number>`` threads to operate on the archive.
+
+    The number of available cores on the machine will be used if set to ``0``.
+    Note that not all compression modes support threading in all environments.
+
   ``WORKING_DIRECTORY <dir>``
     .. versionadded:: 3.31
 

Help/manual/cmake.1.rst

@@ -1497,6 +1497,17 @@ Available commands are:
 
     Specify modification time recorded in tarball entries.
 
+  .. option:: --cmake-tar-threads=<number>
+
+    .. versionadded:: 4.3
+
+    Use the ``<number>`` threads to operate on the archive. Currently only
+    multi-threaded compression is supported.
+
+    If set to ``0``, the number of available cores on the machine will be
+    used instead. Note that not all compression modes support threading
+    in all environments.
+
   .. option:: --touch
 
     .. versionadded:: 3.24

Help/release/dev/cli-tar-multithread.rst

@@ -0,0 +1,6 @@
+cli-tar-multithread
+-------------------
+
+* The :manual:`cmake(1)` ``-E tar`` tool supports multithreading operations
+* The :command:`file(ARCHIVE_CREATE)` command supports multithreading via the
+  ``THREADS`` option

Source/cmFileCommand.cxx

@@ -3666,6 +3666,7 @@ bool HandleArchiveCreateCommand(std::vector<std::string> const& args,
     // accepted without one and treated as if an empty value were given.
     // Fixing this would require a policy.
     ArgumentParser::Maybe<std::string> MTime;
+    std::string Threads;
     std::string WorkingDirectory;
     bool Verbose = false;
     // "PATHS" requires at least one value, but use a custom check below.
@@ -3679,6 +3680,7 @@ bool HandleArchiveCreateCommand(std::vector<std::string> const& args,
       .Bind("COMPRESSION"_s, &Arguments::Compression)
       .Bind("COMPRESSION_LEVEL"_s, &Arguments::CompressionLevel)
       .Bind("MTIME"_s, &Arguments::MTime)
+      .Bind("THREADS"_s, &Arguments::Threads)
       .Bind("WORKING_DIRECTORY"_s, &Arguments::WorkingDirectory)
       .Bind("VERBOSE"_s, &Arguments::Verbose)
       .Bind("PATHS"_s, &Arguments::Paths);
@@ -3740,7 +3742,7 @@ bool HandleArchiveCreateCommand(std::vector<std::string> const& args,
   }
 
   int compressionLevel = 0;
-  int minCompressionLevel = 0;
+  constexpr int minCompressionLevel = 0;
   int maxCompressionLevel = 9;
   if (compress == cmSystemTools::TarCompressZstd) {
     maxCompressionLevel = 19;
@@ -3775,16 +3777,36 @@ bool HandleArchiveCreateCommand(std::vector<std::string> const& args,
     }
   }
 
+  // Use the single thread by default for backward compatibility
+  int threads = 1;
+  constexpr int minThreads = 0;
+  if (!parsedArgs.Threads.empty()) {
+    if (parsedArgs.Threads.size() != 1 &&
+        !std::isdigit(parsedArgs.Threads[0])) {
+      status.SetError(cmStrCat("number of threads ", parsedArgs.Threads,
+                               " should be at least ", minThreads));
+      cmSystemTools::SetFatalErrorOccurred();
+      return false;
+    }
+    threads = std::stoi(parsedArgs.Threads);
+    if (threads < minThreads) {
+      status.SetError(cmStrCat("number of threads ", parsedArgs.Threads,
+                               " should be at least ", minThreads));
+      cmSystemTools::SetFatalErrorOccurred();
+      return false;
+    }
+  }
+
   if (parsedArgs.Paths.empty()) {
     status.SetError("ARCHIVE_CREATE requires a non-empty list of PATHS");
     cmSystemTools::SetFatalErrorOccurred();
     return false;
   }
 
-  if (!cmSystemTools::CreateTar(parsedArgs.Output, parsedArgs.Paths,
-                                parsedArgs.WorkingDirectory, compress,
-                                parsedArgs.Verbose, parsedArgs.MTime,
-                                parsedArgs.Format, compressionLevel)) {
+  if (!cmSystemTools::CreateTar(
+        parsedArgs.Output, parsedArgs.Paths, parsedArgs.WorkingDirectory,
+        compress, parsedArgs.Verbose, parsedArgs.MTime, parsedArgs.Format,
+        compressionLevel, threads)) {
     status.SetError(cmStrCat("failed to compress: ", parsedArgs.Output));
     cmSystemTools::SetFatalErrorOccurred();
     return false;

Source/cmSystemTools.cxx

@@ -2378,7 +2378,8 @@ bool cmSystemTools::CreateTar(std::string const& outFileName,
                               std::string const& workingDirectory,
                               cmTarCompression compressType, bool verbose,
                               std::string const& mtime,
-                              std::string const& format, int compressionLevel)
+                              std::string const& format, int compressionLevel,
+                              int numThreads)
 {
 #if !defined(CMAKE_BOOTSTRAP)
   cmWorkingDirectory workdir(cmSystemTools::GetLogicalWorkingDirectory());
@@ -2417,7 +2418,7 @@ bool cmSystemTools::CreateTar(std::string const& outFileName,
   }
 
   cmArchiveWrite a(fout, compress, format.empty() ? "paxr" : format,
-                   compressionLevel);
+                   compressionLevel, numThreads);
 
   if (!a.Open()) {
     cmSystemTools::Error(a.GetError());

Source/cmSystemTools.h

@@ -567,7 +567,7 @@ public:
                         cmTarCompression compressType, bool verbose,
                         std::string const& mtime = std::string(),
                         std::string const& format = std::string(),
-                        int compressionLevel = 0);
+                        int compressionLevel = 0, int numThreads = 1);
   static bool ExtractTar(std::string const& inFileName,
                          std::vector<std::string> const& files,
                          cmTarExtractTimestamps extractTimestamps,

Source/cmcmd.cxx

@@ -5,6 +5,7 @@
 #include <functional>
 #include <iomanip>
 #include <iterator>
+#include <limits>
 
 #include <cm/optional>
 #include <cmext/algorithm>
@@ -1582,6 +1583,7 @@ int cmcmd::ExecuteCMakeCommand(std::vector<std::string> const& args,
       std::vector<std::string> files;
       std::string mtime;
       std::string format;
+      int numThreads = 1;
       cmSystemTools::cmTarExtractTimestamps extractTimestamps =
         cmSystemTools::cmTarExtractTimestamps::Yes;
       cmSystemTools::cmTarCompression compress =
@@ -1600,6 +1602,31 @@ int cmcmd::ExecuteCMakeCommand(std::vector<std::string> const& args,
             ++nCompress;
           } else if (cmHasLiteralPrefix(arg, "--mtime=")) {
             mtime = arg.substr(8);
+          } else if (cmHasLiteralPrefix(arg, "--cmake-tar-threads=")) {
+            std::string const& numThreadsStr = arg.substr(20);
+            long numThreadsLong = 0;
+            if (!cmStrToLong(numThreadsStr, &numThreadsLong)) {
+              cmSystemTools::Error(
+                cmStrCat("Invalid --cmake-tar-threads value: '", numThreadsStr,
+                         "' - not a number"));
+              return 1;
+            }
+            if (numThreadsLong >
+                std::numeric_limits<decltype(numThreads)>::max()) {
+              cmSystemTools::Error(
+                cmStrCat("Invalid --cmake-tar-threads value: '", numThreadsStr,
+                         "' - too large"));
+              return 1;
+            }
+            if (numThreadsLong <
+                std::numeric_limits<decltype(numThreads)>::min()) {
+              cmSystemTools::Error(
+                cmStrCat("Invalid --cmake-tar-threads value: '", numThreadsStr,
+                         "' - too small"));
+              return 1;
+            }
+
+            numThreads = static_cast<decltype(numThreads)>(numThreadsLong);
           } else if (cmHasLiteralPrefix(arg, "--files-from=")) {
             std::string const& files_from = arg.substr(13);
             if (!cmTarFilesFrom(files_from, files)) {
@@ -1680,7 +1707,7 @@ int cmcmd::ExecuteCMakeCommand(std::vector<std::string> const& args,
           std::cerr << "tar: No files or directories specified\n";
         }
         if (!cmSystemTools::CreateTar(outFile, files, {}, compress, verbose,
-                                      mtime, format)) {
+                                      mtime, format, 0, numThreads)) {
           cmSystemTools::Error("Problem creating tar: " + outFile);
           return 1;
         }

Tests/RunCMake/CommandLineTar/RunCMakeTest.cmake

@@ -33,6 +33,16 @@ run_cmake(paxr)
 run_cmake(paxr-bz2)
 run_cmake(zip)
 
+# Check the --cmake-tar-threads option
+external_command_test(bad-threads-not-a-number  tar cvf bad.tar --cmake-tar-threads=nan .)
+
+run_cmake(threads-7zip)
+run_cmake(threads-bz2)
+run_cmake(threads-gz)
+run_cmake(threads-xz)
+run_cmake(threads-zstd)
+run_cmake(threads-zip)
+
 # Extracting only selected files or directories
 run_cmake(zip-filtered)
 

Tests/RunCMake/CommandLineTar/bad-threads-not-a-number-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CommandLineTar/bad-threads-not-a-number-stderr.txt

@@ -0,0 +1 @@
+^CMake Error: Invalid --cmake-tar-threads value: 'nan' - not a number$

Tests/RunCMake/CommandLineTar/threads-7zip.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.7z")
+
+set(COMPRESSION_FLAGS cvf)
+set(COMPRESSION_OPTIONS --format=7zip --cmake-tar-threads=4)
+
+set(DECOMPRESSION_FLAGS xvf)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("377abcaf271c" LIMIT 6 HEX)

Tests/RunCMake/CommandLineTar/threads-bz2.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.tar.bz2")
+
+set(COMPRESSION_FLAGS cvjf)
+set(COMPRESSION_OPTIONS --cmake-tar-threads=4)
+
+set(DECOMPRESSION_FLAGS xvjf)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("425a68" LIMIT 3 HEX)

Tests/RunCMake/CommandLineTar/threads-gz.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.tar.gz")
+
+set(COMPRESSION_FLAGS -cvzf)
+set(COMPRESSION_OPTIONS --cmake-tar-threads=4)
+
+set(DECOMPRESSION_FLAGS -xvzf)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("1f8b" LIMIT 2 HEX)

Tests/RunCMake/CommandLineTar/threads-xz.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.tar.xz")
+
+set(COMPRESSION_FLAGS cvJf)
+set(COMPRESSION_OPTIONS --cmake-tar-threads=4)
+
+set(DECOMPRESSION_FLAGS xvJf)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("fd377a585a00" LIMIT 6 HEX)

Tests/RunCMake/CommandLineTar/threads-zip.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.zip")
+
+set(COMPRESSION_FLAGS cvf)
+set(COMPRESSION_OPTIONS --format=zip --cmake-tar-threads=4)
+
+set(DECOMPRESSION_FLAGS xvf)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("504b0304" LIMIT 4 HEX)

Tests/RunCMake/CommandLineTar/threads-zstd.cmake

@@ -0,0 +1,11 @@
+set(OUTPUT_NAME "test.tar.zstd")
+
+set(COMPRESSION_FLAGS cvf)
+set(COMPRESSION_OPTIONS --zstd --cmake-tar-threads=4)
+
+set(DECOMPRESSION_FLAGS xvf)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+# libarchive 3.8.2 enables a checksum feature; older versions do not.
+check_magic("^28b52ffd0[04]58$" LIMIT 6 HEX)

Tests/RunCMake/File_Archive/RunCMakeTest.cmake

@@ -14,6 +14,13 @@ run_cmake(zip)
 
 run_cmake(working-directory)
 
+# Check the --cmake-tar-threads option
+run_cmake(argument-validation-threads)
+run_cmake(threads-bz2)
+run_cmake(threads-gz)
+run_cmake(threads-xz)
+run_cmake(threads-zstd)
+
 # Extracting only selected files or directories
 run_cmake(zip-filtered)
 

Tests/RunCMake/File_Archive/argument-validation-compression-level-1-stderr.txt

@@ -1,5 +1,5 @@
-CMake Error at compression-level\.cmake:39 \(file\):
+^CMake Error at compression-level\.cmake:39 \(file\):
   file compression level 100 for GZip should be in range 0 to 9
 Call Stack \(most recent call first\):
   argument-validation-compression-level-1\.cmake:8 \(check_compression_level\)
-  CMakeLists\.txt:3 \(include\)
+  CMakeLists\.txt:3 \(include\)$

Tests/RunCMake/File_Archive/argument-validation-compression-level-2-stderr.txt

@@ -1,5 +1,5 @@
-CMake Error at compression-level\.cmake:39 \(file\):
+^CMake Error at compression-level\.cmake:39 \(file\):
   file compression level high for GZip should be in range 0 to 9
 Call Stack \(most recent call first\):
   argument-validation-compression-level-2\.cmake:8 \(check_compression_level\)
-  CMakeLists\.txt:3 \(include\)
+  CMakeLists\.txt:3 \(include\)$

Tests/RunCMake/File_Archive/argument-validation-threads-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/File_Archive/argument-validation-threads-stderr.txt

@@ -0,0 +1,5 @@
+^CMake Error at threads\.cmake:39 \(file\):
+  file number of threads -1 should be at least 0
+Call Stack \(most recent call first\):
+  argument-validation-threads\.cmake:8 \(check_threads\)
+  CMakeLists\.txt:3 \(include\)$

Tests/RunCMake/File_Archive/argument-validation-threads.cmake

@@ -0,0 +1,8 @@
+set(OUTPUT_NAME "test.tar.gz")
+
+set(ARCHIVE_FORMAT gnutar)
+set(COMPRESSION_TYPE GZip)
+
+include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake)
+
+check_threads("-1")

Tests/RunCMake/File_Archive/roundtrip.cmake

@@ -95,20 +95,3 @@ function(check_magic EXPECTED)
       "Actual [${ACTUAL}] does not match expected [${EXPECTED}]")
   endif()
 endfunction()
-
-
-function(check_compression_level COMPRESSION_LEVEL)
-  file(ARCHIVE_CREATE
-    OUTPUT "${FULL_OUTPUT_NAME}_compression_level"
-    FORMAT "${ARCHIVE_FORMAT}"
-    COMPRESSION_LEVEL ${COMPRESSION_LEVEL}
-    COMPRESSION "${COMPRESSION_TYPE}"
-    VERBOSE
-    PATHS ${COMPRESS_DIR})
-
-  file(ARCHIVE_EXTRACT
-    INPUT "${FULL_OUTPUT_NAME}_compression_level"
-    ${DECOMPRESSION_OPTIONS}
-    DESTINATION ${FULL_DECOMPRESS_DIR}
-    VERBOSE)
-endfunction()

Tests/RunCMake/File_Archive/threads-bz2.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.tar.bz2")
+
+set(ARCHIVE_FORMAT paxr)
+set(COMPRESSION_TYPE BZip2)
+
+include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake)
+
+check_threads("0")
+check_threads("1")
+check_threads("4")

Tests/RunCMake/File_Archive/threads-gz.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.tar.gz")
+
+set(ARCHIVE_FORMAT gnutar)
+set(COMPRESSION_TYPE GZip)
+
+include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake)
+
+check_threads("0")
+check_threads("1")
+check_threads("4")

Tests/RunCMake/File_Archive/threads-xz.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.tar.xz")
+
+set(ARCHIVE_FORMAT pax)
+set(COMPRESSION_TYPE XZ)
+
+include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake)
+
+check_threads("0")
+check_threads("1")
+check_threads("4")

Tests/RunCMake/File_Archive/threads-zstd.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.tar.zstd")
+
+set(ARCHIVE_FORMAT pax)
+set(COMPRESSION_TYPE Zstd)
+
+include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake)
+
+check_threads("0")
+check_threads("1")
+check_threads("4")

Tests/RunCMake/File_Archive/threads.cmake

@@ -0,0 +1,85 @@
+foreach(parameter OUTPUT_NAME ARCHIVE_FORMAT)
+  if(NOT DEFINED ${parameter})
+    message(FATAL_ERROR "missing required parameter ${parameter}")
+  endif()
+endforeach()
+
+set(COMPRESS_DIR compress_dir)
+set(FULL_COMPRESS_DIR ${CMAKE_CURRENT_BINARY_DIR}/${COMPRESS_DIR})
+
+set(DECOMPRESS_DIR decompress_dir)
+set(FULL_DECOMPRESS_DIR ${CMAKE_CURRENT_BINARY_DIR}/${DECOMPRESS_DIR})
+
+set(FULL_OUTPUT_NAME ${CMAKE_CURRENT_BINARY_DIR}/${OUTPUT_NAME})
+
+set(CHECK_FILES
+  "f1.txt"
+  "d1/f1.txt"
+  "d 2/f1.txt"
+  "d + 3/f1.txt"
+  "d_4/f1.txt"
+  "d-4/f1.txt"
+  "My Special Directory/f1.txt"
+)
+
+function(check_threads THREADS)
+  foreach(file ${CHECK_FILES})
+    configure_file(${CMAKE_CURRENT_LIST_FILE} ${FULL_COMPRESS_DIR}/${file} COPYONLY)
+  endforeach()
+
+  if(UNIX)
+    execute_process(COMMAND ln -sf f1.txt ${FULL_COMPRESS_DIR}/d1/f2.txt)
+    list(APPEND CHECK_FILES "d1/f2.txt")
+  endif()
+
+  file(REMOVE ${FULL_OUTPUT_NAME})
+  file(REMOVE_RECURSE ${FULL_DECOMPRESS_DIR})
+  file(MAKE_DIRECTORY ${FULL_DECOMPRESS_DIR})
+
+  file(ARCHIVE_CREATE
+    OUTPUT ${FULL_OUTPUT_NAME}
+    FORMAT "${ARCHIVE_FORMAT}"
+    COMPRESSION "${COMPRESSION_TYPE}"
+    THREADS ${THREADS}
+    VERBOSE
+    PATHS ${COMPRESS_DIR})
+
+  file(ARCHIVE_EXTRACT
+    INPUT ${FULL_OUTPUT_NAME}
+    ${DECOMPRESSION_OPTIONS}
+    DESTINATION ${FULL_DECOMPRESS_DIR}
+    VERBOSE)
+
+  if(CUSTOM_CHECK_FILES)
+    set(CHECK_FILES ${CUSTOM_CHECK_FILES})
+  endif()
+
+  foreach(file ${CHECK_FILES})
+    set(input ${FULL_COMPRESS_DIR}/${file})
+    set(output ${FULL_DECOMPRESS_DIR}/${COMPRESS_DIR}/${file})
+
+    if(NOT EXISTS ${input})
+      message(SEND_ERROR "Cannot find input file ${output}")
+    endif()
+
+    if(NOT EXISTS ${output})
+      message(SEND_ERROR "Cannot find output file ${output}")
+    endif()
+
+    file(MD5 ${input} input_md5)
+    file(MD5 ${output} output_md5)
+
+    if(NOT input_md5 STREQUAL output_md5)
+      message(SEND_ERROR "Files \"${input}\" and \"${output}\" are different")
+    endif()
+  endforeach()
+
+  foreach(file ${NOT_EXISTING_FILES_CHECK})
+    set(output ${FULL_DECOMPRESS_DIR}/${COMPRESS_DIR}/${file})
+
+    if(EXISTS ${output})
+      message(SEND_ERROR "File ${output} exists but it shouldn't")
+    endif()
+  endforeach()
+
+endfunction()