file(ARCHIVE_CREATE): support multithreading compression

Relates: #27420

Help/command/file.rst

@@ -915,6 +915,7 @@ Archiving
     [COMPRESSION <compression>
     [COMPRESSION_LEVEL <compression-level>]]
     [MTIME <mtime>]
+    [THREADS <number>]
     [WORKING_DIRECTORY <dir>]
     [VERBOSE])
   :target: ARCHIVE_CREATE
@@ -959,6 +960,14 @@ Archiving
   ``MTIME <mtime>``
     Specify the modification time recorded in tarball entries.
 
+  ``THREADS <number>``
+    .. versionadded:: 4.3
+
+    Use the ``<number>`` threads to operate on the archive.
+
+    The number of available cores on the machine will be used if set to ``0``.
+    Note that not all compression modes support threading in all environments.
+
   ``WORKING_DIRECTORY <dir>``
     .. versionadded:: 3.31
 

Help/release/dev/cli-tar-multithread.rst

@@ -2,3 +2,5 @@ cli-tar-multithread
 -------------------
 
 * The :manual:`cmake(1)` ``-E tar`` tool supports multithreading operations
+* The :command:`file(ARCHIVE_CREATE)` command supports multithreading via the
+  ``THREADS`` option

Source/cmFileCommand.cxx

@@ -3666,6 +3666,7 @@ bool HandleArchiveCreateCommand(std::vector<std::string> const& args,
     // accepted without one and treated as if an empty value were given.
     // Fixing this would require a policy.
     ArgumentParser::Maybe<std::string> MTime;
+    std::string Threads;
     std::string WorkingDirectory;
     bool Verbose = false;
     // "PATHS" requires at least one value, but use a custom check below.
@@ -3679,6 +3680,7 @@ bool HandleArchiveCreateCommand(std::vector<std::string> const& args,
       .Bind("COMPRESSION"_s, &Arguments::Compression)
       .Bind("COMPRESSION_LEVEL"_s, &Arguments::CompressionLevel)
       .Bind("MTIME"_s, &Arguments::MTime)
+      .Bind("THREADS"_s, &Arguments::Threads)
       .Bind("WORKING_DIRECTORY"_s, &Arguments::WorkingDirectory)
       .Bind("VERBOSE"_s, &Arguments::Verbose)
       .Bind("PATHS"_s, &Arguments::Paths);
@@ -3738,7 +3740,7 @@ bool HandleArchiveCreateCommand(std::vector<std::string> const& args,
   }
 
   int compressionLevel = 0;
-  int minCompressionLevel = 0;
+  constexpr int minCompressionLevel = 0;
   int maxCompressionLevel = 9;
   if (compress == cmSystemTools::TarCompressZstd) {
     maxCompressionLevel = 19;
@@ -3773,16 +3775,36 @@ bool HandleArchiveCreateCommand(std::vector<std::string> const& args,
     }
   }
 
+  // Use the single thread by default for backward compatibility
+  int threads = 1;
+  constexpr int minThreads = 0;
+  if (!parsedArgs.Threads.empty()) {
+    if (parsedArgs.Threads.size() != 1 &&
+        !std::isdigit(parsedArgs.Threads[0])) {
+      status.SetError(cmStrCat("number of threads ", parsedArgs.Threads,
+                               " should be at least ", minThreads));
+      cmSystemTools::SetFatalErrorOccurred();
+      return false;
+    }
+    threads = std::stoi(parsedArgs.Threads);
+    if (threads < minThreads) {
+      status.SetError(cmStrCat("number of threads ", parsedArgs.Threads,
+                               " should be at least ", minThreads));
+      cmSystemTools::SetFatalErrorOccurred();
+      return false;
+    }
+  }
+
   if (parsedArgs.Paths.empty()) {
     status.SetError("ARCHIVE_CREATE requires a non-empty list of PATHS");
     cmSystemTools::SetFatalErrorOccurred();
     return false;
   }
 
-  if (!cmSystemTools::CreateTar(parsedArgs.Output, parsedArgs.Paths,
-                                parsedArgs.WorkingDirectory, compress,
-                                parsedArgs.Verbose, parsedArgs.MTime,
-                                parsedArgs.Format, compressionLevel)) {
+  if (!cmSystemTools::CreateTar(
+        parsedArgs.Output, parsedArgs.Paths, parsedArgs.WorkingDirectory,
+        compress, parsedArgs.Verbose, parsedArgs.MTime, parsedArgs.Format,
+        compressionLevel, threads)) {
     status.SetError(cmStrCat("failed to compress: ", parsedArgs.Output));
     cmSystemTools::SetFatalErrorOccurred();
     return false;

Tests/RunCMake/File_Archive/RunCMakeTest.cmake

@@ -12,6 +12,13 @@ run_cmake(zip)
 
 run_cmake(working-directory)
 
+# Check the --cmake-tar-threads option
+run_cmake(argument-validation-threads)
+run_cmake(threads-bz2)
+run_cmake(threads-gz)
+run_cmake(threads-xz)
+run_cmake(threads-zstd)
+
 # Extracting only selected files or directories
 run_cmake(zip-filtered)
 

Tests/RunCMake/File_Archive/argument-validation-compression-level-1-stderr.txt

@@ -1,5 +1,5 @@
-CMake Error at compression-level\.cmake:39 \(file\):
+^CMake Error at compression-level\.cmake:39 \(file\):
   file compression level 100 for GZip should be in range 0 to 9
 Call Stack \(most recent call first\):
   argument-validation-compression-level-1\.cmake:8 \(check_compression_level\)
-  CMakeLists\.txt:3 \(include\)
+  CMakeLists\.txt:3 \(include\)$

Tests/RunCMake/File_Archive/argument-validation-compression-level-2-stderr.txt

@@ -1,5 +1,5 @@
-CMake Error at compression-level\.cmake:39 \(file\):
+^CMake Error at compression-level\.cmake:39 \(file\):
   file compression level high for GZip should be in range 0 to 9
 Call Stack \(most recent call first\):
   argument-validation-compression-level-2\.cmake:8 \(check_compression_level\)
-  CMakeLists\.txt:3 \(include\)
+  CMakeLists\.txt:3 \(include\)$

Tests/RunCMake/File_Archive/argument-validation-threads-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/File_Archive/argument-validation-threads-stderr.txt

@@ -0,0 +1,5 @@
+^CMake Error at threads\.cmake:39 \(file\):
+  file number of threads -1 should be at least 0
+Call Stack \(most recent call first\):
+  argument-validation-threads\.cmake:8 \(check_threads\)
+  CMakeLists\.txt:3 \(include\)$

Tests/RunCMake/File_Archive/argument-validation-threads.cmake

@@ -0,0 +1,8 @@
+set(OUTPUT_NAME "test.tar.gz")
+
+set(ARCHIVE_FORMAT gnutar)
+set(COMPRESSION_TYPE GZip)
+
+include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake)
+
+check_threads("-1")

Tests/RunCMake/File_Archive/roundtrip.cmake

@@ -95,20 +95,3 @@ function(check_magic EXPECTED)
       "Actual [${ACTUAL}] does not match expected [${EXPECTED}]")
   endif()
 endfunction()
-
-
-function(check_compression_level COMPRESSION_LEVEL)
-  file(ARCHIVE_CREATE
-    OUTPUT "${FULL_OUTPUT_NAME}_compression_level"
-    FORMAT "${ARCHIVE_FORMAT}"
-    COMPRESSION_LEVEL ${COMPRESSION_LEVEL}
-    COMPRESSION "${COMPRESSION_TYPE}"
-    VERBOSE
-    PATHS ${COMPRESS_DIR})
-
-  file(ARCHIVE_EXTRACT
-    INPUT "${FULL_OUTPUT_NAME}_compression_level"
-    ${DECOMPRESSION_OPTIONS}
-    DESTINATION ${FULL_DECOMPRESS_DIR}
-    VERBOSE)
-endfunction()

Tests/RunCMake/File_Archive/threads-bz2.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.tar.bz2")
+
+set(ARCHIVE_FORMAT paxr)
+set(COMPRESSION_TYPE BZip2)
+
+include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake)
+
+check_threads("0")
+check_threads("1")
+check_threads("4")

Tests/RunCMake/File_Archive/threads-gz.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.tar.gz")
+
+set(ARCHIVE_FORMAT gnutar)
+set(COMPRESSION_TYPE GZip)
+
+include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake)
+
+check_threads("0")
+check_threads("1")
+check_threads("4")

Tests/RunCMake/File_Archive/threads-xz.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.tar.xz")
+
+set(ARCHIVE_FORMAT pax)
+set(COMPRESSION_TYPE XZ)
+
+include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake)
+
+check_threads("0")
+check_threads("1")
+check_threads("4")

Tests/RunCMake/File_Archive/threads-zstd.cmake

@@ -0,0 +1,10 @@
+set(OUTPUT_NAME "test.tar.zstd")
+
+set(ARCHIVE_FORMAT pax)
+set(COMPRESSION_TYPE Zstd)
+
+include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake)
+
+check_threads("0")
+check_threads("1")
+check_threads("4")

Tests/RunCMake/File_Archive/threads.cmake

@@ -0,0 +1,85 @@
+foreach(parameter OUTPUT_NAME ARCHIVE_FORMAT)
+  if(NOT DEFINED ${parameter})
+    message(FATAL_ERROR "missing required parameter ${parameter}")
+  endif()
+endforeach()
+
+set(COMPRESS_DIR compress_dir)
+set(FULL_COMPRESS_DIR ${CMAKE_CURRENT_BINARY_DIR}/${COMPRESS_DIR})
+
+set(DECOMPRESS_DIR decompress_dir)
+set(FULL_DECOMPRESS_DIR ${CMAKE_CURRENT_BINARY_DIR}/${DECOMPRESS_DIR})
+
+set(FULL_OUTPUT_NAME ${CMAKE_CURRENT_BINARY_DIR}/${OUTPUT_NAME})
+
+set(CHECK_FILES
+  "f1.txt"
+  "d1/f1.txt"
+  "d 2/f1.txt"
+  "d + 3/f1.txt"
+  "d_4/f1.txt"
+  "d-4/f1.txt"
+  "My Special Directory/f1.txt"
+)
+
+function(check_threads THREADS)
+  foreach(file ${CHECK_FILES})
+    configure_file(${CMAKE_CURRENT_LIST_FILE} ${FULL_COMPRESS_DIR}/${file} COPYONLY)
+  endforeach()
+
+  if(UNIX)
+    execute_process(COMMAND ln -sf f1.txt ${FULL_COMPRESS_DIR}/d1/f2.txt)
+    list(APPEND CHECK_FILES "d1/f2.txt")
+  endif()
+
+  file(REMOVE ${FULL_OUTPUT_NAME})
+  file(REMOVE_RECURSE ${FULL_DECOMPRESS_DIR})
+  file(MAKE_DIRECTORY ${FULL_DECOMPRESS_DIR})
+
+  file(ARCHIVE_CREATE
+    OUTPUT ${FULL_OUTPUT_NAME}
+    FORMAT "${ARCHIVE_FORMAT}"
+    COMPRESSION "${COMPRESSION_TYPE}"
+    THREADS ${THREADS}
+    VERBOSE
+    PATHS ${COMPRESS_DIR})
+
+  file(ARCHIVE_EXTRACT
+    INPUT ${FULL_OUTPUT_NAME}
+    ${DECOMPRESSION_OPTIONS}
+    DESTINATION ${FULL_DECOMPRESS_DIR}
+    VERBOSE)
+
+  if(CUSTOM_CHECK_FILES)
+    set(CHECK_FILES ${CUSTOM_CHECK_FILES})
+  endif()
+
+  foreach(file ${CHECK_FILES})
+    set(input ${FULL_COMPRESS_DIR}/${file})
+    set(output ${FULL_DECOMPRESS_DIR}/${COMPRESS_DIR}/${file})
+
+    if(NOT EXISTS ${input})
+      message(SEND_ERROR "Cannot find input file ${output}")
+    endif()
+
+    if(NOT EXISTS ${output})
+      message(SEND_ERROR "Cannot find output file ${output}")
+    endif()
+
+    file(MD5 ${input} input_md5)
+    file(MD5 ${output} output_md5)
+
+    if(NOT input_md5 STREQUAL output_md5)
+      message(SEND_ERROR "Files \"${input}\" and \"${output}\" are different")
+    endif()
+  endforeach()
+
+  foreach(file ${NOT_EXISTING_FILES_CHECK})
+    set(output ${FULL_DECOMPRESS_DIR}/${COMPRESS_DIR}/${file})
+
+    if(EXISTS ${output})
+      message(SEND_ERROR "File ${output} exists but it shouldn't")
+    endif()
+  endforeach()
+
+endfunction()