From cbf71b21b29867293f5c7c8cd209a0b32e7e0685 Mon Sep 17 00:00:00 2001 From: AJIOB Date: Fri, 28 Nov 2025 23:49:53 +0300 Subject: [PATCH] cli tar: support multithreading compression Fixes: #27420 --- Help/manual/cmake.1.rst | 11 +++++++ Help/release/dev/cli-tar-multithread.rst | 4 +++ Source/cmSystemTools.cxx | 5 ++-- Source/cmSystemTools.h | 2 +- Source/cmcmd.cxx | 29 ++++++++++++++++++- .../CommandLineTar/RunCMakeTest.cmake | 10 +++++++ .../bad-threads-not-a-number-result.txt | 1 + .../bad-threads-not-a-number-stderr.txt | 1 + .../CommandLineTar/threads-7zip.cmake | 10 +++++++ .../RunCMake/CommandLineTar/threads-bz2.cmake | 10 +++++++ .../RunCMake/CommandLineTar/threads-gz.cmake | 10 +++++++ .../RunCMake/CommandLineTar/threads-xz.cmake | 10 +++++++ .../RunCMake/CommandLineTar/threads-zip.cmake | 10 +++++++ .../CommandLineTar/threads-zstd.cmake | 11 +++++++ 14 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 Help/release/dev/cli-tar-multithread.rst create mode 100644 Tests/RunCMake/CommandLineTar/bad-threads-not-a-number-result.txt create mode 100644 Tests/RunCMake/CommandLineTar/bad-threads-not-a-number-stderr.txt create mode 100644 Tests/RunCMake/CommandLineTar/threads-7zip.cmake create mode 100644 Tests/RunCMake/CommandLineTar/threads-bz2.cmake create mode 100644 Tests/RunCMake/CommandLineTar/threads-gz.cmake create mode 100644 Tests/RunCMake/CommandLineTar/threads-xz.cmake create mode 100644 Tests/RunCMake/CommandLineTar/threads-zip.cmake create mode 100644 Tests/RunCMake/CommandLineTar/threads-zstd.cmake diff --git a/Help/manual/cmake.1.rst b/Help/manual/cmake.1.rst index d19ad007d9..d79c876f25 100644 --- a/Help/manual/cmake.1.rst +++ b/Help/manual/cmake.1.rst @@ -1491,6 +1491,17 @@ Available commands are: Specify modification time recorded in tarball entries. + .. option:: --cmake-tar-threads= + + .. versionadded:: 4.3 + + Use the ```` threads to operate on the archive. Currently only + multi-threaded compression is supported. + + If set to ``0``, the number of available cores on the machine will be + used instead. Note that not all compression modes support threading + in all environments. + .. option:: --touch .. versionadded:: 3.24 diff --git a/Help/release/dev/cli-tar-multithread.rst b/Help/release/dev/cli-tar-multithread.rst new file mode 100644 index 0000000000..7065d70206 --- /dev/null +++ b/Help/release/dev/cli-tar-multithread.rst @@ -0,0 +1,4 @@ +cli-tar-multithread +------------------- + +* The :manual:`cmake(1)` ``-E tar`` tool supports multithreading operations diff --git a/Source/cmSystemTools.cxx b/Source/cmSystemTools.cxx index ee1dba4731..07fcb1ea8b 100644 --- a/Source/cmSystemTools.cxx +++ b/Source/cmSystemTools.cxx @@ -2378,7 +2378,8 @@ bool cmSystemTools::CreateTar(std::string const& outFileName, std::string const& workingDirectory, cmTarCompression compressType, bool verbose, std::string const& mtime, - std::string const& format, int compressionLevel) + std::string const& format, int compressionLevel, + int numThreads) { #if !defined(CMAKE_BOOTSTRAP) cmWorkingDirectory workdir(cmSystemTools::GetLogicalWorkingDirectory()); @@ -2414,7 +2415,7 @@ bool cmSystemTools::CreateTar(std::string const& outFileName, } cmArchiveWrite a(fout, compress, format.empty() ? "paxr" : format, - compressionLevel); + compressionLevel, numThreads); if (!a.Open()) { cmSystemTools::Error(a.GetError()); diff --git a/Source/cmSystemTools.h b/Source/cmSystemTools.h index 946dea93c2..44ae8080c3 100644 --- a/Source/cmSystemTools.h +++ b/Source/cmSystemTools.h @@ -566,7 +566,7 @@ public: cmTarCompression compressType, bool verbose, std::string const& mtime = std::string(), std::string const& format = std::string(), - int compressionLevel = 0); + int compressionLevel = 0, int numThreads = 1); static bool ExtractTar(std::string const& inFileName, std::vector const& files, cmTarExtractTimestamps extractTimestamps, diff --git a/Source/cmcmd.cxx b/Source/cmcmd.cxx index 09c35c0c96..1bb104b4e9 100644 --- a/Source/cmcmd.cxx +++ b/Source/cmcmd.cxx @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -1582,6 +1583,7 @@ int cmcmd::ExecuteCMakeCommand(std::vector const& args, std::vector files; std::string mtime; std::string format; + int numThreads = 1; cmSystemTools::cmTarExtractTimestamps extractTimestamps = cmSystemTools::cmTarExtractTimestamps::Yes; cmSystemTools::cmTarCompression compress = @@ -1597,6 +1599,31 @@ int cmcmd::ExecuteCMakeCommand(std::vector const& args, ++nCompress; } else if (cmHasLiteralPrefix(arg, "--mtime=")) { mtime = arg.substr(8); + } else if (cmHasLiteralPrefix(arg, "--cmake-tar-threads=")) { + std::string const& numThreadsStr = arg.substr(20); + long numThreadsLong = 0; + if (!cmStrToLong(numThreadsStr, &numThreadsLong)) { + cmSystemTools::Error( + cmStrCat("Invalid --cmake-tar-threads value: '", numThreadsStr, + "' - not a number")); + return 1; + } + if (numThreadsLong > + std::numeric_limits::max()) { + cmSystemTools::Error( + cmStrCat("Invalid --cmake-tar-threads value: '", numThreadsStr, + "' - too large")); + return 1; + } + if (numThreadsLong < + std::numeric_limits::min()) { + cmSystemTools::Error( + cmStrCat("Invalid --cmake-tar-threads value: '", numThreadsStr, + "' - too small")); + return 1; + } + + numThreads = static_cast(numThreadsLong); } else if (cmHasLiteralPrefix(arg, "--files-from=")) { std::string const& files_from = arg.substr(13); if (!cmTarFilesFrom(files_from, files)) { @@ -1677,7 +1704,7 @@ int cmcmd::ExecuteCMakeCommand(std::vector const& args, std::cerr << "tar: No files or directories specified\n"; } if (!cmSystemTools::CreateTar(outFile, files, {}, compress, verbose, - mtime, format)) { + mtime, format, 0, numThreads)) { cmSystemTools::Error("Problem creating tar: " + outFile); return 1; } diff --git a/Tests/RunCMake/CommandLineTar/RunCMakeTest.cmake b/Tests/RunCMake/CommandLineTar/RunCMakeTest.cmake index a487f372dc..343faceee5 100644 --- a/Tests/RunCMake/CommandLineTar/RunCMakeTest.cmake +++ b/Tests/RunCMake/CommandLineTar/RunCMakeTest.cmake @@ -32,6 +32,16 @@ run_cmake(paxr) run_cmake(paxr-bz2) run_cmake(zip) +# Check the --cmake-tar-threads option +external_command_test(bad-threads-not-a-number tar cvf bad.tar --cmake-tar-threads=nan .) + +run_cmake(threads-7zip) +run_cmake(threads-bz2) +run_cmake(threads-gz) +run_cmake(threads-xz) +run_cmake(threads-zstd) +run_cmake(threads-zip) + # Extracting only selected files or directories run_cmake(zip-filtered) diff --git a/Tests/RunCMake/CommandLineTar/bad-threads-not-a-number-result.txt b/Tests/RunCMake/CommandLineTar/bad-threads-not-a-number-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/CommandLineTar/bad-threads-not-a-number-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CommandLineTar/bad-threads-not-a-number-stderr.txt b/Tests/RunCMake/CommandLineTar/bad-threads-not-a-number-stderr.txt new file mode 100644 index 0000000000..c7a99c06ef --- /dev/null +++ b/Tests/RunCMake/CommandLineTar/bad-threads-not-a-number-stderr.txt @@ -0,0 +1 @@ +^CMake Error: Invalid --cmake-tar-threads value: 'nan' - not a number$ diff --git a/Tests/RunCMake/CommandLineTar/threads-7zip.cmake b/Tests/RunCMake/CommandLineTar/threads-7zip.cmake new file mode 100644 index 0000000000..114d1ef4b6 --- /dev/null +++ b/Tests/RunCMake/CommandLineTar/threads-7zip.cmake @@ -0,0 +1,10 @@ +set(OUTPUT_NAME "test.7z") + +set(COMPRESSION_FLAGS cvf) +set(COMPRESSION_OPTIONS --format=7zip --cmake-tar-threads=4) + +set(DECOMPRESSION_FLAGS xvf) + +include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake) + +check_magic("377abcaf271c" LIMIT 6 HEX) diff --git a/Tests/RunCMake/CommandLineTar/threads-bz2.cmake b/Tests/RunCMake/CommandLineTar/threads-bz2.cmake new file mode 100644 index 0000000000..024496a5ea --- /dev/null +++ b/Tests/RunCMake/CommandLineTar/threads-bz2.cmake @@ -0,0 +1,10 @@ +set(OUTPUT_NAME "test.tar.bz2") + +set(COMPRESSION_FLAGS cvjf) +set(COMPRESSION_OPTIONS --cmake-tar-threads=4) + +set(DECOMPRESSION_FLAGS xvjf) + +include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake) + +check_magic("425a68" LIMIT 3 HEX) diff --git a/Tests/RunCMake/CommandLineTar/threads-gz.cmake b/Tests/RunCMake/CommandLineTar/threads-gz.cmake new file mode 100644 index 0000000000..3d6379d5aa --- /dev/null +++ b/Tests/RunCMake/CommandLineTar/threads-gz.cmake @@ -0,0 +1,10 @@ +set(OUTPUT_NAME "test.tar.gz") + +set(COMPRESSION_FLAGS -cvzf) +set(COMPRESSION_OPTIONS --cmake-tar-threads=4) + +set(DECOMPRESSION_FLAGS -xvzf) + +include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake) + +check_magic("1f8b" LIMIT 2 HEX) diff --git a/Tests/RunCMake/CommandLineTar/threads-xz.cmake b/Tests/RunCMake/CommandLineTar/threads-xz.cmake new file mode 100644 index 0000000000..bae3a76805 --- /dev/null +++ b/Tests/RunCMake/CommandLineTar/threads-xz.cmake @@ -0,0 +1,10 @@ +set(OUTPUT_NAME "test.tar.xz") + +set(COMPRESSION_FLAGS cvJf) +set(COMPRESSION_OPTIONS --cmake-tar-threads=4) + +set(DECOMPRESSION_FLAGS xvJf) + +include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake) + +check_magic("fd377a585a00" LIMIT 6 HEX) diff --git a/Tests/RunCMake/CommandLineTar/threads-zip.cmake b/Tests/RunCMake/CommandLineTar/threads-zip.cmake new file mode 100644 index 0000000000..7d70e7b9db --- /dev/null +++ b/Tests/RunCMake/CommandLineTar/threads-zip.cmake @@ -0,0 +1,10 @@ +set(OUTPUT_NAME "test.zip") + +set(COMPRESSION_FLAGS cvf) +set(COMPRESSION_OPTIONS --format=zip --cmake-tar-threads=4) + +set(DECOMPRESSION_FLAGS xvf) + +include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake) + +check_magic("504b0304" LIMIT 4 HEX) diff --git a/Tests/RunCMake/CommandLineTar/threads-zstd.cmake b/Tests/RunCMake/CommandLineTar/threads-zstd.cmake new file mode 100644 index 0000000000..405b0c9d64 --- /dev/null +++ b/Tests/RunCMake/CommandLineTar/threads-zstd.cmake @@ -0,0 +1,11 @@ +set(OUTPUT_NAME "test.tar.zstd") + +set(COMPRESSION_FLAGS cvf) +set(COMPRESSION_OPTIONS --zstd --cmake-tar-threads=4) + +set(DECOMPRESSION_FLAGS xvf) + +include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake) + +# libarchive 3.8.2 enables a checksum feature; older versions do not. +check_magic("^28b52ffd0[04]58$" LIMIT 6 HEX)