Merge topic 'file-download-range'

231872ddb0 file(DOWNLOAD): Add options to download a range

Acked-by: Kitware Robot <kwrobot@kitware.com>
Acked-by: buildbot <buildbot@kitware.com>
Merge-request: !6986

Help/command/file.rst

@@ -1128,6 +1128,18 @@ Additional options to ``DOWNLOAD`` are:
   Historical short-hand for ``EXPECTED_HASH MD5=<value>``. It is an error to
   specify this if ``DOWNLOAD`` is not given a ``<file>``.
 
+``RANGE_START <value>``
+  .. versionadded:: 3.24
+
+  Offset of the start of the range in file in bytes. Could be omitted to
+  download up to the specified ``RANGE_END``.
+
+``RANGE_END <value>``
+  .. versionadded:: 3.24
+
+  Offset of the end of the range in file in bytes. Could be omitted to
+  download everything from the specified ``RANGE_START`` to the end of file.
+
 Locking
 ^^^^^^^
 

Help/release/dev/file-download-range.rst

@@ -0,0 +1,6 @@
+file-download-range
+-------------------
+
+* Add the fields ``RANGE_START`` and ``RANGE_END`` to ``file(DOWNLOAD)``.
+  Those fields provide a convenient way to specify the range, passed to the
+  libcurl, which can be useful for downloading parts of big binary files.

Source/cmFileCommand.cxx

@@ -15,6 +15,7 @@
 #include <vector>
 
 #include <cm/memory>
+#include <cm/optional>
 #include <cm/string_view>
 #include <cmext/algorithm>
 #include <cmext/string_view>
@@ -1778,6 +1779,7 @@ bool HandleDownloadCommand(std::vector<std::string> const& args,
   std::string userpwd;
 
   std::vector<std::string> curl_headers;
+  std::vector<std::pair<std::string, cm::optional<std::string>>> curl_ranges;
 
   while (i != args.end()) {
     if (*i == "TIMEOUT") {
@@ -1890,6 +1892,27 @@ bool HandleDownloadCommand(std::vector<std::string> const& args,
         return false;
       }
       curl_headers.push_back(*i);
+    } else if (*i == "RANGE_START") {
+      ++i;
+      if (i == args.end()) {
+        status.SetError("DOWNLOAD missing value for RANGE_START.");
+        return false;
+      }
+      curl_ranges.emplace_back(*i, cm::nullopt);
+    } else if (*i == "RANGE_END") {
+      ++i;
+      if (curl_ranges.empty()) {
+        curl_ranges.emplace_back("0", *i);
+      } else {
+        auto& last_range = curl_ranges.back();
+        if (!last_range.second.has_value()) {
+          last_range.second = *i;
+        } else {
+          status.SetError("Multiple RANGE_END values is provided without "
+                          "the corresponding RANGE_START.");
+          return false;
+        }
+      }
     } else if (file.empty()) {
       file = *i;
     } else {
@@ -1899,6 +1922,7 @@ bool HandleDownloadCommand(std::vector<std::string> const& args,
     }
     ++i;
   }
+
   // Can't calculate hash if we don't save the file.
   // TODO Incrementally calculate hash in the write callback as the file is
   // being downloaded so this check can be relaxed.
@@ -1984,6 +2008,13 @@ bool HandleDownloadCommand(std::vector<std::string> const& args,
     check_curl_result(res, "DOWNLOAD cannot set TLS/SSL Verify off: ");
   }
 
+  for (const auto& range : curl_ranges) {
+    std::string curl_range = range.first + '-' +
+      (range.second.has_value() ? range.second.value() : "");
+    res = ::curl_easy_setopt(curl, CURLOPT_RANGE, curl_range.c_str());
+    check_curl_result(res, "DOWNLOAD cannot set range: ");
+  }
+
   // check to see if a CAINFO file has been specified
   // command arg comes first
   std::string const& cainfo_err = cmCurlSetCAInfo(curl, cainfo);

Tests/CMakeTests/FileDownloadTest.cmake.in

@@ -179,3 +179,51 @@ if(EXISTS TIMEOUT)
   message(SEND_ERROR "TIMEOUT argument was incorrectly interpreted as a filename")
 endif()
 message(STATUS "${status}")
+
+message(STATUS "FileDownload:14")
+file(DOWNLOAD
+  ${url}
+  ${dir}/file14.bin
+  TIMEOUT ${timeout}
+  STATUS status
+  RANGE_START 0
+  EXPECTED_MD5 dbd330d52f4dbd60115d4191904ded92
+  )
+__reportIfWrongStatus("${status}" 0)
+
+message(STATUS "FileDownload:15")
+file(DOWNLOAD
+  ${url}
+  ${dir}/file15.bin
+  TIMEOUT ${timeout}
+  STATUS status
+  RANGE_END 50
+  EXPECTED_MD5 8592e5665b839b5d23825dc84c135b61
+  )
+__reportIfWrongStatus("${status}" 0)
+
+message(STATUS "FileDownload:16")
+file(DOWNLOAD
+  ${url}
+  ${dir}/file16.bin
+  TIMEOUT ${timeout}
+  STATUS status
+  RANGE_START 10
+  RANGE_END 50
+  EXPECTED_MD5 36cd52681e6c6c8fef85fcd9e86fc30d
+  )
+__reportIfWrongStatus("${status}" 0)
+
+message(STATUS "FileDownload:17")
+file(DOWNLOAD
+  ${url}
+  ${dir}/file17.bin
+  TIMEOUT ${timeout}
+  STATUS status
+  RANGE_START 0
+  RANGE_END 50
+  RANGE_START 60
+  RANGE_END 100
+  EXPECTED_MD5 c5c9e74e82d493dd901eecccd659cebc
+  )
+__reportIfWrongStatus("${status}" 0)