Download: add a 'resume' option

A download interrupted halfway currently starts again from zero,
which for a data set of any size is the difference between a retry
and an afternoon.

With 'resume' set and a target given, the curl and wget methods
continue the partial file. The clean-up in 'Download' skips the
target for the same reason: otherwise it would throw away the very
thing the next attempt is meant to continue.

Methods that cannot resume decline the request rather than
proceeding to avoid destroying the partial file the resuming
methods need.

'resume' is deliberately left unbound when the caller does not ask
for it, rather than defaulted to false.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: fingolfin

doc/download.xml

@@ -69,7 +69,28 @@ The following components are supported.
   and the function writes the downloaded contents to this file;
   the returned record does not have a <C>result</C> component in this case.
   <P/>
-  If the download fails then this file is not left behind.
+  If the download fails then this file is not left behind,
+  unless <C>resume</C> is set.
+</Item>
+<Mark><C>resume</C></Mark>
+<Item>
+  If this component is bound and has the value <K>true</K>,
+  and <C>target</C> is given,
+  then a partially downloaded file is continued rather than fetched again,
+  and it is kept if the download fails again.
+  <P/>
+  Methods that cannot resume decline the request, since they would discard
+  the partial file that a method which can resume needs; currently the
+  methods based on the external programs <C>curl</C> and <C>wget</C> resume.
+  If none is available the download fails, and the caller can retry without
+  <C>resume</C>.
+  <P/>
+  It is the caller's responsibility that an existing <C>target</C> really is
+  a prefix of what <A>url</A> delivers; otherwise the two get concatenated.
+  <P/>
+  If the server does not support range requests then no data is lost:
+  <C>wget</C> fetches the file again from the start, and <C>curl</C> fails
+  and leaves the partial file alone.
 </Item>
 <Mark><C>verifyCert</C></Mark>
 <Item>

lib/download.gi

@@ -36,6 +36,13 @@ Add( Download_Methods, rec(
   download:= function( url, opt )
     local res;
 
+    if IsBound( opt.resume ) and opt.resume = true and
+       IsBound( opt.target ) and IsString( opt.target ) then
+      # Declining matters: this method would truncate the partial file that a
+      # method which can resume needs.
+      return rec( success:= false, error:= "no support for resuming" );
+    fi;
+
     opt:= ShallowCopy( opt );
     if not IsBound( opt.failOnError ) then
       opt.failOnError:= true;
@@ -61,6 +68,11 @@ Add( Download_Methods, rec(
       return rec( success:= false, error:= "protocol is not http" );
     elif IsBound( opt.maxTime ) and opt.maxTime <> 0 then
       return rec( success:= false, error:= "no support for given timeout" );
+    elif IsBound( opt.resume ) and opt.resume = true and
+         IsBound( opt.target ) and IsString( opt.target ) then
+      # No range request, so this would overwrite the partial file that a
+      # method which can resume needs.
+      return rec( success:= false, error:= "no support for resuming" );
     fi;
 
     # Split the URL after 'http://' into the authority and HTTP request target.
@@ -164,6 +176,10 @@ Add( Download_Methods, rec(
     else
       args:= [ "--quiet", "-O", "-", url ];
     fi;
+    if IsBound( opt.resume ) and opt.resume = true and
+       IsBound( opt.target ) and IsString( opt.target ) then
+      Add( args, "-c" );
+    fi;
     if IsBound( opt.verifyCert ) and opt.verifyCert = false then
       Add( args, "--no-check-certificate" );
     fi;
@@ -173,8 +189,10 @@ Add( Download_Methods, rec(
     code:= Process( DirectoryCurrent(), exec, InputTextNone(), outstream, args );
     CloseStream( outstream );
     if code <> 0 then
-      # wget may have created the target file; try to remove it
+      # wget may have created the target file; try to remove it, unless the
+      # caller wants to resume from what is there
       if IsBound( opt.target ) and IsString( opt.target ) and
+         not ( IsBound( opt.resume ) and opt.resume = true ) and
          IsExistingFile( opt.target ) and RemoveFile( opt.target ) <> true then
         Error( "Download cannot remove unwanted file ", opt.target );
       fi;
@@ -211,6 +229,11 @@ Add( Download_Methods, rec(
     else
       Add( args, "-" );
     fi;
+    if IsBound( opt.resume ) and opt.resume = true and
+       IsBound( opt.target ) and IsString( opt.target ) then
+      Add( args, "-C" );
+      Add( args, "-" );
+    fi;
     if IsBound( opt.maxTime ) and IsPosInt( opt.maxTime ) then
       Add( args, "--max-time" );
       Add( args, opt.maxTime );
@@ -275,8 +298,10 @@ InstallMethod( Download,
         fi;
         # A failed method may have left a partial or bogus target file behind.
         # Remove it here, so that the guarantee holds for every method,
-        # including ones added to 'Download_Methods' from outside.
+        # including ones added to 'Download_Methods' from outside -- but not
+        # when resuming, where the partial file is the whole point.
         if IsBound( opt.target ) and IsString( opt.target ) and
+           not ( IsBound( opt.resume ) and opt.resume = true ) and
            IsExistingFile( opt.target ) then
           RemoveFile( opt.target );
         fi;

tst/download.tst

@@ -1,4 +1,4 @@
-#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad, server, baseurl, iometh, opt, oldpref
+#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad, server, baseurl, iometh, opt, oldpref, resumers, name
 ############################################################################
 ##
 #W  download.tst               Utils Package                   Thomas Breuer
@@ -164,6 +164,55 @@ true
 gap> IsExistingFile( file );
 false
 
+##  'resume' continues a partial file rather than fetching it again.  The
+##  test server answers a Range request with the remainder in upper case, so
+##  a resumed download is distinguishable from a restarted one.
+gap> resumers:= Filtered( meths, r -> r.name in [ "via curl", "via wget" ] );;
+gap> for r in resumers do
+>      FileString( file, "abcde" );;
+>      res1:= r.download( Concatenation( baseurl, "/resumable" ),
+>                         rec( target:= file, resume:= true ) );
+>      if res1.success <> true then
+>        Print( "resume failed for ", r.name, ": ", res1.error, "\n" );
+>      elif StringFile( file ) <> "abcdeFGHIJKLMNOPQRST" then
+>        Print( "did not resume for ", r.name, ": ", StringFile( file ), "\n" );
+>      fi;
+>      RemoveFile( file );
+>    od;
+
+##  Without 'resume' the target is replaced, not appended to.
+gap> FileString( file, "abcde" );;
+gap> res1:= Download( Concatenation( baseurl, "/resumable" ),
+>                     rec( target:= file ) );;
+gap> StringFile( file );
+"abcdefghijklmnopqrst"
+
+##  Methods that cannot resume decline, rather than discarding the partial
+##  file that a method which can resume needs.
+gap> iometh.download( Concatenation( baseurl, "/resumable" ),
+>                     rec( target:= file, resume:= true ) ).error;
+"no support for resuming"
+
+##  With 'resume', a failed download keeps the partial file to continue from.
+##  Without this, the clean-up in 'Download' would throw away the very thing
+##  the next attempt is meant to continue.
+gap> FileString( file, "abcde" );;
+gap> res1:= Download( Concatenation( baseurl, "/missing" ),
+>                     rec( target:= file, resume:= true ) );;
+gap> res1.success;
+false
+gap> StringFile( file );
+"abcde"
+gap> RemoveFile( file );;
+
+##  'resume' is only declined when it is actually requested: a method must
+##  not be skipped merely because the component is present.
+gap> res1:= Download( Concatenation( baseurl, "/file" ),
+>                     rec( target:= file, resume:= false ) );;
+gap> res1.success;
+true
+gap> RemoveFile( file );;
+
 ##  test errors and redirects
 gap> res1:= Download( Concatenation( baseurl, "/missing" ) );;
 gap> res1.success = false;

tst/http-server.g

@@ -4,7 +4,7 @@
 ##
 
 BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket )
-  local connection, line, parts, uri, body, status, location;
+  local connection, line, parts, uri, body, status, location, range, from;
 
   IO_close( listener );
   connection:= IO_WrapFD( socket, IO.DefaultBufSize, IO.DefaultBufSize );
@@ -16,8 +16,12 @@ BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket )
   fi;
   uri:= parts[2];
 
+  range:= fail;
   repeat
     line:= IO_ReadLine( connection );
+    if IsString( line ) and StartsWith( LowercaseString( line ), "range:" ) then
+      range:= line;
+    fi;
   until line = fail or line = "" or line = "\n" or line = "\r\n";
 
   body:= "download test response\n";
@@ -32,6 +36,27 @@ BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket )
     body:= "";
     status:= "302 Found";
     location:= "Location: /success\r\n";
+  elif StartsWith( uri, "/resumable" ) then
+    # Answer a 'Range: bytes=N-' request with the remainder, but in upper
+    # case, so that a test can tell a resumed download from a restarted one.
+    body:= "abcdefghijklmnopqrst";
+    if range <> fail then
+      from:= Int( Filtered( range, c -> c in "0123456789" ) );
+      if from <> fail and 0 < from and from < Length( body ) then
+        IO_Write( connection,
+            "HTTP/1.1 206 Partial Content\r\n",
+            "Content-Type: text/plain\r\n",
+            "Content-Range: bytes ", String( from ), "-",
+                String( Length( body ) - 1 ), "/", String( Length( body ) ),
+                "\r\n",
+            "Content-Length: ", String( Length( body ) - from ), "\r\n",
+            "Connection: close\r\n\r\n",
+            UppercaseString( body{ [ from+1 .. Length( body ) ] } ) );
+        IO_Flush( connection );
+        IO_Close( connection );
+        IO_exit( 0 );
+      fi;
+    fi;
   fi;
 
   IO_Write( connection,