[fix] Improve stale upload slice resilience

src/ByteSync.Client/Services/Communications/Transfers/Strategies/UploadFailureClassifier.cs

@@ -9,6 +9,12 @@
 
 public static class UploadFailureClassifier
 {
+    private static readonly string[] UnexpectedTransportClosureMessageFragments =
+    {
+        "unexpected EOF",
+        "0 bytes from the transport stream",
+    };
+
     public static UploadFileResponse Classify(Exception exception, CancellationToken cancellationToken)
     {
         if (exception is OperationCanceledException && cancellationToken.IsCancellationRequested)
@@ -21,6 +27,11 @@
             return UploadFileResponse.ClientTimeout(exception);
         }
 
+        if (cancellationToken.IsCancellationRequested)
+        {
+            return UploadFileResponse.ClientCancellation(exception);
+        }
+
         if (IsClientNetworkError(exception))
         {
             return UploadFileResponse.ClientNetworkError(exception);
@@ -43,16 +54,35 @@
             {
                 return socketException.SocketErrorCode is SocketError.ConnectionReset
                     or SocketError.ConnectionAborted
+                    or SocketError.OperationAborted
                     or SocketError.TimedOut
                     or SocketError.NetworkDown
                     or SocketError.NetworkUnreachable
                     or SocketError.HostDown
                     or SocketError.HostUnreachable;
             }
 
+            if (current is IOException ioException && HasUnexpectedTransportClosureMessage(ioException))
+            {
+                return true;
+            }
+
             current = current.InnerException;
         }
 
         return false;
     }
+
+    private static bool HasUnexpectedTransportClosureMessage(IOException exception)
+    {
+        foreach (var fragment in UnexpectedTransportClosureMessageFragments)
+        {
+            if (exception.Message.Contains(fragment, StringComparison.OrdinalIgnoreCase))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
 }

src/ByteSync.Client/Services/Communications/Transfers/Uploading/AdaptiveUploadController.cs

@@ -15,6 +15,7 @@
     private const int MIN_PARALLELISM = 2;
     private const int MAX_PARALLELISM = 4;
     private const int CLIENT_NETWORK_ISSUES_BEFORE_DOWNSCALE = 2;
+    private const double CLIENT_NETWORK_ISSUE_CHUNK_FACTOR = 0.5;
 
     private const double MULTIPLIER_2_X = 2.0;
     private const double MULTIPLIER_1_75_X = 1.75;
@@ -116,13 +117,13 @@
 
             var maxElapsed = GetMaxElapsedInWindow();
 
             _logger.LogDebug(
                 "Adaptive: file {FileId} maxElapsed={MaxElapsedMs} ms, window={Window}, parallelism={Parallelism}, chunkSize={ChunkKb} KB",
                 uploadResult.FileId ?? "-",
                 maxElapsed.TotalMilliseconds,
                 _windowSize,
                 _currentParallelism,
                 Math.Round(_currentChunkSizeBytes / 1024d));
 
             if (TryHandleDownscale(maxElapsed, uploadResult.FileId))
             {
@@ -167,22 +168,22 @@
         _consecutiveClientNetworkIssues += 1;
         if (_consecutiveClientNetworkIssues < CLIENT_NETWORK_ISSUES_BEFORE_DOWNSCALE)
         {
             _logger.LogDebug(
                 "Adaptive: file {FileId} client network issue {FailureKind} {IssueCount}/{Threshold}. Waiting before downscale",
                 fileId ?? "-",
                 failureKind,
                 _consecutiveClientNetworkIssues,
                 CLIENT_NETWORK_ISSUES_BEFORE_DOWNSCALE);
 
             return;
         }
 
         _logger.LogInformation(
             "Adaptive: file {FileId} client network issue threshold reached ({IssueCount}). Downscaling upload settings",
             fileId ?? "-",
             _consecutiveClientNetworkIssues);
         _consecutiveClientNetworkIssues = 0;
-        Downscale(fileId, "client network issues");
+        DownscaleForClientNetworkIssue(fileId, failureKind);
     }
 
     private bool HandleBandwidthReset(bool isSuccess, int? statusCode)
@@ -232,13 +233,13 @@
     {
         if (_currentParallelism > MIN_PARALLELISM)
         {
             _logger.LogInformation(
                 "Adaptive: file {FileId} Downscale ({Reason}). Reducing parallelism {Prev} -> {Next}. Resetting window (window before {WindowBefore})",
                 fileId ?? "-",
                 reason,
                 _currentParallelism,
                 _currentParallelism - 1,
                 _windowSize);
             _currentParallelism -= 1;
             _windowSize = _currentParallelism;
             ResetWindow();
@@ -250,15 +251,38 @@
         if (reduced != _currentChunkSizeBytes)
         {
             _currentChunkSizeBytes = reduced;
             _logger.LogInformation(
                 "Adaptive: file {FileId} Downscale ({Reason}). New chunkSize={ChunkKb} KB",
                 fileId ?? "-",
                 reason,
                 Math.Round(_currentChunkSizeBytes / 1024d));
         }
 
         ResetWindow();
     }
+
+    private void DownscaleForClientNetworkIssue(string? fileId, UploadFailureKind failureKind)
+    {
+        var previousParallelism = _currentParallelism;
+        var previousChunkSizeBytes = _currentChunkSizeBytes;
+
+        _currentParallelism = MIN_PARALLELISM;
+        _currentChunkSizeBytes = Math.Max(
+            MIN_CHUNK_SIZE_BYTES,
+            (int)Math.Round(_currentChunkSizeBytes * CLIENT_NETWORK_ISSUE_CHUNK_FACTOR));
+        _windowSize = _currentParallelism;
+
+        _logger.LogInformation(
+            "Adaptive: file {FileId} client network issue downscale ({FailureKind}). Parallelism {PrevParallelism}->{NextParallelism}, chunkSize {PrevChunkKb}->{NextChunkKb} KB",
+            fileId ?? "-",
+            failureKind,
+            previousParallelism,
+            _currentParallelism,
+            Math.Round(previousChunkSizeBytes / 1024d),
+            Math.Round(_currentChunkSizeBytes / 1024d));
+
+        ResetWindow();
+    }
 
     private void TryHandleUpscale(string? fileId)
     {
@@ -307,12 +331,12 @@
                 var increased = (int)Math.Round(_currentChunkSizeBytes * multiplier);
                 _currentChunkSizeBytes = Math.Clamp(increased, MIN_CHUNK_SIZE_BYTES, MAX_CHUNK_SIZE_BYTES);
 
                 _logger.LogInformation(
                     "Adaptive: file {FileId} Upscale. maxElapsed={MaxElapsedMs} ms <= {ThresholdMs} ms. New chunkSize={ChunkKb} KB",
                     fileId ?? "-",
                     maxElapsedEligible.TotalMilliseconds,
                     _upscaleThreshold.TotalMilliseconds,
                     Math.Round(_currentChunkSizeBytes / 1024d));
 
                 UpdateParallelismOnUpscale(fileId);
                 _currentParallelism = Math.Min(_currentParallelism, MAX_PARALLELISM);
@@ -350,8 +374,8 @@
             _currentParallelism = Math.Max(_currentParallelism, 4);
             if (_currentParallelism != prev)
             {
                 _logger.LogInformation("Adaptive: file {FileId} Upscale. Increasing parallelism {Prev} -> {Next} due to chunk>=8MB",
                     fileId ?? "-", prev, _currentParallelism);
             }
         }
         else if (_currentChunkSizeBytes >= FOUR_MB)
@@ -360,8 +384,8 @@
             _currentParallelism = Math.Max(_currentParallelism, 3);
             if (_currentParallelism != prev)
             {
                 _logger.LogInformation("Adaptive: file {FileId} Upscale. Increasing parallelism {Prev} -> {Next} due to chunk>=4MB",
                     fileId ?? "-", prev, _currentParallelism);
             }
         }
     }

src/ByteSync.Client/Services/Communications/Transfers/Uploading/FileUploadWorker.cs

@@ -121,8 +121,8 @@
 
                     var fileName = _sharedFileDefinition.GetFileName(slice.PartNumber);
                     var assertSw = Stopwatch.StartNew();
                     _logger.LogDebug("UploadAvailableSlice: worker {WorkerId} start asserting slice {Number} for {FileName}",
                         workerId, slice.PartNumber, fileName);
 
                     var transferParameters = new TransferParameters
                     {
@@ -134,9 +134,9 @@
 
                     await AssertSliceUploadedAsync(policy, transferParameters, workerId, slice.PartNumber, fileName, assertSw);
                     assertSw.Stop();
                     _logger.LogDebug(
                         "UploadAvailableSlice: worker {WorkerId} finished asserting slice {Number} for {FileName} in {ElapsedMs} ms",
                         workerId, slice.PartNumber, fileName, assertSw.ElapsedMilliseconds);
 
                     await UpdateProgressOnSuccessAsync(progressState, slice, sliceStart);
                 }
@@ -174,18 +174,22 @@
             slice.MemoryStream.Length,
             attempt,
             currentChunkSizeBytes);
+        var chunkRatio = currentChunkSizeBytes > 0
+            ? slice.MemoryStream.Length / (double)currentChunkSizeBytes
+            : 0d;
         using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(globalToken);
         attemptCts.CancelAfter(TimeSpan.FromSeconds(timeoutSec));
 
         var beforeWait = _uploadSlots.CurrentCount;
         _logger.LogDebug(
-            "UploadAvailableSlice: worker {WorkerId} waiting for upload slot (available {Available}), attempt {Attempt}, timeout {TimeoutSec}s, slice {SliceKb} KB, currentChunk {CurrentChunkKb} KB",
+            "UploadAvailableSlice: worker {WorkerId} waiting for upload slot (available {Available}), attempt {Attempt}, timeout {TimeoutSec}s, slice {SliceKb} KB, currentChunk {CurrentChunkKb} KB, chunkRatio {ChunkRatio}",
             workerId,
             beforeWait,
             attempt,
             timeoutSec,
             Math.Round(slice.MemoryStream.Length / 1024d),
-            Math.Round(currentChunkSizeBytes / 1024d));
+            Math.Round(currentChunkSizeBytes / 1024d),
+            Math.Round(chunkRatio, 2));
 
         var acquired = false;
         try
@@ -194,8 +198,8 @@
             acquired = true;
 
             var afterWait = _uploadSlots.CurrentCount;
             _logger.LogDebug("UploadAvailableSlice: worker {WorkerId} acquired upload slot (available now {Available}), attempt {Attempt}",
                 workerId, afterWait, attempt);
 
             var uploadTask = DoUpload(slice, workerId, attemptCts.Token);
             var heartbeat = TimeSpan.FromSeconds(30);
@@ -208,9 +212,9 @@
                 }
 
                 var fileNameHb = _sharedFileDefinition.GetFileName(slice.PartNumber);
                 _logger.LogDebug(
                     "UploadAvailableSlice: worker {WorkerId} uploading slice {Number} for {FileName}... attempt {Attempt}, elapsed {ElapsedMs} ms",
                     workerId, slice.PartNumber, fileNameHb, attempt, (DateTime.UtcNow - attemptStart).TotalMilliseconds);
 
                 if (attemptCts.IsCancellationRequested)
                 {
@@ -275,15 +279,15 @@
                     var beforeRelease = _uploadSlots.CurrentCount;
                     _uploadSlots.Release();
                     var afterRelease = _uploadSlots.CurrentCount;
                     _logger.LogDebug(
                         "UploadAvailableSlice: worker {WorkerId} released upload slot after attempt {Attempt} (available {Before}->{After})",
                         workerId, attempt, beforeRelease, afterRelease);
                 }
                 else
                 {
                     _logger.LogDebug(
                         "UploadAvailableSlice: worker {WorkerId} did not acquire upload slot (canceled before acquire) for attempt {Attempt}",
                         workerId, attempt);
                 }
             }
             catch (Exception ex)
@@ -340,9 +344,9 @@
             var completed = await Task.WhenAny(assertTask, Task.Delay(TimeSpan.FromSeconds(30)));
             if (completed != assertTask)
             {
                 _logger.LogDebug(
                     "UploadAvailableSlice: worker {WorkerId} asserting slice {Number} for {FileName}... elapsed {ElapsedMs} ms",
                     workerId, partNumber, fileName, assertSw.ElapsedMilliseconds);
             }
         }
 
@@ -468,8 +472,8 @@
             if (progressState.TotalUploadedSlices == progressState.TotalCreatedSlices)
             {
                 _uploadingIsFinished.Set();
                 _logger.LogDebug("UploadAvailableSlice: all slices uploaded ({Uploaded}/{Created}) - signaling completion",
                     progressState.TotalUploadedSlices, progressState.TotalCreatedSlices);
             }
         }
         finally
@@ -492,16 +496,16 @@
             var uploadLocation = await _fileTransferApiClient.GetUploadFileStorageLocation(transferParameters);
             var lengthKbRounded = (long)Math.Round((slice.MemoryStream.Length) / 1024d);
             var fileName = _sharedFileDefinition.GetFileName(slice.PartNumber);
             _logger.LogDebug("UploadAvailableSlice: worker {WorkerId} start uploading slice {Number} for {FileName} ({LengthKb} KB)",
                 workerId, slice.PartNumber, fileName, lengthKbRounded);
 
             var uploadStrategy = _strategies[uploadLocation.StorageProvider];
             var sw = Stopwatch.StartNew();
             var response = await uploadStrategy.UploadAsync(slice, uploadLocation, cancellationToken);
             sw.Stop();
             _logger.LogDebug(
                 "UploadAvailableSlice: worker {WorkerId} finished uploading slice {Number} for {FileName} ({LengthKb} KB) in {ElapsedMs} ms (status {Status})",
                 workerId, slice.PartNumber, fileName, lengthKbRounded, sw.ElapsedMilliseconds, response.StatusCode);
 
             return response;
         }

src/ByteSync.Client/Services/Communications/Transfers/Uploading/UploadAttemptTimeoutPolicy.cs

@@ -6,22 +6,26 @@ public static class UploadAttemptTimeoutPolicy
 {
     private const int AttemptTimeoutFloorSeconds = 60;
     private const int AttemptTimeoutCeilingSeconds = 180;
+    private const int ExtendedOversizedSliceTimeoutCeilingSeconds = 300;
     private const int SecondsPerMegabyteHeuristic = 3;
     private const int RetryGrowthSeconds = 15;
-    private const int StaleChunkPenaltySeconds = 5;
+    private const int OversizedSliceSecondsPerCurrentChunk = 30;
+    private const int ExtendedOversizedSliceThresholdBytes = 1024 * 1024;
+    private const int ExtendedOversizedSliceChunkRatioThreshold = 8;
 
     public static int ComputeTimeoutSeconds(long sliceLengthBytes, int attempt, int currentChunkSizeBytes)
     {
-        var timeoutSec = (long)ComputeBaseTimeoutSeconds(sliceLengthBytes);
-        if (attempt <= 1)
+        var timeoutCeilingSeconds = ComputeTimeoutCeilingSeconds(sliceLengthBytes, currentChunkSizeBytes);
+        var timeoutSec = Math.Max(
+            (long)ComputeBaseTimeoutSeconds(sliceLengthBytes),
+            ComputeOversizedSliceTimeoutSeconds(sliceLengthBytes, currentChunkSizeBytes, timeoutCeilingSeconds));
+
+        if (attempt > 1)
         {
-            return (int)timeoutSec;
+            timeoutSec += (long)(attempt - 1) * RetryGrowthSeconds;
         }
 
-        var staleChunkPenalty = ComputeStaleChunkPenaltySeconds(sliceLengthBytes, currentChunkSizeBytes);
-        timeoutSec += (long)(attempt - 1) * RetryGrowthSeconds + staleChunkPenalty;
-
-        return (int)Math.Clamp(timeoutSec, AttemptTimeoutFloorSeconds, AttemptTimeoutCeilingSeconds);
+        return (int)Math.Clamp(timeoutSec, AttemptTimeoutFloorSeconds, timeoutCeilingSeconds);
     }
 
     private static int ComputeBaseTimeoutSeconds(long sliceLengthBytes)
@@ -39,19 +43,36 @@ private static int ComputeBaseTimeoutSeconds(long sliceLengthBytes)
             AttemptTimeoutCeilingSeconds);
     }
 
-    private static long ComputeStaleChunkPenaltySeconds(long sliceLengthBytes, int currentChunkSizeBytes)
+    private static int ComputeTimeoutCeilingSeconds(long sliceLengthBytes, int currentChunkSizeBytes)
+    {
+        if (currentChunkSizeBytes <= 0 || sliceLengthBytes <= currentChunkSizeBytes)
+        {
+            return AttemptTimeoutCeilingSeconds;
+        }
+
+        var chunkRatio = Math.Ceiling(sliceLengthBytes / (double)currentChunkSizeBytes);
+        if (sliceLengthBytes >= ExtendedOversizedSliceThresholdBytes
+            && chunkRatio >= ExtendedOversizedSliceChunkRatioThreshold)
+        {
+            return ExtendedOversizedSliceTimeoutCeilingSeconds;
+        }
+
+        return AttemptTimeoutCeilingSeconds;
+    }
+
+    private static long ComputeOversizedSliceTimeoutSeconds(long sliceLengthBytes, int currentChunkSizeBytes, int timeoutCeilingSeconds)
     {
         if (currentChunkSizeBytes <= 0 || sliceLengthBytes <= currentChunkSizeBytes)
         {
             return 0;
         }
 
         var chunkRatio = Math.Ceiling(sliceLengthBytes / (double)currentChunkSizeBytes);
-        if (chunkRatio >= AttemptTimeoutCeilingSeconds / (double)StaleChunkPenaltySeconds + 1)
+        if (chunkRatio >= timeoutCeilingSeconds / (double)OversizedSliceSecondsPerCurrentChunk)
         {
-            return AttemptTimeoutCeilingSeconds;
+            return timeoutCeilingSeconds;
         }
 
-        return (long)(chunkRatio - 1) * StaleChunkPenaltySeconds;
+        return (long)chunkRatio * OversizedSliceSecondsPerCurrentChunk;
     }
 }

tests/ByteSync.Client.UnitTests/Services/Communications/Transfers/Strategies/UploadFailureClassifierTests.cs

@@ -70,7 +70,6 @@ public void Classify_TaskCanceledException_WithNonCancelledToken_ShouldReturnCli
     public void Classify_GenericException_ShouldReturnServerError500()
     {
         using var cts = new CancellationTokenSource();
-        cts.Cancel();
         var ex = new InvalidOperationException("broken");
 
         var response = UploadFailureClassifier.Classify(ex, cts.Token);
@@ -111,6 +110,22 @@ public void Classify_HttpRequestExceptionWithConnectionReset_ShouldReturnClientN
         response.Exception.Should().BeSameAs(ex);
     }
 
+    [TestCase("Received an unexpected EOF from the transport stream.")]
+    [TestCase("Received 0 bytes from the transport stream.")]
+    public void Classify_HttpRequestExceptionWithUnexpectedTransportClosureMessage_ShouldReturnClientNetworkError(string message)
+    {
+        using var cts = new CancellationTokenSource();
+        var ioException = new IOException(message);
+        var ex = new HttpRequestException("The SSL connection could not be established, see inner exception.", ioException);
+
+        var response = UploadFailureClassifier.Classify(ex, cts.Token);
+
+        response.IsSuccess.Should().BeFalse();
+        response.StatusCode.Should().Be(0);
+        response.FailureKind.Should().Be(UploadFailureKind.ClientNetworkError);
+        response.Exception.Should().BeSameAs(ex);
+    }
+
     [Test]
     public void Classify_DirectSocketExceptionWithConnectionReset_ShouldReturnClientNetworkError()
     {
@@ -124,4 +139,33 @@ public void Classify_DirectSocketExceptionWithConnectionReset_ShouldReturnClient
         response.FailureKind.Should().Be(UploadFailureKind.ClientNetworkError);
         response.Exception.Should().BeSameAs(ex);
     }
+
+    [Test]
+    public void Classify_DirectSocketExceptionWithOperationAborted_ShouldReturnClientNetworkError()
+    {
+        using var cts = new CancellationTokenSource();
+        var ex = new SocketException((int)SocketError.OperationAborted);
+
+        var response = UploadFailureClassifier.Classify(ex, cts.Token);
+
+        response.IsSuccess.Should().BeFalse();
+        response.StatusCode.Should().Be(0);
+        response.FailureKind.Should().Be(UploadFailureKind.ClientNetworkError);
+        response.Exception.Should().BeSameAs(ex);
+    }
+
+    [Test]
+    public void Classify_DirectSocketExceptionWithOperationAbortedAndCancelledToken_ShouldReturnClientCancellation()
+    {
+        using var cts = new CancellationTokenSource();
+        cts.Cancel();
+        var ex = new SocketException((int)SocketError.OperationAborted);
+
+        var response = UploadFailureClassifier.Classify(ex, cts.Token);
+
+        response.IsSuccess.Should().BeFalse();
+        response.StatusCode.Should().Be(0);
+        response.FailureKind.Should().Be(UploadFailureKind.ClientCancellation);
+        response.Exception.Should().BeSameAs(ex);
+    }
 }

tests/ByteSync.Client.UnitTests/Services/Communications/Transfers/Uploading/AdaptiveUploadControllerTests.cs

@@ -120,6 +120,7 @@ public void Downscale_ReducesChunkSize_WhenAtMinParallelism()
         _controller.CurrentParallelism.Should().Be(2);
         _controller.CurrentChunkSizeBytes.Should().BeLessThan(before);
         _controller.CurrentChunkSizeBytes.Should().BeGreaterThanOrEqualTo(64 * 1024);
+        _controller.CurrentChunkSizeBytes.Should().Be(375 * 1024);
     }
 
     [Test]
@@ -197,7 +198,7 @@ public void ClientTimeouts_DownscaleBelowInitialChunkSize_WhenAtMinParallelism()
         // Assert
         _controller.CurrentParallelism.Should().Be(2);
         _controller.CurrentChunkSizeBytes.Should().BeLessThan(500 * 1024);
-        _controller.CurrentChunkSizeBytes.Should().Be(375 * 1024);
+        _controller.CurrentChunkSizeBytes.Should().Be(250 * 1024);
     }
 
     [Test]
@@ -212,11 +213,11 @@ public void ClientNetworkErrors_DownscaleBelowInitialChunkSize_WhenAtMinParallel
 
         // Assert
         _controller.CurrentParallelism.Should().Be(2);
-        _controller.CurrentChunkSizeBytes.Should().Be(375 * 1024);
+        _controller.CurrentChunkSizeBytes.Should().Be(250 * 1024);
     }
 
     [Test]
-    public void ClientTimeouts_ReduceParallelismFirst_WhenAboveMinParallelism()
+    public void ClientTimeouts_DownscaleParallelismAndChunk_WhenAboveMinParallelism()
     {
         // Arrange
         var safety = 100;
@@ -233,8 +234,9 @@ public void ClientTimeouts_ReduceParallelismFirst_WhenAboveMinParallelism()
         FeedClientTimeouts(_controller, 2);
 
         // Assert
-        _controller.CurrentParallelism.Should().Be(beforeParallelism - 1);
-        _controller.CurrentChunkSizeBytes.Should().Be(beforeChunk);
+        _controller.CurrentParallelism.Should().Be(2);
+        _controller.CurrentParallelism.Should().BeLessThan(beforeParallelism);
+        _controller.CurrentChunkSizeBytes.Should().Be(Math.Max(64 * 1024, (int)Math.Round(beforeChunk * 0.5)));
     }
 
     [Test]