Rework retry policy

Fixes #832

DiscordChatExporter.Core/Discord/DiscordClient.cs

@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
-using System.Net.Http.Headers;
 using System.Runtime.CompilerServices;
 using System.Text.Json;
 using System.Threading;
@@ -52,7 +51,7 @@ public class DiscordClient
         string url,
         CancellationToken cancellationToken = default)
     {
-        return await Http.ResponsePolicy.ExecuteAsync(async innerCancellationToken =>
+        return await Http.ResponseResiliencePolicy.ExecuteAsync(async innerCancellationToken =>
         {
             if (_tokenKind == TokenKind.User)
                 return await GetResponseAsync(url, false, innerCancellationToken);

DiscordChatExporter.Core/Exporting/MediaDownloader.cs

@@ -35,20 +35,16 @@ internal partial class MediaDownloader
         var filePath = Path.Combine(_workingDirPath, fileName);
 
         // Reuse existing files if we're allowed to
-        if (_reuseMedia && File.Exists(filePath))
+        if (!_reuseMedia || !File.Exists(filePath))
-            return _pathCache[url] = filePath;
+        {
-
             Directory.CreateDirectory(_workingDirPath);
 
-        // This retries on IOExceptions which is dangerous as we're also working with files
+            await Http.ResiliencePolicy.ExecuteAsync(async () =>
-        await Http.ExceptionPolicy.ExecuteAsync(async () =>
             {
                 // Download the file
                 using var response = await Http.Client.GetAsync(url, cancellationToken);
                 await using (var output = File.Create(filePath))
-            {
                     await response.Content.CopyToAsync(output, cancellationToken);
-            }
 
                 // Try to set the file date according to the last-modified header
                 try
@@ -74,6 +70,7 @@ internal partial class MediaDownloader
                     // ignore exceptions thrown here.
                 }
             });
+        }
 
         return _pathCache[url] = filePath;
     }

DiscordChatExporter.Core/Utils/Extensions/ExceptionExtensions.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Net;
+using System.Net.Http;
+using System.Text.RegularExpressions;
+
+namespace DiscordChatExporter.Core.Utils.Extensions;
+
+public static class ExceptionExtensions
+{
+    private static void PopulateChildren(this Exception exception, ICollection<Exception> children)
+    {
+        if (exception is AggregateException aggregateException)
+        {
+            foreach (var innerException in aggregateException.InnerExceptions)
+            {
+                children.Add(innerException);
+                PopulateChildren(innerException, children);
+            }
+        }
+        else if (exception.InnerException is not null)
+        {
+            children.Add(exception.InnerException);
+            PopulateChildren(exception.InnerException, children);
+        }
+    }
+
+    public static IReadOnlyList<Exception> GetSelfAndChildren(this Exception exception)
+    {
+        var children = new List<Exception> {exception};
+        PopulateChildren(exception, children);
+        return children;
+    }
+
+    public static HttpStatusCode? TryGetStatusCode(this HttpRequestException ex) =>
+        // This is extremely frail, but there's no other way
+        Regex
+            .Match(ex.Message, @": (\d+) \(")
+            .Groups[1]
+            .Value
+            .NullIfWhiteSpace()?
+            .Pipe(s => (HttpStatusCode) int.Parse(s, CultureInfo.InvariantCulture));
+}

DiscordChatExporter.Core/Utils/Http.cs

@@ -1,9 +1,8 @@
 using System;
-using System.Globalization;
+using System.Linq;
-using System.IO;
 using System.Net;
 using System.Net.Http;
-using System.Text.RegularExpressions;
+using System.Net.Sockets;
 using System.Threading.Tasks;
 using DiscordChatExporter.Core.Utils.Extensions;
 using Polly;
@@ -14,13 +13,26 @@ public static class Http
 {
     public static HttpClient Client { get; } = new();
 
-    public static IAsyncPolicy<HttpResponseMessage> ResponsePolicy { get; } =
+    private static bool IsRetryableStatusCode(HttpStatusCode statusCode) => statusCode is
+        HttpStatusCode.TooManyRequests or
+        HttpStatusCode.RequestTimeout or
+        HttpStatusCode.InternalServerError;
+
+    private static bool IsRetryableException(Exception exception) =>
+        exception.GetSelfAndChildren().Any(ex =>
+            ex is TimeoutException or SocketException ||
+            ex is HttpRequestException hrex && IsRetryableStatusCode(hrex.TryGetStatusCode() ?? HttpStatusCode.OK)
+        );
+
+    public static IAsyncPolicy ResiliencePolicy { get; } =
         Policy
-            .Handle<IOException>()
+            .Handle<Exception>(IsRetryableException)
-            .Or<HttpRequestException>()
+            .WaitAndRetryAsync(4, i => TimeSpan.FromSeconds(Math.Pow(2, i) + 1));
-            .OrResult<HttpResponseMessage>(m => m.StatusCode == HttpStatusCode.TooManyRequests)
+
-            .OrResult(m => m.StatusCode == HttpStatusCode.RequestTimeout)
+    public static IAsyncPolicy<HttpResponseMessage> ResponseResiliencePolicy { get; } =
-            .OrResult(m => m.StatusCode >= HttpStatusCode.InternalServerError)
+        Policy
+            .Handle<Exception>(IsRetryableException)
+            .OrResult<HttpResponseMessage>(m => IsRetryableStatusCode(m.StatusCode))
             .WaitAndRetryAsync(
                 8,
                 (i, result, _) =>
@@ -43,24 +55,4 @@ public static class Http
                 },
                 (_, _, _, _) => Task.CompletedTask
             );
-
-    private static HttpStatusCode? TryGetStatusCodeFromException(HttpRequestException ex) =>
-        // This is extremely frail, but there's no other way
-        Regex
-            .Match(ex.Message, @": (\d+) \(")
-            .Groups[1]
-            .Value
-            .NullIfWhiteSpace()?
-            .Pipe(s => (HttpStatusCode) int.Parse(s, CultureInfo.InvariantCulture));
-
-    public static IAsyncPolicy ExceptionPolicy { get; } =
-        Policy
-            .Handle<IOException>() // dangerous
-            .Or<HttpRequestException>(ex =>
-                TryGetStatusCodeFromException(ex) is
-                    HttpStatusCode.TooManyRequests or
-                    HttpStatusCode.RequestTimeout or
-                    HttpStatusCode.InternalServerError
-            )
-            .WaitAndRetryAsync(4, i => TimeSpan.FromSeconds(Math.Pow(2, i) + 1));
 }