Add option to reverse message order in exports (#1487)

Co-authored-by: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>

DiscordChatExporter.Cli.Tests/Specs/HtmlContentSpecs.cs

@@ -1,7 +1,12 @@
-using System.Linq;
+using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
 using AngleSharp.Dom;
+using CliFx.Infrastructure;
+using DiscordChatExporter.Cli.Commands;
 using DiscordChatExporter.Cli.Tests.Infra;
+using DiscordChatExporter.Cli.Tests.Utils;
+using DiscordChatExporter.Core.Exporting;
 using FluentAssertions;
 using Xunit;
 
@@ -44,4 +49,41 @@ public class HtmlContentSpecs
                 "Yeet"
             );
     }
+
+    [Fact]
+    public async Task I_can_export_a_channel_in_the_HTML_format_in_the_reverse_order()
+    {
+        // Arrange
+        using var file = TempFile.Create();
+
+        // Act
+        await new ExportChannelsCommand
+        {
+            Token = Secrets.DiscordToken,
+            ChannelIds = [ChannelIds.DateRangeTestCases],
+            ExportFormat = ExportFormat.HtmlDark,
+            OutputPath = file.Path,
+            Locale = "en-US",
+            IsUtcNormalizationEnabled = true,
+            IsReverseMessageOrder = true,
+        }.ExecuteAsync(new FakeConsole());
+
+        var document = Html.Parse(await File.ReadAllTextAsync(file.Path));
+        var messages = document.QuerySelectorAll("[data-message-id]").ToArray();
+
+        // Assert
+        messages
+            .Select(e => e.GetAttribute("data-message-id"))
+            .Should()
+            .Equal(
+                "885169254029213696",
+                "868505973294268457",
+                "868505969821364245",
+                "868505966528835604",
+                "868490009366396958",
+                "866732113319428096",
+                "866710679758045195",
+                "866674314627121232"
+            );
+    }
 }

DiscordChatExporter.Cli.Tests/Specs/JsonContentSpecs.cs

@@ -1,7 +1,13 @@
-using System.Linq;
+using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
+using CliFx.Infrastructure;
+using DiscordChatExporter.Cli.Commands;
 using DiscordChatExporter.Cli.Tests.Infra;
+using DiscordChatExporter.Cli.Tests.Utils;
+using DiscordChatExporter.Core.Exporting;
 using FluentAssertions;
+using JsonExtensions;
 using Xunit;
 
 namespace DiscordChatExporter.Cli.Tests.Specs;
@@ -43,4 +49,43 @@ public class JsonContentSpecs
                 "Yeet"
             );
     }
+
+    [Fact]
+    public async Task I_can_export_a_channel_in_the_JSON_format_in_the_reverse_order()
+    {
+        // Arrange
+        using var file = TempFile.Create();
+
+        // Act
+        await new ExportChannelsCommand
+        {
+            Token = Secrets.DiscordToken,
+            ChannelIds = [ChannelIds.DateRangeTestCases],
+            ExportFormat = ExportFormat.Json,
+            OutputPath = file.Path,
+            Locale = "en-US",
+            IsUtcNormalizationEnabled = true,
+            IsReverseMessageOrder = true,
+        }.ExecuteAsync(new FakeConsole());
+
+        var messages = Json.Parse(await File.ReadAllTextAsync(file.Path))
+            .GetProperty("messages")
+            .EnumerateArray()
+            .ToArray();
+
+        // Assert
+        messages
+            .Select(j => j.GetProperty("id").GetString())
+            .Should()
+            .Equal(
+                "885169254029213696",
+                "868505973294268457",
+                "868505969821364245",
+                "868505966528835604",
+                "868490009366396958",
+                "866732113319428096",
+                "866710679758045195",
+                "866674314627121232"
+            );
+    }
 }

DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs

@@ -83,6 +83,12 @@ public abstract class ExportCommandBase : DiscordCommandBase
     )]
     public int ParallelLimit { get; init; } = 1;
 
+    [CommandOption(
+        "reverse",
+        Description = "Export messages in reverse chronological order (newest first)."
+    )]
+    public bool IsReverseMessageOrder { get; init; }
+
     [CommandOption(
         "markdown",
         Description = "Process markdown, mentions, and other special tokens."
@@ -267,6 +273,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
                                         Before,
                                         PartitionLimit,
                                         MessageFilter,
+                                        IsReverseMessageOrder,
                                         ShouldFormatMarkdown,
                                         ShouldDownloadAssets,
                                         ShouldReuseAssets,

DiscordChatExporter.Core/Discord/DiscordClient.cs

@@ -578,6 +578,28 @@ public class DiscordClient(
         }
     }
 
+    private async ValueTask<Message?> TryGetFirstMessageAsync(
+        Snowflake channelId,
+        Snowflake? after = null,
+        Snowflake? before = null,
+        CancellationToken cancellationToken = default
+    )
+    {
+        var url = new UrlBuilder()
+            .SetPath($"channels/{channelId}/messages")
+            .SetQueryParameter("limit", "1")
+            .SetQueryParameter("after", (after ?? Snowflake.Zero).ToString())
+            .Build();
+
+        var response = await GetJsonResponseAsync(url, cancellationToken);
+        var message = response.EnumerateArray().Select(Message.Parse).FirstOrDefault();
+
+        if (message is null || before is not null && message.Timestamp > before.Value.ToDate())
+            return null;
+
+        return message;
+    }
+
     private async ValueTask<Message?> TryGetLastMessageAsync(
         Snowflake channelId,
         Snowflake? before = null,
@@ -684,6 +706,86 @@ public class DiscordClient(
         }
     }
 
+    public async IAsyncEnumerable<Message> GetMessagesInReverseAsync(
+        Snowflake channelId,
+        Snowflake? after = null,
+        Snowflake? before = null,
+        IProgress<Percentage>? progress = null,
+        [EnumeratorCancellation] CancellationToken cancellationToken = default
+    )
+    {
+        // Get the first message (oldest) in the range to use as the lower bound for
+        // progress calculation.
+        var firstMessage = await TryGetFirstMessageAsync(
+            channelId,
+            after,
+            before,
+            cancellationToken
+        );
+        if (firstMessage is null)
+            yield break;
+
+        // Keep track of the last message in range in order to calculate the progress
+        var lastMessage = default(Message);
+        var currentBefore = before;
+        while (true)
+        {
+            var url = new UrlBuilder()
+                .SetPath($"channels/{channelId}/messages")
+                .SetQueryParameter("limit", "100")
+                .SetQueryParameter("before", currentBefore?.ToString())
+                .Build();
+
+            var response = await GetJsonResponseAsync(url, cancellationToken);
+
+            var messages = response.EnumerateArray().Select(Message.Parse).ToArray();
+
+            // Break if there are no messages (can happen if messages are deleted during execution)
+            if (!messages.Any())
+                yield break;
+
+            // If all messages are empty, make sure that it's not because the bot account doesn't
+            // have the Message Content Intent enabled.
+            // https://github.com/Tyrrrz/DiscordChatExporter/issues/1106#issuecomment-1741548959
+            if (
+                messages.All(m => m.IsEmpty)
+                && await ResolveTokenKindAsync(cancellationToken) == TokenKind.Bot
+            )
+            {
+                var application = await GetApplicationAsync(cancellationToken);
+                if (!application.IsMessageContentIntentEnabled)
+                {
+                    throw new DiscordChatExporterException(
+                        "Provided bot account does not have the Message Content Intent enabled.",
+                        true
+                    );
+                }
+            }
+
+            foreach (var message in messages)
+            {
+                lastMessage ??= message;
+
+                // Report progress based on timestamps
+                if (progress is not null)
+                {
+                    var exportedDuration = (lastMessage.Timestamp - message.Timestamp).Duration();
+                    var totalDuration = (lastMessage.Timestamp - firstMessage.Timestamp).Duration();
+
+                    progress.Report(
+                        Percentage.FromFraction(
+                            totalDuration > TimeSpan.Zero ? exportedDuration / totalDuration : 1
+                        )
+                    );
+                }
+
+                yield return message;
+            }
+
+            currentBefore = messages.Last().Id;
+        }
+    }
+
     public async IAsyncEnumerable<User> GetMessageReactionsAsync(
         Snowflake channelId,
         Snowflake messageId,

DiscordChatExporter.Core/Exporting/ChannelExporter.cs

@@ -64,15 +64,23 @@ public class ChannelExporter(DiscordClient discord)
             );
         }
 
-        await foreach (
+        var messages = !request.IsReverseMessageOrder
-            var message in discord.GetMessagesAsync(
+            ? discord.GetMessagesAsync(
                 request.Channel.Id,
                 request.After,
                 request.Before,
                 progress,
                 cancellationToken
             )
-        )
+            : discord.GetMessagesInReverseAsync(
+                request.Channel.Id,
+                request.After,
+                request.Before,
+                progress,
+                cancellationToken
+            );
+
+        await foreach (var message in messages)
         {
             try
             {

DiscordChatExporter.Core/Exporting/ExportRequest.cs

@@ -33,6 +33,8 @@ public partial class ExportRequest
 
     public MessageFilter MessageFilter { get; }
 
+    public bool IsReverseMessageOrder { get; }
+
     public bool ShouldFormatMarkdown { get; }
 
     public bool ShouldDownloadAssets { get; }
@@ -55,6 +57,7 @@ public partial class ExportRequest
         Snowflake? before,
         PartitionLimit partitionLimit,
         MessageFilter messageFilter,
+        bool isReverseMessageOrder,
         bool shouldFormatMarkdown,
         bool shouldDownloadAssets,
         bool shouldReuseAssets,
@@ -69,6 +72,7 @@ public partial class ExportRequest
         Before = before;
         PartitionLimit = partitionLimit;
         MessageFilter = messageFilter;
+        IsReverseMessageOrder = isReverseMessageOrder;
         ShouldFormatMarkdown = shouldFormatMarkdown;
         ShouldDownloadAssets = shouldDownloadAssets;
         ShouldReuseAssets = shouldReuseAssets;

DiscordChatExporter.Core/Exporting/HtmlMessageWriter.cs

@@ -17,6 +17,7 @@ internal class HtmlMessageWriter(Stream stream, ExportContext context, string th
     private readonly HtmlMinifier _minifier = new();
     private readonly List<Message> _messageGroup = [];
 
+    // Note: in reverse order, last message is earlier than first message
     private bool CanJoinGroup(Message message)
     {
         // If the group is empty, any message can join it

DiscordChatExporter.Core/Exporting/PlainTextMessageExtensions.cs

@@ -23,13 +23,11 @@ internal static class PlainTextMessageExtensions
                     : "Removed a recipient.",
 
                 MessageKind.Call =>
-                    $"Started a call that lasted {
+                    $"Started a call that lasted {message
-                        message
                             .CallEndedTimestamp?
                             .Pipe(t => t - message.Timestamp)
                             .Pipe(t => t.TotalMinutes)
-                            .ToString("n0", CultureInfo.InvariantCulture) ?? "0"
+                            .ToString("n0", CultureInfo.InvariantCulture) ?? "0"} minutes.",
-                    } minutes.",
 
                 MessageKind.ChannelNameChange => !string.IsNullOrWhiteSpace(message.Content)
                     ? $"Changed the channel name: {message.Content}"

DiscordChatExporter.Gui/Localization/LocalizationManager.English.cs

@@ -104,6 +104,9 @@ public partial class LocalizationManager
             [nameof(MessageFilterLabel)] = "Message filter",
             [nameof(MessageFilterTooltip)] =
                 "Only include messages that satisfy this filter (e.g. 'from:foo#1234' or 'has:image'). See the documentation for more info.",
+            [nameof(ReverseMessageOrderLabel)] = "Reverse messages",
+            [nameof(ReverseMessageOrderTooltip)] =
+                "Export messages in reverse chronological order (newest first)",
             [nameof(FormatMarkdownLabel)] = "Format markdown",
             [nameof(FormatMarkdownTooltip)] =
                 "Process markdown, mentions, and other special tokens",

DiscordChatExporter.Gui/Localization/LocalizationManager.French.cs

@@ -106,6 +106,9 @@ public partial class LocalizationManager
         [nameof(MessageFilterLabel)] = "Filtre de messages",
         [nameof(MessageFilterTooltip)] =
             "Inclure uniquement les messages satisfaisant ce filtre (ex. 'from:foo#1234' ou 'has:image'). Voir la documentation pour plus d'informations.",
+        [nameof(ReverseMessageOrderLabel)] = "Inverser l'ordre des messages",
+        [nameof(ReverseMessageOrderTooltip)] =
+            "Exporter les messages en ordre chronologique inversé (les plus récents en premier)",
         [nameof(FormatMarkdownLabel)] = "Formater le markdown",
         [nameof(FormatMarkdownTooltip)] =
             "Traiter le markdown, les mentions et autres tokens spéciaux",

DiscordChatExporter.Gui/Localization/LocalizationManager.German.cs

@@ -110,6 +110,9 @@ public partial class LocalizationManager
         [nameof(MessageFilterLabel)] = "Nachrichtenfilter",
         [nameof(MessageFilterTooltip)] =
             "Nur Nachrichten einschließen, die diesem Filter entsprechen (z. B. 'from:foo#1234' oder 'has:image'). Weitere Informationen finden Sie in der Dokumentation.",
+        [nameof(ReverseMessageOrderLabel)] = "Nachrichtenreihenfolge umkehren",
+        [nameof(ReverseMessageOrderTooltip)] =
+            "Nachrichten in umgekehrter chronologischer Reihenfolge exportieren (neueste zuerst)",
         [nameof(FormatMarkdownLabel)] = "Markdown formatieren",
         [nameof(FormatMarkdownTooltip)] =
             "Markdown, Erwähnungen und andere spezielle Token verarbeiten",

DiscordChatExporter.Gui/Localization/LocalizationManager.Spanish.cs

@@ -104,6 +104,9 @@ public partial class LocalizationManager
             [nameof(MessageFilterLabel)] = "Filtro de mensajes",
             [nameof(MessageFilterTooltip)] =
                 "Solo incluir mensajes que satisfagan este filtro (p. ej. 'from:foo#1234' o 'has:image'). Consulte la documentación para más información.",
+            [nameof(ReverseMessageOrderLabel)] = "Invertir orden de mensajes",
+            [nameof(ReverseMessageOrderTooltip)] =
+                "Exportar mensajes en orden cronológico inverso (los más recientes primero)",
             [nameof(FormatMarkdownLabel)] = "Formatear markdown",
             [nameof(FormatMarkdownTooltip)] =
                 "Procesar markdown, menciones y otros tokens especiales",

DiscordChatExporter.Gui/Localization/LocalizationManager.Ukrainian.cs

@@ -104,6 +104,9 @@ public partial class LocalizationManager
             [nameof(MessageFilterLabel)] = "Фільтр повідомлень",
             [nameof(MessageFilterTooltip)] =
                 "Включати лише повідомлення, що відповідають цьому фільтру (напр. 'from:foo#1234' або 'has:image'). Дивіться документацію для більш детальної інформації.",
+            [nameof(ReverseMessageOrderLabel)] = "Зворотній порядок повідомлень",
+            [nameof(ReverseMessageOrderTooltip)] =
+                "Експортувати повідомлення у зворотному хронологічному порядку (найновіші спочатку)",
             [nameof(FormatMarkdownLabel)] = "Форматувати markdown",
             [nameof(FormatMarkdownTooltip)] =
                 "Обробляти markdown, згадки та інші спеціальні токени",

DiscordChatExporter.Gui/Localization/LocalizationManager.cs

@@ -135,6 +135,8 @@ public partial class LocalizationManager
     public string PartitionLimitTooltip => Get();
     public string MessageFilterLabel => Get();
     public string MessageFilterTooltip => Get();
+    public string ReverseMessageOrderLabel => Get();
+    public string ReverseMessageOrderTooltip => Get();
     public string FormatMarkdownLabel => Get();
     public string FormatMarkdownTooltip => Get();
     public string DownloadAssetsLabel => Get();

DiscordChatExporter.Gui/Services/SettingsService.cs

@@ -61,6 +61,9 @@ public partial class SettingsService()
     [ObservableProperty]
     public partial string? LastMessageFilterValue { get; set; }
 
+    [ObservableProperty]
+    public partial bool LastIsReverseMessageOrder { get; set; }
+
     [ObservableProperty]
     public partial bool LastShouldFormatMarkdown { get; set; } = true;
 

DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs

@@ -275,6 +275,7 @@ public partial class DashboardViewModel : ViewModelBase
                             dialog.Before?.Pipe(Snowflake.FromDate),
                             dialog.PartitionLimit,
                             dialog.MessageFilter,
+                            dialog.IsReverseMessageOrder,
                             dialog.ShouldFormatMarkdown,
                             dialog.ShouldDownloadAssets,
                             dialog.ShouldReuseAssets,

DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs

@@ -62,6 +62,9 @@ public partial class ExportSetupViewModel(
     [NotifyPropertyChangedFor(nameof(MessageFilter))]
     public partial string? MessageFilterValue { get; set; }
 
+    [ObservableProperty]
+    public partial bool IsReverseMessageOrder { get; set; }
+
     [ObservableProperty]
     public partial bool ShouldFormatMarkdown { get; set; }
 
@@ -106,6 +109,7 @@ public partial class ExportSetupViewModel(
         SelectedFormat = settingsService.LastExportFormat;
         PartitionLimitValue = settingsService.LastPartitionLimitValue;
         MessageFilterValue = settingsService.LastMessageFilterValue;
+        IsReverseMessageOrder = settingsService.LastIsReverseMessageOrder;
         ShouldFormatMarkdown = settingsService.LastShouldFormatMarkdown;
         ShouldDownloadAssets = settingsService.LastShouldDownloadAssets;
         ShouldReuseAssets = settingsService.LastShouldReuseAssets;
@@ -120,7 +124,8 @@ public partial class ExportSetupViewModel(
             || !string.IsNullOrWhiteSpace(MessageFilterValue)
             || ShouldDownloadAssets
             || ShouldReuseAssets
-            || !string.IsNullOrWhiteSpace(AssetsDirPath);
+            || !string.IsNullOrWhiteSpace(AssetsDirPath)
+            || IsReverseMessageOrder;
     }
 
     [RelayCommand]
@@ -184,6 +189,7 @@ public partial class ExportSetupViewModel(
         settingsService.LastExportFormat = SelectedFormat;
         settingsService.LastPartitionLimitValue = PartitionLimitValue;
         settingsService.LastMessageFilterValue = MessageFilterValue;
+        settingsService.LastIsReverseMessageOrder = IsReverseMessageOrder;
         settingsService.LastShouldFormatMarkdown = ShouldFormatMarkdown;
         settingsService.LastShouldDownloadAssets = ShouldDownloadAssets;
         settingsService.LastShouldReuseAssets = ShouldReuseAssets;

DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.axaml

@@ -196,6 +196,15 @@
                             Theme="{DynamicResource FilledTextBox}"
                             ToolTip.Tip="{Binding LocalizationManager.MessageFilterTooltip}" />
 
+                        <!--  Reverse message order  -->
+                        <DockPanel
+                            Margin="16,8"
+                            LastChildFill="False"
+                            ToolTip.Tip="{Binding LocalizationManager.ReverseMessageOrderTooltip}">
+                            <TextBlock DockPanel.Dock="Left" Text="{Binding LocalizationManager.ReverseMessageOrderLabel}" />
+                            <ToggleSwitch DockPanel.Dock="Right" IsChecked="{Binding IsReverseMessageOrder}" />
+                        </DockPanel>
+
                         <!--  Markdown formatting  -->
                         <DockPanel
                             Margin="16,8"