Added existing export search

Previously, an existing export could only be detected if it existed at the current file target path.
However, if the name of the channel, the channel parent or the guild has changed or if the default file name formatting has changed, the existing export's file name would be different, and it could therefore not be detected.
Therefore, the option to explicitly search for the existing export in the target directory has been added.
If it's activated (and there's no existing export at the current file target path), all file names in the target directory will be compared to a regex that matches any file name the channel export (with the same date range) might have had in the past.
If several existing exports have been detected, an error is logged and the channel export is aborted. Otherwise, it continues as before (only additionally moving the existing export files to the to the new file paths if they should be appended).
Whether this new option is activated is currently hardcoded.
The enum FileExistsHandling has been renamed to ExportExistsHandling to clarify that the setting applies to existing exports in general.

If file names of a directory are needed, they are collected lazily and stored for future use in other channel exports to avoid unnecessary I/O operations.
This commit is contained in:
Kornelius Rohrschneider 2025-10-30 00:04:37 +01:00
parent 1ae03ad206
commit 90ed829375
No known key found for this signature in database
GPG key ID: 51CF1ED1E24F7D78
12 changed files with 312 additions and 86 deletions

View file

@ -118,7 +118,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
"prev-export",
Description = "What the exporter should do if the channel had already been exported."
)]
public FileExistsHandling FileExistsHandling { get; init; } = FileExistsHandling.Abort;
public ExportExistsHandling ExportExistsHandling { get; init; } = ExportExistsHandling.Abort;
[Obsolete("This option doesn't do anything. Kept for backwards compatibility.")]
[CommandOption(
@ -230,6 +230,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
// Export
await console.Output.WriteLineAsync($"Exporting {unwrappedChannels.Count} channel(s)...");
var outputDirFilesDict = new ConcurrentDictionary<string, string[]>();
var (progressTicker, logger) = console.CreateProgressTicker();
await progressTicker
.HideCompleted(
@ -270,7 +271,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
ExportFormat,
After,
Before,
FileExistsHandling,
ExportExistsHandling,
PartitionLimit,
MessageFilter,
ShouldFormatMarkdown,
@ -284,6 +285,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
logger,
ParallelLimit > 1,
request,
outputDirFilesDict,
progress.ToPercentageBased(),
innerCancellationToken
);
@ -298,7 +300,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
);
});
logger.PrintExportSummary(FileExistsHandling);
logger.PrintExportSummary(ExportExistsHandling);
// Fail the command only if ALL channels failed to export.
// If only some channels failed to export, it's okay.

View file

@ -128,7 +128,7 @@ public class ConsoleProgressLogger(IAnsiConsole console) : ProgressLogger
/// Prints a summary on all previously logged exports and their respective results to the console.
/// </summary>
/// <param name="updateType">The file exists handling of the export whose summary should be printed.</param>
public void PrintExportSummary(FileExistsHandling updateType)
public void PrintExportSummary(ExportExistsHandling updateType)
{
var exportSummary = GetExportSummary(updateType);
exportSummary.TryGetValue(ExportResult.NewExportSuccess, out var newExportSuccessMessage);

View file

@ -1,5 +1,7 @@
using System;
using System.Collections.Concurrent;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord;
@ -16,6 +18,7 @@ public class ChannelExporter(DiscordClient discord)
ProgressLogger logger,
bool logSuccess,
ExportRequest request,
ConcurrentDictionary<string, string[]> outputDirFilesDict,
IProgress<Percentage>? progress = null,
CancellationToken cancellationToken = default
)
@ -32,62 +35,27 @@ public class ChannelExporter(DiscordClient discord)
return;
}
var currentPartitionIndex = 0;
var exportExists = false;
// TODO: Maybe add a way to search for old files after a username change
if (File.Exists(request.OutputFilePath))
{
exportExists = true;
// TODO: Maybe add an "Ask" option in the future
switch (request.FileExistsHandling)
{
case FileExistsHandling.Abort:
logger.LogError(request, "Aborted export due to existing export files");
return;
case FileExistsHandling.Overwrite:
logger.LogWarning(request, "Removing existing export files");
MessageExporter.RemoveExistingFiles(request.OutputFilePath);
break;
case FileExistsHandling.Append:
var lastMessageSnowflake = MessageExporter.GetLastMessageSnowflake(
request.OutputFilePath,
request.Format
);
if (lastMessageSnowflake != null)
{
if (!request.Channel.MayHaveMessagesAfter(lastMessageSnowflake.Value))
{
logger.IncrementCounter(ExportResult.UpdateExportSkip);
logger.LogInfo(request, "Existing export already up to date");
return;
}
request.LastPriorMessage = lastMessageSnowflake.Value;
logger.LogInfo(
// TODO: Add a way for the user to choose the setting
var searchForExistingExport = true;
if (
!DetectExistingExport(
request,
"Appending existing export starting at "
+ lastMessageSnowflake.Value.ToDate()
);
}
else
{
if (request.Channel.IsEmpty)
{
logger.IncrementCounter(ExportResult.UpdateExportSkip);
logger.LogInfo(request, "Existing empty export already up to date");
logger,
searchForExistingExport,
outputDirFilesDict,
out var existingExportFile
)
)
return;
if (
!HandleExistingExport(
request,
logger,
existingExportFile,
out var currentPartitionIndex
)
)
return;
}
logger.LogInfo(request, "Appending existing empty export.");
}
currentPartitionIndex = MessageExporter.GetPartitionCount(
request.OutputFilePath
);
break;
default:
throw new InvalidOperationException(
$"Unknown FileExistsHandling value '{request.FileExistsHandling}'."
);
}
}
// Build context
var context = new ExportContext(discord, request);
@ -95,7 +63,10 @@ public class ChannelExporter(DiscordClient discord)
// Initialize the exporter before further checks to ensure the file is created even if
// an exception is thrown after this point.
await using var messageExporter = new MessageExporter(context, currentPartitionIndex);
await using var messageExporter = new MessageExporter(
context,
currentPartitionIndex!.Value
);
// Check if the channel is empty
if (request.Channel.IsEmpty)
@ -160,13 +131,13 @@ public class ChannelExporter(DiscordClient discord)
}
}
if (!exportExists)
if (existingExportFile == null)
{
logger.IncrementCounter(ExportResult.NewExportSuccess);
if (logSuccess)
logger.LogSuccess(request, "Successfully exported the channel");
}
else if (request.FileExistsHandling == FileExistsHandling.Append)
else if (request.ExportExistsHandling == ExportExistsHandling.Append)
{
logger.IncrementCounter(ExportResult.UpdateExportSuccess);
if (logSuccess)
@ -179,4 +150,171 @@ public class ChannelExporter(DiscordClient discord)
logger.LogSuccess(request, "Successfully overwrote the channel export");
}
}
/// <summary>
/// Detects whether an existing export of the given request exists.
/// </summary>
/// <param name="request">The request specifying the current channel export.</param>
/// <param name="logger">The logger that's used to log progress updates about the export.</param>
/// <param name="searchForExistingExport">
/// If false, it will only be detected whether an existing export exists at the current target file path.
/// This means that an existing export won't be detected if the name of the channel, the channel parent or the
/// guild has changed or if the default file name formatting has changed.
/// If true, the entire directory will be searched for an existing export file of the given request (if there is
/// none at the current target file path).
/// </param>
/// <param name="outputDirFilesDict">
/// A thread-safe dictionary mapping lists of filenames to the directory they're in.
/// If the directory files are needed, they will be collected lazily and stored for future use in other channel
/// exports.
/// </param>
/// <param name="existingExportFile">
/// The absolute base file path of the existing export of this request, if one has been detected. Null otherwise.
/// </param>
/// <returns>
/// Whether the export should continue normally (true) or return (false).
/// This is true if no, or exactly one, existing export of this request has been detected, and false if
/// several ones have been detected.
/// </returns>
private static bool DetectExistingExport(
ExportRequest request,
ProgressLogger logger,
bool searchForExistingExport,
ConcurrentDictionary<string, string[]> outputDirFilesDict,
out string? existingExportFile
)
{
existingExportFile = null;
if (File.Exists(request.OutputFilePath))
{
existingExportFile = request.OutputFilePath;
return true;
}
if (!searchForExistingExport)
return true;
// Look for an existing export under a different file name
var outputFileRegex = request.GetDefaultOutputFileNameRegex();
var outputDirFiles = outputDirFilesDict.GetOrAdd(
request.OutputDirPath,
outputDirPath =>
Directory
.GetFiles(outputDirPath)
.Select(Path.GetFileName)
.Select(fileName => fileName!)
.ToArray()
);
var regexFiles = outputDirFiles
.Where(fileName => outputFileRegex.IsMatch(fileName))
.ToArray();
if (regexFiles.Length == 0)
return true;
if (regexFiles.Length > 1)
{
logger.LogError(
request,
"Found multiple existing channel exports under different file names: "
+ string.Join(", ", regexFiles)
+ "."
);
return false;
}
logger.LogInfo(
request,
"Found existing channel export under file name " + regexFiles[0] + "."
);
existingExportFile = Path.Combine(request.OutputDirPath, regexFiles[0]);
return true;
}
/// <summary>
/// Handles the existing export files of the current request according to the set file exists handling.
/// </summary>
/// <param name="request">The request specifying the current channel export.</param>
/// <param name="logger">The logger that's used to log progress updates about the export.</param>
/// <param name="existingExportFile">
/// The absolute base file path of the existing export of this request, if one has been detected.
/// If this is null, the function will immediately return.
/// </param>
/// <param name="currentPartitionIndex">
/// The index of the current export partition the newly exported messages should be written to.
/// </param>
/// <returns>
/// Whether the export should continue normally (true) or return (false).
/// This is false both if the export should be aborted and if the export is already up to date, and true otherwise.
/// </returns>
/// <exception cref="InvalidOperationException"></exception>
private static bool HandleExistingExport(
ExportRequest request,
ProgressLogger logger,
string? existingExportFile,
out int? currentPartitionIndex
)
{
currentPartitionIndex = null;
if (existingExportFile == null)
{
currentPartitionIndex = 0;
return true;
}
// TODO: Maybe add an "Ask" option in the future
switch (request.ExportExistsHandling)
{
case ExportExistsHandling.Abort:
logger.LogError(request, "Aborted export due to existing export files");
return false;
case ExportExistsHandling.Overwrite:
logger.LogWarning(request, "Removing existing export files");
MessageExporter.RemoveExistingExport(existingExportFile);
currentPartitionIndex = 0;
return true;
case ExportExistsHandling.Append:
if (existingExportFile != request.OutputFilePath)
{
logger.LogInfo(request, "Moving existing export files to the new file names");
MessageExporter.MoveExistingExport(existingExportFile, request.OutputFilePath);
}
var lastMessageSnowflake = MessageExporter.GetLastMessageSnowflake(
request.OutputFilePath,
request.Format
);
if (lastMessageSnowflake != null)
{
if (!request.Channel.MayHaveMessagesAfter(lastMessageSnowflake.Value))
{
logger.IncrementCounter(ExportResult.UpdateExportSkip);
logger.LogInfo(request, "Existing export already up to date");
return false;
}
request.LastPriorMessage = lastMessageSnowflake.Value;
logger.LogInfo(
request,
"Appending existing export starting at "
+ lastMessageSnowflake.Value.ToDate()
);
}
else
{
if (request.Channel.IsEmpty)
{
logger.IncrementCounter(ExportResult.UpdateExportSkip);
logger.LogInfo(request, "Existing empty export already up to date");
return false;
}
logger.LogInfo(request, "Appending existing empty export.");
}
currentPartitionIndex = MessageExporter.GetPartitionCount(request.OutputFilePath);
return true;
default:
throw new InvalidOperationException(
$"Unknown FileExistsHandling value '{request.ExportExistsHandling}'."
);
}
}
}

View file

@ -3,7 +3,7 @@ namespace DiscordChatExporter.Core.Exporting;
/// <summary>
/// Represents the setting on how to handle the export of a channel that has already been exported.
/// </summary>
public enum FileExistsHandling
public enum ExportExistsHandling
{
/// <summary>
/// If a channel had previously been exported, its export will be aborted.

View file

@ -30,7 +30,7 @@ public partial class ExportRequest
public Snowflake? Before { get; }
public FileExistsHandling FileExistsHandling { get; }
public ExportExistsHandling ExportExistsHandling { get; }
public Snowflake? LastPriorMessage { get; set; }
@ -58,7 +58,7 @@ public partial class ExportRequest
ExportFormat format,
Snowflake? after,
Snowflake? before,
FileExistsHandling fileExistsHandling,
ExportExistsHandling exportExistsHandling,
PartitionLimit partitionLimit,
MessageFilter messageFilter,
bool shouldFormatMarkdown,
@ -73,7 +73,7 @@ public partial class ExportRequest
Format = format;
After = after;
Before = before;
FileExistsHandling = fileExistsHandling;
ExportExistsHandling = exportExistsHandling;
PartitionLimit = partitionLimit;
MessageFilter = messageFilter;
ShouldFormatMarkdown = shouldFormatMarkdown;
@ -104,6 +104,7 @@ public partial class ExportRequest
Snowflake? before = null
)
{
// Do not change this without adding the new version to the corresponding regex below
var buffer = new StringBuilder();
// Guild name
@ -154,6 +155,63 @@ public partial class ExportRequest
return PathEx.EscapeFileName(buffer.ToString());
}
/// <summary>
/// Returns a regex that matches any default file name this channel export might have had in the past.
/// This can be used to detect existing exports of this channel with a different guild, parent and / or channel
/// name.
/// This only matches existing exports with the same date range as the current export.
/// </summary>
/// <returns>A regex that matches any default file name this channel might have had in the past.</returns>
public Regex GetDefaultOutputFileNameRegex()
{
// While this code looks similar to GetDefaultOutputFileName, the two functions are intentionally independent
// Even if the default output file name gets changed, the previous default file names should still be matched
// by this; the new version should just be added additionally to this regex
var buffer = new StringBuilder();
// Guild name
buffer.Append(".*?");
// Parent name
if (Channel.Parent is not null)
buffer.Append(" - ").Append(".*?");
// Channel name and ID
buffer
.Append(" - ")
.Append(".*?")
.Append(' ')
.Append("\\[")
.Append(Channel.Id)
.Append("\\]");
// Date range
if (After is not null || Before is not null)
{
buffer.Append(' ').Append("\\(");
if (After is not null && Before is not null)
{
buffer.Append(
$"{After.Value.ToDate():yyyy-MM-dd} to {Before.Value.ToDate():yyyy-MM-dd}"
);
}
else if (After is not null)
{
buffer.Append($"after {After.Value.ToDate():yyyy-MM-dd}");
}
else if (Before is not null)
{
buffer.Append($"before {Before.Value.ToDate():yyyy-MM-dd}");
}
buffer.Append("\\)");
}
// File extension
buffer.Append("\\.").Append(Format.GetFileExtension());
return new Regex(buffer.ToString());
}
private static string FormatPath(
string path,
Guild guild,

View file

@ -27,7 +27,7 @@ public abstract class ProgressLogger
/// </summary>
/// <param name="updateType">The file exists handling of the export whose summary should be returned.</param>
/// <returns>A summary on all previously logged exports and their respective results.</returns>
protected Dictionary<ExportResult, string> GetExportSummary(FileExistsHandling updateType)
protected Dictionary<ExportResult, string> GetExportSummary(ExportExistsHandling updateType)
{
_counters.TryGetValue(ExportResult.NewExportSuccess, out var newExportSuccessCount);
_counters.TryGetValue(
@ -48,7 +48,7 @@ public abstract class ProgressLogger
if (updateExportSuccessCount > 0)
exportSummary[ExportResult.UpdateExportSuccess] =
"Successfully "
+ (updateType == FileExistsHandling.Append ? "appended" : "overrode")
+ (updateType == ExportExistsHandling.Append ? "appended" : "overrode")
+ $" {updateExportSuccessCount} existing channel export(s).";
if (updateExportSkipCount > 0)
exportSummary[ExportResult.UpdateExportSkip] =

View file

@ -109,7 +109,7 @@ internal partial class MessageExporter
if (File.Exists(filePath))
{
throw new InvalidOperationException(
"Error: An exported file already exists. This should never happen."
"Error: The target export file already exists. This should never happen."
);
}
@ -184,10 +184,9 @@ internal partial class MessageExporter
/// <param name="baseFilePath">
/// The path of the first partition of the Discord channel export that should be removed.
/// </param>
public static void RemoveExistingFiles(string baseFilePath)
public static void RemoveExistingExport(string baseFilePath)
{
var currentPartition = 0;
while (true)
for (var currentPartition = 0; ; currentPartition++)
{
var currentFilePath = GetPartitionFilePath(baseFilePath, currentPartition);
if (File.Exists(currentFilePath))
@ -198,7 +197,33 @@ internal partial class MessageExporter
{
return;
}
currentPartition++;
}
}
/// <summary>
/// Moves all partitions of the previously exported Discord channel with the given old base file path to the given
/// new base file path.
/// </summary>
/// <param name="oldBaseFilePath">
/// The old path of the first partition of the Discord channel export that should be moved.
/// </param>
/// <param name="newBaseFilePath">
/// The new path to which the first partition of the Discord channel export should be moved.
/// </param>
public static void MoveExistingExport(string oldBaseFilePath, string newBaseFilePath)
{
for (var currentPartition = 0; ; currentPartition++)
{
var currentOldFilePath = GetPartitionFilePath(oldBaseFilePath, currentPartition);
if (File.Exists(currentOldFilePath))
{
var currentNewFilePath = GetPartitionFilePath(newBaseFilePath, currentPartition);
File.Move(currentOldFilePath, currentNewFilePath);
}
else
{
return;
}
}
}

View file

@ -104,7 +104,7 @@ public class SnackbarProgressLogger(SnackbarManager snackbarManager) : ProgressL
/// Prints a summary on all previously logged exports and their respective results in the GUI snackbar.
/// </summary>
/// <param name="updateType">The file exists handling of the export whose summary should be printed.</param>
public void PrintExportSummary(FileExistsHandling updateType)
public void PrintExportSummary(ExportExistsHandling updateType)
{
var exportSummary = GetExportSummary(updateType);
exportSummary.TryGetValue(ExportResult.NewExportSuccess, out var newExportSuccessMessage);

View file

@ -37,7 +37,7 @@ public partial class SettingsService()
public partial ThreadInclusionMode ThreadInclusionMode { get; set; }
[ObservableProperty]
public partial FileExistsHandling FileExistsHandling { get; set; }
public partial ExportExistsHandling ExportExistsHandling { get; set; }
[ObservableProperty]
public partial string? Locale { get; set; }

View file

@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
@ -243,6 +244,7 @@ public partial class DashboardViewModel : ViewModelBase
var exporter = new ChannelExporter(_discord);
var logger = new SnackbarProgressLogger(_snackbarManager);
var outputDirFilesDict = new ConcurrentDictionary<string, string[]>();
var channelProgressPairs = dialog
.Channels!.Select(c => new { Channel = c, Progress = _progressMuxer.CreateInput() })
@ -269,7 +271,7 @@ public partial class DashboardViewModel : ViewModelBase
dialog.SelectedFormat,
dialog.After?.Pipe(timestamp => Snowflake.FromDate(timestamp, true)),
dialog.Before?.Pipe(timestamp => Snowflake.FromDate(timestamp)),
_settingsService.FileExistsHandling,
_settingsService.ExportExistsHandling,
dialog.PartitionLimit,
dialog.MessageFilter,
dialog.ShouldFormatMarkdown,
@ -283,6 +285,7 @@ public partial class DashboardViewModel : ViewModelBase
logger,
true,
request,
outputDirFilesDict,
progress,
cancellationToken
);
@ -298,7 +301,7 @@ public partial class DashboardViewModel : ViewModelBase
}
);
logger.PrintExportSummary(_settingsService.FileExistsHandling);
logger.PrintExportSummary(_settingsService.ExportExistsHandling);
}
catch (Exception ex)
{

View file

@ -62,13 +62,13 @@ public class SettingsViewModel : DialogViewModelBase
set => _settingsService.ThreadInclusionMode = value;
}
public IReadOnlyList<FileExistsHandling> AvailableFileExistHandlingOptions { get; } =
Enum.GetValues<FileExistsHandling>();
public IReadOnlyList<ExportExistsHandling> AvailableExportExistsHandlingOptions { get; } =
Enum.GetValues<ExportExistsHandling>();
public FileExistsHandling FileExistsHandling
public ExportExistsHandling ExportExistsHandling
{
get => _settingsService.FileExistsHandling;
set => _settingsService.FileExistsHandling = value;
get => _settingsService.ExportExistsHandling;
set => _settingsService.ExportExistsHandling = value;
}
// These items have to be non-nullable because Avalonia ComboBox doesn't allow a null value to be selected

View file

@ -132,7 +132,7 @@
</StackPanel>
</DockPanel>
<!-- File Exists Handling -->
<!-- Export Exists Handling -->
<DockPanel
Margin="16,8"
LastChildFill="False"
@ -141,8 +141,8 @@
<ComboBox
Width="150"
DockPanel.Dock="Right"
ItemsSource="{Binding AvailableFileExistHandlingOptions}"
SelectedItem="{Binding FileExistsHandling}" />
ItemsSource="{Binding AvailableExportExistsHandlingOptions}"
SelectedItem="{Binding ExportExistsHandling}" />
</DockPanel>
</StackPanel>
</ScrollViewer>