diff --git a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs index 09c85212..d4ff651b 100644 --- a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs @@ -31,8 +31,9 @@ namespace DiscordChatExporter.Cli.Commands.Base [CommandOption("before", Description = "Only include messages sent before this date or message ID.")] public Snowflake? Before { get; init; } - [CommandOption("partition", 'p', Description = "Split output into partitions limited to this number of messages.")] - public int? PartitionLimit { get; init; } + [CommandOption("partition", 'p', Converter = typeof(PartitionConverter), + Description = "Split output into partitions limited to this number of messages or a maximum file size (e.g. \"25mb\").")] + public IPartitioner Partitoner { get; init; } = new NullPartitioner(); [CommandOption("parallel", Description = "Limits how many channels can be exported in parallel.")] public int ParallelLimit { get; init; } = 1; @@ -74,7 +75,7 @@ namespace DiscordChatExporter.Cli.Commands.Base ExportFormat, After, Before, - PartitionLimit, + Partitoner, ShouldDownloadMedia, ShouldReuseMedia, DateFormat diff --git a/DiscordChatExporter.Cli/Commands/Base/PartitionConverter.cs b/DiscordChatExporter.Cli/Commands/Base/PartitionConverter.cs new file mode 100644 index 00000000..bac10078 --- /dev/null +++ b/DiscordChatExporter.Cli/Commands/Base/PartitionConverter.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Text; +using ByteSizeLib; +using CliFx.Extensibility; +using DiscordChatExporter.Core; +using DiscordChatExporter.Core.Exporting; + +namespace DiscordChatExporter.Cli.Commands.Base +{ + public class PartitionConverter : BindingConverter + { + public override IPartitioner Convert(string? rawValue) + { + if (rawValue == null) return new NullPartitioner(); + + if (ByteSize.TryParse(rawValue, out ByteSize filesize)) + { + return new FileSizePartitioner((long)filesize.Bytes); + } + else + { + int messageLimit = int.Parse(rawValue); + return new MessageCountPartitioner(messageLimit); + } + } + } +} diff --git a/DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj b/DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj index 01332f16..0f79b9bf 100644 --- a/DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj +++ b/DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj @@ -8,6 +8,8 @@ + + diff --git a/DiscordChatExporter.Core/Discord/Data/Attachment.cs b/DiscordChatExporter.Core/Discord/Data/Attachment.cs index 799b3446..9a445a21 100644 --- a/DiscordChatExporter.Core/Discord/Data/Attachment.cs +++ b/DiscordChatExporter.Core/Discord/Data/Attachment.cs @@ -5,6 +5,7 @@ using System.Text.Json; using DiscordChatExporter.Core.Discord.Data.Common; using DiscordChatExporter.Core.Utils.Extensions; using JsonExtensions.Reading; +using FileSize = DiscordChatExporter.Core.Discord.Data.Common.FileSize; namespace DiscordChatExporter.Core.Discord.Data { diff --git a/DiscordChatExporter.Core/Discord/Data/Common/FileSize.cs b/DiscordChatExporter.Core/Discord/Data/Common/FileSize.cs index c7754247..5141bb9b 100644 --- a/DiscordChatExporter.Core/Discord/Data/Common/FileSize.cs +++ b/DiscordChatExporter.Core/Discord/Data/Common/FileSize.cs @@ -62,4 +62,4 @@ namespace DiscordChatExporter.Core.Discord.Data.Common { public static FileSize FromBytes(long bytes) => new(bytes); } -} \ No newline at end of file +} diff --git a/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj b/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj index b3b8fc85..acda34c2 100644 --- a/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj +++ b/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj @@ -5,6 +5,7 @@ + diff --git a/DiscordChatExporter.Core/Exporting/ExportRequest.cs b/DiscordChatExporter.Core/Exporting/ExportRequest.cs index c2c5a9e8..d71aa5f1 100644 --- a/DiscordChatExporter.Core/Exporting/ExportRequest.cs +++ b/DiscordChatExporter.Core/Exporting/ExportRequest.cs @@ -28,7 +28,7 @@ namespace DiscordChatExporter.Core.Exporting public Snowflake? Before { get; } - public int? PartitionLimit { get; } + public IPartitioner Partitoner { get; } public bool ShouldDownloadMedia { get; } @@ -43,7 +43,7 @@ namespace DiscordChatExporter.Core.Exporting ExportFormat format, Snowflake? after, Snowflake? before, - int? partitionLimit, + IPartitioner partitioner, bool shouldDownloadMedia, bool shouldReuseMedia, string dateFormat) @@ -54,7 +54,7 @@ namespace DiscordChatExporter.Core.Exporting Format = format; After = after; Before = before; - PartitionLimit = partitionLimit; + Partitoner = partitioner; ShouldDownloadMedia = shouldDownloadMedia; ShouldReuseMedia = shouldReuseMedia; DateFormat = dateFormat; diff --git a/DiscordChatExporter.Core/Exporting/MessageExporter.cs b/DiscordChatExporter.Core/Exporting/MessageExporter.cs index d5b76241..66cd503d 100644 --- a/DiscordChatExporter.Core/Exporting/MessageExporter.cs +++ b/DiscordChatExporter.Core/Exporting/MessageExporter.cs @@ -1,13 +1,18 @@ using System; using System.IO; +using System.Text.RegularExpressions; using System.Threading.Tasks; +using ByteSizeLib; using DiscordChatExporter.Core.Discord.Data; +using DiscordChatExporter.Core.Exporting; +using DiscordChatExporter.Core.Exporting.Partitioners; using DiscordChatExporter.Core.Exporting.Writers; namespace DiscordChatExporter.Core.Exporting { internal partial class MessageExporter : IAsyncDisposable { + private readonly ExportContext _context; private long _messageCount; @@ -19,11 +24,16 @@ namespace DiscordChatExporter.Core.Exporting _context = context; } - private bool IsPartitionLimitReached() => - _messageCount > 0 && - _context.Request.PartitionLimit is not null && - _context.Request.PartitionLimit != 0 && - _messageCount % _context.Request.PartitionLimit == 0; + private bool IsPartitionLimitReached() + { + if (_writer is null) + { + return false; + } + + return _context.Request.Partitoner.IsLimitReached( + new ExportPartitioningContext(_messageCount, _writer.SizeInBytes)); + } private async ValueTask ResetWriterAsync() { @@ -38,7 +48,7 @@ namespace DiscordChatExporter.Core.Exporting private async ValueTask GetWriterAsync() { // Ensure partition limit has not been exceeded - if (IsPartitionLimitReached()) + if (_writer != null && IsPartitionLimitReached()) { await ResetWriterAsync(); _partitionIndex++; diff --git a/DiscordChatExporter.Core/Exporting/Partitioners/ExportPartitioningContext.cs b/DiscordChatExporter.Core/Exporting/Partitioners/ExportPartitioningContext.cs new file mode 100644 index 00000000..5deb81e4 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Partitioners/ExportPartitioningContext.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace DiscordChatExporter.Core.Exporting.Partitioners +{ + public class ExportPartitioningContext + { + public long MessageCount { get; } + public long SizeInBytes { get; } + + public ExportPartitioningContext(long messageCount, long sizeInBytes) + { + MessageCount = messageCount; + SizeInBytes = sizeInBytes; + } + } +} diff --git a/DiscordChatExporter.Core/Exporting/Partitioners/FileSizePartitioner.cs b/DiscordChatExporter.Core/Exporting/Partitioners/FileSizePartitioner.cs new file mode 100644 index 00000000..b1b8f174 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Partitioners/FileSizePartitioner.cs @@ -0,0 +1,21 @@ +using DiscordChatExporter.Core.Exporting.Partitioners; +using System; +using System.Collections.Generic; +using System.Text; + +namespace DiscordChatExporter.Core.Exporting +{ + public class FileSizePartitioner : IPartitioner + { + private long _bytesPerFile; + + public FileSizePartitioner(long bytesPerFile) + { + _bytesPerFile = bytesPerFile; + } + public bool IsLimitReached(ExportPartitioningContext context) + { + return context.SizeInBytes >= _bytesPerFile; + } + } +} diff --git a/DiscordChatExporter.Core/Exporting/Partitioners/IPartitioner.cs b/DiscordChatExporter.Core/Exporting/Partitioners/IPartitioner.cs new file mode 100644 index 00000000..47bc5e21 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Partitioners/IPartitioner.cs @@ -0,0 +1,12 @@ +using DiscordChatExporter.Core.Exporting.Partitioners; +using System; +using System.Collections.Generic; +using System.Text; + +namespace DiscordChatExporter.Core.Exporting +{ + public interface IPartitioner + { + bool IsLimitReached(ExportPartitioningContext context); + } +} diff --git a/DiscordChatExporter.Core/Exporting/Partitioners/MessageCountPartitioner.cs b/DiscordChatExporter.Core/Exporting/Partitioners/MessageCountPartitioner.cs new file mode 100644 index 00000000..53e0737c --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Partitioners/MessageCountPartitioner.cs @@ -0,0 +1,25 @@ +using DiscordChatExporter.Core.Exporting.Partitioners; +using System; +using System.Collections.Generic; +using System.Text; + +namespace DiscordChatExporter.Core.Exporting +{ + public class MessageCountPartitioner : IPartitioner + { + + private int _messagesPerPartition; + + public MessageCountPartitioner(int messagesPerPartition) + { + _messagesPerPartition = messagesPerPartition; + } + + public bool IsLimitReached(ExportPartitioningContext context) + { + return context.MessageCount > 0 && + _messagesPerPartition != 0 && + context.MessageCount % _messagesPerPartition == 0; + } + } +} diff --git a/DiscordChatExporter.Core/Exporting/Partitioners/NullPartitioner.cs b/DiscordChatExporter.Core/Exporting/Partitioners/NullPartitioner.cs new file mode 100644 index 00000000..dc3de638 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Partitioners/NullPartitioner.cs @@ -0,0 +1,16 @@ +using DiscordChatExporter.Core.Exporting; +using DiscordChatExporter.Core.Exporting.Partitioners; +using System; +using System.Collections.Generic; +using System.Text; + +namespace DiscordChatExporter.Core.Exporting +{ + public class NullPartitioner : IPartitioner + { + public bool IsLimitReached(ExportPartitioningContext context) + { + return false; + } + } +} diff --git a/DiscordChatExporter.Core/Exporting/Writers/MessageWriter.cs b/DiscordChatExporter.Core/Exporting/Writers/MessageWriter.cs index d0ab0bf6..f8ca7adc 100644 --- a/DiscordChatExporter.Core/Exporting/Writers/MessageWriter.cs +++ b/DiscordChatExporter.Core/Exporting/Writers/MessageWriter.cs @@ -17,6 +17,8 @@ namespace DiscordChatExporter.Core.Exporting.Writers Context = context; } + public long SizeInBytes => Stream.Length; + public virtual ValueTask WritePreambleAsync() => default; public abstract ValueTask WriteMessageAsync(Message message); diff --git a/DiscordChatExporter.Gui/Converters/PartitionFormatToStringConverter.cs b/DiscordChatExporter.Gui/Converters/PartitionFormatToStringConverter.cs new file mode 100644 index 00000000..8fafab53 --- /dev/null +++ b/DiscordChatExporter.Gui/Converters/PartitionFormatToStringConverter.cs @@ -0,0 +1,27 @@ +using DiscordChatExporter.Core.Exporting; +using DiscordChatExporter.Gui.Internal; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Windows.Data; + +namespace DiscordChatExporter.Gui.Converters +{ + [ValueConversion(typeof(ExportFormat), typeof(string))] + public class PartitionFormatToStringConverter : IValueConverter + { + public static PartitionFormatToStringConverter Instance { get; } = new(); + + public object? Convert(object value, Type targetType, object parameter, CultureInfo culture) + { + if (value is PartitionFormat partitionFormatValue) + return partitionFormatValue.GetDisplayName(); + + return default(string); + } + + public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture) => + throw new NotSupportedException(); + } +} diff --git a/DiscordChatExporter.Gui/Converters/PartitionFormatToTextBoxHintConverter.cs b/DiscordChatExporter.Gui/Converters/PartitionFormatToTextBoxHintConverter.cs new file mode 100644 index 00000000..86ec2813 --- /dev/null +++ b/DiscordChatExporter.Gui/Converters/PartitionFormatToTextBoxHintConverter.cs @@ -0,0 +1,33 @@ +using DiscordChatExporter.Gui.Internal; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Windows.Data; + +namespace DiscordChatExporter.Gui.Converters +{ + [ValueConversion(typeof(DateTimeOffset?), typeof(DateTime?))] + public class PartitionFormatToTextBoxHintConverter : IValueConverter + { + public static PartitionFormatToTextBoxHintConverter Instance { get; } = new(); + + public object? Convert(object value, Type targetType, object parameter, CultureInfo culture) + { + if (value is PartitionFormat partitionFormat) + return partitionFormat switch + { + PartitionFormat.FileSize => "MB per partition", + PartitionFormat.MessageCount => "Messages per partition", + _ => default(string) + }; + + return default(DateTime?); + } + + public object? ConvertBack(object value, Type targetType, object parameter, CultureInfo culture) + { + throw new NotSupportedException(); + } + } +} diff --git a/DiscordChatExporter.Gui/Converters/PartitionFormatToTooltipConverter.cs b/DiscordChatExporter.Gui/Converters/PartitionFormatToTooltipConverter.cs new file mode 100644 index 00000000..a4b96aaf --- /dev/null +++ b/DiscordChatExporter.Gui/Converters/PartitionFormatToTooltipConverter.cs @@ -0,0 +1,33 @@ +using DiscordChatExporter.Gui.Internal; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Windows.Data; + +namespace DiscordChatExporter.Gui.Converters +{ + [ValueConversion(typeof(DateTimeOffset?), typeof(DateTime?))] + public class PartitionFormatToTooltipConverter : IValueConverter + { + public static PartitionFormatToTextBoxHintConverter Instance { get; } = new(); + + public object? Convert(object value, Type targetType, object parameter, CultureInfo culture) + { + if (value is PartitionFormat partitionFormat) + return partitionFormat switch + { + PartitionFormat.FileSize => "Split output into partitions close to this file size", + PartitionFormat.MessageCount => "Split output into partitions limited to this number of messages", + _ => default(string) + }; + + return default(DateTime?); + } + + public object? ConvertBack(object value, Type targetType, object parameter, CultureInfo culture) + { + throw new NotSupportedException(); + } + } +} diff --git a/DiscordChatExporter.Gui/Services/SettingsService.cs b/DiscordChatExporter.Gui/Services/SettingsService.cs index 1104f1dc..9bfa709a 100644 --- a/DiscordChatExporter.Gui/Services/SettingsService.cs +++ b/DiscordChatExporter.Gui/Services/SettingsService.cs @@ -1,5 +1,6 @@ using DiscordChatExporter.Core.Discord; using DiscordChatExporter.Core.Exporting; +using DiscordChatExporter.Gui.Internal; using Tyrrrz.Settings; namespace DiscordChatExporter.Gui.Services @@ -22,6 +23,8 @@ namespace DiscordChatExporter.Gui.Services public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark; + public PartitionFormat LastPartitionFormat { get; set; } = PartitionFormat.MessageCount; + public int? LastPartitionLimit { get; set; } public bool LastShouldDownloadMedia { get; set; } diff --git a/DiscordChatExporter.Gui/Utils/PartitionFormat.cs b/DiscordChatExporter.Gui/Utils/PartitionFormat.cs new file mode 100644 index 00000000..ae363956 --- /dev/null +++ b/DiscordChatExporter.Gui/Utils/PartitionFormat.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace DiscordChatExporter.Gui.Internal +{ + public enum PartitionFormat + { + MessageCount, + FileSize, + } + + public static class PartitionFormatExtensions + { + public static string GetDisplayName(this PartitionFormat format) => format switch + { + PartitionFormat.MessageCount => "Message count", + PartitionFormat.FileSize => "File size (MB)", + _ => throw new ArgumentOutOfRangeException(nameof(format)) + }; + } +} diff --git a/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs b/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs index afe16e83..9785b260 100644 --- a/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using DiscordChatExporter.Gui.Internal; using DiscordChatExporter.Core.Discord; using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Exporting; @@ -46,6 +47,11 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs public DateTimeOffset? Before => BeforeDate?.Add(BeforeTime ?? TimeSpan.Zero); + public IReadOnlyList AvailablePartitionFormats => + Enum.GetValues(typeof(PartitionFormat)).Cast().ToArray(); + + public PartitionFormat SelectedPartitionFormat { get; set; } + public int? PartitionLimit { get; set; } public bool ShouldDownloadMedia { get; set; } @@ -67,6 +73,8 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs SelectedFormat = _settingsService.LastExportFormat; PartitionLimit = _settingsService.LastPartitionLimit; ShouldDownloadMedia = _settingsService.LastShouldDownloadMedia; + SelectedPartitionFormat = _settingsService.LastPartitionFormat; + } public void Confirm() @@ -74,6 +82,7 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs // Persist preferences _settingsService.LastExportFormat = SelectedFormat; _settingsService.LastPartitionLimit = PartitionLimit; + _settingsService.LastPartitionFormat = SelectedPartitionFormat; _settingsService.LastShouldDownloadMedia = ShouldDownloadMedia; // If single channel - prompt file path diff --git a/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs b/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs index d4f4e86f..d839f5cc 100644 --- a/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs @@ -8,6 +8,7 @@ using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Exceptions; using DiscordChatExporter.Core.Exporting; using DiscordChatExporter.Core.Utils.Extensions; +using DiscordChatExporter.Gui.Internal; using DiscordChatExporter.Gui.Services; using DiscordChatExporter.Gui.Utils; using DiscordChatExporter.Gui.ViewModels.Dialogs; @@ -213,7 +214,7 @@ namespace DiscordChatExporter.Gui.ViewModels dialog.SelectedFormat, dialog.After?.Pipe(Snowflake.FromDate), dialog.Before?.Pipe(Snowflake.FromDate), - dialog.PartitionLimit, + CreatePartitioner(), dialog.ShouldDownloadMedia, _settingsService.ShouldReuseMedia, _settingsService.DateFormat @@ -236,6 +237,19 @@ namespace DiscordChatExporter.Gui.ViewModels // Notify of overall completion if (successfulExportCount > 0) Notifications.Enqueue($"Successfully exported {successfulExportCount} channel(s)"); + + IPartitioner CreatePartitioner() + { + var partitionFormat = dialog.SelectedPartitionFormat; + var partitionLimit = dialog.PartitionLimit; + + return (partitionFormat, partitionLimit) switch + { + (PartitionFormat.MessageCount, int messageLimit) => new MessageCountPartitioner(messageLimit), + (PartitionFormat.FileSize, int fileSizeLimit) => new FileSizePartitioner(fileSizeLimit), + _ => new NullPartitioner() + }; + } } } } \ No newline at end of file diff --git a/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml b/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml index 91bd67bd..34d371d3 100644 --- a/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml +++ b/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml @@ -126,12 +126,43 @@ - + + + + + + + + + + + + + + + + + + + + diff --git a/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml.cs b/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml.cs index c2011898..bd0022f7 100644 --- a/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml.cs +++ b/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml.cs @@ -1,4 +1,4 @@ -namespace DiscordChatExporter.Gui.Views.Dialogs +namespace DiscordChatExporter.Gui.Views.Dialogs { public partial class ExportSetupView { @@ -7,4 +7,4 @@ InitializeComponent(); } } -} \ No newline at end of file +}