mirror of
https://github.com/Tyrrrz/DiscordChatExporter.git
synced 2024-09-19 12:18:48 -04:00
Add partition by file size (#497)
This commit is contained in:
parent
ad3655396f
commit
eb89ea5b40
23 changed files with 331 additions and 22 deletions
|
@ -31,8 +31,9 @@ namespace DiscordChatExporter.Cli.Commands.Base
|
|||
[CommandOption("before", Description = "Only include messages sent before this date or message ID.")]
|
||||
public Snowflake? Before { get; init; }
|
||||
|
||||
[CommandOption("partition", 'p', Description = "Split output into partitions limited to this number of messages.")]
|
||||
public int? PartitionLimit { get; init; }
|
||||
[CommandOption("partition", 'p', Converter = typeof(PartitionConverter),
|
||||
Description = "Split output into partitions limited to this number of messages or a maximum file size (e.g. \"25mb\").")]
|
||||
public IPartitioner Partitoner { get; init; } = new NullPartitioner();
|
||||
|
||||
[CommandOption("parallel", Description = "Limits how many channels can be exported in parallel.")]
|
||||
public int ParallelLimit { get; init; } = 1;
|
||||
|
@ -74,7 +75,7 @@ namespace DiscordChatExporter.Cli.Commands.Base
|
|||
ExportFormat,
|
||||
After,
|
||||
Before,
|
||||
PartitionLimit,
|
||||
Partitoner,
|
||||
ShouldDownloadMedia,
|
||||
ShouldReuseMedia,
|
||||
DateFormat
|
||||
|
|
28
DiscordChatExporter.Cli/Commands/Base/PartitionConverter.cs
Normal file
28
DiscordChatExporter.Cli/Commands/Base/PartitionConverter.cs
Normal file
|
@ -0,0 +1,28 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using ByteSizeLib;
|
||||
using CliFx.Extensibility;
|
||||
using DiscordChatExporter.Core;
|
||||
using DiscordChatExporter.Core.Exporting;
|
||||
|
||||
namespace DiscordChatExporter.Cli.Commands.Base
|
||||
{
|
||||
public class PartitionConverter : BindingConverter<IPartitioner>
|
||||
{
|
||||
public override IPartitioner Convert(string? rawValue)
|
||||
{
|
||||
if (rawValue == null) return new NullPartitioner();
|
||||
|
||||
if (ByteSize.TryParse(rawValue, out ByteSize filesize))
|
||||
{
|
||||
return new FileSizePartitioner((long)filesize.Bytes);
|
||||
}
|
||||
else
|
||||
{
|
||||
int messageLimit = int.Parse(rawValue);
|
||||
return new MessageCountPartitioner(messageLimit);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -8,6 +8,8 @@
|
|||
<ItemGroup>
|
||||
<PackageReference Include="CliFx" Version="2.0.1" />
|
||||
<PackageReference Include="Spectre.Console" Version="0.38.0" />
|
||||
<PackageReference Include="Gress" Version="1.2.0" />
|
||||
<PackageReference Include="OneOf" Version="3.0.174" />
|
||||
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
@ -5,6 +5,7 @@ using System.Text.Json;
|
|||
using DiscordChatExporter.Core.Discord.Data.Common;
|
||||
using DiscordChatExporter.Core.Utils.Extensions;
|
||||
using JsonExtensions.Reading;
|
||||
using FileSize = DiscordChatExporter.Core.Discord.Data.Common.FileSize;
|
||||
|
||||
namespace DiscordChatExporter.Core.Discord.Data
|
||||
{
|
||||
|
|
|
@ -5,6 +5,7 @@
|
|||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="ByteSize" Version="2.0.0" />
|
||||
<PackageReference Include="JsonExtensions" Version="1.0.1" />
|
||||
<PackageReference Include="MiniRazor.CodeGen" Version="2.1.2" />
|
||||
<PackageReference Include="Polly" Version="7.2.1" />
|
||||
|
|
|
@ -28,7 +28,7 @@ namespace DiscordChatExporter.Core.Exporting
|
|||
|
||||
public Snowflake? Before { get; }
|
||||
|
||||
public int? PartitionLimit { get; }
|
||||
public IPartitioner Partitoner { get; }
|
||||
|
||||
public bool ShouldDownloadMedia { get; }
|
||||
|
||||
|
@ -43,7 +43,7 @@ namespace DiscordChatExporter.Core.Exporting
|
|||
ExportFormat format,
|
||||
Snowflake? after,
|
||||
Snowflake? before,
|
||||
int? partitionLimit,
|
||||
IPartitioner partitioner,
|
||||
bool shouldDownloadMedia,
|
||||
bool shouldReuseMedia,
|
||||
string dateFormat)
|
||||
|
@ -54,7 +54,7 @@ namespace DiscordChatExporter.Core.Exporting
|
|||
Format = format;
|
||||
After = after;
|
||||
Before = before;
|
||||
PartitionLimit = partitionLimit;
|
||||
Partitoner = partitioner;
|
||||
ShouldDownloadMedia = shouldDownloadMedia;
|
||||
ShouldReuseMedia = shouldReuseMedia;
|
||||
DateFormat = dateFormat;
|
||||
|
|
|
@ -1,13 +1,18 @@
|
|||
using System;
|
||||
using System.IO;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
using ByteSizeLib;
|
||||
using DiscordChatExporter.Core.Discord.Data;
|
||||
using DiscordChatExporter.Core.Exporting;
|
||||
using DiscordChatExporter.Core.Exporting.Partitioners;
|
||||
using DiscordChatExporter.Core.Exporting.Writers;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting
|
||||
{
|
||||
internal partial class MessageExporter : IAsyncDisposable
|
||||
{
|
||||
|
||||
private readonly ExportContext _context;
|
||||
|
||||
private long _messageCount;
|
||||
|
@ -19,11 +24,16 @@ namespace DiscordChatExporter.Core.Exporting
|
|||
_context = context;
|
||||
}
|
||||
|
||||
private bool IsPartitionLimitReached() =>
|
||||
_messageCount > 0 &&
|
||||
_context.Request.PartitionLimit is not null &&
|
||||
_context.Request.PartitionLimit != 0 &&
|
||||
_messageCount % _context.Request.PartitionLimit == 0;
|
||||
private bool IsPartitionLimitReached()
|
||||
{
|
||||
if (_writer is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return _context.Request.Partitoner.IsLimitReached(
|
||||
new ExportPartitioningContext(_messageCount, _writer.SizeInBytes));
|
||||
}
|
||||
|
||||
private async ValueTask ResetWriterAsync()
|
||||
{
|
||||
|
@ -38,7 +48,7 @@ namespace DiscordChatExporter.Core.Exporting
|
|||
private async ValueTask<MessageWriter> GetWriterAsync()
|
||||
{
|
||||
// Ensure partition limit has not been exceeded
|
||||
if (IsPartitionLimitReached())
|
||||
if (_writer != null && IsPartitionLimitReached())
|
||||
{
|
||||
await ResetWriterAsync();
|
||||
_partitionIndex++;
|
||||
|
|
|
@ -0,0 +1,18 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting.Partitioners
|
||||
{
|
||||
public class ExportPartitioningContext
|
||||
{
|
||||
public long MessageCount { get; }
|
||||
public long SizeInBytes { get; }
|
||||
|
||||
public ExportPartitioningContext(long messageCount, long sizeInBytes)
|
||||
{
|
||||
MessageCount = messageCount;
|
||||
SizeInBytes = sizeInBytes;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,21 @@
|
|||
using DiscordChatExporter.Core.Exporting.Partitioners;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting
|
||||
{
|
||||
public class FileSizePartitioner : IPartitioner
|
||||
{
|
||||
private long _bytesPerFile;
|
||||
|
||||
public FileSizePartitioner(long bytesPerFile)
|
||||
{
|
||||
_bytesPerFile = bytesPerFile;
|
||||
}
|
||||
public bool IsLimitReached(ExportPartitioningContext context)
|
||||
{
|
||||
return context.SizeInBytes >= _bytesPerFile;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,12 @@
|
|||
using DiscordChatExporter.Core.Exporting.Partitioners;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting
|
||||
{
|
||||
public interface IPartitioner
|
||||
{
|
||||
bool IsLimitReached(ExportPartitioningContext context);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,25 @@
|
|||
using DiscordChatExporter.Core.Exporting.Partitioners;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting
|
||||
{
|
||||
public class MessageCountPartitioner : IPartitioner
|
||||
{
|
||||
|
||||
private int _messagesPerPartition;
|
||||
|
||||
public MessageCountPartitioner(int messagesPerPartition)
|
||||
{
|
||||
_messagesPerPartition = messagesPerPartition;
|
||||
}
|
||||
|
||||
public bool IsLimitReached(ExportPartitioningContext context)
|
||||
{
|
||||
return context.MessageCount > 0 &&
|
||||
_messagesPerPartition != 0 &&
|
||||
context.MessageCount % _messagesPerPartition == 0;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,16 @@
|
|||
using DiscordChatExporter.Core.Exporting;
|
||||
using DiscordChatExporter.Core.Exporting.Partitioners;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting
|
||||
{
|
||||
public class NullPartitioner : IPartitioner
|
||||
{
|
||||
public bool IsLimitReached(ExportPartitioningContext context)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -17,6 +17,8 @@ namespace DiscordChatExporter.Core.Exporting.Writers
|
|||
Context = context;
|
||||
}
|
||||
|
||||
public long SizeInBytes => Stream.Length;
|
||||
|
||||
public virtual ValueTask WritePreambleAsync() => default;
|
||||
|
||||
public abstract ValueTask WriteMessageAsync(Message message);
|
||||
|
|
|
@ -0,0 +1,27 @@
|
|||
using DiscordChatExporter.Core.Exporting;
|
||||
using DiscordChatExporter.Gui.Internal;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using System.Windows.Data;
|
||||
|
||||
namespace DiscordChatExporter.Gui.Converters
|
||||
{
|
||||
[ValueConversion(typeof(ExportFormat), typeof(string))]
|
||||
public class PartitionFormatToStringConverter : IValueConverter
|
||||
{
|
||||
public static PartitionFormatToStringConverter Instance { get; } = new();
|
||||
|
||||
public object? Convert(object value, Type targetType, object parameter, CultureInfo culture)
|
||||
{
|
||||
if (value is PartitionFormat partitionFormatValue)
|
||||
return partitionFormatValue.GetDisplayName();
|
||||
|
||||
return default(string);
|
||||
}
|
||||
|
||||
public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture) =>
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,33 @@
|
|||
using DiscordChatExporter.Gui.Internal;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using System.Windows.Data;
|
||||
|
||||
namespace DiscordChatExporter.Gui.Converters
|
||||
{
|
||||
[ValueConversion(typeof(DateTimeOffset?), typeof(DateTime?))]
|
||||
public class PartitionFormatToTextBoxHintConverter : IValueConverter
|
||||
{
|
||||
public static PartitionFormatToTextBoxHintConverter Instance { get; } = new();
|
||||
|
||||
public object? Convert(object value, Type targetType, object parameter, CultureInfo culture)
|
||||
{
|
||||
if (value is PartitionFormat partitionFormat)
|
||||
return partitionFormat switch
|
||||
{
|
||||
PartitionFormat.FileSize => "MB per partition",
|
||||
PartitionFormat.MessageCount => "Messages per partition",
|
||||
_ => default(string)
|
||||
};
|
||||
|
||||
return default(DateTime?);
|
||||
}
|
||||
|
||||
public object? ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,33 @@
|
|||
using DiscordChatExporter.Gui.Internal;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using System.Windows.Data;
|
||||
|
||||
namespace DiscordChatExporter.Gui.Converters
|
||||
{
|
||||
[ValueConversion(typeof(DateTimeOffset?), typeof(DateTime?))]
|
||||
public class PartitionFormatToTooltipConverter : IValueConverter
|
||||
{
|
||||
public static PartitionFormatToTextBoxHintConverter Instance { get; } = new();
|
||||
|
||||
public object? Convert(object value, Type targetType, object parameter, CultureInfo culture)
|
||||
{
|
||||
if (value is PartitionFormat partitionFormat)
|
||||
return partitionFormat switch
|
||||
{
|
||||
PartitionFormat.FileSize => "Split output into partitions close to this file size",
|
||||
PartitionFormat.MessageCount => "Split output into partitions limited to this number of messages",
|
||||
_ => default(string)
|
||||
};
|
||||
|
||||
return default(DateTime?);
|
||||
}
|
||||
|
||||
public object? ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,5 +1,6 @@
|
|||
using DiscordChatExporter.Core.Discord;
|
||||
using DiscordChatExporter.Core.Exporting;
|
||||
using DiscordChatExporter.Gui.Internal;
|
||||
using Tyrrrz.Settings;
|
||||
|
||||
namespace DiscordChatExporter.Gui.Services
|
||||
|
@ -22,6 +23,8 @@ namespace DiscordChatExporter.Gui.Services
|
|||
|
||||
public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark;
|
||||
|
||||
public PartitionFormat LastPartitionFormat { get; set; } = PartitionFormat.MessageCount;
|
||||
|
||||
public int? LastPartitionLimit { get; set; }
|
||||
|
||||
public bool LastShouldDownloadMedia { get; set; }
|
||||
|
|
22
DiscordChatExporter.Gui/Utils/PartitionFormat.cs
Normal file
22
DiscordChatExporter.Gui/Utils/PartitionFormat.cs
Normal file
|
@ -0,0 +1,22 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace DiscordChatExporter.Gui.Internal
|
||||
{
|
||||
public enum PartitionFormat
|
||||
{
|
||||
MessageCount,
|
||||
FileSize,
|
||||
}
|
||||
|
||||
public static class PartitionFormatExtensions
|
||||
{
|
||||
public static string GetDisplayName(this PartitionFormat format) => format switch
|
||||
{
|
||||
PartitionFormat.MessageCount => "Message count",
|
||||
PartitionFormat.FileSize => "File size (MB)",
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(format))
|
||||
};
|
||||
}
|
||||
}
|
|
@ -1,6 +1,7 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using DiscordChatExporter.Gui.Internal;
|
||||
using DiscordChatExporter.Core.Discord;
|
||||
using DiscordChatExporter.Core.Discord.Data;
|
||||
using DiscordChatExporter.Core.Exporting;
|
||||
|
@ -46,6 +47,11 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
|
|||
|
||||
public DateTimeOffset? Before => BeforeDate?.Add(BeforeTime ?? TimeSpan.Zero);
|
||||
|
||||
public IReadOnlyList<PartitionFormat> AvailablePartitionFormats =>
|
||||
Enum.GetValues(typeof(PartitionFormat)).Cast<PartitionFormat>().ToArray();
|
||||
|
||||
public PartitionFormat SelectedPartitionFormat { get; set; }
|
||||
|
||||
public int? PartitionLimit { get; set; }
|
||||
|
||||
public bool ShouldDownloadMedia { get; set; }
|
||||
|
@ -67,6 +73,8 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
|
|||
SelectedFormat = _settingsService.LastExportFormat;
|
||||
PartitionLimit = _settingsService.LastPartitionLimit;
|
||||
ShouldDownloadMedia = _settingsService.LastShouldDownloadMedia;
|
||||
SelectedPartitionFormat = _settingsService.LastPartitionFormat;
|
||||
|
||||
}
|
||||
|
||||
public void Confirm()
|
||||
|
@ -74,6 +82,7 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
|
|||
// Persist preferences
|
||||
_settingsService.LastExportFormat = SelectedFormat;
|
||||
_settingsService.LastPartitionLimit = PartitionLimit;
|
||||
_settingsService.LastPartitionFormat = SelectedPartitionFormat;
|
||||
_settingsService.LastShouldDownloadMedia = ShouldDownloadMedia;
|
||||
|
||||
// If single channel - prompt file path
|
||||
|
|
|
@ -8,6 +8,7 @@ using DiscordChatExporter.Core.Discord.Data;
|
|||
using DiscordChatExporter.Core.Exceptions;
|
||||
using DiscordChatExporter.Core.Exporting;
|
||||
using DiscordChatExporter.Core.Utils.Extensions;
|
||||
using DiscordChatExporter.Gui.Internal;
|
||||
using DiscordChatExporter.Gui.Services;
|
||||
using DiscordChatExporter.Gui.Utils;
|
||||
using DiscordChatExporter.Gui.ViewModels.Dialogs;
|
||||
|
@ -213,7 +214,7 @@ namespace DiscordChatExporter.Gui.ViewModels
|
|||
dialog.SelectedFormat,
|
||||
dialog.After?.Pipe(Snowflake.FromDate),
|
||||
dialog.Before?.Pipe(Snowflake.FromDate),
|
||||
dialog.PartitionLimit,
|
||||
CreatePartitioner(),
|
||||
dialog.ShouldDownloadMedia,
|
||||
_settingsService.ShouldReuseMedia,
|
||||
_settingsService.DateFormat
|
||||
|
@ -236,6 +237,19 @@ namespace DiscordChatExporter.Gui.ViewModels
|
|||
// Notify of overall completion
|
||||
if (successfulExportCount > 0)
|
||||
Notifications.Enqueue($"Successfully exported {successfulExportCount} channel(s)");
|
||||
|
||||
IPartitioner CreatePartitioner()
|
||||
{
|
||||
var partitionFormat = dialog.SelectedPartitionFormat;
|
||||
var partitionLimit = dialog.PartitionLimit;
|
||||
|
||||
return (partitionFormat, partitionLimit) switch
|
||||
{
|
||||
(PartitionFormat.MessageCount, int messageLimit) => new MessageCountPartitioner(messageLimit),
|
||||
(PartitionFormat.FileSize, int fileSizeLimit) => new FileSizePartitioner(fileSizeLimit),
|
||||
_ => new NullPartitioner()
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -126,12 +126,43 @@
|
|||
</Grid>
|
||||
|
||||
<!-- Partitioning -->
|
||||
<TextBox
|
||||
<Grid Name="PartitioningGrid">
|
||||
<Grid.ColumnDefinitions>
|
||||
<ColumnDefinition Width="1*" />
|
||||
<ColumnDefinition Width="1*" />
|
||||
</Grid.ColumnDefinitions>
|
||||
<Grid.RowDefinitions>
|
||||
<RowDefinition Height="*" />
|
||||
</Grid.RowDefinitions>
|
||||
|
||||
<ComboBox
|
||||
Name="PartitionFormatComboBox"
|
||||
Grid.Row="0"
|
||||
Grid.Column="0"
|
||||
Margin="16,8"
|
||||
materialDesign:HintAssist.Hint="Messages per partition"
|
||||
materialDesign:HintAssist.Hint="Partition by"
|
||||
materialDesign:HintAssist.IsFloating="True"
|
||||
IsReadOnly="True"
|
||||
ItemsSource="{Binding AvailablePartitionFormats}"
|
||||
SelectedItem="{Binding SelectedPartitionFormat}">
|
||||
<ComboBox.ItemTemplate>
|
||||
<DataTemplate>
|
||||
<TextBlock Text="{Binding Converter={x:Static converters:PartitionFormatToStringConverter.Instance}}" />
|
||||
</DataTemplate>
|
||||
</ComboBox.ItemTemplate>
|
||||
</ComboBox>
|
||||
|
||||
<TextBox
|
||||
Name="PartitionTextBox"
|
||||
Grid.Row="0"
|
||||
Grid.Column="1"
|
||||
Margin="16,8"
|
||||
materialDesign:HintAssist.Hint="{Binding SelectedPartitionFormat, Converter={x:Static converters:PartitionFormatToTooltipConverter.Instance}}"
|
||||
materialDesign:HintAssist.IsFloating="True"
|
||||
Text="{Binding PartitionLimit, TargetNullValue=''}"
|
||||
ToolTip="Split output into partitions limited to this number of messages" />
|
||||
ToolTip="{Binding SelectedPartitionFormat, Converter={x:Static converters:PartitionFormatToTooltipConverter.Instance}}" />
|
||||
|
||||
</Grid>
|
||||
|
||||
<!-- Download media -->
|
||||
<Grid Margin="16,16" ToolTip="Download referenced media content (user avatars, attached files, embedded images, etc)">
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
namespace DiscordChatExporter.Gui.Views.Dialogs
|
||||
namespace DiscordChatExporter.Gui.Views.Dialogs
|
||||
{
|
||||
public partial class ExportSetupView
|
||||
{
|
||||
|
|
Loading…
Reference in a new issue