mirrors/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2024-09-19 12:18:48 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add partition by file size (#497)

Browse source
This commit is contained in:
Andrew Kolos 2021-04-12 06:50:32 -04:00 committed by GitHub
parent ad3655396f
commit eb89ea5b40
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 331 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs
View file

@ -31,8 +31,9 @@ namespace DiscordChatExporter.Cli.Commands.Base
[CommandOption("before", Description = "Only include messages sent before this date or message ID.")]
public Snowflake? Before { get; init; }
[CommandOption("partition", 'p', Description = "Split output into partitions limited to this number of messages.")]
public int? PartitionLimit { get; init; }
[CommandOption("partition", 'p', Converter = typeof(PartitionConverter),
Description = "Split output into partitions limited to this number of messages or a maximum file size (e.g. \"25mb\").")]
public IPartitioner Partitoner { get; init; } = new NullPartitioner();
[CommandOption("parallel", Description = "Limits how many channels can be exported in parallel.")]
public int ParallelLimit { get; init; } = 1;
@ -74,7 +75,7 @@ namespace DiscordChatExporter.Cli.Commands.Base
ExportFormat,
After,
Before,
PartitionLimit,
Partitoner,
ShouldDownloadMedia,
ShouldReuseMedia,
DateFormat

28
DiscordChatExporter.Cli/Commands/Base/PartitionConverter.cs Normal file
View file

@ -0,0 +1,28 @@
using System;
using System.Collections.Generic;
using System.Text;
using ByteSizeLib;
using CliFx.Extensibility;
using DiscordChatExporter.Core;
using DiscordChatExporter.Core.Exporting;
namespace DiscordChatExporter.Cli.Commands.Base
{
public class PartitionConverter : BindingConverter<IPartitioner>
{
public override IPartitioner Convert(string? rawValue)
{
if (rawValue == null) return new NullPartitioner();
if (ByteSize.TryParse(rawValue, out ByteSize filesize))
{
return new FileSizePartitioner((long)filesize.Bytes);
}
else
{
int messageLimit = int.Parse(rawValue);
return new MessageCountPartitioner(messageLimit);
}
}
}
}

2
DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj
View file

@ -8,6 +8,8 @@
<ItemGroup>
<PackageReference Include="CliFx" Version="2.0.1" />
<PackageReference Include="Spectre.Console" Version="0.38.0" />
<PackageReference Include="Gress" Version="1.2.0" />
<PackageReference Include="OneOf" Version="3.0.174" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
</ItemGroup>

1
DiscordChatExporter.Core/Discord/Data/Attachment.cs
View file

@ -5,6 +5,7 @@ using System.Text.Json;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
using FileSize = DiscordChatExporter.Core.Discord.Data.Common.FileSize;
namespace DiscordChatExporter.Core.Discord.Data
{

2
DiscordChatExporter.Core/Discord/Data/Common/FileSize.cs
View file

@ -62,4 +62,4 @@ namespace DiscordChatExporter.Core.Discord.Data.Common
{
public static FileSize FromBytes(long bytes) => new(bytes);
}
}
}

1
DiscordChatExporter.Core/DiscordChatExporter.Core.csproj
View file

@ -5,6 +5,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ByteSize" Version="2.0.0" />
<PackageReference Include="JsonExtensions" Version="1.0.1" />
<PackageReference Include="MiniRazor.CodeGen" Version="2.1.2" />
<PackageReference Include="Polly" Version="7.2.1" />

6
DiscordChatExporter.Core/Exporting/ExportRequest.cs
View file

@ -28,7 +28,7 @@ namespace DiscordChatExporter.Core.Exporting
public Snowflake? Before { get; }
public int? PartitionLimit { get; }
public IPartitioner Partitoner { get; }
public bool ShouldDownloadMedia { get; }
@ -43,7 +43,7 @@ namespace DiscordChatExporter.Core.Exporting
ExportFormat format,
Snowflake? after,
Snowflake? before,
int? partitionLimit,
IPartitioner partitioner,
bool shouldDownloadMedia,
bool shouldReuseMedia,
string dateFormat)
@ -54,7 +54,7 @@ namespace DiscordChatExporter.Core.Exporting
Format = format;
After = after;
Before = before;
PartitionLimit = partitionLimit;
Partitoner = partitioner;
ShouldDownloadMedia = shouldDownloadMedia;
ShouldReuseMedia = shouldReuseMedia;
DateFormat = dateFormat;

22
DiscordChatExporter.Core/Exporting/MessageExporter.cs
View file

@ -1,13 +1,18 @@
using System;
using System.IO;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using ByteSizeLib;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting;
using DiscordChatExporter.Core.Exporting.Partitioners;
using DiscordChatExporter.Core.Exporting.Writers;
namespace DiscordChatExporter.Core.Exporting
{
internal partial class MessageExporter : IAsyncDisposable
{
private readonly ExportContext _context;
private long _messageCount;
@ -19,11 +24,16 @@ namespace DiscordChatExporter.Core.Exporting
_context = context;
}
private bool IsPartitionLimitReached() =>
_messageCount > 0 &&
_context.Request.PartitionLimit is not null &&
_context.Request.PartitionLimit != 0 &&
_messageCount % _context.Request.PartitionLimit == 0;
private bool IsPartitionLimitReached()
{
if (_writer is null)
{
return false;
}
return _context.Request.Partitoner.IsLimitReached(
new ExportPartitioningContext(_messageCount, _writer.SizeInBytes));
}
private async ValueTask ResetWriterAsync()
{
@ -38,7 +48,7 @@ namespace DiscordChatExporter.Core.Exporting
private async ValueTask<MessageWriter> GetWriterAsync()
{
// Ensure partition limit has not been exceeded
if (IsPartitionLimitReached())
if (_writer != null && IsPartitionLimitReached())
{
await ResetWriterAsync();
_partitionIndex++;

18
DiscordChatExporter.Core/Exporting/Partitioners/ExportPartitioningContext.cs Normal file
View file

@ -0,0 +1,18 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Core.Exporting.Partitioners
{
public class ExportPartitioningContext
{
public long MessageCount { get; }
public long SizeInBytes { get; }
public ExportPartitioningContext(long messageCount, long sizeInBytes)
{
MessageCount = messageCount;
SizeInBytes = sizeInBytes;
}
}
}

21
DiscordChatExporter.Core/Exporting/Partitioners/FileSizePartitioner.cs Normal file
View file

@ -0,0 +1,21 @@
using DiscordChatExporter.Core.Exporting.Partitioners;
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Core.Exporting
{
public class FileSizePartitioner : IPartitioner
{
private long _bytesPerFile;
public FileSizePartitioner(long bytesPerFile)
{
_bytesPerFile = bytesPerFile;
}
public bool IsLimitReached(ExportPartitioningContext context)
{
return context.SizeInBytes >= _bytesPerFile;
}
}
}

12
DiscordChatExporter.Core/Exporting/Partitioners/IPartitioner.cs Normal file
View file

@ -0,0 +1,12 @@
using DiscordChatExporter.Core.Exporting.Partitioners;
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Core.Exporting
{
public interface IPartitioner
{
bool IsLimitReached(ExportPartitioningContext context);
}
}

25
DiscordChatExporter.Core/Exporting/Partitioners/MessageCountPartitioner.cs Normal file
View file

@ -0,0 +1,25 @@
using DiscordChatExporter.Core.Exporting.Partitioners;
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Core.Exporting
{
public class MessageCountPartitioner : IPartitioner
{
private int _messagesPerPartition;
public MessageCountPartitioner(int messagesPerPartition)
{
_messagesPerPartition = messagesPerPartition;
}
public bool IsLimitReached(ExportPartitioningContext context)
{
return context.MessageCount > 0 &&
_messagesPerPartition != 0 &&
context.MessageCount % _messagesPerPartition == 0;
}
}
}

16
DiscordChatExporter.Core/Exporting/Partitioners/NullPartitioner.cs Normal file
View file

@ -0,0 +1,16 @@
using DiscordChatExporter.Core.Exporting;
using DiscordChatExporter.Core.Exporting.Partitioners;
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Core.Exporting
{
public class NullPartitioner : IPartitioner
{
public bool IsLimitReached(ExportPartitioningContext context)
{
return false;
}
}
}

2
DiscordChatExporter.Core/Exporting/Writers/MessageWriter.cs
View file

@ -17,6 +17,8 @@ namespace DiscordChatExporter.Core.Exporting.Writers
Context = context;
}
public long SizeInBytes => Stream.Length;
public virtual ValueTask WritePreambleAsync() => default;
public abstract ValueTask WriteMessageAsync(Message message);

27
DiscordChatExporter.Gui/Converters/PartitionFormatToStringConverter.cs Normal file
View file

@ -0,0 +1,27 @@
using DiscordChatExporter.Core.Exporting;
using DiscordChatExporter.Gui.Internal;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text;
using System.Windows.Data;
namespace DiscordChatExporter.Gui.Converters
{
[ValueConversion(typeof(ExportFormat), typeof(string))]
public class PartitionFormatToStringConverter : IValueConverter
{
public static PartitionFormatToStringConverter Instance { get; } = new();
public object? Convert(object value, Type targetType, object parameter, CultureInfo culture)
{
if (value is PartitionFormat partitionFormatValue)
return partitionFormatValue.GetDisplayName();
return default(string);
}
public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture) =>
throw new NotSupportedException();
}
}

33
DiscordChatExporter.Gui/Converters/PartitionFormatToTextBoxHintConverter.cs Normal file
View file

@ -0,0 +1,33 @@
using DiscordChatExporter.Gui.Internal;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text;
using System.Windows.Data;
namespace DiscordChatExporter.Gui.Converters
{
[ValueConversion(typeof(DateTimeOffset?), typeof(DateTime?))]
public class PartitionFormatToTextBoxHintConverter : IValueConverter
{
public static PartitionFormatToTextBoxHintConverter Instance { get; } = new();
public object? Convert(object value, Type targetType, object parameter, CultureInfo culture)
{
if (value is PartitionFormat partitionFormat)
return partitionFormat switch
{
PartitionFormat.FileSize => "MB per partition",
PartitionFormat.MessageCount => "Messages per partition",
_ => default(string)
};
return default(DateTime?);
}
public object? ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
{
throw new NotSupportedException();
}
}
}

33
DiscordChatExporter.Gui/Converters/PartitionFormatToTooltipConverter.cs Normal file
View file

@ -0,0 +1,33 @@
using DiscordChatExporter.Gui.Internal;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text;
using System.Windows.Data;
namespace DiscordChatExporter.Gui.Converters
{
[ValueConversion(typeof(DateTimeOffset?), typeof(DateTime?))]
public class PartitionFormatToTooltipConverter : IValueConverter
{
public static PartitionFormatToTextBoxHintConverter Instance { get; } = new();
public object? Convert(object value, Type targetType, object parameter, CultureInfo culture)
{
if (value is PartitionFormat partitionFormat)
return partitionFormat switch
{
PartitionFormat.FileSize => "Split output into partitions close to this file size",
PartitionFormat.MessageCount => "Split output into partitions limited to this number of messages",
_ => default(string)
};
return default(DateTime?);
}
public object? ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
{
throw new NotSupportedException();
}
}
}

3
DiscordChatExporter.Gui/Services/SettingsService.cs
View file

@ -1,5 +1,6 @@
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Exporting;
using DiscordChatExporter.Gui.Internal;
using Tyrrrz.Settings;
namespace DiscordChatExporter.Gui.Services
@ -22,6 +23,8 @@ namespace DiscordChatExporter.Gui.Services
public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark;
public PartitionFormat LastPartitionFormat { get; set; } = PartitionFormat.MessageCount;
public int? LastPartitionLimit { get; set; }
public bool LastShouldDownloadMedia { get; set; }

22
DiscordChatExporter.Gui/Utils/PartitionFormat.cs Normal file
View file

@ -0,0 +1,22 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Gui.Internal
{
public enum PartitionFormat
{
MessageCount,
FileSize,
}
public static class PartitionFormatExtensions
{
public static string GetDisplayName(this PartitionFormat format) => format switch
{
PartitionFormat.MessageCount => "Message count",
PartitionFormat.FileSize => "File size (MB)",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
}
}

9
DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using DiscordChatExporter.Gui.Internal;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting;
@ -46,6 +47,11 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
public DateTimeOffset? Before => BeforeDate?.Add(BeforeTime ?? TimeSpan.Zero);
public IReadOnlyList<PartitionFormat> AvailablePartitionFormats =>
Enum.GetValues(typeof(PartitionFormat)).Cast<PartitionFormat>().ToArray();
public PartitionFormat SelectedPartitionFormat { get; set; }
public int? PartitionLimit { get; set; }
public bool ShouldDownloadMedia { get; set; }
@ -67,6 +73,8 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
SelectedFormat = _settingsService.LastExportFormat;
PartitionLimit = _settingsService.LastPartitionLimit;
ShouldDownloadMedia = _settingsService.LastShouldDownloadMedia;
SelectedPartitionFormat = _settingsService.LastPartitionFormat;
}
public void Confirm()
@ -74,6 +82,7 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
// Persist preferences
_settingsService.LastExportFormat = SelectedFormat;
_settingsService.LastPartitionLimit = PartitionLimit;
_settingsService.LastPartitionFormat = SelectedPartitionFormat;
_settingsService.LastShouldDownloadMedia = ShouldDownloadMedia;
// If single channel - prompt file path

16
DiscordChatExporter.Gui/ViewModels/RootViewModel.cs
View file

@ -8,6 +8,7 @@ using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exceptions;
using DiscordChatExporter.Core.Exporting;
using DiscordChatExporter.Core.Utils.Extensions;
using DiscordChatExporter.Gui.Internal;
using DiscordChatExporter.Gui.Services;
using DiscordChatExporter.Gui.Utils;
using DiscordChatExporter.Gui.ViewModels.Dialogs;
@ -213,7 +214,7 @@ namespace DiscordChatExporter.Gui.ViewModels
dialog.SelectedFormat,
dialog.After?.Pipe(Snowflake.FromDate),
dialog.Before?.Pipe(Snowflake.FromDate),
dialog.PartitionLimit,
CreatePartitioner(),
dialog.ShouldDownloadMedia,
_settingsService.ShouldReuseMedia,
_settingsService.DateFormat
@ -236,6 +237,19 @@ namespace DiscordChatExporter.Gui.ViewModels
// Notify of overall completion
if (successfulExportCount > 0)
Notifications.Enqueue($"Successfully exported {successfulExportCount} channel(s)");
IPartitioner CreatePartitioner()
{
var partitionFormat = dialog.SelectedPartitionFormat;
var partitionLimit = dialog.PartitionLimit;
return (partitionFormat, partitionLimit) switch
{
(PartitionFormat.MessageCount, int messageLimit) => new MessageCountPartitioner(messageLimit),
(PartitionFormat.FileSize, int fileSizeLimit) => new FileSizePartitioner(fileSizeLimit),
_ => new NullPartitioner()
};
}
}
}
}

43
DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml
View file

@ -126,12 +126,43 @@
</Grid>
<!-- Partitioning -->
<TextBox
Margin="16,8"
materialDesign:HintAssist.Hint="Messages per partition"
materialDesign:HintAssist.IsFloating="True"
Text="{Binding PartitionLimit, TargetNullValue=''}"
ToolTip="Split output into partitions limited to this number of messages" />
<Grid Name="PartitioningGrid">
<Grid.ColumnDefinitions>
<ColumnDefinition Width="1*" />
<ColumnDefinition Width="1*" />
</Grid.ColumnDefinitions>
<Grid.RowDefinitions>
<RowDefinition Height="*" />
</Grid.RowDefinitions>
<ComboBox
Name="PartitionFormatComboBox"
Grid.Row="0"
Grid.Column="0"
Margin="16,8"
materialDesign:HintAssist.Hint="Partition by"
materialDesign:HintAssist.IsFloating="True"
IsReadOnly="True"
ItemsSource="{Binding AvailablePartitionFormats}"
SelectedItem="{Binding SelectedPartitionFormat}">
<ComboBox.ItemTemplate>
<DataTemplate>
<TextBlock Text="{Binding Converter={x:Static converters:PartitionFormatToStringConverter.Instance}}" />
</DataTemplate>
</ComboBox.ItemTemplate>
</ComboBox>
<TextBox
Name="PartitionTextBox"
Grid.Row="0"
Grid.Column="1"
Margin="16,8"
materialDesign:HintAssist.Hint="{Binding SelectedPartitionFormat, Converter={x:Static converters:PartitionFormatToTooltipConverter.Instance}}"
materialDesign:HintAssist.IsFloating="True"
Text="{Binding PartitionLimit, TargetNullValue=''}"
ToolTip="{Binding SelectedPartitionFormat, Converter={x:Static converters:PartitionFormatToTooltipConverter.Instance}}" />
</Grid>
<!-- Download media -->
<Grid Margin="16,16" ToolTip="Download referenced media content (user avatars, attached files, embedded images, etc)">

4
DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml.cs
View file

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Gui.Views.Dialogs
namespace DiscordChatExporter.Gui.Views.Dialogs
{
public partial class ExportSetupView
{
@ -7,4 +7,4 @@
InitializeComponent();
}
}
}
}