Refactor partitioning, don't create empty files

Closes #246
This commit is contained in:
Alexey Golub 2020-01-10 21:05:07 +02:00
parent b4df267372
commit bf56902134
11 changed files with 194 additions and 91 deletions

View file

@ -3,6 +3,7 @@ using System.Net;
using System.Threading.Tasks; using System.Threading.Tasks;
using CliFx.Attributes; using CliFx.Attributes;
using CliFx.Services; using CliFx.Services;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Services; using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Exceptions; using DiscordChatExporter.Core.Services.Exceptions;
@ -39,6 +40,10 @@ namespace DiscordChatExporter.Cli.Commands
{ {
console.Error.WriteLine("This channel doesn't exist."); console.Error.WriteLine("This channel doesn't exist.");
} }
catch (DomainException ex)
{
console.Error.WriteLine(ex.Message);
}
} }
} }
} }

View file

@ -4,6 +4,7 @@ using System.Threading.Tasks;
using CliFx.Attributes; using CliFx.Attributes;
using CliFx.Services; using CliFx.Services;
using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Services; using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Exceptions; using DiscordChatExporter.Core.Services.Exceptions;
@ -43,6 +44,10 @@ namespace DiscordChatExporter.Cli.Commands
{ {
console.Error.WriteLine("This channel doesn't exist."); console.Error.WriteLine("This channel doesn't exist.");
} }
catch (DomainException ex)
{
console.Error.WriteLine(ex.Message);
}
} }
} }
} }

View file

@ -0,0 +1,12 @@
using System;
namespace DiscordChatExporter.Core.Models.Exceptions
{
public class DomainException : Exception
{
public DomainException(string message)
: base(message)
{
}
}
}

View file

@ -1,4 +1,5 @@
using System.Threading.Tasks; using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic; using DiscordChatExporter.Core.Rendering.Logic;
@ -8,8 +9,8 @@ namespace DiscordChatExporter.Core.Rendering
{ {
private bool _isHeaderRendered; private bool _isHeaderRendered;
public CsvMessageRenderer(string filePath, RenderContext context) public CsvMessageRenderer(TextWriter writer, RenderContext context)
: base(filePath, context) : base(writer, context)
{ {
} }

View file

@ -0,0 +1,112 @@
using System;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Rendering
{
public partial class FacadeMessageRenderer : IMessageRenderer
{
private readonly string _baseFilePath;
private readonly ExportFormat _format;
private readonly RenderContext _context;
private int _partitionIndex;
private TextWriter _writer;
private IMessageRenderer _innerRenderer;
public FacadeMessageRenderer(string baseFilePath, ExportFormat format, RenderContext context)
{
_baseFilePath = baseFilePath;
_format = format;
_context = context;
}
private void EnsureInnerRendererInitialized()
{
if (_writer != null && _innerRenderer != null)
return;
// Get partition file path
var filePath = GetPartitionFilePath(_baseFilePath, _partitionIndex);
// Create output directory
var dirPath = Path.GetDirectoryName(_baseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
Directory.CreateDirectory(dirPath);
// Create writer (will be disposed by renderer)
_writer = File.CreateText(filePath);
// Create inner renderer
if (_format == ExportFormat.PlainText)
{
_innerRenderer = new PlainTextMessageRenderer(_writer, _context);
}
else if (_format == ExportFormat.Csv)
{
_innerRenderer = new CsvMessageRenderer(_writer, _context);
}
else if (_format == ExportFormat.HtmlDark)
{
_innerRenderer = new HtmlMessageRenderer(_writer, _context, "Dark");
}
else if (_format == ExportFormat.HtmlLight)
{
_innerRenderer = new HtmlMessageRenderer(_writer, _context, "Light");
}
else
{
throw new InvalidOperationException($"Unknown export format [{_format}].");
}
}
public async Task NextPartitionAsync()
{
// Dispose writer and inner renderer
await DisposeAsync();
_writer = null;
_innerRenderer = null;
// Increment partition index
_partitionIndex++;
}
public async Task RenderMessageAsync(Message message)
{
EnsureInnerRendererInitialized();
await _innerRenderer.RenderMessageAsync(message);
}
public async ValueTask DisposeAsync()
{
if (_innerRenderer != null)
await _innerRenderer.DisposeAsync();
if (_writer != null)
await _writer.DisposeAsync();
}
}
public partial class FacadeMessageRenderer
{
private static string GetPartitionFilePath(string baseFilePath, int partitionIndex)
{
// First partition - no changes
if (partitionIndex <= 0)
return baseFilePath;
// Inject partition index into file name
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(baseFilePath);
var fileExt = Path.GetExtension(baseFilePath);
var fileName = $"{fileNameWithoutExt} [part {partitionIndex + 1}]{fileExt}";
// Generate new path
var dirPath = Path.GetDirectoryName(baseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
return Path.Combine(dirPath, fileName);
return fileName;
}
}
}

View file

@ -1,5 +1,6 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO;
using System.Linq; using System.Linq;
using System.Reflection; using System.Reflection;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -22,8 +23,8 @@ namespace DiscordChatExporter.Core.Rendering
private bool _isLeadingBlockRendered; private bool _isLeadingBlockRendered;
public HtmlMessageRenderer(string filePath, RenderContext context, string themeName) public HtmlMessageRenderer(TextWriter writer, RenderContext context, string themeName)
: base(filePath, context) : base(writer, context)
{ {
_themeName = themeName; _themeName = themeName;

View file

@ -10,14 +10,14 @@ namespace DiscordChatExporter.Core.Rendering
protected RenderContext Context { get; } protected RenderContext Context { get; }
protected MessageRendererBase(string filePath, RenderContext context) protected MessageRendererBase(TextWriter writer, RenderContext context)
{ {
Writer = File.CreateText(filePath); Writer = writer;
Context = context; Context = context;
} }
public abstract Task RenderMessageAsync(Message message); public abstract Task RenderMessageAsync(Message message);
public virtual ValueTask DisposeAsync() => Writer.DisposeAsync(); public virtual ValueTask DisposeAsync() => default;
} }
} }

View file

@ -1,4 +1,5 @@
using System.Threading.Tasks; using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic; using DiscordChatExporter.Core.Rendering.Logic;
@ -8,8 +9,8 @@ namespace DiscordChatExporter.Core.Rendering
{ {
private bool _isPreambleRendered; private bool _isPreambleRendered;
public PlainTextMessageRenderer(string filePath, RenderContext context) public PlainTextMessageRenderer(TextWriter writer, RenderContext context)
: base(filePath, context) : base(writer, context)
{ {
} }

View file

@ -3,13 +3,14 @@ using System.Collections.Generic;
using System.IO; using System.IO;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Rendering; using DiscordChatExporter.Core.Rendering;
using DiscordChatExporter.Core.Services.Logic; using DiscordChatExporter.Core.Services.Logic;
using Tyrrrz.Extensions; using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Services namespace DiscordChatExporter.Core.Services
{ {
public class ExportService public partial class ExportService
{ {
private readonly SettingsService _settingsService; private readonly SettingsService _settingsService;
private readonly DataService _dataService; private readonly DataService _dataService;
@ -20,47 +21,6 @@ namespace DiscordChatExporter.Core.Services
_dataService = dataService; _dataService = dataService;
} }
private string GetFilePathFromOutputPath(string outputPath, ExportFormat format, RenderContext context)
{
// Output is a directory
if (Directory.Exists(outputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath)))
{
var fileName = ExportLogic.GetDefaultExportFileName(format, context.Guild, context.Channel, context.After, context.Before);
return Path.Combine(outputPath, fileName);
}
// Output is a file
return outputPath;
}
private IMessageRenderer CreateRenderer(string outputPath, int partitionIndex, ExportFormat format, RenderContext context)
{
var filePath = ExportLogic.GetExportPartitionFilePath(
GetFilePathFromOutputPath(outputPath, format, context),
partitionIndex);
// Create output directory
var dirPath = Path.GetDirectoryName(filePath);
if (!string.IsNullOrWhiteSpace(dirPath))
Directory.CreateDirectory(dirPath);
// Create renderer
if (format == ExportFormat.PlainText)
return new PlainTextMessageRenderer(filePath, context);
if (format == ExportFormat.Csv)
return new CsvMessageRenderer(filePath, context);
if (format == ExportFormat.HtmlDark)
return new HtmlMessageRenderer(filePath, context, "Dark");
if (format == ExportFormat.HtmlLight)
return new HtmlMessageRenderer(filePath, context, "Light");
throw new InvalidOperationException($"Unknown export format [{format}].");
}
public async Task ExportChatLogAsync(AuthToken token, Guild guild, Channel channel, public async Task ExportChatLogAsync(AuthToken token, Guild guild, Channel channel,
string outputPath, ExportFormat format, int? partitionLimit, string outputPath, ExportFormat format, int? partitionLimit,
DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null) DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null)
@ -76,35 +36,50 @@ namespace DiscordChatExporter.Core.Services
mentionableUsers, mentionableChannels, mentionableRoles mentionableUsers, mentionableChannels, mentionableRoles
); );
// Render messages // Create renderer
var partitionIndex = 0; var baseFilePath = GetFilePathFromOutputPath(outputPath, format, context);
var partitionMessageCount = 0; await using var renderer = new FacadeMessageRenderer(baseFilePath, format, context);
var renderer = CreateRenderer(outputPath, partitionIndex, format, context);
// Render messages
var messageCount = 0L;
await foreach (var message in _dataService.GetMessagesAsync(token, channel.Id, after, before, progress)) await foreach (var message in _dataService.GetMessagesAsync(token, channel.Id, after, before, progress))
{ {
// Add encountered users to the list of mentionable users // Add encountered users to the list of mentionable users
mentionableUsers.Add(message.Author); mentionableUsers.Add(message.Author);
mentionableUsers.AddRange(message.MentionedUsers); mentionableUsers.AddRange(message.MentionedUsers);
// If new partition is required, reset renderer
if (partitionLimit != null && partitionLimit > 0 && partitionMessageCount >= partitionLimit)
{
partitionIndex++;
partitionMessageCount = 0;
// Flush old renderer and create a new one
await renderer.DisposeAsync();
renderer = CreateRenderer(outputPath, partitionIndex, format, context);
}
// Render message // Render message
await renderer.RenderMessageAsync(message); await renderer.RenderMessageAsync(message);
partitionMessageCount++; messageCount++;
// Trigger next partition when needed
if (partitionLimit != null &&
partitionLimit != 0 &&
messageCount % partitionLimit.Value == 0)
{
await renderer.NextPartitionAsync();
}
} }
// Flush last renderer // Throw if no messages were rendered
await renderer.DisposeAsync(); if (messageCount == 0)
throw new DomainException($"Channel [{channel.Name}] contains no messages for specified period");
}
}
public partial class ExportService
{
private static string GetFilePathFromOutputPath(string outputPath, ExportFormat format, RenderContext context)
{
// Output is a directory
if (Directory.Exists(outputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath)))
{
var fileName = ExportLogic.GetDefaultExportFileName(format, context.Guild, context.Channel, context.After, context.Before);
return Path.Combine(outputPath, fileName);
}
// Output is a file
return outputPath;
} }
} }
} }

View file

@ -49,24 +49,5 @@ namespace DiscordChatExporter.Core.Services.Logic
return buffer.ToString(); return buffer.ToString();
} }
public static string GetExportPartitionFilePath(string baseFilePath, int partitionIndex)
{
// First partition - no changes
if (partitionIndex <= 0)
return baseFilePath;
// Inject partition index into file name
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(baseFilePath);
var fileExt = Path.GetExtension(baseFilePath);
var fileName = $"{fileNameWithoutExt} [part {partitionIndex + 1}]{fileExt}";
// Generate new path
var dirPath = Path.GetDirectoryName(baseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
return Path.Combine(dirPath, fileName);
return fileName;
}
} }
} }

View file

@ -4,6 +4,7 @@ using System.Linq;
using System.Net; using System.Net;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Services; using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Exceptions; using DiscordChatExporter.Core.Services.Exceptions;
using DiscordChatExporter.Gui.Services; using DiscordChatExporter.Gui.Services;
@ -230,6 +231,10 @@ namespace DiscordChatExporter.Gui.ViewModels
{ {
Notifications.Enqueue("Forbidden account may be locked by 2FA"); Notifications.Enqueue("Forbidden account may be locked by 2FA");
} }
catch (DomainException ex)
{
Notifications.Enqueue(ex.Message);
}
finally finally
{ {
operation.Dispose(); operation.Dispose();
@ -276,6 +281,10 @@ namespace DiscordChatExporter.Gui.ViewModels
{ {
Notifications.Enqueue($"Channel [{channel.Model.Name}] doesn't exist"); Notifications.Enqueue($"Channel [{channel.Model.Name}] doesn't exist");
} }
catch (DomainException ex)
{
Notifications.Enqueue(ex.Message);
}
finally finally
{ {
operation.Dispose(); operation.Dispose();
@ -283,7 +292,8 @@ namespace DiscordChatExporter.Gui.ViewModels
} }
// Notify of overall completion // Notify of overall completion
Notifications.Enqueue($"Successfully exported {successfulExportCount} channel(s)"); if (successfulExportCount > 0)
Notifications.Enqueue($"Successfully exported {successfulExportCount} channel(s)");
} }
} }
} }