HellionChat/HellionChat/Export/MessageExporter.cs

using System.Globalization;
using System.Text;
using HellionChat.Code;
using HellionChat.Util;

namespace HellionChat.Export;

internal enum ExportFormat
{
    Markdown,
    Json,
    Csv,
}

internal static class ExportFormatExt
{
    internal static string Extension(this ExportFormat fmt) =>
        fmt switch
        {
            ExportFormat.Markdown => "md",
            ExportFormat.Json => "json",
            ExportFormat.Csv => "csv",
            _ => "txt",
        };

    internal static string Filter(this ExportFormat fmt) =>
        fmt switch
        {
            ExportFormat.Markdown => ".md",
            ExportFormat.Json => ".json",
            ExportFormat.Csv => ".csv",
            _ => ".txt",
        };
}

// Serializes message snapshots to Markdown, JSON, or CSV.
//
// Text comes from the chunk lists, never from SenderSource/ContentSource. Those
// are raw SeStrings, and reading TextValue on one containing an auto-translate
// phrase reaches SeStringEvaluator, which asserts it is on the main thread and
// throws unconditionally when a macro resolves a global number. An export runs on
// a worker, so that would abort it partway and leave half a file behind.
//
// The chunks are already resolved: ChunkUtil turns auto-translate into text at
// ingest, and the full-text index reads them exactly this way. Same strings, no
// evaluator, no thread affinity.
//
// The caller pre-filters by channel and date via StreamForExport; only the sender
// substring is applied here.
internal static class MessageExporter
{
    private static readonly UTF8Encoding Utf8NoBom = new(encoderShouldEmitUTF8Identifier: false);

    internal record FilterDescription(
        IReadOnlyCollection<int>? ChatTypes,
        DateTimeOffset? From,
        DateTimeOffset? To,
        string? SenderSubstring
    );

    internal static int ExportToFile(
        string path,
        ExportFormat format,
        IEnumerable<Message> messages,
        FilterDescription filter
    )
    {
        // Rejected before the file is touched. The old order opened the stream
        // first, so an unknown format left a zero-byte file where the user's
        // previous export had been.
        if (!Enum.IsDefined(format))
            throw new ArgumentOutOfRangeException(nameof(format), format, null);

        var matching = filter.SenderSubstring is { Length: > 0 } needle
            ? messages.Where(m => MatchesSender(m, needle))
            : messages;

        // Written beside the target and moved into place at the end. A crash or
        // an unplugged drive halfway through would otherwise leave a file that
        // opens fine and is quietly incomplete -- and this is the path a GDPR
        // access request goes out on, where "looks complete" is the dangerous
        // failure.
        var temp = path + ".part";
        int written;
        try
        {
            // Encoding.UTF8 writes a byte order mark, and that is not a
            // cosmetic detail here: a leading U+FEFF makes the JSON invalid for
            // every strict parser, Python's json.load included. CSV is the one
            // format that wants it -- without a BOM Excel guesses the codepage
            // and mangles every non-ASCII name in the file.
            var encoding = format == ExportFormat.Csv ? Encoding.UTF8 : Utf8NoBom;

            using (var writer = new StreamWriter(temp, append: false, encoding))
            {
                written = format switch
                {
                    ExportFormat.Markdown => WriteMarkdown(writer, matching, filter),
                    ExportFormat.Json => WriteJson(writer, matching, filter),
                    _ => WriteCsv(writer, matching, filter),
                };
            }

            // An export that matched nothing does not replace anything. The
            // file still has a header and a footer, so moving it would put a
            // near-empty file where the user's previous export was -- and then
            // report "no message matched the filter", which reads as "nothing
            // happened". Dalamud's save dialog has no overwrite confirmation to
            // fall back on.
            if (written == 0)
            {
                TryDeleteTemp(temp);
                return 0;
            }

            File.Move(temp, path, overwrite: true);
            return written;
        }
        catch
        {
            TryDeleteTemp(temp);
            throw;
        }
    }

    // Best effort: the export already failed, and a leftover .part file is a
    // smaller problem than masking the original exception with an IO one.
    private static void TryDeleteTemp(string temp)
    {
        try
        {
            if (File.Exists(temp))
                File.Delete(temp);
        }
        catch (IOException) { }
        catch (UnauthorizedAccessException) { }
    }

    private static bool MatchesSender(Message m, string needle) =>
        SenderText(m).Contains(needle, StringComparison.OrdinalIgnoreCase);

    private static string SenderText(Message m) => ChunkUtil.ToRawString(m.Sender);

    private static string ContentText(Message m) => ChunkUtil.ToRawString(m.Content);

    private static int WriteMarkdown(
        StreamWriter w,
        IEnumerable<Message> messages,
        FilterDescription filter
    )
    {
        w.WriteLine("# Hellion Chat Export");
        w.WriteLine();
        w.WriteLine($"Generated: {DateTimeOffset.Now:yyyy-MM-dd HH:mm zzz}");
        WriteFilterSummaryMarkdown(w, filter);
        w.WriteLine();

        DateTimeOffset? lastDate = null;
        var count = 0;
        foreach (var m in messages)
        {
            count++;
            var localDate = m.Date.ToLocalTime();
            if (lastDate is null || localDate.Date != lastDate.Value.Date)
            {
                w.WriteLine();
                w.WriteLine($"## {localDate:yyyy-MM-dd}");
                w.WriteLine();
                lastDate = localDate;
            }

            var chatType = (ChatType)(ushort)m.Code.Type;
            var sender = SenderText(m).Trim().Trim('<', '>', '[', ']', ':').Trim();
            var content = ContentText(m);

            if (string.IsNullOrEmpty(sender))
                w.WriteLine($"**[{localDate:HH:mm}] {chatType}:** {content}");
            else
                w.WriteLine($"**[{localDate:HH:mm}] {chatType} {sender}:** {content}");
        }

        w.WriteLine();
        w.WriteLine($"---");
        w.WriteLine($"Total messages: {count}");
        return count;
    }

    private static void WriteFilterSummaryMarkdown(StreamWriter w, FilterDescription filter)
    {
        if (filter.ChatTypes is { Count: > 0 })
            w.WriteLine(
                $"ChatTypes: {string.Join(", ", filter.ChatTypes.Select(t => $"{(ChatType)(ushort)t}({t})"))}"
            );
        if (filter.From is not null)
            w.WriteLine($"From: {filter.From.Value.ToLocalTime():yyyy-MM-dd HH:mm}");
        if (filter.To is not null)
            w.WriteLine($"To: {filter.To.Value.ToLocalTime():yyyy-MM-dd HH:mm}");
        if (filter.SenderSubstring is { Length: > 0 })
            w.WriteLine($"Sender contains: \"{filter.SenderSubstring}\"");
    }

    private static int WriteJson(
        StreamWriter w,
        IEnumerable<Message> messages,
        FilterDescription filter
    )
    {
        // Manual JSON to avoid System.Text.Json policy coupling.
        w.Write("{\n  \"exported_at\": \"");
        w.Write(DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture));
        w.Write("\",\n  \"plugin\": \"Hellion Chat\",\n");
        w.Write("  \"filter\": {\n");
        w.Write("    \"chat_types\": ");
        if (filter.ChatTypes is { Count: > 0 })
            w.Write("[" + string.Join(",", filter.ChatTypes) + "]");
        else
            w.Write("null");
        w.Write(",\n    \"from\": ");
        w.Write(
            filter.From is null
                ? "null"
                : "\"" + filter.From.Value.ToString("O", CultureInfo.InvariantCulture) + "\""
        );
        w.Write(",\n    \"to\": ");
        w.Write(
            filter.To is null
                ? "null"
                : "\"" + filter.To.Value.ToString("O", CultureInfo.InvariantCulture) + "\""
        );
        w.Write(",\n    \"sender_substring\": ");
        w.Write(filter.SenderSubstring is null ? "null" : JsonString(filter.SenderSubstring));
        w.Write("\n  },\n  \"messages\": [\n");

        var first = true;
        var count = 0;
        foreach (var m in messages)
        {
            if (!first)
                w.Write(",\n");
            first = false;
            count++;

            var chatType = (ChatType)(ushort)m.Code.Type;
            w.Write("    {");
            w.Write($"\"id\":\"{m.Id}\"");
            w.Write($",\"date\":\"{m.Date.ToString("O", CultureInfo.InvariantCulture)}\"");
            w.Write($",\"chat_type\":{(int)m.Code.Type}");
            w.Write($",\"chat_type_name\":\"{chatType}\"");
            // Cast, not interpolate. These are XivChatRelationKind, and string
            // interpolation of an enum writes the member name -- so every
            // message with a recognised relation produced
            // "source_kind":LocalPlayer, which no parser accepts. This is the
            // file an access request goes out on.
            w.Write($",\"source_kind\":{(int)m.Code.Source}");
            w.Write($",\"target_kind\":{(int)m.Code.Target}");
            w.Write($",\"receiver\":{m.Receiver}");
            w.Write($",\"content_id\":{m.ContentId}");
            w.Write($",\"sender\":{JsonString(SenderText(m))}");
            w.Write($",\"content\":{JsonString(ContentText(m))}");
            w.Write("}");
        }

        w.Write("\n  ],\n");
        w.Write($"  \"total\": {count}\n}}\n");
        return count;
    }

    private static int WriteCsv(
        StreamWriter w,
        IEnumerable<Message> messages,
        FilterDescription filter
    )
    {
        // Header always written so empty exports remain importable.
        w.WriteLine("Date,ChatType,ChatTypeName,Sender,Content,Receiver,ContentId");
        var count = 0;
        foreach (var m in messages)
        {
            count++;
            var chatType = (ChatType)(ushort)m.Code.Type;
            w.Write(m.Date.ToString("O", CultureInfo.InvariantCulture));
            w.Write(',');
            w.Write((int)m.Code.Type);
            w.Write(',');
            w.Write(CsvString(chatType.ToString()));
            w.Write(',');
            w.Write(CsvString(SenderText(m)));
            w.Write(',');
            w.Write(CsvString(ContentText(m)));
            w.Write(',');
            w.Write(m.Receiver);
            w.Write(',');
            w.Write(m.ContentId);
            w.WriteLine();
        }
        return count;
    }

    private static string JsonString(string s)
    {
        var sb = new StringBuilder(s.Length + 2);
        sb.Append('"');
        foreach (var c in s)
        {
            switch (c)
            {
                case '"':
                    sb.Append("\\\"");
                    break;
                case '\\':
                    sb.Append("\\\\");
                    break;
                case '\b':
                    sb.Append("\\b");
                    break;
                case '\f':
                    sb.Append("\\f");
                    break;
                case '\n':
                    sb.Append("\\n");
                    break;
                case '\r':
                    sb.Append("\\r");
                    break;
                case '\t':
                    sb.Append("\\t");
                    break;
                default:
                    if (c < 0x20)
                        sb.Append($"\\u{(int)c:x4}");
                    else
                        sb.Append(c);
                    break;
            }
        }
        sb.Append('"');
        return sb.ToString();
    }

    private static string CsvString(string s)
    {
        // Leading =, +, - and @ make a spreadsheet treat the cell as a formula.
        // Every value here is text somebody else typed into a chat channel, and
        // this file exists to be opened in Excel, so a prefixed apostrophe goes
        // in front. It is the standard defence and it costs one character that
        // spreadsheets hide.
        if (s.Length > 0 && s[0] is '=' or '+' or '-' or '@' or '\t' or '\r')
            s = "'" + s;

        if (s.IndexOfAny(['"', ',', '\n', '\r']) < 0)
            return s;

        return "\"" + s.Replace("\"", "\"\"") + "\"";
    }
}