HellionChat/HellionChat/Util/AutoTranslate.cs

using System.Diagnostics;
using System.Globalization;
using System.Text;
using Dalamud.Game;
using Dalamud.Utility;
using Lumina.Excel;
using Lumina.Text.Payloads;
using Lumina.Text.ReadOnly;
using Pidgin;
using static Pidgin.Parser;
using static Pidgin.Parser<char>;

namespace HellionChat.Util;

internal static class AutoTranslate
{
    private static readonly Dictionary<ClientLanguage, List<AutoTranslateEntry>> Entries = new();
    private static readonly HashSet<(uint, uint)> ValidEntries = [];

    // Serialises all reads/writes against Entries and ValidEntries.
    // PreloadCache fills both from a worker thread while the main thread
    // reads via Matching/ReplaceWithPayload/StartsWithCommand.
    private static readonly object EntriesLock = new();

    private static Parser<char, (string name, Maybe<IEnumerable<ISelectorPart>> selector)> Parser()
    {
        var sheetName = Any.AtLeastOnceUntil(Lookahead(Char('[').IgnoreResult().Or(End)))
            .Select(string.Concat)
            .Labelled("sheetName");
        var numPair = Map(
                ISelectorPart (first, second) =>
                    new IndexRange(
                        uint.Parse(string.Concat(first)),
                        uint.Parse(string.Concat(second))
                    ),
                Digit.AtLeastOnce().Before(Char('-')),
                Digit.AtLeastOnce()
            )
            .Labelled("numPair");
        var singleRow = Digit
            .AtLeastOnce()
            .Select(string.Concat)
            .Select(ISelectorPart (num) => new SingleRow(uint.Parse(num)));
        var column = String("col-")
            .Then(Digit.AtLeastOnce())
            .Select(string.Concat)
            .Select(ISelectorPart (num) => new ColumnSpecifier(uint.Parse(num)));
        var noun = String("noun").Select(ISelectorPart (_) => new NounMarker());
        var selectorItems = OneOf(Try(numPair), singleRow, column, noun)
            .Separated(Char(','))
            .Labelled("selectorItems");
        var selector = selectorItems.Between(Char('['), Char(']')).Labelled("selector");
        return Map((name, sel) => (name, sel), sheetName, selector.Optional());
    }

    // Warms the auto-translate cache on a background thread so the first
    // message send doesn't hitch the main thread. IsBackground keeps plugin
    // unload non-blocking even if the warmup is still in flight.
    internal static void PreloadCache()
    {
        var thread = new Thread(() =>
        {
            var sw = Stopwatch.StartNew();
            AllEntries();
            // v1.4.9 R3 profiling: Information so the xllog tail surfaces this
            // without a Debug filter. Belt-and-suspenders for future plugin-load
            // regressions; remains in place after Sub-Task 3.4 Befund.
            Plugin.LogProxy.Information(
                $"Warming up auto-translate took {sw.ElapsedMilliseconds}ms"
            );
        })
        {
            IsBackground = true,
            Name = "HellionChat-AutoTranslate-Warmup",
        };
        thread.Start();
    }

    private static List<AutoTranslateEntry> AllEntries()
    {
        lock (EntriesLock)
        {
            if (Entries.TryGetValue(Plugin.DataManager.Language, out var entries))
                return entries;

            return BuildEntriesLocked();
        }
    }

    private static List<AutoTranslateEntry> BuildEntriesLocked()
    {
        var shouldAdd = ValidEntries.Count == 0;

        var parser = Parser();
        var list = new List<AutoTranslateEntry>();
        foreach (var row in Sheets.CompletionSheet)
        {
            var lookup = string.Concat(
                row.LookupTable.Select(p =>
                    p.Type == ReadOnlySePayloadType.Text ? Encoding.UTF8.GetString(p.Body.Span)
                    : p.MacroCode == MacroCode.Num
                    && p.TryGetExpression(out var num)
                    && num.TryGetInt(out var val)
                        ? val.ToString(CultureInfo.InvariantCulture)
                    : ",,,unexpected macro code,,,"
                )
            );
            try
            {
                if (lookup is not ("" or "@"))
                {
                    // SE added whitespace to newer entries; strip it before parsing.
                    lookup = lookup.Replace(" ", "");

                    var (sheetName, selector) = parser.ParseOrThrow(lookup);
                    var sheet = Plugin.DataManager.Excel.GetSheet<RawRow>(name: sheetName);

                    var columns = new List<int>();
                    var rows = new List<Range>();
                    if (selector.HasValue)
                    {
                        columns.Clear();
                        rows.Clear();
                        foreach (var part in selector.Value)
                        {
                            switch (part)
                            {
                                case IndexRange range:
                                {
                                    var start = (int)range.Start;
                                    var end = (int)(range.End + 1);
                                    rows.Add(start..end);
                                    break;
                                }
                                case SingleRow single:
                                {
                                    var idx = (int)single.Row;
                                    rows.Add(idx..(idx + 1));
                                    break;
                                }
                                case ColumnSpecifier col:
                                    columns.Add((int)col.Column);
                                    break;
                            }
                        }
                    }

                    if (columns.Count == 0)
                        columns.Add(0);

                    if (rows.Count == 0)
                        // Can't use index-from-end here because we iterate over integers,
                        // not an array directly. `0..^0` would silently skip the sheet.
                        rows.Add(..Index.FromStart((int)sheet.GetRowAt(sheet.Count - 1).RowId + 1));

                    foreach (var range in rows)
                    {
                        // Integer iteration -- can't use index-from-end (see above).
                        for (var i = range.Start.Value; i < range.End.Value; i++)
                        {
                            if (!sheet.TryGetRow((uint)i, out var rowParser))
                                continue;

                            foreach (var col in columns)
                            {
                                var rawName = rowParser.ReadStringColumn(col);
                                if (!rawName.IsEmpty)
                                {
                                    list.Add(
                                        new AutoTranslateEntry(
                                            row.Group,
                                            (uint)i,
                                            rawName.ToString(),
                                            string.Empty
                                        )
                                    );

                                    if (shouldAdd)
                                        ValidEntries.Add((row.Group, (uint)i));
                                }
                            }
                        }
                    }
                }
                else if (lookup is not "@")
                {
                    if (row.Text.IsEmpty)
                        continue;

                    list.Add(
                        new AutoTranslateEntry(
                            row.Group,
                            row.RowId,
                            row.Text.ToString(),
                            row.GroupTitle.ToString()
                        )
                    );

                    if (shouldAdd)
                        ValidEntries.Add((row.Group, row.RowId));
                }
            }
            catch (Exception ex)
            {
                Plugin.LogProxy.Error(ex, $"failed to translate: {lookup}");
            }
        }

        Entries[Plugin.DataManager.Language] = list;
        return list;
    }

    internal static List<AutoTranslateEntry> Matching(string prefix, bool sort)
    {
        var wholeMatches = new List<AutoTranslateEntry>();
        var prefixMatches = new List<AutoTranslateEntry>();
        var otherMatches = new List<AutoTranslateEntry>();
        foreach (var entry in AllEntries())
        {
            if (entry.Text.Equals(prefix, StringComparison.OrdinalIgnoreCase))
            {
                wholeMatches.Add(entry);
            }
            else if (entry.Text.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
            {
                prefixMatches.Add(entry);
            }
            else if (entry.Text.Contains(prefix, StringComparison.OrdinalIgnoreCase))
            {
                otherMatches.Add(entry);
            }
            else if (entry.Title.Length > 0)
            {
                if (entry.Title.Equals(prefix, StringComparison.OrdinalIgnoreCase))
                    wholeMatches.Add(entry);
                else if (entry.Title.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
                    prefixMatches.Add(entry);
                else if (entry.Title.Contains(prefix, StringComparison.OrdinalIgnoreCase))
                    otherMatches.Add(entry);
            }
        }

        if (sort)
        {
            return wholeMatches
                .OrderBy(entry => entry.Text, StringComparer.OrdinalIgnoreCase)
                .Concat(
                    prefixMatches.OrderBy(entry => entry.Text, StringComparer.OrdinalIgnoreCase)
                )
                .Concat(otherMatches.OrderBy(entry => entry.Text, StringComparer.OrdinalIgnoreCase))
                .ToList();
        }

        return wholeMatches.Concat(prefixMatches).Concat(otherMatches).ToList();
    }

    internal static void ReplaceWithPayload(ref byte[] bytes)
    {
        var search = "<at:"u8.ToArray();
        if (bytes.Length <= search.Length)
            return;

        bool needBuild;
        lock (EntriesLock)
            needBuild = ValidEntries.Count == 0;
        if (needBuild)
            AllEntries();

        var start = -1;
        for (var i = 0; i < bytes.Length; i++)
        {
            if (start != -1)
            {
                if (bytes[i] != '>')
                    continue;

                var tag = Encoding.UTF8.GetString(bytes[start..(i + 1)]);
                var parts = tag[4..^1].Split(',', 2);
                if (
                    parts.Length == 2
                    && uint.TryParse(parts[0], out var group)
                    && uint.TryParse(parts[1], out var key)
                )
                {
                    bool isValid;
                    lock (EntriesLock)
                        isValid = ValidEntries.Contains((group, key));
                    var payload = isValid ? CreateFixedTranslation(group, key) : [];

                    var oldBytes = bytes.ToArray();
                    var lengthDiff = payload.Length - (i - start);
                    bytes = new byte[oldBytes.Length + lengthDiff];
                    Array.Copy(oldBytes, bytes, start);
                    Array.Copy(payload, 0, bytes, start, payload.Length);
                    Array.Copy(
                        oldBytes,
                        i + 1,
                        bytes,
                        start + payload.Length,
                        oldBytes.Length - (i + 1)
                    );

                    i += lengthDiff;
                }

                start = -1;
            }

            // Span comparison avoids the msvcrt.dll P/Invoke which is fragile
            // under Wine and caused an extra managed-to-unmanaged copy per check.
            if (
                i + search.Length < bytes.Length
                && bytes.AsSpan(i, search.Length).SequenceEqual(search)
            )
                start = i;
        }
    }

    public static bool StartsWithCommand(ref byte[] bytes)
    {
        var search = "<at:"u8;
        if (bytes.Length <= search.Length)
            return false;

        bool needBuild;
        lock (EntriesLock)
            needBuild = ValidEntries.Count == 0;
        if (needBuild)
            AllEntries();

        for (var i = 0; i < search.Length; i++)
        {
            if (bytes[i] != search[i])
                return false;
        }

        for (var i = 0; i < bytes.Length; i++)
        {
            if (bytes[i] != '>')
                continue;

            var tag = Encoding.UTF8.GetString(bytes[..(i + 1)]);
            var parts = tag[4..^1].Split(',', 2);
            if (
                parts.Length == 2
                && uint.TryParse(parts[0], out var group)
                && uint.TryParse(parts[1], out var key)
            )
            {
                bool isValid;
                lock (EntriesLock)
                    isValid = ValidEntries.Contains((group, key));
                if (!isValid)
                    return false;

                var evaluated = Plugin
                    .Evaluator.Evaluate(new ReadOnlySeString(CreateFixedTranslation(group, key)))
                    .ToString();
                if (!evaluated.StartsWith('/'))
                    return false;

                bytes = Encoding.UTF8.GetBytes(evaluated);
                return true;
            }
        }

        return false;
    }

    private static byte[] CreateFixedTranslation(uint group, uint key)
    {
        using var rssb = new RentedSeStringBuilder();
        return rssb
            .Builder.BeginMacro(MacroCode.Fixed)
            .AppendUIntExpression(group - 1)
            .AppendUIntExpression(key)
            .EndMacro()
            .ToArray();
    }
}

internal interface ISelectorPart { }

internal class SingleRow(uint row) : ISelectorPart
{
    public uint Row { get; } = row;
}

internal class IndexRange(uint start, uint end) : ISelectorPart
{
    public uint Start { get; } = start;
    public uint End { get; } = end;
}

internal class NounMarker : ISelectorPart { }

internal class ColumnSpecifier(uint column) : ISelectorPart
{
    public uint Column { get; } = column;
}

internal class AutoTranslateEntry(uint group, uint row, string str, string title)
{
    internal uint Group { get; } = group;
    internal uint Row { get; } = row;
    internal string Text { get; } = str;
    internal string Title { get; } = title;
}