HellionChat/HellionChat/Ui/DbViewer.cs

using System.Collections.Concurrent;
using System.Globalization;
using System.Numerics;
using System.Text;
using System.Threading;
using Dalamud.Bindings.ImGui;
using Dalamud.Game.Text.SeStringHandling.Payloads;
using Dalamud.Interface;
using Dalamud.Interface.Colors;
using Dalamud.Interface.Components;
using Dalamud.Interface.ImGuiNotification;
using Dalamud.Interface.Utility;
using Dalamud.Interface.Utility.Raii;
using Dalamud.Interface.Windowing;
using HellionChat.Code;
using HellionChat.Resources;
using HellionChat.Util;
using Lumina.Data.Files;
using Lumina.Text.ReadOnly;
using Microsoft.Extensions.Logging;
using MoreLinq;

namespace HellionChat.Ui;

public class DbViewer : Window
{
    public const int RowPerPage = 1000;

    private readonly Plugin Plugin;

    private static readonly DateTime MinimalDate = new(2021, 1, 1);

    private DateTime AfterDate;
    private DateTime BeforeDate;

    private int CurrentPage = 1;
    private string SimpleSearchTerm = "";

    // v1.4.8 H2: opt-in full-text search across the whole DB via FTS5.
    // Transient UI state (per-session), not persisted -- users opt in fresh
    // every time so they always see the page-filter as the default mode.
    private bool UseFullTextSearch;

    private bool OnlyCurrentCharacter = true;
    private readonly Dictionary<ChatType, (ChatSource, ChatSource)> SelectedChannels;

    private bool IsProcessing;
    private long ProcessingStart = Environment.TickCount64;

    // Bumped per trigger so a late worker drops itself instead of overwriting
    // a newer result.
    private long _ftsFilterSeq;
    private (DateTime Min, DateTime Max, int Page, bool Local, int ChannelCount) LastProcessed;

    private string MinDateString = "";
    private string MaxDateString = "";

    private readonly string DateFormat;
    private readonly string DateTimeFormat;

    private long Count;
    private Message[] Messages = []; // Messages are only touched while processing is false
    private ConcurrentStack<Message> Filtered = []; // Is used every frame, so ConcurrentStack for safety

    private bool IsExporting;
    private string InputPath = string.Empty;
    private IActiveNotification Notification = null!;

    private bool NeedsScrollReset;

    private readonly ILogger<DbViewer> _logger;

    public DbViewer(Plugin plugin, ILogger<DbViewer> logger)
        : base("DBViewer###chat2-dbviewer")
    {
        Plugin = plugin;
        _logger = logger;
        SelectedChannels = TabsUtil.MostlyPlayer;

        DateFormat = CultureInfo.CurrentCulture.DateTimeFormat.ShortDatePattern;
        DateTimeFormat = "ddd, dd MMM yyy HH:mm:ss";

        LastProcessed = (
            AfterDate,
            BeforeDate,
            CurrentPage,
            OnlyCurrentCharacter,
            SelectedChannels.Count
        );
        DateReset();

        SizeConstraints = new WindowSizeConstraints
        {
            MinimumSize = new Vector2(475, 600),
            MaximumSize = new Vector2(float.MaxValue, float.MaxValue),
        };

        RespectCloseHotkey = false;
        DisableWindowSounds = true;
    }

    public void Dispose()
    {
        // Slash-command tear-down moved to Plugin.TearDownCommands.
    }

    public override void Draw()
    {
        var totalPages = (int)Math.Ceiling((double)Count / RowPerPage);
        if (totalPages < 1)
            totalPages = 1;

        if (CurrentPage > totalPages)
            CurrentPage = 1;

        // First row

        ImGui.AlignTextToFramePadding();
        ImGui.TextColored(ImGuiColors.DalamudViolet, Language.DbViewer_DatePicker_FromTo);
        ImGui.SameLine();

        var spacing = 3.0f * ImGuiHelpers.GlobalScale;
        DateWidget.DatePickerWithInput(
            "##FromDate",
            1,
            ref MinDateString,
            ref AfterDate,
            DateFormat
        );
        DateWidget.DatePickerWithInput(
            "##ToDate",
            2,
            ref MaxDateString,
            ref BeforeDate,
            DateFormat,
            true
        );

        ImGui.SameLine(0, spacing);

        if (ImGuiUtil.IconButton(FontAwesomeIcon.Recycle))
            DateReset();

        if (ImGui.IsItemHovered())
            ImGui.SetTooltip(Language.DbViewer_Date_Reset_Tooltip);

        ImGui.SameLine(0, spacing);

        ChannelSelection();

        var skipText = Language.DbViewer_CharacterOption;
        var textLength =
            ImGui.GetTextLineHeight()
            + ImGui.CalcTextSize(skipText).X
            + ImGui.GetStyle().ItemInnerSpacing.X
            + ImGui.GetStyle().FramePadding.X * 2;
        ImGui.SameLine(ImGui.GetContentRegionMax().X - textLength);
        ImGui.Checkbox(skipText, ref OnlyCurrentCharacter);

        // Second row

        ImGui.AlignTextToFramePadding();
        ImGui.TextColored(ImGuiColors.DalamudViolet, "Export:");
        ImGui.SameLine(0, spacing);
        ImGui.SetNextItemWidth(ImGui.GetWindowWidth() * 0.4f);
        ImGui.InputText("##InputPath", ref InputPath, 255);
        ImGui.SameLine(0, spacing);
        if (ImGuiUtil.IconButton(FontAwesomeIcon.FolderClosed, "##folderPicker"))
            ImGui.OpenPopup("InputPathDialog");

        if (ImGui.IsItemHovered(ImGuiHoveredFlags.AllowWhenDisabled))
            ImGui.SetTooltip(Language.Folder_Export_Location_Tooltip);

        using (var innerPopup = ImRaii.Popup("InputPathDialog"))
        {
            if (innerPopup.Success)
                Plugin.FileDialogManager.OpenFolderDialog(
                    Language.Folder_Selection_Header,
                    (b, s) =>
                    {
                        if (b)
                            InputPath = s;
                    },
                    null,
                    true
                );
        }

        ImGui.SameLine(0, spacing);
        using (ImRaii.Disabled(InputPath.Length == 0 || IsExporting))
        {
            if (ImGuiUtil.IconButton(FontAwesomeIcon.Save))
            {
                Notification = Plugin.Notification.AddNotification(
                    new Notification
                    {
                        Title = "Chat2 Text Export",
                        Content = Language.ChatExport_Initial,
                        Type = NotificationType.Info,
                        Minimized = false,
                        UserDismissable = false,
                        InitialDuration = TimeSpan.FromSeconds(10000),
                        Progress = 0.0f,
                    }
                );
                CreateTxtBackup();
            }
        }

        if (ImGui.IsItemHovered(ImGuiHoveredFlags.AllowWhenDisabled))
            ImGui.SetTooltip(Language.Export_Txt_Tooltip);

        var width = 350 * ImGuiHelpers.GlobalScale;
        var loadingIndicator = IsProcessing && ProcessingStart < Environment.TickCount64;

        ImGui.AlignTextToFramePadding();
        ImGui.TextUnformatted(
            string.Format(
                Language.DbViewer_Page,
                CurrentPage,
                totalPages,
                Count,
                loadingIndicator ? Language.DbViewer_LoadingIndicator : ""
            )
        );
        ImGuiUtil.DrawArrows(
            ref CurrentPage,
            1,
            totalPages,
            spacing,
            tooltipLeft: Language.Page_ArrowLeft_Tooltip,
            tooltipRight: Language.Page_ArrowRight_Tooltip
        );

        // Full-text search toggle (v1.4.8 H2). IsFtsIndexBuilt is a cached
        // volatile bool in MessageStore -- single field read per frame, no
        // SELECT count(*). ImRaii.Disabled blocks any click while the index
        // is still being built, so no defensive force-off branch needed
        // inside the if-body. UseFullTextSearch is transient UI state, so we
        // do not call SaveConfig here.
        var ftsReady = Plugin.MessageManager.Store.IsFtsIndexBuilt;
        using (ImRaii.Disabled(!ftsReady))
        {
            if (ImGui.Checkbox(HellionStrings.DbViewer_FullTextToggle, ref UseFullTextSearch))
                TriggerFilterRefresh();
        }
        ImGuiUtil.HelpMarker(
            ftsReady
                ? HellionStrings.DbViewer_FullTextToggle_Hint_PhraseMode
                : HellionStrings.DbViewer_FullTextToggle_Hint_Indexing
        );

        ImGui.SameLine(ImGui.GetContentRegionMax().X - width);
        ImGui.SetNextItemWidth(width);
        if (
            ImGui.InputTextWithHint(
                "##searchbar",
                Language.DbViewer_SearcHint,
                ref SimpleSearchTerm,
                30
            )
        )
            TriggerFilterRefresh();

        // Third row

        if (DateWidget.Validate(MinimalDate, ref AfterDate, ref BeforeDate))
            DateRefresh();

        if (
            !IsProcessing
            && LastProcessed
                != (
                    AfterDate,
                    BeforeDate,
                    CurrentPage,
                    OnlyCurrentCharacter,
                    SelectedChannels.Count
                )
        )
        {
            // Page hasn't changed, so we reset it back to 1
            if (LastProcessed.Page == CurrentPage)
                CurrentPage = 1;

            AdjustDates();
            IsProcessing = true;
            ProcessingStart = Environment.TickCount64 + 1_000; // + 1 second
            LastProcessed = (
                AfterDate,
                BeforeDate,
                CurrentPage,
                OnlyCurrentCharacter,
                SelectedChannels.Count
            );
            Task.Run(() =>
            {
                try
                {
                    ulong? character = OnlyCurrentCharacter ? Plugin.PlayerState.ContentId : null;
                    var channels = SelectedChannels.Select(pair => (byte)pair.Key).ToArray();

                    // We only want to fetch count if this is the first page
                    if (CurrentPage == 1)
                        Count = Plugin.MessageManager.Store.CountDateRange(
                            AfterDate,
                            BeforeDate,
                            channels,
                            character
                        );

                    using var rangeMessageEnumerator =
                        Plugin.MessageManager.Store.GetPagedDateRange(
                            AfterDate,
                            BeforeDate,
                            channels,
                            character,
                            CurrentPage - 1
                        );
                    Messages = rangeMessageEnumerator.ToArray();

                    Filtered = Filter(Messages);
                    NeedsScrollReset = true;
                }
                catch (Exception ex)
                {
                    _logger.LogError(ex, "Failed reading messages from database");
                }
                finally
                {
                    IsProcessing = false;
                }
            });
        }

        ImGuiHelpers.ScaledDummy(5.0f);

        if (Filtered.IsEmpty)
        {
            ImGui.TextUnformatted(
                SimpleSearchTerm == ""
                    ? Language.DbViewer_Status_NothingFound
                    : Language.DbViewer_Status_NoSearchResult
            );
            return;
        }

        using var child = ImRaii.Child("##tableChild");
        if (!child.Success)
            return;

        if (NeedsScrollReset)
        {
            NeedsScrollReset = false;
            ImGui.SetScrollY(0.0f);
        }

        using var table = ImRaii.Table(
            "##messageHistory",
            4,
            ImGuiTableFlags.Borders | ImGuiTableFlags.RowBg | ImGuiTableFlags.Resizable
        );
        if (!table.Success)
            return;

        var columnWidth = ImGui.CalcTextSize(Language.DbViewer_TableField_Type);
        ImGui.TableSetupColumn(
            Language.DbViewer_TableField_Date,
            ImGuiTableColumnFlags.WidthFixed | ImGuiTableColumnFlags.NoResize
        );
        ImGui.TableSetupColumn(
            Language.DbViewer_TableField_Type,
            ImGuiTableColumnFlags.WidthFixed | ImGuiTableColumnFlags.NoResize,
            columnWidth.X
        );
        ImGui.TableSetupColumn(Language.DbViewer_TableField_Sender);
        ImGui.TableSetupColumn(Language.DbViewer_TableField_Content);

        ImGui.TableHeadersRow();
        foreach (var message in Filtered)
        {
            ImGui.TableNextColumn();
            ImGui.TextUnformatted(message.Date.ToLocalTime().ToString(DateTimeFormat));

            ImGui.TableNextColumn();
            var pos = ImGui.GetCursorPos();
            ImGuiUtil.CenterText($"{(byte)message.Code.Type}");
            ImGui.SetCursorPos(pos);
            ImGui.Dummy(columnWidth);
            if (ImGui.IsItemHovered())
                ImGuiUtil.Tooltip(message.Code.Type.Name());

            ImGui.TableNextColumn();
            Plugin.ChatLogWindow.DrawChunks(message.Sender);

            ImGui.TableNextColumn();
            Plugin.ChatLogWindow.DrawChunks(message.Content);
        }
    }

    private void ChannelSelection()
    {
        const string addTabPopup = "add-channel-popup";
        var spacing = 3.0f * ImGuiHelpers.GlobalScale;

        if (ImGui.Button("Channels"))
            ImGui.OpenPopup(addTabPopup);

        using var popup = ImRaii.Popup(addTabPopup);
        if (!popup.Success)
            return;

        using var channelNode = ImRaii.TreeNode(Language.Options_Tabs_Channels);
        if (!channelNode.Success)
            return;

        foreach (var (header, types) in ChatTypeExt.SortOrder)
        {
            using var pushedId = ImRaii.PushId(header);

            if (ImGuiComponents.IconButton(FontAwesomeIcon.Check))
            {
                foreach (var type in types)
                    SelectedChannels.TryAdd(type, (ChatSourceExt.All, ChatSourceExt.All));
            }

            if (ImGui.IsItemHovered())
                ImGui.SetTooltip("Select all");

            ImGui.SameLine(0, spacing);

            if (ImGuiComponents.IconButton(FontAwesomeIcon.Times))
            {
                foreach (var type in types)
                    SelectedChannels.Remove(type);
            }

            if (ImGui.IsItemHovered())
                ImGui.SetTooltip("Unselect all");

            ImGui.SameLine(0, spacing);

            using var headerNode = ImRaii.TreeNode(header);
            if (!headerNode.Success)
                continue;

            foreach (var type in types)
            {
                if (type.IsGm())
                    continue;

                var enabled = SelectedChannels.ContainsKey(type);
                if (ImGui.Checkbox($"##{type.Name()}", ref enabled))
                {
                    if (enabled)
                        SelectedChannels[type] = (ChatSourceExt.All, ChatSourceExt.All);
                    else
                        SelectedChannels.Remove(type);
                }

                ImGui.SameLine();
                ImGui.TextUnformatted(type.Name());
            }
        }
    }

    // FTS path hits SQLite per keystroke -- dispatch to a worker, drop stale
    // results via _ftsFilterSeq. Page-filter path is in-memory LINQ, stays
    // inline.
    private void TriggerFilterRefresh()
    {
        if (!UseFullTextSearch || !Plugin.MessageManager.Store.IsFtsIndexBuilt)
        {
            Filtered = Filter(Messages);
            return;
        }

        var snapshot = Messages;
        var mySeq = Interlocked.Increment(ref _ftsFilterSeq);
        Task.Run(() =>
        {
            try
            {
                var result = Filter(snapshot);
                if (Interlocked.Read(ref _ftsFilterSeq) == mySeq)
                    Filtered = result;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "FTS filter worker failed");
            }
        });
    }

    private ConcurrentStack<Message> Filter(Message[] messages)
    {
        if (SimpleSearchTerm == "")
            return new ConcurrentStack<Message>(messages.Reverse().OrderByDescending(m => m.Date));

        // Full-text mode bypasses the page-bounded messages array and queries
        // the FTS5 index across the whole DB. IsFtsIndexBuilt re-check guards
        // against the (rare) case of the toggle being on while the index is
        // mid-rebuild -- ImRaii.Disabled prevents the user from flipping it,
        // but a Dispose-and-reopen during indexing could leave UseFullTextSearch
        // true while ftsReady flipped back to false; the local fallback below
        // still serves the page.
        if (UseFullTextSearch && Plugin.MessageManager.Store.IsFtsIndexBuilt)
        {
            var guidHits = Plugin.MessageManager.Store.FullTextSearch(SimpleSearchTerm);
            var resolved = Plugin.MessageManager.Store.LoadByGuids(guidHits);
            return new ConcurrentStack<Message>(resolved.OrderByDescending(m => m.Date));
        }

        return new ConcurrentStack<Message>(
            messages
                .Reverse()
                .Where(m =>
                    ChunkUtil
                        .ToRawString(m.Sender)
                        .Contains(SimpleSearchTerm, StringComparison.InvariantCultureIgnoreCase)
                    || ChunkUtil
                        .ToRawString(m.Content)
                        .Contains(SimpleSearchTerm, StringComparison.InvariantCultureIgnoreCase)
                )
                .OrderByDescending(m => m.Date)
        );
    }

    private void DateRefresh()
    {
        MinDateString = AfterDate.ToString(DateFormat);
        MaxDateString = BeforeDate.ToString(DateFormat);
    }

    private void AdjustDates()
    {
        AfterDate = new DateTime(AfterDate.Year, AfterDate.Month, AfterDate.Day, 0, 0, 0);
        BeforeDate = new DateTime(BeforeDate.Year, BeforeDate.Month, BeforeDate.Day, 23, 59, 59);
    }

    private void DateReset()
    {
        AfterDate = DateTime.Now.AddDays(-5);
        BeforeDate = DateTime.Now;

        AdjustDates();
        DateRefresh();
    }

    private void CreateTxtBackup()
    {
        IsExporting = true;
        Task.Run(async () =>
        {
            try
            {
                ulong? character = OnlyCurrentCharacter ? Plugin.PlayerState.ContentId : null;
                var channels = SelectedChannels.Select(pair => (byte)pair.Key).ToArray();

                var rangeMessageEnumerator = Plugin.MessageManager.Store.GetDateRange(
                    AfterDate,
                    BeforeDate,
                    channels,
                    character
                );
                var messageHistory = rangeMessageEnumerator.ToArray();
                await rangeMessageEnumerator.DisposeAsync();

                var filteredHistory = Filter(messageHistory);
                // Materialize Count once — re-enumerating the IEnumerable on
                // every batch (twice per batch in the Notification update)
                // turned the export into an O(N²) hot loop on large histories.
                var totalCount = filteredHistory.Count;

                var sb = new StringBuilder();
                await using var stream = new StreamWriter(
                    Path.Join(InputPath, $"Chat2_{DateTime.Now:yyyy_dd_M__HH_mm_ss}.txt")
                );

                var batch = 0;
                foreach (var messages in filteredHistory.Batch(5000))
                {
                    await Plugin.Framework.RunOnTick(
                        () =>
                        {
                            foreach (var message in messages)
                            {
                                if (
                                    !Sheets.LogKindSheet.TryGetRow(
                                        (uint)message.Code.Type,
                                        out var logKind
                                    )
                                )
                                    logKind = Sheets.LogKindSheet.GetRow(10); // default to say

                                var rossSender = new ReadOnlySeString(
                                    message.SenderSource.Encode()
                                );
                                var rossMessage = new ReadOnlySeString(
                                    message.ContentSource.Encode()
                                );

                                var timestamp = message.Date.ToLocalTime().ToString(DateTimeFormat);
                                var text = Plugin
                                    .Evaluator.Evaluate(logKind.Format, [rossSender, rossMessage])
                                    .ToString();
                                sb.AppendLine(
                                    $"[{timestamp}][{message.Code.Type.Name()}]   {text}"
                                );

                                batch++;
                            }
                        },
                        delayTicks: 5
                    );

                    Notification.Progress = (float)batch / totalCount;
                    Notification.Content = $"Exported {batch} of {totalCount} messages";
                    await stream.WriteAsync(sb.ToString());
                    sb.Clear();
                }

                await stream.WriteAsync(sb.ToString());
                sb.Clear();

                Notification.Progress = 1.0f;
                Notification.Content = "Done!!!";
                Notification.Type = NotificationType.Success;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed creating txt backup");

                Notification.Content = "Error ...";
                Notification.Type = NotificationType.Error;
            }
            finally
            {
                IsExporting = false;
                Notification.UserDismissable = true;
            }
        });
    }
}