- Implement message history export to text file

ChatTwo.Tests/ChatTwo.Tests.csproj

@@ -9,6 +9,7 @@
   <ItemGroup>
     <PackageReference Include="JetBrains.Annotations" Version="2025.2.2" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
+    <PackageReference Include="morelinq" Version="4.4.0" />
     <PackageReference Include="MSTest.TestAdapter" Version="3.6.3" />
     <PackageReference Include="MSTest.TestFramework" Version="3.6.3" />
   </ItemGroup>

ChatTwo/ChatTwo.csproj

@@ -15,6 +15,7 @@
     <ItemGroup>
         <PackageReference Include="MessagePack" Version="3.1.4" />
         <PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.0" />
+        <PackageReference Include="morelinq" Version="4.4.0" />
         <PackageReference Include="Pidgin" Version="3.3.0" />
         <PackageReference Include="SixLabors.ImageSharp" Version="3.1.12" />
         <PackageReference Include="Watson.Lite" Version="6.3.9" />

ChatTwo/MessageStore.cs

@@ -387,7 +387,7 @@ internal class MessageStore : IDisposable
         cmd.ExecuteNonQuery();
     }
 
-    internal long CountDateRange(DateTime after, DateTime before, uint[] channels, ulong? receiver = null)
+    internal long CountDateRange(DateTime after, DateTime before, IEnumerable<uint> channels, ulong? receiver = null)
     {
         List<string> whereClauses = ["deleted = false"];
         if (receiver != null)
@@ -416,7 +416,47 @@ internal class MessageStore : IDisposable
         return (long) cmd.ExecuteScalar()!;
     }
 
-    internal MessageEnumerator GetDateRange(DateTime after, DateTime before, uint[] channels, ulong? receiver = null, int page = 0)
+    internal MessageEnumerator GetDateRange(DateTime after, DateTime before, IEnumerable<uint> channels, ulong? receiver = null)
+    {
+        List<string> whereClauses = ["deleted = false"];
+        if (receiver != null)
+            whereClauses.Add("Receiver = $Receiver");
+
+        whereClauses.Add("Date BETWEEN $After AND $Before");
+        whereClauses.Add($"Channel IN ({string.Join(", ", channels)})");
+
+        var whereClause = $"WHERE {string.Join(" AND ", whereClauses)}";
+
+        var cmd = Connection.CreateCommand();
+        // Select last N messages by date DESC, but reverse the order to get
+        // them in ascending order.
+        cmd.CommandText = @"
+            SELECT
+                Id,
+                Receiver,
+                ContentId,
+                Date,
+                Code,
+                Sender,
+                Content,
+                SenderSource,
+                ContentSource,
+                SortCode,
+                ExtraChatChannel
+            FROM messages
+            " + whereClause;
+        cmd.CommandTimeout = 120; // this could take a while on slow computers
+
+        if (receiver != null)
+            cmd.Parameters.AddWithValue("$Receiver", receiver);
+
+        cmd.Parameters.AddWithValue("$After", ((DateTimeOffset) after).ToUnixTimeMilliseconds());
+        cmd.Parameters.AddWithValue("$Before", ((DateTimeOffset) before).ToUnixTimeMilliseconds());
+
+        return new MessageEnumerator(cmd.ExecuteReader());
+    }
+
+    internal MessageEnumerator GetPagedDateRange(DateTime after, DateTime before, IEnumerable<uint> channels, ulong? receiver = null, int page = 0)
     {
         List<string> whereClauses = ["deleted = false"];
         if (receiver != null)

ChatTwo/Plugin.cs

@@ -12,6 +12,7 @@ using Dalamud.IoC;
 using Dalamud.Plugin;
 using Dalamud.Plugin.Services;
 using Dalamud.Bindings.ImGui;
+using Dalamud.Interface.ImGuiFileDialog;
 
 namespace ChatTwo;
 
@@ -42,6 +43,7 @@ public sealed class Plugin : IDalamudPlugin
     [PluginService] internal static ISeStringEvaluator Evaluator { get; private set; } = null!;
 
     internal static Configuration Config = null!;
+    public static FileDialogManager FileDialogManager { get; private set; } = null!;
 
     public readonly WindowSystem WindowSystem = new(PluginName);
     public SettingsWindow SettingsWindow { get; }
@@ -93,6 +95,8 @@ public sealed class Plugin : IDalamudPlugin
             LanguageChanged(Interface.UiLanguage);
             ImGuiUtil.Initialize(this);
 
+            FileDialogManager = new FileDialogManager();
+
             // Functions calls this in its ctor if the player is already logged in
             ServerCore = new ServerCore(this);
 
@@ -216,6 +220,8 @@ public sealed class Plugin : IDalamudPlugin
 
         ChatLogWindow.FinalizeFrame();
         TypingIpc.Update();
+
+        FileDialogManager.Draw();
     }
 
     internal void SaveConfig()

ChatTwo/Ui/DbViewer.cs

@@ -1,6 +1,7 @@
 using System.Collections.Concurrent;
 using System.Globalization;
 using System.Numerics;
+using System.Text;
 using ChatTwo.Code;
 using ChatTwo.Resources;
 using ChatTwo.Util;
@@ -10,6 +11,9 @@ using Dalamud.Interface.Utility;
 using Dalamud.Interface.Utility.Raii;
 using Dalamud.Interface.Windowing;
 using Dalamud.Bindings.ImGui;
+using Dalamud.Interface.ImGuiNotification;
+using Lumina.Text.ReadOnly;
+using MoreLinq;
 
 namespace ChatTwo.Ui;
 
@@ -43,6 +47,10 @@ public class DbViewer : Window
     private Message[] Messages = [];  // Messages are only touched while processing is false
     private ConcurrentStack<Message> Filtered = [];  // Is used every frame, so ConcurrentStack for safety
 
+    private bool IsExporting;
+    private string InputPath = string.Empty;
+    private IActiveNotification Notification = null!;
+
     public DbViewer(Plugin plugin) : base("DBViewer###chat2-dbviewer")
     {
         Plugin = plugin;
@@ -63,12 +71,12 @@ public class DbViewer : Window
         RespectCloseHotkey = false;
         DisableWindowSounds = true;
 
-        Plugin.Commands.Register("/chat2Viewer", "Database Viewer", true).Execute += Toggle;
+        Plugin.Commands.Register("/chat2Viewer", "Get access to your message history, with simple filter options.", true).Execute += Toggle;
     }
 
     public void Dispose()
     {
-        Plugin.Commands.Register("/chat2Viewer", "Database Viewer", true).Execute -= Toggle;
+        Plugin.Commands.Register("/chat2Viewer", "Get access to your message history, with simple filter options.", true).Execute -= Toggle;
     }
 
     private void Toggle(string _, string __) => Toggle();
@@ -82,6 +90,8 @@ public class DbViewer : Window
         if (CurrentPage > totalPages)
             CurrentPage = 1;
 
+        // First row
+
         ImGui.AlignTextToFramePadding();
         ImGui.TextColored(ImGuiColors.DalamudViolet, Language.DbViewer_DatePicker_FromTo);
         ImGui.SameLine();
@@ -101,6 +111,49 @@ public class DbViewer : Window
         ImGui.SameLine(ImGui.GetContentRegionMax().X - textLength);
         ImGui.Checkbox(skipText, ref OnlyCurrentCharacter);
 
+        // Second row
+
+        ImGui.AlignTextToFramePadding();
+        ImGui.TextColored(ImGuiColors.DalamudViolet, "Export:");
+        ImGui.SameLine(0, spacing);
+        ImGui.SetNextItemWidth(ImGui.GetWindowWidth() * 0.4f);
+        ImGui.InputText("##InputPath", ref InputPath, 255);
+        ImGui.SameLine(0, spacing);
+        if (ImGuiUtil.IconButton(FontAwesomeIcon.FolderClosed, "##folderPicker"))
+            ImGui.OpenPopup("InputPathDialog");
+
+        if (ImGui.IsItemHovered(ImGuiHoveredFlags.AllowWhenDisabled))
+            ImGui.SetTooltip("Pick a folder location for export.");
+
+        using (var innerPopup = ImRaii.Popup("InputPathDialog"))
+        {
+            if (innerPopup.Success)
+                Plugin.FileDialogManager.OpenFolderDialog("Pick an export location", (b, s) => { if (b) InputPath = s; }, null, true);
+        }
+
+        ImGui.SameLine(0, spacing);
+        using (ImRaii.Disabled(InputPath.Length == 0 || IsExporting))
+        {
+            if (ImGuiUtil.IconButton(FontAwesomeIcon.Save))
+            {
+                Notification = Plugin.Notification.AddNotification(
+                    new Notification
+                    {
+                        Title = "Chat2 Export",
+                        Content = "Loading logs ...",
+                        Type = NotificationType.Info,
+                        Minimized = false,
+                        UserDismissable = false,
+                        InitialDuration = TimeSpan.FromSeconds(10000),
+                        Progress = 0.0f,
+                    });
+                CreateTxtBackup();
+            }
+        }
+
+        if (ImGui.IsItemHovered(ImGuiHoveredFlags.AllowWhenDisabled))
+            ImGui.SetTooltip("Export the message history to a text file.");
+
         var width = 350 * ImGuiHelpers.GlobalScale;
         var loadingIndicator = IsProcessing && ProcessingStart < Environment.TickCount64;
 
@@ -109,7 +162,9 @@ public class DbViewer : Window
         ImGui.SameLine(ImGui.GetContentRegionMax().X - width);
         ImGui.SetNextItemWidth(width);
         if (ImGui.InputTextWithHint("##searchbar", Language.DbViewer_SearcHint, ref SimpleSearchTerm, 30))
-            Filter();
+            Filtered = Filter(Messages);
+
+        // Third row
 
         if (DateWidget.Validate(MinimalDate, ref AfterDate, ref BeforeDate))
             DateRefresh();
@@ -135,10 +190,10 @@ public class DbViewer : Window
                     if (CurrentPage == 1)
                         Count = Plugin.MessageManager.Store.CountDateRange(AfterDate, BeforeDate, channels, character);
 
-                    using var dateRangeMessageEnumerator = Plugin.MessageManager.Store.GetDateRange(AfterDate, BeforeDate, channels, character, CurrentPage - 1);
+                    using var rangeMessageEnumerator = Plugin.MessageManager.Store.GetPagedDateRange(AfterDate, BeforeDate, channels, character, CurrentPage - 1);
-                    Messages = dateRangeMessageEnumerator.ToArray();
+                    Messages = rangeMessageEnumerator.ToArray();
 
-                    Filter();
+                    Filtered = Filter(Messages);
                 }
                 catch (Exception ex)
                 {
@@ -236,16 +291,13 @@ public class DbViewer : Window
         }
     }
 
-    private void Filter()
+    private ConcurrentStack<Message> Filter(Message[] messages)
     {
         if (SimpleSearchTerm == "")
-        {
+            return new ConcurrentStack<Message>(messages.Reverse().OrderByDescending(m => m.Date));
-            Filtered = new ConcurrentStack<Message>(Messages.Reverse().OrderByDescending(m => m.Date));
-            return;
-        }
 
-        Filtered = new ConcurrentStack<Message>(
+        return new ConcurrentStack<Message>(
-            Messages.Reverse().Where(m =>
+            messages.Reverse().Where(m =>
                 ChunkUtil.ToRawString(m.Sender).Contains(SimpleSearchTerm, StringComparison.InvariantCultureIgnoreCase) ||
                 ChunkUtil.ToRawString(m.Content).Contains(SimpleSearchTerm, StringComparison.InvariantCultureIgnoreCase)
                 ).OrderByDescending(m => m.Date));
@@ -271,4 +323,72 @@ public class DbViewer : Window
         AdjustDates();
         DateRefresh();
     }
+
+    private void CreateTxtBackup()
+    {
+        IsExporting = true;
+        Task.Run(async () =>
+        {
+            try
+            {
+                ulong? character = OnlyCurrentCharacter ? Plugin.PlayerState.ContentId : null;
+                var channels = ChatCodes.Select(c => (uint)c.Key).ToArray();
+
+                var rangeMessageEnumerator = Plugin.MessageManager.Store.GetDateRange(AfterDate, BeforeDate, channels, character);
+                var messageHistory = rangeMessageEnumerator.ToArray();
+                await rangeMessageEnumerator.DisposeAsync();
+
+                var filteredHistory = Filter(messageHistory);
+
+                var sb = new StringBuilder();
+                await using var stream = new StreamWriter(Path.Join(InputPath, $"Chat2_{DateTime.Now:yyyy_dd_M__HH_mm_ss}.txt"));
+
+                var batch = 0;
+                foreach (var messages in filteredHistory.Batch(5000))
+                {
+                    await Plugin.Framework.RunOnTick(() =>
+                    {
+                        foreach (var message in messages)
+                        {
+                            if (!Sheets.LogKindSheet.TryGetRow((uint)message.Code.Type, out var logKind))
+                                logKind = Sheets.LogKindSheet.GetRow(10); // default to say
+
+                            var rossSender = new ReadOnlySeString(message.SenderSource.Encode());
+                            var rossMessage = new ReadOnlySeString(message.ContentSource.Encode());
+
+                            var timestamp = message.Date.ToLocalTime().ToString(DateTimeFormat);
+                            var text = Plugin.Evaluator.Evaluate(logKind.Format, [rossSender, rossMessage]).ToString();
+                            sb.AppendLine($"[{timestamp}][{message.Code.Type.Name()}]   {text}");
+
+                            batch++;
+                        }
+                    }, delayTicks: 5);
+
+                    Notification.Progress = (float)batch / filteredHistory.Count;
+                    Notification.Content = $"Exported {batch} of {filteredHistory.Count} messages";
+                    await stream.WriteAsync(sb.ToString());
+                    sb.Clear();
+                }
+
+                await stream.WriteAsync(sb.ToString());
+                sb.Clear();
+
+                Notification.Progress = 1.0f;
+                Notification.Content = "Done!!!";
+                Notification.Type = NotificationType.Success;
+            }
+            catch (Exception ex)
+            {
+                Plugin.Log.Error(ex, "Failed creating txt backup");
+
+                Notification.Content = "Error ...";
+                Notification.Type = NotificationType.Error;
+            }
+            finally
+            {
+                IsExporting = false;
+                Notification.UserDismissable = true;
+            }
+        });
+    }
 }

ChatTwo/packages.lock.json

@@ -36,6 +36,12 @@
           "SQLitePCLRaw.core": "2.1.10"
         }
       },
+      "morelinq": {
+        "type": "Direct",
+        "requested": "[4.4.0, )",
+        "resolved": "4.4.0",
+        "contentHash": "QX3bsK9oFeUXk8tFsc9NkI6NnCr8Ar/ex027p+ZZ/jdLCdX2RlryDtxUqZW5j45NVwn4E4Z4hzupsoMQd6Yxtg=="
+      },
       "Pidgin": {
         "type": "Direct",
         "requested": "[3.3.0, )",