Merge pull request #37

* chore: use threads for DB reading and writing

ChatTwo/Code/ChatCode.cs

@@ -92,6 +92,9 @@ internal class ChatCode
     {
         switch (Type)
         {
+            // Error isn't a battle message, but it can be just as spammy if you
+            // use macros with unavailable actions.
+            case ChatType.Error:
             case ChatType.Damage:
             case ChatType.Miss:
             case ChatType.Action:

ChatTwo/LegacyMessageImporter.cs

@@ -166,11 +166,6 @@ internal class LegacyMessageImporter : IAsyncDisposable
         WorkingThread.Start();
     }
 
-    public void Dispose()
-    {
-        _database?.Dispose();
-    }
-
     public async ValueTask DisposeAsync()
     {
         await CancellationToken.CancelAsync();
@@ -339,8 +334,7 @@ internal class LegacyMessageImporter : IAsyncDisposable
         _database.Dispose();
         _database = null;
 
-        if (Plugin != null)
+        Plugin?.MessageManager.FilterAllTabsAsync(false);
-            Plugin.Framework.Run(() => Plugin.MessageManager.FilterAllTabs(false), token);
     }
 
     private static Message BsonDocumentToMessage(BsonDocument doc)

ChatTwo/Message.cs

@@ -70,8 +70,9 @@ internal class Message {
     internal Dictionary<Guid, float?> Height { get; } = new();
     internal Dictionary<Guid, bool> IsVisible { get; } = new();
 
-    internal Message(ulong receiver, ChatCode code, List<Chunk> sender, List<Chunk> content, SeString senderSource, SeString contentSource) {
+    internal Message(ulong receiver, ulong contentId, ChatCode code, List<Chunk> sender, List<Chunk> content, SeString senderSource, SeString contentSource) {
         Receiver = receiver;
+        ContentId = contentId;
         Date = DateTimeOffset.UtcNow;
         Code = code;
         Sender = sender;

ChatTwo/MessageManager.cs

@@ -11,17 +11,22 @@ using Lumina.Excel.GeneratedSheets;
 
 namespace ChatTwo;
 
-internal class MessageManager : IDisposable
+internal class MessageManager : IAsyncDisposable
 {
     internal const int MessageDisplayLimit = 10_000;
 
     private Plugin Plugin { get; }
     internal MessageStore Store { get; }
 
-    private ConcurrentQueue<(uint, Message)> Pending { get; } = new();
     private Dictionary<ChatType, NameFormatting> Formats { get; } = new();
     private ulong LastContentId { get; set; }
 
+    private ConcurrentQueue<PendingMessage> Pending { get; } = new();
+    private ulong LastMessageIndex { get; set; }
+
+    private readonly Thread PendingMessageThread;
+    private readonly CancellationTokenSource PendingThreadCancellationToken = new();
+
     internal ulong CurrentContentId
     {
         get
@@ -36,19 +41,32 @@ internal class MessageManager : IDisposable
         Plugin = plugin;
         Store = new MessageStore(DatabasePath());
 
+        PendingMessageThread = new Thread(() => ProcessPendingMessages(PendingThreadCancellationToken.Token));
+        PendingMessageThread.Start();
+
         Plugin.ChatGui.ChatMessageUnhandled += ChatMessage;
-        Plugin.Framework.Update += GetMessageInfo;
         Plugin.Framework.Update += UpdateReceiver;
         Plugin.ClientState.Logout += Logout;
     }
 
-    public void Dispose()
+    public async ValueTask DisposeAsync()
     {
         Plugin.ClientState.Logout -= Logout;
         Plugin.Framework.Update -= UpdateReceiver;
-        Plugin.Framework.Update -= GetMessageInfo;
         Plugin.ChatGui.ChatMessageUnhandled -= ChatMessage;
 
+        await PendingThreadCancellationToken.CancelAsync();
+        var timeout = 10_000; // 10s
+        while (timeout > 0)
+        {
+            if (!PendingMessageThread.IsAlive)
+                break;
+
+            timeout -= 100;
+            await Task.Delay(100);
+            Plugin.Log.Debug("Sleeping because PendingMessageThread thread still alive");
+        }
+
         Store.Dispose();
     }
 
@@ -69,30 +87,28 @@ internal class MessageManager : IDisposable
             LastContentId = contentId;
     }
 
-    private void GetMessageInfo(IFramework framework)
+    private void ProcessPendingMessages(CancellationToken token)
     {
-        if (!Pending.TryDequeue(out var entry))
+        while (!token.IsCancellationRequested)
-            return;
+        {
-
+            if (Pending.TryDequeue(out var pendingMessage))
-        var contentId = Plugin.Functions.Chat.GetContentIdForEntry(entry.Item1);
+                try
-        entry.Item2.ContentId = contentId ?? 0;
+                {
-        if (Plugin.Config.DatabaseBattleMessages || !entry.Item2.Code.IsBattle())
+                    ProcessMessage(pendingMessage);
-            Store.UpsertMessage(entry.Item2);
+                }
+                catch (Exception ex)
+                {
+                    Plugin.Log.Error(ex, "Error processing pending message");
+                }
+            else
+                Thread.Sleep(1);
+        }
     }
 
-    private void AddMessage(Message message, Tab? currentTab)
+    internal void ClearAllTabs()
     {
-        if (Plugin.Config.DatabaseBattleMessages || !message.Code.IsBattle())
-            Store.UpsertMessage(message);
-
-        var currentMatches = currentTab?.Matches(message) ?? false;
         foreach (var tab in Plugin.Config.Tabs)
-        {
+            tab.Clear();
-            var unread = !(tab.UnreadMode == UnreadMode.Unseen && currentTab != tab && currentMatches);
-
-            if (tab.Matches(message))
-                tab.AddMessage(message, unread);
-        }
     }
 
     internal void FilterAllTabs(bool unread = true)
@@ -109,17 +125,68 @@ internal class MessageManager : IDisposable
         if (messages.DidError)
             WrapperUtil.AddNotification(Language.LoadMessages_Error, NotificationType.Error);
     }
+    internal void FilterAllTabsAsync(bool unread = true)
+    {
+        Task.Run(() =>
+        {
+            var stopwatch = Stopwatch.StartNew();
+            try
+            {
+                FilterAllTabs(unread);
+            }
+            catch (Exception ex)
+            {
+                Plugin.Log.Error(ex, "Error in FilterAllTabs");
+            }
+
+            Plugin.Log.Debug($"FilterAllTabs took {stopwatch.ElapsedMilliseconds}ms");
+        });
+    }
 
     public (SeString? Sender, SeString? Message) LastMessage = (null, null);
     private void ChatMessage(XivChatType type, uint senderId, SeString sender, SeString message)
     {
-        var chatCode = new ChatCode((ushort)type);
+        LastMessage = (sender, message);
+        var pendingMessage = new PendingMessage
+        {
+            ReceiverId = CurrentContentId,
+            ContentId = 0,
+            Type = type,
+            SenderId = senderId,
+            Sender = sender,
+            Content = message,
+        };
+
+        // If the message was rendered in the vanilla chat log window it has an
+        // index, and we can use that to get the sender's content ID. The
+        // content ID is used to show "invite to party" buttons in the context
+        // menu.
+        var idx = Plugin.Functions.GetCurrentChatLogEntryIndex() ?? 0;
+        if (idx > LastMessageIndex)
+        {
+            LastMessageIndex = idx;
+            // You can't call GetContentIdForEntry in the same framework tick
+            // that you received the message, or you just get null.
+            Plugin.Framework.RunOnTick(() =>
+            {
+                var contentId = Plugin.Functions.Chat.GetContentIdForEntry(idx - 1);
+                pendingMessage.ContentId = contentId ?? 0;
+                Pending.Enqueue(pendingMessage);
+            });
+            return;
+        }
+
+        Pending.Enqueue(pendingMessage);
+    }
+
+    private void ProcessMessage(PendingMessage pendingMessage)
+    {
+        var chatCode = new ChatCode((ushort)pendingMessage.Type);
 
         NameFormatting? formatting = null;
-        if (sender.Payloads.Count > 0)
+        if (pendingMessage.Sender.Payloads.Count > 0)
             formatting = FormatFor(chatCode.Type);
 
-        LastMessage = (sender, message);
         var senderChunks = new List<Chunk>();
         if (formatting is { IsPresent: true })
         {
@@ -127,21 +194,29 @@ internal class MessageManager : IDisposable
             {
                 FallbackColour = chatCode.Type,
             });
-            senderChunks.AddRange(ChunkUtil.ToChunks(sender, ChunkSource.Sender, chatCode.Type));
+            senderChunks.AddRange(ChunkUtil.ToChunks(pendingMessage.Sender, ChunkSource.Sender, chatCode.Type));
             senderChunks.Add(new TextChunk(ChunkSource.None, null, formatting.After)
             {
                 FallbackColour = chatCode.Type,
             });
         }
 
-        var messageChunks = ChunkUtil.ToChunks(message, ChunkSource.Content, chatCode.Type).ToList();
+        var contentChunks = ChunkUtil.ToChunks(pendingMessage.Content, ChunkSource.Content, chatCode.Type).ToList();
 
-        var msg = new Message(CurrentContentId, chatCode, senderChunks, messageChunks, sender, message);
+        var msg = new Message(pendingMessage.ReceiverId, pendingMessage.ContentId, chatCode, senderChunks, contentChunks, pendingMessage.Sender, pendingMessage.Content);
-        AddMessage(msg, Plugin.ChatLogWindow.CurrentTab ?? null);
 
-        var idx = Plugin.Functions.GetCurrentChatLogEntryIndex();
+        if (Plugin.Config.DatabaseBattleMessages || !msg.Code.IsBattle())
-        if (idx != null)
+            Store.UpsertMessage(msg);
-            Pending.Enqueue((idx.Value - 1, msg));
+
+        var currentMatches = Plugin.ChatLogWindow.CurrentTab?.Matches(msg) ?? false;
+
+        foreach (var tab in Plugin.Config.Tabs)
+        {
+            var unread = !(tab.UnreadMode == UnreadMode.Unseen && Plugin.ChatLogWindow.CurrentTab != tab && currentMatches);
+
+            if (tab.Matches(msg))
+                tab.AddMessage(msg, unread);
+        }
     }
 
     internal class NameFormatting
@@ -208,4 +283,14 @@ internal class MessageManager : IDisposable
 
         return nameFormatting;
     }
+
+    private class PendingMessage
+    {
+        internal ulong ReceiverId { get; set; }
+        internal ulong ContentId { get; set; } // 0 if unknown
+        internal XivChatType Type { get; set; }
+        internal uint SenderId { get; set; }
+        internal SeString Sender { get; set; }
+        internal SeString Content { get; set; }
+    }
 }

ChatTwo/Plugin.cs

@@ -113,7 +113,7 @@ public sealed class Plugin : IDalamudPlugin
             Commands.Initialise();
 
             if (Interface.Reason is not PluginLoadReason.Boot) {
-                MessageManager.FilterAllTabs(false);
+                MessageManager.FilterAllTabsAsync(false);
             }
 
             Framework.Update += FrameworkUpdate;
@@ -154,7 +154,7 @@ public sealed class Plugin : IDalamudPlugin
 
         ExtraChat?.Dispose();
         Ipc?.Dispose();
-        MessageManager?.Dispose();
+        MessageManager?.DisposeAsync().AsTask().Wait();
         Functions?.Dispose();
         TextureCache?.Dispose();
         Common?.Dispose();

ChatTwo/Ui/ChatLogWindow.cs

@@ -145,13 +145,12 @@ public sealed class ChatLogWindow : Window
 
     private void Logout()
     {
-        foreach (var tab in Plugin.Config.Tabs)
+        Plugin.MessageManager.ClearAllTabs();
-            tab.Clear();
     }
 
     private void Login()
     {
-        Plugin.MessageManager.FilterAllTabs(false);
+        Plugin.MessageManager.FilterAllTabsAsync(false);
     }
 
     private void Activated(ChatActivatedArgs args)
@@ -229,8 +228,7 @@ public sealed class ChatLogWindow : Window
         switch (arguments)
         {
             case "all":
-                foreach (var tab in Plugin.Config.Tabs)
+                Plugin.MessageManager.ClearAllTabs();
-                    tab.Clear();
                 break;
             case "help":
                 Plugin.ChatGui.Print("- /clearlog2: clears the active tab's log");

ChatTwo/Ui/LegacyMessageImporterWindow.cs

@@ -35,7 +35,7 @@ internal class LegacyMessageImporterWindow : Window
 
     public void Dispose()
     {
-        Importer?.Dispose();
+        Importer?.DisposeAsync().AsTask().Wait();
     }
 
     private void NotificationClicked(INotificationClickArgs args)

ChatTwo/Ui/Settings.cs

@@ -158,7 +158,8 @@ public sealed class SettingsWindow : Window
         // save after 60 frames have passed, which should hopefully not
         // commit any changes that cause a crash
         Plugin.DeferredSaveFrames = 60;
-        Plugin.MessageManager.FilterAllTabs(false);
+        Plugin.MessageManager.ClearAllTabs();
+        Plugin.MessageManager.FilterAllTabsAsync(false);
 
         if (fontChanged || fontSizeChanged)
             Plugin.FontManager.BuildFonts();

ChatTwo/Ui/SettingsTabs/Database.cs

@@ -133,8 +133,7 @@ internal sealed class Database : ISettingsTab
             {
                 Plugin.Log.Warning("Clearing messages from database");
                 Plugin.MessageManager.Store.ClearMessages();
-                foreach (var tab in Plugin.Config.Tabs)
+                Plugin.MessageManager.ClearAllTabs();
-                    tab.Clear();
 
                 // Refresh on next draw
                 DatabaseLastRefreshTicks = 0;
@@ -156,10 +155,8 @@ internal sealed class Database : ISettingsTab
 
         if (ImGuiUtil.CtrlShiftButton("Reload messages from database", "Ctrl+Shift: MessageManager.FilterAllTabs(false)"))
         {
-            foreach (var tab in Plugin.Config.Tabs)
+            Plugin.MessageManager.ClearAllTabs();
-                tab.Clear();
+            Plugin.MessageManager.FilterAllTabsAsync(false);
-
-            Plugin.MessageManager.FilterAllTabs(false);
         }
 
         if (ImGuiUtil.CtrlShiftButton("Inject 10,000 messages", "Ctrl+Shift: creates 10,000 unique messages (async)"))
@@ -226,9 +223,7 @@ internal sealed class Database : ISettingsTab
         Plugin.Framework.Run(() =>
         {
             stopwatch = Stopwatch.StartNew();
-            foreach (var tab in Plugin.Config.Tabs)
+            Plugin.MessageManager.ClearAllTabs();
-                tab.Clear();
-
             elapsedTicks = stopwatch.ElapsedTicks;
             stopwatch.Stop();
             Plugin.Log.Info($"Cleared {Plugin.Config.Tabs.Count} tabs in {elapsedTicks} ticks ({elapsedTicks / TimeSpan.TicksPerMillisecond}ms)");
@@ -238,6 +233,7 @@ internal sealed class Database : ISettingsTab
         Plugin.Framework.Run(() =>
         {
             stopwatch = Stopwatch.StartNew();
+            // Intentionally synchronous
             Plugin.MessageManager.FilterAllTabs(false);
             elapsedTicks = stopwatch.ElapsedTicks;
             stopwatch.Stop();