feat: replace LiteDB with Sqlite

- Replace LiteDB database engine with Sqlite
  Note: old databases will not be deleted
- Message duplication detection improvements
- Tolerate parse errors in release builds, log them

.gitignore

@@ -363,3 +363,7 @@ MigrationBackup/
 
 # Fody - auto-generated XML schema
 FodyWeavers.xsd
+
+TestResults
+*.db-shm
+*.db-wal

ChatTwo.Tests/ChatTwo.Tests.csproj

@@ -0,0 +1,26 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>net8.0-windows</TargetFrameworks>
+    
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="JetBrains.Annotations" Version="2023.3.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.3.1" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.3.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ChatTwo\ChatTwo.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Reference Include="Dalamud">
+      <HintPath>..\..\AppData\Roaming\XIVLauncher\addon\Hooks\dev\Dalamud.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+  
+</Project>

ChatTwo.Tests/MessageStoreTest.cs

@@ -0,0 +1,289 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using ChatTwo.Code;
+using ChatTwo.Util;
+using Dalamud.Game.Text.SeStringHandling;
+using Dalamud.Game.Text.SeStringHandling.Payloads;
+using JetBrains.Annotations;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Chat2PartyFinderPayload = ChatTwo.Util.PartyFinderPayload;
+
+namespace ChatTwo.Tests;
+
+[TestClass]
+[TestSubject(typeof(MessageStore))]
+public class MessageStoreTest {
+    // From Message.cs
+    private static readonly byte[] ExtraChatChannelPayloadBytes = [0, 0x27, 18, 0x20];
+
+    public TestContext TestContext { get; set; }
+
+    public static string GetImportPath() {
+        string[] importPaths = [
+            @".\TestData",
+            @"..\TestData",
+            @"..\..\TestData",
+            @"..\..\..\TestData",
+        ];
+        var importPath = importPaths.FirstOrDefault(Directory.Exists);
+        if (string.IsNullOrEmpty(importPath)) {
+            throw new DirectoryNotFoundException("Could not find the import path");
+        }
+        return importPath;
+    }
+
+    [TestMethod]
+    [Timeout(5000)]
+    public void StoreAndRetrieve() {
+        var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_");
+        var dbPath = Path.Join(tempDir.FullName, "test.db");
+        TestContext.WriteLine("Using database path: " + dbPath);
+        using var store = new MessageStore(dbPath);
+
+        // Write the message.
+        var input = BigMessage();
+        store.UpsertMessage(input);
+
+        // Read the message back.
+        var messages = store.GetMostRecentMessages().ToList();
+        Assert.AreEqual(1, messages.Count);
+        AssertMessagesEqual(input, messages.First());
+    }
+
+    [TestMethod]
+    [Timeout(5000)]
+    public void RetrieveMultiple() {
+        var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_");
+        var dbPath = Path.Join(tempDir.FullName, "test.db");
+        TestContext.WriteLine("Using database path: " + dbPath);
+        using var store = new MessageStore(dbPath);
+
+        // Insert 10 messages in the wrong order of date.
+        var messages = new List<Message>();
+        const uint receiver = 12345;
+        var now = DateTimeOffset.UtcNow;
+        for (var i = 0; i < 10; i++) {
+            var message = BigMessage(true, receiver, now.AddSeconds(-i));
+            TestContext.WriteLine($"Inserting message {i}: {message.Id}");
+            store.UpsertMessage(message);
+            messages.Add(message);
+        }
+
+        // Insert a message for a different receiver. This shouldn't be returned
+        // because of the receiver filtering.
+        var otherReceiverMsg = BigMessage(receiver: receiver + 1, dateTime: now.AddSeconds(1));
+        TestContext.WriteLine($"Inserting other receiver message: {otherReceiverMsg.Id}");
+        store.UpsertMessage(otherReceiverMsg);
+
+        // Query the most recent 5 messages. Should return the 4 newest messages
+        // from the list, as well as the different receiver message because we
+        // aren't filtering.
+        var outputMessages = store.GetMostRecentMessages(count: 5).ToList();
+        var gotIds = outputMessages.Select(m => m.Id).ToList();
+        TestContext.WriteLine($"Query 1 got IDs: {string.Join(", ", gotIds)}");
+        AssertGuidsEqual(new List<Guid> {
+            messages[3].Id,
+            messages[2].Id,
+            messages[1].Id,
+            messages[0].Id,
+            otherReceiverMsg.Id
+        }, gotIds);
+
+        // Query the most recent 5 messages but filter by receiver ID.
+        outputMessages = store.GetMostRecentMessages(receiver: receiver, count: 5).ToList();
+        gotIds = outputMessages.Select(m => m.Id).ToList();
+        TestContext.WriteLine($"Query 2 got IDs: {string.Join(", ", gotIds)}");
+        AssertGuidsEqual(new List<Guid> {
+            messages[4].Id,
+            messages[3].Id,
+            messages[2].Id,
+            messages[1].Id,
+            messages[0].Id,
+        }, gotIds);
+
+        // Query the most recent 5 messages but only since a specific date.
+        outputMessages = store.GetMostRecentMessages(receiver, since: messages[1].Date, count: 5).ToList();
+        gotIds = outputMessages.Select(m => m.Id).ToList();
+        TestContext.WriteLine($"Query 3 got IDs: {string.Join(", ", gotIds)}");
+        AssertGuidsEqual(new List<Guid> {
+            messages[1].Id,
+            messages[0].Id,
+        }, gotIds);
+    }
+
+    [TestMethod]
+    [Timeout(5000)]
+    // This test guards against the data format changing in an incompatible way.
+    public void RetrieveExisting() {
+        var input = BigMessage(uniqId: false);
+
+        var dbPath = Path.Join(GetImportPath(), "existing.db");
+        TestContext.WriteLine($"Using existing database: {dbPath}");
+        Assert.IsTrue(File.Exists(dbPath));
+
+        // Uncomment this section to regenerate the existing database.
+        /*
+        File.Delete(dbPath);
+        using (var newStore = new MessageStore(dbPath)) {
+            newStore.UpsertMessage(input);
+        }
+        */
+
+        using var store = new MessageStore(dbPath);
+        var output = store.GetMostRecentMessages().ToList();
+        Assert.AreEqual(1, output.Count);
+        AssertMessagesEqual(input, output[0]);
+    }
+
+    [TestMethod]
+    [Timeout(30_000)]
+    public void ProfileMany() {
+        const int count = 20_000;
+
+        var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_");
+        var dbPath = Path.Join(tempDir.FullName, "test.db");
+        TestContext.WriteLine("Using database path: " + dbPath);
+        using var store = new MessageStore(dbPath);
+
+        for (var i = 0; i < count; i++) {
+            var message = BigMessage(uniqId: true);
+            store.UpsertMessage(message);
+        }
+
+        var messages = store.GetMostRecentMessages(count: count).ToList();
+        Assert.AreEqual(count, messages.Count);
+        foreach (var message in messages) {
+            // Load the message because they are lazily parsed.
+            Assert.IsTrue(message.Id != Guid.Empty);
+        }
+    }
+
+    private static Message BigMessage(bool uniqId = true, uint receiver = 12345, DateTimeOffset? dateTime = null) {
+        // NOTE: These values aren't valid in the game.
+        // NOTE: we can't test UiForeground, UiGlow, or AutoTranslatePayload
+        // because they load data from the game.
+        var senderSeString = new SeStringBuilder()
+            .AddText("<")
+            .Add(new PlayerPayload("Player Name", 12345))
+            .AddItalics("Player Name")
+            .Add(RawPayload.LinkTerminator)
+            .AddText(">: ")
+            .Build();
+        var extraChatId = Guid.Parse("03d9e6d4-dc1a-4005-bbe7-66b8c3529277");
+        var contentSeString = new SeStringBuilder()
+            .Add(new RawPayload(ExtraChatChannelPayloadBytes.Concat(extraChatId.ToByteArray()).ToArray()))
+            .AddIcon(BitmapFontIcon.IslandSanctuary)
+            .AddMapLink(1, 2, 3, 4)
+            .AddText("map")
+            .Add(RawPayload.LinkTerminator)
+            .AddQuestLink(12345)
+            .AddText("quest")
+            .Add(RawPayload.LinkTerminator)
+            .Add(new DalamudLinkPayload())
+            .AddText("dalamud")
+            .Add(RawPayload.LinkTerminator)
+            .AddStatusLink(12345)
+            .AddText("status")
+            .Add(RawPayload.LinkTerminator)
+            .AddPartyFinderLink(12345)
+            .AddText("party finder")
+            .Add(RawPayload.LinkTerminator)
+            .Build();
+
+        // Add Chat 2 specific payloads (that can't be serialized into the
+        // SeString).
+        var contentChunks = ChunkUtil.ToChunks(contentSeString, ChunkSource.Content, ChatType.Say).ToList();
+        contentChunks = contentChunks.Concat([
+            new TextChunk(ChunkSource.Content, new Chat2PartyFinderPayload(12345), "chat 2 party finder"),
+            new TextChunk(ChunkSource.Content, new AchievementPayload(12345), "chat 2 achievement"),
+            new TextChunk(ChunkSource.Content, new UriPayload(new Uri("https://dalamud.dev")), "chat 2 uri"),
+        ]).ToList();
+
+        return new Message(
+            uniqId ? Guid.NewGuid() : Guid.Parse("f011343e-6a21-49e5-a6f9-238f0f1f8c2c"),
+            receiver,
+            54321,
+            dateTime ?? DateTimeOffset.FromUnixTimeMilliseconds(1713520182440),
+            new ChatCode(12345),
+            ChunkUtil.ToChunks(senderSeString, ChunkSource.Sender, ChatType.Debug).ToList(),
+            contentChunks,
+            senderSeString,
+            contentSeString,
+            new SortCode(ChatType.Crafting, ChatSource.AlliancePet),
+            extraChatId
+        );
+    }
+
+    private void AssertMessagesEqual(Message input, Message output) {
+        // Check basic fields.
+        Assert.AreEqual(input.Id, output.Id);
+        Assert.AreEqual(input.Receiver, output.Receiver);
+        Assert.AreEqual(input.ContentId, output.ContentId);
+        // Assert time is within 1 second
+        TestContext.WriteLine($"Input date: {input.Date.ToUniversalTime()}");
+        TestContext.WriteLine($"Output date: {output.Date.ToUniversalTime()}");
+        var timeDifference = Math.Abs(input.Date.ToUniversalTime().Subtract(output.Date.ToUniversalTime()).TotalSeconds);
+        TestContext.WriteLine($"Time difference: {timeDifference}s");
+        Assert.IsTrue(timeDifference < 1);
+        Assert.AreEqual(input.Code.Raw, output.Code.Raw);
+        Assert.AreEqual($"{input.SenderSource.Encode():X}", $"{output.SenderSource.Encode():X}");
+        Assert.AreEqual($"{input.ContentSource.Encode():X}", $"{output.ContentSource.Encode():X}");
+        Assert.AreEqual(input.SortCode, output.SortCode);
+        Assert.AreEqual(input.ExtraChatChannel, output.ExtraChatChannel);
+
+        // Check chunks.
+        AssertChunksEqual(input.Sender, output.Sender);
+        AssertChunksEqual(input.Content, output.Content);
+    }
+
+    private static void AssertChunksEqual(IReadOnlyList<Chunk> inputChunks, IReadOnlyList<Chunk> outputChunks) {
+        Assert.AreEqual(inputChunks.Count, outputChunks.Count);
+        for (var i = 0; i < inputChunks.Count; i++) {
+            var inputChunk = inputChunks[i];
+            var outputChunk = outputChunks[i];
+            Assert.AreEqual(inputChunk.Source, outputChunk.Source);
+            switch (inputChunk.Link) {
+                case AchievementPayload inputAchievementPayload:
+                    Assert.AreEqual(inputAchievementPayload.Id, ((AchievementPayload) outputChunk.Link)!.Id);
+                    break;
+                case Chat2PartyFinderPayload inputPartyFinderPayload:
+                    Assert.AreEqual(inputPartyFinderPayload.Id, ((Chat2PartyFinderPayload) outputChunk.Link)!.Id);
+                    break;
+                case UriPayload inputUriPayload:
+                    Assert.AreEqual(inputUriPayload.Uri, ((UriPayload) outputChunk.Link)!.Uri);
+                    break;
+                case null:
+                    Assert.IsTrue(outputChunk.Link == null);
+                    break;
+                default:
+                    Assert.AreEqual($"{inputChunk.Link.Encode():X}", $"{outputChunk.Link!.Encode():X}");
+                    break;
+            }
+
+            switch (inputChunk) {
+                case TextChunk inputTextChunk:
+                    var outputTextChunk = (TextChunk)outputChunk;
+                    Assert.AreEqual(inputTextChunk.FallbackColour, outputTextChunk.FallbackColour);
+                    Assert.AreEqual(inputTextChunk.Foreground, outputTextChunk.Foreground);
+                    Assert.AreEqual(inputTextChunk.Glow, outputTextChunk.Glow);
+                    Assert.AreEqual(inputTextChunk.Italic, outputTextChunk.Italic);
+                    Assert.AreEqual(inputTextChunk.Content, outputTextChunk.Content);
+                    break;
+                case IconChunk inputIconChunk:
+                    Assert.AreEqual(inputIconChunk.Icon, ((IconChunk) outputChunk).Icon);
+                    break;
+                default:
+                    throw new Exception("Unknown chunk type");
+            }
+        }
+    }
+
+    private static void AssertGuidsEqual(IReadOnlyList<Guid> expected, IReadOnlyList<Guid> got) {
+        Assert.AreEqual(expected.Count, got.Count);
+        for (var i = 0; i < expected.Count; i++) {
+            Assert.AreEqual(expected[i].ToString(), got[i].ToString());
+        }
+    }
+}

ChatTwo.sln

@@ -2,6 +2,8 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ChatTwo", "ChatTwo\ChatTwo.csproj", "{739F75E6-B65F-41EF-9D90-F7BC519E4875}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ChatTwo.Tests", "ChatTwo.Tests\ChatTwo.Tests.csproj", "{A9FE423A-240C-4EDA-ACC6-21474B562128}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -12,5 +14,9 @@ Global
 		{739F75E6-B65F-41EF-9D90-F7BC519E4875}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{739F75E6-B65F-41EF-9D90-F7BC519E4875}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{739F75E6-B65F-41EF-9D90-F7BC519E4875}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A9FE423A-240C-4EDA-ACC6-21474B562128}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A9FE423A-240C-4EDA-ACC6-21474B562128}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A9FE423A-240C-4EDA-ACC6-21474B562128}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A9FE423A-240C-4EDA-ACC6-21474B562128}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

ChatTwo/ChatTwo.csproj

@@ -50,7 +50,8 @@
 
     <ItemGroup>
         <PackageReference Include="DalamudPackager" Version="2.1.12"/>
-        <PackageReference Include="LiteDB" Version="5.0.17"/>
+        <PackageReference Include="MessagePack" Version="2.5.140" />
+        <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.4" />
         <PackageReference Include="Pidgin" Version="3.2.2"/>
         <PackageReference Include="SharpDX.Direct2D1" Version="4.2.0"/>
         <PackageReference Include="XivCommon" Version="9.0.0"/>

ChatTwo/Chunk.cs

@@ -1,15 +1,22 @@
 using ChatTwo.Code;
 using Dalamud.Game.Text.SeStringHandling;
-using LiteDB;
+using MessagePack;
 
 namespace ChatTwo;
 
-internal abstract class Chunk {
+[Union(0, typeof(TextChunk))]
-    [BsonIgnore]
+[Union(1, typeof(IconChunk))]
+[MessagePackObject]
+public abstract class Chunk {
+    [IgnoreMember]
     internal Message? Message { get; set; }
 
-    internal ChunkSource Source { get; set; }
+    [Key(0)]
-    internal Payload? Link { get; set; }
+    public ChunkSource Source { get; set; }
+
+    [Key(1)]
+    [MessagePackFormatter(typeof(PayloadMessagePackFormatter))]
+    public Payload? Link { get; set; }
 
     protected Chunk(ChunkSource source, Payload? link) {
         Source = source;
@@ -38,27 +45,38 @@ internal abstract class Chunk {
     }
 }
 
-internal enum ChunkSource {
+public enum ChunkSource {
     None,
     Sender,
     Content,
 }
 
-internal class TextChunk : Chunk {
+[MessagePackObject]
-    internal ChatType? FallbackColour { get; set; }
+public class TextChunk : Chunk {
-    internal uint? Foreground { get; set; }
+    [Key(2)]
-    internal uint? Glow { get; set; }
+    public ChatType? FallbackColour { get; set; }
-    internal bool Italic { get; set; }
+    [Key(3)]
-    internal string Content { get; set; }
+    public uint? Foreground { get; set; }
+    [Key(4)]
+    public uint? Glow { get; set; }
+    [Key(5)]
+    public bool Italic { get; set; }
+    [Key(6)]
+    public string Content { get; set; }
 
     internal TextChunk(ChunkSource source, Payload? link, string content) : base(source, link) {
         Content = content;
     }
 
-    #pragma warning disable CS8618
+    // ReSharper disable once UnusedMember.Global // Used by MessagePack
-    public TextChunk() : base(ChunkSource.None, null) {
+    public TextChunk(ChunkSource source, Payload? link, ChatType? fallbackColour, uint? foreground, uint? glow,
+        bool italic, string content) : base(source, link) {
+        FallbackColour = fallbackColour;
+        Foreground = foreground;
+        Glow = glow;
+        Italic = italic;
+        Content = content;
     }
-    #pragma warning restore CS8618
 
     /// <summary>
     /// Creates a new TextChunk with identical styling to this one.
@@ -74,13 +92,12 @@ internal class TextChunk : Chunk {
     }
 }
 
-internal class IconChunk : Chunk {
+[MessagePackObject]
-    internal BitmapFontIcon Icon { get; set; }
+public class IconChunk : Chunk {
+    [Key(2)]
+    public BitmapFontIcon Icon { get; set; }
 
     public IconChunk(ChunkSource source, Payload? link, BitmapFontIcon icon) : base(source, link) {
         Icon = icon;
     }
-
-    public IconChunk() : base(ChunkSource.None, null) {
-    }
 }

ChatTwo/Code/ChatCode.cs

@@ -1,5 +1,3 @@
-using LiteDB;
-
 namespace ChatTwo.Code;
 
 internal class ChatCode
@@ -21,15 +19,6 @@ internal class ChatCode
         Target = SourceFrom(7);
     }
 
-    [BsonCtor]
-    public ChatCode(ushort raw, ChatType type, ChatSource source, ChatSource target)
-    {
-        Raw = raw;
-        Type = type;
-        Source = source;
-        Target = target;
-    }
-
     internal ChatType Parent() => Type switch
     {
         ChatType.Say => ChatType.Say,

ChatTwo/Code/ChatType.cs

@@ -1,7 +1,7 @@
 namespace ChatTwo.Code;
 
 [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1028:Enum Storage should be Int32")]
-internal enum ChatType : ushort
+public enum ChatType : ushort
 {
     Debug = 1,
     Urgent = 2,

ChatTwo/Configuration.cs

@@ -35,7 +35,6 @@ internal class Configuration : IPluginConfiguration
     public bool DatabaseBattleMessages;
     public bool LoadPreviousSession;
     public bool FilterIncludePreviousSessions;
-    public bool SharedMode;
     public bool SortAutoTranslate;
     public bool CollapseDuplicateMessages;
     public bool PlaySounds = true;
@@ -80,7 +79,6 @@ internal class Configuration : IPluginConfiguration
         DatabaseBattleMessages = other.DatabaseBattleMessages;
         LoadPreviousSession = other.LoadPreviousSession;
         FilterIncludePreviousSessions = other.FilterIncludePreviousSessions;
-        SharedMode = other.SharedMode;
         SortAutoTranslate = other.SortAutoTranslate;
         CollapseDuplicateMessages = other.CollapseDuplicateMessages;
         PlaySounds = other.PlaySounds;
@@ -197,16 +195,16 @@ internal class Tab
 
     [NonSerialized]
     public List<Message> Messages = new();
+    [NonSerialized]
+    public HashSet<Guid> TrackedMessageIds = new();
 
     ~Tab() { MessagesMutex.Dispose(); }
 
-    internal bool Contains(Message message)
+    internal bool Contains(Message message) {
-    {
+        return TrackedMessageIds.Contains(message.Id);
-        return Messages.Any(m => m.Hash == message.Hash);
     }
 
-    internal bool Matches(Message message)
+    internal bool Matches(Message message) {
-    {
         if (message.ExtraChatChannel != Guid.Empty)
             return ExtraChatAll || ExtraChatChannels.Contains(message.ExtraChatChannel);
 
@@ -216,23 +214,25 @@ internal class Tab
                    || sources.HasFlag(message.Code.Source));
     }
 
-    internal void AddMessage(Message message, bool unread = true)
+    internal void AddMessage(Message message, bool unread = true) {
-    {
+        if (Contains(message)) return;
         MessagesMutex.Wait();
+        TrackedMessageIds.Add(message.Id);
         Messages.Add(message);
-        while (Messages.Count > Store.MessagesLimit)
+        while (Messages.Count > MessageManager.MessageDisplayLimit) {
+            TrackedMessageIds.Remove(Messages[0].Id);
             Messages.RemoveAt(0);
-
+        }
         MessagesMutex.Release();
 
         if (unread)
             Unread += 1;
     }
 
-    internal void Clear()
+    internal void Clear() {
-    {
         MessagesMutex.Wait();
         Messages.Clear();
+        TrackedMessageIds.Clear();
         MessagesMutex.Release();
     }
 

ChatTwo/Message.cs

@@ -2,21 +2,26 @@ using ChatTwo.Code;
 using ChatTwo.Util;
 using Dalamud.Game.Text.SeStringHandling;
 using Dalamud.Game.Text.SeStringHandling.Payloads;
-using LiteDB;
 using System.Text.RegularExpressions;
 
 namespace ChatTwo;
 
 internal class SortCode {
-    internal ChatType Type { get; set; }
+    internal ChatType Type { get; }
-    internal ChatSource Source { get; set; }
+    internal ChatSource Source { get; }
 
     internal SortCode(ChatType type, ChatSource source) {
         Type = type;
         Source = source;
     }
 
-    public SortCode() {
+    internal SortCode(uint raw) {
+        Type = (ChatType)(raw >> 16);
+        Source = (ChatSource)(raw & 0xFFFF);
+    }
+
+    internal uint Encode() {
+        return ((uint) Type << 16) | (uint) Source;
     }
 
     private bool Equals(SortCode other) {
@@ -43,18 +48,11 @@ internal class SortCode {
 }
 
 internal class Message {
-    // ReSharper disable once UnusedMember.Global
+    internal Guid Id { get; } = Guid.NewGuid();
-    internal ObjectId Id { get; } = ObjectId.NewObjectId();
     internal ulong Receiver { get; }
     internal ulong ContentId { get; set; }
 
-    [BsonIgnore]
+    internal DateTimeOffset Date { get; }
-    internal float? Height;
-
-    [BsonIgnore]
-    internal bool IsVisible;
-
-    internal DateTime Date { get; }
     internal ChatCode Code { get; }
     internal List<Chunk> Sender { get; }
     internal List<Chunk> Content { get; }
@@ -65,11 +63,14 @@ internal class Message {
     internal SortCode SortCode { get; }
     internal Guid ExtraChatChannel { get; }
 
+    // Not stored in the database:
     internal int Hash { get; }
+    internal float? Height { get; set; }
+    internal bool IsVisible { get; set; }
 
     internal Message(ulong receiver, ChatCode code, List<Chunk> sender, List<Chunk> content, SeString senderSource, SeString contentSource) {
         Receiver = receiver;
-        Date = DateTime.UtcNow;
+        Date = DateTimeOffset.UtcNow;
         Code = code;
         Sender = sender;
         Content = ReplaceContentURLs(content);
@@ -84,44 +85,23 @@ internal class Message {
         }
     }
 
-    internal Message(ObjectId id, ulong receiver, ulong contentId, DateTime date, BsonDocument code, BsonArray sender, BsonArray content, BsonValue senderSource, BsonValue contentSource, BsonDocument sortCode) {
+    internal Message(Guid id, ulong receiver, ulong contentId, DateTimeOffset date, ChatCode code, List<Chunk> sender, List<Chunk> content, SeString senderSource, SeString contentSource, SortCode sortCode, Guid extraChatChannel) {
         Id = id;
         Receiver = receiver;
         ContentId = contentId;
         Date = date;
-        Code = BsonMapper.Global.ToObject<ChatCode>(code);
+        Code = code;
-        Sender = BsonMapper.Global.Deserialize<List<Chunk>>(sender);
+        Sender = sender;
         // Don't call ReplaceContentURLs here since we're loading the message
         // from the database and it should already have parsed URL data.
-        Content = BsonMapper.Global.Deserialize<List<Chunk>>(content);
+        Content = content;
-        SenderSource = BsonMapper.Global.Deserialize<SeString>(senderSource);
+        SenderSource = senderSource;
-        ContentSource = BsonMapper.Global.Deserialize<SeString>(contentSource);
+        ContentSource = contentSource;
-        SortCode = BsonMapper.Global.ToObject<SortCode>(sortCode);
+        SortCode = sortCode;
-        ExtraChatChannel = ExtractExtraChatChannel();
+        ExtraChatChannel = extraChatChannel;
         Hash = GenerateHash();
 
-        foreach (var chunk in Sender.Concat(Content)) {
+        foreach (var chunk in sender.Concat(content)) {
-            chunk.Message = this;
-        }
-    }
-
-    internal Message(ObjectId id, ulong receiver, ulong contentId, DateTime date, BsonDocument code, BsonArray sender, BsonArray content, BsonValue senderSource, BsonValue contentSource, BsonDocument sortCode, BsonValue extraChatChannel) {
-        Id = id;
-        Receiver = receiver;
-        ContentId = contentId;
-        Date = date;
-        Code = BsonMapper.Global.ToObject<ChatCode>(code);
-        Sender = BsonMapper.Global.Deserialize<List<Chunk>>(sender);
-        // Don't call ReplaceContentURLs here since we're loading the message
-        // from the database and it should already have parsed URL data.
-        Content = BsonMapper.Global.Deserialize<List<Chunk>>(content);
-        SenderSource = BsonMapper.Global.Deserialize<SeString>(senderSource);
-        ContentSource = BsonMapper.Global.Deserialize<SeString>(contentSource);
-        SortCode = BsonMapper.Global.ToObject<SortCode>(sortCode);
-        ExtraChatChannel = BsonMapper.Global.Deserialize<Guid>(extraChatChannel);
-        Hash = GenerateHash();
-
-        foreach (var chunk in Sender.Concat(Content)) {
             chunk.Message = this;
         }
     }
@@ -203,7 +183,7 @@ internal class Message {
                 // Create a new TextChunk with a URIPayload for the URL text.
                 try
                 {
-                    var link = URIPayload.ResolveURI(match.Value);
+                    var link = UriPayload.ResolveURI(match.Value);
                     AddChunkWithMessage(text.NewWithStyle(chunk.Source, link, match.Value));
                 }
                 catch (UriFormatException)

ChatTwo/MessageManager.cs

@@ -0,0 +1,201 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using ChatTwo.Code;
+using ChatTwo.Resources;
+using ChatTwo.Util;
+using Dalamud.Game.Text;
+using Dalamud.Game.Text.SeStringHandling;
+using Dalamud.Interface.Internal.Notifications;
+using Dalamud.Plugin.Services;
+using Lumina.Excel.GeneratedSheets;
+
+namespace ChatTwo;
+
+internal class MessageManager : IDisposable {
+    internal const int MessageDisplayLimit = 10_000;
+
+    private Plugin Plugin { get; }
+    internal MessageStore Store { get; }
+
+    private ConcurrentQueue<(uint, Message)> Pending { get; } = new();
+    private Stopwatch MaintenanceTimer { get; } = new();
+    private Dictionary<ChatType, NameFormatting> Formats { get; } = new();
+    private ulong LastContentId { get; set; }
+
+    internal ulong CurrentContentId {
+        get {
+            var contentId = Plugin.ClientState.LocalContentId;
+            return contentId == 0 ? LastContentId : contentId;
+        }
+    }
+
+    internal MessageManager(Plugin plugin) {
+        Plugin = plugin;
+        MaintenanceTimer.Start();
+        Store = new MessageStore(DatabasePath());
+
+        Plugin.ChatGui.ChatMessageUnhandled += ChatMessage;
+        Plugin.Framework.Update += GetMessageInfo;
+        Plugin.Framework.Update += UpdateReceiver;
+        Plugin.ClientState.Logout += Logout;
+    }
+
+    public void Dispose() {
+        Plugin.ClientState.Logout -= Logout;
+        Plugin.Framework.Update -= UpdateReceiver;
+        Plugin.Framework.Update -= GetMessageInfo;
+        Plugin.ChatGui.ChatMessageUnhandled -= ChatMessage;
+
+        Store.Dispose();
+    }
+
+    internal static string DatabasePath() {
+        var dir = Plugin.Interface.ConfigDirectory;
+        dir.Create();
+        return Path.Join(dir.FullName, "chat-sqlite.db");
+    }
+
+    private void Logout() {
+        LastContentId = 0;
+    }
+
+    private void UpdateReceiver(IFramework framework) {
+        var contentId = Plugin.ClientState.LocalContentId;
+        if (contentId != 0)
+            LastContentId = contentId;
+    }
+
+    private void GetMessageInfo(IFramework framework) {
+        if (MaintenanceTimer.Elapsed > TimeSpan.FromMinutes(5)) {
+            MaintenanceTimer.Restart();
+            new Thread(() => Store.PerformMaintenance()).Start();
+        }
+
+        if (!Pending.TryDequeue(out var entry))
+            return;
+
+        var contentId = Plugin.Functions.Chat.GetContentIdForEntry(entry.Item1);
+        entry.Item2.ContentId = contentId ?? 0;
+        if (Plugin.Config.DatabaseBattleMessages || !entry.Item2.Code.IsBattle())
+            Store.UpsertMessage(entry.Item2);
+    }
+
+    internal void AddMessage(Message message, Tab? currentTab) {
+        if (Plugin.Config.DatabaseBattleMessages || !message.Code.IsBattle())
+            Store.UpsertMessage(message);
+
+        var currentMatches = currentTab?.Matches(message) ?? false;
+
+        foreach (var tab in Plugin.Config.Tabs) {
+            var unread = !(tab.UnreadMode == UnreadMode.Unseen && currentTab != tab && currentMatches);
+
+            if (tab.Matches(message))
+                tab.AddMessage(message, unread);
+        }
+    }
+
+    internal void FilterAllTabs(bool unread = true) {
+        DateTimeOffset? since = null;
+        if (!Plugin.Config.FilterIncludePreviousSessions)
+            since = Plugin.GameStarted;
+
+        var messages = Store.GetMostRecentMessages(CurrentContentId, since);
+        foreach (var message in messages) {
+            foreach (var tab in Plugin.Config.Tabs.Where(tab => tab.Matches(message))) {
+                tab.AddMessage(message, unread);
+            }
+        }
+
+        if (messages.DidError)
+            WrapperUtil.AddNotification(Language.LoadMessages_Error, NotificationType.Error);
+    }
+
+    public (SeString? Sender, SeString? Message) LastMessage = (null, null);
+    private void ChatMessage(XivChatType type, uint senderId, SeString sender, SeString message) {
+        var chatCode = new ChatCode((ushort) type);
+
+        NameFormatting? formatting = null;
+        if (sender.Payloads.Count > 0)
+            formatting = FormatFor(chatCode.Type);
+
+        LastMessage = (sender, message);
+        var senderChunks = new List<Chunk>();
+        if (formatting is { IsPresent: true }) {
+            senderChunks.Add(new TextChunk(ChunkSource.None, null, formatting.Before) {
+                FallbackColour = chatCode.Type,
+            });
+            senderChunks.AddRange(ChunkUtil.ToChunks(sender, ChunkSource.Sender, chatCode.Type));
+            senderChunks.Add(new TextChunk(ChunkSource.None, null, formatting.After) {
+                FallbackColour = chatCode.Type,
+            });
+        }
+
+        var messageChunks = ChunkUtil.ToChunks(message, ChunkSource.Content, chatCode.Type).ToList();
+
+        var msg = new Message(CurrentContentId, chatCode, senderChunks, messageChunks, sender, message);
+        AddMessage(msg, Plugin.ChatLogWindow.CurrentTab ?? null);
+
+        var idx = Plugin.Functions.GetCurrentChatLogEntryIndex();
+        if (idx != null)
+            Pending.Enqueue((idx.Value - 1, msg));
+    }
+
+    internal class NameFormatting {
+        internal string Before { get; private set; } = string.Empty;
+        internal string After { get; private set; } = string.Empty;
+        internal bool IsPresent { get; private set; } = true;
+
+        internal static NameFormatting Empty() {
+            return new NameFormatting { IsPresent = false, };
+        }
+
+        internal static NameFormatting Of(string before, string after) {
+            return new NameFormatting
+            {
+                Before = before,
+                After = after,
+            };
+        }
+    }
+
+    private NameFormatting? FormatFor(ChatType type) {
+        if (Formats.TryGetValue(type, out var cached))
+            return cached;
+
+        var logKind = Plugin.DataManager.GetExcelSheet<LogKind>()!.GetRow((ushort) type);
+        if (logKind == null)
+            return null;
+
+        var format = (SeString) logKind.Format;
+        static bool IsStringParam(Payload payload, byte num) {
+            var data = payload.Encode();
+            return data.Length >= 5 && data[1] == 0x29 && data[4] == num + 1;
+        }
+
+        var firstStringParam = format.Payloads.FindIndex(payload => IsStringParam(payload, 1));
+        var secondStringParam = format.Payloads.FindIndex(payload => IsStringParam(payload, 2));
+
+        if (firstStringParam == -1 || secondStringParam == -1)
+            return NameFormatting.Empty();
+
+        var before = format.Payloads
+            .GetRange(0, firstStringParam)
+            .Where(payload => payload is ITextProvider)
+            .Cast<ITextProvider>()
+            .Select(text => text.Text);
+        var after = format.Payloads
+            .GetRange(firstStringParam + 1, secondStringParam - firstStringParam)
+            .Where(payload => payload is ITextProvider)
+            .Cast<ITextProvider>()
+            .Select(text => text.Text);
+
+        var nameFormatting = NameFormatting.Of(
+            string.Join("", before),
+            string.Join("", after)
+        );
+
+        Formats[type] = nameFormatting;
+
+        return nameFormatting;
+    }
+}

ChatTwo/MessageStore.cs

@@ -0,0 +1,356 @@
+using System.Buffers;
+using System.Collections;
+using System.Data.Common;
+using ChatTwo.Code;
+using ChatTwo.Util;
+using Dalamud.Game.Text.SeStringHandling;
+using MessagePack;
+using MessagePack.Formatters;
+using MessagePack.Resolvers;
+using Microsoft.Data.Sqlite;
+using DalamudUtil = Dalamud.Utility.Util;
+using Encoding = System.Text.Encoding;
+
+namespace ChatTwo;
+
+internal static class DbExtensions {
+    internal static void Execute(this DbConnection conn, string sql) {
+        using var cmd = conn.CreateCommand();
+        cmd.CommandText = sql;
+        cmd.ExecuteNonQuery();
+    }
+}
+
+internal enum PayloadMessagePackType : byte {
+    Achievement,
+    PartyFinder,
+    Uri,
+    Other = 255,
+}
+
+public class PayloadMessagePackFormatter : IMessagePackFormatter<Payload?> {
+    public void Serialize(ref MessagePackWriter writer, Payload? value, MessagePackSerializerOptions options) {
+        if (value == null) {
+            writer.WriteNil();
+            return;
+        }
+
+        writer.WriteArrayHeader(2);
+        switch (value) {
+            case AchievementPayload achievementPayload:
+                writer.WriteUInt8((byte) PayloadMessagePackType.Achievement);
+                writer.WriteUInt32(achievementPayload.Id);
+                break;
+            case PartyFinderPayload partyFinderPayload:
+                writer.WriteUInt8((byte) PayloadMessagePackType.PartyFinder);
+                writer.WriteUInt32(partyFinderPayload.Id);
+                break;
+            case UriPayload uriPayload:
+                writer.WriteUInt8((byte) PayloadMessagePackType.Uri);
+                writer.WriteString(Encoding.UTF8.GetBytes(uriPayload.Uri.ToString()));
+                break;
+            default:
+                writer.WriteUInt8((byte) PayloadMessagePackType.Other);
+                writer.Write(value.Encode());
+                break;
+        }
+    }
+
+    public Payload? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) {
+        if (reader.TryReadNil())
+            return null;
+
+        if (reader.ReadArrayHeader() != 2)
+            throw new InvalidOperationException("Invalid array count for Payload object");
+
+        var type = (PayloadMessagePackType) reader.ReadByte();
+        switch (type) {
+           case PayloadMessagePackType.Achievement:
+               return new AchievementPayload(reader.ReadUInt32());
+           case PayloadMessagePackType.PartyFinder:
+               return new PartyFinderPayload(reader.ReadUInt32());
+           case PayloadMessagePackType.Uri:
+               return new UriPayload(new Uri(reader.ReadString() ?? ""));
+           case PayloadMessagePackType.Other:
+           default:
+               var bytes = reader.ReadBytes() ?? new ReadOnlySequence<byte>();
+               var binReader = new BinaryReader(new MemoryStream(bytes.ToArray()));
+               return Payload.Decode(binReader);
+        }
+    }
+}
+
+public class SeStringMessagePackFormatter : IMessagePackFormatter<SeString> {
+    public void Serialize(ref MessagePackWriter writer, SeString value, MessagePackSerializerOptions options) {
+        options.Resolver.GetFormatter<List<Payload>>()!.Serialize(ref writer, value.Payloads, options);
+    }
+
+    public SeString Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) {
+        return new SeString(options.Resolver.GetFormatter<List<Payload>>()!.Deserialize(ref reader, options));
+    }
+}
+
+internal class MessageStore : IDisposable {
+    internal const int MessageQueryLimit = 10_000;
+
+    private string DbPath { get; }
+
+    private SqliteConnection Connection { get; set; }
+
+    internal static readonly MessagePackSerializerOptions MsgPackOptions = MessagePackSerializerOptions.Standard
+        .WithResolver(CompositeResolver.Create(
+            new IMessagePackFormatter[] {
+                new PayloadMessagePackFormatter(),
+                new SeStringMessagePackFormatter(),
+            },
+            new IFormatterResolver[] { StandardResolver.Instance }));
+
+    internal MessageStore(string dbPath) {
+        DbPath = dbPath;
+        Connection = Connect();
+        Migrate();
+    }
+
+    public void Dispose() {
+        Connection.Close();
+        Connection.Dispose();
+        // Closing the connection doesn't immediately release the file.
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+    }
+
+    private SqliteConnection Connect() {
+        var uriBuilder = new SqliteConnectionStringBuilder {
+            DataSource = DbPath,
+            DefaultTimeout = 5,
+            Pooling = false,
+            Mode = SqliteOpenMode.ReadWriteCreate,
+        };
+        var conn = new SqliteConnection(uriBuilder.ToString());
+        conn.Open();
+        conn.Execute(@"PRAGMA journal_mode=WAL;");
+        conn.Execute(@"PRAGMA synchronous=NORMAL;");
+        if (DalamudUtil.IsWine())
+            conn.Execute(@"PRAGMA cache_size = 32768;");
+        return conn;
+    }
+
+    private void Migrate() {
+        // TODO: this should be improved/swapped out for a library at some
+        // point.
+        Connection.Execute(@"
+            CREATE TABLE IF NOT EXISTS messages (
+                Id BLOB PRIMARY KEY NOT NULL,  -- Guid
+                Receiver INTEGER NOT NULL,     -- uint64 (first bits are always 0)
+                ContentId INTEGER NOT NULL,    -- uint64 (first bits are always 0)
+                Date INTEGER NOT NULL,         -- unix timestamp with millisecond precision
+                Code INTEGER NOT NULL,         -- ChatCode encoding
+                Sender BLOB NOT NULL,          -- Chunk[] msgpack
+                Content BLOB NOT NULL,         -- Chunk[] msgpack
+                SenderSource BLOB NOT NULL,    -- SeString
+                ContentSource BLOB NOT NULL,   -- SeString
+                SortCode INTEGER NOT NULL,     -- SortCode encoding
+                ExtraChatChannel BLOB NOT NULL -- Guid
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_messages_receiver ON messages (Receiver);
+            CREATE INDEX IF NOT EXISTS idx_messages_date ON messages (Date);
+        ");
+    }
+
+    internal void Reconnect() {
+        Connection.Close();
+        Connection.Dispose();
+        Connection = Connect();
+    }
+
+    internal void ClearMessages() {
+        Connection.Execute("DELETE FROM messages;");
+        PerformMaintenance();
+    }
+
+    internal void PerformMaintenance() {
+        Connection.Execute(@"
+            VACUUM;
+            REINDEX messages;
+            ANALYZE;
+        ");
+    }
+
+    internal long DatabaseSize() {
+        return !File.Exists(DbPath) ? 0 : new FileInfo(DbPath).Length;
+    }
+
+    private string LogPath => DbPath + "-wal";
+
+    internal long DatabaseLogSize() {
+        return !File.Exists(LogPath) ? 0 : new FileInfo(LogPath).Length;
+    }
+
+    internal int MessageCount()
+    {
+        var cmd = Connection.CreateCommand();
+        cmd.CommandText = "SELECT COUNT(*) FROM messages;";
+        return Convert.ToInt32(cmd.ExecuteScalar());
+    }
+
+    internal void UpsertMessage(Message message) {
+        var cmd = Connection.CreateCommand();
+        cmd.CommandText = @"
+            INSERT INTO messages (
+                Id,
+                Receiver,
+                ContentId,
+                Date,
+                Code,
+                Sender,
+                Content,
+                SenderSource,
+                ContentSource,
+                SortCode,
+                ExtraChatChannel
+            ) VALUES (
+                $Id,
+                $Receiver,
+                $ContentId,
+                $Date,
+                $Code,
+                $Sender,
+                $Content,
+                $SenderSource,
+                $ContentSource,
+                $SortCode,
+                $ExtraChatChannel
+            )
+            ON CONFLICT (id) DO UPDATE SET
+                Receiver = excluded.Receiver,
+                ContentId = excluded.ContentId,
+                Date = excluded.Date,
+                Code = excluded.Code,
+                Sender = excluded.Sender,
+                Content = excluded.Content,
+                SenderSource = excluded.SenderSource,
+                ContentSource = excluded.ContentSource,
+                SortCode = excluded.SortCode,
+                ExtraChatChannel = excluded.ExtraChatChannel;
+        ";
+
+        cmd.Parameters.AddWithValue("$Id", message.Id);
+        cmd.Parameters.AddWithValue("$Receiver", message.Receiver);
+        cmd.Parameters.AddWithValue("$ContentId", message.ContentId);
+        cmd.Parameters.AddWithValue("$Date", message.Date.ToUnixTimeMilliseconds());
+        cmd.Parameters.AddWithValue("$Code", message.Code.Raw);
+        cmd.Parameters.AddWithValue("$Sender", MessagePackSerializer.Serialize(message.Sender, MsgPackOptions));
+        cmd.Parameters.AddWithValue("$Content", MessagePackSerializer.Serialize(message.Content, MsgPackOptions));
+        cmd.Parameters.AddWithValue("$SenderSource", MessagePackSerializer.Serialize(message.SenderSource, MsgPackOptions));
+        cmd.Parameters.AddWithValue("$ContentSource", MessagePackSerializer.Serialize(message.ContentSource, MsgPackOptions));
+        cmd.Parameters.AddWithValue("$SortCode", message.SortCode.Encode());
+        cmd.Parameters.AddWithValue("$ExtraChatChannel", message.ExtraChatChannel);
+
+        cmd.ExecuteNonQuery();
+    }
+
+    /// <summary>
+    /// Get the most recent messages.
+    /// </summary>
+    /// <param name="receiver">The receiver content ID to filter by. If null, no filtering is performed.</param>
+    /// <param name="since">Only show messages since this date. If null, no filtering is performed.</param>
+    /// <param name="count">The amount to return. Defaults to 10,000.</param>
+    internal MessageEnumerator GetMostRecentMessages(ulong? receiver = null, DateTimeOffset? since = null, int count = MessageQueryLimit) {
+        var whereClauses = new List<string>();
+        if (receiver != null)
+            whereClauses.Add("Receiver = $Receiver");
+        if (since != null)
+            whereClauses.Add("Date >= $Since");
+
+        var whereClause = whereClauses.Count > 0 ? "WHERE " + string.Join(" AND ", whereClauses) : "";
+
+        var cmd = Connection.CreateCommand();
+        // Select last N messages by date DESC, but reverse the order to get
+        // them in ascending order.
+        cmd.CommandText = @"
+            SELECT *
+            FROM (
+                SELECT
+                    Id,
+                    Receiver,
+                    ContentId,
+                    Date,
+                    Code,
+                    Sender,
+                    Content,
+                    SenderSource,
+                    ContentSource,
+                    SortCode,
+                    ExtraChatChannel
+                FROM messages
+                " + whereClause + @"
+                ORDER BY Date DESC
+                LIMIT $Count
+            )
+            ORDER BY Date ASC;
+        ";
+        cmd.CommandTimeout = 120; // this could take a while on slow computers
+
+        if (receiver != null)
+            cmd.Parameters.AddWithValue("$Receiver", receiver);
+        if (since != null)
+            cmd.Parameters.AddWithValue("$Since", since.Value.ToUnixTimeMilliseconds());
+
+        cmd.Parameters.AddWithValue("$Count", count);
+
+        return new MessageEnumerator(cmd.ExecuteReader());
+    }
+}
+
+internal class MessageEnumerator(DbDataReader reader) : IEnumerable<Message> {
+    private const int MaxErrorLogs = 10;
+
+    private int _errorCount;
+    public bool DidError => _errorCount > 0;
+
+    public IEnumerator<Message> GetEnumerator() {
+        while (reader.Read()) {
+            var id = Guid.Empty;
+            Message msg;
+            try {
+                id = reader.GetGuid(0);
+                msg = new Message(
+                    id,
+                    (ulong)reader.GetInt64(1),
+                    (ulong)reader.GetInt64(2),
+                    DateTimeOffset.FromUnixTimeMilliseconds(reader.GetInt64(3)),
+                    new ChatCode((ushort)reader.GetInt32(4)),
+                    MessagePackSerializer.Deserialize<List<Chunk>>(reader.GetFieldValue<byte[]>(5),
+                        MessageStore.MsgPackOptions),
+                    MessagePackSerializer.Deserialize<List<Chunk>>(reader.GetFieldValue<byte[]>(6),
+                        MessageStore.MsgPackOptions),
+                    MessagePackSerializer.Deserialize<SeString>(reader.GetFieldValue<byte[]>(7),
+                        MessageStore.MsgPackOptions),
+                    MessagePackSerializer.Deserialize<SeString>(reader.GetFieldValue<byte[]>(8),
+                        MessageStore.MsgPackOptions),
+                    new SortCode((uint)reader.GetInt32(9)),
+                    reader.GetGuid(10)
+                );
+            } catch (Exception e) {
+                if (_errorCount < MaxErrorLogs)
+                    Plugin.Log.Error($"Exception while reading message '{id}' from database: {e}");
+                _errorCount++;
+                if (_errorCount == MaxErrorLogs)
+                    Plugin.Log.Error("Further parsing errors will not be logged");
+
+#if DEBUG
+                throw;
+#else
+                continue;
+#endif
+            }
+
+            yield return msg;
+        }
+    }
+
+    IEnumerator IEnumerable.GetEnumerator() {
+        return GetEnumerator();
+    }
+}

ChatTwo/PayloadHandler.cs

@@ -92,7 +92,7 @@ public sealed class PayloadHandler {
                 DrawItemPopup(item);
                 drawn = true;
                 break;
-            case URIPayload uri:
+            case UriPayload uri:
                 DrawUriPopup(uri);
                 drawn = true;
                 break;
@@ -252,7 +252,7 @@ public sealed class PayloadHandler {
 
                 DoHover(() => HoverItem(item), hoverSize);
                 break;
-            case URIPayload uri:
+            case UriPayload uri:
                 DoHover(() => HoverURI(uri), hoverSize);
                 break;
         }
@@ -376,7 +376,7 @@ public sealed class PayloadHandler {
         }
     }
 
-    private void HoverURI(URIPayload uri)
+    private void HoverURI(UriPayload uri)
     {
         ImGui.TextUnformatted(string.Format(Language.Context_URLDomain, uri.Uri.Authority));
         ImGuiUtil.WarningText(Language.Context_URLWarning);
@@ -411,7 +411,7 @@ public sealed class PayloadHandler {
                 if (Equals(raw, ChunkUtil.PeriodicRecruitmentLink))
                     GameFunctions.GameFunctions.OpenPartyFinder();
                 break;
-            case URIPayload uri:
+            case UriPayload uri:
                 TryOpenURI(uri.Uri);
                 break;
             default:
@@ -659,7 +659,7 @@ public sealed class PayloadHandler {
         return null;
     }
 
-    private void DrawUriPopup(URIPayload uri)
+    private void DrawUriPopup(UriPayload uri)
     {
         ImGui.TextUnformatted(string.Format(Language.Context_URLDomain, uri.Uri.Authority));
         ImGuiUtil.WarningText(Language.Context_URLWarning, false);

ChatTwo/Plugin.cs

@@ -55,7 +55,7 @@ public sealed class Plugin : IDalamudPlugin
     internal XivCommonBase Common { get; }
     internal TextureCache TextureCache { get; }
     internal GameFunctions.GameFunctions Functions { get; }
-    internal Store Store { get; }
+    internal MessageManager MessageManager { get; }
     internal IpcManager Ipc { get; }
     internal ExtraChat ExtraChat { get; }
     internal FontManager FontManager { get; }
@@ -102,13 +102,13 @@ public sealed class Plugin : IDalamudPlugin
             Interface.UiBuilder.DisableCutsceneUiHide = true;
             Interface.UiBuilder.DisableGposeUiHide = true;
 
-            Store = new Store(this);  // requires Ui
+            MessageManager = new MessageManager(this); // requires Ui
 
             // let all the other components register, then initialise commands
             Commands.Initialise();
 
             if (Interface.Reason is not PluginLoadReason.Boot) {
-                Store.FilterAllTabs(false);
+                MessageManager.FilterAllTabs(false);
             }
 
             Framework.Update += FrameworkUpdate;
@@ -141,7 +141,7 @@ public sealed class Plugin : IDalamudPlugin
 
         ExtraChat?.Dispose();
         Ipc?.Dispose();
-        Store?.Dispose();
+        MessageManager?.Dispose();
         Functions?.Dispose();
         TextureCache?.Dispose();
         Common?.Dispose();

ChatTwo/Properties/AssemblyInfo.cs

@@ -0,0 +1 @@
+[assembly: System.Runtime.CompilerServices.InternalsVisibleTo("ChatTwo.Tests")]

ChatTwo/Resources/Language.Designer.cs

@@ -1463,6 +1463,15 @@ namespace ChatTwo.Resources {
             }
         }
         
+        /// <summary>
+        ///   Looks up a localized string similar to An error occurred while loading chat history. Please see plugin logs for more information to report this issue..
+        /// </summary>
+        internal static string LoadMessages_Error {
+            get {
+                return ResourceManager.GetString("LoadMessages_Error", resourceCulture);
+            }
+        }
+        
         /// <summary>
         ///   Looks up a localized string similar to {0} is performing a database migration..
         /// </summary>
@@ -2237,33 +2246,6 @@ namespace ChatTwo.Resources {
             }
         }
         
-        /// <summary>
-        ///   Looks up a localized string similar to Allow multiple clients to run {0} at the same time, sharing the same database..
-        /// </summary>
-        internal static string Options_SharedMode_Description {
-            get {
-                return ResourceManager.GetString("Options_SharedMode_Description", resourceCulture);
-            }
-        }
-        
-        /// <summary>
-        ///   Looks up a localized string similar to Enable multi-client mode.
-        /// </summary>
-        internal static string Options_SharedMode_Name {
-            get {
-                return ResourceManager.GetString("Options_SharedMode_Name", resourceCulture);
-            }
-        }
-        
-        /// <summary>
-        ///   Looks up a localized string similar to This option is not recommended. No support will be offered if you enable this option. This option will hurt the performance of {0}..
-        /// </summary>
-        internal static string Options_SharedMode_Warning {
-            get {
-                return ResourceManager.GetString("Options_SharedMode_Warning", resourceCulture);
-            }
-        }
-        
         /// <summary>
         ///   Looks up a localized string similar to Show the Novice Network join button next to the settings button if logged in as a mentor..
         /// </summary>

ChatTwo/Resources/Language.de.resx

@@ -461,15 +461,6 @@ Sie wurden gewarnt.</value>
     <data name="Options_Database_Advanced">
         <value>Erweitert</value>
     </data>
-    <data name="Options_SharedMode_Name">
-        <value>Mehrere Clients verwenden</value>
-    </data>
-    <data name="Options_SharedMode_Description">
-        <value>Erlaubt es mehreren Clients, {0} gleichzeitig zu verwenden und Daten zu teilen.</value>
-    </data>
-    <data name="Options_SharedMode_Warning">
-        <value>Diese Einstellung ist nicht empfohlen. Es wird keine Hilfe angeboten, falls diese Option genutzt wird. Dadurch sinkt zudem die Leistung von {0}.</value>
-    </data>
     <data name="ChatLog_Tabs_PopOut">
         <value>Herauslösen</value>
     </data>

ChatTwo/Resources/Language.es.resx

@@ -460,15 +460,6 @@
     <data name="Options_Database_Advanced">
         <value>Avanzado</value>
     </data>
-    <data name="Options_SharedMode_Name">
-        <value>Habilitar modo multiventana</value>
-    </data>
-    <data name="Options_SharedMode_Description">
-        <value>Permitir que múltiples clientes ejecuten {0} al mismo tiempo, compartiendo la misma base de datos.</value>
-    </data>
-    <data name="Options_SharedMode_Warning">
-        <value>Esta opción no es recomendada. No se ofrecerá ningún soporte si activas esta opción. Esta opción perjudicará el rendimiento dé {0}.</value>
-    </data>
     <data name="ChatLog_Tabs_PopOut">
         <value>Nueva pestaña</value>
     </data>

ChatTwo/Resources/Language.fr.resx

@@ -460,15 +460,6 @@
     <data name="Options_Database_Advanced">
         <value>Avancé</value>
     </data>
-    <data name="Options_SharedMode_Name">
-        <value>Activer le mode multi-client</value>
-    </data>
-    <data name="Options_SharedMode_Description">
-        <value>Permet à plusieurs clients d'exécuter {0} en même temps, en partageant la même base de données.</value>
-    </data>
-    <data name="Options_SharedMode_Warning">
-        <value>Cette option n'est pas recommandée. Aucune assistance ne sera offerte si vous activez cette option. Cette option nuira aux performances de {0}.</value>
-    </data>
     <data name="ChatLog_Tabs_PopOut">
         <value>Détacher</value>
     </data>

ChatTwo/Resources/Language.nl.resx

@@ -460,15 +460,6 @@
     <data name="Options_Database_Advanced">
         <value>Geavanceerd</value>
     </data>
-    <data name="Options_SharedMode_Name">
-        <value>Modus voor meerdere vensters inschakelen</value>
-    </data>
-    <data name="Options_SharedMode_Description">
-        <value>Meerdere vensters toestaan om {0} tegelijkertijd uit te voeren en dezelfde database te delen.</value>
-    </data>
-    <data name="Options_SharedMode_Warning">
-        <value>Deze optie wordt niet aanbevolen. Er wordt geen ondersteuning aangeboden als je deze optie inschakelt. Deze optie is schadelijk voor de snelheid van {0}.</value>
-    </data>
     <data name="ChatLog_Tabs_PopOut">
         <value>Uitvouwen</value>
     </data>

ChatTwo/Resources/Language.pt-BR.resx

@@ -460,15 +460,6 @@
     <data name="Options_Database_Advanced">
         <value>Avançado</value>
     </data>
-    <data name="Options_SharedMode_Name">
-        <value>Ativar modo de mútiplos clientes</value>
-    </data>
-    <data name="Options_SharedMode_Description">
-        <value>Permitir que vários clientes sejam executados {0} ao mesmo tempo, compartilhando o mesmo banco de dados.</value>
-    </data>
-    <data name="Options_SharedMode_Warning">
-        <value>Esta opção não é recomendada. Nenhum suporte será oferecido se você ativar esta opção. Esta opção irá prejudicar o desempenho de {0}.</value>
-    </data>
     <data name="ChatLog_Tabs_PopOut">
         <value>Separar da janela</value>
     </data>

ChatTwo/Resources/Language.resx

@@ -460,15 +460,6 @@
     <data name="Options_Database_Advanced">
         <value>Advanced</value>
     </data>
-    <data name="Options_SharedMode_Name">
-        <value>Enable multi-client mode</value>
-    </data>
-    <data name="Options_SharedMode_Description">
-        <value>Allow multiple clients to run {0} at the same time, sharing the same database.</value>
-    </data>
-    <data name="Options_SharedMode_Warning">
-        <value>This option is not recommended. No support will be offered if you enable this option. This option will hurt the performance of {0}.</value>
-    </data>
     <data name="ChatLog_Tabs_PopOut">
         <value>Pop out</value>
     </data>
@@ -1000,4 +991,7 @@
     <data name="Options_MaxLinesToShow_Description" xml:space="preserve">
         <value>Limits the amount of log lines to show in the chat window. This may slightly improve performance.</value>
     </data>
+    <data name="LoadMessages_Error" xml:space="preserve">
+        <value>An error occurred while loading chat history. Please see plugin logs for more information to report this issue.</value>
+    </data>
 </root>

ChatTwo/Resources/Language.ro.resx

@@ -460,15 +460,6 @@
     <data name="Options_Database_Advanced">
         <value>Avansat</value>
     </data>
-    <data name="Options_SharedMode_Name">
-        <value>Activează modul instanțelor multiple</value>
-    </data>
-    <data name="Options_SharedMode_Description">
-        <value>Permite mai multor instanțe sa folosească {0} simultan, folosind aceeași data de baze.</value>
-    </data>
-    <data name="Options_SharedMode_Warning">
-        <value>Această opțiune nu este recomandată. Dacă o activați nu veți primi niciun suport. Această opțiune va afecta performanța {0}-ului.</value>
-    </data>
     <data name="ChatLog_Tabs_PopOut">
         <value>Mută tabul într-o fereastra noua</value>
     </data>

ChatTwo/Resources/Language.ru.resx

@@ -460,15 +460,6 @@
     <data name="Options_Database_Advanced">
         <value>Расширенные</value>
     </data>
-    <data name="Options_SharedMode_Name">
-        <value>Включить режим нескольких клиентов</value>
-    </data>
-    <data name="Options_SharedMode_Description">
-        <value>Разрешить нескольким клиентам запускать {0} одновременно, обмениваясь одной и той же базой данных.</value>
-    </data>
-    <data name="Options_SharedMode_Warning">
-        <value>Эта опция не рекомендуется. Если вы включите эту опцию, то поддержка не будет предоставляться. Эта опция значитьльно навредит производительности {0}.</value>
-    </data>
     <data name="ChatLog_Tabs_PopOut">
         <value>Отделить</value>
     </data>

ChatTwo/Resources/Language.sv.resx

@@ -460,15 +460,6 @@
     <data name="Options_Database_Advanced">
         <value>Avancerat</value>
     </data>
-    <data name="Options_SharedMode_Name">
-        <value>Aktivera flerklientsläge</value>
-    </data>
-    <data name="Options_SharedMode_Description">
-        <value>Tillåt flera klienter att köra {0} samtidigt med samma databas.</value>
-    </data>
-    <data name="Options_SharedMode_Warning">
-        <value>Den här inställningen rekommenderas inte. Inget stöd kommer erbjudas om du aktiverar den här inställningen. Den här inställningen kommer att förvärra prestandan av {0}.</value>
-    </data>
     <data name="ChatLog_Tabs_PopOut">
         <value>Separera</value>
     </data>

ChatTwo/Resources/Language.zh-Hans.resx

@@ -460,15 +460,6 @@
     <data name="Options_Database_Advanced">
         <value>高级选项</value>
     </data>
-    <data name="Options_SharedMode_Name">
-        <value>启用多客户端模式</value>
-    </data>
-    <data name="Options_SharedMode_Description">
-        <value>允许多个客户端同时运行 {0} ，共享同一个数据库。</value>
-    </data>
-    <data name="Options_SharedMode_Warning">
-        <value>不推荐此选项。如果您启用此选项，不会提供任何支持。此选项将损害 {0} 的性能。</value>
-    </data>
     <data name="ChatLog_Tabs_PopOut">
         <value>弹出</value>
     </data>

ChatTwo/Resources/Language.zh-Hant.resx

@@ -461,15 +461,6 @@
     <data name="Options_Database_Advanced">
         <value>高階選項</value>
     </data>
-    <data name="Options_SharedMode_Name">
-        <value>啓用多客戸端模式</value>
-    </data>
-    <data name="Options_SharedMode_Description">
-        <value>允許多個客戸端同時執行 {0} ，共享同一個資料庫。</value>
-    </data>
-    <data name="Options_SharedMode_Warning">
-        <value>不推薦此選項。 如果啓用此選項，不會提供任何支持。 此選項將損害 {0} 的效能。</value>
-    </data>
     <data name="ChatLog_Tabs_PopOut">
         <value>彈出</value>
     </data>

ChatTwo/Store.cs

@@ -1,387 +0,0 @@
-using System.Collections.Concurrent;
-using System.Diagnostics;
-using ChatTwo.Code;
-using ChatTwo.Util;
-using Dalamud.Game.Text;
-using Dalamud.Game.Text.SeStringHandling;
-using Dalamud.Plugin.Services;
-using LiteDB;
-using Lumina.Excel.GeneratedSheets;
-
-namespace ChatTwo;
-
-internal class Store : IDisposable
-{
-    internal const int MessagesLimit = 10_000;
-
-    private Plugin Plugin { get; }
-
-    private ConcurrentQueue<(uint, Message)> Pending { get; } = new();
-    private Stopwatch CheckpointTimer { get; } = new();
-    internal ILiteDatabase Database { get; private set; }
-    private ILiteCollection<Message> Messages => Database.GetCollection<Message>("messages");
-
-    private Dictionary<ChatType, NameFormatting> Formats { get; } = new();
-    private ulong LastContentId { get; set; }
-
-    private ulong CurrentContentId
-    {
-        get
-        {
-            var contentId = Plugin.ClientState.LocalContentId;
-            return contentId == 0 ? LastContentId : contentId;
-        }
-    }
-
-    internal Store(Plugin plugin)
-    {
-        Plugin = plugin;
-        CheckpointTimer.Start();
-
-        BsonMapper.Global = new BsonMapper
-        {
-            IncludeNonPublic = true,
-            TrimWhitespace = false,
-            // EnumAsInteger = true,
-        };
-
-        BsonMapper.Global.Entity<Message>()
-            .Id(msg => msg.Id)
-            .Ctor(doc => new Message(
-                doc["_id"].AsObjectId,
-                (ulong) doc["Receiver"].AsInt64,
-                (ulong) doc["ContentId"].AsInt64,
-                DateTime.UnixEpoch.AddMilliseconds(doc["Date"].AsInt64),
-                doc["Code"].AsDocument,
-                doc["Sender"].AsArray,
-                doc["Content"].AsArray,
-                doc["SenderSource"],
-                doc["ContentSource"],
-                doc["SortCode"].AsDocument,
-                doc["ExtraChatChannel"]
-            ));
-
-        BsonMapper.Global.RegisterType<Payload?>(
-            payload =>
-            {
-                switch (payload)
-                {
-                    case AchievementPayload achievement:
-                        return new BsonDocument(new Dictionary<string, BsonValue> {
-                            ["Type"] = new("Achievement"),
-                            ["Id"] = new(achievement.Id),
-                        });
-                    case PartyFinderPayload partyFinder:
-                        return new BsonDocument(new Dictionary<string, BsonValue> {
-                            ["Type"] = new("PartyFinder"),
-                            ["Id"] = new(partyFinder.Id),
-                        });
-                    case URIPayload uri:
-                        return new BsonDocument(new Dictionary<string, BsonValue> {
-                            ["Type"] = new("URI"),
-                            ["Uri"] = new(uri.Uri.ToString()),
-                        });
-                }
-
-                return payload?.Encode();
-            },
-            bson =>
-            {
-                if (bson.IsNull)
-                    return null;
-
-                if (bson.IsDocument)
-                {
-                    return bson["Type"].AsString switch
-                    {
-                        "Achievement" => new AchievementPayload((uint) bson["Id"].AsInt64),
-                        "PartyFinder" => new PartyFinderPayload((uint) bson["Id"].AsInt64),
-                        "URI" => new URIPayload(new Uri(bson["Uri"].AsString)),
-                        _ => null,
-                    };
-                }
-
-                return Payload.Decode(new BinaryReader(new MemoryStream(bson.AsBinary)));
-            });
-        BsonMapper.Global.RegisterType<SeString?>(
-            seString => seString == null
-                ? null
-                : new BsonArray(seString.Payloads.Select(payload => new BsonValue(payload.Encode()))),
-            bson =>
-            {
-                if (bson.IsNull)
-                    return null;
-
-                var array = bson.IsArray ? bson.AsArray : bson["Payloads"].AsArray;
-                var payloads = array
-                    .Select(payload => Payload.Decode(new BinaryReader(new MemoryStream(payload.AsBinary))))
-                    .ToList();
-                return new SeString(payloads);
-            }
-        );
-        BsonMapper.Global.RegisterType(
-            type => (int) type,
-            bson => (ChatType) bson.AsInt32
-        );
-        BsonMapper.Global.RegisterType(
-            source => (int) source,
-            bson => (ChatSource) bson.AsInt32
-        );
-        BsonMapper.Global.RegisterType(
-            dateTime => dateTime.Subtract(DateTime.UnixEpoch).TotalMilliseconds,
-            bson => DateTime.UnixEpoch.AddMilliseconds(bson.AsInt64)
-        );
-        Database = Connect();
-
-        Plugin.ChatGui.ChatMessageUnhandled += ChatMessage;
-        Plugin.Framework.Update += GetMessageInfo;
-        Plugin.Framework.Update += UpdateReceiver;
-        Plugin.ClientState.Logout += Logout;
-    }
-
-    public void Dispose() {
-        Plugin.ClientState.Logout -= Logout;
-        Plugin.Framework.Update -= UpdateReceiver;
-        Plugin.Framework.Update -= GetMessageInfo;
-        Plugin.ChatGui.ChatMessageUnhandled -= ChatMessage;
-
-        Database.Dispose();
-    }
-
-    internal static string DatabasePath()
-    {
-        var dir = Plugin.Interface.ConfigDirectory;
-        dir.Create();
-        return Path.Join(dir.FullName, "chat.db");
-    }
-
-    private LiteDatabase Connect() {
-        var dbPath = DatabasePath();
-        var connection = Plugin.Config.SharedMode ? "shared" : "direct";
-        var connString = $"Filename='{dbPath}';Connection={connection}";
-        var conn = new LiteDatabase(connString, BsonMapper.Global)
-        {
-            CheckpointSize = 1_000,
-            Timeout = TimeSpan.FromSeconds(1),
-        };
-        var messages = conn.GetCollection<Message>("messages");
-        messages.EnsureIndex(msg => msg.Date);
-        messages.EnsureIndex(msg => msg.SortCode);
-        messages.EnsureIndex(msg => msg.ExtraChatChannel);
-        return conn;
-    }
-
-    internal void Reconnect()
-    {
-        Database.Dispose();
-        Database = Connect();
-    }
-
-    internal void ClearDatabase()
-    {
-        Messages.DeleteAll();
-        Database.Rebuild();
-    }
-
-    internal static long DatabaseSize()
-    {
-        var dbPath = DatabasePath();
-        return !File.Exists(dbPath) ? 0 : new FileInfo(dbPath).Length;
-    }
-
-    internal static long DatabaseLogSize()
-    {
-        var dbLogPath = Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-log.db");
-        return !File.Exists(dbLogPath) ? 0 : new FileInfo(dbLogPath).Length;
-    }
-
-    internal int MessageCount() => Messages.Count();
-
-    private void Logout()
-    {
-        LastContentId = 0;
-    }
-
-    private void UpdateReceiver(IFramework framework)
-    {
-        var contentId = Plugin.ClientState.LocalContentId;
-        if (contentId != 0)
-            LastContentId = contentId;
-    }
-
-    private void GetMessageInfo(IFramework framework)
-    {
-        if (CheckpointTimer.Elapsed > TimeSpan.FromMinutes(5))
-        {
-            CheckpointTimer.Restart();
-            new Thread(() => Database.Checkpoint()).Start();
-        }
-
-        if (!Pending.TryDequeue(out var entry))
-            return;
-
-        var contentId = Plugin.Functions.Chat.GetContentIdForEntry(entry.Item1);
-        entry.Item2.ContentId = contentId ?? 0;
-        if (Plugin.Config.DatabaseBattleMessages || !entry.Item2.Code.IsBattle())
-            Messages.Update(entry.Item2);
-    }
-
-    internal void AddMessage(Message message, Tab? currentTab)
-    {
-        if (Plugin.Config.DatabaseBattleMessages || !message.Code.IsBattle())
-            Messages.Insert(message);
-
-        var currentMatches = currentTab?.Matches(message) ?? false;
-
-        foreach (var tab in Plugin.Config.Tabs)
-        {
-            var unread = !(tab.UnreadMode == UnreadMode.Unseen && currentTab != tab && currentMatches);
-
-            if (tab.Matches(message))
-                tab.AddMessage(message, unread);
-        }
-    }
-
-    internal void FilterAllTabs(bool unread = true)
-    {
-        foreach (var tab in Plugin.Config.Tabs)
-            FilterTab(tab, unread);
-    }
-
-    internal void FilterTab(Tab tab, bool unread)
-    {
-        var sortCodes = new List<SortCode>();
-        foreach (var (type, sources) in tab.ChatCodes)
-        {
-            sortCodes.Add(new SortCode(type, 0));
-            sortCodes.Add(new SortCode(type, (ChatSource) 1));
-
-            if (!type.HasSource())
-                continue;
-
-            foreach (var source in Enum.GetValues<ChatSource>())
-                if (sources.HasFlag(source))
-                    sortCodes.Add(new SortCode(type, source));
-        }
-
-        var query = Messages
-            .Query()
-            .OrderByDescending(msg => msg.Date)
-            .Where(msg => sortCodes.Contains(msg.SortCode) || msg.ExtraChatChannel != Guid.Empty)
-            .Where(msg => msg.Receiver == CurrentContentId);
-
-        if (!Plugin.Config.FilterIncludePreviousSessions)
-            query = query.Where(msg => msg.Date >= Plugin.GameStarted);
-
-        var messages = query.Limit(MessagesLimit).ToEnumerable().Reverse();
-
-        foreach (var message in messages)
-        {
-            // check primarily for startup double posting messages
-            if (tab.Contains(message))
-                continue;
-
-            // redundant matches check for extrachat
-            if (tab.Matches(message))
-                tab.AddMessage(message, unread);
-        }
-    }
-
-    public (SeString? Sender, SeString? Message) LastMessage = (null, null);
-    private void ChatMessage(XivChatType type, uint senderId, SeString sender, SeString message)
-    {
-        var chatCode = new ChatCode((ushort) type);
-
-        NameFormatting? formatting = null;
-        if (sender.Payloads.Count > 0)
-            formatting = FormatFor(chatCode.Type);
-
-        LastMessage = (sender, message);
-        var senderChunks = new List<Chunk>();
-        if (formatting is { IsPresent: true })
-        {
-            senderChunks.Add(new TextChunk(ChunkSource.None, null, formatting.Before)
-            {
-                FallbackColour = chatCode.Type,
-            });
-            senderChunks.AddRange(ChunkUtil.ToChunks(sender, ChunkSource.Sender, chatCode.Type));
-            senderChunks.Add(new TextChunk(ChunkSource.None, null, formatting.After)
-            {
-                FallbackColour = chatCode.Type,
-            });
-        }
-
-        var messageChunks = ChunkUtil.ToChunks(message, ChunkSource.Content, chatCode.Type).ToList();
-
-        var msg = new Message(CurrentContentId, chatCode, senderChunks, messageChunks, sender, message);
-        AddMessage(msg, Plugin.ChatLogWindow.CurrentTab ?? null);
-
-        var idx = Plugin.Functions.GetCurrentChatLogEntryIndex();
-        if (idx != null)
-            Pending.Enqueue((idx.Value - 1, msg));
-    }
-
-    internal class NameFormatting
-    {
-        internal string Before { get; private set; } = string.Empty;
-        internal string After { get; private set; } = string.Empty;
-        internal bool IsPresent { get; private set; } = true;
-
-        internal static NameFormatting Empty()
-        {
-            return new NameFormatting { IsPresent = false, };
-        }
-
-        internal static NameFormatting Of(string before, string after)
-        {
-            return new NameFormatting
-            {
-                Before = before,
-                After = after,
-            };
-        }
-    }
-
-    private NameFormatting? FormatFor(ChatType type)
-    {
-        if (Formats.TryGetValue(type, out var cached))
-            return cached;
-
-        var logKind = Plugin.DataManager.GetExcelSheet<LogKind>()!.GetRow((ushort) type);
-        if (logKind == null)
-            return null;
-
-        var format = (SeString) logKind.Format;
-        static bool IsStringParam(Payload payload, byte num)
-        {
-            var data = payload.Encode();
-            return data.Length >= 5 && data[1] == 0x29 && data[4] == num + 1;
-        }
-
-        var firstStringParam = format.Payloads.FindIndex(payload => IsStringParam(payload, 1));
-        var secondStringParam = format.Payloads.FindIndex(payload => IsStringParam(payload, 2));
-
-        if (firstStringParam == -1 || secondStringParam == -1)
-            return NameFormatting.Empty();
-
-        var before = format.Payloads
-            .GetRange(0, firstStringParam)
-            .Where(payload => payload is ITextProvider)
-            .Cast<ITextProvider>()
-            .Select(text => text.Text);
-        var after = format.Payloads
-            .GetRange(firstStringParam + 1, secondStringParam - firstStringParam)
-            .Where(payload => payload is ITextProvider)
-            .Cast<ITextProvider>()
-            .Select(text => text.Text);
-
-        var nameFormatting = NameFormatting.Of(
-            string.Join("", before),
-            string.Join("", after)
-        );
-
-        Formats[type] = nameFormatting;
-
-        return nameFormatting;
-    }
-}

ChatTwo/Ui/ChatLogWindow.cs

@@ -139,7 +139,7 @@ public sealed class ChatLogWindow : Window, IUiComponent
 
     private void Login()
     {
-        Plugin.Store.FilterAllTabs(false);
+        Plugin.MessageManager.FilterAllTabs(false);
     }
 
     private void Activated(ChatActivatedArgs args) {

ChatTwo/Ui/SeStringDebugger.cs

@@ -42,7 +42,7 @@ public class SeStringDebugger : Window
 
     public override void Draw()
     {
-        if (Plugin.Store.LastMessage.Sender == null)
+        if (Plugin.MessageManager.LastMessage.Sender == null)
         {
             ImGui.TextUnformatted("Nothing to show");
             return;
@@ -51,15 +51,15 @@ public class SeStringDebugger : Window
         // TODO: Make SeString freely selectable through chat
         ImGui.TextUnformatted("Sender Content");
         ImGui.Spacing();
-        if (Plugin.Store.LastMessage.Sender != null)
+        if (Plugin.MessageManager.LastMessage.Sender != null)
-            ProcessPayloads(Plugin.Store.LastMessage.Sender.Payloads);
+            ProcessPayloads(Plugin.MessageManager.LastMessage.Sender.Payloads);
         else
             ImGui.TextUnformatted("Nothing to show");
 
         ImGui.TextUnformatted("Message Content");
         ImGui.Spacing();
-        if (Plugin.Store.LastMessage.Message != null)
+        if (Plugin.MessageManager.LastMessage.Message != null)
-            ProcessPayloads(Plugin.Store.LastMessage.Message.Payloads);
+            ProcessPayloads(Plugin.MessageManager.LastMessage.Message.Payloads);
         else
             ImGui.TextUnformatted("Nothing to show");
     }

ChatTwo/Ui/Settings.cs

@@ -151,7 +151,6 @@ public sealed class SettingsWindow : Window, IUiComponent
                                   || Math.Abs(Mutable.JapaneseFontSize - Plugin.Config.JapaneseFontSize) > 0.001
                                   || Math.Abs(Mutable.SymbolsFontSize - Plugin.Config.SymbolsFontSize) > 0.001;
             var langChanged = Mutable.LanguageOverride != Plugin.Config.LanguageOverride;
-            var sharedChanged = Mutable.SharedMode != Plugin.Config.SharedMode;
 
             config.UpdateFrom(Mutable);
 
@@ -159,7 +158,7 @@ public sealed class SettingsWindow : Window, IUiComponent
             // commit any changes that cause a crash
             Plugin.DeferredSaveFrames = 60;
 
-            Plugin.Store.FilterAllTabs(false);
+            Plugin.MessageManager.FilterAllTabs(false);
 
             if (fontChanged || fontSizeChanged) {
                 Plugin.FontManager.BuildFonts();
@@ -169,10 +168,6 @@ public sealed class SettingsWindow : Window, IUiComponent
                 Plugin.LanguageChanged(Plugin.Interface.UiLanguage);
             }
 
-            if (sharedChanged) {
-                Plugin.Store.Reconnect();
-            }
-
             if (!Mutable.HideChat && hideChatChanged) {
                 GameFunctions.GameFunctions.SetChatInteractable(true);
             }

ChatTwo/Ui/SettingsTabs/Database.cs

@@ -1,5 +1,9 @@
+using System.Diagnostics;
+using ChatTwo.Code;
 using ChatTwo.Resources;
 using ChatTwo.Util;
+using Dalamud.Game.Text.SeStringHandling;
+using Dalamud.Game.Text.SeStringHandling.Payloads;
 using Dalamud.Interface.Internal.Notifications;
 using ImGuiNET;
 
@@ -46,13 +50,6 @@ internal sealed class Database : ISettingsTab
                 Mutable.LoadPreviousSession = false;
         }
 
-        ImGuiUtil.OptionCheckbox(
-            ref Mutable.SharedMode,
-            Language.Options_SharedMode_Name,
-            string.Format(Language.Options_SharedMode_Description, Plugin.PluginName)
-        );
-        ImGuiUtil.WarningText(string.Format(Language.Options_SharedMode_Warning, Plugin.PluginName));
-
         ImGui.Spacing();
         ImGui.Separator();
         ImGui.Spacing();
@@ -65,18 +62,18 @@ internal sealed class Database : ISettingsTab
         // constant stat calls and spamming the database.
         if (DatabaseLastRefreshTicks + 5 * 1000 < Environment.TickCount64)
         {
-            DatabaseSize = Store.DatabaseSize();
+            DatabaseSize = Plugin.MessageManager.Store.DatabaseSize();
-            DatabaseLogSize = Store.DatabaseLogSize();
+            DatabaseLogSize = Plugin.MessageManager.Store.DatabaseLogSize();
-            DatabaseMessageCount = Plugin.Store.MessageCount();
+            DatabaseMessageCount = Plugin.MessageManager.Store.MessageCount();
             DatabaseLastRefreshTicks = Environment.TickCount64;
         }
 
-        ImGuiUtil.HelpText(string.Format(Language.Options_Database_Metadata_Path, Store.DatabasePath()));
+        ImGuiUtil.HelpText(string.Format(Language.Options_Database_Metadata_Path, MessageManager.DatabasePath()));
         if (ImGui.IsItemClicked(ImGuiMouseButton.Left))
         {
             // Copy the directory path instead of the file path so people can
             // paste it into their file explorer.
-            var path = Path.GetDirectoryName(Store.DatabasePath());
+            var path = Path.GetDirectoryName(MessageManager.DatabasePath());
             ImGui.SetClipboardText(path);
             WrapperUtil.AddNotification(Language.Options_Database_Metadata_CopyConfigPathNotification, NotificationType.Info);
         }
@@ -95,12 +92,12 @@ internal sealed class Database : ISettingsTab
         if (ImGui.IsItemHovered())
             ImGui.SetTooltip(DatabaseLogSize.ToString("N0") + "B");
 
-        ImGuiUtil.HelpText(string.Format(Language.Options_Database_Metadata_MessageCount, DatabaseMessageCount, Store.MessagesLimit));
+        ImGuiUtil.HelpText(string.Format(Language.Options_Database_Metadata_MessageCount, DatabaseMessageCount, MessageStore.MessageQueryLimit));
 
         if (ImGuiUtil.CtrlShiftButton(Language.Options_ClearDatabase_Button, Language.Options_ClearDatabase_Tooltip))
         {
-            Plugin.Log.Warning("Clearing database");
+            Plugin.Log.Warning("Clearing messages from database");
-            Plugin.Store.ClearDatabase();
+            Plugin.MessageManager.Store.ClearMessages();
             foreach (var tab in Plugin.Config.Tabs)
                 tab.Clear();
 
@@ -117,11 +114,18 @@ internal sealed class Database : ISettingsTab
             ImGui.PushTextWrapPos();
             ImGuiUtil.WarningText(Language.Options_Database_Advanced_Warning);
 
-            if (ImGuiUtil.CtrlShiftButton("Checkpoint", "Ctrl+Shift: Database.Checkpoint()"))
+            if (ImGuiUtil.CtrlShiftButton("Perform maintenance", "Ctrl+Shift: MessageManager.Store.PerformMaintenance()"))
-                Plugin.Store.Database.Checkpoint();
+                Plugin.MessageManager.Store.PerformMaintenance();
 
-            if (ImGuiUtil.CtrlShiftButton("Rebuild", "Ctrl+Shift: Database.Rebuild()"))
+            if (ImGuiUtil.CtrlShiftButton("Reload messages from database",
-                Plugin.Store.Database.Rebuild();
+                    "Ctrl+Shift: MessageManager.FilterAllTabs(false)")) {
+                foreach (var tab in Plugin.Config.Tabs)
+                    tab.Clear();
+                Plugin.MessageManager.FilterAllTabs(false);
+            }
+
+            if (ImGuiUtil.CtrlShiftButton("Inject 10,000 messages", "Ctrl+Shift: creates 10,000 unique messages (async)"))
+                new Thread(() => InsertMessages(10_000)).Start();
 
             ImGui.PopTextWrapPos();
             ImGui.TreePop();
@@ -129,4 +133,78 @@ internal sealed class Database : ISettingsTab
 
         ImGui.Spacing();
     }
+
+    private void InsertMessages(int count) {
+        Plugin.Log.Info($"Inserting {count} messages due to user request");
+
+        // Generate
+        var stopwatch = Stopwatch.StartNew();
+        var playerName = Plugin.ClientState.LocalPlayer?.Name.ToString() ?? "Unknown Player";
+        var worldId = Plugin.ClientState.LocalPlayer?.HomeWorld.Id ?? 0;
+        var senderSource = new SeStringBuilder()
+            .AddText("<")
+            .Add(new PlayerPayload(playerName, worldId))
+            .AddText("Random Message")
+            .Add(RawPayload.LinkTerminator)
+            .AddText(">: ")
+            .Build();
+        var senderChunks = ChunkUtil.ToChunks(senderSource, ChunkSource.Sender, ChatType.Debug).ToList();
+        var messages = new List<Message>(count);
+        for (var i = 0; i < count; i++) {
+            var contentSource = new SeStringBuilder()
+                .AddText("Random message payload - ")
+                .AddItalics(Guid.NewGuid().ToString())
+                .Build();
+            var contentChunks = ChunkUtil.ToChunks(contentSource, ChunkSource.Content, ChatType.Debug).ToList();
+
+            messages.Add(new Message(
+                Guid.NewGuid(),
+                Plugin.MessageManager.CurrentContentId,
+                Plugin.MessageManager.CurrentContentId,
+                DateTimeOffset.UtcNow,
+                new ChatCode(10),
+                senderChunks,
+                contentChunks,
+                senderSource,
+                contentSource,
+                new SortCode(ChatType.Debug, ChatSource.Self),
+                Guid.Empty
+            ));
+        }
+
+        var elapsedTicks = stopwatch.ElapsedTicks;
+        stopwatch.Stop();
+        Plugin.Log.Info($"Crafted {count} messages in {elapsedTicks} ticks ({elapsedTicks / TimeSpan.TicksPerMillisecond}ms)");
+
+        // Insert
+        stopwatch = Stopwatch.StartNew();
+        foreach (var message in messages) {
+            Plugin.MessageManager.Store.UpsertMessage(message);
+        }
+
+        elapsedTicks = stopwatch.ElapsedTicks;
+        stopwatch.Stop();
+        Plugin.Log.Info($"Upserted {count} messages in {elapsedTicks} ticks ({elapsedTicks / TimeSpan.TicksPerMillisecond}ms)");
+
+        // Clear tabs during framework frame
+        Plugin.Framework.Run(() => {
+            stopwatch = Stopwatch.StartNew();
+            foreach (var tab in Plugin.Config.Tabs)
+                tab.Clear();
+
+            elapsedTicks = stopwatch.ElapsedTicks;
+            stopwatch.Stop();
+            Plugin.Log.Info(
+                $"Cleared {Plugin.Config.Tabs.Count} tabs in {elapsedTicks} ticks ({elapsedTicks / TimeSpan.TicksPerMillisecond}ms)");
+        }).Wait();
+
+        // Fetch and filter during framework frame
+        Plugin.Framework.Run(() => {
+            stopwatch = Stopwatch.StartNew();
+            Plugin.MessageManager.FilterAllTabs(false);
+            elapsedTicks = stopwatch.ElapsedTicks;
+            stopwatch.Stop();
+            Plugin.Log.Info($"Fetched and filtered all tabs in {elapsedTicks} ticks ({elapsedTicks / TimeSpan.TicksPerMillisecond}ms)");
+        }).Wait();
+    }
 }

ChatTwo/Util/ChunkUtil.cs

@@ -108,7 +108,7 @@ internal static class ChunkUtil {
                     } else if (rawPayload.Data.Length > 5 && rawPayload.Data[1] == 0x27 && rawPayload.Data[3] == 0x07) {
                         // uri payload
                         var uri = new Uri(Encoding.UTF8.GetString(rawPayload.Data[4..]));
-                        link = new URIPayload(uri);
+                        link = new UriPayload(uri);
                     } else if (Equals(rawPayload, RawPayload.LinkTerminator)) {
                         link = null;
                     }

ChatTwo/Util/Payloads.cs

@@ -39,11 +39,11 @@ internal class AchievementPayload : Payload {
 }
 
 
-internal class URIPayload(Uri uri) : Payload
+internal class UriPayload(Uri uri) : Payload
 {
     public override PayloadType Type => (PayloadType) 0x52;
 
-    public Uri Uri { get; init; } = uri;
+    public Uri Uri { get; } = uri;
 
     private static readonly string[] ExpectedSchemes = ["http", "https"];
     private static readonly string DefaultScheme = "https";
@@ -55,7 +55,7 @@ internal class URIPayload(Uri uri) : Payload
     /// <exception cref="UriFormatException">
     /// If the URI is invalid, or if the scheme is not supported.
     /// </exception>
-    public static URIPayload ResolveURI(string rawURI)
+    public static UriPayload ResolveURI(string rawURI)
     {
         ArgumentNullException.ThrowIfNull(rawURI);
 
@@ -64,7 +64,7 @@ internal class URIPayload(Uri uri) : Payload
         {
             if (rawURI.StartsWith($"{scheme}://"))
             {
-                return new URIPayload(new Uri(rawURI));
+                return new UriPayload(new Uri(rawURI));
             }
         }
         if (rawURI.Contains("://"))
@@ -72,7 +72,7 @@ internal class URIPayload(Uri uri) : Payload
             throw new UriFormatException($"Unsupported scheme in URL: {rawURI}");
         }
 
-        return new URIPayload(new Uri($"{DefaultScheme}://{rawURI}"));
+        return new UriPayload(new Uri($"{DefaultScheme}://{rawURI}"));
     }
 
     protected override void DecodeImpl(BinaryReader reader, long endOfStream)

ChatTwo/packages.lock.json

@@ -8,11 +8,26 @@
         "resolved": "2.1.12",
         "contentHash": "Sc0PVxvgg4NQjcI8n10/VfUQBAS4O+Fw2pZrAqBdRMbthYGeogzu5+xmIGCGmsEZ/ukMOBuAqiNiB5qA3MRalg=="
       },
-      "LiteDB": {
+      "MessagePack": {
         "type": "Direct",
-        "requested": "[5.0.17, )",
+        "requested": "[2.5.140, )",
-        "resolved": "5.0.17",
+        "resolved": "2.5.140",
-        "contentHash": "cKPvkdlzIts3ZKu/BzoIc/Y71e4VFKlij4LyioPFATZMot+wB7EAm1FFbZSJez6coJmQUoIg/3yHE1MMU+zOdg=="
+        "contentHash": "nkIsgy8BkIfv40bSz9XZb4q//scI1PF3AYeB5X66nSlIhBIqbdpLz8Qk3gHvnjV3RZglQLO/ityK3eNfLii2NA==",
+        "dependencies": {
+          "MessagePack.Annotations": "2.5.140",
+          "Microsoft.NET.StringTools": "17.6.3",
+          "System.Runtime.CompilerServices.Unsafe": "6.0.0"
+        }
+      },
+      "Microsoft.Data.Sqlite": {
+        "type": "Direct",
+        "requested": "[8.0.4, )",
+        "resolved": "8.0.4",
+        "contentHash": "vgLm03wS+CfsolO7qk4KVuvt0CtzgdjKmoORuwxMmiIF1ow1JlOo1vwfDHfwXnGa5+QEbvOUy3169bBcHshfTg==",
+        "dependencies": {
+          "Microsoft.Data.Sqlite.Core": "8.0.4",
+          "SQLitePCLRaw.bundle_e_sqlite3": "2.1.6"
+        }
       },
       "Pidgin": {
         "type": "Direct",
@@ -37,6 +52,24 @@
         "resolved": "9.0.0",
         "contentHash": "avaBp3FmSCi/PiQhntCeBDYOHejdyTWmFtz4pRBVQQ8vHkmRx+YTk1la9dkYBMlXxRXKckEdH1iI1Fu61JlE7w=="
       },
+      "MessagePack.Annotations": {
+        "type": "Transitive",
+        "resolved": "2.5.140",
+        "contentHash": "JE3vwluOrsJ4t3hnfXzIxJUh6lhx6M/KR8Sark/HOUw1DJ5UKu5JsAnnuaQngg6poFkRx1lzHSLTkxHNJO7+uQ=="
+      },
+      "Microsoft.Data.Sqlite.Core": {
+        "type": "Transitive",
+        "resolved": "8.0.4",
+        "contentHash": "x5FE5m1h31UIDEk0j3r38HtYvsa0fxd5jXzvE/SARI7LecXt/jm4z2SUl6TEoJGQOo9Ow2wg3a0MU2E1TVVAdA==",
+        "dependencies": {
+          "SQLitePCLRaw.core": "2.1.6"
+        }
+      },
+      "Microsoft.NET.StringTools": {
+        "type": "Transitive",
+        "resolved": "17.6.3",
+        "contentHash": "N0ZIanl1QCgvUumEL1laasU0a7sOE5ZwLZVTn0pAePnfhq8P7SvTjF8Axq+CnavuQkmdQpGNXQ1efZtu5kDFbA=="
+      },
       "Microsoft.NETCore.Platforms": {
         "type": "Transitive",
         "resolved": "1.1.0",
@@ -232,6 +265,36 @@
           "SharpDX": "4.2.0"
         }
       },
+      "SQLitePCLRaw.bundle_e_sqlite3": {
+        "type": "Transitive",
+        "resolved": "2.1.6",
+        "contentHash": "BmAf6XWt4TqtowmiWe4/5rRot6GerAeklmOPfviOvwLoF5WwgxcJHAxZtySuyW9r9w+HLILnm8VfJFLCUJYW8A==",
+        "dependencies": {
+          "SQLitePCLRaw.lib.e_sqlite3": "2.1.6",
+          "SQLitePCLRaw.provider.e_sqlite3": "2.1.6"
+        }
+      },
+      "SQLitePCLRaw.core": {
+        "type": "Transitive",
+        "resolved": "2.1.6",
+        "contentHash": "wO6v9GeMx9CUngAet8hbO7xdm+M42p1XeJq47ogyRoYSvNSp0NGLI+MgC0bhrMk9C17MTVFlLiN6ylyExLCc5w==",
+        "dependencies": {
+          "System.Memory": "4.5.3"
+        }
+      },
+      "SQLitePCLRaw.lib.e_sqlite3": {
+        "type": "Transitive",
+        "resolved": "2.1.6",
+        "contentHash": "2ObJJLkIUIxRpOUlZNGuD4rICpBnrBR5anjyfUFQep4hMOIeqW+XGQYzrNmHSVz5xSWZ3klSbh7sFR6UyDj68Q=="
+      },
+      "SQLitePCLRaw.provider.e_sqlite3": {
+        "type": "Transitive",
+        "resolved": "2.1.6",
+        "contentHash": "PQ2Oq3yepLY4P7ll145P3xtx2bX8xF4PzaKPRpw9jZlKvfe4LE/saAV82inND9usn1XRpmxXk7Lal3MTI+6CNg==",
+        "dependencies": {
+          "SQLitePCLRaw.core": "2.1.6"
+        }
+      },
       "System.AppContext": {
         "type": "Transitive",
         "resolved": "4.3.0",
@@ -476,6 +539,11 @@
           "System.Threading": "4.3.0"
         }
       },
+      "System.Memory": {
+        "type": "Transitive",
+        "resolved": "4.5.3",
+        "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA=="
+      },
       "System.Net.Http": {
         "type": "Transitive",
         "resolved": "4.3.0",
@@ -641,6 +709,11 @@
           "Microsoft.NETCore.Targets": "1.1.0"
         }
       },
+      "System.Runtime.CompilerServices.Unsafe": {
+        "type": "Transitive",
+        "resolved": "6.0.0",
+        "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg=="
+      },
       "System.Runtime.Extensions": {
         "type": "Transitive",
         "resolved": "4.3.0",