From bb6c6b0034acb97cf74d8feda8d623e317a24efa Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Fri, 19 Apr 2024 16:57:19 +1000 Subject: [PATCH] feat: replace LiteDB with Sqlite - Replace LiteDB database engine with Sqlite Note: old databases will not be deleted - Message duplication detection improvements - Tolerate parse errors in release builds, log them --- .gitignore | 4 + ChatTwo.Tests/ChatTwo.Tests.csproj | 26 ++ ChatTwo.Tests/MessageStoreTest.cs | 289 ++++++++++++++++++ ChatTwo.Tests/testdata/existing.db | Bin 0 -> 20480 bytes ChatTwo.sln | 6 + ChatTwo/ChatTwo.csproj | 3 +- ChatTwo/Chunk.cs | 57 ++-- ChatTwo/Code/ChatCode.cs | 11 - ChatTwo/Code/ChatType.cs | 2 +- ChatTwo/Configuration.cs | 26 +- ChatTwo/Message.cs | 70 ++--- ChatTwo/MessageManager.cs | 201 ++++++++++++ ChatTwo/MessageStore.cs | 356 ++++++++++++++++++++++ ChatTwo/PayloadHandler.cs | 10 +- ChatTwo/Plugin.cs | 8 +- ChatTwo/Properties/AssemblyInfo.cs | 1 + ChatTwo/Resources/Language.Designer.cs | 36 +-- ChatTwo/Resources/Language.de.resx | 65 ++-- ChatTwo/Resources/Language.es.resx | 65 ++-- ChatTwo/Resources/Language.fr.resx | 65 ++-- ChatTwo/Resources/Language.nl.resx | 9 - ChatTwo/Resources/Language.pt-BR.resx | 65 ++-- ChatTwo/Resources/Language.resx | 12 +- ChatTwo/Resources/Language.ro.resx | 65 ++-- ChatTwo/Resources/Language.ru.resx | 65 ++-- ChatTwo/Resources/Language.sv.resx | 65 ++-- ChatTwo/Resources/Language.zh-Hans.resx | 65 ++-- ChatTwo/Resources/Language.zh-Hant.resx | 65 ++-- ChatTwo/Store.cs | 387 ------------------------ ChatTwo/Ui/ChatLogWindow.cs | 2 +- ChatTwo/Ui/SeStringDebugger.cs | 10 +- ChatTwo/Ui/Settings.cs | 7 +- ChatTwo/Ui/SettingsTabs/Database.cs | 116 +++++-- ChatTwo/Util/ChunkUtil.cs | 2 +- ChatTwo/Util/Payloads.cs | 10 +- ChatTwo/packages.lock.json | 81 ++++- 36 files changed, 1421 insertions(+), 906 deletions(-) create mode 100644 ChatTwo.Tests/ChatTwo.Tests.csproj create mode 100644 ChatTwo.Tests/MessageStoreTest.cs create mode 100644 ChatTwo.Tests/testdata/existing.db create mode 100644 ChatTwo/MessageManager.cs create mode 100644 ChatTwo/MessageStore.cs create mode 100644 ChatTwo/Properties/AssemblyInfo.cs delete mode 100755 ChatTwo/Store.cs diff --git a/.gitignore b/.gitignore index 1db30bf..b9fb6f1 100644 --- a/.gitignore +++ b/.gitignore @@ -363,3 +363,7 @@ MigrationBackup/ # Fody - auto-generated XML schema FodyWeavers.xsd + +TestResults +*.db-shm +*.db-wal diff --git a/ChatTwo.Tests/ChatTwo.Tests.csproj b/ChatTwo.Tests/ChatTwo.Tests.csproj new file mode 100644 index 0000000..2771b9e --- /dev/null +++ b/ChatTwo.Tests/ChatTwo.Tests.csproj @@ -0,0 +1,26 @@ + + + + net8.0-windows + + false + + + + + + + + + + + + + + + + ..\..\AppData\Roaming\XIVLauncher\addon\Hooks\dev\Dalamud.dll + + + + \ No newline at end of file diff --git a/ChatTwo.Tests/MessageStoreTest.cs b/ChatTwo.Tests/MessageStoreTest.cs new file mode 100644 index 0000000..e7199f7 --- /dev/null +++ b/ChatTwo.Tests/MessageStoreTest.cs @@ -0,0 +1,289 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using ChatTwo.Code; +using ChatTwo.Util; +using Dalamud.Game.Text.SeStringHandling; +using Dalamud.Game.Text.SeStringHandling.Payloads; +using JetBrains.Annotations; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Chat2PartyFinderPayload = ChatTwo.Util.PartyFinderPayload; + +namespace ChatTwo.Tests; + +[TestClass] +[TestSubject(typeof(MessageStore))] +public class MessageStoreTest { + // From Message.cs + private static readonly byte[] ExtraChatChannelPayloadBytes = [0, 0x27, 18, 0x20]; + + public TestContext TestContext { get; set; } + + public static string GetImportPath() { + string[] importPaths = [ + @".\TestData", + @"..\TestData", + @"..\..\TestData", + @"..\..\..\TestData", + ]; + var importPath = importPaths.FirstOrDefault(Directory.Exists); + if (string.IsNullOrEmpty(importPath)) { + throw new DirectoryNotFoundException("Could not find the import path"); + } + return importPath; + } + + [TestMethod] + [Timeout(5000)] + public void StoreAndRetrieve() { + var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_"); + var dbPath = Path.Join(tempDir.FullName, "test.db"); + TestContext.WriteLine("Using database path: " + dbPath); + using var store = new MessageStore(dbPath); + + // Write the message. + var input = BigMessage(); + store.UpsertMessage(input); + + // Read the message back. + var messages = store.GetMostRecentMessages().ToList(); + Assert.AreEqual(1, messages.Count); + AssertMessagesEqual(input, messages.First()); + } + + [TestMethod] + [Timeout(5000)] + public void RetrieveMultiple() { + var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_"); + var dbPath = Path.Join(tempDir.FullName, "test.db"); + TestContext.WriteLine("Using database path: " + dbPath); + using var store = new MessageStore(dbPath); + + // Insert 10 messages in the wrong order of date. + var messages = new List(); + const uint receiver = 12345; + var now = DateTimeOffset.UtcNow; + for (var i = 0; i < 10; i++) { + var message = BigMessage(true, receiver, now.AddSeconds(-i)); + TestContext.WriteLine($"Inserting message {i}: {message.Id}"); + store.UpsertMessage(message); + messages.Add(message); + } + + // Insert a message for a different receiver. This shouldn't be returned + // because of the receiver filtering. + var otherReceiverMsg = BigMessage(receiver: receiver + 1, dateTime: now.AddSeconds(1)); + TestContext.WriteLine($"Inserting other receiver message: {otherReceiverMsg.Id}"); + store.UpsertMessage(otherReceiverMsg); + + // Query the most recent 5 messages. Should return the 4 newest messages + // from the list, as well as the different receiver message because we + // aren't filtering. + var outputMessages = store.GetMostRecentMessages(count: 5).ToList(); + var gotIds = outputMessages.Select(m => m.Id).ToList(); + TestContext.WriteLine($"Query 1 got IDs: {string.Join(", ", gotIds)}"); + AssertGuidsEqual(new List { + messages[3].Id, + messages[2].Id, + messages[1].Id, + messages[0].Id, + otherReceiverMsg.Id + }, gotIds); + + // Query the most recent 5 messages but filter by receiver ID. + outputMessages = store.GetMostRecentMessages(receiver: receiver, count: 5).ToList(); + gotIds = outputMessages.Select(m => m.Id).ToList(); + TestContext.WriteLine($"Query 2 got IDs: {string.Join(", ", gotIds)}"); + AssertGuidsEqual(new List { + messages[4].Id, + messages[3].Id, + messages[2].Id, + messages[1].Id, + messages[0].Id, + }, gotIds); + + // Query the most recent 5 messages but only since a specific date. + outputMessages = store.GetMostRecentMessages(receiver, since: messages[1].Date, count: 5).ToList(); + gotIds = outputMessages.Select(m => m.Id).ToList(); + TestContext.WriteLine($"Query 3 got IDs: {string.Join(", ", gotIds)}"); + AssertGuidsEqual(new List { + messages[1].Id, + messages[0].Id, + }, gotIds); + } + + [TestMethod] + [Timeout(5000)] + // This test guards against the data format changing in an incompatible way. + public void RetrieveExisting() { + var input = BigMessage(uniqId: false); + + var dbPath = Path.Join(GetImportPath(), "existing.db"); + TestContext.WriteLine($"Using existing database: {dbPath}"); + Assert.IsTrue(File.Exists(dbPath)); + + // Uncomment this section to regenerate the existing database. + /* + File.Delete(dbPath); + using (var newStore = new MessageStore(dbPath)) { + newStore.UpsertMessage(input); + } + */ + + using var store = new MessageStore(dbPath); + var output = store.GetMostRecentMessages().ToList(); + Assert.AreEqual(1, output.Count); + AssertMessagesEqual(input, output[0]); + } + + [TestMethod] + [Timeout(30_000)] + public void ProfileMany() { + const int count = 20_000; + + var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_"); + var dbPath = Path.Join(tempDir.FullName, "test.db"); + TestContext.WriteLine("Using database path: " + dbPath); + using var store = new MessageStore(dbPath); + + for (var i = 0; i < count; i++) { + var message = BigMessage(uniqId: true); + store.UpsertMessage(message); + } + + var messages = store.GetMostRecentMessages(count: count).ToList(); + Assert.AreEqual(count, messages.Count); + foreach (var message in messages) { + // Load the message because they are lazily parsed. + Assert.IsTrue(message.Id != Guid.Empty); + } + } + + private static Message BigMessage(bool uniqId = true, uint receiver = 12345, DateTimeOffset? dateTime = null) { + // NOTE: These values aren't valid in the game. + // NOTE: we can't test UiForeground, UiGlow, or AutoTranslatePayload + // because they load data from the game. + var senderSeString = new SeStringBuilder() + .AddText("<") + .Add(new PlayerPayload("Player Name", 12345)) + .AddItalics("Player Name") + .Add(RawPayload.LinkTerminator) + .AddText(">: ") + .Build(); + var extraChatId = Guid.Parse("03d9e6d4-dc1a-4005-bbe7-66b8c3529277"); + var contentSeString = new SeStringBuilder() + .Add(new RawPayload(ExtraChatChannelPayloadBytes.Concat(extraChatId.ToByteArray()).ToArray())) + .AddIcon(BitmapFontIcon.IslandSanctuary) + .AddMapLink(1, 2, 3, 4) + .AddText("map") + .Add(RawPayload.LinkTerminator) + .AddQuestLink(12345) + .AddText("quest") + .Add(RawPayload.LinkTerminator) + .Add(new DalamudLinkPayload()) + .AddText("dalamud") + .Add(RawPayload.LinkTerminator) + .AddStatusLink(12345) + .AddText("status") + .Add(RawPayload.LinkTerminator) + .AddPartyFinderLink(12345) + .AddText("party finder") + .Add(RawPayload.LinkTerminator) + .Build(); + + // Add Chat 2 specific payloads (that can't be serialized into the + // SeString). + var contentChunks = ChunkUtil.ToChunks(contentSeString, ChunkSource.Content, ChatType.Say).ToList(); + contentChunks = contentChunks.Concat([ + new TextChunk(ChunkSource.Content, new Chat2PartyFinderPayload(12345), "chat 2 party finder"), + new TextChunk(ChunkSource.Content, new AchievementPayload(12345), "chat 2 achievement"), + new TextChunk(ChunkSource.Content, new UriPayload(new Uri("https://dalamud.dev")), "chat 2 uri"), + ]).ToList(); + + return new Message( + uniqId ? Guid.NewGuid() : Guid.Parse("f011343e-6a21-49e5-a6f9-238f0f1f8c2c"), + receiver, + 54321, + dateTime ?? DateTimeOffset.FromUnixTimeMilliseconds(1713520182440), + new ChatCode(12345), + ChunkUtil.ToChunks(senderSeString, ChunkSource.Sender, ChatType.Debug).ToList(), + contentChunks, + senderSeString, + contentSeString, + new SortCode(ChatType.Crafting, ChatSource.AlliancePet), + extraChatId + ); + } + + private void AssertMessagesEqual(Message input, Message output) { + // Check basic fields. + Assert.AreEqual(input.Id, output.Id); + Assert.AreEqual(input.Receiver, output.Receiver); + Assert.AreEqual(input.ContentId, output.ContentId); + // Assert time is within 1 second + TestContext.WriteLine($"Input date: {input.Date.ToUniversalTime()}"); + TestContext.WriteLine($"Output date: {output.Date.ToUniversalTime()}"); + var timeDifference = Math.Abs(input.Date.ToUniversalTime().Subtract(output.Date.ToUniversalTime()).TotalSeconds); + TestContext.WriteLine($"Time difference: {timeDifference}s"); + Assert.IsTrue(timeDifference < 1); + Assert.AreEqual(input.Code.Raw, output.Code.Raw); + Assert.AreEqual($"{input.SenderSource.Encode():X}", $"{output.SenderSource.Encode():X}"); + Assert.AreEqual($"{input.ContentSource.Encode():X}", $"{output.ContentSource.Encode():X}"); + Assert.AreEqual(input.SortCode, output.SortCode); + Assert.AreEqual(input.ExtraChatChannel, output.ExtraChatChannel); + + // Check chunks. + AssertChunksEqual(input.Sender, output.Sender); + AssertChunksEqual(input.Content, output.Content); + } + + private static void AssertChunksEqual(IReadOnlyList inputChunks, IReadOnlyList outputChunks) { + Assert.AreEqual(inputChunks.Count, outputChunks.Count); + for (var i = 0; i < inputChunks.Count; i++) { + var inputChunk = inputChunks[i]; + var outputChunk = outputChunks[i]; + Assert.AreEqual(inputChunk.Source, outputChunk.Source); + switch (inputChunk.Link) { + case AchievementPayload inputAchievementPayload: + Assert.AreEqual(inputAchievementPayload.Id, ((AchievementPayload) outputChunk.Link)!.Id); + break; + case Chat2PartyFinderPayload inputPartyFinderPayload: + Assert.AreEqual(inputPartyFinderPayload.Id, ((Chat2PartyFinderPayload) outputChunk.Link)!.Id); + break; + case UriPayload inputUriPayload: + Assert.AreEqual(inputUriPayload.Uri, ((UriPayload) outputChunk.Link)!.Uri); + break; + case null: + Assert.IsTrue(outputChunk.Link == null); + break; + default: + Assert.AreEqual($"{inputChunk.Link.Encode():X}", $"{outputChunk.Link!.Encode():X}"); + break; + } + + switch (inputChunk) { + case TextChunk inputTextChunk: + var outputTextChunk = (TextChunk)outputChunk; + Assert.AreEqual(inputTextChunk.FallbackColour, outputTextChunk.FallbackColour); + Assert.AreEqual(inputTextChunk.Foreground, outputTextChunk.Foreground); + Assert.AreEqual(inputTextChunk.Glow, outputTextChunk.Glow); + Assert.AreEqual(inputTextChunk.Italic, outputTextChunk.Italic); + Assert.AreEqual(inputTextChunk.Content, outputTextChunk.Content); + break; + case IconChunk inputIconChunk: + Assert.AreEqual(inputIconChunk.Icon, ((IconChunk) outputChunk).Icon); + break; + default: + throw new Exception("Unknown chunk type"); + } + } + } + + private static void AssertGuidsEqual(IReadOnlyList expected, IReadOnlyList got) { + Assert.AreEqual(expected.Count, got.Count); + for (var i = 0; i < expected.Count; i++) { + Assert.AreEqual(expected[i].ToString(), got[i].ToString()); + } + } +} diff --git a/ChatTwo.Tests/testdata/existing.db b/ChatTwo.Tests/testdata/existing.db new file mode 100644 index 0000000000000000000000000000000000000000..b67eb8cfe092b9c41713b5598cad421c4d947b56 GIT binary patch literal 20480 zcmeI3PiWg#9LL{#vXa_ya9aj}g5llVjJc8j#EC;0v1O%%Wl3izqh*YsSaK|C%dYgC zCYOz}($m->J(L|s*&nu@wjIWH3CmzRjP26H20eGM+jr_mJ?j zf8XC%KYf0Z;0auLVaZ}9t+kw{&S(amMuebwN)bYY?=s(mA@NBl_`*MN&-|z@oJ&Cr1jJ{9-&W+MS~WXR*Lj;dGYx| z`6_)*yGoZ=R_XHE($X`OCX@71$1(=$K6J=K=YOq=nNhMN8>PtlR#c3lv{)3&xLvp71YH{0}v#p<+aH5!&{ zR$I0~+q@N)YqjiQ1si{3uGV#?wz$f)t1ZK_HwR5BCcpk1uM0iogFf*J)q2Oi_R`C= z>29|5>a{@&AKXzM$FcCLsI)px)g0o(Eh=V(IYZCDKm1Y2f0wxFa^nQTT& z<_hUlGMm??lZD(uKAFzUE+`ABg;_PNDtUB2h2QvXKlGlGzq5<(;yt{#_r+Tm#dP-WBfNF zY@;L*4*iNrNal$Ve#CoCy&VK2+*AHp$zw0{_Vo^5zz4xmk_g3+7kSsv8+x;21hH|F zh=jX|2ZVNQ-{C8N)^P*D7)hLp2*EF&!G~?#VLP-Y7K)B1!rvmKgvF{PzU;qIi00e*l5C8%|00;m9AOHk_ z01yBICyYQsS_>cNb(pvQe~e;}PuK}Uxj+C200AHX1b_e#00KY&2mk>f00e-*F$l;K y2`hPT{r?EX9vveEBtQTN00AHX1b_e#00KY&2mk>f00e-*2_g`d - + + diff --git a/ChatTwo/Chunk.cs b/ChatTwo/Chunk.cs index 245b26c..fe5aa44 100755 --- a/ChatTwo/Chunk.cs +++ b/ChatTwo/Chunk.cs @@ -1,15 +1,22 @@ using ChatTwo.Code; using Dalamud.Game.Text.SeStringHandling; -using LiteDB; +using MessagePack; namespace ChatTwo; -internal abstract class Chunk { - [BsonIgnore] +[Union(0, typeof(TextChunk))] +[Union(1, typeof(IconChunk))] +[MessagePackObject] +public abstract class Chunk { + [IgnoreMember] internal Message? Message { get; set; } - internal ChunkSource Source { get; set; } - internal Payload? Link { get; set; } + [Key(0)] + public ChunkSource Source { get; set; } + + [Key(1)] + [MessagePackFormatter(typeof(PayloadMessagePackFormatter))] + public Payload? Link { get; set; } protected Chunk(ChunkSource source, Payload? link) { Source = source; @@ -38,27 +45,38 @@ internal abstract class Chunk { } } -internal enum ChunkSource { +public enum ChunkSource { None, Sender, Content, } -internal class TextChunk : Chunk { - internal ChatType? FallbackColour { get; set; } - internal uint? Foreground { get; set; } - internal uint? Glow { get; set; } - internal bool Italic { get; set; } - internal string Content { get; set; } +[MessagePackObject] +public class TextChunk : Chunk { + [Key(2)] + public ChatType? FallbackColour { get; set; } + [Key(3)] + public uint? Foreground { get; set; } + [Key(4)] + public uint? Glow { get; set; } + [Key(5)] + public bool Italic { get; set; } + [Key(6)] + public string Content { get; set; } internal TextChunk(ChunkSource source, Payload? link, string content) : base(source, link) { Content = content; } - #pragma warning disable CS8618 - public TextChunk() : base(ChunkSource.None, null) { + // ReSharper disable once UnusedMember.Global // Used by MessagePack + public TextChunk(ChunkSource source, Payload? link, ChatType? fallbackColour, uint? foreground, uint? glow, + bool italic, string content) : base(source, link) { + FallbackColour = fallbackColour; + Foreground = foreground; + Glow = glow; + Italic = italic; + Content = content; } - #pragma warning restore CS8618 /// /// Creates a new TextChunk with identical styling to this one. @@ -74,13 +92,12 @@ internal class TextChunk : Chunk { } } -internal class IconChunk : Chunk { - internal BitmapFontIcon Icon { get; set; } +[MessagePackObject] +public class IconChunk : Chunk { + [Key(2)] + public BitmapFontIcon Icon { get; set; } public IconChunk(ChunkSource source, Payload? link, BitmapFontIcon icon) : base(source, link) { Icon = icon; } - - public IconChunk() : base(ChunkSource.None, null) { - } } diff --git a/ChatTwo/Code/ChatCode.cs b/ChatTwo/Code/ChatCode.cs index 029c06d..a90e0ac 100755 --- a/ChatTwo/Code/ChatCode.cs +++ b/ChatTwo/Code/ChatCode.cs @@ -1,5 +1,3 @@ -using LiteDB; - namespace ChatTwo.Code; internal class ChatCode @@ -21,15 +19,6 @@ internal class ChatCode Target = SourceFrom(7); } - [BsonCtor] - public ChatCode(ushort raw, ChatType type, ChatSource source, ChatSource target) - { - Raw = raw; - Type = type; - Source = source; - Target = target; - } - internal ChatType Parent() => Type switch { ChatType.Say => ChatType.Say, diff --git a/ChatTwo/Code/ChatType.cs b/ChatTwo/Code/ChatType.cs index 28d836f..63a653d 100755 --- a/ChatTwo/Code/ChatType.cs +++ b/ChatTwo/Code/ChatType.cs @@ -1,7 +1,7 @@ namespace ChatTwo.Code; [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1028:Enum Storage should be Int32")] -internal enum ChatType : ushort +public enum ChatType : ushort { Debug = 1, Urgent = 2, diff --git a/ChatTwo/Configuration.cs b/ChatTwo/Configuration.cs index 14446cb..3a697bf 100755 --- a/ChatTwo/Configuration.cs +++ b/ChatTwo/Configuration.cs @@ -35,7 +35,6 @@ internal class Configuration : IPluginConfiguration public bool DatabaseBattleMessages; public bool LoadPreviousSession; public bool FilterIncludePreviousSessions; - public bool SharedMode; public bool SortAutoTranslate; public bool CollapseDuplicateMessages; public bool PlaySounds = true; @@ -80,7 +79,6 @@ internal class Configuration : IPluginConfiguration DatabaseBattleMessages = other.DatabaseBattleMessages; LoadPreviousSession = other.LoadPreviousSession; FilterIncludePreviousSessions = other.FilterIncludePreviousSessions; - SharedMode = other.SharedMode; SortAutoTranslate = other.SortAutoTranslate; CollapseDuplicateMessages = other.CollapseDuplicateMessages; PlaySounds = other.PlaySounds; @@ -197,16 +195,16 @@ internal class Tab [NonSerialized] public List Messages = new(); + [NonSerialized] + public HashSet TrackedMessageIds = new(); ~Tab() { MessagesMutex.Dispose(); } - internal bool Contains(Message message) - { - return Messages.Any(m => m.Hash == message.Hash); + internal bool Contains(Message message) { + return TrackedMessageIds.Contains(message.Id); } - internal bool Matches(Message message) - { + internal bool Matches(Message message) { if (message.ExtraChatChannel != Guid.Empty) return ExtraChatAll || ExtraChatChannels.Contains(message.ExtraChatChannel); @@ -216,23 +214,25 @@ internal class Tab || sources.HasFlag(message.Code.Source)); } - internal void AddMessage(Message message, bool unread = true) - { + internal void AddMessage(Message message, bool unread = true) { + if (Contains(message)) return; MessagesMutex.Wait(); + TrackedMessageIds.Add(message.Id); Messages.Add(message); - while (Messages.Count > Store.MessagesLimit) + while (Messages.Count > MessageManager.MessageDisplayLimit) { + TrackedMessageIds.Remove(Messages[0].Id); Messages.RemoveAt(0); - + } MessagesMutex.Release(); if (unread) Unread += 1; } - internal void Clear() - { + internal void Clear() { MessagesMutex.Wait(); Messages.Clear(); + TrackedMessageIds.Clear(); MessagesMutex.Release(); } diff --git a/ChatTwo/Message.cs b/ChatTwo/Message.cs index 4f5803e..1f83b63 100755 --- a/ChatTwo/Message.cs +++ b/ChatTwo/Message.cs @@ -2,21 +2,26 @@ using ChatTwo.Code; using ChatTwo.Util; using Dalamud.Game.Text.SeStringHandling; using Dalamud.Game.Text.SeStringHandling.Payloads; -using LiteDB; using System.Text.RegularExpressions; namespace ChatTwo; internal class SortCode { - internal ChatType Type { get; set; } - internal ChatSource Source { get; set; } + internal ChatType Type { get; } + internal ChatSource Source { get; } internal SortCode(ChatType type, ChatSource source) { Type = type; Source = source; } - public SortCode() { + internal SortCode(uint raw) { + Type = (ChatType)(raw >> 16); + Source = (ChatSource)(raw & 0xFFFF); + } + + internal uint Encode() { + return ((uint) Type << 16) | (uint) Source; } private bool Equals(SortCode other) { @@ -43,18 +48,11 @@ internal class SortCode { } internal class Message { - // ReSharper disable once UnusedMember.Global - internal ObjectId Id { get; } = ObjectId.NewObjectId(); + internal Guid Id { get; } = Guid.NewGuid(); internal ulong Receiver { get; } internal ulong ContentId { get; set; } - [BsonIgnore] - internal float? Height; - - [BsonIgnore] - internal bool IsVisible; - - internal DateTime Date { get; } + internal DateTimeOffset Date { get; } internal ChatCode Code { get; } internal List Sender { get; } internal List Content { get; } @@ -65,11 +63,14 @@ internal class Message { internal SortCode SortCode { get; } internal Guid ExtraChatChannel { get; } + // Not stored in the database: internal int Hash { get; } + internal float? Height { get; set; } + internal bool IsVisible { get; set; } internal Message(ulong receiver, ChatCode code, List sender, List content, SeString senderSource, SeString contentSource) { Receiver = receiver; - Date = DateTime.UtcNow; + Date = DateTimeOffset.UtcNow; Code = code; Sender = sender; Content = ReplaceContentURLs(content); @@ -84,44 +85,23 @@ internal class Message { } } - internal Message(ObjectId id, ulong receiver, ulong contentId, DateTime date, BsonDocument code, BsonArray sender, BsonArray content, BsonValue senderSource, BsonValue contentSource, BsonDocument sortCode) { + internal Message(Guid id, ulong receiver, ulong contentId, DateTimeOffset date, ChatCode code, List sender, List content, SeString senderSource, SeString contentSource, SortCode sortCode, Guid extraChatChannel) { Id = id; Receiver = receiver; ContentId = contentId; Date = date; - Code = BsonMapper.Global.ToObject(code); - Sender = BsonMapper.Global.Deserialize>(sender); + Code = code; + Sender = sender; // Don't call ReplaceContentURLs here since we're loading the message // from the database and it should already have parsed URL data. - Content = BsonMapper.Global.Deserialize>(content); - SenderSource = BsonMapper.Global.Deserialize(senderSource); - ContentSource = BsonMapper.Global.Deserialize(contentSource); - SortCode = BsonMapper.Global.ToObject(sortCode); - ExtraChatChannel = ExtractExtraChatChannel(); + Content = content; + SenderSource = senderSource; + ContentSource = contentSource; + SortCode = sortCode; + ExtraChatChannel = extraChatChannel; Hash = GenerateHash(); - foreach (var chunk in Sender.Concat(Content)) { - chunk.Message = this; - } - } - - internal Message(ObjectId id, ulong receiver, ulong contentId, DateTime date, BsonDocument code, BsonArray sender, BsonArray content, BsonValue senderSource, BsonValue contentSource, BsonDocument sortCode, BsonValue extraChatChannel) { - Id = id; - Receiver = receiver; - ContentId = contentId; - Date = date; - Code = BsonMapper.Global.ToObject(code); - Sender = BsonMapper.Global.Deserialize>(sender); - // Don't call ReplaceContentURLs here since we're loading the message - // from the database and it should already have parsed URL data. - Content = BsonMapper.Global.Deserialize>(content); - SenderSource = BsonMapper.Global.Deserialize(senderSource); - ContentSource = BsonMapper.Global.Deserialize(contentSource); - SortCode = BsonMapper.Global.ToObject(sortCode); - ExtraChatChannel = BsonMapper.Global.Deserialize(extraChatChannel); - Hash = GenerateHash(); - - foreach (var chunk in Sender.Concat(Content)) { + foreach (var chunk in sender.Concat(content)) { chunk.Message = this; } } @@ -203,7 +183,7 @@ internal class Message { // Create a new TextChunk with a URIPayload for the URL text. try { - var link = URIPayload.ResolveURI(match.Value); + var link = UriPayload.ResolveURI(match.Value); AddChunkWithMessage(text.NewWithStyle(chunk.Source, link, match.Value)); } catch (UriFormatException) diff --git a/ChatTwo/MessageManager.cs b/ChatTwo/MessageManager.cs new file mode 100644 index 0000000..f79d7e5 --- /dev/null +++ b/ChatTwo/MessageManager.cs @@ -0,0 +1,201 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using ChatTwo.Code; +using ChatTwo.Resources; +using ChatTwo.Util; +using Dalamud.Game.Text; +using Dalamud.Game.Text.SeStringHandling; +using Dalamud.Interface.Internal.Notifications; +using Dalamud.Plugin.Services; +using Lumina.Excel.GeneratedSheets; + +namespace ChatTwo; + +internal class MessageManager : IDisposable { + internal const int MessageDisplayLimit = 10_000; + + private Plugin Plugin { get; } + internal MessageStore Store { get; } + + private ConcurrentQueue<(uint, Message)> Pending { get; } = new(); + private Stopwatch MaintenanceTimer { get; } = new(); + private Dictionary Formats { get; } = new(); + private ulong LastContentId { get; set; } + + internal ulong CurrentContentId { + get { + var contentId = Plugin.ClientState.LocalContentId; + return contentId == 0 ? LastContentId : contentId; + } + } + + internal MessageManager(Plugin plugin) { + Plugin = plugin; + MaintenanceTimer.Start(); + Store = new MessageStore(DatabasePath()); + + Plugin.ChatGui.ChatMessageUnhandled += ChatMessage; + Plugin.Framework.Update += GetMessageInfo; + Plugin.Framework.Update += UpdateReceiver; + Plugin.ClientState.Logout += Logout; + } + + public void Dispose() { + Plugin.ClientState.Logout -= Logout; + Plugin.Framework.Update -= UpdateReceiver; + Plugin.Framework.Update -= GetMessageInfo; + Plugin.ChatGui.ChatMessageUnhandled -= ChatMessage; + + Store.Dispose(); + } + + internal static string DatabasePath() { + var dir = Plugin.Interface.ConfigDirectory; + dir.Create(); + return Path.Join(dir.FullName, "chat-sqlite.db"); + } + + private void Logout() { + LastContentId = 0; + } + + private void UpdateReceiver(IFramework framework) { + var contentId = Plugin.ClientState.LocalContentId; + if (contentId != 0) + LastContentId = contentId; + } + + private void GetMessageInfo(IFramework framework) { + if (MaintenanceTimer.Elapsed > TimeSpan.FromMinutes(5)) { + MaintenanceTimer.Restart(); + new Thread(() => Store.PerformMaintenance()).Start(); + } + + if (!Pending.TryDequeue(out var entry)) + return; + + var contentId = Plugin.Functions.Chat.GetContentIdForEntry(entry.Item1); + entry.Item2.ContentId = contentId ?? 0; + if (Plugin.Config.DatabaseBattleMessages || !entry.Item2.Code.IsBattle()) + Store.UpsertMessage(entry.Item2); + } + + internal void AddMessage(Message message, Tab? currentTab) { + if (Plugin.Config.DatabaseBattleMessages || !message.Code.IsBattle()) + Store.UpsertMessage(message); + + var currentMatches = currentTab?.Matches(message) ?? false; + + foreach (var tab in Plugin.Config.Tabs) { + var unread = !(tab.UnreadMode == UnreadMode.Unseen && currentTab != tab && currentMatches); + + if (tab.Matches(message)) + tab.AddMessage(message, unread); + } + } + + internal void FilterAllTabs(bool unread = true) { + DateTimeOffset? since = null; + if (!Plugin.Config.FilterIncludePreviousSessions) + since = Plugin.GameStarted; + + var messages = Store.GetMostRecentMessages(CurrentContentId, since); + foreach (var message in messages) { + foreach (var tab in Plugin.Config.Tabs.Where(tab => tab.Matches(message))) { + tab.AddMessage(message, unread); + } + } + + if (messages.DidError) + WrapperUtil.AddNotification(Language.LoadMessages_Error, NotificationType.Error); + } + + public (SeString? Sender, SeString? Message) LastMessage = (null, null); + private void ChatMessage(XivChatType type, uint senderId, SeString sender, SeString message) { + var chatCode = new ChatCode((ushort) type); + + NameFormatting? formatting = null; + if (sender.Payloads.Count > 0) + formatting = FormatFor(chatCode.Type); + + LastMessage = (sender, message); + var senderChunks = new List(); + if (formatting is { IsPresent: true }) { + senderChunks.Add(new TextChunk(ChunkSource.None, null, formatting.Before) { + FallbackColour = chatCode.Type, + }); + senderChunks.AddRange(ChunkUtil.ToChunks(sender, ChunkSource.Sender, chatCode.Type)); + senderChunks.Add(new TextChunk(ChunkSource.None, null, formatting.After) { + FallbackColour = chatCode.Type, + }); + } + + var messageChunks = ChunkUtil.ToChunks(message, ChunkSource.Content, chatCode.Type).ToList(); + + var msg = new Message(CurrentContentId, chatCode, senderChunks, messageChunks, sender, message); + AddMessage(msg, Plugin.ChatLogWindow.CurrentTab ?? null); + + var idx = Plugin.Functions.GetCurrentChatLogEntryIndex(); + if (idx != null) + Pending.Enqueue((idx.Value - 1, msg)); + } + + internal class NameFormatting { + internal string Before { get; private set; } = string.Empty; + internal string After { get; private set; } = string.Empty; + internal bool IsPresent { get; private set; } = true; + + internal static NameFormatting Empty() { + return new NameFormatting { IsPresent = false, }; + } + + internal static NameFormatting Of(string before, string after) { + return new NameFormatting + { + Before = before, + After = after, + }; + } + } + + private NameFormatting? FormatFor(ChatType type) { + if (Formats.TryGetValue(type, out var cached)) + return cached; + + var logKind = Plugin.DataManager.GetExcelSheet()!.GetRow((ushort) type); + if (logKind == null) + return null; + + var format = (SeString) logKind.Format; + static bool IsStringParam(Payload payload, byte num) { + var data = payload.Encode(); + return data.Length >= 5 && data[1] == 0x29 && data[4] == num + 1; + } + + var firstStringParam = format.Payloads.FindIndex(payload => IsStringParam(payload, 1)); + var secondStringParam = format.Payloads.FindIndex(payload => IsStringParam(payload, 2)); + + if (firstStringParam == -1 || secondStringParam == -1) + return NameFormatting.Empty(); + + var before = format.Payloads + .GetRange(0, firstStringParam) + .Where(payload => payload is ITextProvider) + .Cast() + .Select(text => text.Text); + var after = format.Payloads + .GetRange(firstStringParam + 1, secondStringParam - firstStringParam) + .Where(payload => payload is ITextProvider) + .Cast() + .Select(text => text.Text); + + var nameFormatting = NameFormatting.Of( + string.Join("", before), + string.Join("", after) + ); + + Formats[type] = nameFormatting; + + return nameFormatting; + } +} diff --git a/ChatTwo/MessageStore.cs b/ChatTwo/MessageStore.cs new file mode 100644 index 0000000..611ebd3 --- /dev/null +++ b/ChatTwo/MessageStore.cs @@ -0,0 +1,356 @@ +using System.Buffers; +using System.Collections; +using System.Data.Common; +using ChatTwo.Code; +using ChatTwo.Util; +using Dalamud.Game.Text.SeStringHandling; +using MessagePack; +using MessagePack.Formatters; +using MessagePack.Resolvers; +using Microsoft.Data.Sqlite; +using DalamudUtil = Dalamud.Utility.Util; +using Encoding = System.Text.Encoding; + +namespace ChatTwo; + +internal static class DbExtensions { + internal static void Execute(this DbConnection conn, string sql) { + using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + cmd.ExecuteNonQuery(); + } +} + +internal enum PayloadMessagePackType : byte { + Achievement, + PartyFinder, + Uri, + Other = 255, +} + +public class PayloadMessagePackFormatter : IMessagePackFormatter { + public void Serialize(ref MessagePackWriter writer, Payload? value, MessagePackSerializerOptions options) { + if (value == null) { + writer.WriteNil(); + return; + } + + writer.WriteArrayHeader(2); + switch (value) { + case AchievementPayload achievementPayload: + writer.WriteUInt8((byte) PayloadMessagePackType.Achievement); + writer.WriteUInt32(achievementPayload.Id); + break; + case PartyFinderPayload partyFinderPayload: + writer.WriteUInt8((byte) PayloadMessagePackType.PartyFinder); + writer.WriteUInt32(partyFinderPayload.Id); + break; + case UriPayload uriPayload: + writer.WriteUInt8((byte) PayloadMessagePackType.Uri); + writer.WriteString(Encoding.UTF8.GetBytes(uriPayload.Uri.ToString())); + break; + default: + writer.WriteUInt8((byte) PayloadMessagePackType.Other); + writer.Write(value.Encode()); + break; + } + } + + public Payload? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { + if (reader.TryReadNil()) + return null; + + if (reader.ReadArrayHeader() != 2) + throw new InvalidOperationException("Invalid array count for Payload object"); + + var type = (PayloadMessagePackType) reader.ReadByte(); + switch (type) { + case PayloadMessagePackType.Achievement: + return new AchievementPayload(reader.ReadUInt32()); + case PayloadMessagePackType.PartyFinder: + return new PartyFinderPayload(reader.ReadUInt32()); + case PayloadMessagePackType.Uri: + return new UriPayload(new Uri(reader.ReadString() ?? "")); + case PayloadMessagePackType.Other: + default: + var bytes = reader.ReadBytes() ?? new ReadOnlySequence(); + var binReader = new BinaryReader(new MemoryStream(bytes.ToArray())); + return Payload.Decode(binReader); + } + } +} + +public class SeStringMessagePackFormatter : IMessagePackFormatter { + public void Serialize(ref MessagePackWriter writer, SeString value, MessagePackSerializerOptions options) { + options.Resolver.GetFormatter>()!.Serialize(ref writer, value.Payloads, options); + } + + public SeString Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { + return new SeString(options.Resolver.GetFormatter>()!.Deserialize(ref reader, options)); + } +} + +internal class MessageStore : IDisposable { + internal const int MessageQueryLimit = 10_000; + + private string DbPath { get; } + + private SqliteConnection Connection { get; set; } + + internal static readonly MessagePackSerializerOptions MsgPackOptions = MessagePackSerializerOptions.Standard + .WithResolver(CompositeResolver.Create( + new IMessagePackFormatter[] { + new PayloadMessagePackFormatter(), + new SeStringMessagePackFormatter(), + }, + new IFormatterResolver[] { StandardResolver.Instance })); + + internal MessageStore(string dbPath) { + DbPath = dbPath; + Connection = Connect(); + Migrate(); + } + + public void Dispose() { + Connection.Close(); + Connection.Dispose(); + // Closing the connection doesn't immediately release the file. + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + + private SqliteConnection Connect() { + var uriBuilder = new SqliteConnectionStringBuilder { + DataSource = DbPath, + DefaultTimeout = 5, + Pooling = false, + Mode = SqliteOpenMode.ReadWriteCreate, + }; + var conn = new SqliteConnection(uriBuilder.ToString()); + conn.Open(); + conn.Execute(@"PRAGMA journal_mode=WAL;"); + conn.Execute(@"PRAGMA synchronous=NORMAL;"); + if (DalamudUtil.IsWine()) + conn.Execute(@"PRAGMA cache_size = 32768;"); + return conn; + } + + private void Migrate() { + // TODO: this should be improved/swapped out for a library at some + // point. + Connection.Execute(@" + CREATE TABLE IF NOT EXISTS messages ( + Id BLOB PRIMARY KEY NOT NULL, -- Guid + Receiver INTEGER NOT NULL, -- uint64 (first bits are always 0) + ContentId INTEGER NOT NULL, -- uint64 (first bits are always 0) + Date INTEGER NOT NULL, -- unix timestamp with millisecond precision + Code INTEGER NOT NULL, -- ChatCode encoding + Sender BLOB NOT NULL, -- Chunk[] msgpack + Content BLOB NOT NULL, -- Chunk[] msgpack + SenderSource BLOB NOT NULL, -- SeString + ContentSource BLOB NOT NULL, -- SeString + SortCode INTEGER NOT NULL, -- SortCode encoding + ExtraChatChannel BLOB NOT NULL -- Guid + ); + + CREATE INDEX IF NOT EXISTS idx_messages_receiver ON messages (Receiver); + CREATE INDEX IF NOT EXISTS idx_messages_date ON messages (Date); + "); + } + + internal void Reconnect() { + Connection.Close(); + Connection.Dispose(); + Connection = Connect(); + } + + internal void ClearMessages() { + Connection.Execute("DELETE FROM messages;"); + PerformMaintenance(); + } + + internal void PerformMaintenance() { + Connection.Execute(@" + VACUUM; + REINDEX messages; + ANALYZE; + "); + } + + internal long DatabaseSize() { + return !File.Exists(DbPath) ? 0 : new FileInfo(DbPath).Length; + } + + private string LogPath => DbPath + "-wal"; + + internal long DatabaseLogSize() { + return !File.Exists(LogPath) ? 0 : new FileInfo(LogPath).Length; + } + + internal int MessageCount() + { + var cmd = Connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM messages;"; + return Convert.ToInt32(cmd.ExecuteScalar()); + } + + internal void UpsertMessage(Message message) { + var cmd = Connection.CreateCommand(); + cmd.CommandText = @" + INSERT INTO messages ( + Id, + Receiver, + ContentId, + Date, + Code, + Sender, + Content, + SenderSource, + ContentSource, + SortCode, + ExtraChatChannel + ) VALUES ( + $Id, + $Receiver, + $ContentId, + $Date, + $Code, + $Sender, + $Content, + $SenderSource, + $ContentSource, + $SortCode, + $ExtraChatChannel + ) + ON CONFLICT (id) DO UPDATE SET + Receiver = excluded.Receiver, + ContentId = excluded.ContentId, + Date = excluded.Date, + Code = excluded.Code, + Sender = excluded.Sender, + Content = excluded.Content, + SenderSource = excluded.SenderSource, + ContentSource = excluded.ContentSource, + SortCode = excluded.SortCode, + ExtraChatChannel = excluded.ExtraChatChannel; + "; + + cmd.Parameters.AddWithValue("$Id", message.Id); + cmd.Parameters.AddWithValue("$Receiver", message.Receiver); + cmd.Parameters.AddWithValue("$ContentId", message.ContentId); + cmd.Parameters.AddWithValue("$Date", message.Date.ToUnixTimeMilliseconds()); + cmd.Parameters.AddWithValue("$Code", message.Code.Raw); + cmd.Parameters.AddWithValue("$Sender", MessagePackSerializer.Serialize(message.Sender, MsgPackOptions)); + cmd.Parameters.AddWithValue("$Content", MessagePackSerializer.Serialize(message.Content, MsgPackOptions)); + cmd.Parameters.AddWithValue("$SenderSource", MessagePackSerializer.Serialize(message.SenderSource, MsgPackOptions)); + cmd.Parameters.AddWithValue("$ContentSource", MessagePackSerializer.Serialize(message.ContentSource, MsgPackOptions)); + cmd.Parameters.AddWithValue("$SortCode", message.SortCode.Encode()); + cmd.Parameters.AddWithValue("$ExtraChatChannel", message.ExtraChatChannel); + + cmd.ExecuteNonQuery(); + } + + /// + /// Get the most recent messages. + /// + /// The receiver content ID to filter by. If null, no filtering is performed. + /// Only show messages since this date. If null, no filtering is performed. + /// The amount to return. Defaults to 10,000. + internal MessageEnumerator GetMostRecentMessages(ulong? receiver = null, DateTimeOffset? since = null, int count = MessageQueryLimit) { + var whereClauses = new List(); + if (receiver != null) + whereClauses.Add("Receiver = $Receiver"); + if (since != null) + whereClauses.Add("Date >= $Since"); + + var whereClause = whereClauses.Count > 0 ? "WHERE " + string.Join(" AND ", whereClauses) : ""; + + var cmd = Connection.CreateCommand(); + // Select last N messages by date DESC, but reverse the order to get + // them in ascending order. + cmd.CommandText = @" + SELECT * + FROM ( + SELECT + Id, + Receiver, + ContentId, + Date, + Code, + Sender, + Content, + SenderSource, + ContentSource, + SortCode, + ExtraChatChannel + FROM messages + " + whereClause + @" + ORDER BY Date DESC + LIMIT $Count + ) + ORDER BY Date ASC; + "; + cmd.CommandTimeout = 120; // this could take a while on slow computers + + if (receiver != null) + cmd.Parameters.AddWithValue("$Receiver", receiver); + if (since != null) + cmd.Parameters.AddWithValue("$Since", since.Value.ToUnixTimeMilliseconds()); + + cmd.Parameters.AddWithValue("$Count", count); + + return new MessageEnumerator(cmd.ExecuteReader()); + } +} + +internal class MessageEnumerator(DbDataReader reader) : IEnumerable { + private const int MaxErrorLogs = 10; + + private int _errorCount; + public bool DidError => _errorCount > 0; + + public IEnumerator GetEnumerator() { + while (reader.Read()) { + var id = Guid.Empty; + Message msg; + try { + id = reader.GetGuid(0); + msg = new Message( + id, + (ulong)reader.GetInt64(1), + (ulong)reader.GetInt64(2), + DateTimeOffset.FromUnixTimeMilliseconds(reader.GetInt64(3)), + new ChatCode((ushort)reader.GetInt32(4)), + MessagePackSerializer.Deserialize>(reader.GetFieldValue(5), + MessageStore.MsgPackOptions), + MessagePackSerializer.Deserialize>(reader.GetFieldValue(6), + MessageStore.MsgPackOptions), + MessagePackSerializer.Deserialize(reader.GetFieldValue(7), + MessageStore.MsgPackOptions), + MessagePackSerializer.Deserialize(reader.GetFieldValue(8), + MessageStore.MsgPackOptions), + new SortCode((uint)reader.GetInt32(9)), + reader.GetGuid(10) + ); + } catch (Exception e) { + if (_errorCount < MaxErrorLogs) + Plugin.Log.Error($"Exception while reading message '{id}' from database: {e}"); + _errorCount++; + if (_errorCount == MaxErrorLogs) + Plugin.Log.Error("Further parsing errors will not be logged"); + +#if DEBUG + throw; +#else + continue; +#endif + } + + yield return msg; + } + } + + IEnumerator IEnumerable.GetEnumerator() { + return GetEnumerator(); + } +} diff --git a/ChatTwo/PayloadHandler.cs b/ChatTwo/PayloadHandler.cs index be86f32..035fc71 100755 --- a/ChatTwo/PayloadHandler.cs +++ b/ChatTwo/PayloadHandler.cs @@ -92,7 +92,7 @@ public sealed class PayloadHandler { DrawItemPopup(item); drawn = true; break; - case URIPayload uri: + case UriPayload uri: DrawUriPopup(uri); drawn = true; break; @@ -252,7 +252,7 @@ public sealed class PayloadHandler { DoHover(() => HoverItem(item), hoverSize); break; - case URIPayload uri: + case UriPayload uri: DoHover(() => HoverURI(uri), hoverSize); break; } @@ -376,7 +376,7 @@ public sealed class PayloadHandler { } } - private void HoverURI(URIPayload uri) + private void HoverURI(UriPayload uri) { ImGui.TextUnformatted(string.Format(Language.Context_URLDomain, uri.Uri.Authority)); ImGuiUtil.WarningText(Language.Context_URLWarning); @@ -411,7 +411,7 @@ public sealed class PayloadHandler { if (Equals(raw, ChunkUtil.PeriodicRecruitmentLink)) GameFunctions.GameFunctions.OpenPartyFinder(); break; - case URIPayload uri: + case UriPayload uri: TryOpenURI(uri.Uri); break; default: @@ -659,7 +659,7 @@ public sealed class PayloadHandler { return null; } - private void DrawUriPopup(URIPayload uri) + private void DrawUriPopup(UriPayload uri) { ImGui.TextUnformatted(string.Format(Language.Context_URLDomain, uri.Uri.Authority)); ImGuiUtil.WarningText(Language.Context_URLWarning, false); diff --git a/ChatTwo/Plugin.cs b/ChatTwo/Plugin.cs index afba168..5186025 100755 --- a/ChatTwo/Plugin.cs +++ b/ChatTwo/Plugin.cs @@ -55,7 +55,7 @@ public sealed class Plugin : IDalamudPlugin internal XivCommonBase Common { get; } internal TextureCache TextureCache { get; } internal GameFunctions.GameFunctions Functions { get; } - internal Store Store { get; } + internal MessageManager MessageManager { get; } internal IpcManager Ipc { get; } internal ExtraChat ExtraChat { get; } internal FontManager FontManager { get; } @@ -102,13 +102,13 @@ public sealed class Plugin : IDalamudPlugin Interface.UiBuilder.DisableCutsceneUiHide = true; Interface.UiBuilder.DisableGposeUiHide = true; - Store = new Store(this); // requires Ui + MessageManager = new MessageManager(this); // requires Ui // let all the other components register, then initialise commands Commands.Initialise(); if (Interface.Reason is not PluginLoadReason.Boot) { - Store.FilterAllTabs(false); + MessageManager.FilterAllTabs(false); } Framework.Update += FrameworkUpdate; @@ -141,7 +141,7 @@ public sealed class Plugin : IDalamudPlugin ExtraChat?.Dispose(); Ipc?.Dispose(); - Store?.Dispose(); + MessageManager?.Dispose(); Functions?.Dispose(); TextureCache?.Dispose(); Common?.Dispose(); diff --git a/ChatTwo/Properties/AssemblyInfo.cs b/ChatTwo/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..123e4be --- /dev/null +++ b/ChatTwo/Properties/AssemblyInfo.cs @@ -0,0 +1 @@ +[assembly: System.Runtime.CompilerServices.InternalsVisibleTo("ChatTwo.Tests")] diff --git a/ChatTwo/Resources/Language.Designer.cs b/ChatTwo/Resources/Language.Designer.cs index 5cb6019..cb6e6a2 100755 --- a/ChatTwo/Resources/Language.Designer.cs +++ b/ChatTwo/Resources/Language.Designer.cs @@ -1463,6 +1463,15 @@ namespace ChatTwo.Resources { } } + /// + /// Looks up a localized string similar to An error occurred while loading chat history. Please see plugin logs for more information to report this issue.. + /// + internal static string LoadMessages_Error { + get { + return ResourceManager.GetString("LoadMessages_Error", resourceCulture); + } + } + /// /// Looks up a localized string similar to {0} is performing a database migration.. /// @@ -2237,33 +2246,6 @@ namespace ChatTwo.Resources { } } - /// - /// Looks up a localized string similar to Allow multiple clients to run {0} at the same time, sharing the same database.. - /// - internal static string Options_SharedMode_Description { - get { - return ResourceManager.GetString("Options_SharedMode_Description", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Enable multi-client mode. - /// - internal static string Options_SharedMode_Name { - get { - return ResourceManager.GetString("Options_SharedMode_Name", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to This option is not recommended. No support will be offered if you enable this option. This option will hurt the performance of {0}.. - /// - internal static string Options_SharedMode_Warning { - get { - return ResourceManager.GetString("Options_SharedMode_Warning", resourceCulture); - } - } - /// /// Looks up a localized string similar to Show the Novice Network join button next to the settings button if logged in as a mentor.. /// diff --git a/ChatTwo/Resources/Language.de.resx b/ChatTwo/Resources/Language.de.resx index 0964a0b..3840875 100644 --- a/ChatTwo/Resources/Language.de.resx +++ b/ChatTwo/Resources/Language.de.resx @@ -1,17 +1,17 @@ - @@ -461,15 +461,6 @@ Sie wurden gewarnt. Erweitert - - Mehrere Clients verwenden - - - Erlaubt es mehreren Clients, {0} gleichzeitig zu verwenden und Daten zu teilen. - - - Diese Einstellung ist nicht empfohlen. Es wird keine Hilfe angeboten, falls diese Option genutzt wird. Dadurch sinkt zudem die Leistung von {0}. - Herauslösen diff --git a/ChatTwo/Resources/Language.es.resx b/ChatTwo/Resources/Language.es.resx index 256defd..08c659d 100644 --- a/ChatTwo/Resources/Language.es.resx +++ b/ChatTwo/Resources/Language.es.resx @@ -1,17 +1,17 @@ - @@ -460,15 +460,6 @@ Avanzado - - Habilitar modo multiventana - - - Permitir que múltiples clientes ejecuten {0} al mismo tiempo, compartiendo la misma base de datos. - - - Esta opción no es recomendada. No se ofrecerá ningún soporte si activas esta opción. Esta opción perjudicará el rendimiento dé {0}. - Nueva pestaña diff --git a/ChatTwo/Resources/Language.fr.resx b/ChatTwo/Resources/Language.fr.resx index 9437c29..58ea691 100644 --- a/ChatTwo/Resources/Language.fr.resx +++ b/ChatTwo/Resources/Language.fr.resx @@ -1,17 +1,17 @@ - @@ -460,15 +460,6 @@ Avancé - - Activer le mode multi-client - - - Permet à plusieurs clients d'exécuter {0} en même temps, en partageant la même base de données. - - - Cette option n'est pas recommandée. Aucune assistance ne sera offerte si vous activez cette option. Cette option nuira aux performances de {0}. - Détacher diff --git a/ChatTwo/Resources/Language.nl.resx b/ChatTwo/Resources/Language.nl.resx index d8d0abe..d23b272 100644 --- a/ChatTwo/Resources/Language.nl.resx +++ b/ChatTwo/Resources/Language.nl.resx @@ -460,15 +460,6 @@ Geavanceerd - - Modus voor meerdere vensters inschakelen - - - Meerdere vensters toestaan om {0} tegelijkertijd uit te voeren en dezelfde database te delen. - - - Deze optie wordt niet aanbevolen. Er wordt geen ondersteuning aangeboden als je deze optie inschakelt. Deze optie is schadelijk voor de snelheid van {0}. - Uitvouwen diff --git a/ChatTwo/Resources/Language.pt-BR.resx b/ChatTwo/Resources/Language.pt-BR.resx index 6b6ca48..e9658f6 100644 --- a/ChatTwo/Resources/Language.pt-BR.resx +++ b/ChatTwo/Resources/Language.pt-BR.resx @@ -1,17 +1,17 @@ - @@ -460,15 +460,6 @@ Avançado - - Ativar modo de mútiplos clientes - - - Permitir que vários clientes sejam executados {0} ao mesmo tempo, compartilhando o mesmo banco de dados. - - - Esta opção não é recomendada. Nenhum suporte será oferecido se você ativar esta opção. Esta opção irá prejudicar o desempenho de {0}. - Separar da janela diff --git a/ChatTwo/Resources/Language.resx b/ChatTwo/Resources/Language.resx index d8c212c..e938260 100644 --- a/ChatTwo/Resources/Language.resx +++ b/ChatTwo/Resources/Language.resx @@ -460,15 +460,6 @@ Advanced - - Enable multi-client mode - - - Allow multiple clients to run {0} at the same time, sharing the same database. - - - This option is not recommended. No support will be offered if you enable this option. This option will hurt the performance of {0}. - Pop out @@ -1000,4 +991,7 @@ Limits the amount of log lines to show in the chat window. This may slightly improve performance. + + An error occurred while loading chat history. Please see plugin logs for more information to report this issue. + diff --git a/ChatTwo/Resources/Language.ro.resx b/ChatTwo/Resources/Language.ro.resx index 35d1804..938e221 100644 --- a/ChatTwo/Resources/Language.ro.resx +++ b/ChatTwo/Resources/Language.ro.resx @@ -1,17 +1,17 @@ - @@ -460,15 +460,6 @@ Avansat - - Activează modul instanțelor multiple - - - Permite mai multor instanțe sa folosească {0} simultan, folosind aceeași data de baze. - - - Această opțiune nu este recomandată. Dacă o activați nu veți primi niciun suport. Această opțiune va afecta performanța {0}-ului. - Mută tabul într-o fereastra noua diff --git a/ChatTwo/Resources/Language.ru.resx b/ChatTwo/Resources/Language.ru.resx index 267ee9a..92428b4 100644 --- a/ChatTwo/Resources/Language.ru.resx +++ b/ChatTwo/Resources/Language.ru.resx @@ -1,17 +1,17 @@ - @@ -460,15 +460,6 @@ Расширенные - - Включить режим нескольких клиентов - - - Разрешить нескольким клиентам запускать {0} одновременно, обмениваясь одной и той же базой данных. - - - Эта опция не рекомендуется. Если вы включите эту опцию, то поддержка не будет предоставляться. Эта опция значитьльно навредит производительности {0}. - Отделить diff --git a/ChatTwo/Resources/Language.sv.resx b/ChatTwo/Resources/Language.sv.resx index b94f6b6..e9bade7 100644 --- a/ChatTwo/Resources/Language.sv.resx +++ b/ChatTwo/Resources/Language.sv.resx @@ -1,17 +1,17 @@ - @@ -460,15 +460,6 @@ Avancerat - - Aktivera flerklientsläge - - - Tillåt flera klienter att köra {0} samtidigt med samma databas. - - - Den här inställningen rekommenderas inte. Inget stöd kommer erbjudas om du aktiverar den här inställningen. Den här inställningen kommer att förvärra prestandan av {0}. - Separera diff --git a/ChatTwo/Resources/Language.zh-Hans.resx b/ChatTwo/Resources/Language.zh-Hans.resx index 3454a92..2b6a197 100644 --- a/ChatTwo/Resources/Language.zh-Hans.resx +++ b/ChatTwo/Resources/Language.zh-Hans.resx @@ -1,17 +1,17 @@ - @@ -460,15 +460,6 @@ 高级选项 - - 启用多客户端模式 - - - 允许多个客户端同时运行 {0} ，共享同一个数据库。 - - - 不推荐此选项。如果您启用此选项，不会提供任何支持。此选项将损害 {0} 的性能。 - 弹出 diff --git a/ChatTwo/Resources/Language.zh-Hant.resx b/ChatTwo/Resources/Language.zh-Hant.resx index 9f4a1ea..153b1c0 100644 --- a/ChatTwo/Resources/Language.zh-Hant.resx +++ b/ChatTwo/Resources/Language.zh-Hant.resx @@ -1,17 +1,17 @@ - @@ -461,15 +461,6 @@ 高階選項 - - 啓用多客戸端模式 - - - 允許多個客戸端同時執行 {0} ，共享同一個資料庫。 - - - 不推薦此選項。 如果啓用此選項，不會提供任何支持。 此選項將損害 {0} 的效能。 - 彈出 diff --git a/ChatTwo/Store.cs b/ChatTwo/Store.cs deleted file mode 100755 index 63be71b..0000000 --- a/ChatTwo/Store.cs +++ /dev/null @@ -1,387 +0,0 @@ -using System.Collections.Concurrent; -using System.Diagnostics; -using ChatTwo.Code; -using ChatTwo.Util; -using Dalamud.Game.Text; -using Dalamud.Game.Text.SeStringHandling; -using Dalamud.Plugin.Services; -using LiteDB; -using Lumina.Excel.GeneratedSheets; - -namespace ChatTwo; - -internal class Store : IDisposable -{ - internal const int MessagesLimit = 10_000; - - private Plugin Plugin { get; } - - private ConcurrentQueue<(uint, Message)> Pending { get; } = new(); - private Stopwatch CheckpointTimer { get; } = new(); - internal ILiteDatabase Database { get; private set; } - private ILiteCollection Messages => Database.GetCollection("messages"); - - private Dictionary Formats { get; } = new(); - private ulong LastContentId { get; set; } - - private ulong CurrentContentId - { - get - { - var contentId = Plugin.ClientState.LocalContentId; - return contentId == 0 ? LastContentId : contentId; - } - } - - internal Store(Plugin plugin) - { - Plugin = plugin; - CheckpointTimer.Start(); - - BsonMapper.Global = new BsonMapper - { - IncludeNonPublic = true, - TrimWhitespace = false, - // EnumAsInteger = true, - }; - - BsonMapper.Global.Entity() - .Id(msg => msg.Id) - .Ctor(doc => new Message( - doc["_id"].AsObjectId, - (ulong) doc["Receiver"].AsInt64, - (ulong) doc["ContentId"].AsInt64, - DateTime.UnixEpoch.AddMilliseconds(doc["Date"].AsInt64), - doc["Code"].AsDocument, - doc["Sender"].AsArray, - doc["Content"].AsArray, - doc["SenderSource"], - doc["ContentSource"], - doc["SortCode"].AsDocument, - doc["ExtraChatChannel"] - )); - - BsonMapper.Global.RegisterType( - payload => - { - switch (payload) - { - case AchievementPayload achievement: - return new BsonDocument(new Dictionary { - ["Type"] = new("Achievement"), - ["Id"] = new(achievement.Id), - }); - case PartyFinderPayload partyFinder: - return new BsonDocument(new Dictionary { - ["Type"] = new("PartyFinder"), - ["Id"] = new(partyFinder.Id), - }); - case URIPayload uri: - return new BsonDocument(new Dictionary { - ["Type"] = new("URI"), - ["Uri"] = new(uri.Uri.ToString()), - }); - } - - return payload?.Encode(); - }, - bson => - { - if (bson.IsNull) - return null; - - if (bson.IsDocument) - { - return bson["Type"].AsString switch - { - "Achievement" => new AchievementPayload((uint) bson["Id"].AsInt64), - "PartyFinder" => new PartyFinderPayload((uint) bson["Id"].AsInt64), - "URI" => new URIPayload(new Uri(bson["Uri"].AsString)), - _ => null, - }; - } - - return Payload.Decode(new BinaryReader(new MemoryStream(bson.AsBinary))); - }); - BsonMapper.Global.RegisterType( - seString => seString == null - ? null - : new BsonArray(seString.Payloads.Select(payload => new BsonValue(payload.Encode()))), - bson => - { - if (bson.IsNull) - return null; - - var array = bson.IsArray ? bson.AsArray : bson["Payloads"].AsArray; - var payloads = array - .Select(payload => Payload.Decode(new BinaryReader(new MemoryStream(payload.AsBinary)))) - .ToList(); - return new SeString(payloads); - } - ); - BsonMapper.Global.RegisterType( - type => (int) type, - bson => (ChatType) bson.AsInt32 - ); - BsonMapper.Global.RegisterType( - source => (int) source, - bson => (ChatSource) bson.AsInt32 - ); - BsonMapper.Global.RegisterType( - dateTime => dateTime.Subtract(DateTime.UnixEpoch).TotalMilliseconds, - bson => DateTime.UnixEpoch.AddMilliseconds(bson.AsInt64) - ); - Database = Connect(); - - Plugin.ChatGui.ChatMessageUnhandled += ChatMessage; - Plugin.Framework.Update += GetMessageInfo; - Plugin.Framework.Update += UpdateReceiver; - Plugin.ClientState.Logout += Logout; - } - - public void Dispose() { - Plugin.ClientState.Logout -= Logout; - Plugin.Framework.Update -= UpdateReceiver; - Plugin.Framework.Update -= GetMessageInfo; - Plugin.ChatGui.ChatMessageUnhandled -= ChatMessage; - - Database.Dispose(); - } - - internal static string DatabasePath() - { - var dir = Plugin.Interface.ConfigDirectory; - dir.Create(); - return Path.Join(dir.FullName, "chat.db"); - } - - private LiteDatabase Connect() { - var dbPath = DatabasePath(); - var connection = Plugin.Config.SharedMode ? "shared" : "direct"; - var connString = $"Filename='{dbPath}';Connection={connection}"; - var conn = new LiteDatabase(connString, BsonMapper.Global) - { - CheckpointSize = 1_000, - Timeout = TimeSpan.FromSeconds(1), - }; - var messages = conn.GetCollection("messages"); - messages.EnsureIndex(msg => msg.Date); - messages.EnsureIndex(msg => msg.SortCode); - messages.EnsureIndex(msg => msg.ExtraChatChannel); - return conn; - } - - internal void Reconnect() - { - Database.Dispose(); - Database = Connect(); - } - - internal void ClearDatabase() - { - Messages.DeleteAll(); - Database.Rebuild(); - } - - internal static long DatabaseSize() - { - var dbPath = DatabasePath(); - return !File.Exists(dbPath) ? 0 : new FileInfo(dbPath).Length; - } - - internal static long DatabaseLogSize() - { - var dbLogPath = Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-log.db"); - return !File.Exists(dbLogPath) ? 0 : new FileInfo(dbLogPath).Length; - } - - internal int MessageCount() => Messages.Count(); - - private void Logout() - { - LastContentId = 0; - } - - private void UpdateReceiver(IFramework framework) - { - var contentId = Plugin.ClientState.LocalContentId; - if (contentId != 0) - LastContentId = contentId; - } - - private void GetMessageInfo(IFramework framework) - { - if (CheckpointTimer.Elapsed > TimeSpan.FromMinutes(5)) - { - CheckpointTimer.Restart(); - new Thread(() => Database.Checkpoint()).Start(); - } - - if (!Pending.TryDequeue(out var entry)) - return; - - var contentId = Plugin.Functions.Chat.GetContentIdForEntry(entry.Item1); - entry.Item2.ContentId = contentId ?? 0; - if (Plugin.Config.DatabaseBattleMessages || !entry.Item2.Code.IsBattle()) - Messages.Update(entry.Item2); - } - - internal void AddMessage(Message message, Tab? currentTab) - { - if (Plugin.Config.DatabaseBattleMessages || !message.Code.IsBattle()) - Messages.Insert(message); - - var currentMatches = currentTab?.Matches(message) ?? false; - - foreach (var tab in Plugin.Config.Tabs) - { - var unread = !(tab.UnreadMode == UnreadMode.Unseen && currentTab != tab && currentMatches); - - if (tab.Matches(message)) - tab.AddMessage(message, unread); - } - } - - internal void FilterAllTabs(bool unread = true) - { - foreach (var tab in Plugin.Config.Tabs) - FilterTab(tab, unread); - } - - internal void FilterTab(Tab tab, bool unread) - { - var sortCodes = new List(); - foreach (var (type, sources) in tab.ChatCodes) - { - sortCodes.Add(new SortCode(type, 0)); - sortCodes.Add(new SortCode(type, (ChatSource) 1)); - - if (!type.HasSource()) - continue; - - foreach (var source in Enum.GetValues()) - if (sources.HasFlag(source)) - sortCodes.Add(new SortCode(type, source)); - } - - var query = Messages - .Query() - .OrderByDescending(msg => msg.Date) - .Where(msg => sortCodes.Contains(msg.SortCode) || msg.ExtraChatChannel != Guid.Empty) - .Where(msg => msg.Receiver == CurrentContentId); - - if (!Plugin.Config.FilterIncludePreviousSessions) - query = query.Where(msg => msg.Date >= Plugin.GameStarted); - - var messages = query.Limit(MessagesLimit).ToEnumerable().Reverse(); - - foreach (var message in messages) - { - // check primarily for startup double posting messages - if (tab.Contains(message)) - continue; - - // redundant matches check for extrachat - if (tab.Matches(message)) - tab.AddMessage(message, unread); - } - } - - public (SeString? Sender, SeString? Message) LastMessage = (null, null); - private void ChatMessage(XivChatType type, uint senderId, SeString sender, SeString message) - { - var chatCode = new ChatCode((ushort) type); - - NameFormatting? formatting = null; - if (sender.Payloads.Count > 0) - formatting = FormatFor(chatCode.Type); - - LastMessage = (sender, message); - var senderChunks = new List(); - if (formatting is { IsPresent: true }) - { - senderChunks.Add(new TextChunk(ChunkSource.None, null, formatting.Before) - { - FallbackColour = chatCode.Type, - }); - senderChunks.AddRange(ChunkUtil.ToChunks(sender, ChunkSource.Sender, chatCode.Type)); - senderChunks.Add(new TextChunk(ChunkSource.None, null, formatting.After) - { - FallbackColour = chatCode.Type, - }); - } - - var messageChunks = ChunkUtil.ToChunks(message, ChunkSource.Content, chatCode.Type).ToList(); - - var msg = new Message(CurrentContentId, chatCode, senderChunks, messageChunks, sender, message); - AddMessage(msg, Plugin.ChatLogWindow.CurrentTab ?? null); - - var idx = Plugin.Functions.GetCurrentChatLogEntryIndex(); - if (idx != null) - Pending.Enqueue((idx.Value - 1, msg)); - } - - internal class NameFormatting - { - internal string Before { get; private set; } = string.Empty; - internal string After { get; private set; } = string.Empty; - internal bool IsPresent { get; private set; } = true; - - internal static NameFormatting Empty() - { - return new NameFormatting { IsPresent = false, }; - } - - internal static NameFormatting Of(string before, string after) - { - return new NameFormatting - { - Before = before, - After = after, - }; - } - } - - private NameFormatting? FormatFor(ChatType type) - { - if (Formats.TryGetValue(type, out var cached)) - return cached; - - var logKind = Plugin.DataManager.GetExcelSheet()!.GetRow((ushort) type); - if (logKind == null) - return null; - - var format = (SeString) logKind.Format; - static bool IsStringParam(Payload payload, byte num) - { - var data = payload.Encode(); - return data.Length >= 5 && data[1] == 0x29 && data[4] == num + 1; - } - - var firstStringParam = format.Payloads.FindIndex(payload => IsStringParam(payload, 1)); - var secondStringParam = format.Payloads.FindIndex(payload => IsStringParam(payload, 2)); - - if (firstStringParam == -1 || secondStringParam == -1) - return NameFormatting.Empty(); - - var before = format.Payloads - .GetRange(0, firstStringParam) - .Where(payload => payload is ITextProvider) - .Cast() - .Select(text => text.Text); - var after = format.Payloads - .GetRange(firstStringParam + 1, secondStringParam - firstStringParam) - .Where(payload => payload is ITextProvider) - .Cast() - .Select(text => text.Text); - - var nameFormatting = NameFormatting.Of( - string.Join("", before), - string.Join("", after) - ); - - Formats[type] = nameFormatting; - - return nameFormatting; - } -} diff --git a/ChatTwo/Ui/ChatLogWindow.cs b/ChatTwo/Ui/ChatLogWindow.cs index b8fe4d6..ea015e2 100644 --- a/ChatTwo/Ui/ChatLogWindow.cs +++ b/ChatTwo/Ui/ChatLogWindow.cs @@ -139,7 +139,7 @@ public sealed class ChatLogWindow : Window, IUiComponent private void Login() { - Plugin.Store.FilterAllTabs(false); + Plugin.MessageManager.FilterAllTabs(false); } private void Activated(ChatActivatedArgs args) { diff --git a/ChatTwo/Ui/SeStringDebugger.cs b/ChatTwo/Ui/SeStringDebugger.cs index 7e7dd92..1c25afa 100644 --- a/ChatTwo/Ui/SeStringDebugger.cs +++ b/ChatTwo/Ui/SeStringDebugger.cs @@ -42,7 +42,7 @@ public class SeStringDebugger : Window public override void Draw() { - if (Plugin.Store.LastMessage.Sender == null) + if (Plugin.MessageManager.LastMessage.Sender == null) { ImGui.TextUnformatted("Nothing to show"); return; @@ -51,15 +51,15 @@ public class SeStringDebugger : Window // TODO: Make SeString freely selectable through chat ImGui.TextUnformatted("Sender Content"); ImGui.Spacing(); - if (Plugin.Store.LastMessage.Sender != null) - ProcessPayloads(Plugin.Store.LastMessage.Sender.Payloads); + if (Plugin.MessageManager.LastMessage.Sender != null) + ProcessPayloads(Plugin.MessageManager.LastMessage.Sender.Payloads); else ImGui.TextUnformatted("Nothing to show"); ImGui.TextUnformatted("Message Content"); ImGui.Spacing(); - if (Plugin.Store.LastMessage.Message != null) - ProcessPayloads(Plugin.Store.LastMessage.Message.Payloads); + if (Plugin.MessageManager.LastMessage.Message != null) + ProcessPayloads(Plugin.MessageManager.LastMessage.Message.Payloads); else ImGui.TextUnformatted("Nothing to show"); } diff --git a/ChatTwo/Ui/Settings.cs b/ChatTwo/Ui/Settings.cs index 2786523..4abae74 100755 --- a/ChatTwo/Ui/Settings.cs +++ b/ChatTwo/Ui/Settings.cs @@ -151,7 +151,6 @@ public sealed class SettingsWindow : Window, IUiComponent || Math.Abs(Mutable.JapaneseFontSize - Plugin.Config.JapaneseFontSize) > 0.001 || Math.Abs(Mutable.SymbolsFontSize - Plugin.Config.SymbolsFontSize) > 0.001; var langChanged = Mutable.LanguageOverride != Plugin.Config.LanguageOverride; - var sharedChanged = Mutable.SharedMode != Plugin.Config.SharedMode; config.UpdateFrom(Mutable); @@ -159,7 +158,7 @@ public sealed class SettingsWindow : Window, IUiComponent // commit any changes that cause a crash Plugin.DeferredSaveFrames = 60; - Plugin.Store.FilterAllTabs(false); + Plugin.MessageManager.FilterAllTabs(false); if (fontChanged || fontSizeChanged) { Plugin.FontManager.BuildFonts(); @@ -169,10 +168,6 @@ public sealed class SettingsWindow : Window, IUiComponent Plugin.LanguageChanged(Plugin.Interface.UiLanguage); } - if (sharedChanged) { - Plugin.Store.Reconnect(); - } - if (!Mutable.HideChat && hideChatChanged) { GameFunctions.GameFunctions.SetChatInteractable(true); } diff --git a/ChatTwo/Ui/SettingsTabs/Database.cs b/ChatTwo/Ui/SettingsTabs/Database.cs index 2ca9f43..3441550 100755 --- a/ChatTwo/Ui/SettingsTabs/Database.cs +++ b/ChatTwo/Ui/SettingsTabs/Database.cs @@ -1,5 +1,9 @@ +using System.Diagnostics; +using ChatTwo.Code; using ChatTwo.Resources; using ChatTwo.Util; +using Dalamud.Game.Text.SeStringHandling; +using Dalamud.Game.Text.SeStringHandling.Payloads; using Dalamud.Interface.Internal.Notifications; using ImGuiNET; @@ -46,13 +50,6 @@ internal sealed class Database : ISettingsTab Mutable.LoadPreviousSession = false; } - ImGuiUtil.OptionCheckbox( - ref Mutable.SharedMode, - Language.Options_SharedMode_Name, - string.Format(Language.Options_SharedMode_Description, Plugin.PluginName) - ); - ImGuiUtil.WarningText(string.Format(Language.Options_SharedMode_Warning, Plugin.PluginName)); - ImGui.Spacing(); ImGui.Separator(); ImGui.Spacing(); @@ -65,18 +62,18 @@ internal sealed class Database : ISettingsTab // constant stat calls and spamming the database. if (DatabaseLastRefreshTicks + 5 * 1000 < Environment.TickCount64) { - DatabaseSize = Store.DatabaseSize(); - DatabaseLogSize = Store.DatabaseLogSize(); - DatabaseMessageCount = Plugin.Store.MessageCount(); + DatabaseSize = Plugin.MessageManager.Store.DatabaseSize(); + DatabaseLogSize = Plugin.MessageManager.Store.DatabaseLogSize(); + DatabaseMessageCount = Plugin.MessageManager.Store.MessageCount(); DatabaseLastRefreshTicks = Environment.TickCount64; } - ImGuiUtil.HelpText(string.Format(Language.Options_Database_Metadata_Path, Store.DatabasePath())); + ImGuiUtil.HelpText(string.Format(Language.Options_Database_Metadata_Path, MessageManager.DatabasePath())); if (ImGui.IsItemClicked(ImGuiMouseButton.Left)) { // Copy the directory path instead of the file path so people can // paste it into their file explorer. - var path = Path.GetDirectoryName(Store.DatabasePath()); + var path = Path.GetDirectoryName(MessageManager.DatabasePath()); ImGui.SetClipboardText(path); WrapperUtil.AddNotification(Language.Options_Database_Metadata_CopyConfigPathNotification, NotificationType.Info); } @@ -95,12 +92,12 @@ internal sealed class Database : ISettingsTab if (ImGui.IsItemHovered()) ImGui.SetTooltip(DatabaseLogSize.ToString("N0") + "B"); - ImGuiUtil.HelpText(string.Format(Language.Options_Database_Metadata_MessageCount, DatabaseMessageCount, Store.MessagesLimit)); + ImGuiUtil.HelpText(string.Format(Language.Options_Database_Metadata_MessageCount, DatabaseMessageCount, MessageStore.MessageQueryLimit)); if (ImGuiUtil.CtrlShiftButton(Language.Options_ClearDatabase_Button, Language.Options_ClearDatabase_Tooltip)) { - Plugin.Log.Warning("Clearing database"); - Plugin.Store.ClearDatabase(); + Plugin.Log.Warning("Clearing messages from database"); + Plugin.MessageManager.Store.ClearMessages(); foreach (var tab in Plugin.Config.Tabs) tab.Clear(); @@ -117,11 +114,18 @@ internal sealed class Database : ISettingsTab ImGui.PushTextWrapPos(); ImGuiUtil.WarningText(Language.Options_Database_Advanced_Warning); - if (ImGuiUtil.CtrlShiftButton("Checkpoint", "Ctrl+Shift: Database.Checkpoint()")) - Plugin.Store.Database.Checkpoint(); + if (ImGuiUtil.CtrlShiftButton("Perform maintenance", "Ctrl+Shift: MessageManager.Store.PerformMaintenance()")) + Plugin.MessageManager.Store.PerformMaintenance(); - if (ImGuiUtil.CtrlShiftButton("Rebuild", "Ctrl+Shift: Database.Rebuild()")) - Plugin.Store.Database.Rebuild(); + if (ImGuiUtil.CtrlShiftButton("Reload messages from database", + "Ctrl+Shift: MessageManager.FilterAllTabs(false)")) { + foreach (var tab in Plugin.Config.Tabs) + tab.Clear(); + Plugin.MessageManager.FilterAllTabs(false); + } + + if (ImGuiUtil.CtrlShiftButton("Inject 10,000 messages", "Ctrl+Shift: creates 10,000 unique messages (async)")) + new Thread(() => InsertMessages(10_000)).Start(); ImGui.PopTextWrapPos(); ImGui.TreePop(); @@ -129,4 +133,78 @@ internal sealed class Database : ISettingsTab ImGui.Spacing(); } + + private void InsertMessages(int count) { + Plugin.Log.Info($"Inserting {count} messages due to user request"); + + // Generate + var stopwatch = Stopwatch.StartNew(); + var playerName = Plugin.ClientState.LocalPlayer?.Name.ToString() ?? "Unknown Player"; + var worldId = Plugin.ClientState.LocalPlayer?.HomeWorld.Id ?? 0; + var senderSource = new SeStringBuilder() + .AddText("<") + .Add(new PlayerPayload(playerName, worldId)) + .AddText("Random Message") + .Add(RawPayload.LinkTerminator) + .AddText(">: ") + .Build(); + var senderChunks = ChunkUtil.ToChunks(senderSource, ChunkSource.Sender, ChatType.Debug).ToList(); + var messages = new List(count); + for (var i = 0; i < count; i++) { + var contentSource = new SeStringBuilder() + .AddText("Random message payload - ") + .AddItalics(Guid.NewGuid().ToString()) + .Build(); + var contentChunks = ChunkUtil.ToChunks(contentSource, ChunkSource.Content, ChatType.Debug).ToList(); + + messages.Add(new Message( + Guid.NewGuid(), + Plugin.MessageManager.CurrentContentId, + Plugin.MessageManager.CurrentContentId, + DateTimeOffset.UtcNow, + new ChatCode(10), + senderChunks, + contentChunks, + senderSource, + contentSource, + new SortCode(ChatType.Debug, ChatSource.Self), + Guid.Empty + )); + } + + var elapsedTicks = stopwatch.ElapsedTicks; + stopwatch.Stop(); + Plugin.Log.Info($"Crafted {count} messages in {elapsedTicks} ticks ({elapsedTicks / TimeSpan.TicksPerMillisecond}ms)"); + + // Insert + stopwatch = Stopwatch.StartNew(); + foreach (var message in messages) { + Plugin.MessageManager.Store.UpsertMessage(message); + } + + elapsedTicks = stopwatch.ElapsedTicks; + stopwatch.Stop(); + Plugin.Log.Info($"Upserted {count} messages in {elapsedTicks} ticks ({elapsedTicks / TimeSpan.TicksPerMillisecond}ms)"); + + // Clear tabs during framework frame + Plugin.Framework.Run(() => { + stopwatch = Stopwatch.StartNew(); + foreach (var tab in Plugin.Config.Tabs) + tab.Clear(); + + elapsedTicks = stopwatch.ElapsedTicks; + stopwatch.Stop(); + Plugin.Log.Info( + $"Cleared {Plugin.Config.Tabs.Count} tabs in {elapsedTicks} ticks ({elapsedTicks / TimeSpan.TicksPerMillisecond}ms)"); + }).Wait(); + + // Fetch and filter during framework frame + Plugin.Framework.Run(() => { + stopwatch = Stopwatch.StartNew(); + Plugin.MessageManager.FilterAllTabs(false); + elapsedTicks = stopwatch.ElapsedTicks; + stopwatch.Stop(); + Plugin.Log.Info($"Fetched and filtered all tabs in {elapsedTicks} ticks ({elapsedTicks / TimeSpan.TicksPerMillisecond}ms)"); + }).Wait(); + } } diff --git a/ChatTwo/Util/ChunkUtil.cs b/ChatTwo/Util/ChunkUtil.cs index 3874c77..ada1040 100755 --- a/ChatTwo/Util/ChunkUtil.cs +++ b/ChatTwo/Util/ChunkUtil.cs @@ -108,7 +108,7 @@ internal static class ChunkUtil { } else if (rawPayload.Data.Length > 5 && rawPayload.Data[1] == 0x27 && rawPayload.Data[3] == 0x07) { // uri payload var uri = new Uri(Encoding.UTF8.GetString(rawPayload.Data[4..])); - link = new URIPayload(uri); + link = new UriPayload(uri); } else if (Equals(rawPayload, RawPayload.LinkTerminator)) { link = null; } diff --git a/ChatTwo/Util/Payloads.cs b/ChatTwo/Util/Payloads.cs index 92311d2..6fc81fc 100755 --- a/ChatTwo/Util/Payloads.cs +++ b/ChatTwo/Util/Payloads.cs @@ -39,11 +39,11 @@ internal class AchievementPayload : Payload { } -internal class URIPayload(Uri uri) : Payload +internal class UriPayload(Uri uri) : Payload { public override PayloadType Type => (PayloadType) 0x52; - public Uri Uri { get; init; } = uri; + public Uri Uri { get; } = uri; private static readonly string[] ExpectedSchemes = ["http", "https"]; private static readonly string DefaultScheme = "https"; @@ -55,7 +55,7 @@ internal class URIPayload(Uri uri) : Payload /// /// If the URI is invalid, or if the scheme is not supported. /// - public static URIPayload ResolveURI(string rawURI) + public static UriPayload ResolveURI(string rawURI) { ArgumentNullException.ThrowIfNull(rawURI); @@ -64,7 +64,7 @@ internal class URIPayload(Uri uri) : Payload { if (rawURI.StartsWith($"{scheme}://")) { - return new URIPayload(new Uri(rawURI)); + return new UriPayload(new Uri(rawURI)); } } if (rawURI.Contains("://")) @@ -72,7 +72,7 @@ internal class URIPayload(Uri uri) : Payload throw new UriFormatException($"Unsupported scheme in URL: {rawURI}"); } - return new URIPayload(new Uri($"{DefaultScheme}://{rawURI}")); + return new UriPayload(new Uri($"{DefaultScheme}://{rawURI}")); } protected override void DecodeImpl(BinaryReader reader, long endOfStream) diff --git a/ChatTwo/packages.lock.json b/ChatTwo/packages.lock.json index b2f267f..18f7d6b 100644 --- a/ChatTwo/packages.lock.json +++ b/ChatTwo/packages.lock.json @@ -8,11 +8,26 @@ "resolved": "2.1.12", "contentHash": "Sc0PVxvgg4NQjcI8n10/VfUQBAS4O+Fw2pZrAqBdRMbthYGeogzu5+xmIGCGmsEZ/ukMOBuAqiNiB5qA3MRalg==" }, - "LiteDB": { + "MessagePack": { "type": "Direct", - "requested": "[5.0.17, )", - "resolved": "5.0.17", - "contentHash": "cKPvkdlzIts3ZKu/BzoIc/Y71e4VFKlij4LyioPFATZMot+wB7EAm1FFbZSJez6coJmQUoIg/3yHE1MMU+zOdg==" + "requested": "[2.5.140, )", + "resolved": "2.5.140", + "contentHash": "nkIsgy8BkIfv40bSz9XZb4q//scI1PF3AYeB5X66nSlIhBIqbdpLz8Qk3gHvnjV3RZglQLO/ityK3eNfLii2NA==", + "dependencies": { + "MessagePack.Annotations": "2.5.140", + "Microsoft.NET.StringTools": "17.6.3", + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + }, + "Microsoft.Data.Sqlite": { + "type": "Direct", + "requested": "[8.0.4, )", + "resolved": "8.0.4", + "contentHash": "vgLm03wS+CfsolO7qk4KVuvt0CtzgdjKmoORuwxMmiIF1ow1JlOo1vwfDHfwXnGa5+QEbvOUy3169bBcHshfTg==", + "dependencies": { + "Microsoft.Data.Sqlite.Core": "8.0.4", + "SQLitePCLRaw.bundle_e_sqlite3": "2.1.6" + } }, "Pidgin": { "type": "Direct", @@ -37,6 +52,24 @@ "resolved": "9.0.0", "contentHash": "avaBp3FmSCi/PiQhntCeBDYOHejdyTWmFtz4pRBVQQ8vHkmRx+YTk1la9dkYBMlXxRXKckEdH1iI1Fu61JlE7w==" }, + "MessagePack.Annotations": { + "type": "Transitive", + "resolved": "2.5.140", + "contentHash": "JE3vwluOrsJ4t3hnfXzIxJUh6lhx6M/KR8Sark/HOUw1DJ5UKu5JsAnnuaQngg6poFkRx1lzHSLTkxHNJO7+uQ==" + }, + "Microsoft.Data.Sqlite.Core": { + "type": "Transitive", + "resolved": "8.0.4", + "contentHash": "x5FE5m1h31UIDEk0j3r38HtYvsa0fxd5jXzvE/SARI7LecXt/jm4z2SUl6TEoJGQOo9Ow2wg3a0MU2E1TVVAdA==", + "dependencies": { + "SQLitePCLRaw.core": "2.1.6" + } + }, + "Microsoft.NET.StringTools": { + "type": "Transitive", + "resolved": "17.6.3", + "contentHash": "N0ZIanl1QCgvUumEL1laasU0a7sOE5ZwLZVTn0pAePnfhq8P7SvTjF8Axq+CnavuQkmdQpGNXQ1efZtu5kDFbA==" + }, "Microsoft.NETCore.Platforms": { "type": "Transitive", "resolved": "1.1.0", @@ -232,6 +265,36 @@ "SharpDX": "4.2.0" } }, + "SQLitePCLRaw.bundle_e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.6", + "contentHash": "BmAf6XWt4TqtowmiWe4/5rRot6GerAeklmOPfviOvwLoF5WwgxcJHAxZtySuyW9r9w+HLILnm8VfJFLCUJYW8A==", + "dependencies": { + "SQLitePCLRaw.lib.e_sqlite3": "2.1.6", + "SQLitePCLRaw.provider.e_sqlite3": "2.1.6" + } + }, + "SQLitePCLRaw.core": { + "type": "Transitive", + "resolved": "2.1.6", + "contentHash": "wO6v9GeMx9CUngAet8hbO7xdm+M42p1XeJq47ogyRoYSvNSp0NGLI+MgC0bhrMk9C17MTVFlLiN6ylyExLCc5w==", + "dependencies": { + "System.Memory": "4.5.3" + } + }, + "SQLitePCLRaw.lib.e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.6", + "contentHash": "2ObJJLkIUIxRpOUlZNGuD4rICpBnrBR5anjyfUFQep4hMOIeqW+XGQYzrNmHSVz5xSWZ3klSbh7sFR6UyDj68Q==" + }, + "SQLitePCLRaw.provider.e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.6", + "contentHash": "PQ2Oq3yepLY4P7ll145P3xtx2bX8xF4PzaKPRpw9jZlKvfe4LE/saAV82inND9usn1XRpmxXk7Lal3MTI+6CNg==", + "dependencies": { + "SQLitePCLRaw.core": "2.1.6" + } + }, "System.AppContext": { "type": "Transitive", "resolved": "4.3.0", @@ -476,6 +539,11 @@ "System.Threading": "4.3.0" } }, + "System.Memory": { + "type": "Transitive", + "resolved": "4.5.3", + "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==" + }, "System.Net.Http": { "type": "Transitive", "resolved": "4.3.0", @@ -641,6 +709,11 @@ "Microsoft.NETCore.Targets": "1.1.0" } }, + "System.Runtime.CompilerServices.Unsafe": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" + }, "System.Runtime.Extensions": { "type": "Transitive", "resolved": "4.3.0",