diff --git a/ChatTwo.Tests/LegacyMessageImporterTest.cs b/ChatTwo.Tests/LegacyMessageImporterTest.cs new file mode 100644 index 0000000..7dcb9bc --- /dev/null +++ b/ChatTwo.Tests/LegacyMessageImporterTest.cs @@ -0,0 +1,220 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using ChatTwo.Code; +using JetBrains.Annotations; +using LiteDB; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace ChatTwo.Tests; + +[TestClass] +[TestSubject(typeof(LegacyMessageImporter))] +public class LegacyMessageImporterTest +{ + public TestContext TestContext { get; set; } + + [TestMethod] + public void ConvertId() + { + for (var i = 0; i < 1000; i++) + { + var originalObjectId = ObjectId.NewObjectId(); + var intermediateGuid = LegacyMessageImporter.ObjectIdToGuid(originalObjectId); + var newObjectId = CustomGuidToObjectId(intermediateGuid); + + TestContext.WriteLine($"original: {originalObjectId}"); + TestContext.WriteLine($"new: {newObjectId}"); + TestContext.WriteLine($"intermediate: {intermediateGuid}"); + Assert.IsTrue(originalObjectId.Equals(newObjectId)); + } + } + + [TestMethod] + [Timeout(10_000)] + public void Import() + { + const int count = 100; + var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_"); + TestContext.WriteLine("Using temp path: " + tempDir); + var liteDbPath = Path.Join(tempDir.FullName, "original.litedb"); + TestContext.WriteLine("Using original DB path: " + liteDbPath); + var migrationDbPath = Path.Join(tempDir.FullName, "migration.litedb"); + TestContext.WriteLine("Using migration DB path: " + migrationDbPath); + var newDbPath = Path.Join(tempDir.FullName, "new.sqlitedb"); + TestContext.WriteLine("Using new DB path: " + newDbPath); + + var expectedMessages = new List(count); + using (var liteDatabase = LegacyMessageImporter.Connect(liteDbPath, readOnly: false)) + { + var messagesCollection = liteDatabase.GetCollection(LegacyMessageImporter.MessagesCollection); + var now = DateTimeOffset.UtcNow; + for (var i = 0; i < count; i++) + { + var messageId = ObjectId.NewObjectId(); + var message = MessageStoreTest.BigMessage(dateTime: now.AddSeconds(-count + i)); + // Use reflection to set Id because we don't want to add a + // setter to it and allow other code to use it. + var guid = LegacyMessageImporter.ObjectIdToGuid(messageId); + message.GetType().GetField("k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(message, guid); + expectedMessages.Add(message); + + messagesCollection.Insert(new BsonDocument { + ["_id"] = messageId, + ["Receiver"] = message.Receiver, + ["ContentId"] = message.ContentId, + ["Date"] = message.Date.ToUnixTimeMilliseconds(), + ["Code"] = BsonMapper.Global.Serialize(message.Code), + ["Sender"] = BsonMapper.Global.Serialize(message.Sender), + ["Content"] = BsonMapper.Global.Serialize(message.Content), + ["SenderSource"] = BsonMapper.Global.Serialize(message.SenderSource), + ["ContentSource"] = BsonMapper.Global.Serialize(message.ContentSource), + ["SortCode"] = BsonMapper.Global.Serialize(message.SortCode), + ["ExtraChatChannel"] = message.ExtraChatChannel, + }); + } + + Assert.AreEqual(count, messagesCollection.Count()); + } + + var dbPath = Path.Join(tempDir.FullName, "test.db"); + using var store = new MessageStore(dbPath); + + var eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath); + Assert.AreEqual(LegacyMessageImporterEligibilityStatus.Eligible, eligibility.Status); + Assert.AreEqual("", eligibility.AdditionalIneligibilityInfo); + Assert.AreEqual(liteDbPath, eligibility.OriginalDbPath); + Assert.AreEqual(migrationDbPath, eligibility.MigrationDbPath); + Assert.IsTrue(eligibility.DatabaseSizeBytes > 0); + Assert.AreEqual(count, eligibility.MessageCount); + + var importer = eligibility.StartImport(store, noLog: true); + while (importer.ImportComplete == null) + System.Threading.Thread.Sleep(10); + + Assert.IsTrue(importer.ImportComplete > importer.ImportStart); + Assert.AreEqual(count, importer.SuccessfulMessages); + Assert.AreEqual(0, importer.FailedMessages); + + var messages = store.GetMostRecentMessages(count: count + 1).ToList(); + Assert.AreEqual(count, messages.Count); + for (var i = 0; i < count; i++) + MessageStoreTest.AssertMessagesEqual(expectedMessages[i], messages[i]); + + // No longer eligible. + eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath); + Assert.AreEqual(LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists, eligibility.Status); + Assert.IsTrue(eligibility.AdditionalIneligibilityInfo.Contains("Original database file")); + Assert.AreEqual("", eligibility.OriginalDbPath); + Assert.AreEqual("", eligibility.MigrationDbPath); + Assert.AreEqual(0, eligibility.DatabaseSizeBytes); + Assert.AreEqual(0, eligibility.MessageCount); + } + + [TestMethod] + [Timeout(10_000)] + public void CorruptedImport() + { + const int count = 100; + const int corruptedIndex = 69; + var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_"); + TestContext.WriteLine("Using temp path: " + tempDir); + var liteDbPath = Path.Join(tempDir.FullName, "original.litedb"); + TestContext.WriteLine("Using original DB path: " + liteDbPath); + var migrationDbPath = Path.Join(tempDir.FullName, "migration.litedb"); + TestContext.WriteLine("Using migration DB path: " + migrationDbPath); + var newDbPath = Path.Join(tempDir.FullName, "new.sqlitedb"); + TestContext.WriteLine("Using new DB path: " + newDbPath); + + var expectedMessages = new List(count); + using (var liteDatabase = LegacyMessageImporter.Connect(liteDbPath, readOnly: false)) + { + var messagesCollection = liteDatabase.GetCollection(LegacyMessageImporter.MessagesCollection); + var now = DateTimeOffset.UtcNow; + for (var i = 0; i < count; i++) + { + if (i == corruptedIndex) + { + // This message will not be imported because it can't be + // parsed into a Message object. + messagesCollection.Insert(new BsonDocument + { + ["_id"] = ObjectId.NewObjectId(), + ["Receiver"] = 0L, + ["ContentId"] = 0L, + ["Date"] = 0L, + ["Code"] = BsonMapper.Global.Serialize(new ChatCode(0)), + ["Sender"] = BsonMapper.Global.Serialize(new List()), + ["Content"] = BsonMapper.Global.Serialize(new List()), + // These are meant to be arrays. + ["SenderSource"] = new BsonDocument(), + ["ContentSource"] = new BsonDocument(), + ["SortCode"] = BsonMapper.Global.Serialize(new SortCode(0)), + ["ExtraChatChannel"] = new Guid(), + }); + continue; + } + + var messageId = ObjectId.NewObjectId(); + var message = MessageStoreTest.BigMessage(dateTime: now.AddSeconds(-count + i)); + // Use reflection to set Id because we don't want to add a + // setter to it and allow other code to use it. + var guid = LegacyMessageImporter.ObjectIdToGuid(messageId); + message.GetType().GetField("k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(message, guid); + expectedMessages.Add(message); + + messagesCollection.Insert(new BsonDocument { + ["_id"] = messageId, + ["Receiver"] = message.Receiver, + ["ContentId"] = message.ContentId, + ["Date"] = message.Date.ToUnixTimeMilliseconds(), + ["Code"] = BsonMapper.Global.Serialize(message.Code), + ["Sender"] = BsonMapper.Global.Serialize(message.Sender), + ["Content"] = BsonMapper.Global.Serialize(message.Content), + ["SenderSource"] = BsonMapper.Global.Serialize(message.SenderSource), + ["ContentSource"] = BsonMapper.Global.Serialize(message.ContentSource), + ["SortCode"] = BsonMapper.Global.Serialize(message.SortCode), + ["ExtraChatChannel"] = message.ExtraChatChannel, + }); + } + + Assert.AreEqual(count, messagesCollection.Count()); + } + + var dbPath = Path.Join(tempDir.FullName, "test.db"); + using var store = new MessageStore(dbPath); + + var eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath); + Assert.AreEqual(LegacyMessageImporterEligibilityStatus.Eligible, eligibility.Status); + + var importer = eligibility.StartImport(store, noLog: true); + while (importer.ImportComplete == null) + System.Threading.Thread.Sleep(10); + + Assert.IsTrue(importer.ImportComplete > importer.ImportStart); + Assert.AreEqual(count - 1, importer.SuccessfulMessages); + Assert.AreEqual(1, importer.FailedMessages); + + var messages = store.GetMostRecentMessages(count: count + 1).ToList(); + Assert.AreEqual(count - 1, messages.Count); + for (var i = 0; i < count - 1; i++) + MessageStoreTest.AssertMessagesEqual(expectedMessages[i], messages[i]); + } + + /// + /// Converts Guids created by LegacyMessageImporter.ObjectIdToGuid() back to + /// their original ObjectId. If any other Guid is passed, the result is + /// lossy. + /// + private ObjectId CustomGuidToObjectId(Guid guid) + { + var guidBytes = guid.ToByteArray(); + var newObjectIdBytes = new byte[12]; + Buffer.BlockCopy(guidBytes, 0, newObjectIdBytes, 0, 7); + newObjectIdBytes[7] = guidBytes[8]; + Buffer.BlockCopy(guidBytes, 10, newObjectIdBytes, 8, 4); + return new ObjectId(newObjectIdBytes); + } +} diff --git a/ChatTwo.Tests/MessageStoreTest.cs b/ChatTwo.Tests/MessageStoreTest.cs index e7199f7..41d26da 100644 --- a/ChatTwo.Tests/MessageStoreTest.cs +++ b/ChatTwo.Tests/MessageStoreTest.cs @@ -160,7 +160,7 @@ public class MessageStoreTest { } } - private static Message BigMessage(bool uniqId = true, uint receiver = 12345, DateTimeOffset? dateTime = null) { + internal static Message BigMessage(bool uniqId = true, uint receiver = 12345, DateTimeOffset? dateTime = null) { // NOTE: These values aren't valid in the game. // NOTE: we can't test UiForeground, UiGlow, or AutoTranslatePayload // because they load data from the game. @@ -216,16 +216,13 @@ public class MessageStoreTest { ); } - private void AssertMessagesEqual(Message input, Message output) { + internal static void AssertMessagesEqual(Message input, Message output) { // Check basic fields. Assert.AreEqual(input.Id, output.Id); Assert.AreEqual(input.Receiver, output.Receiver); Assert.AreEqual(input.ContentId, output.ContentId); // Assert time is within 1 second - TestContext.WriteLine($"Input date: {input.Date.ToUniversalTime()}"); - TestContext.WriteLine($"Output date: {output.Date.ToUniversalTime()}"); var timeDifference = Math.Abs(input.Date.ToUniversalTime().Subtract(output.Date.ToUniversalTime()).TotalSeconds); - TestContext.WriteLine($"Time difference: {timeDifference}s"); Assert.IsTrue(timeDifference < 1); Assert.AreEqual(input.Code.Raw, output.Code.Raw); Assert.AreEqual($"{input.SenderSource.Encode():X}", $"{output.SenderSource.Encode():X}"); diff --git a/ChatTwo/ChatTwo.csproj b/ChatTwo/ChatTwo.csproj index 201b3a9..3089455 100755 --- a/ChatTwo/ChatTwo.csproj +++ b/ChatTwo/ChatTwo.csproj @@ -50,6 +50,7 @@ + diff --git a/ChatTwo/Chunk.cs b/ChatTwo/Chunk.cs index fe5aa44..c60e2ac 100755 --- a/ChatTwo/Chunk.cs +++ b/ChatTwo/Chunk.cs @@ -1,5 +1,6 @@ using ChatTwo.Code; using Dalamud.Game.Text.SeStringHandling; +using LiteDB; using MessagePack; namespace ChatTwo; @@ -9,6 +10,7 @@ namespace ChatTwo; [MessagePackObject] public abstract class Chunk { [IgnoreMember] + [BsonIgnore] // used by LegacyMessageImporter internal Message? Message { get; set; } [Key(0)] diff --git a/ChatTwo/Code/ChatCode.cs b/ChatTwo/Code/ChatCode.cs index a90e0ac..aae2caa 100755 --- a/ChatTwo/Code/ChatCode.cs +++ b/ChatTwo/Code/ChatCode.cs @@ -1,3 +1,5 @@ +using LiteDB; + namespace ChatTwo.Code; internal class ChatCode @@ -19,6 +21,15 @@ internal class ChatCode Target = SourceFrom(7); } + [BsonCtor] // Used by LegacyMessageImporter + public ChatCode(ushort raw, ChatType type, ChatSource source, ChatSource target) + { + Raw = raw; + Type = type; + Source = source; + Target = target; + } + internal ChatType Parent() => Type switch { ChatType.Say => ChatType.Say, diff --git a/ChatTwo/LegacyMessageImporter.cs b/ChatTwo/LegacyMessageImporter.cs new file mode 100644 index 0000000..b45aaf8 --- /dev/null +++ b/ChatTwo/LegacyMessageImporter.cs @@ -0,0 +1,356 @@ +using System.Diagnostics; +using ChatTwo.Code; +using ChatTwo.Util; +using Dalamud.Game.Text.SeStringHandling; +using Dalamud.Plugin.Services; +using LiteDB; + +namespace ChatTwo; + +internal enum LegacyMessageImporterEligibilityStatus +{ + Eligible, + IneligibleOriginalDbNotExists, + IneligibleMigrationDbExists, + IneligibleLiteDbFailed, + IneligibleNoMessages, +} + +internal class LegacyMessageImporterEligibility +{ + internal LegacyMessageImporterEligibilityStatus Status { get; private set; } + internal string AdditionalIneligibilityInfo { get; private set; } + + internal string OriginalDbPath { get; } + internal string MigrationDbPath { get; } + + internal long DatabaseSizeBytes { get; } + internal int MessageCount { get; } + + private LegacyMessageImporterEligibility(LegacyMessageImporterEligibilityStatus status, string additionalIneligibilityInfo, string originalDbPath, string migrationDbPath, long databaseSizeBytes, int messageCount) + { + Status = status; + AdditionalIneligibilityInfo = additionalIneligibilityInfo; + OriginalDbPath = originalDbPath; + MigrationDbPath = migrationDbPath; + DatabaseSizeBytes = databaseSizeBytes; + MessageCount = messageCount; + } + + private static LegacyMessageImporterEligibility NewEligible(string originalDbPath, string migrationDbPath, + long databaseSizeBytes, int messageCount) + { + return new LegacyMessageImporterEligibility(LegacyMessageImporterEligibilityStatus.Eligible, "", originalDbPath, migrationDbPath, databaseSizeBytes, messageCount); + } + + private static LegacyMessageImporterEligibility NewIneligible(LegacyMessageImporterEligibilityStatus status, string additionalIneligibilityReason) + { + return new LegacyMessageImporterEligibility(status, additionalIneligibilityReason, "", "", 0, 0); + } + + internal static LegacyMessageImporterEligibility CheckEligibility(string? originalDbPath = null, string? migrationDbPath = null) + { + originalDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat.db"); + migrationDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-litedb.db"); + + // Condition 1: the database file must exist in its original path. + if (!File.Exists(originalDbPath)) + { + return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists, $"Original database file '{originalDbPath}' does not exist"); + } + + // Condition 2: the migration file must not exist. + if (File.Exists(migrationDbPath)) + { + return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists, $"Migration database file '{migrationDbPath}' already exists, migration was already started in the past"); + } + + // Condition 3: we need to be able to connect to the original database + // path. + try + { + using var db = LegacyMessageImporter.Connect(originalDbPath); + var size = new FileInfo(originalDbPath).Length; + var count = db.GetCollection(LegacyMessageImporter.MessagesCollection).Count(); + if (count <= 0) + NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleNoMessages, $"No messages in original database file '{originalDbPath}'"); + return NewEligible(originalDbPath, migrationDbPath, size, count); + } + catch (Exception e) + { + // Notify the user about this error, because they might be wondering + // why they weren't offered a migration. + return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed, $"LiteDB connection to original database file '{originalDbPath}' failed: {e}"); + } + } + + internal LegacyMessageImporter StartImport(MessageStore targetStore, bool noLog = false) + { + if (Status != LegacyMessageImporterEligibilityStatus.Eligible) + throw new InvalidOperationException($"Migration not eligible: status is {Status}"); + + return new LegacyMessageImporter(targetStore, originalDbPath: OriginalDbPath, migrationDbPath: MigrationDbPath, noLog: noLog); + } + + /// + /// Makes the migration ineligible so the user won't be asked again. + /// + internal void RenameOldDatabase() + { + File.Move(OriginalDbPath, MigrationDbPath); + Status = LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists; + AdditionalIneligibilityInfo = "User chose to rename the old database file"; + } +} + +internal class LegacyMessageImporter : IDisposable +{ + internal const string MessagesCollection = "messages"; + private const int MaxFailedMessageLogs = 10; + + private readonly MessageStore _targetStore; + private readonly IPluginLog? _log; + + private LiteDatabase? _database; + + internal long ImportStart { get; } // ticks + internal int ImportCount { get; private set; } + internal int SuccessfulMessages { get; private set; } + internal int FailedMessages { get; private set; } + internal int ProcessedMessages => SuccessfulMessages + FailedMessages; + internal int RemainingMessages => ImportCount - ProcessedMessages; + // Progress from 0 to 1. + internal float Progress => ImportCount > 0 ? ProcessedMessages / (float)ImportCount : 1; + // Message count processed in the last second. + internal float CurrentMessageRate { get; private set; } + // ETA based on CurrentMessageRate. + internal TimeSpan EstimatedTimeRemaining => TimeSpan.FromSeconds(CurrentMessageRate > 0 ? (ImportCount - SuccessfulMessages - FailedMessages) / CurrentMessageRate : 0); + internal long? ImportComplete { get; private set; } // ticks + + // This can be set by the user to limit the rate at which messages are + // imported. If the rate exceeds this value, the importer will sleep for the + // remainder of the second. + internal int MaxMessageRate { get; set; } = 250; // start low + + // Do not call this directly, use + // LegacyMessageImporterEligibility.StartImport instead. + internal LegacyMessageImporter(MessageStore targetStore, string? originalDbPath = null, string? migrationDbPath = null, bool noLog = false) + { + _targetStore = targetStore; + originalDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat.db"); + migrationDbPath ??= migrationDbPath ?? Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-litedb.db"); + _log = noLog ? null : Plugin.Log; + + _log?.Info($"[Migration] Moving '{originalDbPath}' to '{migrationDbPath}'"); + File.Move(originalDbPath, migrationDbPath); + _log?.Info($"[Migration] Opening '{migrationDbPath}'"); + _database = Connect(migrationDbPath); + + ImportStart = Environment.TickCount64; + new Thread(DoImport).Start(); + } + + public void Dispose() + { + // TODO: cancel thread and wait for it to close + _database?.Dispose(); + } + + internal static LiteDatabase Connect(string dbPath, bool readOnly = true) + { + BsonMapper.Global = new BsonMapper + { + IncludeNonPublic = true, + TrimWhitespace = false + }; + + BsonMapper.Global.RegisterType( + payload => + { + switch (payload) + { + case AchievementPayload achievement: + return new BsonDocument(new Dictionary + { + ["Type"] = new("Achievement"), + ["Id"] = new(achievement.Id) + }); + case PartyFinderPayload partyFinder: + return new BsonDocument(new Dictionary + { + ["Type"] = new("PartyFinder"), + ["Id"] = new(partyFinder.Id) + }); + case UriPayload uri: + return new BsonDocument(new Dictionary + { + ["Type"] = new("URI"), + ["Uri"] = new(uri.Uri.ToString()) + }); + } + + return payload?.Encode(); + }, + bson => + { + if (bson.IsNull) + return null; + + if (bson.IsDocument) + return bson["Type"].AsString switch + { + "Achievement" => new AchievementPayload((uint)bson["Id"].AsInt64), + "PartyFinder" => new PartyFinderPayload((uint)bson["Id"].AsInt64), + "URI" => new UriPayload(new Uri(bson["Uri"].AsString)), + _ => null + }; + + return Payload.Decode(new BinaryReader(new MemoryStream(bson.AsBinary))); + }); + + BsonMapper.Global.RegisterType( + seString => seString == null + ? null + : new BsonArray(seString.Payloads.Select(payload => new BsonValue(payload.Encode()))), + bson => + { + if (bson.IsNull) + return null; + + var array = bson.IsArray ? bson.AsArray : bson["Payloads"].AsArray; + var payloads = array + .Select(payload => Payload.Decode(new BinaryReader(new MemoryStream(payload.AsBinary)))) + .ToList(); + return new SeString(payloads); + } + ); + BsonMapper.Global.RegisterType( + type => (int)type, + bson => (ChatType)bson.AsInt32 + ); + BsonMapper.Global.RegisterType( + source => (int)source, + bson => (ChatSource)bson.AsInt32 + ); + BsonMapper.Global.RegisterType( + dateTime => dateTime.Subtract(DateTime.UnixEpoch).TotalMilliseconds, + bson => DateTime.UnixEpoch.AddMilliseconds(bson.AsInt64) + ); + + var connString = $"Filename='{dbPath}';Connection=direct;ReadOnly={readOnly}"; + var conn = new LiteDatabase(connString, BsonMapper.Global) + { + CheckpointSize = 1_000, + Timeout = TimeSpan.FromSeconds(1) + }; + var messages = conn.GetCollection(MessagesCollection); + messages.EnsureIndex(msg => msg.Date); + return conn; + } + + private void DoImport() + { + var importRateTimer = Stopwatch.StartNew(); + var messagesInLastSecond = 0; + + // Query raw BsonDocuments, so we can convert them in individual + // try-catch blocks. + var messagesCollection = _database!.GetCollection(MessagesCollection); + var totalMessages = messagesCollection.Count(); + ImportCount = totalMessages; + var messages = messagesCollection.Query().OrderBy(msg => msg.Date).ToDocuments(); + foreach (var messageDoc in messages) + { + try + { + var message = BsonDocumentToMessage(messageDoc); + _targetStore.UpsertMessage(message); + SuccessfulMessages++; + } + catch (Exception e) + { + FailedMessages++; + if (FailedMessages <= MaxFailedMessageLogs) + _log?.Error( + $"[Migration] Failed to import message '{messageDoc["_id"].AsObjectId}' (usually due to corruption): {e}"); + if (FailedMessages == MaxFailedMessageLogs) + _log?.Error("[Migration] Further failed message logs will be suppressed"); + } + + messagesInLastSecond++; + if (MaxMessageRate > 0 && messagesInLastSecond > MaxMessageRate) + { + var sleepTime = 1000 - (int)importRateTimer.ElapsedMilliseconds; + if (sleepTime > 0) + Thread.Sleep(sleepTime); + } + if (importRateTimer.ElapsedMilliseconds > 1000) + { + CurrentMessageRate = messagesInLastSecond / (float)importRateTimer.ElapsedMilliseconds * 1000; + importRateTimer.Restart(); + messagesInLastSecond = 0; + } + + // Log every 1,000 messages + if ((SuccessfulMessages + FailedMessages) % 1000 == 0) + _log?.Information( + $"[Migration] Progress: successfully imported {SuccessfulMessages}/{totalMessages} messages ({FailedMessages} failures)"); + } + + _log?.Information($"[Migration] Imported {SuccessfulMessages}/{FailedMessages} messages, {FailedMessages} failed"); + + if (ProcessedMessages != totalMessages) + _log?.Warning( + $"[Migration] Total message count mismatch: expected {totalMessages}, got {SuccessfulMessages + FailedMessages}"); + + ImportComplete = Environment.TickCount64; + _database.Dispose(); + _database = null; + } + + private static Message BsonDocumentToMessage(BsonDocument doc) + { + return new Message( + ObjectIdToGuid(doc["_id"].AsObjectId), + (ulong)doc["Receiver"].AsInt64, + (ulong)doc["ContentId"].AsInt64, + DateTimeOffset.FromUnixTimeMilliseconds(doc["Date"].AsInt64), + BsonMapper.Global.Deserialize(doc["Code"].AsDocument), + BsonMapper.Global.Deserialize>(doc["Sender"].AsArray), + BsonMapper.Global.Deserialize>(doc["Content"].AsArray), + BsonMapper.Global.Deserialize(doc["SenderSource"].AsArray), + BsonMapper.Global.Deserialize(doc["ContentSource"].AsArray), + BsonMapper.Global.Deserialize(doc["SortCode"].AsDocument), + doc["ExtraChatChannel"].AsGuid + ); + } + + internal static Guid ObjectIdToGuid(ObjectId objectId) + { + // "Generate" a new Guid based on the ObjectId from the original + // database. We want to have a stable unique identifier for each message + // so that if the migration somehow happens twice the objects won't be + // duplicated. + // + // Technically, when Guids are generated they follow a specific pattern. + // However, in practice it doesn't matter at all, and we can just + // generate whatever we want. + var objectIdBytes = objectId.ToByteArray(); + var guidBytes = new byte[16]; + // Copy the first 7 bytes directly + Buffer.BlockCopy(objectIdBytes, 0, guidBytes, 0, 7); + // Fixed byte for version + guidBytes[7] = 0b11111111; + // Copy the next byte. + guidBytes[8] = objectIdBytes[7]; + // Fixed reserved byte + guidBytes[9] = 0b11111111; + // Copy the last 4 bytes. + Buffer.BlockCopy(objectIdBytes, 8, guidBytes, 10, 4); + // Set the last 2 bytes to beef + guidBytes[14] = 0xbe; + guidBytes[15] = 0xef; + + return new Guid(guidBytes); + } +} diff --git a/ChatTwo/Message.cs b/ChatTwo/Message.cs index 1f83b63..8424321 100755 --- a/ChatTwo/Message.cs +++ b/ChatTwo/Message.cs @@ -3,6 +3,7 @@ using ChatTwo.Util; using Dalamud.Game.Text.SeStringHandling; using Dalamud.Game.Text.SeStringHandling.Payloads; using System.Text.RegularExpressions; +using LiteDB; namespace ChatTwo; @@ -10,7 +11,8 @@ internal class SortCode { internal ChatType Type { get; } internal ChatSource Source { get; } - internal SortCode(ChatType type, ChatSource source) { + [BsonCtor] // Used by LegacyMessageImporter + public SortCode(ChatType type, ChatSource source) { Type = type; Source = source; } diff --git a/ChatTwo/MessageManager.cs b/ChatTwo/MessageManager.cs index 78d63e2..c2baa68 100644 --- a/ChatTwo/MessageManager.cs +++ b/ChatTwo/MessageManager.cs @@ -19,7 +19,6 @@ internal class MessageManager : IDisposable internal MessageStore Store { get; } private ConcurrentQueue<(uint, Message)> Pending { get; } = new(); - private Stopwatch MaintenanceTimer { get; } = new(); private Dictionary Formats { get; } = new(); private ulong LastContentId { get; set; } @@ -35,7 +34,6 @@ internal class MessageManager : IDisposable internal MessageManager(Plugin plugin) { Plugin = plugin; - MaintenanceTimer.Start(); Store = new MessageStore(DatabasePath()); Plugin.ChatGui.ChatMessageUnhandled += ChatMessage; @@ -73,12 +71,6 @@ internal class MessageManager : IDisposable private void GetMessageInfo(IFramework framework) { - if (MaintenanceTimer.Elapsed > TimeSpan.FromMinutes(5)) - { - MaintenanceTimer.Restart(); - new Thread(() => Store.PerformMaintenance()).Start(); - } - if (!Pending.TryDequeue(out var entry)) return; diff --git a/ChatTwo/Plugin.cs b/ChatTwo/Plugin.cs index 5186025..cec74f4 100755 --- a/ChatTwo/Plugin.cs +++ b/ChatTwo/Plugin.cs @@ -49,6 +49,7 @@ public sealed class Plugin : IDalamudPlugin public ChatLogWindow ChatLogWindow { get; } public CommandHelpWindow CommandHelpWindow { get; } public SeStringDebugger SeStringDebugger { get; } + internal LegacyMesasgeImporterWindow LegacyMesasgeImporterWindow { get; } internal Configuration Config { get; } internal Commands Commands { get; } @@ -104,6 +105,10 @@ public sealed class Plugin : IDalamudPlugin MessageManager = new MessageManager(this); // requires Ui + // Requires MessageManager + LegacyMesasgeImporterWindow = new LegacyMesasgeImporterWindow(MessageManager.Store); + WindowSystem.AddWindow(LegacyMesasgeImporterWindow); + // let all the other components register, then initialise commands Commands.Initialise(); @@ -138,6 +143,7 @@ public sealed class Plugin : IDalamudPlugin ChatLogWindow?.Dispose(); SettingsWindow?.Dispose(); SeStringDebugger?.Dispose(); + LegacyMesasgeImporterWindow?.Dispose(); ExtraChat?.Dispose(); Ipc?.Dispose(); diff --git a/ChatTwo/Ui/LegacyMesasgeImporterWindow.cs b/ChatTwo/Ui/LegacyMesasgeImporterWindow.cs new file mode 100644 index 0000000..0c2eece --- /dev/null +++ b/ChatTwo/Ui/LegacyMesasgeImporterWindow.cs @@ -0,0 +1,209 @@ +using System.Numerics; +using ChatTwo.Resources; +using ChatTwo.Util; +using Dalamud.Interface.ImGuiNotification; +using Dalamud.Interface.Internal.Notifications; +using Dalamud.Interface.Windowing; +using ImGuiNET; + +namespace ChatTwo.Ui; + +internal class LegacyMesasgeImporterWindow : Window +{ + private readonly MessageStore _store; + + private LegacyMessageImporterEligibility Eligibility { get; set; } + private LegacyMessageImporter? Importer { get; set; } + + internal LegacyMesasgeImporterWindow(MessageStore store) : base("Chat 2 Legacy Importer###chat2-legacy-importer") + { + _store = store; + Eligibility = LegacyMessageImporterEligibility.CheckEligibility(); + LogAndNotify(); + } + + public void Dispose() + { + Importer?.Dispose(); + } + + private void LogAndNotify() + { + Plugin.Log.Info( + $"[Migration] Checked migration eligibility: {Eligibility.Status} - '{Eligibility.AdditionalIneligibilityInfo}'"); + + switch (Eligibility.Status) + { + case LegacyMessageImporterEligibilityStatus.Eligible: + { + var notification = Plugin.Notification.AddNotification(new Notification + { + Type = NotificationType.Info, + // The user needs to dismiss this for it to go away. + InitialDuration = TimeSpan.FromHours(6), + Title = "Chat 2 Migration", + Content = "Import messages from old database into new database? Click for more information.", + }); + // TODO: clicking does not dismiss + notification.Click += _ => IsOpen = true; + notification.Dismiss += _ => WriteChatMessage(); + break; + } + + case LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed: + { + var notification = Plugin.Notification.AddNotification(new Notification + { + Type = NotificationType.Warning, + InitialDuration = TimeSpan.FromMinutes(1), + Title = "Chat Two Migration", + Content = + "Migration is not possible because the old database could not be opened. Click for more information." + }); + notification.Click += _ => IsOpen = true; + notification.Dismiss += _ => WriteChatMessage(); + break; + } + } + } + + private void WriteChatMessage() + { + // TODO: write a message to chat saying how to open the window again + // TODO: add a way of opening the window again, maybe a command or in + // database settings + } + + public override void Draw() + { + if (Importer != null) + { + DrawImportStatus(); + return; + } + + if (Eligibility.Status == LegacyMessageImporterEligibilityStatus.Eligible) + DrawEligible(); + else + DrawIneligible(); + } + + private void DrawEligible() + { + // TODO: pretty + ImGui.Text("Import database messages from legacy LiteDB database to Sqlite database?"); + ImGui.Text($"Message count: {Eligibility.MessageCount}"); + ImGui.Text($"Database size: {Eligibility.DatabaseSizeBytes}"); + + if (ImGui.Button("Yes, import messages")) + { + // Next draw call will run DrawImportStatus(). + Importer = Eligibility.StartImport(_store); + return; + } + + ImGui.SameLine(); + + if (ImGuiUtil.CtrlShiftButton("No, do not import messages", + "Ctrl+Shift: renames old database to avoid prompting again")) + { + Eligibility.RenameOldDatabase(); + IsOpen = false; + } + } + + private void DrawIneligible() + { + // TODO: pretty + ImGui.Text("Your legacy LiteDB database is not eligible for import:"); + switch (Eligibility.Status) + { + case LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists: + ImGui.Text("The old database could not be found."); + break; + case LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists: + ImGui.Text("The migration process was already started."); + break; + case LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed: + ImGui.Text("The old database could not be opened."); + break; + case LegacyMessageImporterEligibilityStatus.IneligibleNoMessages: + ImGui.Text("The old database contains no messages."); + break; + case LegacyMessageImporterEligibilityStatus.Eligible: + default: + throw new ArgumentOutOfRangeException(); + } + if (!string.IsNullOrWhiteSpace(Eligibility.AdditionalIneligibilityInfo)) + ImGui.Text(Eligibility.AdditionalIneligibilityInfo); + + // LiteDB failures notify the user, so give them a chance to rename the + // database to avoid prompting again. + if (Eligibility.Status == LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed) + { + if (ImGuiUtil.CtrlShiftButton("Rename old database", + "Ctrl+Shift: rename old database to avoid import prompt in the future")) + { + Eligibility.RenameOldDatabase(); + // TODO: notify success as this changes the status + } + } + } + + private void DrawImportStatus() + { + // TODO: pretty + if (Importer == null) + return; + + var importStart = Importer.ImportStart; + var importEnd = Importer.ImportComplete; + var total = Importer.ImportCount; + var successful = Importer!.SuccessfulMessages; + var failed = Importer.FailedMessages; + var remaining = Importer.RemainingMessages; + + if (importEnd != null) + { + ImGui.Text($"Completed migration in {Duration(importStart, importEnd.Value)}"); + ImGui.Text($"Successfully imported: {successful} messages"); + ImGui.Text($"Failed to import: {failed} messages"); + ImGui.Text($"Unaccounted for: {remaining}"); + ImGui.Text("See logs for more details: /xllog"); + return; + } + + // TODO: implement Importer.MaxMessageRate slider in UI, values 0 (infinity) => 10000 + + ImGui.Text($"Importing messages... {Importer.Progress:P}%"); + ImGui.Text($"Duration: {Duration(importStart, Environment.TickCount64)}"); + ImGui.Text($"Successfully imported: {successful} messages"); + ImGui.Text($"Failed to import: {failed} messages"); + ImGui.Text($"Progress: {Importer.ProcessedMessages}/{total} messages"); + ImGui.Text($"Remaining: {remaining} messages"); + ImGui.Text($"Messages per second: {Importer.CurrentMessageRate}"); + ImGui.Text($"Estimated time remaining: {Importer.EstimatedTimeRemaining}"); + ImGui.Text("See logs for more details: /xllog"); + + // TODO: this doesn't render properly + ImGui.ProgressBar(Importer.Progress, new Vector2(0.0f, 0.0f), $"{Importer.Progress:P}%"); + + if (ImGuiUtil.CtrlShiftButton("Cancel import", "Ctrl+Shift: cancel import and close window")) + { + // TODO: This currently crashes the whole game because we don't ask + // the importer thread to stop and wait for it to stop before + // disposing it. + // See LegacyMessageImporter.Dispose() for more details. + /* + Importer.Dispose(); + Importer = null; + Eligibility = LegacyMessageImporterEligibility.CheckEligibility(); + */ + } + } + + private static TimeSpan Duration(long startTicks, long endTicks) + { + return endTicks < startTicks ? TimeSpan.Zero : TimeSpan.FromTicks(endTicks - startTicks); + } +} diff --git a/ChatTwo/packages.lock.json b/ChatTwo/packages.lock.json index 18f7d6b..f3eb1f5 100644 --- a/ChatTwo/packages.lock.json +++ b/ChatTwo/packages.lock.json @@ -8,6 +8,12 @@ "resolved": "2.1.12", "contentHash": "Sc0PVxvgg4NQjcI8n10/VfUQBAS4O+Fw2pZrAqBdRMbthYGeogzu5+xmIGCGmsEZ/ukMOBuAqiNiB5qA3MRalg==" }, + "LiteDB": { + "type": "Direct", + "requested": "[5.0.19, )", + "resolved": "5.0.19", + "contentHash": "O8XPBptE4SygW2SN6skqg/VDVTrjpJ0p6+i7Cp8x8razbu2ORLSd6ep3JdhDIdL6h57Bcxv2BuVaN70IKpXI0Q==" + }, "MessagePack": { "type": "Direct", "requested": "[2.5.140, )",