feat: add LiteDB => Sqlite importer

2024-04-22 02:43:48 +10:00
parent c152f3dfde
commit fcdb5625f3
11 changed files with 816 additions and 14 deletions
@@ -0,0 +1,220 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using ChatTwo.Code;
+using JetBrains.Annotations;
+using LiteDB;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace ChatTwo.Tests;
+
+[TestClass]
+[TestSubject(typeof(LegacyMessageImporter))]
+public class LegacyMessageImporterTest
+{
+    public TestContext TestContext { get; set; }
+
+    [TestMethod]
+    public void ConvertId()
+    {
+        for (var i = 0; i < 1000; i++)
+        {
+            var originalObjectId = ObjectId.NewObjectId();
+            var intermediateGuid = LegacyMessageImporter.ObjectIdToGuid(originalObjectId);
+            var newObjectId = CustomGuidToObjectId(intermediateGuid);
+
+            TestContext.WriteLine($"original:     {originalObjectId}");
+            TestContext.WriteLine($"new:          {newObjectId}");
+            TestContext.WriteLine($"intermediate: {intermediateGuid}");
+            Assert.IsTrue(originalObjectId.Equals(newObjectId));
+        }
+    }
+
+    [TestMethod]
+    [Timeout(10_000)]
+    public void Import()
+    {
+        const int count = 100;
+        var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_");
+        TestContext.WriteLine("Using temp path: " + tempDir);
+        var liteDbPath = Path.Join(tempDir.FullName, "original.litedb");
+        TestContext.WriteLine("Using original DB path: " + liteDbPath);
+        var migrationDbPath = Path.Join(tempDir.FullName, "migration.litedb");
+        TestContext.WriteLine("Using migration DB path: " + migrationDbPath);
+        var newDbPath = Path.Join(tempDir.FullName, "new.sqlitedb");
+        TestContext.WriteLine("Using new DB path: " + newDbPath);
+
+        var expectedMessages = new List<Message>(count);
+        using (var liteDatabase = LegacyMessageImporter.Connect(liteDbPath, readOnly: false))
+        {
+            var messagesCollection = liteDatabase.GetCollection(LegacyMessageImporter.MessagesCollection);
+            var now = DateTimeOffset.UtcNow;
+            for (var i = 0; i < count; i++)
+            {
+                var messageId = ObjectId.NewObjectId();
+                var message = MessageStoreTest.BigMessage(dateTime: now.AddSeconds(-count + i));
+                // Use reflection to set Id because we don't want to add a
+                // setter to it and allow other code to use it.
+                var guid = LegacyMessageImporter.ObjectIdToGuid(messageId);
+                message.GetType().GetField("<Id>k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(message, guid);
+                expectedMessages.Add(message);
+
+                messagesCollection.Insert(new BsonDocument {
+                    ["_id"] = messageId,
+                    ["Receiver"] = message.Receiver,
+                    ["ContentId"] = message.ContentId,
+                    ["Date"] = message.Date.ToUnixTimeMilliseconds(),
+                    ["Code"] = BsonMapper.Global.Serialize(message.Code),
+                    ["Sender"] = BsonMapper.Global.Serialize(message.Sender),
+                    ["Content"] = BsonMapper.Global.Serialize(message.Content),
+                    ["SenderSource"] = BsonMapper.Global.Serialize(message.SenderSource),
+                    ["ContentSource"] = BsonMapper.Global.Serialize(message.ContentSource),
+                    ["SortCode"] = BsonMapper.Global.Serialize(message.SortCode),
+                    ["ExtraChatChannel"] = message.ExtraChatChannel,
+                });
+            }
+
+            Assert.AreEqual(count, messagesCollection.Count());
+        }
+
+        var dbPath = Path.Join(tempDir.FullName, "test.db");
+        using var store = new MessageStore(dbPath);
+
+        var eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath);
+        Assert.AreEqual(LegacyMessageImporterEligibilityStatus.Eligible, eligibility.Status);
+        Assert.AreEqual("", eligibility.AdditionalIneligibilityInfo);
+        Assert.AreEqual(liteDbPath, eligibility.OriginalDbPath);
+        Assert.AreEqual(migrationDbPath, eligibility.MigrationDbPath);
+        Assert.IsTrue(eligibility.DatabaseSizeBytes > 0);
+        Assert.AreEqual(count, eligibility.MessageCount);
+
+        var importer = eligibility.StartImport(store, noLog: true);
+        while (importer.ImportComplete == null)
+            System.Threading.Thread.Sleep(10);
+
+        Assert.IsTrue(importer.ImportComplete > importer.ImportStart);
+        Assert.AreEqual(count, importer.SuccessfulMessages);
+        Assert.AreEqual(0, importer.FailedMessages);
+
+        var messages = store.GetMostRecentMessages(count: count + 1).ToList();
+        Assert.AreEqual(count, messages.Count);
+        for (var i = 0; i < count; i++)
+            MessageStoreTest.AssertMessagesEqual(expectedMessages[i], messages[i]);
+
+        // No longer eligible.
+        eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath);
+        Assert.AreEqual(LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists, eligibility.Status);
+        Assert.IsTrue(eligibility.AdditionalIneligibilityInfo.Contains("Original database file"));
+        Assert.AreEqual("", eligibility.OriginalDbPath);
+        Assert.AreEqual("", eligibility.MigrationDbPath);
+        Assert.AreEqual(0, eligibility.DatabaseSizeBytes);
+        Assert.AreEqual(0, eligibility.MessageCount);
+    }
+
+    [TestMethod]
+    [Timeout(10_000)]
+    public void CorruptedImport()
+    {
+        const int count = 100;
+        const int corruptedIndex = 69;
+        var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_");
+        TestContext.WriteLine("Using temp path: " + tempDir);
+        var liteDbPath = Path.Join(tempDir.FullName, "original.litedb");
+        TestContext.WriteLine("Using original DB path: " + liteDbPath);
+        var migrationDbPath = Path.Join(tempDir.FullName, "migration.litedb");
+        TestContext.WriteLine("Using migration DB path: " + migrationDbPath);
+        var newDbPath = Path.Join(tempDir.FullName, "new.sqlitedb");
+        TestContext.WriteLine("Using new DB path: " + newDbPath);
+
+        var expectedMessages = new List<Message>(count);
+        using (var liteDatabase = LegacyMessageImporter.Connect(liteDbPath, readOnly: false))
+        {
+            var messagesCollection = liteDatabase.GetCollection(LegacyMessageImporter.MessagesCollection);
+            var now = DateTimeOffset.UtcNow;
+            for (var i = 0; i < count; i++)
+            {
+                if (i == corruptedIndex)
+                {
+                    // This message will not be imported because it can't be
+                    // parsed into a Message object.
+                    messagesCollection.Insert(new BsonDocument
+                    {
+                        ["_id"] = ObjectId.NewObjectId(),
+                        ["Receiver"] = 0L,
+                        ["ContentId"] = 0L,
+                        ["Date"] = 0L,
+                        ["Code"] = BsonMapper.Global.Serialize(new ChatCode(0)),
+                        ["Sender"] = BsonMapper.Global.Serialize(new List<Chunk>()),
+                        ["Content"] = BsonMapper.Global.Serialize(new List<Chunk>()),
+                        // These are meant to be arrays.
+                        ["SenderSource"] = new BsonDocument(),
+                        ["ContentSource"] = new BsonDocument(),
+                        ["SortCode"] = BsonMapper.Global.Serialize(new SortCode(0)),
+                        ["ExtraChatChannel"] = new Guid(),
+                    });
+                    continue;
+                }
+
+                var messageId = ObjectId.NewObjectId();
+                var message = MessageStoreTest.BigMessage(dateTime: now.AddSeconds(-count + i));
+                // Use reflection to set Id because we don't want to add a
+                // setter to it and allow other code to use it.
+                var guid = LegacyMessageImporter.ObjectIdToGuid(messageId);
+                message.GetType().GetField("<Id>k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(message, guid);
+                expectedMessages.Add(message);
+
+                messagesCollection.Insert(new BsonDocument {
+                    ["_id"] = messageId,
+                    ["Receiver"] = message.Receiver,
+                    ["ContentId"] = message.ContentId,
+                    ["Date"] = message.Date.ToUnixTimeMilliseconds(),
+                    ["Code"] = BsonMapper.Global.Serialize(message.Code),
+                    ["Sender"] = BsonMapper.Global.Serialize(message.Sender),
+                    ["Content"] = BsonMapper.Global.Serialize(message.Content),
+                    ["SenderSource"] = BsonMapper.Global.Serialize(message.SenderSource),
+                    ["ContentSource"] = BsonMapper.Global.Serialize(message.ContentSource),
+                    ["SortCode"] = BsonMapper.Global.Serialize(message.SortCode),
+                    ["ExtraChatChannel"] = message.ExtraChatChannel,
+                });
+            }
+
+            Assert.AreEqual(count, messagesCollection.Count());
+        }
+
+        var dbPath = Path.Join(tempDir.FullName, "test.db");
+        using var store = new MessageStore(dbPath);
+
+        var eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath);
+        Assert.AreEqual(LegacyMessageImporterEligibilityStatus.Eligible, eligibility.Status);
+
+        var importer = eligibility.StartImport(store, noLog: true);
+        while (importer.ImportComplete == null)
+            System.Threading.Thread.Sleep(10);
+
+        Assert.IsTrue(importer.ImportComplete > importer.ImportStart);
+        Assert.AreEqual(count - 1, importer.SuccessfulMessages);
+        Assert.AreEqual(1, importer.FailedMessages);
+
+        var messages = store.GetMostRecentMessages(count: count + 1).ToList();
+        Assert.AreEqual(count - 1, messages.Count);
+        for (var i = 0; i < count - 1; i++)
+            MessageStoreTest.AssertMessagesEqual(expectedMessages[i], messages[i]);
+    }
+
+    /// <summary>
+    /// Converts Guids created by LegacyMessageImporter.ObjectIdToGuid() back to
+    /// their original ObjectId. If any other Guid is passed, the result is
+    /// lossy.
+    /// </summary>
+    private ObjectId CustomGuidToObjectId(Guid guid)
+    {
+        var guidBytes = guid.ToByteArray();
+        var newObjectIdBytes = new byte[12];
+        Buffer.BlockCopy(guidBytes, 0, newObjectIdBytes, 0, 7);
+        newObjectIdBytes[7] = guidBytes[8];
+        Buffer.BlockCopy(guidBytes, 10, newObjectIdBytes, 8, 4);
+        return new ObjectId(newObjectIdBytes);
+    }
+}
@@ -160,7 +160,7 @@ public class MessageStoreTest {
        }
    }

-    private static Message BigMessage(bool uniqId = true, uint receiver = 12345, DateTimeOffset? dateTime = null) {
+    internal static Message BigMessage(bool uniqId = true, uint receiver = 12345, DateTimeOffset? dateTime = null) {
        // NOTE: These values aren't valid in the game.
        // NOTE: we can't test UiForeground, UiGlow, or AutoTranslatePayload
        // because they load data from the game.
@@ -216,16 +216,13 @@ public class MessageStoreTest {
        );
    }

-    private void AssertMessagesEqual(Message input, Message output) {
+    internal static void AssertMessagesEqual(Message input, Message output) {
        // Check basic fields.
        Assert.AreEqual(input.Id, output.Id);
        Assert.AreEqual(input.Receiver, output.Receiver);
        Assert.AreEqual(input.ContentId, output.ContentId);
        // Assert time is within 1 second
-        TestContext.WriteLine($"Input date: {input.Date.ToUniversalTime()}");
-        TestContext.WriteLine($"Output date: {output.Date.ToUniversalTime()}");
        var timeDifference = Math.Abs(input.Date.ToUniversalTime().Subtract(output.Date.ToUniversalTime()).TotalSeconds);
-        TestContext.WriteLine($"Time difference: {timeDifference}s");
        Assert.IsTrue(timeDifference < 1);
        Assert.AreEqual(input.Code.Raw, output.Code.Raw);
        Assert.AreEqual($"{input.SenderSource.Encode():X}", $"{output.SenderSource.Encode():X}");