feat: add LiteDB => Sqlite importer

ChatTwo.Tests/LegacyMessageImporterTest.cs

@@ -0,0 +1,220 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using ChatTwo.Code;
+using JetBrains.Annotations;
+using LiteDB;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace ChatTwo.Tests;
+
+[TestClass]
+[TestSubject(typeof(LegacyMessageImporter))]
+public class LegacyMessageImporterTest
+{
+    public TestContext TestContext { get; set; }
+
+    [TestMethod]
+    public void ConvertId()
+    {
+        for (var i = 0; i < 1000; i++)
+        {
+            var originalObjectId = ObjectId.NewObjectId();
+            var intermediateGuid = LegacyMessageImporter.ObjectIdToGuid(originalObjectId);
+            var newObjectId = CustomGuidToObjectId(intermediateGuid);
+
+            TestContext.WriteLine($"original:     {originalObjectId}");
+            TestContext.WriteLine($"new:          {newObjectId}");
+            TestContext.WriteLine($"intermediate: {intermediateGuid}");
+            Assert.IsTrue(originalObjectId.Equals(newObjectId));
+        }
+    }
+
+    [TestMethod]
+    [Timeout(10_000)]
+    public void Import()
+    {
+        const int count = 100;
+        var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_");
+        TestContext.WriteLine("Using temp path: " + tempDir);
+        var liteDbPath = Path.Join(tempDir.FullName, "original.litedb");
+        TestContext.WriteLine("Using original DB path: " + liteDbPath);
+        var migrationDbPath = Path.Join(tempDir.FullName, "migration.litedb");
+        TestContext.WriteLine("Using migration DB path: " + migrationDbPath);
+        var newDbPath = Path.Join(tempDir.FullName, "new.sqlitedb");
+        TestContext.WriteLine("Using new DB path: " + newDbPath);
+
+        var expectedMessages = new List<Message>(count);
+        using (var liteDatabase = LegacyMessageImporter.Connect(liteDbPath, readOnly: false))
+        {
+            var messagesCollection = liteDatabase.GetCollection(LegacyMessageImporter.MessagesCollection);
+            var now = DateTimeOffset.UtcNow;
+            for (var i = 0; i < count; i++)
+            {
+                var messageId = ObjectId.NewObjectId();
+                var message = MessageStoreTest.BigMessage(dateTime: now.AddSeconds(-count + i));
+                // Use reflection to set Id because we don't want to add a
+                // setter to it and allow other code to use it.
+                var guid = LegacyMessageImporter.ObjectIdToGuid(messageId);
+                message.GetType().GetField("<Id>k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(message, guid);
+                expectedMessages.Add(message);
+
+                messagesCollection.Insert(new BsonDocument {
+                    ["_id"] = messageId,
+                    ["Receiver"] = message.Receiver,
+                    ["ContentId"] = message.ContentId,
+                    ["Date"] = message.Date.ToUnixTimeMilliseconds(),
+                    ["Code"] = BsonMapper.Global.Serialize(message.Code),
+                    ["Sender"] = BsonMapper.Global.Serialize(message.Sender),
+                    ["Content"] = BsonMapper.Global.Serialize(message.Content),
+                    ["SenderSource"] = BsonMapper.Global.Serialize(message.SenderSource),
+                    ["ContentSource"] = BsonMapper.Global.Serialize(message.ContentSource),
+                    ["SortCode"] = BsonMapper.Global.Serialize(message.SortCode),
+                    ["ExtraChatChannel"] = message.ExtraChatChannel,
+                });
+            }
+
+            Assert.AreEqual(count, messagesCollection.Count());
+        }
+
+        var dbPath = Path.Join(tempDir.FullName, "test.db");
+        using var store = new MessageStore(dbPath);
+
+        var eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath);
+        Assert.AreEqual(LegacyMessageImporterEligibilityStatus.Eligible, eligibility.Status);
+        Assert.AreEqual("", eligibility.AdditionalIneligibilityInfo);
+        Assert.AreEqual(liteDbPath, eligibility.OriginalDbPath);
+        Assert.AreEqual(migrationDbPath, eligibility.MigrationDbPath);
+        Assert.IsTrue(eligibility.DatabaseSizeBytes > 0);
+        Assert.AreEqual(count, eligibility.MessageCount);
+
+        var importer = eligibility.StartImport(store, noLog: true);
+        while (importer.ImportComplete == null)
+            System.Threading.Thread.Sleep(10);
+
+        Assert.IsTrue(importer.ImportComplete > importer.ImportStart);
+        Assert.AreEqual(count, importer.SuccessfulMessages);
+        Assert.AreEqual(0, importer.FailedMessages);
+
+        var messages = store.GetMostRecentMessages(count: count + 1).ToList();
+        Assert.AreEqual(count, messages.Count);
+        for (var i = 0; i < count; i++)
+            MessageStoreTest.AssertMessagesEqual(expectedMessages[i], messages[i]);
+
+        // No longer eligible.
+        eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath);
+        Assert.AreEqual(LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists, eligibility.Status);
+        Assert.IsTrue(eligibility.AdditionalIneligibilityInfo.Contains("Original database file"));
+        Assert.AreEqual("", eligibility.OriginalDbPath);
+        Assert.AreEqual("", eligibility.MigrationDbPath);
+        Assert.AreEqual(0, eligibility.DatabaseSizeBytes);
+        Assert.AreEqual(0, eligibility.MessageCount);
+    }
+
+    [TestMethod]
+    [Timeout(10_000)]
+    public void CorruptedImport()
+    {
+        const int count = 100;
+        const int corruptedIndex = 69;
+        var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_");
+        TestContext.WriteLine("Using temp path: " + tempDir);
+        var liteDbPath = Path.Join(tempDir.FullName, "original.litedb");
+        TestContext.WriteLine("Using original DB path: " + liteDbPath);
+        var migrationDbPath = Path.Join(tempDir.FullName, "migration.litedb");
+        TestContext.WriteLine("Using migration DB path: " + migrationDbPath);
+        var newDbPath = Path.Join(tempDir.FullName, "new.sqlitedb");
+        TestContext.WriteLine("Using new DB path: " + newDbPath);
+
+        var expectedMessages = new List<Message>(count);
+        using (var liteDatabase = LegacyMessageImporter.Connect(liteDbPath, readOnly: false))
+        {
+            var messagesCollection = liteDatabase.GetCollection(LegacyMessageImporter.MessagesCollection);
+            var now = DateTimeOffset.UtcNow;
+            for (var i = 0; i < count; i++)
+            {
+                if (i == corruptedIndex)
+                {
+                    // This message will not be imported because it can't be
+                    // parsed into a Message object.
+                    messagesCollection.Insert(new BsonDocument
+                    {
+                        ["_id"] = ObjectId.NewObjectId(),
+                        ["Receiver"] = 0L,
+                        ["ContentId"] = 0L,
+                        ["Date"] = 0L,
+                        ["Code"] = BsonMapper.Global.Serialize(new ChatCode(0)),
+                        ["Sender"] = BsonMapper.Global.Serialize(new List<Chunk>()),
+                        ["Content"] = BsonMapper.Global.Serialize(new List<Chunk>()),
+                        // These are meant to be arrays.
+                        ["SenderSource"] = new BsonDocument(),
+                        ["ContentSource"] = new BsonDocument(),
+                        ["SortCode"] = BsonMapper.Global.Serialize(new SortCode(0)),
+                        ["ExtraChatChannel"] = new Guid(),
+                    });
+                    continue;
+                }
+
+                var messageId = ObjectId.NewObjectId();
+                var message = MessageStoreTest.BigMessage(dateTime: now.AddSeconds(-count + i));
+                // Use reflection to set Id because we don't want to add a
+                // setter to it and allow other code to use it.
+                var guid = LegacyMessageImporter.ObjectIdToGuid(messageId);
+                message.GetType().GetField("<Id>k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(message, guid);
+                expectedMessages.Add(message);
+
+                messagesCollection.Insert(new BsonDocument {
+                    ["_id"] = messageId,
+                    ["Receiver"] = message.Receiver,
+                    ["ContentId"] = message.ContentId,
+                    ["Date"] = message.Date.ToUnixTimeMilliseconds(),
+                    ["Code"] = BsonMapper.Global.Serialize(message.Code),
+                    ["Sender"] = BsonMapper.Global.Serialize(message.Sender),
+                    ["Content"] = BsonMapper.Global.Serialize(message.Content),
+                    ["SenderSource"] = BsonMapper.Global.Serialize(message.SenderSource),
+                    ["ContentSource"] = BsonMapper.Global.Serialize(message.ContentSource),
+                    ["SortCode"] = BsonMapper.Global.Serialize(message.SortCode),
+                    ["ExtraChatChannel"] = message.ExtraChatChannel,
+                });
+            }
+
+            Assert.AreEqual(count, messagesCollection.Count());
+        }
+
+        var dbPath = Path.Join(tempDir.FullName, "test.db");
+        using var store = new MessageStore(dbPath);
+
+        var eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath);
+        Assert.AreEqual(LegacyMessageImporterEligibilityStatus.Eligible, eligibility.Status);
+
+        var importer = eligibility.StartImport(store, noLog: true);
+        while (importer.ImportComplete == null)
+            System.Threading.Thread.Sleep(10);
+
+        Assert.IsTrue(importer.ImportComplete > importer.ImportStart);
+        Assert.AreEqual(count - 1, importer.SuccessfulMessages);
+        Assert.AreEqual(1, importer.FailedMessages);
+
+        var messages = store.GetMostRecentMessages(count: count + 1).ToList();
+        Assert.AreEqual(count - 1, messages.Count);
+        for (var i = 0; i < count - 1; i++)
+            MessageStoreTest.AssertMessagesEqual(expectedMessages[i], messages[i]);
+    }
+
+    /// <summary>
+    /// Converts Guids created by LegacyMessageImporter.ObjectIdToGuid() back to
+    /// their original ObjectId. If any other Guid is passed, the result is
+    /// lossy.
+    /// </summary>
+    private ObjectId CustomGuidToObjectId(Guid guid)
+    {
+        var guidBytes = guid.ToByteArray();
+        var newObjectIdBytes = new byte[12];
+        Buffer.BlockCopy(guidBytes, 0, newObjectIdBytes, 0, 7);
+        newObjectIdBytes[7] = guidBytes[8];
+        Buffer.BlockCopy(guidBytes, 10, newObjectIdBytes, 8, 4);
+        return new ObjectId(newObjectIdBytes);
+    }
+}

ChatTwo.Tests/MessageStoreTest.cs

@@ -160,7 +160,7 @@ public class MessageStoreTest {
         }
     }
 
-    private static Message BigMessage(bool uniqId = true, uint receiver = 12345, DateTimeOffset? dateTime = null) {
+    internal static Message BigMessage(bool uniqId = true, uint receiver = 12345, DateTimeOffset? dateTime = null) {
         // NOTE: These values aren't valid in the game.
         // NOTE: we can't test UiForeground, UiGlow, or AutoTranslatePayload
         // because they load data from the game.
@@ -216,16 +216,13 @@ public class MessageStoreTest {
         );
     }
 
-    private void AssertMessagesEqual(Message input, Message output) {
+    internal static void AssertMessagesEqual(Message input, Message output) {
         // Check basic fields.
         Assert.AreEqual(input.Id, output.Id);
         Assert.AreEqual(input.Receiver, output.Receiver);
         Assert.AreEqual(input.ContentId, output.ContentId);
         // Assert time is within 1 second
-        TestContext.WriteLine($"Input date: {input.Date.ToUniversalTime()}");
-        TestContext.WriteLine($"Output date: {output.Date.ToUniversalTime()}");
         var timeDifference = Math.Abs(input.Date.ToUniversalTime().Subtract(output.Date.ToUniversalTime()).TotalSeconds);
-        TestContext.WriteLine($"Time difference: {timeDifference}s");
         Assert.IsTrue(timeDifference < 1);
         Assert.AreEqual(input.Code.Raw, output.Code.Raw);
         Assert.AreEqual($"{input.SenderSource.Encode():X}", $"{output.SenderSource.Encode():X}");

ChatTwo/ChatTwo.csproj

@@ -50,6 +50,7 @@
 
     <ItemGroup>
         <PackageReference Include="DalamudPackager" Version="2.1.12"/>
+        <PackageReference Include="LiteDB" Version="5.0.19" />
         <PackageReference Include="MessagePack" Version="2.5.140" />
         <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.4" />
         <PackageReference Include="Pidgin" Version="3.2.2"/>

ChatTwo/Chunk.cs

@@ -1,5 +1,6 @@
 using ChatTwo.Code;
 using Dalamud.Game.Text.SeStringHandling;
+using LiteDB;
 using MessagePack;
 
 namespace ChatTwo;
@@ -9,6 +10,7 @@ namespace ChatTwo;
 [MessagePackObject]
 public abstract class Chunk {
     [IgnoreMember]
+    [BsonIgnore] // used by LegacyMessageImporter
     internal Message? Message { get; set; }
 
     [Key(0)]

ChatTwo/Code/ChatCode.cs

@@ -1,3 +1,5 @@
+using LiteDB;
+
 namespace ChatTwo.Code;
 
 internal class ChatCode
@@ -19,6 +21,15 @@ internal class ChatCode
         Target = SourceFrom(7);
     }
 
+    [BsonCtor] // Used by LegacyMessageImporter
+    public ChatCode(ushort raw, ChatType type, ChatSource source, ChatSource target)
+    {
+        Raw = raw;
+        Type = type;
+        Source = source;
+        Target = target;
+    }
+
     internal ChatType Parent() => Type switch
     {
         ChatType.Say => ChatType.Say,

ChatTwo/LegacyMessageImporter.cs

@@ -0,0 +1,356 @@
+using System.Diagnostics;
+using ChatTwo.Code;
+using ChatTwo.Util;
+using Dalamud.Game.Text.SeStringHandling;
+using Dalamud.Plugin.Services;
+using LiteDB;
+
+namespace ChatTwo;
+
+internal enum LegacyMessageImporterEligibilityStatus
+{
+    Eligible,
+    IneligibleOriginalDbNotExists,
+    IneligibleMigrationDbExists,
+    IneligibleLiteDbFailed,
+    IneligibleNoMessages,
+}
+
+internal class LegacyMessageImporterEligibility
+{
+    internal LegacyMessageImporterEligibilityStatus Status { get; private set; }
+    internal string AdditionalIneligibilityInfo { get; private set; }
+
+    internal string OriginalDbPath { get; }
+    internal string MigrationDbPath { get; }
+
+    internal long DatabaseSizeBytes { get; }
+    internal int MessageCount { get; }
+
+    private LegacyMessageImporterEligibility(LegacyMessageImporterEligibilityStatus status, string additionalIneligibilityInfo, string originalDbPath, string migrationDbPath, long databaseSizeBytes, int messageCount)
+    {
+        Status = status;
+        AdditionalIneligibilityInfo = additionalIneligibilityInfo;
+        OriginalDbPath = originalDbPath;
+        MigrationDbPath = migrationDbPath;
+        DatabaseSizeBytes = databaseSizeBytes;
+        MessageCount = messageCount;
+    }
+
+    private static LegacyMessageImporterEligibility NewEligible(string originalDbPath, string migrationDbPath,
+        long databaseSizeBytes, int messageCount)
+    {
+        return new LegacyMessageImporterEligibility(LegacyMessageImporterEligibilityStatus.Eligible, "", originalDbPath, migrationDbPath, databaseSizeBytes, messageCount);
+    }
+
+    private static LegacyMessageImporterEligibility NewIneligible(LegacyMessageImporterEligibilityStatus status, string additionalIneligibilityReason)
+    {
+        return new LegacyMessageImporterEligibility(status, additionalIneligibilityReason, "", "", 0, 0);
+    }
+
+    internal static LegacyMessageImporterEligibility CheckEligibility(string? originalDbPath = null, string? migrationDbPath = null)
+    {
+        originalDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat.db");
+        migrationDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-litedb.db");
+
+        // Condition 1: the database file must exist in its original path.
+        if (!File.Exists(originalDbPath))
+        {
+            return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists, $"Original database file '{originalDbPath}' does not exist");
+        }
+
+        // Condition 2: the migration file must not exist.
+        if (File.Exists(migrationDbPath))
+        {
+            return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists, $"Migration database file '{migrationDbPath}' already exists, migration was already started in the past");
+        }
+
+        // Condition 3: we need to be able to connect to the original database
+        // path.
+        try
+        {
+            using var db = LegacyMessageImporter.Connect(originalDbPath);
+            var size = new FileInfo(originalDbPath).Length;
+            var count = db.GetCollection(LegacyMessageImporter.MessagesCollection).Count();
+            if (count <= 0)
+                NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleNoMessages, $"No messages in original database file '{originalDbPath}'");
+            return NewEligible(originalDbPath, migrationDbPath, size, count);
+        }
+        catch (Exception e)
+        {
+            // Notify the user about this error, because they might be wondering
+            // why they weren't offered a migration.
+            return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed, $"LiteDB connection to original database file '{originalDbPath}' failed: {e}");
+        }
+    }
+
+    internal LegacyMessageImporter StartImport(MessageStore targetStore, bool noLog = false)
+    {
+        if (Status != LegacyMessageImporterEligibilityStatus.Eligible)
+            throw new InvalidOperationException($"Migration not eligible: status is {Status}");
+
+        return new LegacyMessageImporter(targetStore, originalDbPath: OriginalDbPath, migrationDbPath: MigrationDbPath, noLog: noLog);
+    }
+
+    /// <summary>
+    /// Makes the migration ineligible so the user won't be asked again.
+    /// </summary>
+    internal void RenameOldDatabase()
+    {
+        File.Move(OriginalDbPath, MigrationDbPath);
+        Status = LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists;
+        AdditionalIneligibilityInfo = "User chose to rename the old database file";
+    }
+}
+
+internal class LegacyMessageImporter : IDisposable
+{
+    internal const string MessagesCollection = "messages";
+    private const int MaxFailedMessageLogs = 10;
+
+    private readonly MessageStore _targetStore;
+    private readonly IPluginLog? _log;
+
+    private LiteDatabase? _database;
+
+    internal long ImportStart { get; } // ticks
+    internal int ImportCount { get; private set; }
+    internal int SuccessfulMessages { get; private set; }
+    internal int FailedMessages { get; private set; }
+    internal int ProcessedMessages => SuccessfulMessages + FailedMessages;
+    internal int RemainingMessages => ImportCount - ProcessedMessages;
+    // Progress from 0 to 1.
+    internal float Progress => ImportCount > 0 ? ProcessedMessages / (float)ImportCount : 1;
+    // Message count processed in the last second.
+    internal float CurrentMessageRate { get; private set; }
+    // ETA based on CurrentMessageRate.
+    internal TimeSpan EstimatedTimeRemaining => TimeSpan.FromSeconds(CurrentMessageRate > 0 ? (ImportCount - SuccessfulMessages - FailedMessages) / CurrentMessageRate : 0);
+    internal long? ImportComplete { get; private set; } // ticks
+
+    // This can be set by the user to limit the rate at which messages are
+    // imported. If the rate exceeds this value, the importer will sleep for the
+    // remainder of the second.
+    internal int MaxMessageRate { get; set; } = 250; // start low
+
+    // Do not call this directly, use
+    // LegacyMessageImporterEligibility.StartImport instead.
+    internal LegacyMessageImporter(MessageStore targetStore, string? originalDbPath = null, string? migrationDbPath = null, bool noLog = false)
+    {
+        _targetStore = targetStore;
+        originalDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat.db");
+        migrationDbPath ??= migrationDbPath ?? Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-litedb.db");
+        _log = noLog ? null : Plugin.Log;
+
+        _log?.Info($"[Migration] Moving '{originalDbPath}' to '{migrationDbPath}'");
+        File.Move(originalDbPath, migrationDbPath);
+        _log?.Info($"[Migration] Opening '{migrationDbPath}'");
+        _database = Connect(migrationDbPath);
+
+        ImportStart = Environment.TickCount64;
+        new Thread(DoImport).Start();
+    }
+
+    public void Dispose()
+    {
+        // TODO: cancel thread and wait for it to close
+        _database?.Dispose();
+    }
+
+    internal static LiteDatabase Connect(string dbPath, bool readOnly = true)
+    {
+        BsonMapper.Global = new BsonMapper
+        {
+            IncludeNonPublic = true,
+            TrimWhitespace = false
+        };
+
+        BsonMapper.Global.RegisterType<Payload?>(
+            payload =>
+            {
+                switch (payload)
+                {
+                    case AchievementPayload achievement:
+                        return new BsonDocument(new Dictionary<string, BsonValue>
+                        {
+                            ["Type"] = new("Achievement"),
+                            ["Id"] = new(achievement.Id)
+                        });
+                    case PartyFinderPayload partyFinder:
+                        return new BsonDocument(new Dictionary<string, BsonValue>
+                        {
+                            ["Type"] = new("PartyFinder"),
+                            ["Id"] = new(partyFinder.Id)
+                        });
+                    case UriPayload uri:
+                        return new BsonDocument(new Dictionary<string, BsonValue>
+                        {
+                            ["Type"] = new("URI"),
+                            ["Uri"] = new(uri.Uri.ToString())
+                        });
+                }
+
+                return payload?.Encode();
+            },
+            bson =>
+            {
+                if (bson.IsNull)
+                    return null;
+
+                if (bson.IsDocument)
+                    return bson["Type"].AsString switch
+                    {
+                        "Achievement" => new AchievementPayload((uint)bson["Id"].AsInt64),
+                        "PartyFinder" => new PartyFinderPayload((uint)bson["Id"].AsInt64),
+                        "URI" => new UriPayload(new Uri(bson["Uri"].AsString)),
+                        _ => null
+                    };
+
+                return Payload.Decode(new BinaryReader(new MemoryStream(bson.AsBinary)));
+            });
+
+        BsonMapper.Global.RegisterType<SeString?>(
+            seString => seString == null
+                ? null
+                : new BsonArray(seString.Payloads.Select(payload => new BsonValue(payload.Encode()))),
+            bson =>
+            {
+                if (bson.IsNull)
+                    return null;
+
+                var array = bson.IsArray ? bson.AsArray : bson["Payloads"].AsArray;
+                var payloads = array
+                    .Select(payload => Payload.Decode(new BinaryReader(new MemoryStream(payload.AsBinary))))
+                    .ToList();
+                return new SeString(payloads);
+            }
+        );
+        BsonMapper.Global.RegisterType(
+            type => (int)type,
+            bson => (ChatType)bson.AsInt32
+        );
+        BsonMapper.Global.RegisterType(
+            source => (int)source,
+            bson => (ChatSource)bson.AsInt32
+        );
+        BsonMapper.Global.RegisterType(
+            dateTime => dateTime.Subtract(DateTime.UnixEpoch).TotalMilliseconds,
+            bson => DateTime.UnixEpoch.AddMilliseconds(bson.AsInt64)
+        );
+
+        var connString = $"Filename='{dbPath}';Connection=direct;ReadOnly={readOnly}";
+        var conn = new LiteDatabase(connString, BsonMapper.Global)
+        {
+            CheckpointSize = 1_000,
+            Timeout = TimeSpan.FromSeconds(1)
+        };
+        var messages = conn.GetCollection<Message>(MessagesCollection);
+        messages.EnsureIndex(msg => msg.Date);
+        return conn;
+    }
+
+    private void DoImport()
+    {
+        var importRateTimer = Stopwatch.StartNew();
+        var messagesInLastSecond = 0;
+
+        // Query raw BsonDocuments, so we can convert them in individual
+        // try-catch blocks.
+        var messagesCollection = _database!.GetCollection<Message>(MessagesCollection);
+        var totalMessages = messagesCollection.Count();
+        ImportCount = totalMessages;
+        var messages = messagesCollection.Query().OrderBy(msg => msg.Date).ToDocuments();
+        foreach (var messageDoc in messages)
+        {
+            try
+            {
+                var message = BsonDocumentToMessage(messageDoc);
+                _targetStore.UpsertMessage(message);
+                SuccessfulMessages++;
+            }
+            catch (Exception e)
+            {
+                FailedMessages++;
+                if (FailedMessages <= MaxFailedMessageLogs)
+                    _log?.Error(
+                        $"[Migration] Failed to import message '{messageDoc["_id"].AsObjectId}' (usually due to corruption): {e}");
+                if (FailedMessages == MaxFailedMessageLogs)
+                    _log?.Error("[Migration] Further failed message logs will be suppressed");
+            }
+
+            messagesInLastSecond++;
+            if (MaxMessageRate > 0 && messagesInLastSecond > MaxMessageRate)
+            {
+                var sleepTime = 1000 - (int)importRateTimer.ElapsedMilliseconds;
+                if (sleepTime > 0)
+                    Thread.Sleep(sleepTime);
+            }
+            if (importRateTimer.ElapsedMilliseconds > 1000)
+            {
+                CurrentMessageRate = messagesInLastSecond / (float)importRateTimer.ElapsedMilliseconds * 1000;
+                importRateTimer.Restart();
+                messagesInLastSecond = 0;
+            }
+
+            // Log every 1,000 messages
+            if ((SuccessfulMessages + FailedMessages) % 1000 == 0)
+                _log?.Information(
+                    $"[Migration] Progress: successfully imported {SuccessfulMessages}/{totalMessages} messages ({FailedMessages} failures)");
+        }
+
+        _log?.Information($"[Migration] Imported {SuccessfulMessages}/{FailedMessages} messages, {FailedMessages} failed");
+
+        if (ProcessedMessages != totalMessages)
+            _log?.Warning(
+                $"[Migration] Total message count mismatch: expected {totalMessages}, got {SuccessfulMessages + FailedMessages}");
+
+        ImportComplete = Environment.TickCount64;
+        _database.Dispose();
+        _database = null;
+    }
+
+    private static Message BsonDocumentToMessage(BsonDocument doc)
+    {
+        return new Message(
+            ObjectIdToGuid(doc["_id"].AsObjectId),
+            (ulong)doc["Receiver"].AsInt64,
+            (ulong)doc["ContentId"].AsInt64,
+            DateTimeOffset.FromUnixTimeMilliseconds(doc["Date"].AsInt64),
+            BsonMapper.Global.Deserialize<ChatCode>(doc["Code"].AsDocument),
+            BsonMapper.Global.Deserialize<List<Chunk>>(doc["Sender"].AsArray),
+            BsonMapper.Global.Deserialize<List<Chunk>>(doc["Content"].AsArray),
+            BsonMapper.Global.Deserialize<SeString>(doc["SenderSource"].AsArray),
+            BsonMapper.Global.Deserialize<SeString>(doc["ContentSource"].AsArray),
+            BsonMapper.Global.Deserialize<SortCode>(doc["SortCode"].AsDocument),
+            doc["ExtraChatChannel"].AsGuid
+        );
+    }
+
+    internal static Guid ObjectIdToGuid(ObjectId objectId)
+    {
+        // "Generate" a new Guid based on the ObjectId from the original
+        // database. We want to have a stable unique identifier for each message
+        // so that if the migration somehow happens twice the objects won't be
+        // duplicated.
+        //
+        // Technically, when Guids are generated they follow a specific pattern.
+        // However, in practice it doesn't matter at all, and we can just
+        // generate whatever we want.
+        var objectIdBytes = objectId.ToByteArray();
+        var guidBytes = new byte[16];
+        // Copy the first 7 bytes directly
+        Buffer.BlockCopy(objectIdBytes, 0, guidBytes, 0, 7);
+        // Fixed byte for version
+        guidBytes[7] = 0b11111111;
+        // Copy the next byte.
+        guidBytes[8] = objectIdBytes[7];
+        // Fixed reserved byte
+        guidBytes[9] = 0b11111111;
+        // Copy the last 4 bytes.
+        Buffer.BlockCopy(objectIdBytes, 8, guidBytes, 10, 4);
+        // Set the last 2 bytes to beef
+        guidBytes[14] = 0xbe;
+        guidBytes[15] = 0xef;
+
+        return new Guid(guidBytes);
+    }
+}

ChatTwo/Message.cs

@@ -3,6 +3,7 @@ using ChatTwo.Util;
 using Dalamud.Game.Text.SeStringHandling;
 using Dalamud.Game.Text.SeStringHandling.Payloads;
 using System.Text.RegularExpressions;
+using LiteDB;
 
 namespace ChatTwo;
 
@@ -10,7 +11,8 @@ internal class SortCode {
     internal ChatType Type { get; }
     internal ChatSource Source { get; }
 
-    internal SortCode(ChatType type, ChatSource source) {
+    [BsonCtor] // Used by LegacyMessageImporter
+    public SortCode(ChatType type, ChatSource source) {
         Type = type;
         Source = source;
     }

ChatTwo/MessageManager.cs

@@ -19,7 +19,6 @@ internal class MessageManager : IDisposable
     internal MessageStore Store { get; }
 
     private ConcurrentQueue<(uint, Message)> Pending { get; } = new();
-    private Stopwatch MaintenanceTimer { get; } = new();
     private Dictionary<ChatType, NameFormatting> Formats { get; } = new();
     private ulong LastContentId { get; set; }
 
@@ -35,7 +34,6 @@ internal class MessageManager : IDisposable
     internal MessageManager(Plugin plugin)
     {
         Plugin = plugin;
-        MaintenanceTimer.Start();
         Store = new MessageStore(DatabasePath());
 
         Plugin.ChatGui.ChatMessageUnhandled += ChatMessage;
@@ -73,12 +71,6 @@ internal class MessageManager : IDisposable
 
     private void GetMessageInfo(IFramework framework)
     {
-        if (MaintenanceTimer.Elapsed > TimeSpan.FromMinutes(5))
-        {
-            MaintenanceTimer.Restart();
-            new Thread(() => Store.PerformMaintenance()).Start();
-        }
-
         if (!Pending.TryDequeue(out var entry))
             return;
 

ChatTwo/Plugin.cs

@@ -49,6 +49,7 @@ public sealed class Plugin : IDalamudPlugin
     public ChatLogWindow ChatLogWindow { get; }
     public CommandHelpWindow CommandHelpWindow { get; }
     public SeStringDebugger SeStringDebugger { get; }
+    internal LegacyMesasgeImporterWindow LegacyMesasgeImporterWindow { get; }
 
     internal Configuration Config { get; }
     internal Commands Commands { get; }
@@ -104,6 +105,10 @@ public sealed class Plugin : IDalamudPlugin
 
             MessageManager = new MessageManager(this); // requires Ui
 
+            // Requires MessageManager
+            LegacyMesasgeImporterWindow = new LegacyMesasgeImporterWindow(MessageManager.Store);
+            WindowSystem.AddWindow(LegacyMesasgeImporterWindow);
+
             // let all the other components register, then initialise commands
             Commands.Initialise();
 
@@ -138,6 +143,7 @@ public sealed class Plugin : IDalamudPlugin
         ChatLogWindow?.Dispose();
         SettingsWindow?.Dispose();
         SeStringDebugger?.Dispose();
+        LegacyMesasgeImporterWindow?.Dispose();
 
         ExtraChat?.Dispose();
         Ipc?.Dispose();

ChatTwo/Ui/LegacyMesasgeImporterWindow.cs

@@ -0,0 +1,209 @@
+using System.Numerics;
+using ChatTwo.Resources;
+using ChatTwo.Util;
+using Dalamud.Interface.ImGuiNotification;
+using Dalamud.Interface.Internal.Notifications;
+using Dalamud.Interface.Windowing;
+using ImGuiNET;
+
+namespace ChatTwo.Ui;
+
+internal class LegacyMesasgeImporterWindow : Window
+{
+    private readonly MessageStore _store;
+
+    private LegacyMessageImporterEligibility Eligibility { get; set; }
+    private LegacyMessageImporter? Importer { get; set; }
+
+    internal LegacyMesasgeImporterWindow(MessageStore store) : base("Chat 2 Legacy Importer###chat2-legacy-importer")
+    {
+        _store = store;
+        Eligibility = LegacyMessageImporterEligibility.CheckEligibility();
+        LogAndNotify();
+    }
+
+    public void Dispose()
+    {
+        Importer?.Dispose();
+    }
+
+    private void LogAndNotify()
+    {
+        Plugin.Log.Info(
+            $"[Migration] Checked migration eligibility: {Eligibility.Status} - '{Eligibility.AdditionalIneligibilityInfo}'");
+
+        switch (Eligibility.Status)
+        {
+            case LegacyMessageImporterEligibilityStatus.Eligible:
+            {
+                var notification = Plugin.Notification.AddNotification(new Notification
+                {
+                    Type = NotificationType.Info,
+                    // The user needs to dismiss this for it to go away.
+                    InitialDuration = TimeSpan.FromHours(6),
+                    Title = "Chat 2 Migration",
+                    Content = "Import messages from old database into new database? Click for more information.",
+                });
+                // TODO: clicking does not dismiss
+                notification.Click += _ => IsOpen = true;
+                notification.Dismiss += _ => WriteChatMessage();
+                break;
+            }
+
+            case LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed:
+            {
+                var notification = Plugin.Notification.AddNotification(new Notification
+                {
+                    Type = NotificationType.Warning,
+                    InitialDuration = TimeSpan.FromMinutes(1),
+                    Title = "Chat Two Migration",
+                    Content =
+                        "Migration is not possible because the old database could not be opened. Click for more information."
+                });
+                notification.Click += _ => IsOpen = true;
+                notification.Dismiss += _ => WriteChatMessage();
+                break;
+            }
+        }
+    }
+
+    private void WriteChatMessage()
+    {
+        // TODO: write a message to chat saying how to open the window again
+        // TODO: add a way of opening the window again, maybe a command or in
+        // database settings
+    }
+
+    public override void Draw()
+    {
+        if (Importer != null)
+        {
+            DrawImportStatus();
+            return;
+        }
+
+        if (Eligibility.Status == LegacyMessageImporterEligibilityStatus.Eligible)
+            DrawEligible();
+        else
+            DrawIneligible();
+    }
+
+    private void DrawEligible()
+    {
+        // TODO: pretty
+        ImGui.Text("Import database messages from legacy LiteDB database to Sqlite database?");
+        ImGui.Text($"Message count: {Eligibility.MessageCount}");
+        ImGui.Text($"Database size: {Eligibility.DatabaseSizeBytes}");
+
+        if (ImGui.Button("Yes, import messages"))
+        {
+            // Next draw call will run DrawImportStatus().
+            Importer = Eligibility.StartImport(_store);
+            return;
+        }
+
+        ImGui.SameLine();
+
+        if (ImGuiUtil.CtrlShiftButton("No, do not import messages",
+                "Ctrl+Shift: renames old database to avoid prompting again"))
+        {
+            Eligibility.RenameOldDatabase();
+            IsOpen = false;
+        }
+    }
+
+    private void DrawIneligible()
+    {
+        // TODO: pretty
+        ImGui.Text("Your legacy LiteDB database is not eligible for import:");
+        switch (Eligibility.Status)
+        {
+            case LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists:
+                ImGui.Text("The old database could not be found.");
+                break;
+            case LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists:
+                ImGui.Text("The migration process was already started.");
+                break;
+            case LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed:
+                ImGui.Text("The old database could not be opened.");
+                break;
+            case LegacyMessageImporterEligibilityStatus.IneligibleNoMessages:
+                ImGui.Text("The old database contains no messages.");
+                break;
+            case LegacyMessageImporterEligibilityStatus.Eligible:
+            default:
+                throw new ArgumentOutOfRangeException();
+        }
+        if (!string.IsNullOrWhiteSpace(Eligibility.AdditionalIneligibilityInfo))
+            ImGui.Text(Eligibility.AdditionalIneligibilityInfo);
+
+        // LiteDB failures notify the user, so give them a chance to rename the
+        // database to avoid prompting again.
+        if (Eligibility.Status == LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed)
+        {
+            if (ImGuiUtil.CtrlShiftButton("Rename old database",
+                    "Ctrl+Shift: rename old database to avoid import prompt in the future"))
+            {
+                Eligibility.RenameOldDatabase();
+                // TODO: notify success as this changes the status
+            }
+        }
+    }
+
+    private void DrawImportStatus()
+    {
+        // TODO: pretty
+        if (Importer == null)
+            return;
+
+        var importStart = Importer.ImportStart;
+        var importEnd = Importer.ImportComplete;
+        var total = Importer.ImportCount;
+        var successful = Importer!.SuccessfulMessages;
+        var failed = Importer.FailedMessages;
+        var remaining = Importer.RemainingMessages;
+
+        if (importEnd != null)
+        {
+            ImGui.Text($"Completed migration in {Duration(importStart, importEnd.Value)}");
+            ImGui.Text($"Successfully imported: {successful} messages");
+            ImGui.Text($"Failed to import: {failed} messages");
+            ImGui.Text($"Unaccounted for: {remaining}");
+            ImGui.Text("See logs for more details: /xllog");
+            return;
+        }
+
+        // TODO: implement Importer.MaxMessageRate slider in UI, values 0 (infinity) => 10000
+
+        ImGui.Text($"Importing messages... {Importer.Progress:P}%");
+        ImGui.Text($"Duration: {Duration(importStart, Environment.TickCount64)}");
+        ImGui.Text($"Successfully imported: {successful} messages");
+        ImGui.Text($"Failed to import: {failed} messages");
+        ImGui.Text($"Progress: {Importer.ProcessedMessages}/{total} messages");
+        ImGui.Text($"Remaining: {remaining} messages");
+        ImGui.Text($"Messages per second: {Importer.CurrentMessageRate}");
+        ImGui.Text($"Estimated time remaining: {Importer.EstimatedTimeRemaining}");
+        ImGui.Text("See logs for more details: /xllog");
+
+        // TODO: this doesn't render properly
+        ImGui.ProgressBar(Importer.Progress, new Vector2(0.0f, 0.0f), $"{Importer.Progress:P}%");
+
+        if (ImGuiUtil.CtrlShiftButton("Cancel import", "Ctrl+Shift: cancel import and close window"))
+        {
+            // TODO: This currently crashes the whole game because we don't ask
+            // the importer thread to stop and wait for it to stop before
+            // disposing it.
+            // See LegacyMessageImporter.Dispose() for more details.
+            /*
+            Importer.Dispose();
+            Importer = null;
+            Eligibility = LegacyMessageImporterEligibility.CheckEligibility();
+            */
+        }
+    }
+
+    private static TimeSpan Duration(long startTicks, long endTicks)
+    {
+        return endTicks < startTicks ? TimeSpan.Zero : TimeSpan.FromTicks(endTicks - startTicks);
+    }
+}

ChatTwo/packages.lock.json

@@ -8,6 +8,12 @@
         "resolved": "2.1.12",
         "contentHash": "Sc0PVxvgg4NQjcI8n10/VfUQBAS4O+Fw2pZrAqBdRMbthYGeogzu5+xmIGCGmsEZ/ukMOBuAqiNiB5qA3MRalg=="
       },
+      "LiteDB": {
+        "type": "Direct",
+        "requested": "[5.0.19, )",
+        "resolved": "5.0.19",
+        "contentHash": "O8XPBptE4SygW2SN6skqg/VDVTrjpJ0p6+i7Cp8x8razbu2ORLSd6ep3JdhDIdL6h57Bcxv2BuVaN70IKpXI0Q=="
+      },
       "MessagePack": {
         "type": "Direct",
         "requested": "[2.5.140, )",