From 906eeb5095182b626578f8ea227846be6160adc4 Mon Sep 17 00:00:00 2001 From: Infi Date: Thu, 25 Jul 2024 15:33:05 +0200 Subject: [PATCH] Remove legacy migration --- ChatTwo.Tests/LegacyMessageImporterTest.cs | 220 ------------ ChatTwo/ChatTwo.csproj | 1 - ChatTwo/ChatTwo.yaml | 13 +- ChatTwo/Chunk.cs | 2 - ChatTwo/Code/ChatCode.cs | 12 - ChatTwo/LegacyMessageImporter.cs | 385 --------------------- ChatTwo/Message.cs | 3 - ChatTwo/Plugin.cs | 6 - ChatTwo/Ui/LegacyMessageImporterWindow.cs | 225 ------------ ChatTwo/Ui/SettingsTabs/Database.cs | 19 +- ChatTwo/packages.lock.json | 6 - 11 files changed, 14 insertions(+), 878 deletions(-) delete mode 100644 ChatTwo.Tests/LegacyMessageImporterTest.cs delete mode 100644 ChatTwo/LegacyMessageImporter.cs delete mode 100644 ChatTwo/Ui/LegacyMessageImporterWindow.cs diff --git a/ChatTwo.Tests/LegacyMessageImporterTest.cs b/ChatTwo.Tests/LegacyMessageImporterTest.cs deleted file mode 100644 index 7dcb9bc..0000000 --- a/ChatTwo.Tests/LegacyMessageImporterTest.cs +++ /dev/null @@ -1,220 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using ChatTwo.Code; -using JetBrains.Annotations; -using LiteDB; -using Microsoft.VisualStudio.TestTools.UnitTesting; - -namespace ChatTwo.Tests; - -[TestClass] -[TestSubject(typeof(LegacyMessageImporter))] -public class LegacyMessageImporterTest -{ - public TestContext TestContext { get; set; } - - [TestMethod] - public void ConvertId() - { - for (var i = 0; i < 1000; i++) - { - var originalObjectId = ObjectId.NewObjectId(); - var intermediateGuid = LegacyMessageImporter.ObjectIdToGuid(originalObjectId); - var newObjectId = CustomGuidToObjectId(intermediateGuid); - - TestContext.WriteLine($"original: {originalObjectId}"); - TestContext.WriteLine($"new: {newObjectId}"); - TestContext.WriteLine($"intermediate: {intermediateGuid}"); - Assert.IsTrue(originalObjectId.Equals(newObjectId)); - } - } - - [TestMethod] - [Timeout(10_000)] - public void Import() - { - const int count = 100; - var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_"); - TestContext.WriteLine("Using temp path: " + tempDir); - var liteDbPath = Path.Join(tempDir.FullName, "original.litedb"); - TestContext.WriteLine("Using original DB path: " + liteDbPath); - var migrationDbPath = Path.Join(tempDir.FullName, "migration.litedb"); - TestContext.WriteLine("Using migration DB path: " + migrationDbPath); - var newDbPath = Path.Join(tempDir.FullName, "new.sqlitedb"); - TestContext.WriteLine("Using new DB path: " + newDbPath); - - var expectedMessages = new List(count); - using (var liteDatabase = LegacyMessageImporter.Connect(liteDbPath, readOnly: false)) - { - var messagesCollection = liteDatabase.GetCollection(LegacyMessageImporter.MessagesCollection); - var now = DateTimeOffset.UtcNow; - for (var i = 0; i < count; i++) - { - var messageId = ObjectId.NewObjectId(); - var message = MessageStoreTest.BigMessage(dateTime: now.AddSeconds(-count + i)); - // Use reflection to set Id because we don't want to add a - // setter to it and allow other code to use it. - var guid = LegacyMessageImporter.ObjectIdToGuid(messageId); - message.GetType().GetField("k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(message, guid); - expectedMessages.Add(message); - - messagesCollection.Insert(new BsonDocument { - ["_id"] = messageId, - ["Receiver"] = message.Receiver, - ["ContentId"] = message.ContentId, - ["Date"] = message.Date.ToUnixTimeMilliseconds(), - ["Code"] = BsonMapper.Global.Serialize(message.Code), - ["Sender"] = BsonMapper.Global.Serialize(message.Sender), - ["Content"] = BsonMapper.Global.Serialize(message.Content), - ["SenderSource"] = BsonMapper.Global.Serialize(message.SenderSource), - ["ContentSource"] = BsonMapper.Global.Serialize(message.ContentSource), - ["SortCode"] = BsonMapper.Global.Serialize(message.SortCode), - ["ExtraChatChannel"] = message.ExtraChatChannel, - }); - } - - Assert.AreEqual(count, messagesCollection.Count()); - } - - var dbPath = Path.Join(tempDir.FullName, "test.db"); - using var store = new MessageStore(dbPath); - - var eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath); - Assert.AreEqual(LegacyMessageImporterEligibilityStatus.Eligible, eligibility.Status); - Assert.AreEqual("", eligibility.AdditionalIneligibilityInfo); - Assert.AreEqual(liteDbPath, eligibility.OriginalDbPath); - Assert.AreEqual(migrationDbPath, eligibility.MigrationDbPath); - Assert.IsTrue(eligibility.DatabaseSizeBytes > 0); - Assert.AreEqual(count, eligibility.MessageCount); - - var importer = eligibility.StartImport(store, noLog: true); - while (importer.ImportComplete == null) - System.Threading.Thread.Sleep(10); - - Assert.IsTrue(importer.ImportComplete > importer.ImportStart); - Assert.AreEqual(count, importer.SuccessfulMessages); - Assert.AreEqual(0, importer.FailedMessages); - - var messages = store.GetMostRecentMessages(count: count + 1).ToList(); - Assert.AreEqual(count, messages.Count); - for (var i = 0; i < count; i++) - MessageStoreTest.AssertMessagesEqual(expectedMessages[i], messages[i]); - - // No longer eligible. - eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath); - Assert.AreEqual(LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists, eligibility.Status); - Assert.IsTrue(eligibility.AdditionalIneligibilityInfo.Contains("Original database file")); - Assert.AreEqual("", eligibility.OriginalDbPath); - Assert.AreEqual("", eligibility.MigrationDbPath); - Assert.AreEqual(0, eligibility.DatabaseSizeBytes); - Assert.AreEqual(0, eligibility.MessageCount); - } - - [TestMethod] - [Timeout(10_000)] - public void CorruptedImport() - { - const int count = 100; - const int corruptedIndex = 69; - var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_"); - TestContext.WriteLine("Using temp path: " + tempDir); - var liteDbPath = Path.Join(tempDir.FullName, "original.litedb"); - TestContext.WriteLine("Using original DB path: " + liteDbPath); - var migrationDbPath = Path.Join(tempDir.FullName, "migration.litedb"); - TestContext.WriteLine("Using migration DB path: " + migrationDbPath); - var newDbPath = Path.Join(tempDir.FullName, "new.sqlitedb"); - TestContext.WriteLine("Using new DB path: " + newDbPath); - - var expectedMessages = new List(count); - using (var liteDatabase = LegacyMessageImporter.Connect(liteDbPath, readOnly: false)) - { - var messagesCollection = liteDatabase.GetCollection(LegacyMessageImporter.MessagesCollection); - var now = DateTimeOffset.UtcNow; - for (var i = 0; i < count; i++) - { - if (i == corruptedIndex) - { - // This message will not be imported because it can't be - // parsed into a Message object. - messagesCollection.Insert(new BsonDocument - { - ["_id"] = ObjectId.NewObjectId(), - ["Receiver"] = 0L, - ["ContentId"] = 0L, - ["Date"] = 0L, - ["Code"] = BsonMapper.Global.Serialize(new ChatCode(0)), - ["Sender"] = BsonMapper.Global.Serialize(new List()), - ["Content"] = BsonMapper.Global.Serialize(new List()), - // These are meant to be arrays. - ["SenderSource"] = new BsonDocument(), - ["ContentSource"] = new BsonDocument(), - ["SortCode"] = BsonMapper.Global.Serialize(new SortCode(0)), - ["ExtraChatChannel"] = new Guid(), - }); - continue; - } - - var messageId = ObjectId.NewObjectId(); - var message = MessageStoreTest.BigMessage(dateTime: now.AddSeconds(-count + i)); - // Use reflection to set Id because we don't want to add a - // setter to it and allow other code to use it. - var guid = LegacyMessageImporter.ObjectIdToGuid(messageId); - message.GetType().GetField("k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(message, guid); - expectedMessages.Add(message); - - messagesCollection.Insert(new BsonDocument { - ["_id"] = messageId, - ["Receiver"] = message.Receiver, - ["ContentId"] = message.ContentId, - ["Date"] = message.Date.ToUnixTimeMilliseconds(), - ["Code"] = BsonMapper.Global.Serialize(message.Code), - ["Sender"] = BsonMapper.Global.Serialize(message.Sender), - ["Content"] = BsonMapper.Global.Serialize(message.Content), - ["SenderSource"] = BsonMapper.Global.Serialize(message.SenderSource), - ["ContentSource"] = BsonMapper.Global.Serialize(message.ContentSource), - ["SortCode"] = BsonMapper.Global.Serialize(message.SortCode), - ["ExtraChatChannel"] = message.ExtraChatChannel, - }); - } - - Assert.AreEqual(count, messagesCollection.Count()); - } - - var dbPath = Path.Join(tempDir.FullName, "test.db"); - using var store = new MessageStore(dbPath); - - var eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath); - Assert.AreEqual(LegacyMessageImporterEligibilityStatus.Eligible, eligibility.Status); - - var importer = eligibility.StartImport(store, noLog: true); - while (importer.ImportComplete == null) - System.Threading.Thread.Sleep(10); - - Assert.IsTrue(importer.ImportComplete > importer.ImportStart); - Assert.AreEqual(count - 1, importer.SuccessfulMessages); - Assert.AreEqual(1, importer.FailedMessages); - - var messages = store.GetMostRecentMessages(count: count + 1).ToList(); - Assert.AreEqual(count - 1, messages.Count); - for (var i = 0; i < count - 1; i++) - MessageStoreTest.AssertMessagesEqual(expectedMessages[i], messages[i]); - } - - /// - /// Converts Guids created by LegacyMessageImporter.ObjectIdToGuid() back to - /// their original ObjectId. If any other Guid is passed, the result is - /// lossy. - /// - private ObjectId CustomGuidToObjectId(Guid guid) - { - var guidBytes = guid.ToByteArray(); - var newObjectIdBytes = new byte[12]; - Buffer.BlockCopy(guidBytes, 0, newObjectIdBytes, 0, 7); - newObjectIdBytes[7] = guidBytes[8]; - Buffer.BlockCopy(guidBytes, 10, newObjectIdBytes, 8, 4); - return new ObjectId(newObjectIdBytes); - } -} diff --git a/ChatTwo/ChatTwo.csproj b/ChatTwo/ChatTwo.csproj index e8d986a..7b46571 100755 --- a/ChatTwo/ChatTwo.csproj +++ b/ChatTwo/ChatTwo.csproj @@ -50,7 +50,6 @@ - diff --git a/ChatTwo/ChatTwo.yaml b/ChatTwo/ChatTwo.yaml index fdab20b..f7fd436 100755 --- a/ChatTwo/ChatTwo.yaml +++ b/ChatTwo/ChatTwo.yaml @@ -22,7 +22,12 @@ tags: - Chat - Replacement changelog: |- - **Updates** - - Loc updates - - Fix multiple issues related to keybinds - - Add more options for auto hide [thanks dean] + **Fonts** + - Uses dalamud font chooser + - Italic font version can be chosen + - Config options related to fonts have been reset + - Old font names and sizes used are displayed for up to a month + + **LiteDB** + - Migration option removed + - Old files can still be removed in the database tab diff --git a/ChatTwo/Chunk.cs b/ChatTwo/Chunk.cs index 7b44562..81ab669 100755 --- a/ChatTwo/Chunk.cs +++ b/ChatTwo/Chunk.cs @@ -1,6 +1,5 @@ using ChatTwo.Code; using Dalamud.Game.Text.SeStringHandling; -using LiteDB; using MessagePack; namespace ChatTwo; @@ -11,7 +10,6 @@ namespace ChatTwo; public abstract class Chunk { [IgnoreMember] - [BsonIgnore] // used by LegacyMessageImporter internal Message? Message { get; set; } [Key(0)] diff --git a/ChatTwo/Code/ChatCode.cs b/ChatTwo/Code/ChatCode.cs index cda3a2e..3606d82 100755 --- a/ChatTwo/Code/ChatCode.cs +++ b/ChatTwo/Code/ChatCode.cs @@ -1,6 +1,3 @@ -using Dalamud.Game.Config; -using LiteDB; - namespace ChatTwo.Code; internal class ChatCode @@ -22,15 +19,6 @@ internal class ChatCode Target = SourceFrom(7); } - [BsonCtor] // Used by LegacyMessageImporter - public ChatCode(ushort raw, ChatType type, ChatSource source, ChatSource target) - { - Raw = raw; - Type = type; - Source = source; - Target = target; - } - internal ChatType Parent() => Type switch { ChatType.Say => ChatType.Say, diff --git a/ChatTwo/LegacyMessageImporter.cs b/ChatTwo/LegacyMessageImporter.cs deleted file mode 100644 index 4358565..0000000 --- a/ChatTwo/LegacyMessageImporter.cs +++ /dev/null @@ -1,385 +0,0 @@ -using System.Diagnostics; -using ChatTwo.Code; -using ChatTwo.Util; -using Dalamud.Game.Text.SeStringHandling; -using Dalamud.Plugin.Services; -using LiteDB; - -namespace ChatTwo; - -internal enum LegacyMessageImporterEligibilityStatus -{ - Eligible, - IneligibleOriginalDbNotExists, - IneligibleMigrationDbExists, - IneligibleLiteDbFailed, - IneligibleNoMessages, -} - -internal class LegacyMessageImporterEligibility -{ - internal LegacyMessageImporterEligibilityStatus Status { get; private set; } - internal string AdditionalIneligibilityInfo { get; private set; } - - internal string OriginalDbPath { get; } - internal string MigrationDbPath { get; } - - internal long DatabaseSizeBytes { get; } - internal int MessageCount { get; } - - private LegacyMessageImporterEligibility(LegacyMessageImporterEligibilityStatus status, string additionalIneligibilityInfo, string originalDbPath, string migrationDbPath, long databaseSizeBytes, int messageCount) - { - Status = status; - AdditionalIneligibilityInfo = additionalIneligibilityInfo; - OriginalDbPath = originalDbPath; - MigrationDbPath = migrationDbPath; - DatabaseSizeBytes = databaseSizeBytes; - MessageCount = messageCount; - } - - private static LegacyMessageImporterEligibility NewEligible(string originalDbPath, string migrationDbPath, - long databaseSizeBytes, int messageCount) - { - return new LegacyMessageImporterEligibility(LegacyMessageImporterEligibilityStatus.Eligible, "", originalDbPath, migrationDbPath, databaseSizeBytes, messageCount); - } - - private static LegacyMessageImporterEligibility NewIneligible(LegacyMessageImporterEligibilityStatus status, string additionalIneligibilityReason) - { - return new LegacyMessageImporterEligibility(status, additionalIneligibilityReason, "", "", 0, 0); - } - - internal static LegacyMessageImporterEligibility CheckEligibility(string? originalDbPath = null, string? migrationDbPath = null) - { - originalDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat.db"); - migrationDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-litedb.db"); - - // Condition 1: the database file must exist in its original path. - if (!File.Exists(originalDbPath)) - { - return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists, $"Original database file '{originalDbPath}' does not exist"); - } - - // Condition 2: the migration file must not exist. - if (File.Exists(migrationDbPath)) - { - return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists, $"Migration database file '{migrationDbPath}' already exists, migration was already started in the past"); - } - - // Condition 3: we need to be able to connect to the original database - // path. - try - { - using var db = LegacyMessageImporter.Connect(originalDbPath); - var size = new FileInfo(originalDbPath).Length; - var count = db.GetCollection(LegacyMessageImporter.MessagesCollection).Count(); - if (count <= 0) - NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleNoMessages, $"No messages in original database file '{originalDbPath}'"); - return NewEligible(originalDbPath, migrationDbPath, size, count); - } - catch (Exception e) - { - // Notify the user about this error, because they might be wondering - // why they weren't offered a migration. - return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed, $"LiteDB connection to original database file '{originalDbPath}' failed: {e}"); - } - } - - internal LegacyMessageImporter StartImport(MessageStore targetStore, bool noLog = false, Plugin? plugin = null) - { - if (Status != LegacyMessageImporterEligibilityStatus.Eligible) - throw new InvalidOperationException($"Migration not eligible: status is {Status}"); - - return new LegacyMessageImporter(targetStore, originalDbPath: OriginalDbPath, migrationDbPath: MigrationDbPath, noLog: noLog, plugin); - } - - /// - /// Makes the migration ineligible so the user won't be asked again. - /// - internal bool RenameOldDatabase() - { - try - { - File.Move(OriginalDbPath, MigrationDbPath); - Status = LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists; - AdditionalIneligibilityInfo = "User chose to rename the old database file"; - return true; - } - catch (Exception ex) - { - Plugin.Log.Error(ex, "Unable to move the old database"); - return false; - } - } -} - -internal class LegacyMessageImporter : IAsyncDisposable -{ - private readonly Plugin? Plugin; - - private readonly CancellationTokenSource CancellationToken = new(); - private Thread? WorkingThread = null; - - internal const string MessagesCollection = "messages"; - private const int MaxFailedMessageLogs = 10; - - private readonly MessageStore _targetStore; - private readonly IPluginLog? _log; - - private LiteDatabase? _database; - - internal long ImportStart { get; } // ticks - internal int ImportCount { get; private set; } - internal int SuccessfulMessages { get; private set; } - internal int FailedMessages { get; private set; } - internal int ProcessedMessages => SuccessfulMessages + FailedMessages; - internal int RemainingMessages => ImportCount - ProcessedMessages; - // Progress from 0 to 1. - internal float Progress => ImportCount > 0 ? ProcessedMessages / (float)ImportCount : 1; - // Message count processed in the last second. - internal float CurrentMessageRate { get; private set; } - // ETA based on CurrentMessageRate. - internal TimeSpan EstimatedTimeRemaining => TimeSpan.FromSeconds(CurrentMessageRate > 0 ? (ImportCount - SuccessfulMessages - FailedMessages) / CurrentMessageRate : 0); - internal long? ImportComplete { get; private set; } // ticks - - // This can be set by the user to limit the rate at which messages are - // imported. If the rate exceeds this value, the importer will sleep for the - // remainder of the second. - internal int MaxMessageRate = 250; // start low - - // Do not call this directly, use - // LegacyMessageImporterEligibility.StartImport instead. - internal LegacyMessageImporter(MessageStore targetStore, string? originalDbPath = null, string? migrationDbPath = null, bool noLog = false, Plugin? plugin = null) - { - _targetStore = targetStore; - originalDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat.db"); - migrationDbPath ??= migrationDbPath ?? Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-litedb.db"); - _log = noLog ? null : Plugin.Log; - Plugin = plugin; - - _log?.Info($"[Migration] Moving '{originalDbPath}' to '{migrationDbPath}'"); - File.Move(originalDbPath, migrationDbPath); - _log?.Info($"[Migration] Opening '{migrationDbPath}'"); - _database = Connect(migrationDbPath); - - ImportStart = Environment.TickCount64; - WorkingThread = new Thread(() => DoImport(CancellationToken.Token)); - WorkingThread.Start(); - } - - public async ValueTask DisposeAsync() - { - await CancellationToken.CancelAsync(); - - var timeout = 10_000; // 10s - while (WorkingThread != null && timeout > 0) - { - if (!WorkingThread.IsAlive) - break; - - timeout -= 100; - await Task.Delay(100); - Plugin.Log.Information("Sleeping because thread still alive"); - } - - _database?.Dispose(); - } - - internal static LiteDatabase Connect(string dbPath, bool readOnly = true) - { - BsonMapper.Global = new BsonMapper - { - IncludeNonPublic = true, - TrimWhitespace = false - }; - - BsonMapper.Global.RegisterType( - payload => - { - switch (payload) - { - case AchievementPayload achievement: - return new BsonDocument(new Dictionary - { - ["Type"] = new("Achievement"), - ["Id"] = new(achievement.Id) - }); - case PartyFinderPayload partyFinder: - return new BsonDocument(new Dictionary - { - ["Type"] = new("PartyFinder"), - ["Id"] = new(partyFinder.Id) - }); - case UriPayload uri: - return new BsonDocument(new Dictionary - { - ["Type"] = new("URI"), - ["Uri"] = new(uri.Uri.ToString()) - }); - } - - return payload?.Encode(); - }, - bson => - { - if (bson.IsNull) - return null; - - if (bson.IsDocument) - return bson["Type"].AsString switch - { - "Achievement" => new AchievementPayload((uint)bson["Id"].AsInt64), - "PartyFinder" => new PartyFinderPayload((uint)bson["Id"].AsInt64), - "URI" => new UriPayload(new Uri(bson["Uri"].AsString)), - _ => null - }; - - return Payload.Decode(new BinaryReader(new MemoryStream(bson.AsBinary))); - }); - - BsonMapper.Global.RegisterType( - seString => seString == null - ? null - : new BsonArray(seString.Payloads.Select(payload => new BsonValue(payload.Encode()))), - bson => - { - if (bson.IsNull) - return null; - - var array = bson.IsArray ? bson.AsArray : bson["Payloads"].AsArray; - var payloads = array - .Select(payload => Payload.Decode(new BinaryReader(new MemoryStream(payload.AsBinary)))) - .ToList(); - return new SeString(payloads); - } - ); - BsonMapper.Global.RegisterType( - type => (int)type, - bson => (ChatType)bson.AsInt32 - ); - BsonMapper.Global.RegisterType( - source => (int)source, - bson => (ChatSource)bson.AsInt32 - ); - BsonMapper.Global.RegisterType( - dateTime => dateTime.Subtract(DateTime.UnixEpoch).TotalMilliseconds, - bson => DateTime.UnixEpoch.AddMilliseconds(bson.AsInt64) - ); - - var connString = $"Filename='{dbPath}';Connection=direct;ReadOnly={readOnly}"; - var conn = new LiteDatabase(connString, BsonMapper.Global) - { - CheckpointSize = 1_000, - Timeout = TimeSpan.FromSeconds(1) - }; - var messages = conn.GetCollection(MessagesCollection); - messages.EnsureIndex(msg => msg.Date); - return conn; - } - - private void DoImport(CancellationToken token) - { - var importRateTimer = Stopwatch.StartNew(); - var messagesInLastSecond = 0; - - // Query raw BsonDocuments, so we can convert them in individual - // try-catch blocks. - var messagesCollection = _database!.GetCollection(MessagesCollection); - var totalMessages = messagesCollection.Count(); - ImportCount = totalMessages; - var messages = messagesCollection.Query().OrderBy(msg => msg.Date).ToDocuments(); - foreach (var messageDoc in messages) - { - if (token.IsCancellationRequested) - return; - - try - { - var message = BsonDocumentToMessage(messageDoc); - _targetStore.UpsertMessage(message); - SuccessfulMessages++; - } - catch (Exception e) - { - FailedMessages++; - if (FailedMessages <= MaxFailedMessageLogs) - _log?.Error($"[Migration] Failed to import message '{messageDoc["_id"].AsObjectId}' (usually due to corruption): {e}"); - if (FailedMessages == MaxFailedMessageLogs) - _log?.Error("[Migration] Further failed message logs will be suppressed"); - } - - messagesInLastSecond++; - if (MaxMessageRate > 0 && messagesInLastSecond > MaxMessageRate) - { - var sleepTime = 1000 - (int)importRateTimer.ElapsedMilliseconds; - if (sleepTime > 0) - Thread.Sleep(sleepTime); - } - if (importRateTimer.ElapsedMilliseconds > 1000) - { - CurrentMessageRate = messagesInLastSecond / (float)importRateTimer.ElapsedMilliseconds * 1000; - importRateTimer.Restart(); - messagesInLastSecond = 0; - } - - // Log every 1,000 messages - if ((SuccessfulMessages + FailedMessages) % 1000 == 0) - _log?.Information($"[Migration] Progress: successfully imported {SuccessfulMessages}/{totalMessages} messages ({FailedMessages} failures)"); - } - - _log?.Information($"[Migration] Imported {SuccessfulMessages}/{FailedMessages} messages, {FailedMessages} failed"); - if (ProcessedMessages != totalMessages) - _log?.Warning($"[Migration] Total message count mismatch: expected {totalMessages}, got {SuccessfulMessages + FailedMessages}"); - - ImportComplete = Environment.TickCount64; - _database.Dispose(); - _database = null; - - Plugin?.MessageManager.FilterAllTabsAsync(); - } - - private static Message BsonDocumentToMessage(BsonDocument doc) - { - return new Message( - ObjectIdToGuid(doc["_id"].AsObjectId), - (ulong)doc["Receiver"].AsInt64, - (ulong)doc["ContentId"].AsInt64, - DateTimeOffset.FromUnixTimeMilliseconds(doc["Date"].AsInt64), - BsonMapper.Global.Deserialize(doc["Code"].AsDocument), - BsonMapper.Global.Deserialize>(doc["Sender"].AsArray), - BsonMapper.Global.Deserialize>(doc["Content"].AsArray), - BsonMapper.Global.Deserialize(doc["SenderSource"].AsArray), - BsonMapper.Global.Deserialize(doc["ContentSource"].AsArray), - BsonMapper.Global.Deserialize(doc["SortCode"].AsDocument), - doc["ExtraChatChannel"].AsGuid - ); - } - - internal static Guid ObjectIdToGuid(ObjectId objectId) - { - // "Generate" a new Guid based on the ObjectId from the original - // database. We want to have a stable unique identifier for each message - // so that if the migration somehow happens twice the objects won't be - // duplicated. - // - // Technically, when Guids are generated they follow a specific pattern. - // However, in practice it doesn't matter at all, and we can just - // generate whatever we want. - var objectIdBytes = objectId.ToByteArray(); - var guidBytes = new byte[16]; - // Copy the first 7 bytes directly - Buffer.BlockCopy(objectIdBytes, 0, guidBytes, 0, 7); - // Fixed byte for version - guidBytes[7] = 0b11111111; - // Copy the next byte. - guidBytes[8] = objectIdBytes[7]; - // Fixed reserved byte - guidBytes[9] = 0b11111111; - // Copy the last 4 bytes. - Buffer.BlockCopy(objectIdBytes, 8, guidBytes, 10, 4); - // Set the last 2 bytes to beef - guidBytes[14] = 0xbe; - guidBytes[15] = 0xef; - - return new Guid(guidBytes); - } -} diff --git a/ChatTwo/Message.cs b/ChatTwo/Message.cs index 499f19d..85f87b9 100755 --- a/ChatTwo/Message.cs +++ b/ChatTwo/Message.cs @@ -5,9 +5,7 @@ using Dalamud.Game.Text.SeStringHandling; using Dalamud.Game.Text.SeStringHandling.Payloads; using System.Text.RegularExpressions; using Dalamud.Game.Text; -using FFXIVClientStructs.FFXIV.Client.Game; using FFXIVClientStructs.FFXIV.Client.UI.Agent; -using LiteDB; using Lumina.Excel.GeneratedSheets; namespace ChatTwo; @@ -17,7 +15,6 @@ internal class SortCode internal ChatType Type { get; } internal ChatSource Source { get; } - [BsonCtor] // Used by LegacyMessageImporter public SortCode(ChatType type, ChatSource source) { Type = type; diff --git a/ChatTwo/Plugin.cs b/ChatTwo/Plugin.cs index f29eebb..11633ed 100755 --- a/ChatTwo/Plugin.cs +++ b/ChatTwo/Plugin.cs @@ -50,7 +50,6 @@ public sealed class Plugin : IDalamudPlugin public CommandHelpWindow CommandHelpWindow { get; } public SeStringDebugger SeStringDebugger { get; } public DebuggerWindow DebuggerWindow { get; } - internal LegacyMessageImporterWindow LegacyMessageImporterWindow { get; } internal Commands Commands { get; } internal ChatCommon Common { get; } @@ -109,10 +108,6 @@ public sealed class Plugin : IDalamudPlugin MessageManager = new MessageManager(this); // requires Ui - // Requires MessageManager - LegacyMessageImporterWindow = new LegacyMessageImporterWindow(this); - WindowSystem.AddWindow(LegacyMessageImporterWindow); - // let all the other components register, then initialise commands Commands.Initialise(); @@ -158,7 +153,6 @@ public sealed class Plugin : IDalamudPlugin SettingsWindow?.Dispose(); DebuggerWindow?.Dispose(); SeStringDebugger?.Dispose(); - LegacyMessageImporterWindow?.Dispose(); ExtraChat?.Dispose(); Ipc?.Dispose(); diff --git a/ChatTwo/Ui/LegacyMessageImporterWindow.cs b/ChatTwo/Ui/LegacyMessageImporterWindow.cs deleted file mode 100644 index 76f5213..0000000 --- a/ChatTwo/Ui/LegacyMessageImporterWindow.cs +++ /dev/null @@ -1,225 +0,0 @@ -using System.Numerics; -using ChatTwo.Util; -using Dalamud.Interface.ImGuiNotification; -using Dalamud.Interface.ImGuiNotification.EventArgs; -using Dalamud.Interface.Utility; -using Dalamud.Interface.Utility.Raii; -using Dalamud.Interface.Windowing; -using ImGuiNET; - -namespace ChatTwo.Ui; - -internal class LegacyMessageImporterWindow : Window -{ - private readonly Plugin Plugin; - private readonly MessageStore _store; - - private LegacyMessageImporterEligibility Eligibility { get; set; } - private LegacyMessageImporter? Importer { get; set; } - - internal LegacyMessageImporterWindow(Plugin plugin) : base("Chat 2 Legacy Importer###chat2-legacy-importer") - { - Plugin = plugin; - - Flags = ImGuiWindowFlags.NoResize; - Size = new Vector2(500, 400); - - RespectCloseHotkey = false; - DisableWindowSounds = true; - - _store = plugin.MessageManager.Store; - Eligibility = LegacyMessageImporterEligibility.CheckEligibility(); - LogAndNotify(); - } - - public void Dispose() - { - Importer?.DisposeAsync().AsTask().Wait(); - } - - private void NotificationClicked(INotificationClickArgs args) - { - IsOpen = true; - args.Notification.DismissNow(); - } - - private void LogAndNotify() - { - Plugin.Log.Info($"[Migration] Checked migration eligibility: {Eligibility.Status} - '{Eligibility.AdditionalIneligibilityInfo}'"); - - switch (Eligibility.Status) - { - case LegacyMessageImporterEligibilityStatus.Eligible: - { - var notification = Plugin.Notification.AddNotification(new Notification - { - // The user needs to dismiss this for it to go away. - Type = NotificationType.Info, - InitialDuration = TimeSpan.FromHours(24), - Title = "Chat2 Migration", - Content = "Import messages from old database into new database?\nClick for more information.", - Minimized = false, - }); - - notification.Click += NotificationClicked; - break; - } - - case LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed: - { - var notification = Plugin.Notification.AddNotification(new Notification - { - Type = NotificationType.Warning, - InitialDuration = TimeSpan.FromMinutes(1), - Title = "Chat2 Migration", - Content = "Migration is not possible because the old database could not be opened.\nClick for more information.", - Minimized = false, - }); - - notification.Click += NotificationClicked; - break; - } - } - } - - public override void Draw() - { - if (Importer != null) - { - DrawImportStatus(); - return; - } - - if (Eligibility.Status == LegacyMessageImporterEligibilityStatus.Eligible) - DrawEligible(); - else - DrawIneligible(); - } - - private void DrawEligible() - { - ImGui.TextWrapped("Import database messages from legacy LiteDB database to SQLite database?"); - ImGui.TextWrapped($"Message count: {Eligibility.MessageCount:N0}"); - ImGui.TextWrapped($"Database size: {StringUtil.BytesToString(Eligibility.DatabaseSizeBytes)}"); - - ImGui.Spacing(); - - var colorNormal = new Vector4(0.0f, 0.70f, 0.0f, 1.0f); - var colorHovered = new Vector4(0.059f, 0.49f, 0.0f, 1.0f); - using (ImRaii.PushColor(ImGuiCol.Button, colorNormal)) - using (ImRaii.PushColor(ImGuiCol.ButtonHovered, colorHovered)) - { - if (ImGui.Button("Yes, import messages")) - { - // Next draw call will run DrawImportStatus(). - Importer = Eligibility.StartImport(_store, plugin: Plugin); - return; - } - } - - ImGui.SameLine(); - - if (ImGuiUtil.CtrlShiftButtonColored("No, do not import messages", "Ctrl+Shift: renames old database to avoid prompting again")) - { - Eligibility.RenameOldDatabase(); - IsOpen = false; - } - } - - private void DrawIneligible() - { - ImGui.TextWrapped("Your legacy LiteDB database is not eligible for import:"); - switch (Eligibility.Status) - { - case LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists: - ImGui.TextWrapped("The old database could not be found."); - break; - case LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists: - ImGui.TextWrapped("The migration process was already started."); - break; - case LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed: - ImGui.TextWrapped("The old database could not be opened."); - break; - case LegacyMessageImporterEligibilityStatus.IneligibleNoMessages: - ImGui.TextWrapped("The old database contains no messages."); - break; - case LegacyMessageImporterEligibilityStatus.Eligible: - default: - throw new ArgumentOutOfRangeException(); - } - - if (!string.IsNullOrWhiteSpace(Eligibility.AdditionalIneligibilityInfo)) - ImGui.TextWrapped(Eligibility.AdditionalIneligibilityInfo); - - // LiteDB failures notify the user, so give them a chance to rename the - // database to avoid prompting again. - if (Eligibility.Status == LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed) - { - if (ImGuiUtil.CtrlShiftButton("Rename old database", "Ctrl+Shift: rename old database to avoid import prompt in the future")) - { - if (Eligibility.RenameOldDatabase()) - WrapperUtil.AddNotification("Successfully renamed the old database.", NotificationType.Success); - else - WrapperUtil.AddNotification("Rename failed, please check /xllog for more information.", NotificationType.Error); - } - } - } - - private void DrawImportStatus() - { - if (Importer == null) - return; - - if (Importer.ImportComplete != null) - { - ImGui.TextUnformatted($"Completed migration in {Duration(Importer.ImportStart, Importer.ImportComplete.Value):g}"); - ImGui.TextUnformatted($"Successfully imported: {Importer.SuccessfulMessages:N0} messages"); - ImGui.TextUnformatted($"Failed to import: {Importer.FailedMessages:N0} messages"); - ImGui.TextUnformatted($"Unaccounted for: {Importer.RemainingMessages:N0}"); - ImGui.TextUnformatted("See logs for more details: /xllog"); - - ImGui.Spacing(); - - if (ImGui.Button("Finish")) - IsOpen = false; - - return; - } - - ImGui.TextUnformatted($"Importing messages ... {Importer.Progress:P}"); - ImGuiHelpers.ScaledDummy(10.0f); - - ImGui.TextUnformatted($"Duration: {Duration(Importer.ImportStart, Environment.TickCount64):g}"); - ImGui.TextUnformatted($"Progress: {Importer.ProcessedMessages:N0}/{Importer.ImportCount:N0} messages ({Importer.FailedMessages:N0} failed)"); - ImGuiHelpers.ScaledDummy(10.0f); - - var width = ImGui.GetContentRegionAvail().X / 2; - ImGui.AlignTextToFramePadding(); - ImGui.TextUnformatted("Import speed:"); - ImGui.SameLine(); - ImGui.SetNextItemWidth(width); - ImGui.SliderInt("##speedSlider", ref Importer.MaxMessageRate, 1, 10000, "%d msgs/sec", ImGuiSliderFlags.AlwaysClamp); - ImGui.TextUnformatted($"Current speed: {Importer.CurrentMessageRate:N0} msgs/sec"); - ImGui.TextUnformatted($"Estimated time remaining: {Importer.EstimatedTimeRemaining:g}"); - ImGui.TextUnformatted("See logs for more details: /xllog"); - ImGuiHelpers.ScaledDummy(10.0f); - - ImGui.ProgressBar(Importer.Progress, new Vector2(-1, 0), $"{Importer.Progress:P}"); - ImGui.Spacing(); - - if (ImGuiUtil.CtrlShiftButton("Cancel import", "Ctrl+Shift: cancel import and close window")) - { - Task.Run(async () => - { - await Importer.DisposeAsync(); - Importer = null; - Eligibility = LegacyMessageImporterEligibility.CheckEligibility(); - }); - } - } - - private static TimeSpan Duration(long startTicks, long endTicks) - { - return endTicks < startTicks ? TimeSpan.Zero : TimeSpan.FromMilliseconds(endTicks - startTicks); - } -} diff --git a/ChatTwo/Ui/SettingsTabs/Database.cs b/ChatTwo/Ui/SettingsTabs/Database.cs index 6ce46b4..775f6f7 100755 --- a/ChatTwo/Ui/SettingsTabs/Database.cs +++ b/ChatTwo/Ui/SettingsTabs/Database.cs @@ -54,19 +54,7 @@ internal sealed class Database : ISettingsTab var old = new FileInfo(Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat.db")); var migratedOld = new FileInfo(Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-litedb.db")); - if (old.Exists) - { - ImGui.TextUnformatted(Language.Options_Database_Old_Heading); - ImGui.Spacing(); - - if (ImGui.Button(Language.Options_Database_Old_Migration)) - Plugin.LegacyMessageImporterWindow.IsOpen = true; - - ImGui.Spacing(); - ImGui.Separator(); - ImGui.Spacing(); - } - else if (migratedOld.Exists) + if (old.Exists || migratedOld.Exists) { ImGui.TextUnformatted(Language.Options_Database_Old_Heading); ImGui.Spacing(); @@ -75,7 +63,10 @@ internal sealed class Database : ISettingsTab { try { - migratedOld.Delete(); + if (old.Exists) + old.Delete(); + else + migratedOld.Delete(); WrapperUtil.AddNotification(Language.Options_Database_Old_Delete_Success, NotificationType.Success); } catch (Exception e) diff --git a/ChatTwo/packages.lock.json b/ChatTwo/packages.lock.json index c062fa4..ce33da2 100644 --- a/ChatTwo/packages.lock.json +++ b/ChatTwo/packages.lock.json @@ -8,12 +8,6 @@ "resolved": "2.1.13", "contentHash": "rMN1omGe8536f4xLMvx9NwfvpAc9YFFfeXJ1t4P4PE6Gu8WCIoFliR1sh07hM+bfODmesk/dvMbji7vNI+B/pQ==" }, - "LiteDB": { - "type": "Direct", - "requested": "[5.0.19, )", - "resolved": "5.0.19", - "contentHash": "O8XPBptE4SygW2SN6skqg/VDVTrjpJ0p6+i7Cp8x8razbu2ORLSd6ep3JdhDIdL6h57Bcxv2BuVaN70IKpXI0Q==" - }, "MessagePack": { "type": "Direct", "requested": "[2.5.140, )",