Infi
GitHub
@@ -0,0 +1,220 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using ChatTwo.Code;
|
||||
using JetBrains.Annotations;
|
||||
using LiteDB;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
|
||||
namespace ChatTwo.Tests;
|
||||
|
||||
[TestClass]
|
||||
[TestSubject(typeof(LegacyMessageImporter))]
|
||||
public class LegacyMessageImporterTest
|
||||
{
|
||||
public TestContext TestContext { get; set; }
|
||||
|
||||
[TestMethod]
|
||||
public void ConvertId()
|
||||
{
|
||||
for (var i = 0; i < 1000; i++)
|
||||
{
|
||||
var originalObjectId = ObjectId.NewObjectId();
|
||||
var intermediateGuid = LegacyMessageImporter.ObjectIdToGuid(originalObjectId);
|
||||
var newObjectId = CustomGuidToObjectId(intermediateGuid);
|
||||
|
||||
TestContext.WriteLine($"original: {originalObjectId}");
|
||||
TestContext.WriteLine($"new: {newObjectId}");
|
||||
TestContext.WriteLine($"intermediate: {intermediateGuid}");
|
||||
Assert.IsTrue(originalObjectId.Equals(newObjectId));
|
||||
}
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[Timeout(10_000)]
|
||||
public void Import()
|
||||
{
|
||||
const int count = 100;
|
||||
var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_");
|
||||
TestContext.WriteLine("Using temp path: " + tempDir);
|
||||
var liteDbPath = Path.Join(tempDir.FullName, "original.litedb");
|
||||
TestContext.WriteLine("Using original DB path: " + liteDbPath);
|
||||
var migrationDbPath = Path.Join(tempDir.FullName, "migration.litedb");
|
||||
TestContext.WriteLine("Using migration DB path: " + migrationDbPath);
|
||||
var newDbPath = Path.Join(tempDir.FullName, "new.sqlitedb");
|
||||
TestContext.WriteLine("Using new DB path: " + newDbPath);
|
||||
|
||||
var expectedMessages = new List<Message>(count);
|
||||
using (var liteDatabase = LegacyMessageImporter.Connect(liteDbPath, readOnly: false))
|
||||
{
|
||||
var messagesCollection = liteDatabase.GetCollection(LegacyMessageImporter.MessagesCollection);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var messageId = ObjectId.NewObjectId();
|
||||
var message = MessageStoreTest.BigMessage(dateTime: now.AddSeconds(-count + i));
|
||||
// Use reflection to set Id because we don't want to add a
|
||||
// setter to it and allow other code to use it.
|
||||
var guid = LegacyMessageImporter.ObjectIdToGuid(messageId);
|
||||
message.GetType().GetField("<Id>k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(message, guid);
|
||||
expectedMessages.Add(message);
|
||||
|
||||
messagesCollection.Insert(new BsonDocument {
|
||||
["_id"] = messageId,
|
||||
["Receiver"] = message.Receiver,
|
||||
["ContentId"] = message.ContentId,
|
||||
["Date"] = message.Date.ToUnixTimeMilliseconds(),
|
||||
["Code"] = BsonMapper.Global.Serialize(message.Code),
|
||||
["Sender"] = BsonMapper.Global.Serialize(message.Sender),
|
||||
["Content"] = BsonMapper.Global.Serialize(message.Content),
|
||||
["SenderSource"] = BsonMapper.Global.Serialize(message.SenderSource),
|
||||
["ContentSource"] = BsonMapper.Global.Serialize(message.ContentSource),
|
||||
["SortCode"] = BsonMapper.Global.Serialize(message.SortCode),
|
||||
["ExtraChatChannel"] = message.ExtraChatChannel,
|
||||
});
|
||||
}
|
||||
|
||||
Assert.AreEqual(count, messagesCollection.Count());
|
||||
}
|
||||
|
||||
var dbPath = Path.Join(tempDir.FullName, "test.db");
|
||||
using var store = new MessageStore(dbPath);
|
||||
|
||||
var eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath);
|
||||
Assert.AreEqual(LegacyMessageImporterEligibilityStatus.Eligible, eligibility.Status);
|
||||
Assert.AreEqual("", eligibility.AdditionalIneligibilityInfo);
|
||||
Assert.AreEqual(liteDbPath, eligibility.OriginalDbPath);
|
||||
Assert.AreEqual(migrationDbPath, eligibility.MigrationDbPath);
|
||||
Assert.IsTrue(eligibility.DatabaseSizeBytes > 0);
|
||||
Assert.AreEqual(count, eligibility.MessageCount);
|
||||
|
||||
var importer = eligibility.StartImport(store, noLog: true);
|
||||
while (importer.ImportComplete == null)
|
||||
System.Threading.Thread.Sleep(10);
|
||||
|
||||
Assert.IsTrue(importer.ImportComplete > importer.ImportStart);
|
||||
Assert.AreEqual(count, importer.SuccessfulMessages);
|
||||
Assert.AreEqual(0, importer.FailedMessages);
|
||||
|
||||
var messages = store.GetMostRecentMessages(count: count + 1).ToList();
|
||||
Assert.AreEqual(count, messages.Count);
|
||||
for (var i = 0; i < count; i++)
|
||||
MessageStoreTest.AssertMessagesEqual(expectedMessages[i], messages[i]);
|
||||
|
||||
// No longer eligible.
|
||||
eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath);
|
||||
Assert.AreEqual(LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists, eligibility.Status);
|
||||
Assert.IsTrue(eligibility.AdditionalIneligibilityInfo.Contains("Original database file"));
|
||||
Assert.AreEqual("", eligibility.OriginalDbPath);
|
||||
Assert.AreEqual("", eligibility.MigrationDbPath);
|
||||
Assert.AreEqual(0, eligibility.DatabaseSizeBytes);
|
||||
Assert.AreEqual(0, eligibility.MessageCount);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[Timeout(10_000)]
|
||||
public void CorruptedImport()
|
||||
{
|
||||
const int count = 100;
|
||||
const int corruptedIndex = 69;
|
||||
var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_");
|
||||
TestContext.WriteLine("Using temp path: " + tempDir);
|
||||
var liteDbPath = Path.Join(tempDir.FullName, "original.litedb");
|
||||
TestContext.WriteLine("Using original DB path: " + liteDbPath);
|
||||
var migrationDbPath = Path.Join(tempDir.FullName, "migration.litedb");
|
||||
TestContext.WriteLine("Using migration DB path: " + migrationDbPath);
|
||||
var newDbPath = Path.Join(tempDir.FullName, "new.sqlitedb");
|
||||
TestContext.WriteLine("Using new DB path: " + newDbPath);
|
||||
|
||||
var expectedMessages = new List<Message>(count);
|
||||
using (var liteDatabase = LegacyMessageImporter.Connect(liteDbPath, readOnly: false))
|
||||
{
|
||||
var messagesCollection = liteDatabase.GetCollection(LegacyMessageImporter.MessagesCollection);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
if (i == corruptedIndex)
|
||||
{
|
||||
// This message will not be imported because it can't be
|
||||
// parsed into a Message object.
|
||||
messagesCollection.Insert(new BsonDocument
|
||||
{
|
||||
["_id"] = ObjectId.NewObjectId(),
|
||||
["Receiver"] = 0L,
|
||||
["ContentId"] = 0L,
|
||||
["Date"] = 0L,
|
||||
["Code"] = BsonMapper.Global.Serialize(new ChatCode(0)),
|
||||
["Sender"] = BsonMapper.Global.Serialize(new List<Chunk>()),
|
||||
["Content"] = BsonMapper.Global.Serialize(new List<Chunk>()),
|
||||
// These are meant to be arrays.
|
||||
["SenderSource"] = new BsonDocument(),
|
||||
["ContentSource"] = new BsonDocument(),
|
||||
["SortCode"] = BsonMapper.Global.Serialize(new SortCode(0)),
|
||||
["ExtraChatChannel"] = new Guid(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
var messageId = ObjectId.NewObjectId();
|
||||
var message = MessageStoreTest.BigMessage(dateTime: now.AddSeconds(-count + i));
|
||||
// Use reflection to set Id because we don't want to add a
|
||||
// setter to it and allow other code to use it.
|
||||
var guid = LegacyMessageImporter.ObjectIdToGuid(messageId);
|
||||
message.GetType().GetField("<Id>k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(message, guid);
|
||||
expectedMessages.Add(message);
|
||||
|
||||
messagesCollection.Insert(new BsonDocument {
|
||||
["_id"] = messageId,
|
||||
["Receiver"] = message.Receiver,
|
||||
["ContentId"] = message.ContentId,
|
||||
["Date"] = message.Date.ToUnixTimeMilliseconds(),
|
||||
["Code"] = BsonMapper.Global.Serialize(message.Code),
|
||||
["Sender"] = BsonMapper.Global.Serialize(message.Sender),
|
||||
["Content"] = BsonMapper.Global.Serialize(message.Content),
|
||||
["SenderSource"] = BsonMapper.Global.Serialize(message.SenderSource),
|
||||
["ContentSource"] = BsonMapper.Global.Serialize(message.ContentSource),
|
||||
["SortCode"] = BsonMapper.Global.Serialize(message.SortCode),
|
||||
["ExtraChatChannel"] = message.ExtraChatChannel,
|
||||
});
|
||||
}
|
||||
|
||||
Assert.AreEqual(count, messagesCollection.Count());
|
||||
}
|
||||
|
||||
var dbPath = Path.Join(tempDir.FullName, "test.db");
|
||||
using var store = new MessageStore(dbPath);
|
||||
|
||||
var eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath);
|
||||
Assert.AreEqual(LegacyMessageImporterEligibilityStatus.Eligible, eligibility.Status);
|
||||
|
||||
var importer = eligibility.StartImport(store, noLog: true);
|
||||
while (importer.ImportComplete == null)
|
||||
System.Threading.Thread.Sleep(10);
|
||||
|
||||
Assert.IsTrue(importer.ImportComplete > importer.ImportStart);
|
||||
Assert.AreEqual(count - 1, importer.SuccessfulMessages);
|
||||
Assert.AreEqual(1, importer.FailedMessages);
|
||||
|
||||
var messages = store.GetMostRecentMessages(count: count + 1).ToList();
|
||||
Assert.AreEqual(count - 1, messages.Count);
|
||||
for (var i = 0; i < count - 1; i++)
|
||||
MessageStoreTest.AssertMessagesEqual(expectedMessages[i], messages[i]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts Guids created by LegacyMessageImporter.ObjectIdToGuid() back to
|
||||
/// their original ObjectId. If any other Guid is passed, the result is
|
||||
/// lossy.
|
||||
/// </summary>
|
||||
private ObjectId CustomGuidToObjectId(Guid guid)
|
||||
{
|
||||
var guidBytes = guid.ToByteArray();
|
||||
var newObjectIdBytes = new byte[12];
|
||||
Buffer.BlockCopy(guidBytes, 0, newObjectIdBytes, 0, 7);
|
||||
newObjectIdBytes[7] = guidBytes[8];
|
||||
Buffer.BlockCopy(guidBytes, 10, newObjectIdBytes, 8, 4);
|
||||
return new ObjectId(newObjectIdBytes);
|
||||
}
|
||||
}
|
||||
@@ -160,7 +160,7 @@ public class MessageStoreTest {
|
||||
}
|
||||
}
|
||||
|
||||
private static Message BigMessage(bool uniqId = true, uint receiver = 12345, DateTimeOffset? dateTime = null) {
|
||||
internal static Message BigMessage(bool uniqId = true, uint receiver = 12345, DateTimeOffset? dateTime = null) {
|
||||
// NOTE: These values aren't valid in the game.
|
||||
// NOTE: we can't test UiForeground, UiGlow, or AutoTranslatePayload
|
||||
// because they load data from the game.
|
||||
@@ -216,16 +216,13 @@ public class MessageStoreTest {
|
||||
);
|
||||
}
|
||||
|
||||
private void AssertMessagesEqual(Message input, Message output) {
|
||||
internal static void AssertMessagesEqual(Message input, Message output) {
|
||||
// Check basic fields.
|
||||
Assert.AreEqual(input.Id, output.Id);
|
||||
Assert.AreEqual(input.Receiver, output.Receiver);
|
||||
Assert.AreEqual(input.ContentId, output.ContentId);
|
||||
// Assert time is within 1 second
|
||||
TestContext.WriteLine($"Input date: {input.Date.ToUniversalTime()}");
|
||||
TestContext.WriteLine($"Output date: {output.Date.ToUniversalTime()}");
|
||||
var timeDifference = Math.Abs(input.Date.ToUniversalTime().Subtract(output.Date.ToUniversalTime()).TotalSeconds);
|
||||
TestContext.WriteLine($"Time difference: {timeDifference}s");
|
||||
Assert.IsTrue(timeDifference < 1);
|
||||
Assert.AreEqual(input.Code.Raw, output.Code.Raw);
|
||||
Assert.AreEqual($"{input.SenderSource.Encode():X}", $"{output.SenderSource.Encode():X}");
|
||||
|
||||
@@ -50,6 +50,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="DalamudPackager" Version="2.1.12"/>
|
||||
<PackageReference Include="LiteDB" Version="5.0.19" />
|
||||
<PackageReference Include="MessagePack" Version="2.5.140" />
|
||||
<PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.4" />
|
||||
<PackageReference Include="Pidgin" Version="3.2.2"/>
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using ChatTwo.Code;
|
||||
using Dalamud.Game.Text.SeStringHandling;
|
||||
using LiteDB;
|
||||
using MessagePack;
|
||||
|
||||
namespace ChatTwo;
|
||||
@@ -9,6 +10,7 @@ namespace ChatTwo;
|
||||
[MessagePackObject]
|
||||
public abstract class Chunk {
|
||||
[IgnoreMember]
|
||||
[BsonIgnore] // used by LegacyMessageImporter
|
||||
internal Message? Message { get; set; }
|
||||
|
||||
[Key(0)]
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using LiteDB;
|
||||
|
||||
namespace ChatTwo.Code;
|
||||
|
||||
internal class ChatCode
|
||||
@@ -19,6 +21,15 @@ internal class ChatCode
|
||||
Target = SourceFrom(7);
|
||||
}
|
||||
|
||||
[BsonCtor] // Used by LegacyMessageImporter
|
||||
public ChatCode(ushort raw, ChatType type, ChatSource source, ChatSource target)
|
||||
{
|
||||
Raw = raw;
|
||||
Type = type;
|
||||
Source = source;
|
||||
Target = target;
|
||||
}
|
||||
|
||||
internal ChatType Parent() => Type switch
|
||||
{
|
||||
ChatType.Say => ChatType.Say,
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
using System.Diagnostics;
|
||||
using ChatTwo.Code;
|
||||
using ChatTwo.Util;
|
||||
using Dalamud.Game.Text.SeStringHandling;
|
||||
using Dalamud.Plugin.Services;
|
||||
using LiteDB;
|
||||
|
||||
namespace ChatTwo;
|
||||
|
||||
internal enum LegacyMessageImporterEligibilityStatus
|
||||
{
|
||||
Eligible,
|
||||
IneligibleOriginalDbNotExists,
|
||||
IneligibleMigrationDbExists,
|
||||
IneligibleLiteDbFailed,
|
||||
IneligibleNoMessages,
|
||||
}
|
||||
|
||||
internal class LegacyMessageImporterEligibility
|
||||
{
|
||||
internal LegacyMessageImporterEligibilityStatus Status { get; private set; }
|
||||
internal string AdditionalIneligibilityInfo { get; private set; }
|
||||
|
||||
internal string OriginalDbPath { get; }
|
||||
internal string MigrationDbPath { get; }
|
||||
|
||||
internal long DatabaseSizeBytes { get; }
|
||||
internal int MessageCount { get; }
|
||||
|
||||
private LegacyMessageImporterEligibility(LegacyMessageImporterEligibilityStatus status, string additionalIneligibilityInfo, string originalDbPath, string migrationDbPath, long databaseSizeBytes, int messageCount)
|
||||
{
|
||||
Status = status;
|
||||
AdditionalIneligibilityInfo = additionalIneligibilityInfo;
|
||||
OriginalDbPath = originalDbPath;
|
||||
MigrationDbPath = migrationDbPath;
|
||||
DatabaseSizeBytes = databaseSizeBytes;
|
||||
MessageCount = messageCount;
|
||||
}
|
||||
|
||||
private static LegacyMessageImporterEligibility NewEligible(string originalDbPath, string migrationDbPath,
|
||||
long databaseSizeBytes, int messageCount)
|
||||
{
|
||||
return new LegacyMessageImporterEligibility(LegacyMessageImporterEligibilityStatus.Eligible, "", originalDbPath, migrationDbPath, databaseSizeBytes, messageCount);
|
||||
}
|
||||
|
||||
private static LegacyMessageImporterEligibility NewIneligible(LegacyMessageImporterEligibilityStatus status, string additionalIneligibilityReason)
|
||||
{
|
||||
return new LegacyMessageImporterEligibility(status, additionalIneligibilityReason, "", "", 0, 0);
|
||||
}
|
||||
|
||||
internal static LegacyMessageImporterEligibility CheckEligibility(string? originalDbPath = null, string? migrationDbPath = null)
|
||||
{
|
||||
originalDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat.db");
|
||||
migrationDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-litedb.db");
|
||||
|
||||
// Condition 1: the database file must exist in its original path.
|
||||
if (!File.Exists(originalDbPath))
|
||||
{
|
||||
return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists, $"Original database file '{originalDbPath}' does not exist");
|
||||
}
|
||||
|
||||
// Condition 2: the migration file must not exist.
|
||||
if (File.Exists(migrationDbPath))
|
||||
{
|
||||
return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists, $"Migration database file '{migrationDbPath}' already exists, migration was already started in the past");
|
||||
}
|
||||
|
||||
// Condition 3: we need to be able to connect to the original database
|
||||
// path.
|
||||
try
|
||||
{
|
||||
using var db = LegacyMessageImporter.Connect(originalDbPath);
|
||||
var size = new FileInfo(originalDbPath).Length;
|
||||
var count = db.GetCollection(LegacyMessageImporter.MessagesCollection).Count();
|
||||
if (count <= 0)
|
||||
NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleNoMessages, $"No messages in original database file '{originalDbPath}'");
|
||||
return NewEligible(originalDbPath, migrationDbPath, size, count);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
// Notify the user about this error, because they might be wondering
|
||||
// why they weren't offered a migration.
|
||||
return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed, $"LiteDB connection to original database file '{originalDbPath}' failed: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
internal LegacyMessageImporter StartImport(MessageStore targetStore, bool noLog = false)
|
||||
{
|
||||
if (Status != LegacyMessageImporterEligibilityStatus.Eligible)
|
||||
throw new InvalidOperationException($"Migration not eligible: status is {Status}");
|
||||
|
||||
return new LegacyMessageImporter(targetStore, originalDbPath: OriginalDbPath, migrationDbPath: MigrationDbPath, noLog: noLog);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Makes the migration ineligible so the user won't be asked again.
|
||||
/// </summary>
|
||||
internal void RenameOldDatabase()
|
||||
{
|
||||
File.Move(OriginalDbPath, MigrationDbPath);
|
||||
Status = LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists;
|
||||
AdditionalIneligibilityInfo = "User chose to rename the old database file";
|
||||
}
|
||||
}
|
||||
|
||||
internal class LegacyMessageImporter : IDisposable
|
||||
{
|
||||
internal const string MessagesCollection = "messages";
|
||||
private const int MaxFailedMessageLogs = 10;
|
||||
|
||||
private readonly MessageStore _targetStore;
|
||||
private readonly IPluginLog? _log;
|
||||
|
||||
private LiteDatabase? _database;
|
||||
|
||||
internal long ImportStart { get; } // ticks
|
||||
internal int ImportCount { get; private set; }
|
||||
internal int SuccessfulMessages { get; private set; }
|
||||
internal int FailedMessages { get; private set; }
|
||||
internal int ProcessedMessages => SuccessfulMessages + FailedMessages;
|
||||
internal int RemainingMessages => ImportCount - ProcessedMessages;
|
||||
// Progress from 0 to 1.
|
||||
internal float Progress => ImportCount > 0 ? ProcessedMessages / (float)ImportCount : 1;
|
||||
// Message count processed in the last second.
|
||||
internal float CurrentMessageRate { get; private set; }
|
||||
// ETA based on CurrentMessageRate.
|
||||
internal TimeSpan EstimatedTimeRemaining => TimeSpan.FromSeconds(CurrentMessageRate > 0 ? (ImportCount - SuccessfulMessages - FailedMessages) / CurrentMessageRate : 0);
|
||||
internal long? ImportComplete { get; private set; } // ticks
|
||||
|
||||
// This can be set by the user to limit the rate at which messages are
|
||||
// imported. If the rate exceeds this value, the importer will sleep for the
|
||||
// remainder of the second.
|
||||
internal int MaxMessageRate { get; set; } = 250; // start low
|
||||
|
||||
// Do not call this directly, use
|
||||
// LegacyMessageImporterEligibility.StartImport instead.
|
||||
internal LegacyMessageImporter(MessageStore targetStore, string? originalDbPath = null, string? migrationDbPath = null, bool noLog = false)
|
||||
{
|
||||
_targetStore = targetStore;
|
||||
originalDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat.db");
|
||||
migrationDbPath ??= migrationDbPath ?? Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-litedb.db");
|
||||
_log = noLog ? null : Plugin.Log;
|
||||
|
||||
_log?.Info($"[Migration] Moving '{originalDbPath}' to '{migrationDbPath}'");
|
||||
File.Move(originalDbPath, migrationDbPath);
|
||||
_log?.Info($"[Migration] Opening '{migrationDbPath}'");
|
||||
_database = Connect(migrationDbPath);
|
||||
|
||||
ImportStart = Environment.TickCount64;
|
||||
new Thread(DoImport).Start();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
// TODO: cancel thread and wait for it to close
|
||||
_database?.Dispose();
|
||||
}
|
||||
|
||||
internal static LiteDatabase Connect(string dbPath, bool readOnly = true)
|
||||
{
|
||||
BsonMapper.Global = new BsonMapper
|
||||
{
|
||||
IncludeNonPublic = true,
|
||||
TrimWhitespace = false
|
||||
};
|
||||
|
||||
BsonMapper.Global.RegisterType<Payload?>(
|
||||
payload =>
|
||||
{
|
||||
switch (payload)
|
||||
{
|
||||
case AchievementPayload achievement:
|
||||
return new BsonDocument(new Dictionary<string, BsonValue>
|
||||
{
|
||||
["Type"] = new("Achievement"),
|
||||
["Id"] = new(achievement.Id)
|
||||
});
|
||||
case PartyFinderPayload partyFinder:
|
||||
return new BsonDocument(new Dictionary<string, BsonValue>
|
||||
{
|
||||
["Type"] = new("PartyFinder"),
|
||||
["Id"] = new(partyFinder.Id)
|
||||
});
|
||||
case UriPayload uri:
|
||||
return new BsonDocument(new Dictionary<string, BsonValue>
|
||||
{
|
||||
["Type"] = new("URI"),
|
||||
["Uri"] = new(uri.Uri.ToString())
|
||||
});
|
||||
}
|
||||
|
||||
return payload?.Encode();
|
||||
},
|
||||
bson =>
|
||||
{
|
||||
if (bson.IsNull)
|
||||
return null;
|
||||
|
||||
if (bson.IsDocument)
|
||||
return bson["Type"].AsString switch
|
||||
{
|
||||
"Achievement" => new AchievementPayload((uint)bson["Id"].AsInt64),
|
||||
"PartyFinder" => new PartyFinderPayload((uint)bson["Id"].AsInt64),
|
||||
"URI" => new UriPayload(new Uri(bson["Uri"].AsString)),
|
||||
_ => null
|
||||
};
|
||||
|
||||
return Payload.Decode(new BinaryReader(new MemoryStream(bson.AsBinary)));
|
||||
});
|
||||
|
||||
BsonMapper.Global.RegisterType<SeString?>(
|
||||
seString => seString == null
|
||||
? null
|
||||
: new BsonArray(seString.Payloads.Select(payload => new BsonValue(payload.Encode()))),
|
||||
bson =>
|
||||
{
|
||||
if (bson.IsNull)
|
||||
return null;
|
||||
|
||||
var array = bson.IsArray ? bson.AsArray : bson["Payloads"].AsArray;
|
||||
var payloads = array
|
||||
.Select(payload => Payload.Decode(new BinaryReader(new MemoryStream(payload.AsBinary))))
|
||||
.ToList();
|
||||
return new SeString(payloads);
|
||||
}
|
||||
);
|
||||
BsonMapper.Global.RegisterType(
|
||||
type => (int)type,
|
||||
bson => (ChatType)bson.AsInt32
|
||||
);
|
||||
BsonMapper.Global.RegisterType(
|
||||
source => (int)source,
|
||||
bson => (ChatSource)bson.AsInt32
|
||||
);
|
||||
BsonMapper.Global.RegisterType(
|
||||
dateTime => dateTime.Subtract(DateTime.UnixEpoch).TotalMilliseconds,
|
||||
bson => DateTime.UnixEpoch.AddMilliseconds(bson.AsInt64)
|
||||
);
|
||||
|
||||
var connString = $"Filename='{dbPath}';Connection=direct;ReadOnly={readOnly}";
|
||||
var conn = new LiteDatabase(connString, BsonMapper.Global)
|
||||
{
|
||||
CheckpointSize = 1_000,
|
||||
Timeout = TimeSpan.FromSeconds(1)
|
||||
};
|
||||
var messages = conn.GetCollection<Message>(MessagesCollection);
|
||||
messages.EnsureIndex(msg => msg.Date);
|
||||
return conn;
|
||||
}
|
||||
|
||||
private void DoImport()
|
||||
{
|
||||
var importRateTimer = Stopwatch.StartNew();
|
||||
var messagesInLastSecond = 0;
|
||||
|
||||
// Query raw BsonDocuments, so we can convert them in individual
|
||||
// try-catch blocks.
|
||||
var messagesCollection = _database!.GetCollection<Message>(MessagesCollection);
|
||||
var totalMessages = messagesCollection.Count();
|
||||
ImportCount = totalMessages;
|
||||
var messages = messagesCollection.Query().OrderBy(msg => msg.Date).ToDocuments();
|
||||
foreach (var messageDoc in messages)
|
||||
{
|
||||
try
|
||||
{
|
||||
var message = BsonDocumentToMessage(messageDoc);
|
||||
_targetStore.UpsertMessage(message);
|
||||
SuccessfulMessages++;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
FailedMessages++;
|
||||
if (FailedMessages <= MaxFailedMessageLogs)
|
||||
_log?.Error(
|
||||
$"[Migration] Failed to import message '{messageDoc["_id"].AsObjectId}' (usually due to corruption): {e}");
|
||||
if (FailedMessages == MaxFailedMessageLogs)
|
||||
_log?.Error("[Migration] Further failed message logs will be suppressed");
|
||||
}
|
||||
|
||||
messagesInLastSecond++;
|
||||
if (MaxMessageRate > 0 && messagesInLastSecond > MaxMessageRate)
|
||||
{
|
||||
var sleepTime = 1000 - (int)importRateTimer.ElapsedMilliseconds;
|
||||
if (sleepTime > 0)
|
||||
Thread.Sleep(sleepTime);
|
||||
}
|
||||
if (importRateTimer.ElapsedMilliseconds > 1000)
|
||||
{
|
||||
CurrentMessageRate = messagesInLastSecond / (float)importRateTimer.ElapsedMilliseconds * 1000;
|
||||
importRateTimer.Restart();
|
||||
messagesInLastSecond = 0;
|
||||
}
|
||||
|
||||
// Log every 1,000 messages
|
||||
if ((SuccessfulMessages + FailedMessages) % 1000 == 0)
|
||||
_log?.Information(
|
||||
$"[Migration] Progress: successfully imported {SuccessfulMessages}/{totalMessages} messages ({FailedMessages} failures)");
|
||||
}
|
||||
|
||||
_log?.Information($"[Migration] Imported {SuccessfulMessages}/{FailedMessages} messages, {FailedMessages} failed");
|
||||
|
||||
if (ProcessedMessages != totalMessages)
|
||||
_log?.Warning(
|
||||
$"[Migration] Total message count mismatch: expected {totalMessages}, got {SuccessfulMessages + FailedMessages}");
|
||||
|
||||
ImportComplete = Environment.TickCount64;
|
||||
_database.Dispose();
|
||||
_database = null;
|
||||
}
|
||||
|
||||
private static Message BsonDocumentToMessage(BsonDocument doc)
|
||||
{
|
||||
return new Message(
|
||||
ObjectIdToGuid(doc["_id"].AsObjectId),
|
||||
(ulong)doc["Receiver"].AsInt64,
|
||||
(ulong)doc["ContentId"].AsInt64,
|
||||
DateTimeOffset.FromUnixTimeMilliseconds(doc["Date"].AsInt64),
|
||||
BsonMapper.Global.Deserialize<ChatCode>(doc["Code"].AsDocument),
|
||||
BsonMapper.Global.Deserialize<List<Chunk>>(doc["Sender"].AsArray),
|
||||
BsonMapper.Global.Deserialize<List<Chunk>>(doc["Content"].AsArray),
|
||||
BsonMapper.Global.Deserialize<SeString>(doc["SenderSource"].AsArray),
|
||||
BsonMapper.Global.Deserialize<SeString>(doc["ContentSource"].AsArray),
|
||||
BsonMapper.Global.Deserialize<SortCode>(doc["SortCode"].AsDocument),
|
||||
doc["ExtraChatChannel"].AsGuid
|
||||
);
|
||||
}
|
||||
|
||||
internal static Guid ObjectIdToGuid(ObjectId objectId)
|
||||
{
|
||||
// "Generate" a new Guid based on the ObjectId from the original
|
||||
// database. We want to have a stable unique identifier for each message
|
||||
// so that if the migration somehow happens twice the objects won't be
|
||||
// duplicated.
|
||||
//
|
||||
// Technically, when Guids are generated they follow a specific pattern.
|
||||
// However, in practice it doesn't matter at all, and we can just
|
||||
// generate whatever we want.
|
||||
var objectIdBytes = objectId.ToByteArray();
|
||||
var guidBytes = new byte[16];
|
||||
// Copy the first 7 bytes directly
|
||||
Buffer.BlockCopy(objectIdBytes, 0, guidBytes, 0, 7);
|
||||
// Fixed byte for version
|
||||
guidBytes[7] = 0b11111111;
|
||||
// Copy the next byte.
|
||||
guidBytes[8] = objectIdBytes[7];
|
||||
// Fixed reserved byte
|
||||
guidBytes[9] = 0b11111111;
|
||||
// Copy the last 4 bytes.
|
||||
Buffer.BlockCopy(objectIdBytes, 8, guidBytes, 10, 4);
|
||||
// Set the last 2 bytes to beef
|
||||
guidBytes[14] = 0xbe;
|
||||
guidBytes[15] = 0xef;
|
||||
|
||||
return new Guid(guidBytes);
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -3,6 +3,7 @@ using ChatTwo.Util;
|
||||
using Dalamud.Game.Text.SeStringHandling;
|
||||
using Dalamud.Game.Text.SeStringHandling.Payloads;
|
||||
using System.Text.RegularExpressions;
|
||||
using LiteDB;
|
||||
|
||||
namespace ChatTwo;
|
||||
|
||||
@@ -10,7 +11,8 @@ internal class SortCode {
|
||||
internal ChatType Type { get; }
|
||||
internal ChatSource Source { get; }
|
||||
|
||||
internal SortCode(ChatType type, ChatSource source) {
|
||||
[BsonCtor] // Used by LegacyMessageImporter
|
||||
public SortCode(ChatType type, ChatSource source) {
|
||||
Type = type;
|
||||
Source = source;
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ internal class MessageManager : IDisposable
|
||||
internal MessageStore Store { get; }
|
||||
|
||||
private ConcurrentQueue<(uint, Message)> Pending { get; } = new();
|
||||
private Stopwatch MaintenanceTimer { get; } = new();
|
||||
private Dictionary<ChatType, NameFormatting> Formats { get; } = new();
|
||||
private ulong LastContentId { get; set; }
|
||||
|
||||
@@ -35,7 +34,6 @@ internal class MessageManager : IDisposable
|
||||
internal MessageManager(Plugin plugin)
|
||||
{
|
||||
Plugin = plugin;
|
||||
MaintenanceTimer.Start();
|
||||
Store = new MessageStore(DatabasePath());
|
||||
|
||||
Plugin.ChatGui.ChatMessageUnhandled += ChatMessage;
|
||||
@@ -73,12 +71,6 @@ internal class MessageManager : IDisposable
|
||||
|
||||
private void GetMessageInfo(IFramework framework)
|
||||
{
|
||||
if (MaintenanceTimer.Elapsed > TimeSpan.FromMinutes(5))
|
||||
{
|
||||
MaintenanceTimer.Restart();
|
||||
new Thread(() => Store.PerformMaintenance()).Start();
|
||||
}
|
||||
|
||||
if (!Pending.TryDequeue(out var entry))
|
||||
return;
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ public sealed class Plugin : IDalamudPlugin
|
||||
public ChatLogWindow ChatLogWindow { get; }
|
||||
public CommandHelpWindow CommandHelpWindow { get; }
|
||||
public SeStringDebugger SeStringDebugger { get; }
|
||||
internal LegacyMesasgeImporterWindow LegacyMesasgeImporterWindow { get; }
|
||||
|
||||
internal Configuration Config { get; }
|
||||
internal Commands Commands { get; }
|
||||
@@ -104,6 +105,10 @@ public sealed class Plugin : IDalamudPlugin
|
||||
|
||||
MessageManager = new MessageManager(this); // requires Ui
|
||||
|
||||
// Requires MessageManager
|
||||
LegacyMesasgeImporterWindow = new LegacyMesasgeImporterWindow(MessageManager.Store);
|
||||
WindowSystem.AddWindow(LegacyMesasgeImporterWindow);
|
||||
|
||||
// let all the other components register, then initialise commands
|
||||
Commands.Initialise();
|
||||
|
||||
@@ -138,6 +143,7 @@ public sealed class Plugin : IDalamudPlugin
|
||||
ChatLogWindow?.Dispose();
|
||||
SettingsWindow?.Dispose();
|
||||
SeStringDebugger?.Dispose();
|
||||
LegacyMesasgeImporterWindow?.Dispose();
|
||||
|
||||
ExtraChat?.Dispose();
|
||||
Ipc?.Dispose();
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
using System.Numerics;
|
||||
using ChatTwo.Resources;
|
||||
using ChatTwo.Util;
|
||||
using Dalamud.Interface.ImGuiNotification;
|
||||
using Dalamud.Interface.Internal.Notifications;
|
||||
using Dalamud.Interface.Windowing;
|
||||
using ImGuiNET;
|
||||
|
||||
namespace ChatTwo.Ui;
|
||||
|
||||
internal class LegacyMesasgeImporterWindow : Window
|
||||
{
|
||||
private readonly MessageStore _store;
|
||||
|
||||
private LegacyMessageImporterEligibility Eligibility { get; set; }
|
||||
private LegacyMessageImporter? Importer { get; set; }
|
||||
|
||||
internal LegacyMesasgeImporterWindow(MessageStore store) : base("Chat 2 Legacy Importer###chat2-legacy-importer")
|
||||
{
|
||||
_store = store;
|
||||
Eligibility = LegacyMessageImporterEligibility.CheckEligibility();
|
||||
LogAndNotify();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Importer?.Dispose();
|
||||
}
|
||||
|
||||
private void LogAndNotify()
|
||||
{
|
||||
Plugin.Log.Info(
|
||||
$"[Migration] Checked migration eligibility: {Eligibility.Status} - '{Eligibility.AdditionalIneligibilityInfo}'");
|
||||
|
||||
switch (Eligibility.Status)
|
||||
{
|
||||
case LegacyMessageImporterEligibilityStatus.Eligible:
|
||||
{
|
||||
var notification = Plugin.Notification.AddNotification(new Notification
|
||||
{
|
||||
Type = NotificationType.Info,
|
||||
// The user needs to dismiss this for it to go away.
|
||||
InitialDuration = TimeSpan.FromHours(6),
|
||||
Title = "Chat 2 Migration",
|
||||
Content = "Import messages from old database into new database? Click for more information.",
|
||||
});
|
||||
// TODO: clicking does not dismiss
|
||||
notification.Click += _ => IsOpen = true;
|
||||
break;
|
||||
}
|
||||
|
||||
case LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed:
|
||||
{
|
||||
var notification = Plugin.Notification.AddNotification(new Notification
|
||||
{
|
||||
Type = NotificationType.Warning,
|
||||
InitialDuration = TimeSpan.FromMinutes(1),
|
||||
Title = "Chat Two Migration",
|
||||
Content =
|
||||
"Migration is not possible because the old database could not be opened. Click for more information."
|
||||
});
|
||||
// TODO: clicking does not dismiss
|
||||
notification.Click += _ => IsOpen = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override void Draw()
|
||||
{
|
||||
if (Importer != null)
|
||||
{
|
||||
DrawImportStatus();
|
||||
return;
|
||||
}
|
||||
|
||||
if (Eligibility.Status == LegacyMessageImporterEligibilityStatus.Eligible)
|
||||
DrawEligible();
|
||||
else
|
||||
DrawIneligible();
|
||||
}
|
||||
|
||||
private void DrawEligible()
|
||||
{
|
||||
// TODO: pretty
|
||||
ImGui.Text("Import database messages from legacy LiteDB database to Sqlite database?");
|
||||
ImGui.Text($"Message count: {Eligibility.MessageCount}");
|
||||
ImGui.Text($"Database size: {Eligibility.DatabaseSizeBytes}");
|
||||
|
||||
if (ImGui.Button("Yes, import messages"))
|
||||
{
|
||||
// Next draw call will run DrawImportStatus().
|
||||
Importer = Eligibility.StartImport(_store);
|
||||
return;
|
||||
}
|
||||
|
||||
ImGui.SameLine();
|
||||
|
||||
if (ImGuiUtil.CtrlShiftButton("No, do not import messages",
|
||||
"Ctrl+Shift: renames old database to avoid prompting again"))
|
||||
{
|
||||
Eligibility.RenameOldDatabase();
|
||||
IsOpen = false;
|
||||
}
|
||||
}
|
||||
|
||||
private void DrawIneligible()
|
||||
{
|
||||
// TODO: pretty
|
||||
ImGui.Text("Your legacy LiteDB database is not eligible for import:");
|
||||
switch (Eligibility.Status)
|
||||
{
|
||||
case LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists:
|
||||
ImGui.Text("The old database could not be found.");
|
||||
break;
|
||||
case LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists:
|
||||
ImGui.Text("The migration process was already started.");
|
||||
break;
|
||||
case LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed:
|
||||
ImGui.Text("The old database could not be opened.");
|
||||
break;
|
||||
case LegacyMessageImporterEligibilityStatus.IneligibleNoMessages:
|
||||
ImGui.Text("The old database contains no messages.");
|
||||
break;
|
||||
case LegacyMessageImporterEligibilityStatus.Eligible:
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException();
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(Eligibility.AdditionalIneligibilityInfo))
|
||||
ImGui.Text(Eligibility.AdditionalIneligibilityInfo);
|
||||
|
||||
// LiteDB failures notify the user, so give them a chance to rename the
|
||||
// database to avoid prompting again.
|
||||
if (Eligibility.Status == LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed)
|
||||
{
|
||||
if (ImGuiUtil.CtrlShiftButton("Rename old database",
|
||||
"Ctrl+Shift: rename old database to avoid import prompt in the future"))
|
||||
{
|
||||
Eligibility.RenameOldDatabase();
|
||||
// TODO: notify success as this changes the status
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void DrawImportStatus()
|
||||
{
|
||||
// TODO: pretty
|
||||
if (Importer == null)
|
||||
return;
|
||||
|
||||
var importStart = Importer.ImportStart;
|
||||
var importEnd = Importer.ImportComplete;
|
||||
var total = Importer.ImportCount;
|
||||
var successful = Importer!.SuccessfulMessages;
|
||||
var failed = Importer.FailedMessages;
|
||||
var remaining = Importer.RemainingMessages;
|
||||
|
||||
if (importEnd != null)
|
||||
{
|
||||
ImGui.Text($"Completed migration in {Duration(importStart, importEnd.Value)}");
|
||||
ImGui.Text($"Successfully imported: {successful} messages");
|
||||
ImGui.Text($"Failed to import: {failed} messages");
|
||||
ImGui.Text($"Unaccounted for: {remaining}");
|
||||
ImGui.Text("See logs for more details: /xllog");
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: implement Importer.MaxMessageRate slider in UI, values 0 (infinity) => 10000
|
||||
|
||||
ImGui.Text($"Importing messages... {Importer.Progress:P}%");
|
||||
ImGui.Text($"Duration: {Duration(importStart, Environment.TickCount64)}");
|
||||
ImGui.Text($"Successfully imported: {successful} messages");
|
||||
ImGui.Text($"Failed to import: {failed} messages");
|
||||
ImGui.Text($"Progress: {Importer.ProcessedMessages}/{total} messages");
|
||||
ImGui.Text($"Remaining: {remaining} messages");
|
||||
ImGui.Text($"Messages per second: {Importer.CurrentMessageRate}");
|
||||
ImGui.Text($"Estimated time remaining: {Importer.EstimatedTimeRemaining}");
|
||||
ImGui.Text("See logs for more details: /xllog");
|
||||
|
||||
// TODO: this doesn't render properly
|
||||
ImGui.ProgressBar(Importer.Progress, new Vector2(0.0f, 0.0f), $"{Importer.Progress:P}%");
|
||||
|
||||
if (ImGuiUtil.CtrlShiftButton("Cancel import", "Ctrl+Shift: cancel import and close window"))
|
||||
{
|
||||
// TODO: This currently crashes the whole game because we don't ask
|
||||
// the importer thread to stop and wait for it to stop before
|
||||
// disposing it.
|
||||
// See LegacyMessageImporter.Dispose() for more details.
|
||||
/*
|
||||
Importer.Dispose();
|
||||
Importer = null;
|
||||
Eligibility = LegacyMessageImporterEligibility.CheckEligibility();
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
private static TimeSpan Duration(long startTicks, long endTicks)
|
||||
{
|
||||
return endTicks < startTicks ? TimeSpan.Zero : TimeSpan.FromTicks(endTicks - startTicks);
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,12 @@
|
||||
"resolved": "2.1.12",
|
||||
"contentHash": "Sc0PVxvgg4NQjcI8n10/VfUQBAS4O+Fw2pZrAqBdRMbthYGeogzu5+xmIGCGmsEZ/ukMOBuAqiNiB5qA3MRalg=="
|
||||
},
|
||||
"LiteDB": {
|
||||
"type": "Direct",
|
||||
"requested": "[5.0.19, )",
|
||||
"resolved": "5.0.19",
|
||||
"contentHash": "O8XPBptE4SygW2SN6skqg/VDVTrjpJ0p6+i7Cp8x8razbu2ORLSd6ep3JdhDIdL6h57Bcxv2BuVaN70IKpXI0Q=="
|
||||
},
|
||||
"MessagePack": {
|
||||
"type": "Direct",
|
||||
"requested": "[2.5.140, )",
|
||||
|
||||
Reference in New Issue
Block a user