JonKazama-Hellion/HellionChat
1
0
Fork 0
Code Issues 1 Pull Requests Actions Releases 28 Activity

Remove legacy migration

Browse Source
This commit is contained in:
Infi
2024-07-25 15:33:05 +02:00
parent 41c59f6aa0
commit 906eeb5095
11 changed files with 14 additions and 878 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ChatTwo.Tests/LegacyMessageImporterTest.cs
-220
View File
@@ -1,220 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using ChatTwo.Code;
using JetBrains.Annotations;
using LiteDB;
using Microsoft.VisualStudio.TestTools.UnitTesting;
namespace ChatTwo.Tests;
[TestClass]
[TestSubject(typeof(LegacyMessageImporter))]
public class LegacyMessageImporterTest
{
public TestContext TestContext { get; set; }
[TestMethod]
public void ConvertId()
{
for (var i = 0; i < 1000; i++)
{
var originalObjectId = ObjectId.NewObjectId();
var intermediateGuid = LegacyMessageImporter.ObjectIdToGuid(originalObjectId);
var newObjectId = CustomGuidToObjectId(intermediateGuid);
TestContext.WriteLine($"original: {originalObjectId}");
TestContext.WriteLine($"new: {newObjectId}");
TestContext.WriteLine($"intermediate: {intermediateGuid}");
Assert.IsTrue(originalObjectId.Equals(newObjectId));
}
}
[TestMethod]
[Timeout(10_000)]
public void Import()
{
const int count = 100;
var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_");
TestContext.WriteLine("Using temp path: " + tempDir);
var liteDbPath = Path.Join(tempDir.FullName, "original.litedb");
TestContext.WriteLine("Using original DB path: " + liteDbPath);
var migrationDbPath = Path.Join(tempDir.FullName, "migration.litedb");
TestContext.WriteLine("Using migration DB path: " + migrationDbPath);
var newDbPath = Path.Join(tempDir.FullName, "new.sqlitedb");
TestContext.WriteLine("Using new DB path: " + newDbPath);
var expectedMessages = new List<Message>(count);
using (var liteDatabase = LegacyMessageImporter.Connect(liteDbPath, readOnly: false))
{
var messagesCollection = liteDatabase.GetCollection(LegacyMessageImporter.MessagesCollection);
var now = DateTimeOffset.UtcNow;
for (var i = 0; i < count; i++)
{
var messageId = ObjectId.NewObjectId();
var message = MessageStoreTest.BigMessage(dateTime: now.AddSeconds(-count + i));
// Use reflection to set Id because we don't want to add a
// setter to it and allow other code to use it.
var guid = LegacyMessageImporter.ObjectIdToGuid(messageId);
message.GetType().GetField("<Id>k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(message, guid);
expectedMessages.Add(message);
messagesCollection.Insert(new BsonDocument {
["_id"] = messageId,
["Receiver"] = message.Receiver,
["ContentId"] = message.ContentId,
["Date"] = message.Date.ToUnixTimeMilliseconds(),
["Code"] = BsonMapper.Global.Serialize(message.Code),
["Sender"] = BsonMapper.Global.Serialize(message.Sender),
["Content"] = BsonMapper.Global.Serialize(message.Content),
["SenderSource"] = BsonMapper.Global.Serialize(message.SenderSource),
["ContentSource"] = BsonMapper.Global.Serialize(message.ContentSource),
["SortCode"] = BsonMapper.Global.Serialize(message.SortCode),
["ExtraChatChannel"] = message.ExtraChatChannel,
});
}
Assert.AreEqual(count, messagesCollection.Count());
}
var dbPath = Path.Join(tempDir.FullName, "test.db");
using var store = new MessageStore(dbPath);
var eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath);
Assert.AreEqual(LegacyMessageImporterEligibilityStatus.Eligible, eligibility.Status);
Assert.AreEqual("", eligibility.AdditionalIneligibilityInfo);
Assert.AreEqual(liteDbPath, eligibility.OriginalDbPath);
Assert.AreEqual(migrationDbPath, eligibility.MigrationDbPath);
Assert.IsTrue(eligibility.DatabaseSizeBytes > 0);
Assert.AreEqual(count, eligibility.MessageCount);
var importer = eligibility.StartImport(store, noLog: true);
while (importer.ImportComplete == null)
System.Threading.Thread.Sleep(10);
Assert.IsTrue(importer.ImportComplete > importer.ImportStart);
Assert.AreEqual(count, importer.SuccessfulMessages);
Assert.AreEqual(0, importer.FailedMessages);
var messages = store.GetMostRecentMessages(count: count + 1).ToList();
Assert.AreEqual(count, messages.Count);
for (var i = 0; i < count; i++)
MessageStoreTest.AssertMessagesEqual(expectedMessages[i], messages[i]);
// No longer eligible.
eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath);
Assert.AreEqual(LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists, eligibility.Status);
Assert.IsTrue(eligibility.AdditionalIneligibilityInfo.Contains("Original database file"));
Assert.AreEqual("", eligibility.OriginalDbPath);
Assert.AreEqual("", eligibility.MigrationDbPath);
Assert.AreEqual(0, eligibility.DatabaseSizeBytes);
Assert.AreEqual(0, eligibility.MessageCount);
}
[TestMethod]
[Timeout(10_000)]
public void CorruptedImport()
{
const int count = 100;
const int corruptedIndex = 69;
var tempDir = Directory.CreateTempSubdirectory("ChatTwo_test_");
TestContext.WriteLine("Using temp path: " + tempDir);
var liteDbPath = Path.Join(tempDir.FullName, "original.litedb");
TestContext.WriteLine("Using original DB path: " + liteDbPath);
var migrationDbPath = Path.Join(tempDir.FullName, "migration.litedb");
TestContext.WriteLine("Using migration DB path: " + migrationDbPath);
var newDbPath = Path.Join(tempDir.FullName, "new.sqlitedb");
TestContext.WriteLine("Using new DB path: " + newDbPath);
var expectedMessages = new List<Message>(count);
using (var liteDatabase = LegacyMessageImporter.Connect(liteDbPath, readOnly: false))
{
var messagesCollection = liteDatabase.GetCollection(LegacyMessageImporter.MessagesCollection);
var now = DateTimeOffset.UtcNow;
for (var i = 0; i < count; i++)
{
if (i == corruptedIndex)
{
// This message will not be imported because it can't be
// parsed into a Message object.
messagesCollection.Insert(new BsonDocument
{
["_id"] = ObjectId.NewObjectId(),
["Receiver"] = 0L,
["ContentId"] = 0L,
["Date"] = 0L,
["Code"] = BsonMapper.Global.Serialize(new ChatCode(0)),
["Sender"] = BsonMapper.Global.Serialize(new List<Chunk>()),
["Content"] = BsonMapper.Global.Serialize(new List<Chunk>()),
// These are meant to be arrays.
["SenderSource"] = new BsonDocument(),
["ContentSource"] = new BsonDocument(),
["SortCode"] = BsonMapper.Global.Serialize(new SortCode(0)),
["ExtraChatChannel"] = new Guid(),
});
continue;
}
var messageId = ObjectId.NewObjectId();
var message = MessageStoreTest.BigMessage(dateTime: now.AddSeconds(-count + i));
// Use reflection to set Id because we don't want to add a
// setter to it and allow other code to use it.
var guid = LegacyMessageImporter.ObjectIdToGuid(messageId);
message.GetType().GetField("<Id>k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(message, guid);
expectedMessages.Add(message);
messagesCollection.Insert(new BsonDocument {
["_id"] = messageId,
["Receiver"] = message.Receiver,
["ContentId"] = message.ContentId,
["Date"] = message.Date.ToUnixTimeMilliseconds(),
["Code"] = BsonMapper.Global.Serialize(message.Code),
["Sender"] = BsonMapper.Global.Serialize(message.Sender),
["Content"] = BsonMapper.Global.Serialize(message.Content),
["SenderSource"] = BsonMapper.Global.Serialize(message.SenderSource),
["ContentSource"] = BsonMapper.Global.Serialize(message.ContentSource),
["SortCode"] = BsonMapper.Global.Serialize(message.SortCode),
["ExtraChatChannel"] = message.ExtraChatChannel,
});
}
Assert.AreEqual(count, messagesCollection.Count());
}
var dbPath = Path.Join(tempDir.FullName, "test.db");
using var store = new MessageStore(dbPath);
var eligibility = LegacyMessageImporterEligibility.CheckEligibility(originalDbPath: liteDbPath, migrationDbPath: migrationDbPath);
Assert.AreEqual(LegacyMessageImporterEligibilityStatus.Eligible, eligibility.Status);
var importer = eligibility.StartImport(store, noLog: true);
while (importer.ImportComplete == null)
System.Threading.Thread.Sleep(10);
Assert.IsTrue(importer.ImportComplete > importer.ImportStart);
Assert.AreEqual(count - 1, importer.SuccessfulMessages);
Assert.AreEqual(1, importer.FailedMessages);
var messages = store.GetMostRecentMessages(count: count + 1).ToList();
Assert.AreEqual(count - 1, messages.Count);
for (var i = 0; i < count - 1; i++)
MessageStoreTest.AssertMessagesEqual(expectedMessages[i], messages[i]);
}
/// <summary>
/// Converts Guids created by LegacyMessageImporter.ObjectIdToGuid() back to
/// their original ObjectId. If any other Guid is passed, the result is
/// lossy.
/// </summary>
private ObjectId CustomGuidToObjectId(Guid guid)
{
var guidBytes = guid.ToByteArray();
var newObjectIdBytes = new byte[12];
Buffer.BlockCopy(guidBytes, 0, newObjectIdBytes, 0, 7);
newObjectIdBytes[7] = guidBytes[8];
Buffer.BlockCopy(guidBytes, 10, newObjectIdBytes, 8, 4);
return new ObjectId(newObjectIdBytes);
}
}
ChatTwo/ChatTwo.csproj
-1
View File
@@ -50,7 +50,6 @@
<ItemGroup>
<PackageReference Include="DalamudPackager" Version="2.1.13" />
<PackageReference Include="LiteDB" Version="5.0.19" />
<PackageReference Include="MessagePack" Version="2.5.140" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.4" />
<PackageReference Include="Pidgin" Version="3.2.2"/>
ChatTwo/ChatTwo.yaml
+9 -4
View File
@@ -22,7 +22,12 @@ tags:
- Chat
- Replacement
changelog: |-
**Updates**
- Loc updates
- Fix multiple issues related to keybinds
- Add more options for auto hide [thanks dean]
**Fonts**
- Uses dalamud font chooser
- Italic font version can be chosen
- Config options related to fonts have been reset
- Old font names and sizes used are displayed for up to a month
**LiteDB**
- Migration option removed
- Old files can still be removed in the database tab
ChatTwo/Chunk.cs
-2
View File
@@ -1,6 +1,5 @@
using ChatTwo.Code;
using Dalamud.Game.Text.SeStringHandling;
using LiteDB;
using MessagePack;
namespace ChatTwo;
@@ -11,7 +10,6 @@ namespace ChatTwo;
public abstract class Chunk
{
[IgnoreMember]
[BsonIgnore] // used by LegacyMessageImporter
internal Message? Message { get; set; }
[Key(0)]
ChatTwo/Code/ChatCode.cs
-12
View File
@@ -1,6 +1,3 @@
using Dalamud.Game.Config;
using LiteDB;
namespace ChatTwo.Code;
internal class ChatCode
@@ -22,15 +19,6 @@ internal class ChatCode
Target = SourceFrom(7);
}
[BsonCtor] // Used by LegacyMessageImporter
public ChatCode(ushort raw, ChatType type, ChatSource source, ChatSource target)
{
Raw = raw;
Type = type;
Source = source;
Target = target;
}
internal ChatType Parent() => Type switch
{
ChatType.Say => ChatType.Say,
ChatTwo/LegacyMessageImporter.cs
-385
View File
@@ -1,385 +0,0 @@
using System.Diagnostics;
using ChatTwo.Code;
using ChatTwo.Util;
using Dalamud.Game.Text.SeStringHandling;
using Dalamud.Plugin.Services;
using LiteDB;
namespace ChatTwo;
internal enum LegacyMessageImporterEligibilityStatus
{
Eligible,
IneligibleOriginalDbNotExists,
IneligibleMigrationDbExists,
IneligibleLiteDbFailed,
IneligibleNoMessages,
}
internal class LegacyMessageImporterEligibility
{
internal LegacyMessageImporterEligibilityStatus Status { get; private set; }
internal string AdditionalIneligibilityInfo { get; private set; }
internal string OriginalDbPath { get; }
internal string MigrationDbPath { get; }
internal long DatabaseSizeBytes { get; }
internal int MessageCount { get; }
private LegacyMessageImporterEligibility(LegacyMessageImporterEligibilityStatus status, string additionalIneligibilityInfo, string originalDbPath, string migrationDbPath, long databaseSizeBytes, int messageCount)
{
Status = status;
AdditionalIneligibilityInfo = additionalIneligibilityInfo;
OriginalDbPath = originalDbPath;
MigrationDbPath = migrationDbPath;
DatabaseSizeBytes = databaseSizeBytes;
MessageCount = messageCount;
}
private static LegacyMessageImporterEligibility NewEligible(string originalDbPath, string migrationDbPath,
long databaseSizeBytes, int messageCount)
{
return new LegacyMessageImporterEligibility(LegacyMessageImporterEligibilityStatus.Eligible, "", originalDbPath, migrationDbPath, databaseSizeBytes, messageCount);
}
private static LegacyMessageImporterEligibility NewIneligible(LegacyMessageImporterEligibilityStatus status, string additionalIneligibilityReason)
{
return new LegacyMessageImporterEligibility(status, additionalIneligibilityReason, "", "", 0, 0);
}
internal static LegacyMessageImporterEligibility CheckEligibility(string? originalDbPath = null, string? migrationDbPath = null)
{
originalDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat.db");
migrationDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-litedb.db");
// Condition 1: the database file must exist in its original path.
if (!File.Exists(originalDbPath))
{
return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists, $"Original database file '{originalDbPath}' does not exist");
}
// Condition 2: the migration file must not exist.
if (File.Exists(migrationDbPath))
{
return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists, $"Migration database file '{migrationDbPath}' already exists, migration was already started in the past");
}
// Condition 3: we need to be able to connect to the original database
// path.
try
{
using var db = LegacyMessageImporter.Connect(originalDbPath);
var size = new FileInfo(originalDbPath).Length;
var count = db.GetCollection(LegacyMessageImporter.MessagesCollection).Count();
if (count <= 0)
NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleNoMessages, $"No messages in original database file '{originalDbPath}'");
return NewEligible(originalDbPath, migrationDbPath, size, count);
}
catch (Exception e)
{
// Notify the user about this error, because they might be wondering
// why they weren't offered a migration.
return NewIneligible(LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed, $"LiteDB connection to original database file '{originalDbPath}' failed: {e}");
}
}
internal LegacyMessageImporter StartImport(MessageStore targetStore, bool noLog = false, Plugin? plugin = null)
{
if (Status != LegacyMessageImporterEligibilityStatus.Eligible)
throw new InvalidOperationException($"Migration not eligible: status is {Status}");
return new LegacyMessageImporter(targetStore, originalDbPath: OriginalDbPath, migrationDbPath: MigrationDbPath, noLog: noLog, plugin);
}
/// <summary>
/// Makes the migration ineligible so the user won't be asked again.
/// </summary>
internal bool RenameOldDatabase()
{
try
{
File.Move(OriginalDbPath, MigrationDbPath);
Status = LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists;
AdditionalIneligibilityInfo = "User chose to rename the old database file";
return true;
}
catch (Exception ex)
{
Plugin.Log.Error(ex, "Unable to move the old database");
return false;
}
}
}
internal class LegacyMessageImporter : IAsyncDisposable
{
private readonly Plugin? Plugin;
private readonly CancellationTokenSource CancellationToken = new();
private Thread? WorkingThread = null;
internal const string MessagesCollection = "messages";
private const int MaxFailedMessageLogs = 10;
private readonly MessageStore _targetStore;
private readonly IPluginLog? _log;
private LiteDatabase? _database;
internal long ImportStart { get; } // ticks
internal int ImportCount { get; private set; }
internal int SuccessfulMessages { get; private set; }
internal int FailedMessages { get; private set; }
internal int ProcessedMessages => SuccessfulMessages + FailedMessages;
internal int RemainingMessages => ImportCount - ProcessedMessages;
// Progress from 0 to 1.
internal float Progress => ImportCount > 0 ? ProcessedMessages / (float)ImportCount : 1;
// Message count processed in the last second.
internal float CurrentMessageRate { get; private set; }
// ETA based on CurrentMessageRate.
internal TimeSpan EstimatedTimeRemaining => TimeSpan.FromSeconds(CurrentMessageRate > 0 ? (ImportCount - SuccessfulMessages - FailedMessages) / CurrentMessageRate : 0);
internal long? ImportComplete { get; private set; } // ticks
// This can be set by the user to limit the rate at which messages are
// imported. If the rate exceeds this value, the importer will sleep for the
// remainder of the second.
internal int MaxMessageRate = 250; // start low
// Do not call this directly, use
// LegacyMessageImporterEligibility.StartImport instead.
internal LegacyMessageImporter(MessageStore targetStore, string? originalDbPath = null, string? migrationDbPath = null, bool noLog = false, Plugin? plugin = null)
{
_targetStore = targetStore;
originalDbPath ??= Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat.db");
migrationDbPath ??= migrationDbPath ?? Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-litedb.db");
_log = noLog ? null : Plugin.Log;
Plugin = plugin;
_log?.Info($"[Migration] Moving '{originalDbPath}' to '{migrationDbPath}'");
File.Move(originalDbPath, migrationDbPath);
_log?.Info($"[Migration] Opening '{migrationDbPath}'");
_database = Connect(migrationDbPath);
ImportStart = Environment.TickCount64;
WorkingThread = new Thread(() => DoImport(CancellationToken.Token));
WorkingThread.Start();
}
public async ValueTask DisposeAsync()
{
await CancellationToken.CancelAsync();
var timeout = 10_000; // 10s
while (WorkingThread != null && timeout > 0)
{
if (!WorkingThread.IsAlive)
break;
timeout -= 100;
await Task.Delay(100);
Plugin.Log.Information("Sleeping because thread still alive");
}
_database?.Dispose();
}
internal static LiteDatabase Connect(string dbPath, bool readOnly = true)
{
BsonMapper.Global = new BsonMapper
{
IncludeNonPublic = true,
TrimWhitespace = false
};
BsonMapper.Global.RegisterType<Payload?>(
payload =>
{
switch (payload)
{
case AchievementPayload achievement:
return new BsonDocument(new Dictionary<string, BsonValue>
{
["Type"] = new("Achievement"),
["Id"] = new(achievement.Id)
});
case PartyFinderPayload partyFinder:
return new BsonDocument(new Dictionary<string, BsonValue>
{
["Type"] = new("PartyFinder"),
["Id"] = new(partyFinder.Id)
});
case UriPayload uri:
return new BsonDocument(new Dictionary<string, BsonValue>
{
["Type"] = new("URI"),
["Uri"] = new(uri.Uri.ToString())
});
}
return payload?.Encode();
},
bson =>
{
if (bson.IsNull)
return null;
if (bson.IsDocument)
return bson["Type"].AsString switch
{
"Achievement" => new AchievementPayload((uint)bson["Id"].AsInt64),
"PartyFinder" => new PartyFinderPayload((uint)bson["Id"].AsInt64),
"URI" => new UriPayload(new Uri(bson["Uri"].AsString)),
_ => null
};
return Payload.Decode(new BinaryReader(new MemoryStream(bson.AsBinary)));
});
BsonMapper.Global.RegisterType<SeString?>(
seString => seString == null
? null
: new BsonArray(seString.Payloads.Select(payload => new BsonValue(payload.Encode()))),
bson =>
{
if (bson.IsNull)
return null;
var array = bson.IsArray ? bson.AsArray : bson["Payloads"].AsArray;
var payloads = array
.Select(payload => Payload.Decode(new BinaryReader(new MemoryStream(payload.AsBinary))))
.ToList();
return new SeString(payloads);
}
);
BsonMapper.Global.RegisterType(
type => (int)type,
bson => (ChatType)bson.AsInt32
);
BsonMapper.Global.RegisterType(
source => (int)source,
bson => (ChatSource)bson.AsInt32
);
BsonMapper.Global.RegisterType(
dateTime => dateTime.Subtract(DateTime.UnixEpoch).TotalMilliseconds,
bson => DateTime.UnixEpoch.AddMilliseconds(bson.AsInt64)
);
var connString = $"Filename='{dbPath}';Connection=direct;ReadOnly={readOnly}";
var conn = new LiteDatabase(connString, BsonMapper.Global)
{
CheckpointSize = 1_000,
Timeout = TimeSpan.FromSeconds(1)
};
var messages = conn.GetCollection<Message>(MessagesCollection);
messages.EnsureIndex(msg => msg.Date);
return conn;
}
private void DoImport(CancellationToken token)
{
var importRateTimer = Stopwatch.StartNew();
var messagesInLastSecond = 0;
// Query raw BsonDocuments, so we can convert them in individual
// try-catch blocks.
var messagesCollection = _database!.GetCollection<Message>(MessagesCollection);
var totalMessages = messagesCollection.Count();
ImportCount = totalMessages;
var messages = messagesCollection.Query().OrderBy(msg => msg.Date).ToDocuments();
foreach (var messageDoc in messages)
{
if (token.IsCancellationRequested)
return;
try
{
var message = BsonDocumentToMessage(messageDoc);
_targetStore.UpsertMessage(message);
SuccessfulMessages++;
}
catch (Exception e)
{
FailedMessages++;
if (FailedMessages <= MaxFailedMessageLogs)
_log?.Error($"[Migration] Failed to import message '{messageDoc["_id"].AsObjectId}' (usually due to corruption): {e}");
if (FailedMessages == MaxFailedMessageLogs)
_log?.Error("[Migration] Further failed message logs will be suppressed");
}
messagesInLastSecond++;
if (MaxMessageRate > 0 && messagesInLastSecond > MaxMessageRate)
{
var sleepTime = 1000 - (int)importRateTimer.ElapsedMilliseconds;
if (sleepTime > 0)
Thread.Sleep(sleepTime);
}
if (importRateTimer.ElapsedMilliseconds > 1000)
{
CurrentMessageRate = messagesInLastSecond / (float)importRateTimer.ElapsedMilliseconds * 1000;
importRateTimer.Restart();
messagesInLastSecond = 0;
}
// Log every 1,000 messages
if ((SuccessfulMessages + FailedMessages) % 1000 == 0)
_log?.Information($"[Migration] Progress: successfully imported {SuccessfulMessages}/{totalMessages} messages ({FailedMessages} failures)");
}
_log?.Information($"[Migration] Imported {SuccessfulMessages}/{FailedMessages} messages, {FailedMessages} failed");
if (ProcessedMessages != totalMessages)
_log?.Warning($"[Migration] Total message count mismatch: expected {totalMessages}, got {SuccessfulMessages + FailedMessages}");
ImportComplete = Environment.TickCount64;
_database.Dispose();
_database = null;
Plugin?.MessageManager.FilterAllTabsAsync();
}
private static Message BsonDocumentToMessage(BsonDocument doc)
{
return new Message(
ObjectIdToGuid(doc["_id"].AsObjectId),
(ulong)doc["Receiver"].AsInt64,
(ulong)doc["ContentId"].AsInt64,
DateTimeOffset.FromUnixTimeMilliseconds(doc["Date"].AsInt64),
BsonMapper.Global.Deserialize<ChatCode>(doc["Code"].AsDocument),
BsonMapper.Global.Deserialize<List<Chunk>>(doc["Sender"].AsArray),
BsonMapper.Global.Deserialize<List<Chunk>>(doc["Content"].AsArray),
BsonMapper.Global.Deserialize<SeString>(doc["SenderSource"].AsArray),
BsonMapper.Global.Deserialize<SeString>(doc["ContentSource"].AsArray),
BsonMapper.Global.Deserialize<SortCode>(doc["SortCode"].AsDocument),
doc["ExtraChatChannel"].AsGuid
);
}
internal static Guid ObjectIdToGuid(ObjectId objectId)
{
// "Generate" a new Guid based on the ObjectId from the original
// database. We want to have a stable unique identifier for each message
// so that if the migration somehow happens twice the objects won't be
// duplicated.
//
// Technically, when Guids are generated they follow a specific pattern.
// However, in practice it doesn't matter at all, and we can just
// generate whatever we want.
var objectIdBytes = objectId.ToByteArray();
var guidBytes = new byte[16];
// Copy the first 7 bytes directly
Buffer.BlockCopy(objectIdBytes, 0, guidBytes, 0, 7);
// Fixed byte for version
guidBytes[7] = 0b11111111;
// Copy the next byte.
guidBytes[8] = objectIdBytes[7];
// Fixed reserved byte
guidBytes[9] = 0b11111111;
// Copy the last 4 bytes.
Buffer.BlockCopy(objectIdBytes, 8, guidBytes, 10, 4);
// Set the last 2 bytes to beef
guidBytes[14] = 0xbe;
guidBytes[15] = 0xef;
return new Guid(guidBytes);
}
}
ChatTwo/Message.cs
-3
View File
@@ -5,9 +5,7 @@ using Dalamud.Game.Text.SeStringHandling;
using Dalamud.Game.Text.SeStringHandling.Payloads;
using System.Text.RegularExpressions;
using Dalamud.Game.Text;
using FFXIVClientStructs.FFXIV.Client.Game;
using FFXIVClientStructs.FFXIV.Client.UI.Agent;
using LiteDB;
using Lumina.Excel.GeneratedSheets;
namespace ChatTwo;
@@ -17,7 +15,6 @@ internal class SortCode
internal ChatType Type { get; }
internal ChatSource Source { get; }
[BsonCtor] // Used by LegacyMessageImporter
public SortCode(ChatType type, ChatSource source)
{
Type = type;
ChatTwo/Plugin.cs
-6
View File
@@ -50,7 +50,6 @@ public sealed class Plugin : IDalamudPlugin
public CommandHelpWindow CommandHelpWindow { get; }
public SeStringDebugger SeStringDebugger { get; }
public DebuggerWindow DebuggerWindow { get; }
internal LegacyMessageImporterWindow LegacyMessageImporterWindow { get; }
internal Commands Commands { get; }
internal ChatCommon Common { get; }
@@ -109,10 +108,6 @@ public sealed class Plugin : IDalamudPlugin
MessageManager = new MessageManager(this); // requires Ui
// Requires MessageManager
LegacyMessageImporterWindow = new LegacyMessageImporterWindow(this);
WindowSystem.AddWindow(LegacyMessageImporterWindow);
// let all the other components register, then initialise commands
Commands.Initialise();
@@ -158,7 +153,6 @@ public sealed class Plugin : IDalamudPlugin
SettingsWindow?.Dispose();
DebuggerWindow?.Dispose();
SeStringDebugger?.Dispose();
LegacyMessageImporterWindow?.Dispose();
ExtraChat?.Dispose();
Ipc?.Dispose();
ChatTwo/Ui/LegacyMessageImporterWindow.cs
-225
View File
@@ -1,225 +0,0 @@
using System.Numerics;
using ChatTwo.Util;
using Dalamud.Interface.ImGuiNotification;
using Dalamud.Interface.ImGuiNotification.EventArgs;
using Dalamud.Interface.Utility;
using Dalamud.Interface.Utility.Raii;
using Dalamud.Interface.Windowing;
using ImGuiNET;
namespace ChatTwo.Ui;
internal class LegacyMessageImporterWindow : Window
{
private readonly Plugin Plugin;
private readonly MessageStore _store;
private LegacyMessageImporterEligibility Eligibility { get; set; }
private LegacyMessageImporter? Importer { get; set; }
internal LegacyMessageImporterWindow(Plugin plugin) : base("Chat 2 Legacy Importer###chat2-legacy-importer")
{
Plugin = plugin;
Flags = ImGuiWindowFlags.NoResize;
Size = new Vector2(500, 400);
RespectCloseHotkey = false;
DisableWindowSounds = true;
_store = plugin.MessageManager.Store;
Eligibility = LegacyMessageImporterEligibility.CheckEligibility();
LogAndNotify();
}
public void Dispose()
{
Importer?.DisposeAsync().AsTask().Wait();
}
private void NotificationClicked(INotificationClickArgs args)
{
IsOpen = true;
args.Notification.DismissNow();
}
private void LogAndNotify()
{
Plugin.Log.Info($"[Migration] Checked migration eligibility: {Eligibility.Status} - '{Eligibility.AdditionalIneligibilityInfo}'");
switch (Eligibility.Status)
{
case LegacyMessageImporterEligibilityStatus.Eligible:
{
var notification = Plugin.Notification.AddNotification(new Notification
{
// The user needs to dismiss this for it to go away.
Type = NotificationType.Info,
InitialDuration = TimeSpan.FromHours(24),
Title = "Chat2 Migration",
Content = "Import messages from old database into new database?\nClick for more information.",
Minimized = false,
});
notification.Click += NotificationClicked;
break;
}
case LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed:
{
var notification = Plugin.Notification.AddNotification(new Notification
{
Type = NotificationType.Warning,
InitialDuration = TimeSpan.FromMinutes(1),
Title = "Chat2 Migration",
Content = "Migration is not possible because the old database could not be opened.\nClick for more information.",
Minimized = false,
});
notification.Click += NotificationClicked;
break;
}
}
}
public override void Draw()
{
if (Importer != null)
{
DrawImportStatus();
return;
}
if (Eligibility.Status == LegacyMessageImporterEligibilityStatus.Eligible)
DrawEligible();
else
DrawIneligible();
}
private void DrawEligible()
{
ImGui.TextWrapped("Import database messages from legacy LiteDB database to SQLite database?");
ImGui.TextWrapped($"Message count: {Eligibility.MessageCount:N0}");
ImGui.TextWrapped($"Database size: {StringUtil.BytesToString(Eligibility.DatabaseSizeBytes)}");
ImGui.Spacing();
var colorNormal = new Vector4(0.0f, 0.70f, 0.0f, 1.0f);
var colorHovered = new Vector4(0.059f, 0.49f, 0.0f, 1.0f);
using (ImRaii.PushColor(ImGuiCol.Button, colorNormal))
using (ImRaii.PushColor(ImGuiCol.ButtonHovered, colorHovered))
{
if (ImGui.Button("Yes, import messages"))
{
// Next draw call will run DrawImportStatus().
Importer = Eligibility.StartImport(_store, plugin: Plugin);
return;
}
}
ImGui.SameLine();
if (ImGuiUtil.CtrlShiftButtonColored("No, do not import messages", "Ctrl+Shift: renames old database to avoid prompting again"))
{
Eligibility.RenameOldDatabase();
IsOpen = false;
}
}
private void DrawIneligible()
{
ImGui.TextWrapped("Your legacy LiteDB database is not eligible for import:");
switch (Eligibility.Status)
{
case LegacyMessageImporterEligibilityStatus.IneligibleOriginalDbNotExists:
ImGui.TextWrapped("The old database could not be found.");
break;
case LegacyMessageImporterEligibilityStatus.IneligibleMigrationDbExists:
ImGui.TextWrapped("The migration process was already started.");
break;
case LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed:
ImGui.TextWrapped("The old database could not be opened.");
break;
case LegacyMessageImporterEligibilityStatus.IneligibleNoMessages:
ImGui.TextWrapped("The old database contains no messages.");
break;
case LegacyMessageImporterEligibilityStatus.Eligible:
default:
throw new ArgumentOutOfRangeException();
}
if (!string.IsNullOrWhiteSpace(Eligibility.AdditionalIneligibilityInfo))
ImGui.TextWrapped(Eligibility.AdditionalIneligibilityInfo);
// LiteDB failures notify the user, so give them a chance to rename the
// database to avoid prompting again.
if (Eligibility.Status == LegacyMessageImporterEligibilityStatus.IneligibleLiteDbFailed)
{
if (ImGuiUtil.CtrlShiftButton("Rename old database", "Ctrl+Shift: rename old database to avoid import prompt in the future"))
{
if (Eligibility.RenameOldDatabase())
WrapperUtil.AddNotification("Successfully renamed the old database.", NotificationType.Success);
else
WrapperUtil.AddNotification("Rename failed, please check /xllog for more information.", NotificationType.Error);
}
}
}
private void DrawImportStatus()
{
if (Importer == null)
return;
if (Importer.ImportComplete != null)
{
ImGui.TextUnformatted($"Completed migration in {Duration(Importer.ImportStart, Importer.ImportComplete.Value):g}");
ImGui.TextUnformatted($"Successfully imported: {Importer.SuccessfulMessages:N0} messages");
ImGui.TextUnformatted($"Failed to import: {Importer.FailedMessages:N0} messages");
ImGui.TextUnformatted($"Unaccounted for: {Importer.RemainingMessages:N0}");
ImGui.TextUnformatted("See logs for more details: /xllog");
ImGui.Spacing();
if (ImGui.Button("Finish"))
IsOpen = false;
return;
}
ImGui.TextUnformatted($"Importing messages ... {Importer.Progress:P}");
ImGuiHelpers.ScaledDummy(10.0f);
ImGui.TextUnformatted($"Duration: {Duration(Importer.ImportStart, Environment.TickCount64):g}");
ImGui.TextUnformatted($"Progress: {Importer.ProcessedMessages:N0}/{Importer.ImportCount:N0} messages ({Importer.FailedMessages:N0} failed)");
ImGuiHelpers.ScaledDummy(10.0f);
var width = ImGui.GetContentRegionAvail().X / 2;
ImGui.AlignTextToFramePadding();
ImGui.TextUnformatted("Import speed:");
ImGui.SameLine();
ImGui.SetNextItemWidth(width);
ImGui.SliderInt("##speedSlider", ref Importer.MaxMessageRate, 1, 10000, "%d msgs/sec", ImGuiSliderFlags.AlwaysClamp);
ImGui.TextUnformatted($"Current speed: {Importer.CurrentMessageRate:N0} msgs/sec");
ImGui.TextUnformatted($"Estimated time remaining: {Importer.EstimatedTimeRemaining:g}");
ImGui.TextUnformatted("See logs for more details: /xllog");
ImGuiHelpers.ScaledDummy(10.0f);
ImGui.ProgressBar(Importer.Progress, new Vector2(-1, 0), $"{Importer.Progress:P}");
ImGui.Spacing();
if (ImGuiUtil.CtrlShiftButton("Cancel import", "Ctrl+Shift: cancel import and close window"))
{
Task.Run(async () =>
{
await Importer.DisposeAsync();
Importer = null;
Eligibility = LegacyMessageImporterEligibility.CheckEligibility();
});
}
}
private static TimeSpan Duration(long startTicks, long endTicks)
{
return endTicks < startTicks ? TimeSpan.Zero : TimeSpan.FromMilliseconds(endTicks - startTicks);
}
}
ChatTwo/Ui/SettingsTabs/Database.cs
+5 -14
View File
@@ -54,19 +54,7 @@ internal sealed class Database : ISettingsTab
var old = new FileInfo(Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat.db"));
var migratedOld = new FileInfo(Path.Join(Plugin.Interface.ConfigDirectory.FullName, "chat-litedb.db"));
if (old.Exists)
{
ImGui.TextUnformatted(Language.Options_Database_Old_Heading);
ImGui.Spacing();
if (ImGui.Button(Language.Options_Database_Old_Migration))
Plugin.LegacyMessageImporterWindow.IsOpen = true;
ImGui.Spacing();
ImGui.Separator();
ImGui.Spacing();
}
else if (migratedOld.Exists)
if (old.Exists || migratedOld.Exists)
{
ImGui.TextUnformatted(Language.Options_Database_Old_Heading);
ImGui.Spacing();
@@ -75,7 +63,10 @@ internal sealed class Database : ISettingsTab
{
try
{
migratedOld.Delete();
if (old.Exists)
old.Delete();
else
migratedOld.Delete();
WrapperUtil.AddNotification(Language.Options_Database_Old_Delete_Success, NotificationType.Success);
}
catch (Exception e)
ChatTwo/packages.lock.json
-6
View File
@@ -8,12 +8,6 @@
"resolved": "2.1.13",
"contentHash": "rMN1omGe8536f4xLMvx9NwfvpAc9YFFfeXJ1t4P4PE6Gu8WCIoFliR1sh07hM+bfODmesk/dvMbji7vNI+B/pQ=="
},
"LiteDB": {
"type": "Direct",
"requested": "[5.0.19, )",
"resolved": "5.0.19",
"contentHash": "O8XPBptE4SygW2SN6skqg/VDVTrjpJ0p6+i7Cp8x8razbu2ORLSd6ep3JdhDIdL6h57Bcxv2BuVaN70IKpXI0Q=="
},
"MessagePack": {
"type": "Direct",
"requested": "[2.5.140, )",