JonKazama-Hellion/HellionChat
1
0
Fork 0
Code Issues Pull Requests Actions Releases 28 Activity

remove message limit text as it doesn't exist for database itself

Browse Source
This commit is contained in:
Infi
2024-04-21 01:59:04 +02:00
parent 42e2fe12a4
commit ed5cedefd2
4 changed files with 93 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ChatTwo/MessageStore.cs
+90 -70
View File
@@ -13,85 +13,98 @@ using Encoding = System.Text.Encoding;
namespace ChatTwo; namespace ChatTwo;
internal static class DbExtensions { internal static class DbExtensions
internal static void Execute(this DbConnection conn, string sql) { {
internal static void Execute(this DbConnection conn, string sql)
{
using var cmd = conn.CreateCommand(); using var cmd = conn.CreateCommand();
cmd.CommandText = sql; cmd.CommandText = sql;
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
} }
} }
internal enum PayloadMessagePackType : byte { internal enum PayloadMessagePackType : byte
{
Achievement, Achievement,
PartyFinder, PartyFinder,
Uri, Uri,
Other = 255, Other = 255,
} }
public class PayloadMessagePackFormatter : IMessagePackFormatter<Payload?> { public class PayloadMessagePackFormatter : IMessagePackFormatter<Payload?>
public void Serialize(ref MessagePackWriter writer, Payload? value, MessagePackSerializerOptions options) { {
if (value == null) { public void Serialize(ref MessagePackWriter writer, Payload? value, MessagePackSerializerOptions options)
{
if (value == null)
{
writer.WriteNil(); writer.WriteNil();
return; return;
} }
writer.WriteArrayHeader(2); writer.WriteArrayHeader(2);
switch (value) { switch (value)
{
case AchievementPayload achievementPayload: case AchievementPayload achievementPayload:
writer.WriteUInt8((byte) PayloadMessagePackType.Achievement); writer.WriteUInt8((byte)PayloadMessagePackType.Achievement);
writer.WriteUInt32(achievementPayload.Id); writer.WriteUInt32(achievementPayload.Id);
break; break;
case PartyFinderPayload partyFinderPayload: case PartyFinderPayload partyFinderPayload:
writer.WriteUInt8((byte) PayloadMessagePackType.PartyFinder); writer.WriteUInt8((byte)PayloadMessagePackType.PartyFinder);
writer.WriteUInt32(partyFinderPayload.Id); writer.WriteUInt32(partyFinderPayload.Id);
break; break;
case UriPayload uriPayload: case UriPayload uriPayload:
writer.WriteUInt8((byte) PayloadMessagePackType.Uri); writer.WriteUInt8((byte)PayloadMessagePackType.Uri);
writer.WriteString(Encoding.UTF8.GetBytes(uriPayload.Uri.ToString())); writer.WriteString(Encoding.UTF8.GetBytes(uriPayload.Uri.ToString()));
break; break;
default: default:
writer.WriteUInt8((byte) PayloadMessagePackType.Other); writer.WriteUInt8((byte)PayloadMessagePackType.Other);
writer.Write(value.Encode()); writer.Write(value.Encode());
break; break;
} }
} }
public Payload? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { public Payload? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
{
if (reader.TryReadNil()) if (reader.TryReadNil())
return null; return null;
if (reader.ReadArrayHeader() != 2) if (reader.ReadArrayHeader() != 2)
throw new InvalidOperationException("Invalid array count for Payload object"); throw new InvalidOperationException("Invalid array count for Payload object");
var type = (PayloadMessagePackType) reader.ReadByte(); var type = (PayloadMessagePackType)reader.ReadByte();
switch (type) { switch (type)
case PayloadMessagePackType.Achievement: {
return new AchievementPayload(reader.ReadUInt32()); case PayloadMessagePackType.Achievement:
case PayloadMessagePackType.PartyFinder: return new AchievementPayload(reader.ReadUInt32());
return new PartyFinderPayload(reader.ReadUInt32()); case PayloadMessagePackType.PartyFinder:
case PayloadMessagePackType.Uri: return new PartyFinderPayload(reader.ReadUInt32());
return new UriPayload(new Uri(reader.ReadString() ?? "")); case PayloadMessagePackType.Uri:
case PayloadMessagePackType.Other: return new UriPayload(new Uri(reader.ReadString() ?? ""));
default: case PayloadMessagePackType.Other:
var bytes = reader.ReadBytes() ?? new ReadOnlySequence<byte>(); default:
var binReader = new BinaryReader(new MemoryStream(bytes.ToArray())); var bytes = reader.ReadBytes() ?? new ReadOnlySequence<byte>();
return Payload.Decode(binReader); var binReader = new BinaryReader(new MemoryStream(bytes.ToArray()));
return Payload.Decode(binReader);
} }
} }
} }
public class SeStringMessagePackFormatter : IMessagePackFormatter<SeString> { public class SeStringMessagePackFormatter : IMessagePackFormatter<SeString>
public void Serialize(ref MessagePackWriter writer, SeString value, MessagePackSerializerOptions options) { {
public void Serialize(ref MessagePackWriter writer, SeString value, MessagePackSerializerOptions options)
{
options.Resolver.GetFormatter<List<Payload>>()!.Serialize(ref writer, value.Payloads, options); options.Resolver.GetFormatter<List<Payload>>()!.Serialize(ref writer, value.Payloads, options);
} }
public SeString Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { public SeString Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
{
return new SeString(options.Resolver.GetFormatter<List<Payload>>()!.Deserialize(ref reader, options)); return new SeString(options.Resolver.GetFormatter<List<Payload>>()!.Deserialize(ref reader, options));
} }
} }
internal class MessageStore : IDisposable { internal class MessageStore : IDisposable
internal const int MessageQueryLimit = 10_000; {
private const int MessageQueryLimit = 10_000;
private string DbPath { get; } private string DbPath { get; }
@@ -99,19 +112,20 @@ internal class MessageStore : IDisposable {
internal static readonly MessagePackSerializerOptions MsgPackOptions = MessagePackSerializerOptions.Standard internal static readonly MessagePackSerializerOptions MsgPackOptions = MessagePackSerializerOptions.Standard
.WithResolver(CompositeResolver.Create( .WithResolver(CompositeResolver.Create(
new IMessagePackFormatter[] { new IMessagePackFormatter[] { new PayloadMessagePackFormatter(), new SeStringMessagePackFormatter(), },
new PayloadMessagePackFormatter(), new IFormatterResolver[] { StandardResolver.Instance }
new SeStringMessagePackFormatter(), )
}, );
new IFormatterResolver[] { StandardResolver.Instance }));
internal MessageStore(string dbPath) { internal MessageStore(string dbPath)
{
DbPath = dbPath; DbPath = dbPath;
Connection = Connect(); Connection = Connect();
Migrate(); Migrate();
} }
public void Dispose() { public void Dispose()
{
Connection.Close(); Connection.Close();
Connection.Dispose(); Connection.Dispose();
// Closing the connection doesn't immediately release the file. // Closing the connection doesn't immediately release the file.
@@ -119,13 +133,16 @@ internal class MessageStore : IDisposable {
GC.WaitForPendingFinalizers(); GC.WaitForPendingFinalizers();
} }
private SqliteConnection Connect() { private SqliteConnection Connect()
var uriBuilder = new SqliteConnectionStringBuilder { {
var uriBuilder = new SqliteConnectionStringBuilder
{
DataSource = DbPath, DataSource = DbPath,
DefaultTimeout = 5, DefaultTimeout = 5,
Pooling = false, Pooling = false,
Mode = SqliteOpenMode.ReadWriteCreate, Mode = SqliteOpenMode.ReadWriteCreate,
}; };
var conn = new SqliteConnection(uriBuilder.ToString()); var conn = new SqliteConnection(uriBuilder.ToString());
conn.Open(); conn.Open();
conn.Execute(@"PRAGMA journal_mode=WAL;"); conn.Execute(@"PRAGMA journal_mode=WAL;");
@@ -135,7 +152,8 @@ internal class MessageStore : IDisposable {
return conn; return conn;
} }
private void Migrate() { private void Migrate()
{
// TODO: this should be improved/swapped out for a library at some // TODO: this should be improved/swapped out for a library at some
// point. // point.
Connection.Execute(@" Connection.Execute(@"
@@ -158,18 +176,21 @@ internal class MessageStore : IDisposable {
"); ");
} }
internal void Reconnect() { internal void Reconnect()
{
Connection.Close(); Connection.Close();
Connection.Dispose(); Connection.Dispose();
Connection = Connect(); Connection = Connect();
} }
internal void ClearMessages() { internal void ClearMessages()
{
Connection.Execute("DELETE FROM messages;"); Connection.Execute("DELETE FROM messages;");
PerformMaintenance(); PerformMaintenance();
} }
internal void PerformMaintenance() { internal void PerformMaintenance()
{
Connection.Execute(@" Connection.Execute(@"
VACUUM; VACUUM;
REINDEX messages; REINDEX messages;
@@ -177,15 +198,9 @@ internal class MessageStore : IDisposable {
"); ");
} }
internal long DatabaseSize() {
return !File.Exists(DbPath) ? 0 : new FileInfo(DbPath).Length;
}
private string LogPath => DbPath + "-wal"; private string LogPath => DbPath + "-wal";
internal long DatabaseSize() => !File.Exists(DbPath) ? 0 : new FileInfo(DbPath).Length;
internal long DatabaseLogSize() { internal long DatabaseLogSize() => !File.Exists(LogPath) ? 0 : new FileInfo(LogPath).Length;
return !File.Exists(LogPath) ? 0 : new FileInfo(LogPath).Length;
}
internal int MessageCount() internal int MessageCount()
{ {
@@ -194,7 +209,8 @@ internal class MessageStore : IDisposable {
return Convert.ToInt32(cmd.ExecuteScalar()); return Convert.ToInt32(cmd.ExecuteScalar());
} }
internal void UpsertMessage(Message message) { internal void UpsertMessage(Message message)
{
var cmd = Connection.CreateCommand(); var cmd = Connection.CreateCommand();
cmd.CommandText = @" cmd.CommandText = @"
INSERT INTO messages ( INSERT INTO messages (
@@ -256,7 +272,8 @@ internal class MessageStore : IDisposable {
/// <param name="receiver">The receiver content ID to filter by. If null, no filtering is performed.</param> /// <param name="receiver">The receiver content ID to filter by. If null, no filtering is performed.</param>
/// <param name="since">Only show messages since this date. If null, no filtering is performed.</param> /// <param name="since">Only show messages since this date. If null, no filtering is performed.</param>
/// <param name="count">The amount to return. Defaults to 10,000.</param> /// <param name="count">The amount to return. Defaults to 10,000.</param>
internal MessageEnumerator GetMostRecentMessages(ulong? receiver = null, DateTimeOffset? since = null, int count = MessageQueryLimit) { internal MessageEnumerator GetMostRecentMessages(ulong? receiver = null, DateTimeOffset? since = null, int count = MessageQueryLimit)
{
var whereClauses = new List<string>(); var whereClauses = new List<string>();
if (receiver != null) if (receiver != null)
whereClauses.Add("Receiver = $Receiver"); whereClauses.Add("Receiver = $Receiver");
@@ -303,17 +320,21 @@ internal class MessageStore : IDisposable {
} }
} }
internal class MessageEnumerator(DbDataReader reader) : IEnumerable<Message> { internal class MessageEnumerator(DbDataReader reader) : IEnumerable<Message>
{
private const int MaxErrorLogs = 10; private const int MaxErrorLogs = 10;
private int _errorCount; private int _errorCount;
public bool DidError => _errorCount > 0; public bool DidError => _errorCount > 0;
public IEnumerator<Message> GetEnumerator() { public IEnumerator<Message> GetEnumerator()
while (reader.Read()) { {
while (reader.Read())
{
var id = Guid.Empty; var id = Guid.Empty;
Message msg; Message msg;
try { try
{
id = reader.GetGuid(0); id = reader.GetGuid(0);
msg = new Message( msg = new Message(
id, id,
@@ -321,36 +342,35 @@ internal class MessageEnumerator(DbDataReader reader) : IEnumerable<Message> {
(ulong)reader.GetInt64(2), (ulong)reader.GetInt64(2),
DateTimeOffset.FromUnixTimeMilliseconds(reader.GetInt64(3)), DateTimeOffset.FromUnixTimeMilliseconds(reader.GetInt64(3)),
new ChatCode((ushort)reader.GetInt32(4)), new ChatCode((ushort)reader.GetInt32(4)),
MessagePackSerializer.Deserialize<List<Chunk>>(reader.GetFieldValue<byte[]>(5), MessagePackSerializer.Deserialize<List<Chunk>>(reader.GetFieldValue<byte[]>(5), MessageStore.MsgPackOptions),
MessageStore.MsgPackOptions), MessagePackSerializer.Deserialize<List<Chunk>>(reader.GetFieldValue<byte[]>(6), MessageStore.MsgPackOptions),
MessagePackSerializer.Deserialize<List<Chunk>>(reader.GetFieldValue<byte[]>(6), MessagePackSerializer.Deserialize<SeString>(reader.GetFieldValue<byte[]>(7), MessageStore.MsgPackOptions),
MessageStore.MsgPackOptions), MessagePackSerializer.Deserialize<SeString>(reader.GetFieldValue<byte[]>(8), MessageStore.MsgPackOptions),
MessagePackSerializer.Deserialize<SeString>(reader.GetFieldValue<byte[]>(7),
MessageStore.MsgPackOptions),
MessagePackSerializer.Deserialize<SeString>(reader.GetFieldValue<byte[]>(8),
MessageStore.MsgPackOptions),
new SortCode((uint)reader.GetInt32(9)), new SortCode((uint)reader.GetInt32(9)),
reader.GetGuid(10) reader.GetGuid(10)
); );
} catch (Exception e) { }
catch (Exception e)
{
if (_errorCount < MaxErrorLogs) if (_errorCount < MaxErrorLogs)
Plugin.Log.Error($"Exception while reading message '{id}' from database: {e}"); Plugin.Log.Error($"Exception while reading message '{id}' from database: {e}");
_errorCount++; _errorCount++;
if (_errorCount == MaxErrorLogs) if (_errorCount == MaxErrorLogs)
Plugin.Log.Error("Further parsing errors will not be logged"); Plugin.Log.Error("Further parsing errors will not be logged");
#if DEBUG #if DEBUG
throw; throw;
#else #else
continue; continue;
#endif #endif
} }
yield return msg; yield return msg;
} }
} }
IEnumerator IEnumerable.GetEnumerator() { IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator(); return GetEnumerator();
} }
} }
ChatTwo/Resources/Language.Designer.cs
Generated
+1 -1
View File
@@ -1761,7 +1761,7 @@ namespace ChatTwo.Resources {
} }
/// <summary> /// <summary>
/// Looks up a localized string similar to Stored messages: {0:N0}/{1:N0}. /// Looks up a localized string similar to Stored messages: {0:N0}.
/// </summary> /// </summary>
internal static string Options_Database_Metadata_MessageCount { internal static string Options_Database_Metadata_MessageCount {
get { get {
ChatTwo/Resources/Language.resx
+1 -1
View File
@@ -965,7 +965,7 @@
<value>Log size: {0}</value> <value>Log size: {0}</value>
</data> </data>
<data name="Options_Database_Metadata_MessageCount"> <data name="Options_Database_Metadata_MessageCount">
<value>Stored messages: {0:N0}/{1:N0}</value> <value>Stored messages: {0:N0}</value>
</data> </data>
<data name="Options_Database_Metadata_Path"> <data name="Options_Database_Metadata_Path">
<value>Path: {0}</value> <value>Path: {0}</value>
ChatTwo/Ui/SettingsTabs/Database.cs
+1 -1
View File
@@ -117,7 +117,7 @@ internal sealed class Database : ISettingsTab
if (ImGui.IsItemHovered()) if (ImGui.IsItemHovered())
ImGui.SetTooltip(DatabaseLogSize.ToString("N0") + "B"); ImGui.SetTooltip(DatabaseLogSize.ToString("N0") + "B");
ImGuiUtil.HelpText(string.Format(Language.Options_Database_Metadata_MessageCount, DatabaseMessageCount, MessageStore.MessageQueryLimit)); ImGuiUtil.HelpText(string.Format(Language.Options_Database_Metadata_MessageCount, DatabaseMessageCount));
if (ImGuiUtil.CtrlShiftButton(Language.Options_ClearDatabase_Button, Language.Options_ClearDatabase_Tooltip)) if (ImGuiUtil.CtrlShiftButton(Language.Options_ClearDatabase_Button, Language.Options_ClearDatabase_Tooltip))
{ {