Add retroactive cleanup for the existing database

The privacy filter only catches new messages. Two new MessageStore
methods support a one-shot retroactive sweep: GetMessageCountsByChatType
returns a (ChatType, count) snapshot so the UI can preview the impact,
and CleanupRetainOnly hard-deletes everything outside the supplied
allowlist and runs VACUUM to reclaim disk space.

The Privacy tab gains a new section with a refresh-preview button, a
keep/delete summary, a per-channel breakdown tree, and a Ctrl+Shift
confirm. The cleanup runs on a background thread so a 800+ MB VACUUM
does not block the settings UI; tabs are rebuilt via the framework
thread once the delete finishes. The cleanup deliberately uses the
saved Plugin.Config whitelist (not unsaved Mutable edits) so it stays
consistent with the prospective filter.
This commit is contained in:
2026-05-01 18:34:28 +02:00
parent 465aadbb1a
commit 2401ea5864
3 changed files with 196 additions and 2 deletions
+47
View File
@@ -310,6 +310,53 @@ internal class MessageStore : IDisposable
PerformMaintenance();
}
/// <summary>
/// Returns a (ChatType, count) snapshot over non-deleted messages.
/// Used by the Privacy tab to preview the impact of a retroactive
/// cleanup before the user confirms.
/// </summary>
internal Dictionary<int, long> GetMessageCountsByChatType()
{
var result = new Dictionary<int, long>();
using var cmd = Connection.CreateCommand();
cmd.CommandText = "SELECT ChatType, COUNT(*) FROM messages WHERE deleted = false GROUP BY ChatType;";
cmd.CommandTimeout = 120;
using var reader = cmd.ExecuteReader();
while (reader.Read())
{
var chatType = reader.GetInt32(0);
var count = reader.GetInt64(1);
result[chatType] = count;
}
return result;
}
/// <summary>
/// Hard-deletes every message whose ChatType is not in the supplied
/// allowlist, then VACUUMs the database to reclaim disk space.
/// Returns the number of rows deleted.
/// </summary>
internal long CleanupRetainOnly(IReadOnlyCollection<int> allowedTypes)
{
if (allowedTypes.Count == 0)
{
// Defensive: refuse a "delete everything" disguised as a filter.
// Use ClearMessages() if a full wipe is actually intended.
throw new InvalidOperationException("CleanupRetainOnly requires at least one allowed ChatType. Use ClearMessages for a full wipe.");
}
var inList = string.Join(",", allowedTypes);
long deleted;
using (var cmd = Connection.CreateCommand())
{
cmd.CommandText = $"DELETE FROM messages WHERE ChatType NOT IN ({inList});";
cmd.CommandTimeout = 600;
deleted = cmd.ExecuteNonQuery();
}
PerformMaintenance();
return deleted;
}
internal void PerformMaintenance()
{
Connection.Execute(@"
+1 -1
View File
@@ -40,7 +40,7 @@ public sealed class SettingsWindow : Window
new Fonts(Mutable),
new ChatColours(Plugin, Mutable),
new Tabs(Plugin, Mutable),
new SettingsTabs.Privacy(Mutable),
new SettingsTabs.Privacy(Plugin, Mutable),
new Database(Plugin, Mutable),
new Webinterface(Plugin, Mutable),
new Miscellaneous(Mutable),
+148 -1
View File
@@ -1,6 +1,7 @@
using ChatTwo.Code;
using ChatTwo.Privacy;
using ChatTwo.Util;
using Dalamud.Interface.ImGuiNotification;
using Dalamud.Interface.Utility.Raii;
using Dalamud.Bindings.ImGui;
@@ -8,12 +9,14 @@ namespace ChatTwo.Ui.SettingsTabs;
internal sealed class Privacy : ISettingsTab
{
private Plugin Plugin { get; }
private Configuration Mutable { get; }
public string Name => "Privacy###tabs-privacy";
internal Privacy(Configuration mutable)
internal Privacy(Plugin plugin, Configuration mutable)
{
Plugin = plugin;
Mutable = mutable;
}
@@ -44,6 +47,14 @@ internal sealed class Privacy : ISettingsTab
]),
];
// Cleanup preview state. Held in the tab so the user can refresh and
// inspect before confirming. Resets when the tab is reopened (acceptable —
// a stale preview against a freshly-edited whitelist would be misleading).
private Dictionary<int, long>? CleanupCounts;
private long CleanupKeepCount;
private long CleanupDeleteCount;
private bool CleanupRunning;
public void Draw(bool changed)
{
ImGuiUtil.OptionCheckbox(
@@ -115,5 +126,141 @@ internal sealed class Privacy : ISettingsTab
"Failsafe for ChatTypes added by future FFXIV patches that this plugin does not yet know about. " +
"Default OFF (Privacy-First). Turn ON if you want a complete log including future channels.");
}
ImGui.Spacing();
ImGui.Separator();
ImGui.Spacing();
DrawCleanupSection();
}
private void DrawCleanupSection()
{
ImGui.TextUnformatted("Apply filter to existing database");
using (ImRaii.PushIndent(ImGui.GetStyle().IndentSpacing, false))
{
ImGuiUtil.HelpText(
"The privacy filter only applies to new messages. " +
"Use the cleanup below to retroactively remove already-stored messages " +
"that don't match your saved whitelist.");
ImGuiUtil.HelpText(
"Cleanup uses your SAVED whitelist (Plugin.Config), not unsaved edits above. " +
"Click Save first if you want to apply your current edits.");
ImGui.Spacing();
using (ImRaii.Disabled(CleanupRunning))
{
if (ImGui.Button("Refresh preview"))
RefreshCleanupPreview();
}
if (CleanupCounts is null)
{
ImGuiUtil.HelpText("No preview yet. Click Refresh to compute the impact.");
return;
}
ImGui.Spacing();
ImGuiUtil.HelpText($"Total stored messages: {CleanupKeepCount + CleanupDeleteCount:N0}");
ImGuiUtil.HelpText($"Will keep: {CleanupKeepCount:N0}");
ImGuiUtil.HelpText($"Will delete: {CleanupDeleteCount:N0}");
using (var tree = ImRaii.TreeNode("Per-channel breakdown"))
{
if (tree.Success)
{
using (ImRaii.PushIndent(ImGui.GetStyle().IndentSpacing, false))
foreach (var (chatType, count) in CleanupCounts.OrderByDescending(p => p.Value))
{
var name = Enum.IsDefined(typeof(ChatType), (ushort)chatType)
? ((ChatType)(ushort)chatType).ToString()
: $"Unknown({chatType})";
var keeps = WouldBeKept(chatType);
var marker = keeps ? "[KEEP] " : "[DELETE]";
ImGuiUtil.HelpText($"{marker} {name} — {count:N0}");
}
}
}
ImGui.Spacing();
using (ImRaii.Disabled(CleanupRunning || CleanupDeleteCount == 0))
{
if (ImGuiUtil.CtrlShiftButton("Apply current filter to database",
$"Ctrl+Shift: Hard-deletes {CleanupDeleteCount:N0} messages, then runs VACUUM. Cannot be undone."))
StartCleanup();
}
if (CleanupRunning)
ImGuiUtil.HelpText("Cleanup running in background…");
}
}
private bool WouldBeKept(int chatType)
{
if (!Plugin.Config.PrivacyFilterEnabled)
return true;
if (Plugin.Config.PrivacyPersistChannels.Contains((ChatType)(ushort)chatType))
return true;
return Plugin.Config.PrivacyPersistUnknownChannels;
}
private void RefreshCleanupPreview()
{
try
{
CleanupCounts = Plugin.MessageManager.Store.GetMessageCountsByChatType();
CleanupKeepCount = 0;
CleanupDeleteCount = 0;
foreach (var (chatType, count) in CleanupCounts)
{
if (WouldBeKept(chatType))
CleanupKeepCount += count;
else
CleanupDeleteCount += count;
}
}
catch (Exception e)
{
Plugin.Log.Error(e, "Failed to compute cleanup preview");
WrapperUtil.AddNotification("Failed to compute cleanup preview, see /xllog", NotificationType.Error);
}
}
private void StartCleanup()
{
if (CleanupRunning)
return;
CleanupRunning = true;
var allowed = Plugin.Config.PrivacyPersistChannels.Select(t => (int)(ushort)t).ToList();
new Thread(() =>
{
try
{
var deleted = Plugin.MessageManager.Store.CleanupRetainOnly(allowed);
Plugin.Log.Information($"Privacy cleanup: deleted {deleted} messages");
Plugin.Framework.Run(() =>
{
Plugin.MessageManager.ClearAllTabs();
Plugin.MessageManager.FilterAllTabsAsync();
}).Wait();
WrapperUtil.AddNotification($"Privacy cleanup complete: {deleted:N0} messages removed.", NotificationType.Success);
}
catch (Exception e)
{
Plugin.Log.Error(e, "Privacy cleanup failed");
WrapperUtil.AddNotification("Privacy cleanup failed, see /xllog", NotificationType.Error);
}
finally
{
CleanupRunning = false;
CleanupCounts = null;
}
}).Start();
}
}