Add message export for GDPR Art. 15 right of access

The privacy story is incomplete without a way to actually hand the
data over. New Export section in the Privacy tab streams matching
messages to a Markdown, JSON or CSV file using Dalamud's file
dialog and a background thread, so the settings UI stays
responsive even when the export crawls a 150k-message archive.

MessageStore.StreamForExport returns a MessageEnumerator over
non-deleted rows filtered by ChatType list and date range, sorted
ascending. MessageExporter.ExportToFile takes that enumerator,
optionally narrows by SenderSource.TextValue substring (case-
insensitive), and writes one of three formats:

  Markdown — human-readable, day headers, [HH:mm] ChatType Sender:
  prefix per line, trailing total.

  JSON — single object with metadata (filter snapshot, exported_at,
  plugin name) and a messages array carrying id, ISO-8601 date,
  numeric and named ChatType, source/target kinds, receiver,
  content_id, sender plaintext, content plaintext.

  CSV — header line plus quoted-when-needed rows for spreadsheet
  ingestion.

Sender plaintext, channel filter, date range and format are
exposed as form fields above the Export button. Empty channel
selection means "all stored channels", a 0-day range means "no
time limit". Result count and target path are reported via
WrapperUtil notifications.
This commit is contained in:
2026-05-01 20:41:58 +02:00
parent 33cfc7effa
commit 68a6910c53
6 changed files with 534 additions and 0 deletions
+229
View File
@@ -0,0 +1,229 @@
using System.Globalization;
using System.Text;
using ChatTwo.Code;
namespace ChatTwo.Export;
internal enum ExportFormat
{
Markdown,
Json,
Csv,
}
internal static class ExportFormatExt
{
internal static string Extension(this ExportFormat fmt) => fmt switch
{
ExportFormat.Markdown => "md",
ExportFormat.Json => "json",
ExportFormat.Csv => "csv",
_ => "txt",
};
internal static string Filter(this ExportFormat fmt) => fmt switch
{
ExportFormat.Markdown => ".md",
ExportFormat.Json => ".json",
ExportFormat.Csv => ".csv",
_ => ".txt",
};
}
/// <summary>
/// Serializes message snapshots into Markdown, JSON, or CSV. The caller is
/// expected to filter the input enumerable; this class only handles
/// formatting and writes to the supplied path. Sender substring filtering
/// happens here because it requires deserialized SeString.TextValue.
/// </summary>
internal static class MessageExporter
{
internal record FilterDescription(
IReadOnlyCollection<int>? ChatTypes,
DateTimeOffset? From,
DateTimeOffset? To,
string? SenderSubstring);
internal static int ExportToFile(
string path,
ExportFormat format,
IEnumerable<Message> messages,
FilterDescription filter)
{
var matching = filter.SenderSubstring is { Length: > 0 } needle
? messages.Where(m => MatchesSender(m, needle))
: messages;
using var writer = new StreamWriter(path, append: false, encoding: Encoding.UTF8);
return format switch
{
ExportFormat.Markdown => WriteMarkdown(writer, matching, filter),
ExportFormat.Json => WriteJson(writer, matching, filter),
ExportFormat.Csv => WriteCsv(writer, matching, filter),
_ => throw new ArgumentOutOfRangeException(nameof(format), format, null),
};
}
private static bool MatchesSender(Message m, string needle)
=> m.SenderSource.TextValue.Contains(needle, StringComparison.OrdinalIgnoreCase);
private static int WriteMarkdown(StreamWriter w, IEnumerable<Message> messages, FilterDescription filter)
{
w.WriteLine("# Hellion Chat Export");
w.WriteLine();
w.WriteLine($"Generated: {DateTimeOffset.Now:yyyy-MM-dd HH:mm zzz}");
WriteFilterSummaryMarkdown(w, filter);
w.WriteLine();
DateTimeOffset? lastDate = null;
var count = 0;
foreach (var m in messages)
{
count++;
var localDate = m.Date.ToLocalTime();
if (lastDate is null || localDate.Date != lastDate.Value.Date)
{
w.WriteLine();
w.WriteLine($"## {localDate:yyyy-MM-dd}");
w.WriteLine();
lastDate = localDate;
}
var chatType = (ChatType)(ushort)m.Code.Type;
var sender = m.SenderSource.TextValue.Trim().Trim('<', '>', '[', ']', ':').Trim();
var content = m.ContentSource.TextValue;
if (string.IsNullOrEmpty(sender))
w.WriteLine($"**[{localDate:HH:mm}] {chatType}:** {content}");
else
w.WriteLine($"**[{localDate:HH:mm}] {chatType} {sender}:** {content}");
}
w.WriteLine();
w.WriteLine($"---");
w.WriteLine($"Total messages: {count}");
return count;
}
private static void WriteFilterSummaryMarkdown(StreamWriter w, FilterDescription filter)
{
if (filter.ChatTypes is { Count: > 0 })
w.WriteLine($"ChatTypes: {string.Join(", ", filter.ChatTypes.Select(t => $"{(ChatType)(ushort)t}({t})"))}");
if (filter.From is not null)
w.WriteLine($"From: {filter.From.Value.ToLocalTime():yyyy-MM-dd HH:mm}");
if (filter.To is not null)
w.WriteLine($"To: {filter.To.Value.ToLocalTime():yyyy-MM-dd HH:mm}");
if (filter.SenderSubstring is { Length: > 0 })
w.WriteLine($"Sender contains: \"{filter.SenderSubstring}\"");
}
private static int WriteJson(StreamWriter w, IEnumerable<Message> messages, FilterDescription filter)
{
// Manual JSON to avoid pulling in System.Text.Json policy choices.
// Output is a single object with metadata and an array of messages.
w.Write("{\n \"exported_at\": \"");
w.Write(DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture));
w.Write("\",\n \"plugin\": \"Hellion Chat\",\n");
w.Write(" \"filter\": {\n");
w.Write(" \"chat_types\": ");
if (filter.ChatTypes is { Count: > 0 })
w.Write("[" + string.Join(",", filter.ChatTypes) + "]");
else
w.Write("null");
w.Write(",\n \"from\": ");
w.Write(filter.From is null ? "null" : "\"" + filter.From.Value.ToString("O", CultureInfo.InvariantCulture) + "\"");
w.Write(",\n \"to\": ");
w.Write(filter.To is null ? "null" : "\"" + filter.To.Value.ToString("O", CultureInfo.InvariantCulture) + "\"");
w.Write(",\n \"sender_substring\": ");
w.Write(filter.SenderSubstring is null ? "null" : JsonString(filter.SenderSubstring));
w.Write("\n },\n \"messages\": [\n");
var first = true;
var count = 0;
foreach (var m in messages)
{
if (!first)
w.Write(",\n");
first = false;
count++;
var chatType = (ChatType)(ushort)m.Code.Type;
w.Write(" {");
w.Write($"\"id\":\"{m.Id}\"");
w.Write($",\"date\":\"{m.Date.ToString("O", CultureInfo.InvariantCulture)}\"");
w.Write($",\"chat_type\":{(int)m.Code.Type}");
w.Write($",\"chat_type_name\":\"{chatType}\"");
w.Write($",\"source_kind\":{m.Code.Source}");
w.Write($",\"target_kind\":{m.Code.Target}");
w.Write($",\"receiver\":{m.Receiver}");
w.Write($",\"content_id\":{m.ContentId}");
w.Write($",\"sender\":{JsonString(m.SenderSource.TextValue)}");
w.Write($",\"content\":{JsonString(m.ContentSource.TextValue)}");
w.Write("}");
}
w.Write("\n ],\n");
w.Write($" \"total\": {count}\n}}\n");
return count;
}
private static int WriteCsv(StreamWriter w, IEnumerable<Message> messages, FilterDescription filter)
{
// Header line always written so empty exports are still importable.
w.WriteLine("Date,ChatType,ChatTypeName,Sender,Content,Receiver,ContentId");
var count = 0;
foreach (var m in messages)
{
count++;
var chatType = (ChatType)(ushort)m.Code.Type;
w.Write(m.Date.ToString("O", CultureInfo.InvariantCulture));
w.Write(',');
w.Write((int)m.Code.Type);
w.Write(',');
w.Write(CsvString(chatType.ToString()));
w.Write(',');
w.Write(CsvString(m.SenderSource.TextValue));
w.Write(',');
w.Write(CsvString(m.ContentSource.TextValue));
w.Write(',');
w.Write(m.Receiver);
w.Write(',');
w.Write(m.ContentId);
w.WriteLine();
}
return count;
}
private static string JsonString(string s)
{
var sb = new StringBuilder(s.Length + 2);
sb.Append('"');
foreach (var c in s)
{
switch (c)
{
case '"': sb.Append("\\\""); break;
case '\\': sb.Append("\\\\"); break;
case '\b': sb.Append("\\b"); break;
case '\f': sb.Append("\\f"); break;
case '\n': sb.Append("\\n"); break;
case '\r': sb.Append("\\r"); break;
case '\t': sb.Append("\\t"); break;
default:
if (c < 0x20)
sb.Append($"\\u{(int)c:x4}");
else
sb.Append(c);
break;
}
}
sb.Append('"');
return sb.ToString();
}
private static string CsvString(string s)
{
if (s.IndexOfAny(['"', ',', '\n', '\r']) < 0)
return s;
return "\"" + s.Replace("\"", "\"\"") + "\"";
}
}
+48
View File
@@ -500,6 +500,54 @@ internal class MessageStore : IDisposable
cmd.ExecuteNonQuery();
}
/// <summary>
/// Streams messages for export. Optional filters:
/// - <paramref name="chatTypes"/>: limit to these ChatTypes
/// - <paramref name="from"/> / <paramref name="to"/>: inclusive date range
/// Result is sorted ascending by Date and excludes soft-deleted rows.
/// Caller is responsible for disposing the enumerator.
/// </summary>
internal MessageEnumerator StreamForExport(
IReadOnlyCollection<int>? chatTypes,
DateTimeOffset? from,
DateTimeOffset? to)
{
var clauses = new List<string> { "deleted = false" };
if (chatTypes is { Count: > 0 })
clauses.Add($"ChatType IN ({string.Join(",", chatTypes)})");
if (from is not null)
clauses.Add("Date >= $From");
if (to is not null)
clauses.Add("Date <= $To");
var cmd = Connection.CreateCommand();
cmd.CommandText = @"
SELECT
Id,
Receiver,
ContentId,
Date,
ChatType,
SourceKind,
TargetKind,
Sender,
Content,
SenderSource,
ContentSource,
ExtraChatChannel
FROM messages
WHERE " + string.Join(" AND ", clauses) + @"
ORDER BY Date ASC;";
cmd.CommandTimeout = 600;
if (from is not null)
cmd.Parameters.AddWithValue("$From", from.Value.ToUnixTimeMilliseconds());
if (to is not null)
cmd.Parameters.AddWithValue("$To", to.Value.ToUnixTimeMilliseconds());
return new MessageEnumerator(cmd.ExecuteReader());
}
/// <summary>
/// Get the most recent messages.
/// </summary>
+17
View File
@@ -113,4 +113,21 @@ internal class HellionStrings
internal static string Wizard_Profile_FullHistory_GdprWarning => Get(nameof(Wizard_Profile_FullHistory_GdprWarning));
internal static string Wizard_Profile_FullHistory_Apply => Get(nameof(Wizard_Profile_FullHistory_Apply));
internal static string Wizard_Reopen_Button => Get(nameof(Wizard_Reopen_Button));
internal static string Export_Heading => Get(nameof(Export_Heading));
internal static string Export_Help => Get(nameof(Export_Help));
internal static string Export_Range_Label => Get(nameof(Export_Range_Label));
internal static string Export_Sender_Label => Get(nameof(Export_Sender_Label));
internal static string Export_Channels_Heading => Get(nameof(Export_Channels_Heading));
internal static string Export_Channels_AllOff => Get(nameof(Export_Channels_AllOff));
internal static string Export_Format_Label => Get(nameof(Export_Format_Label));
internal static string Export_Format_Markdown => Get(nameof(Export_Format_Markdown));
internal static string Export_Format_Json => Get(nameof(Export_Format_Json));
internal static string Export_Format_Csv => Get(nameof(Export_Format_Csv));
internal static string Export_Button => Get(nameof(Export_Button));
internal static string Export_Dialog_Title => Get(nameof(Export_Dialog_Title));
internal static string Export_Running => Get(nameof(Export_Running));
internal static string Export_Success => Get(nameof(Export_Success));
internal static string Export_Empty => Get(nameof(Export_Empty));
internal static string Export_Error => Get(nameof(Export_Error));
}
+48
View File
@@ -216,4 +216,52 @@
<data name="Wizard_Reopen_Button" xml:space="preserve">
<value>Wizard erneut zeigen</value>
</data>
<data name="Export_Heading" xml:space="preserve">
<value>Export (DSGVO Art. 15 — Auskunftsrecht)</value>
</data>
<data name="Export_Help" xml:space="preserve">
<value>Gespeicherte Nachrichten als Markdown, JSON oder CSV exportieren. Damit kannst du einer Auskunftsanfrage einer Person nachkommen, deren Nachrichten du gespeichert hast, oder deine eigene Historie mitnehmen.</value>
</data>
<data name="Export_Range_Label" xml:space="preserve">
<value>Letzte X Tage (0 = ohne Zeitlimit)</value>
</data>
<data name="Export_Sender_Label" xml:space="preserve">
<value>Sender enthält (optional, Groß-/Kleinschreibung egal)</value>
</data>
<data name="Export_Channels_Heading" xml:space="preserve">
<value>Auf Kanäle einschränken</value>
</data>
<data name="Export_Channels_AllOff" xml:space="preserve">
<value>(nichts ausgewählt = alle gespeicherten Kanäle)</value>
</data>
<data name="Export_Format_Label" xml:space="preserve">
<value>Format</value>
</data>
<data name="Export_Format_Markdown" xml:space="preserve">
<value>Markdown</value>
</data>
<data name="Export_Format_Json" xml:space="preserve">
<value>JSON</value>
</data>
<data name="Export_Format_Csv" xml:space="preserve">
<value>CSV</value>
</data>
<data name="Export_Button" xml:space="preserve">
<value>In Datei exportieren…</value>
</data>
<data name="Export_Dialog_Title" xml:space="preserve">
<value>Export speichern</value>
</data>
<data name="Export_Running" xml:space="preserve">
<value>Export läuft im Hintergrund…</value>
</data>
<data name="Export_Success" xml:space="preserve">
<value>Export abgeschlossen, {0:N0} Nachrichten in {1} geschrieben</value>
</data>
<data name="Export_Empty" xml:space="preserve">
<value>Export abgeschlossen, keine Nachricht passte zum Filter.</value>
</data>
<data name="Export_Error" xml:space="preserve">
<value>Export fehlgeschlagen, siehe /xllog</value>
</data>
</root>
+48
View File
@@ -216,4 +216,52 @@
<data name="Wizard_Reopen_Button" xml:space="preserve">
<value>Show wizard again</value>
</data>
<data name="Export_Heading" xml:space="preserve">
<value>Export (GDPR Art. 15 — right of access)</value>
</data>
<data name="Export_Help" xml:space="preserve">
<value>Export stored messages to Markdown, JSON or CSV. Use this to fulfil a request for access from someone whose messages you have, or to take your own history with you.</value>
</data>
<data name="Export_Range_Label" xml:space="preserve">
<value>Last X days (0 = all time)</value>
</data>
<data name="Export_Sender_Label" xml:space="preserve">
<value>Sender contains (optional, case-insensitive)</value>
</data>
<data name="Export_Channels_Heading" xml:space="preserve">
<value>Limit to channels</value>
</data>
<data name="Export_Channels_AllOff" xml:space="preserve">
<value>(none selected = all stored channels)</value>
</data>
<data name="Export_Format_Label" xml:space="preserve">
<value>Format</value>
</data>
<data name="Export_Format_Markdown" xml:space="preserve">
<value>Markdown</value>
</data>
<data name="Export_Format_Json" xml:space="preserve">
<value>JSON</value>
</data>
<data name="Export_Format_Csv" xml:space="preserve">
<value>CSV</value>
</data>
<data name="Export_Button" xml:space="preserve">
<value>Export to file…</value>
</data>
<data name="Export_Dialog_Title" xml:space="preserve">
<value>Save export</value>
</data>
<data name="Export_Running" xml:space="preserve">
<value>Export running in background…</value>
</data>
<data name="Export_Success" xml:space="preserve">
<value>Export complete: {0:N0} messages written to {1}</value>
</data>
<data name="Export_Empty" xml:space="preserve">
<value>Export complete: no messages matched the filter.</value>
</data>
<data name="Export_Error" xml:space="preserve">
<value>Export failed, see /xllog</value>
</data>
</root>
+144
View File
@@ -1,4 +1,5 @@
using ChatTwo.Code;
using ChatTwo.Export;
using ChatTwo.Privacy;
using ChatTwo.Resources;
using ChatTwo.Util;
@@ -56,6 +57,13 @@ internal sealed class Privacy : ISettingsTab
private bool RetentionRunning;
// Export form state
private int ExportRangeDays = 30;
private string ExportSenderSubstring = string.Empty;
private readonly HashSet<ChatType> ExportSelectedChannels = [];
private ExportFormat ExportFormat = ExportFormat.Markdown;
private bool ExportRunning;
public void Draw(bool changed)
{
if (ImGui.Button(HellionStrings.Wizard_Reopen_Button))
@@ -138,6 +146,142 @@ internal sealed class Privacy : ISettingsTab
ImGui.Spacing();
DrawCleanupSection();
ImGui.Spacing();
ImGui.Separator();
ImGui.Spacing();
DrawExportSection();
}
private void DrawExportSection()
{
ImGui.TextUnformatted(HellionStrings.Export_Heading);
using (ImRaii.PushIndent(ImGui.GetStyle().IndentSpacing, false))
{
ImGuiUtil.HelpText(HellionStrings.Export_Help);
ImGui.Spacing();
if (ImGui.InputInt(HellionStrings.Export_Range_Label, ref ExportRangeDays))
ExportRangeDays = Math.Max(0, ExportRangeDays);
ImGui.InputText(HellionStrings.Export_Sender_Label, ref ExportSenderSubstring, 256);
using (var tree = ImRaii.TreeNode(HellionStrings.Export_Channels_Heading))
{
if (tree.Success)
{
using (ImRaii.PushIndent(ImGui.GetStyle().IndentSpacing, false))
{
ImGuiUtil.HelpText(HellionStrings.Export_Channels_AllOff);
foreach (var (heading, types) in Groups)
{
using var subTree = ImRaii.TreeNode($"{heading()}##export-group-{heading()}");
if (!subTree.Success)
continue;
using (ImRaii.PushIndent(ImGui.GetStyle().IndentSpacing, false))
foreach (var type in types)
{
var enabled = ExportSelectedChannels.Contains(type);
if (ImGui.Checkbox($"{type}##export-{(int)type}", ref enabled))
{
if (enabled)
ExportSelectedChannels.Add(type);
else
ExportSelectedChannels.Remove(type);
}
}
}
}
}
}
ImGui.Spacing();
ImGui.TextUnformatted(HellionStrings.Export_Format_Label);
ImGui.SameLine();
var fmt = (int)ExportFormat;
if (ImGui.RadioButton(HellionStrings.Export_Format_Markdown, ref fmt, (int)ExportFormat.Markdown))
ExportFormat = ExportFormat.Markdown;
ImGui.SameLine();
if (ImGui.RadioButton(HellionStrings.Export_Format_Json, ref fmt, (int)ExportFormat.Json))
ExportFormat = ExportFormat.Json;
ImGui.SameLine();
if (ImGui.RadioButton(HellionStrings.Export_Format_Csv, ref fmt, (int)ExportFormat.Csv))
ExportFormat = ExportFormat.Csv;
ImGui.Spacing();
using (ImRaii.Disabled(ExportRunning))
{
if (ImGui.Button(HellionStrings.Export_Button))
PromptExport();
}
if (ExportRunning)
ImGuiUtil.HelpText(HellionStrings.Export_Running);
}
}
private void PromptExport()
{
var defaultName = $"hellion-chat-export-{DateTimeOffset.Now:yyyyMMdd-HHmm}";
var ext = ExportFormat.Extension();
Plugin.FileDialogManager.SaveFileDialog(
HellionStrings.Export_Dialog_Title,
ExportFormat.Filter(),
defaultName,
ext,
(success, path) =>
{
if (!success || string.IsNullOrWhiteSpace(path))
return;
StartExport(path);
});
}
private void StartExport(string path)
{
if (ExportRunning)
return;
ExportRunning = true;
var types = ExportSelectedChannels.Count > 0
? ExportSelectedChannels.Select(t => (int)(ushort)t).ToList()
: null;
DateTimeOffset? from = ExportRangeDays > 0
? DateTimeOffset.UtcNow.AddDays(-ExportRangeDays)
: null;
var senderSubstring = string.IsNullOrWhiteSpace(ExportSenderSubstring) ? null : ExportSenderSubstring.Trim();
var format = ExportFormat;
var filterDesc = new MessageExporter.FilterDescription(types, from, null, senderSubstring);
new Thread(() =>
{
try
{
using var enumerator = Plugin.MessageManager.Store.StreamForExport(types, from, null);
var written = MessageExporter.ExportToFile(path, format, enumerator, filterDesc);
if (written > 0)
WrapperUtil.AddNotification(string.Format(HellionStrings.Export_Success, written, path), NotificationType.Success);
else
WrapperUtil.AddNotification(HellionStrings.Export_Empty, NotificationType.Info);
}
catch (Exception e)
{
Plugin.Log.Error(e, "Export failed");
WrapperUtil.AddNotification(HellionStrings.Export_Error, NotificationType.Error);
}
finally
{
ExportRunning = false;
}
}) { IsBackground = true }.Start();
}
private void DrawRetentionSection()