using System.Globalization; using System.Text; using HellionChat.Code; using HellionChat.Util; namespace HellionChat.Export; internal enum ExportFormat { Markdown, Json, Csv, } internal static class ExportFormatExt { internal static string Extension(this ExportFormat fmt) => fmt switch { ExportFormat.Markdown => "md", ExportFormat.Json => "json", ExportFormat.Csv => "csv", _ => "txt", }; internal static string Filter(this ExportFormat fmt) => fmt switch { ExportFormat.Markdown => ".md", ExportFormat.Json => ".json", ExportFormat.Csv => ".csv", _ => ".txt", }; } // Serializes message snapshots to Markdown, JSON, or CSV. // // Text comes from the chunk lists, never from SenderSource/ContentSource. Those // are raw SeStrings, and reading TextValue on one containing an auto-translate // phrase reaches SeStringEvaluator, which asserts it is on the main thread and // throws unconditionally when a macro resolves a global number. An export runs on // a worker, so that would abort it partway and leave half a file behind. // // The chunks are already resolved: ChunkUtil turns auto-translate into text at // ingest, and the full-text index reads them exactly this way. Same strings, no // evaluator, no thread affinity. // // The caller pre-filters by channel and date via StreamForExport; only the sender // substring is applied here. internal static class MessageExporter { private static readonly UTF8Encoding Utf8NoBom = new(encoderShouldEmitUTF8Identifier: false); internal record FilterDescription( IReadOnlyCollection? ChatTypes, DateTimeOffset? From, DateTimeOffset? To, string? SenderSubstring ); internal static int ExportToFile( string path, ExportFormat format, IEnumerable messages, FilterDescription filter ) { // Rejected before the file is touched. The old order opened the stream // first, so an unknown format left a zero-byte file where the user's // previous export had been. if (!Enum.IsDefined(format)) throw new ArgumentOutOfRangeException(nameof(format), format, null); var matching = filter.SenderSubstring is { Length: > 0 } needle ? messages.Where(m => MatchesSender(m, needle)) : messages; // Written beside the target and moved into place at the end. A crash or // an unplugged drive halfway through would otherwise leave a file that // opens fine and is quietly incomplete -- and this is the path a GDPR // access request goes out on, where "looks complete" is the dangerous // failure. var temp = path + ".part"; int written; try { // Encoding.UTF8 writes a byte order mark, and that is not a // cosmetic detail here: a leading U+FEFF makes the JSON invalid for // every strict parser, Python's json.load included. CSV is the one // format that wants it -- without a BOM Excel guesses the codepage // and mangles every non-ASCII name in the file. var encoding = format == ExportFormat.Csv ? Encoding.UTF8 : Utf8NoBom; using (var writer = new StreamWriter(temp, append: false, encoding)) { written = format switch { ExportFormat.Markdown => WriteMarkdown(writer, matching, filter), ExportFormat.Json => WriteJson(writer, matching, filter), _ => WriteCsv(writer, matching, filter), }; } // An export that matched nothing does not replace anything. The // file still has a header and a footer, so moving it would put a // near-empty file where the user's previous export was -- and then // report "no message matched the filter", which reads as "nothing // happened". Dalamud's save dialog has no overwrite confirmation to // fall back on. if (written == 0) { TryDeleteTemp(temp); return 0; } File.Move(temp, path, overwrite: true); return written; } catch { TryDeleteTemp(temp); throw; } } // Best effort: the export already failed, and a leftover .part file is a // smaller problem than masking the original exception with an IO one. private static void TryDeleteTemp(string temp) { try { if (File.Exists(temp)) File.Delete(temp); } catch (IOException) { } catch (UnauthorizedAccessException) { } } private static bool MatchesSender(Message m, string needle) => SenderText(m).Contains(needle, StringComparison.OrdinalIgnoreCase); private static string SenderText(Message m) => ChunkUtil.ToRawString(m.Sender); private static string ContentText(Message m) => ChunkUtil.ToRawString(m.Content); private static int WriteMarkdown( StreamWriter w, IEnumerable messages, FilterDescription filter ) { w.WriteLine("# Hellion Chat Export"); w.WriteLine(); w.WriteLine($"Generated: {DateTimeOffset.Now:yyyy-MM-dd HH:mm zzz}"); WriteFilterSummaryMarkdown(w, filter); w.WriteLine(); DateTimeOffset? lastDate = null; var count = 0; foreach (var m in messages) { count++; var localDate = m.Date.ToLocalTime(); if (lastDate is null || localDate.Date != lastDate.Value.Date) { w.WriteLine(); w.WriteLine($"## {localDate:yyyy-MM-dd}"); w.WriteLine(); lastDate = localDate; } var chatType = (ChatType)(ushort)m.Code.Type; var sender = SenderText(m).Trim().Trim('<', '>', '[', ']', ':').Trim(); var content = ContentText(m); if (string.IsNullOrEmpty(sender)) w.WriteLine($"**[{localDate:HH:mm}] {chatType}:** {content}"); else w.WriteLine($"**[{localDate:HH:mm}] {chatType} {sender}:** {content}"); } w.WriteLine(); w.WriteLine($"---"); w.WriteLine($"Total messages: {count}"); return count; } private static void WriteFilterSummaryMarkdown(StreamWriter w, FilterDescription filter) { if (filter.ChatTypes is { Count: > 0 }) w.WriteLine( $"ChatTypes: {string.Join(", ", filter.ChatTypes.Select(t => $"{(ChatType)(ushort)t}({t})"))}" ); if (filter.From is not null) w.WriteLine($"From: {filter.From.Value.ToLocalTime():yyyy-MM-dd HH:mm}"); if (filter.To is not null) w.WriteLine($"To: {filter.To.Value.ToLocalTime():yyyy-MM-dd HH:mm}"); if (filter.SenderSubstring is { Length: > 0 }) w.WriteLine($"Sender contains: \"{filter.SenderSubstring}\""); } private static int WriteJson( StreamWriter w, IEnumerable messages, FilterDescription filter ) { // Manual JSON to avoid System.Text.Json policy coupling. w.Write("{\n \"exported_at\": \""); w.Write(DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture)); w.Write("\",\n \"plugin\": \"Hellion Chat\",\n"); w.Write(" \"filter\": {\n"); w.Write(" \"chat_types\": "); if (filter.ChatTypes is { Count: > 0 }) w.Write("[" + string.Join(",", filter.ChatTypes) + "]"); else w.Write("null"); w.Write(",\n \"from\": "); w.Write( filter.From is null ? "null" : "\"" + filter.From.Value.ToString("O", CultureInfo.InvariantCulture) + "\"" ); w.Write(",\n \"to\": "); w.Write( filter.To is null ? "null" : "\"" + filter.To.Value.ToString("O", CultureInfo.InvariantCulture) + "\"" ); w.Write(",\n \"sender_substring\": "); w.Write(filter.SenderSubstring is null ? "null" : JsonString(filter.SenderSubstring)); w.Write("\n },\n \"messages\": [\n"); var first = true; var count = 0; foreach (var m in messages) { if (!first) w.Write(",\n"); first = false; count++; var chatType = (ChatType)(ushort)m.Code.Type; w.Write(" {"); w.Write($"\"id\":\"{m.Id}\""); w.Write($",\"date\":\"{m.Date.ToString("O", CultureInfo.InvariantCulture)}\""); w.Write($",\"chat_type\":{(int)m.Code.Type}"); w.Write($",\"chat_type_name\":\"{chatType}\""); // Cast, not interpolate. These are XivChatRelationKind, and string // interpolation of an enum writes the member name -- so every // message with a recognised relation produced // "source_kind":LocalPlayer, which no parser accepts. This is the // file an access request goes out on. w.Write($",\"source_kind\":{(int)m.Code.Source}"); w.Write($",\"target_kind\":{(int)m.Code.Target}"); w.Write($",\"receiver\":{m.Receiver}"); w.Write($",\"content_id\":{m.ContentId}"); w.Write($",\"sender\":{JsonString(SenderText(m))}"); w.Write($",\"content\":{JsonString(ContentText(m))}"); w.Write("}"); } w.Write("\n ],\n"); w.Write($" \"total\": {count}\n}}\n"); return count; } private static int WriteCsv( StreamWriter w, IEnumerable messages, FilterDescription filter ) { // Header always written so empty exports remain importable. w.WriteLine("Date,ChatType,ChatTypeName,Sender,Content,Receiver,ContentId"); var count = 0; foreach (var m in messages) { count++; var chatType = (ChatType)(ushort)m.Code.Type; w.Write(m.Date.ToString("O", CultureInfo.InvariantCulture)); w.Write(','); w.Write((int)m.Code.Type); w.Write(','); w.Write(CsvString(chatType.ToString())); w.Write(','); w.Write(CsvString(SenderText(m))); w.Write(','); w.Write(CsvString(ContentText(m))); w.Write(','); w.Write(m.Receiver); w.Write(','); w.Write(m.ContentId); w.WriteLine(); } return count; } private static string JsonString(string s) { var sb = new StringBuilder(s.Length + 2); sb.Append('"'); foreach (var c in s) { switch (c) { case '"': sb.Append("\\\""); break; case '\\': sb.Append("\\\\"); break; case '\b': sb.Append("\\b"); break; case '\f': sb.Append("\\f"); break; case '\n': sb.Append("\\n"); break; case '\r': sb.Append("\\r"); break; case '\t': sb.Append("\\t"); break; default: if (c < 0x20) sb.Append($"\\u{(int)c:x4}"); else sb.Append(c); break; } } sb.Append('"'); return sb.ToString(); } private static string CsvString(string s) { // Leading =, +, - and @ make a spreadsheet treat the cell as a formula. // Every value here is text somebody else typed into a chat channel, and // this file exists to be opened in Excel, so a prefixed apostrophe goes // in front. It is the standard defence and it costs one character that // spreadsheets hide. if (s.Length > 0 && s[0] is '=' or '+' or '-' or '@' or '\t' or '\r') s = "'" + s; if (s.IndexOfAny(['"', ',', '\n', '\r']) < 0) return s; return "\"" + s.Replace("\"", "\"\"") + "\""; } }