diff --git a/ChatTwo.Tests/ChatTwo.Tests.csproj b/ChatTwo.Tests/ChatTwo.Tests.csproj index 68fabd5..8ea66bc 100644 --- a/ChatTwo.Tests/ChatTwo.Tests.csproj +++ b/ChatTwo.Tests/ChatTwo.Tests.csproj @@ -9,6 +9,7 @@ + diff --git a/ChatTwo/ChatTwo.csproj b/ChatTwo/ChatTwo.csproj index 3cb0275..352f979 100755 --- a/ChatTwo/ChatTwo.csproj +++ b/ChatTwo/ChatTwo.csproj @@ -15,6 +15,7 @@ + diff --git a/ChatTwo/MessageStore.cs b/ChatTwo/MessageStore.cs index 578b0bc..a23d77f 100644 --- a/ChatTwo/MessageStore.cs +++ b/ChatTwo/MessageStore.cs @@ -387,7 +387,7 @@ internal class MessageStore : IDisposable cmd.ExecuteNonQuery(); } - internal long CountDateRange(DateTime after, DateTime before, uint[] channels, ulong? receiver = null) + internal long CountDateRange(DateTime after, DateTime before, IEnumerable channels, ulong? receiver = null) { List whereClauses = ["deleted = false"]; if (receiver != null) @@ -416,7 +416,47 @@ internal class MessageStore : IDisposable return (long) cmd.ExecuteScalar()!; } - internal MessageEnumerator GetDateRange(DateTime after, DateTime before, uint[] channels, ulong? receiver = null, int page = 0) + internal MessageEnumerator GetDateRange(DateTime after, DateTime before, IEnumerable channels, ulong? receiver = null) + { + List whereClauses = ["deleted = false"]; + if (receiver != null) + whereClauses.Add("Receiver = $Receiver"); + + whereClauses.Add("Date BETWEEN $After AND $Before"); + whereClauses.Add($"Channel IN ({string.Join(", ", channels)})"); + + var whereClause = $"WHERE {string.Join(" AND ", whereClauses)}"; + + var cmd = Connection.CreateCommand(); + // Select last N messages by date DESC, but reverse the order to get + // them in ascending order. + cmd.CommandText = @" + SELECT + Id, + Receiver, + ContentId, + Date, + Code, + Sender, + Content, + SenderSource, + ContentSource, + SortCode, + ExtraChatChannel + FROM messages + " + whereClause; + cmd.CommandTimeout = 120; // this could take a while on slow computers + + if (receiver != null) + cmd.Parameters.AddWithValue("$Receiver", receiver); + + cmd.Parameters.AddWithValue("$After", ((DateTimeOffset) after).ToUnixTimeMilliseconds()); + cmd.Parameters.AddWithValue("$Before", ((DateTimeOffset) before).ToUnixTimeMilliseconds()); + + return new MessageEnumerator(cmd.ExecuteReader()); + } + + internal MessageEnumerator GetPagedDateRange(DateTime after, DateTime before, IEnumerable channels, ulong? receiver = null, int page = 0) { List whereClauses = ["deleted = false"]; if (receiver != null) diff --git a/ChatTwo/Plugin.cs b/ChatTwo/Plugin.cs index 442741d..991f1bd 100755 --- a/ChatTwo/Plugin.cs +++ b/ChatTwo/Plugin.cs @@ -12,6 +12,7 @@ using Dalamud.IoC; using Dalamud.Plugin; using Dalamud.Plugin.Services; using Dalamud.Bindings.ImGui; +using Dalamud.Interface.ImGuiFileDialog; namespace ChatTwo; @@ -42,6 +43,7 @@ public sealed class Plugin : IDalamudPlugin [PluginService] internal static ISeStringEvaluator Evaluator { get; private set; } = null!; internal static Configuration Config = null!; + public static FileDialogManager FileDialogManager { get; private set; } = null!; public readonly WindowSystem WindowSystem = new(PluginName); public SettingsWindow SettingsWindow { get; } @@ -93,6 +95,8 @@ public sealed class Plugin : IDalamudPlugin LanguageChanged(Interface.UiLanguage); ImGuiUtil.Initialize(this); + FileDialogManager = new FileDialogManager(); + // Functions calls this in its ctor if the player is already logged in ServerCore = new ServerCore(this); @@ -216,6 +220,8 @@ public sealed class Plugin : IDalamudPlugin ChatLogWindow.FinalizeFrame(); TypingIpc.Update(); + + FileDialogManager.Draw(); } internal void SaveConfig() diff --git a/ChatTwo/Ui/DbViewer.cs b/ChatTwo/Ui/DbViewer.cs index 8dd1a3b..5fc8798 100644 --- a/ChatTwo/Ui/DbViewer.cs +++ b/ChatTwo/Ui/DbViewer.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using System.Globalization; using System.Numerics; +using System.Text; using ChatTwo.Code; using ChatTwo.Resources; using ChatTwo.Util; @@ -10,6 +11,9 @@ using Dalamud.Interface.Utility; using Dalamud.Interface.Utility.Raii; using Dalamud.Interface.Windowing; using Dalamud.Bindings.ImGui; +using Dalamud.Interface.ImGuiNotification; +using Lumina.Text.ReadOnly; +using MoreLinq; namespace ChatTwo.Ui; @@ -43,6 +47,10 @@ public class DbViewer : Window private Message[] Messages = []; // Messages are only touched while processing is false private ConcurrentStack Filtered = []; // Is used every frame, so ConcurrentStack for safety + private bool IsExporting; + private string InputPath = string.Empty; + private IActiveNotification Notification = null!; + public DbViewer(Plugin plugin) : base("DBViewer###chat2-dbviewer") { Plugin = plugin; @@ -63,12 +71,12 @@ public class DbViewer : Window RespectCloseHotkey = false; DisableWindowSounds = true; - Plugin.Commands.Register("/chat2Viewer", "Database Viewer", true).Execute += Toggle; + Plugin.Commands.Register("/chat2Viewer", "Get access to your message history, with simple filter options.", true).Execute += Toggle; } public void Dispose() { - Plugin.Commands.Register("/chat2Viewer", "Database Viewer", true).Execute -= Toggle; + Plugin.Commands.Register("/chat2Viewer", "Get access to your message history, with simple filter options.", true).Execute -= Toggle; } private void Toggle(string _, string __) => Toggle(); @@ -82,6 +90,8 @@ public class DbViewer : Window if (CurrentPage > totalPages) CurrentPage = 1; + // First row + ImGui.AlignTextToFramePadding(); ImGui.TextColored(ImGuiColors.DalamudViolet, Language.DbViewer_DatePicker_FromTo); ImGui.SameLine(); @@ -101,6 +111,49 @@ public class DbViewer : Window ImGui.SameLine(ImGui.GetContentRegionMax().X - textLength); ImGui.Checkbox(skipText, ref OnlyCurrentCharacter); + // Second row + + ImGui.AlignTextToFramePadding(); + ImGui.TextColored(ImGuiColors.DalamudViolet, "Export:"); + ImGui.SameLine(0, spacing); + ImGui.SetNextItemWidth(ImGui.GetWindowWidth() * 0.4f); + ImGui.InputText("##InputPath", ref InputPath, 255); + ImGui.SameLine(0, spacing); + if (ImGuiUtil.IconButton(FontAwesomeIcon.FolderClosed, "##folderPicker")) + ImGui.OpenPopup("InputPathDialog"); + + if (ImGui.IsItemHovered(ImGuiHoveredFlags.AllowWhenDisabled)) + ImGui.SetTooltip("Pick a folder location for export."); + + using (var innerPopup = ImRaii.Popup("InputPathDialog")) + { + if (innerPopup.Success) + Plugin.FileDialogManager.OpenFolderDialog("Pick an export location", (b, s) => { if (b) InputPath = s; }, null, true); + } + + ImGui.SameLine(0, spacing); + using (ImRaii.Disabled(InputPath.Length == 0 || IsExporting)) + { + if (ImGuiUtil.IconButton(FontAwesomeIcon.Save)) + { + Notification = Plugin.Notification.AddNotification( + new Notification + { + Title = "Chat2 Export", + Content = "Loading logs ...", + Type = NotificationType.Info, + Minimized = false, + UserDismissable = false, + InitialDuration = TimeSpan.FromSeconds(10000), + Progress = 0.0f, + }); + CreateTxtBackup(); + } + } + + if (ImGui.IsItemHovered(ImGuiHoveredFlags.AllowWhenDisabled)) + ImGui.SetTooltip("Export the message history to a text file."); + var width = 350 * ImGuiHelpers.GlobalScale; var loadingIndicator = IsProcessing && ProcessingStart < Environment.TickCount64; @@ -109,7 +162,9 @@ public class DbViewer : Window ImGui.SameLine(ImGui.GetContentRegionMax().X - width); ImGui.SetNextItemWidth(width); if (ImGui.InputTextWithHint("##searchbar", Language.DbViewer_SearcHint, ref SimpleSearchTerm, 30)) - Filter(); + Filtered = Filter(Messages); + + // Third row if (DateWidget.Validate(MinimalDate, ref AfterDate, ref BeforeDate)) DateRefresh(); @@ -135,10 +190,10 @@ public class DbViewer : Window if (CurrentPage == 1) Count = Plugin.MessageManager.Store.CountDateRange(AfterDate, BeforeDate, channels, character); - using var dateRangeMessageEnumerator = Plugin.MessageManager.Store.GetDateRange(AfterDate, BeforeDate, channels, character, CurrentPage - 1); - Messages = dateRangeMessageEnumerator.ToArray(); + using var rangeMessageEnumerator = Plugin.MessageManager.Store.GetPagedDateRange(AfterDate, BeforeDate, channels, character, CurrentPage - 1); + Messages = rangeMessageEnumerator.ToArray(); - Filter(); + Filtered = Filter(Messages); } catch (Exception ex) { @@ -236,16 +291,13 @@ public class DbViewer : Window } } - private void Filter() + private ConcurrentStack Filter(Message[] messages) { if (SimpleSearchTerm == "") - { - Filtered = new ConcurrentStack(Messages.Reverse().OrderByDescending(m => m.Date)); - return; - } + return new ConcurrentStack(messages.Reverse().OrderByDescending(m => m.Date)); - Filtered = new ConcurrentStack( - Messages.Reverse().Where(m => + return new ConcurrentStack( + messages.Reverse().Where(m => ChunkUtil.ToRawString(m.Sender).Contains(SimpleSearchTerm, StringComparison.InvariantCultureIgnoreCase) || ChunkUtil.ToRawString(m.Content).Contains(SimpleSearchTerm, StringComparison.InvariantCultureIgnoreCase) ).OrderByDescending(m => m.Date)); @@ -271,4 +323,72 @@ public class DbViewer : Window AdjustDates(); DateRefresh(); } + + private void CreateTxtBackup() + { + IsExporting = true; + Task.Run(async () => + { + try + { + ulong? character = OnlyCurrentCharacter ? Plugin.PlayerState.ContentId : null; + var channels = ChatCodes.Select(c => (uint)c.Key).ToArray(); + + var rangeMessageEnumerator = Plugin.MessageManager.Store.GetDateRange(AfterDate, BeforeDate, channels, character); + var messageHistory = rangeMessageEnumerator.ToArray(); + await rangeMessageEnumerator.DisposeAsync(); + + var filteredHistory = Filter(messageHistory); + + var sb = new StringBuilder(); + await using var stream = new StreamWriter(Path.Join(InputPath, $"Chat2_{DateTime.Now:yyyy_dd_M__HH_mm_ss}.txt")); + + var batch = 0; + foreach (var messages in filteredHistory.Batch(5000)) + { + await Plugin.Framework.RunOnTick(() => + { + foreach (var message in messages) + { + if (!Sheets.LogKindSheet.TryGetRow((uint)message.Code.Type, out var logKind)) + logKind = Sheets.LogKindSheet.GetRow(10); // default to say + + var rossSender = new ReadOnlySeString(message.SenderSource.Encode()); + var rossMessage = new ReadOnlySeString(message.ContentSource.Encode()); + + var timestamp = message.Date.ToLocalTime().ToString(DateTimeFormat); + var text = Plugin.Evaluator.Evaluate(logKind.Format, [rossSender, rossMessage]).ToString(); + sb.AppendLine($"[{timestamp}][{message.Code.Type.Name()}] {text}"); + + batch++; + } + }, delayTicks: 5); + + Notification.Progress = (float)batch / filteredHistory.Count; + Notification.Content = $"Exported {batch} of {filteredHistory.Count} messages"; + await stream.WriteAsync(sb.ToString()); + sb.Clear(); + } + + await stream.WriteAsync(sb.ToString()); + sb.Clear(); + + Notification.Progress = 1.0f; + Notification.Content = "Done!!!"; + Notification.Type = NotificationType.Success; + } + catch (Exception ex) + { + Plugin.Log.Error(ex, "Failed creating txt backup"); + + Notification.Content = "Error ..."; + Notification.Type = NotificationType.Error; + } + finally + { + IsExporting = false; + Notification.UserDismissable = true; + } + }); + } } diff --git a/ChatTwo/packages.lock.json b/ChatTwo/packages.lock.json index 019ea19..480ffe7 100644 --- a/ChatTwo/packages.lock.json +++ b/ChatTwo/packages.lock.json @@ -36,6 +36,12 @@ "SQLitePCLRaw.core": "2.1.10" } }, + "morelinq": { + "type": "Direct", + "requested": "[4.4.0, )", + "resolved": "4.4.0", + "contentHash": "QX3bsK9oFeUXk8tFsc9NkI6NnCr8Ar/ex027p+ZZ/jdLCdX2RlryDtxUqZW5j45NVwn4E4Z4hzupsoMQd6Yxtg==" + }, "Pidgin": { "type": "Direct", "requested": "[3.3.0, )",