diff --git a/HellionChat/MessageStore.cs b/HellionChat/MessageStore.cs index fd4378d..eaea976 100644 --- a/HellionChat/MessageStore.cs +++ b/HellionChat/MessageStore.cs @@ -181,6 +181,14 @@ internal class MessageStore : IDisposable private readonly IPlatformUtil _platformUtil; private readonly IPluginLogProxy _logger; + // Readiness gate for the FTS5 full-text index. Volatile so the DbViewer's + // per-frame IsFtsIndexBuilt read sees the flip the moment the bulk-insert + // worker calls MarkFtsIndexBuilt(). Set in the ctor by InitFtsReadyCache: + // true when the index already has rows (no rebuild needed) or when the + // messages table itself is empty (nothing to index yet); false otherwise. + private volatile bool _ftsReady; + public bool IsFtsIndexBuilt => _ftsReady; + internal MessageStore(string dbPath, IPlatformUtil platformUtil, IPluginLogProxy logger) { DbPath = dbPath; @@ -188,6 +196,7 @@ internal class MessageStore : IDisposable _logger = logger; Connection = Connect(); Migrate(); + InitFtsReadyCache(); } public void Dispose() @@ -547,6 +556,155 @@ internal class MessageStore : IDisposable return (long)(cmd.ExecuteScalar() ?? 0L) > 0; } + // Decides whether the FTS index already covers the messages table. Called + // once after Migrate -- empty messages-table is "ready" because there is + // nothing to index yet; a populated fts-table is "ready" because some + // previous run filled it. A populated messages-table with an empty + // fts-table is the "needs rebuild" case the worker (Plugin.cs LoadAsync) + // picks up. + internal void InitFtsReadyCache() + { + using (var cmd = Connection.CreateCommand()) + { + cmd.CommandText = "SELECT count(*) FROM messages_fts;"; + var ftsRows = (long)(cmd.ExecuteScalar() ?? 0L); + if (ftsRows > 0) + { + _ftsReady = true; + return; + } + } + + using var cmd2 = Connection.CreateCommand(); + cmd2.CommandText = "SELECT count(*) FROM messages;"; + var messageRows = (long)(cmd2.ExecuteScalar() ?? 0L); + _ftsReady = messageRows == 0; + } + + // Opens a worker-owned SqliteConnection on the same db path. Used by the + // FTS rebuild worker so the bulk-insert writer stream does not contend + // with the live UpsertMessage path on the primary Connection (WAL allows + // N readers + 1 writer; two writer sessions on the same connection are + // not safe per Microsoft.Data.Sqlite). Caller closes+disposes the + // returned connection and only then calls MarkFtsIndexBuilt() -- the + // DbViewer never sees IsFtsIndexBuilt=true while the worker connection + // is still alive. + internal SqliteConnection OpenSecondaryConnection() + { + var conn = new SqliteConnection(BuildConnectionString(DbPath)); + conn.Open(); + ApplyPragmas(conn); + return conn; + } + + // Worker-only mutator. The bulk-insert worker is the single legitimate + // caller; the flag flips after the worker has closed its own connection. + internal void MarkFtsIndexBuilt() => _ftsReady = true; + + // Builds the FTS5 index from scratch on a worker-owned SqliteConnection. + // Chunked-commit (every 500 rows + 5ms sleep) releases the WAL writer + // lock between transactions so the live PendingMessageThread UpsertMessage + // path on the primary Connection does not hit "database is locked" after + // DefaultTimeout=5s. The Thread.Sleep is intentional: it gives the live + // writer a deterministic window to acquire the lock before we re-take + // it for the next chunk. + // + // Cancellation: checked at the top of each row and again after each + // chunk commit, so a Dispose-during-rebuild collapses on the next row + // without trashing the half-built index (DELETE FROM messages_fts at + // the start makes the next run idempotent). + public long RebuildFtsIndex( + SqliteConnection conn, + IProgress progress, + CancellationToken ct + ) + { + const int ChunkSize = 500; + + using (var clear = conn.CreateCommand()) + { + clear.CommandText = "DELETE FROM messages_fts;"; + clear.ExecuteNonQuery(); + } + + long total; + using (var totalCmd = conn.CreateCommand()) + { + totalCmd.CommandText = "SELECT count(*) FROM messages;"; + total = (long)(totalCmd.ExecuteScalar() ?? 0L); + } + + long done = 0; + + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT Id, Sender, Content FROM messages ORDER BY Id;"; + using var reader = cmd.ExecuteReader(); + + using var insert = conn.CreateCommand(); + insert.CommandText = + "INSERT INTO messages_fts(message_guid, sender_text, content_text) VALUES ($g, $s, $c);"; + var pG = insert.CreateParameter(); + pG.ParameterName = "$g"; + insert.Parameters.Add(pG); + var pS = insert.CreateParameter(); + pS.ParameterName = "$s"; + insert.Parameters.Add(pS); + var pC = insert.CreateParameter(); + pC.ParameterName = "$c"; + insert.Parameters.Add(pC); + + // Nullable so the finally can dispose exactly once whether the loop + // ends normally, via cancellation between Dispose and BeginTransaction, + // or via an exception in the body. + SqliteTransaction? transaction = conn.BeginTransaction(); + insert.Transaction = transaction; + try + { + while (reader.Read()) + { + ct.ThrowIfCancellationRequested(); + + var guidBytes = (byte[])reader.GetValue(0); + var senderChunks = MessagePackSerializer.Deserialize>( + reader.GetFieldValue(1), + MsgPackOptions + ); + var contentChunks = MessagePackSerializer.Deserialize>( + reader.GetFieldValue(2), + MsgPackOptions + ); + + pG.Value = Convert.ToHexString(guidBytes); + pS.Value = ChunkUtil.ToRawString(senderChunks); + pC.Value = ChunkUtil.ToRawString(contentChunks); + insert.ExecuteNonQuery(); + done++; + + if (done % ChunkSize == 0) + { + transaction.Commit(); + transaction.Dispose(); + transaction = null; + progress.Report(done); + + Thread.Sleep(5); + ct.ThrowIfCancellationRequested(); + + transaction = conn.BeginTransaction(); + insert.Transaction = transaction; + } + } + transaction?.Commit(); + } + finally + { + transaction?.Dispose(); + } + + progress.Report(done); + return total; + } + internal void UpsertMessage(Message message) { // Privacy filter -- drop disallowed ChatTypes before they reach storage. diff --git a/HellionChat/Plugin.cs b/HellionChat/Plugin.cs index ea1594b..7263120 100755 --- a/HellionChat/Plugin.cs +++ b/HellionChat/Plugin.cs @@ -14,6 +14,7 @@ using HellionChat.Ipc; using HellionChat.Resources; using HellionChat.Ui; using HellionChat.Util; +using Microsoft.Data.Sqlite; namespace HellionChat; @@ -127,6 +128,12 @@ public sealed class Plugin : IAsyncDalamudPlugin internal int DeferredSaveFrames = -1; + // Cancels the v1.4.8 FTS5 bulk-insert worker on plugin teardown. The + // worker runs off the framework thread on its own SqliteConnection, so a + // Dispose mid-rebuild must signal cancellation before MessageManager + // tears down (the worker logs "rebuild failed" via Log on error paths). + private CancellationTokenSource? _ftsRebuildCts; + // Serialises retention sweeps so a manual trigger and the 24h auto-sweep // can't run in parallel. Volatile because the ImGui thread reads it outside // the lock to gate the manual button. @@ -282,6 +289,113 @@ public sealed class Plugin : IAsyncDalamudPlugin if (Interface.Reason is not PluginLoadReason.Boot) MessageManager.FilterAllTabsAsync(); + // Kick the FTS5 rebuild worker if Migrate4 just added the schema or + // a previous run was cut short (InitFtsReadyCache leaves _ftsReady + // false in that case). Runs off the framework thread on its own + // SqliteConnection so the live UpsertMessage path keeps flowing + // through the chunked-commit windows. + _ftsRebuildCts = new CancellationTokenSource(); + if (!MessageManager.Store.IsFtsIndexBuilt) + { + var token = _ftsRebuildCts.Token; + _ = Task.Run( + async () => + { + // FQN: Plugin.Notification (Z.74) shadows the type name. + Dalamud.Interface.ImGuiNotification.IActiveNotification? notif = null; + try + { + notif = Notification.AddNotification( + new Dalamud.Interface.ImGuiNotification.Notification + { + Title = "Hellion Chat", + Content = "Indexing chat history for full-text search...", + Type = Dalamud + .Interface + .ImGuiNotification + .NotificationType + .Info, + Minimized = false, + InitialDuration = TimeSpan.FromMinutes(10), + } + ); + + // Progress raises this callback on the captured + // sync-context (Task.Run worker pool). IActiveNotification + // is ImGui-backed and mutates the UI, so marshal the + // mutation onto the framework thread via RunOnTick. + var progress = new Progress(done => + { + Framework.RunOnTick(() => + { + if (notif is { } n) + n.Content = $"Indexing chat history: {done:N0} messages..."; + }); + }); + + // Worker-owned connection. Closed+disposed before we + // flip the readiness flag so the DbViewer never sees + // IsFtsIndexBuilt=true while the worker connection + // is still alive. + SqliteConnection? workerConn = null; + try + { + workerConn = MessageManager.Store.OpenSecondaryConnection(); + var total = await Task.Run( + () => + MessageManager.Store.RebuildFtsIndex( + workerConn, + progress, + token + ), + token + ) + .ConfigureAwait(false); + + workerConn.Close(); + workerConn.Dispose(); + workerConn = null; + MessageManager.Store.MarkFtsIndexBuilt(); + + if (notif is { } final) + { + final.Content = $"Indexed {total:N0} messages."; + final.Type = Dalamud + .Interface + .ImGuiNotification + .NotificationType + .Success; + final.InitialDuration = TimeSpan.FromSeconds(5); + } + } + finally + { + workerConn?.Dispose(); + } + } + catch (OperationCanceledException) + { + notif?.DismissNow(); + } + catch (Exception ex) + { + Log.Error(ex, "FTS index rebuild failed"); + if (notif is { } err) + { + err.Content = + "Full-text indexing failed -- search will use local filter only."; + err.Type = Dalamud + .Interface + .ImGuiNotification + .NotificationType + .Error; + } + } + }, + _ftsRebuildCts.Token + ); + } + Interface.UiBuilder.DisableCutsceneUiHide = true; Interface.UiBuilder.DisableGposeUiHide = true; @@ -328,6 +442,19 @@ public sealed class Plugin : IAsyncDalamudPlugin failure = CaptureFailure(failure, () => Interface.UiBuilder.Draw -= Draw); failure = CaptureFailure(failure, () => Framework.Update -= FrameworkUpdate); + // Signal the FTS rebuild worker to bail. Runs before MessageManager + // tears down so the worker's "rebuild failed" log path still finds + // a live Log static. Worker owns its own SqliteConnection and disposes + // it itself; we only flip the cancellation flag here. + failure = CaptureFailure( + failure, + () => + { + _ftsRebuildCts?.Cancel(); + _ftsRebuildCts?.Dispose(); + } + ); + // Flush a pending DeferredSave — FrameworkUpdate won't fire it anymore. failure = CaptureFailure( failure,