Fix FTS5 syntax errors when searching with punctuation characters (#2085)

* Initial plan

* Add failing tests for punctuation in search and fix FTS query escaping

Co-authored-by: myieye <12587509+myieye@users.noreply.github.com>

* Improve FTS query escaping to handle punctuation correctly

Co-authored-by: myieye <12587509+myieye@users.noreply.github.com>

* Address code review feedback - extract constants and improve matching logic

Co-authored-by: myieye <12587509+myieye@users.noreply.github.com>

* Simplify FTS5 escaping - always wrap user queries in quotes

Co-authored-by: myieye <12587509+myieye@users.noreply.github.com>

* Fix Non-FTS test cases to use < 3 character search terms

Co-authored-by: myieye <12587509+myieye@users.noreply.github.com>

* Simplify comments and interpolation

* Fix comment

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: myieye <12587509+myieye@users.noreply.github.com>
Co-authored-by: Tim Haasdyk <tim_haasdyk@sil.org>

Author: Copilot

backend/FwLite/LcmCrdt/FullTextSearch/EntrySearchService.cs

@@ -26,18 +26,20 @@ public class EntrySearchService(LcmCrdtDbContext dbContext, ILogger<EntrySearchS
 {
     internal IQueryable<EntrySearchRecord> EntrySearchRecords => dbContext.Set<EntrySearchRecord>();
 
-    //ling2db table
+    //linq2db table
     private ITable<EntrySearchRecord> EntrySearchRecordsTable => dbContext.GetTable<EntrySearchRecord>();
 
     public IQueryable<Entry> FilterAndRank(IQueryable<Entry> queryable,
         string query,
         bool rankResults,
         bool orderAscending)
     {
+        var ftsString = ToFts5LiteralString(query);
+
         //starting from EntrySearchRecordsTable rather than queryable otherwise linq2db loses track of the table
         var filtered = from searchRecord in EntrySearchRecordsTable
             from entry in queryable.InnerJoin(r => r.Id == searchRecord.Id)
-            where Sql.Ext.SQLite().Match(searchRecord, query) && (entry.LexemeForm.SearchValue(query)
+            where Sql.Ext.SQLite().Match(searchRecord, ftsString) && (entry.LexemeForm.SearchValue(query)
                                                                   || entry.CitationForm.SearchValue(query)
                                                                   || entry.Senses.Any(s => s.Gloss.SearchValue(query)))
             select new { entry, searchRecord };
@@ -57,6 +59,14 @@ where Sql.Ext.SQLite().Match(searchRecord, query) && (entry.LexemeForm.SearchVal
         return filtered.Select(t => t.entry);
     }
 
+    private static string ToFts5LiteralString(string query)
+    {
+        // https://sqlite.org/fts5.html#fts5_strings
+        // - escape double quotes by doubling them
+        // - wrap the entire query in quotes
+        return $"\"{query.Replace("\"", "\"\"")}\"";
+    }
+
     public bool ValidSearchTerm(string query) => query.Normalize(NormalizationForm.FormC).Length >= 3;
 
     public static string? Best(MultiString ms, WritingSystem[] wss, WritingSystemType type)
@@ -243,6 +253,9 @@ private static EntrySearchRecord ToEntrySearchRecord(Entry entry, WritingSystem[
 
     public IAsyncEnumerable<EntrySearchRecord> Search(string query)
     {
+        // (Currently only used by tests)
+        // This method is for advanced queries with FTS5 syntax (wildcards, operators, etc.).
+        // So, we don't use ToFts5LiteralString.
         return EntrySearchRecords
             .ToLinqToDB()
             .Where(e => Sql.Ext.SQLite().Match(e, query))

backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs

@@ -438,6 +438,22 @@ public async Task HeadwordOrder(string searchTerm, string wordsAndGlosses, strin
                 .Select(e => e.LexemeForm["en"]);
         string.Join(",", result).Should().Be(expectedOrder);
     }
+
+    [Theory]
+    [InlineData("a;", "a;test")]  // Non-FTS search (2 chars total)
+    [InlineData("abc;", "abc;test")] // FTS search (4 chars total)
+    [InlineData("a:", "a:test")]  // Non-FTS with colon (2 chars total)
+    [InlineData("abc:", "abc:test")] // FTS with colon (4 chars total)
+    [InlineData("a\"", "a\"test")] // Non-FTS with quote (2 chars total)
+    [InlineData("abc\"", "abc\"test")] // FTS with quote (4 chars total)
+    [InlineData("a'", "a'test")]  // Non-FTS with apostrophe (U+0027, 2 chars total)
+    [InlineData("abc'", "abc'test")] // FTS with apostrophe (U+0027, 4 chars total)
+    public async Task PunctuationWorks(string searchTerm, string word)
+    {
+        await Api.CreateEntry(new Entry { LexemeForm = { ["en"] = word } });
+        var results = await Api.SearchEntries(searchTerm).Select(e => e.LexemeForm["en"]).ToArrayAsync();
+        results.Should().Contain(word);
+    }
 }
 
 // A seperate class to preserve the readability of the results in the main test class