Merge pull request #41 from CVEProject/hk/006a_exact_phrase_search

work-in-progress for #6 (exact phrase search)

Author: hkong-mitre

src/search/SearchRequest.ts

@@ -87,18 +87,23 @@ export class SearchRequest {
    */
   static tokenizeSearchText(searchText: string): string[] {
     // based on code generated by gemini
-    const regex = /"([^"]*)"|\S+/g; // tokenizes all words between double quotes as well as every word outside of quotes
-    let tokens = [];
-    let match;
-
-    while ((match = regex.exec(searchText)) !== null) {
-      if (match[1]) {
-        tokens.push(match[1]); // Add the content within quotes
-      } else {
-        tokens.push(match[0]); // Add the non-quoted token
+    let matches = SearchRequest.extractQuotedStrings(searchText);
+    // filter out ',' tokens
+    matches = matches.flatMap(item => item === ',' ? [] : item);
+
+    // Process each match to unescape characters in quoted strings.
+    return matches.map(match => {
+      // Check if the match is a quoted string.
+      if (match.startsWith('"') && match.endsWith('"')) {
+        // Remove leading/trailing quotes and then unescape the characters.
+        let unescaped = match.slice(1, -1);
+        unescaped = unescaped.replace(/\\"/g, '"');
+        unescaped = unescaped.replace(/\\\\/g, '\\');
+        return unescaped;
       }
-    }
-    return tokens;
+      // For unquoted words, return as-is.
+      return match;
+    });
   }
 
 
@@ -403,6 +408,25 @@ export class SearchRequest {
   }
 
 
+  /** returns true iff searchText is a quoted string
+   */
+  static isQuotedString = (searchText: string): boolean => {
+    return new RegExp(/"([^"]*)"/g).test(searchText);
+    // return new RegExp(/"(?:[^"\\]|\\.)+"|[^\s,]+/g).test(searchText);
+  };
+
+
+  static extractQuotedStrings = (searchText: string): string[] => {
+    // based on code generated by gemini
+    // Regex to find tokens: quoted strings with escaped quotes, or unquoted words.
+    const regex = /"[^"]*"|[\S]+/g
+    // const regex = /"([^"]*)"/g;
+    const matches = searchText.match(regex);
+    return matches ?? [];
+  }
+
+
+
   /**
    *  determine the SearchReuestTypeId based on searchText
    *  @param searchText the search text
@@ -414,7 +438,12 @@ export class SearchRequest {
     if (searchText.includes('{')) {
       return 'SEARCH_STRING_CANNOT_CONTAIN_RESERVED_CHARACTERS'
     }
-    // disallow wildcards
+    // else if (searchText[0] == '"' && searchText[searchText.length - 1] == '"') {
+    // else if (new RegExp(/"([^"]*)"/g).test(searchText)) {
+    // else if (new RegExp(/"(?:[^"\\]|\\.)+"|[^\s,]+/g).test(searchText)) {
+    else if (SearchRequest.isQuotedString(searchText)) {
+      return 'SEARCH_PHRASE';
+    }
     else if (searchText.includes('*')) {
       return 'SEARCH_AS_WILDCARD_ASTERISK'
     }
@@ -429,11 +458,6 @@ export class SearchRequest {
     else if (searchText.includes('%')) {
       return 'WILDCARD_PERCENT_SEARCH_NOT_SUPPORTED'
     }
-    // else if (searchText[0] == '"' && searchText[searchText.length - 1] == '"') {
-    else if (new RegExp(/"([^"]*)"/g).test(searchText)) {
-      return 'SEARCH_PHRASE'
-    }
-
     // process urls
     else if (SearchRequest.isUrl(searchText)) {
       // original from https://jsfiddle.net/DanielD/8S4nq/