[NOCSL] Adds extra url validation for android referrers (#420)

* add url validation for android referres

* lint spell check

* add url trimming

* remove mutation

Author: stanlp1

cspell.json

@@ -48,6 +48,7 @@
     "atcs",
     "testdata",
     "Bytespider",
-    "Timespans"
+    "Timespans",
+    "googlequicksearchbox"
   ]
 }

spec/src/utils/helpers.js

@@ -15,6 +15,7 @@ const {
   isNil,
   getWindowLocation,
   getCanonicalUrl,
+  getDocumentReferrer,
   dispatchEvent,
   createCustomEvent,
   hasOrderIdRecord,
@@ -23,6 +24,8 @@ const {
   stringify,
   convertResponseToJson,
   addHTTPSToString,
+  trimUrl,
+  cleanAndValidateUrl,
 } = require('../../../test/utils/helpers'); // eslint-disable-line import/extensions
 const jsdom = require('./jsdom-global');
 const store = require('../../../test/utils/store'); // eslint-disable-line import/extensions
@@ -215,6 +218,18 @@ describe('ConstructorIO - Utils - Helpers', () => {
     });
 
     describe('getCanonicalUrl', () => {
+      it('Should return android app referrers in a valid url structure', () => {
+        const cleanup = jsdom();
+
+        const canonicalUrl = 'android-app://com.google.android.googlequicksearchbox/';
+        const canonicalEle = document.querySelector('[rel=canonical]');
+        canonicalEle.setAttribute('href', canonicalUrl);
+
+        expect(getCanonicalUrl()).to.equal('https://com.google.android.googlequicksearchbox/');
+
+        cleanup();
+      });
+
       it('Should return the canonical URL from the DOM link element', () => {
         const cleanup = jsdom();
 
@@ -264,6 +279,67 @@ describe('ConstructorIO - Utils - Helpers', () => {
       });
     });
 
+    describe('getDocumentReferrer', () => {
+      it('Should return android app referrers in a valid url structure', () => {
+        const cleanup = jsdom();
+
+        const referrerUrl = 'android-app://com.google.android.googlequicksearchbox/';
+        Object.defineProperty(document, 'referrer', {
+          value: referrerUrl,
+          configurable: true,
+        });
+
+        expect(getDocumentReferrer()).to.equal('https://com.google.android.googlequicksearchbox/');
+
+        cleanup();
+      });
+
+      it('Should return the referrer URL from the document', () => {
+        const cleanup = jsdom();
+
+        const referrerUrl = 'https://constructor.io/products/item';
+        Object.defineProperty(document, 'referrer', {
+          value: referrerUrl,
+          configurable: true,
+        });
+
+        expect(getDocumentReferrer()).to.equal(referrerUrl);
+
+        cleanup();
+      });
+
+      it('Should return null for a relative url', () => {
+        const cleanup = jsdom();
+
+        const relativeUrl = '/products/item';
+        Object.defineProperty(document, 'referrer', {
+          value: relativeUrl,
+          configurable: true,
+        });
+
+        const result = getDocumentReferrer();
+        expect(result).to.be.null;
+
+        cleanup();
+      });
+
+      it('Should return null when referrer is empty', () => {
+        const cleanup = jsdom();
+
+        Object.defineProperty(document, 'referrer', {
+          value: '',
+          configurable: true,
+        });
+
+        expect(getDocumentReferrer()).to.be.null;
+        cleanup();
+      });
+
+      it('Should return null when not in a DOM context', () => {
+        expect(getDocumentReferrer()).to.be.null;
+      });
+    });
+
     describe('dispatchEvent', () => {
       it('Should dispatch an event if in a DOM context', () => {
         const cleanup = jsdom();
@@ -539,5 +615,88 @@ describe('ConstructorIO - Utils - Helpers', () => {
         expect(addHTTPSToString(testUrl)).to.equal(null);
       });
     });
+
+    describe('trimUrl', () => {
+      it('Should return the URL as-is if it is under the max length', () => {
+        const testUrl = new URL('https://www.constructor.io/search?q=test');
+        const result = trimUrl(testUrl);
+
+        expect(result).to.equal('https://www.constructor.io/search?q=test');
+      });
+
+      it('Should remove the longest parameter when URL exceeds max length', () => {
+        const longValue = 'a'.repeat(2000);
+        const testUrl = new URL(`https://www.constructor.io/search?short=b&long=${longValue}`);
+        const result = trimUrl(testUrl, 100);
+
+        expect(result).to.include('short=b');
+        expect(result).to.not.include('long=');
+        expect(result.length).to.be.at.most(100);
+      });
+
+      it('Should remove multiple parameters starting with the longest', () => {
+        const testUrl = new URL('https://www.constructor.io/search?a=1&b=22&c=333&d=4444');
+        const result = trimUrl(testUrl, 50);
+
+        expect(result.length).to.be.at.most(50);
+      });
+
+      it('Should truncate URL if removing all parameters is not enough', () => {
+        const longPath = 'a'.repeat(2000);
+        const testUrl = new URL(`https://www.constructor.io/${longPath}`);
+        const result = trimUrl(testUrl, 100);
+
+        expect(result.length).to.equal(100);
+      });
+
+      it('Should use custom maxLen parameter', () => {
+        const testUrl = new URL('https://www.constructor.io/search?param=value');
+        const customMaxLen = 30;
+        const result = trimUrl(testUrl, customMaxLen);
+
+        expect(result.length).to.be.at.most(customMaxLen);
+      });
+    });
+
+    describe('cleanAndValidateUrl', () => {
+      it('Should return a valid URL string', () => {
+        const testUrl = 'https://www.constructor.io/search?q=test';
+        const result = cleanAndValidateUrl(testUrl);
+
+        expect(result).to.equal(testUrl);
+      });
+
+      it('Should handle android-app referrers by converting to https', () => {
+        const androidUrl = 'android-app://com.google.android.googlequicksearchbox/path';
+        const result = cleanAndValidateUrl(androidUrl);
+
+        expect(result).to.include('https://');
+        expect(result).to.include('com.google.android.googlequicksearchbox');
+      });
+
+      it('Should return null for invalid URLs', () => {
+        const invalidUrl = 'not a valid url';
+        const result = cleanAndValidateUrl(invalidUrl);
+
+        expect(result).to.be.null;
+      });
+
+      it('Should handle relative URLs with baseUrl', () => {
+        const relativeUrl = '/search?q=test';
+        const baseUrl = 'https://www.constructor.io';
+        const result = cleanAndValidateUrl(relativeUrl, baseUrl);
+
+        expect(result).to.include('https://www.constructor.io/search?q=test');
+      });
+
+      it('Should trim URLs that exceed max length', () => {
+        const longValue = 'a'.repeat(2000);
+        const testUrl = `https://www.constructor.io/search?param=${longValue}`;
+        const result = cleanAndValidateUrl(testUrl);
+
+        expect(result).to.not.be.null;
+        expect(result.length).to.be.at.most(2000);
+      });
+    });
   }
 });

src/utils/helpers.js

@@ -19,6 +19,7 @@ const PII_REGEX = [
   },
   // Add more PII REGEX
 ];
+const URL_MAX_LEN = 2000;
 
 const utils = {
   trimNonBreakingSpaces: (string) => string.replace(/\s/g, ' ').trim(),
@@ -43,6 +44,61 @@ const utils = {
 
     return cleanedParams;
   },
+  trimUrl: (urlObj, maxLen = URL_MAX_LEN) => {
+    let urlString = urlObj.toString();
+
+    if (urlString.length <= maxLen) {
+      return urlString;
+    }
+
+    const urlCopy = new URL(urlObj.toString());
+    const { searchParams } = urlCopy;
+    const paramEntries = Array.from(searchParams.entries());
+
+    if (paramEntries.length === 0) {
+      return utils.truncateString(urlString, maxLen);
+    }
+
+    paramEntries.sort((a, b) => {
+      const aLength = a[0].length + a[1].length;
+      const bLength = b[0].length + b[1].length;
+      return bLength - aLength;
+    });
+
+    for (let i = 0; i < paramEntries.length; i += 1) {
+      if (urlString.length <= maxLen) {
+        break;
+      }
+      searchParams.delete(paramEntries[i][0]);
+      urlString = urlCopy.toString();
+    }
+
+    if (urlString.length > maxLen) {
+      return utils.truncateString(urlString, maxLen);
+    }
+
+    return urlString;
+  },
+
+  cleanAndValidateUrl: (url, baseUrl = undefined) => {
+    let validatedUrl = null;
+
+    try {
+      // Handle android app referrers
+      if (url?.startsWith('android-app')) {
+        url = url?.replace('android-app', 'https');
+      }
+
+      const urlObj = new URL(url, baseUrl);
+      const trimmedUrl = new URL(utils.trimUrl(urlObj));
+
+      validatedUrl = trimmedUrl.toString();
+    } catch (e) {
+      // do nothing
+    }
+
+    return validatedUrl;
+  },
 
   throwHttpErrorFromResponse: (error, response) => response.json().then((json) => {
     error.message = json.message;
@@ -92,11 +148,17 @@ const utils = {
   },
 
   getDocumentReferrer: () => {
-    if (utils.canUseDOM()) {
-      return document?.referrer;
+    let documentReferrer = null;
+
+    try {
+      if (utils.canUseDOM()) {
+        documentReferrer = utils.cleanAndValidateUrl(document.referrer);
+      }
+    } catch (e) {
+      // do nothing
     }
 
-    return null;
+    return documentReferrer;
   },
 
   getCanonicalUrl: () => {
@@ -108,8 +170,7 @@ const utils = {
         const href = linkEle?.getAttribute('href');
 
         if (href) {
-          const url = new URL(href, document.location.href);
-          canonicalURL = url.toString();
+          canonicalURL = utils.cleanAndValidateUrl(href, document.location.href);
         }
       }
     } catch (e) {