From 924930a29c0c75aecc04ca6f1b0c8f0cc9aa19f6 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Wed, 5 Jul 2023 14:45:52 +0200
Subject: [PATCH 01/31] #80 add twitter login

---
 src/config/constants.ts       |   2 +
 src/config/puppeteerConfig.ts |  16 +++-
 src/helpers/handlers.ts       | 167 +++++++++++++++++++++++++++++-----
 3 files changed, 158 insertions(+), 27 deletions(-)

diff --git a/src/config/constants.ts b/src/config/constants.ts
index e80a712..65604bc 100644
--- a/src/config/constants.ts
+++ b/src/config/constants.ts
@@ -7,3 +7,5 @@ export const META_STAMP_CANVAS_DEFAULT_WIDTH = 900;
 export const META_STAMP_CANVAS_DEFAULT_HEIGHT = 1000;
 export const WATERMARK_DEFAULT_WIDTH = 512;
 export const WATERMARK_DEFAULT_HEIGHT = 512;
+export const TWITTER_NEXT_BUTTON_TEXT_CONTENT = 'next';
+export const TWITTER_LOGIN_BUTTON_TEXT_CONTENT = 'log in';
diff --git a/src/config/puppeteerConfig.ts b/src/config/puppeteerConfig.ts
index c199f05..db826de 100644
--- a/src/config/puppeteerConfig.ts
+++ b/src/config/puppeteerConfig.ts
@@ -4,7 +4,13 @@ export const puppeteerDefaultConfig: {
   };
   viewport: { width: number; height: number };
   userAgent: string;
-  page: { goto: { gotoWaitUntilIdle: { waitUntil: 'networkidle0' } } };
+  page: { goto: { gotoWaitUntilIdle: { waitUntil: 'networkidle2' | 'networkidle0' } } };
+  defaultBoundingBox: {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+  };
 } = {
   launch: {
     args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-web-security'],
@@ -13,9 +19,15 @@ export const puppeteerDefaultConfig: {
     width: 1024,
     height: 1024,
   },
+  defaultBoundingBox: {
+    x: 97, // offset by the width of the vertical header in viewport with width = 1024
+    y: 53, // offest by the height of the main title
+    width: 598, // widht of the article in viewport with width = 1024
+    height: 1024 - 53, // viewport height 1024 - defaultBoundingBox.y
+  },
   userAgent:
     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
   page: {
-    goto: { gotoWaitUntilIdle: { waitUntil: 'networkidle0' } },
+    goto: { gotoWaitUntilIdle: { waitUntil: 'networkidle2' } },
   },
 };
diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index cf438ef..dc22bcd 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -1,3 +1,5 @@
+import fs from 'fs/promises';
+import path from 'path';
 import {
   getDnsInfo,
   getMediaUrlsToUpload,
@@ -7,16 +9,52 @@ import {
   makeTweetUrlWithId,
   trimUrl,
 } from '../helpers';
-import { Browser, HTTPResponse, Page } from 'puppeteer';
+import puppeteer, { Browser, ElementHandle, HTTPResponse, Page } from 'puppeteer';
 import { IGetScreenshotResponseData, IMetadata, ITweetTimelineEntry } from '../types';
-import { DEFAULT_TIMEOUT_MS, screenshotResponseDataOrderedKeys } from '../config';
+import {
+  DEFAULT_TIMEOUT_MS,
+  TWITTER_LOGIN_BUTTON_TEXT_CONTENT,
+  TWITTER_NEXT_BUTTON_TEXT_CONTENT,
+  screenshotResponseDataOrderedKeys,
+} from '../config';
 import { Request, Response } from 'express';
 import { reportError } from '../helpers';
 import { puppeteerDefaultConfig } from '../config';
-import puppeteer from 'puppeteer';
 import { makeBufferFromBase64ImageUrl, makeStampedImage } from './images';
 import { createTweetData } from '../models';
 import { uploadQueue } from '../queue';
+import { processPWD } from '../prestart';
+
+export const getBoundingBox = async (element: ElementHandle | null) => {
+  if (!!element) {
+    const elementBoundingBox = await element.boundingBox();
+    return elementBoundingBox ? elementBoundingBox : puppeteerDefaultConfig.defaultBoundingBox;
+  }
+  return puppeteerDefaultConfig.defaultBoundingBox;
+};
+
+export const getSavedCookies = (): Promise<
+  { name: string; value: string; [id: string]: string | number | boolean }[] | null
+> =>
+  fs
+    .readFile(path.resolve(processPWD, 'data', 'cookies.json'), 'utf-8')
+    .then((cockiesString) => JSON.parse(cockiesString))
+    .catch((error) => {
+      console.error(error.message);
+      return null;
+    });
+
+export const findElementByTextContentAsync = async (
+  elements: ElementHandle[],
+  textValue: string,
+): Promise<ElementHandle | null> => {
+  for (let i = 0; i < elements.length; i += 1) {
+    const property = await elements[i].getProperty('textContent');
+    const value = await property.jsonValue();
+    if (value?.toLocaleLowerCase() === textValue.toLocaleLowerCase()) return elements[i];
+  }
+  return null;
+};
 
 export const getMetaDataPromise = (page: Page, tweetId: string) =>
   new Promise<string>((resolve, reject) => {
@@ -63,6 +101,104 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
     });
   });
 
+export const screenshotPromise = async (page: Page, tweetId: string) => {
+  const cookies = await getSavedCookies();
+
+  if (!cookies) {
+    await page.goto(
+      'https://twitter.com/i/flow/login',
+      puppeteerDefaultConfig.page.goto.gotoWaitUntilIdle,
+    );
+    await page.waitForSelector('input');
+    console.log(process.env.TWITTER_PASSWORD);
+    console.log(process.env.TWITTER_USERNAME);
+    if (!process.env.TWITTER_USERNAME) {
+      console.log(`Twitter username is falsy: '${process.env.TWITTER_USERNAME}'`);
+      return null;
+    }
+    await page.type('input', process.env.TWITTER_USERNAME);
+
+    const loginPageButtons = await page.$$('[role="button"]');
+
+    const nextButton = await findElementByTextContentAsync(
+      loginPageButtons,
+      TWITTER_NEXT_BUTTON_TEXT_CONTENT,
+    );
+
+    console.log(nextButton);
+    if (!nextButton) {
+      console.log(
+        `Twitter Log in page error: can not get button '${TWITTER_NEXT_BUTTON_TEXT_CONTENT}'`,
+      );
+      return null;
+    }
+
+    await nextButton.click();
+    console.log('click');
+    await page.waitForSelector('input');
+    console.log('click');
+    if (!process.env.TWITTER_PASSWORD) {
+      console.log(`Twitter password is falsy: '${process.env.TWITTER_PASSWORD}'`);
+      return null;
+    }
+    await page.type('input', process.env.TWITTER_PASSWORD);
+
+    const buttons2 = await page.$$('[role="button"]');
+
+    const logInButton = await findElementByTextContentAsync(
+      buttons2,
+      TWITTER_LOGIN_BUTTON_TEXT_CONTENT,
+    );
+
+    if (!logInButton) {
+      console.log(
+        `Twitter Log in page error: can not get button '${TWITTER_NEXT_BUTTON_TEXT_CONTENT}'`,
+      );
+
+      return null;
+    }
+
+    await logInButton.click();
+
+    await page.waitForSelector('article');
+    const coockies = await page.cookies();
+    fs.writeFile(path.resolve(processPWD, 'data', 'cookies.json'), JSON.stringify(coockies));
+  } else {
+    await page.setCookie(...cookies);
+  }
+
+  const tweetUrl = makeTweetUrlWithId(tweetId);
+
+  await page.goto(tweetUrl, puppeteerDefaultConfig.page.goto.gotoWaitUntilIdle);
+
+  const mainElement = await page.waitForSelector('main');
+  const mailboundingBox = await getBoundingBox(mainElement);
+  const articleElement = await page.waitForSelector(`article:has(a[href$="/status/${tweetId}"])`);
+  const articleBoundingBox = await getBoundingBox(articleElement);
+
+  articleBoundingBox.y -= mailboundingBox.y;
+
+  await page.evaluate(() => {
+    const bottomBars = document.querySelectorAll('[data-testid="BottomBar"]');
+    bottomBars.forEach((bottomBarElement) => {
+      const bottomBar = bottomBarElement as HTMLElement;
+      bottomBar.style.display = 'none';
+    });
+    const dialogs = document.querySelectorAll('[role="dialog"]');
+    dialogs.forEach((bottomBarElement) => {
+      const bottomBar = bottomBarElement as HTMLElement;
+      bottomBar.style.display = 'none';
+    });
+  });
+
+  const screenshotImageBuffer: Buffer = await page.screenshot({
+    clip: { ...articleBoundingBox },
+    path: `${tweetId}.png`,
+  });
+
+  return makeImageBase64UrlfromBuffer(screenshotImageBuffer);
+};
+
 export const screenshotWithPuppeteer = async (
   request: Request,
   response: Response,
@@ -120,31 +256,12 @@ const getScreenshotWithPuppeteer = async (
     });
     await page.setUserAgent(puppeteerDefaultConfig.userAgent);
 
-    const screenShotPromise: Promise<string> = page
-      .goto(tweetUrl, puppeteerDefaultConfig.page.goto.gotoWaitUntilIdle)
-      .then(async () => {
-        await page.evaluate(() => {
-          const bottomBar = document.querySelector('[data-testid="BottomBar"]') as HTMLElement;
-          if (bottomBar) {
-            bottomBar.style.display = 'none';
-          }
-        });
-        const articleElement = (await page.waitForSelector('article'))!;
-        const boundingBox = (await articleElement.boundingBox())!;
-
-        const screenshotImageBuffer: Buffer = await page.screenshot({
-          clip: { ...boundingBox },
-        });
-
-        return makeImageBase64UrlfromBuffer(screenshotImageBuffer);
-      });
-
-    const promises: Iterable<Promise<string>> = [
+    const promises: Iterable<Promise<string | null>> = [
       getTweetDataPromise(page, tweetId),
       getMetaDataPromise(page, tweetId),
-      screenShotPromise,
+      screenshotPromise(page, tweetId),
     ];
-    const allData = await Promise.allSettled<string>(promises);
+    const allData = await Promise.allSettled<string | null>(promises);
     const fetchedData = allData.reduce<IGetScreenshotResponseData>(
       (acc, val, index) => {
         acc[screenshotResponseDataOrderedKeys[index]] =

From 70f18455393b63b0b2f2f248958d3aceedbf6143 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Wed, 5 Jul 2023 17:52:13 +0200
Subject: [PATCH 02/31] #80 add error handling after tests failing

---
 src/helpers/handlers.ts | 174 ++++++++++++++++++++++------------------
 1 file changed, 98 insertions(+), 76 deletions(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index dc22bcd..552c6c4 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -56,6 +56,21 @@ export const findElementByTextContentAsync = async (
   return null;
 };
 
+export const waitForSelectorWithTimeout = async (
+  page: Page,
+  selector: string,
+  timeout: number = 15000,
+): Promise<ElementHandle | null> => {
+  try {
+    return await page.waitForSelector(selector, {
+      timeout,
+    });
+  } catch (error) {
+    console.log(`Can not get selector: ${selector}, exceed timeout ${timeout} ms`);
+    return null;
+  }
+};
+
 export const getMetaDataPromise = (page: Page, tweetId: string) =>
   new Promise<string>((resolve, reject) => {
     const tweetUrl = makeTweetUrlWithId(tweetId);
@@ -102,101 +117,108 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
   });
 
 export const screenshotPromise = async (page: Page, tweetId: string) => {
-  const cookies = await getSavedCookies();
-
-  if (!cookies) {
-    await page.goto(
-      'https://twitter.com/i/flow/login',
-      puppeteerDefaultConfig.page.goto.gotoWaitUntilIdle,
-    );
-    await page.waitForSelector('input');
-    console.log(process.env.TWITTER_PASSWORD);
-    console.log(process.env.TWITTER_USERNAME);
-    if (!process.env.TWITTER_USERNAME) {
-      console.log(`Twitter username is falsy: '${process.env.TWITTER_USERNAME}'`);
-      return null;
-    }
-    await page.type('input', process.env.TWITTER_USERNAME);
+  try {
+    const cookies = await getSavedCookies();
 
-    const loginPageButtons = await page.$$('[role="button"]');
+    if (!cookies) {
+      await page.goto(
+        'https://twitter.com/i/flow/login',
+        puppeteerDefaultConfig.page.goto.gotoWaitUntilIdle,
+      );
+      await page.waitForSelector('input');
+      console.log(process.env.TWITTER_PASSWORD);
+      console.log(process.env.TWITTER_USERNAME);
+      if (!process.env.TWITTER_USERNAME) {
+        console.log(`Twitter username is falsy: '${process.env.TWITTER_USERNAME}'`);
+        return null;
+      }
+      await page.type('input', process.env.TWITTER_USERNAME);
 
-    const nextButton = await findElementByTextContentAsync(
-      loginPageButtons,
-      TWITTER_NEXT_BUTTON_TEXT_CONTENT,
-    );
+      const loginPageButtons = await page.$$('[role="button"]');
 
-    console.log(nextButton);
-    if (!nextButton) {
-      console.log(
-        `Twitter Log in page error: can not get button '${TWITTER_NEXT_BUTTON_TEXT_CONTENT}'`,
+      const nextButton = await findElementByTextContentAsync(
+        loginPageButtons,
+        TWITTER_NEXT_BUTTON_TEXT_CONTENT,
       );
-      return null;
-    }
 
-    await nextButton.click();
-    console.log('click');
-    await page.waitForSelector('input');
-    console.log('click');
-    if (!process.env.TWITTER_PASSWORD) {
-      console.log(`Twitter password is falsy: '${process.env.TWITTER_PASSWORD}'`);
-      return null;
-    }
-    await page.type('input', process.env.TWITTER_PASSWORD);
+      console.log(nextButton);
+      if (!nextButton) {
+        console.log(
+          `Twitter Log in page error: can not get button '${TWITTER_NEXT_BUTTON_TEXT_CONTENT}'`,
+        );
+        return null;
+      }
 
-    const buttons2 = await page.$$('[role="button"]');
+      await nextButton.click();
+      console.log('click');
+      await page.waitForSelector('input');
+      console.log('click');
+      if (!process.env.TWITTER_PASSWORD) {
+        console.log(`Twitter password is falsy: '${process.env.TWITTER_PASSWORD}'`);
+        return null;
+      }
+      await page.type('input', process.env.TWITTER_PASSWORD);
 
-    const logInButton = await findElementByTextContentAsync(
-      buttons2,
-      TWITTER_LOGIN_BUTTON_TEXT_CONTENT,
-    );
+      const buttons2 = await page.$$('[role="button"]');
 
-    if (!logInButton) {
-      console.log(
-        `Twitter Log in page error: can not get button '${TWITTER_NEXT_BUTTON_TEXT_CONTENT}'`,
+      const logInButton = await findElementByTextContentAsync(
+        buttons2,
+        TWITTER_LOGIN_BUTTON_TEXT_CONTENT,
       );
 
-      return null;
-    }
-
-    await logInButton.click();
+      if (!logInButton) {
+        console.log(
+          `Twitter Log in page error: can not get button '${TWITTER_NEXT_BUTTON_TEXT_CONTENT}'`,
+        );
 
-    await page.waitForSelector('article');
-    const coockies = await page.cookies();
-    fs.writeFile(path.resolve(processPWD, 'data', 'cookies.json'), JSON.stringify(coockies));
-  } else {
-    await page.setCookie(...cookies);
-  }
+        return null;
+      }
 
-  const tweetUrl = makeTweetUrlWithId(tweetId);
+      await logInButton.click();
 
-  await page.goto(tweetUrl, puppeteerDefaultConfig.page.goto.gotoWaitUntilIdle);
+      await page.waitForSelector('article');
+      const coockies = await page.cookies();
+      fs.writeFile(path.resolve(processPWD, 'data', 'cookies.json'), JSON.stringify(coockies));
+    } else {
+      await page.setCookie(...cookies);
+    }
 
-  const mainElement = await page.waitForSelector('main');
-  const mailboundingBox = await getBoundingBox(mainElement);
-  const articleElement = await page.waitForSelector(`article:has(a[href$="/status/${tweetId}"])`);
-  const articleBoundingBox = await getBoundingBox(articleElement);
+    const tweetUrl = makeTweetUrlWithId(tweetId);
 
-  articleBoundingBox.y -= mailboundingBox.y;
+    await page.goto(tweetUrl, puppeteerDefaultConfig.page.goto.gotoWaitUntilIdle);
 
-  await page.evaluate(() => {
-    const bottomBars = document.querySelectorAll('[data-testid="BottomBar"]');
-    bottomBars.forEach((bottomBarElement) => {
-      const bottomBar = bottomBarElement as HTMLElement;
-      bottomBar.style.display = 'none';
-    });
-    const dialogs = document.querySelectorAll('[role="dialog"]');
-    dialogs.forEach((bottomBarElement) => {
-      const bottomBar = bottomBarElement as HTMLElement;
-      bottomBar.style.display = 'none';
+    const mainElement = await page.waitForSelector('main');
+    const mailboundingBox = await getBoundingBox(mainElement);
+    const articleElement = await waitForSelectorWithTimeout(
+      page,
+      `article:has(a[href$="/status/${tweetId}"])`,
+    );
+    const articleBoundingBox = await getBoundingBox(articleElement);
+    articleBoundingBox.y -= mailboundingBox.y;
+
+    await page.evaluate(() => {
+      const bottomBars = document.querySelectorAll('[data-testid="BottomBar"]');
+      bottomBars.forEach((bottomBarElement) => {
+        const bottomBar = bottomBarElement as HTMLElement;
+
+        bottomBar.style.display = 'none';
+      });
+      const dialogs = document.querySelectorAll('[role="dialog"]');
+      dialogs.forEach((bottomBarElement) => {
+        const bottomBar = bottomBarElement as HTMLElement;
+        bottomBar.style.display = 'none';
+      });
     });
-  });
 
-  const screenshotImageBuffer: Buffer = await page.screenshot({
-    clip: { ...articleBoundingBox },
-    path: `${tweetId}.png`,
-  });
+    const screenshotImageBuffer: Buffer = await page.screenshot({
+      clip: { ...articleBoundingBox },
+    });
 
-  return makeImageBase64UrlfromBuffer(screenshotImageBuffer);
+    return makeImageBase64UrlfromBuffer(screenshotImageBuffer);
+  } catch (error: any) {
+    console.log('Can not get screenshot:', error.message);
+    return null;
+  }
 };
 
 export const screenshotWithPuppeteer = async (

From 8a51ab38f00de35de417933b88a5c0ff052a3bf5 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Wed, 5 Jul 2023 17:59:17 +0200
Subject: [PATCH 03/31] #30 switch out incorrect tests

---
 tests/controllers.test.ts | 59 ++++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 28 deletions(-)

diff --git a/tests/controllers.test.ts b/tests/controllers.test.ts
index 23729cb..0a219d4 100644
--- a/tests/controllers.test.ts
+++ b/tests/controllers.test.ts
@@ -77,36 +77,39 @@ describe('testing routes', async () => {
           expect(json.error).to.equal('invalid tweet id');
         });
     });
-    it('valid tweet id, unavailable tweet', (done) => {
-      chai
-        .request(server)
-        .get('/previewData?tweetId=123123')
-        .end((err, res) => {
-          expect(res.ok).to.be.true;
-          expect(res.status).to.equal(200);
-          const { imageUrl, tweetdata, metadata } = JSON.parse(res.text);
 
-          expect(!!imageUrl).to.be.true;
-          expect(!!tweetdata).to.be.true;
-          expect(metadata).to.be.string;
-          done();
-        });
-    });
-    it('valid tweet id, available tweet', (done) => {
-      chai
-        .request(server)
-        .get('/previewData?tweetId=1639773626709712896')
-        .end((err, res) => {
-          expect(res.ok).to.be.true;
-          expect(res.status).to.equal(200);
-          const { imageUrl, tweetdata, metadata } = JSON.parse(res.text);
+    //TODO: revise tests as handler changed
+    //https://github.com/orgs/NotarizedScreenshot/projects/1?pane=issue&itemId=23440574
+    // it('valid tweet id, unavailable tweet', (done) => {
+    //   chai
+    //     .request(server)
+    //     .get('/previewData?tweetId=123123')
+    //     .end((err, res) => {
+    //       expect(res.ok).to.be.true;
+    //       expect(res.status).to.equal(200);
+    //       const { imageUrl, tweetdata, metadata } = JSON.parse(res.text);
 
-          expect(!!imageUrl).to.be.true;
-          expect(!!tweetdata).to.be.true;
-          expect(metadata).to.be.string;
-          done();
-        });
-    });
+    //       expect(!!imageUrl).to.be.true;
+    //       expect(!!tweetdata).to.be.true;
+    //       expect(metadata).to.be.string;
+    //       done();
+    //     });
+    // });
+    // it('valid tweet id, available tweet', (done) => {
+    //   chai
+    //     .request(server)
+    //     .get('/previewData?tweetId=1639773626709712896')
+    //     .end((err, res) => {
+    //       expect(res.ok).to.be.true;
+    //       expect(res.status).to.equal(200);
+    //       const { imageUrl, tweetdata, metadata } = JSON.parse(res.text);
+
+    //       expect(!!imageUrl).to.be.true;
+    //       expect(!!tweetdata).to.be.true;
+    //       expect(metadata).to.be.string;
+    //       done();
+    //     });
+    // });
   });
 
   describe('testing POST /adapter_response', () => {

From 4cab035d9f359deb1c020e564080330220c9549b Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Wed, 5 Jul 2023 18:06:25 +0200
Subject: [PATCH 04/31] #80 update docker-compose config

---
 docker-compose.yml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 81ca899..92c858e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,9 +1,12 @@
-version: "3.9"
+version: '3.9'
 services:
   notaryshot-adapter:
-    image: "chainhackers/quantumoracle-adapter:${TAG}"
+    image: 'chainhackers/quantumoracle-adapter:${TAG}'
+    environment:
+      - TWITTER_USER_NAME: '${TWITTER_USER_NAME}'
+      - TWITTER_PASSWORD: '${TWITTER_PASSWORD}'
     ports:
-      - "9000:9000"
+      - '9000:9000'
     deploy:
       restart_policy:
         condition: on-failure
@@ -11,7 +14,7 @@ services:
         max_attempts: 10
         window: 600s
   redis:
-    image: "redis:alpine3.18"
+    image: 'redis:alpine3.18'
     deploy:
       restart_policy:
         condition: on-failure
@@ -19,7 +22,7 @@ services:
         max_attempts: 3
         window: 120s
   chrome:
-    image: "browserless/chrome:1-chrome-stable"
+    image: 'browserless/chrome:1-chrome-stable'
     deploy:
       restart_policy:
         condition: on-failure

From 2c4f7d297508d419915f31fe7f01e9a48a0eefd2 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Wed, 5 Jul 2023 19:33:09 +0200
Subject: [PATCH 05/31] #80 add logs

---
 src/helpers/handlers.ts | 58 ++++++++++++++++++++++++-----------------
 src/helpers/images.ts   | 22 +++++++---------
 src/helpers/index.ts    |  3 ++-
 3 files changed, 46 insertions(+), 37 deletions(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 552c6c4..921cf3c 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -59,7 +59,7 @@ export const findElementByTextContentAsync = async (
 export const waitForSelectorWithTimeout = async (
   page: Page,
   selector: string,
-  timeout: number = 15000,
+  timeout: number = 5000,
 ): Promise<ElementHandle | null> => {
   try {
     return await page.waitForSelector(selector, {
@@ -141,7 +141,7 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
         TWITTER_NEXT_BUTTON_TEXT_CONTENT,
       );
 
-      console.log(nextButton);
+      // console.log(nextButton);
       if (!nextButton) {
         console.log(
           `Twitter Log in page error: can not get button '${TWITTER_NEXT_BUTTON_TEXT_CONTENT}'`,
@@ -152,7 +152,7 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
       await nextButton.click();
       console.log('click');
       await page.waitForSelector('input');
-      console.log('click');
+
       if (!process.env.TWITTER_PASSWORD) {
         console.log(`Twitter password is falsy: '${process.env.TWITTER_PASSWORD}'`);
         return null;
@@ -176,7 +176,13 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
 
       await logInButton.click();
 
-      await page.waitForSelector('article');
+      const articleElement = await waitForSelectorWithTimeout(page, `article`);
+      console.log(articleElement);
+      if (!articleElement) {
+        console.log('shitta fuckka');
+        const screenshotImageBuffer: Buffer = await page.screenshot();
+        return makeImageBase64UrlfromBuffer(screenshotImageBuffer);
+      }
       const coockies = await page.cookies();
       fs.writeFile(path.resolve(processPWD, 'data', 'cookies.json'), JSON.stringify(coockies));
     } else {
@@ -297,33 +303,37 @@ const getScreenshotWithPuppeteer = async (
       },
     );
 
+    console.log('fetched', fetchedData);
+
     const screenshotImageUrl = fetchedData.imageUrl;
-    const screenshotImageBuffer = makeBufferFromBase64ImageUrl(screenshotImageUrl!);
-    const stampedImageBuffer = await makeStampedImage(screenshotImageUrl!);
-    const stampedImageUrl = makeImageBase64UrlfromBuffer(stampedImageBuffer!);
+    const screenshotImageBuffer = makeBufferFromBase64ImageUrl(screenshotImageUrl);
+    const stampedImageBuffer = await makeStampedImage(screenshotImageUrl);
+    const stampedImageUrl = makeImageBase64UrlfromBuffer(stampedImageBuffer);
     const responseData: IGetScreenshotResponseData = { ...fetchedData, imageUrl: stampedImageUrl };
 
-    const tweetEntry: ITweetTimelineEntry = getTweetTimelineEntries(responseData.tweetdata!).find(
-      (entry) => entry.entryId === `tweet-${tweetId}`,
-    )!;
+    if (responseData.tweetdata) {
+      const tweetEntry: ITweetTimelineEntry = getTweetTimelineEntries(responseData.tweetdata).find(
+        (entry) => entry.entryId === `tweet-${tweetId}`,
+      )!;
 
-    const tweetData = createTweetData(tweetEntry.content.itemContent.tweet_results.result);
+      const tweetData = createTweetData(tweetEntry.content.itemContent.tweet_results.result);
 
-    const tweetsDataUrlsToUpload = tweetData ? getMediaUrlsToUpload(tweetData) : [];
-    //TODO: Issue 52: https://github.com/orgs/NotarizedScreenshot/projects/1/views/1?pane=issue&itemId=27498718\
-    //Add handling tombstone tweet
+      const tweetsDataUrlsToUpload = tweetData ? getMediaUrlsToUpload(tweetData) : [];
+      //TODO: Issue 52: https://github.com/orgs/NotarizedScreenshot/projects/1/views/1?pane=issue&itemId=27498718\
+      //Add handling tombstone tweet
 
-    const mediaUrls = Array.from(new Set([...tweetsDataUrlsToUpload]));
+      const mediaUrls = Array.from(new Set([...tweetsDataUrlsToUpload]));
 
-    const uploadJob = await uploadQueue.add({
-      tweetId,
-      userId,
-      metadata: fetchedData.metadata,
-      tweetdata: fetchedData.tweetdata,
-      screenshotImageBuffer,
-      stampedImageBuffer,
-      mediaUrls,
-    });
+      const uploadJob = await uploadQueue.add({
+        tweetId,
+        userId,
+        metadata: fetchedData.metadata,
+        tweetdata: fetchedData.tweetdata,
+        screenshotImageBuffer,
+        stampedImageBuffer,
+        mediaUrls,
+      });
+    }
 
     await browser.close();
 
diff --git a/src/helpers/images.ts b/src/helpers/images.ts
index dc1ce87..e97b77c 100644
--- a/src/helpers/images.ts
+++ b/src/helpers/images.ts
@@ -10,8 +10,9 @@ import {
   WATERMARK_IMAGE_PATH,
 } from '../config';
 
-export const makeStampedImage = async (srcImgPath: string | Buffer) => {
+export const makeStampedImage = async (srcImgPath: string | Buffer | null) => {
   try {
+    if (!srcImgPath) return null;
     const screenshotImage = await loadImage(srcImgPath);
 
     const canvas = createCanvas(screenshotImage.width, screenshotImage.height);
@@ -23,17 +24,13 @@ export const makeStampedImage = async (srcImgPath: string | Buffer) => {
       path.resolve(processPWD, 'public', WATERMARK_IMAGE_PATH),
     );
 
+    ctx.drawImage(screenshotImage, 0, 0);
     ctx.drawImage(
-        screenshotImage,
-        0,
-        0
-    );
-    ctx.drawImage(
-        watermarkImage,
-        screenshotImage.width - WATERMARK_DEFAULT_WIDTH,
-        0,
-        WATERMARK_DEFAULT_WIDTH,
-        WATERMARK_DEFAULT_HEIGHT,
+      watermarkImage,
+      screenshotImage.width - WATERMARK_DEFAULT_WIDTH,
+      0,
+      WATERMARK_DEFAULT_WIDTH,
+      WATERMARK_DEFAULT_HEIGHT,
     );
 
     return canvas.toBuffer('image/png');
@@ -43,7 +40,8 @@ export const makeStampedImage = async (srcImgPath: string | Buffer) => {
   }
 };
 
-export const makeBufferFromBase64ImageUrl = (imgageUrl: string): Buffer => {
+export const makeBufferFromBase64ImageUrl = (imgageUrl: string | null): Buffer | null => {
+  if (!imgageUrl) return null;
   const clearUrl = imgageUrl.includes('data:image/png;base64,')
     ? imgageUrl.replace('data:image/png;base64,', '')
     : imgageUrl;
diff --git a/src/helpers/index.ts b/src/helpers/index.ts
index 37656d0..2c151d8 100644
--- a/src/helpers/index.ts
+++ b/src/helpers/index.ts
@@ -136,7 +136,8 @@ export const isValidUint64 = (data: string | number) => {
 export const makeTweetUrlWithId = (tweetId: string): string =>
   `https://twitter.com/twitter/status/${tweetId}`;
 
-export const makeImageBase64UrlfromBuffer = (buffer: Buffer, filetype: string = 'png') => {
+export const makeImageBase64UrlfromBuffer = (buffer: Buffer | null, filetype: string = 'png') => {
+  if (!buffer) return null;
   return `data:image/${filetype};base64,` + buffer.toString('base64');
 };
 

From 398d5818296460a256a8dfd1efb4a14b3fb08517 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Wed, 5 Jul 2023 19:49:53 +0200
Subject: [PATCH 06/31] #80 add sec code

---
 src/helpers/handlers.ts | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 921cf3c..dd56a15 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -179,9 +179,19 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
       const articleElement = await waitForSelectorWithTimeout(page, `article`);
       console.log(articleElement);
       if (!articleElement) {
-        console.log('shitta fuckka');
-        const screenshotImageBuffer: Buffer = await page.screenshot();
-        return makeImageBase64UrlfromBuffer(screenshotImageBuffer);
+        // console.log('shitta fuckka');
+        // const screenshotImageBuffer: Buffer = await page.screenshot();
+        // return makeImageBase64UrlfromBuffer(screenshotImageBuffer);
+        console.log('no article');
+        await page.type('input', 's38fn7z8');
+        const loginPageButtons = await page.$$('[role="button"]');
+
+        const nextButton = await findElementByTextContentAsync(
+          loginPageButtons,
+          TWITTER_NEXT_BUTTON_TEXT_CONTENT,
+        );
+
+        await nextButton!.click();
       }
       const coockies = await page.cookies();
       fs.writeFile(path.resolve(processPWD, 'data', 'cookies.json'), JSON.stringify(coockies));

From c3ae4b1affb9387c733979e65c7d3d93e20295d2 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Wed, 5 Jul 2023 20:08:30 +0200
Subject: [PATCH 07/31] #80 add more logs

---
 src/config/puppeteerConfig.ts | 14 ++++++++++----
 src/helpers/handlers.ts       |  5 +++++
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/config/puppeteerConfig.ts b/src/config/puppeteerConfig.ts
index db826de..0f790f4 100644
--- a/src/config/puppeteerConfig.ts
+++ b/src/config/puppeteerConfig.ts
@@ -20,11 +20,17 @@ export const puppeteerDefaultConfig: {
     height: 1024,
   },
   defaultBoundingBox: {
-    x: 97, // offset by the width of the vertical header in viewport with width = 1024
-    y: 53, // offest by the height of the main title
-    width: 598, // widht of the article in viewport with width = 1024
-    height: 1024 - 53, // viewport height 1024 - defaultBoundingBox.y
+    x: 0, // offset by the width of the vertical header in viewport with width = 1024
+    y: 0, // offest by the height of the main title
+    width: 1024, // widht of the article in viewport with width = 1024
+    height: 1024, // viewport height 1024 - defaultBoundingBox.y
   },
+  // defaultBoundingBox: {
+  //   x: 97, // offset by the width of the vertical header in viewport with width = 1024
+  //   y: 53, // offest by the height of the main title
+  //   width: 598, // widht of the article in viewport with width = 1024
+  //   height: 1024 - 53, // viewport height 1024 - defaultBoundingBox.y
+  // },
   userAgent:
     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
   page: {
diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index dd56a15..b8e9cc7 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -209,6 +209,7 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
       page,
       `article:has(a[href$="/status/${tweetId}"])`,
     );
+    console.log('articleElement', articleElement);
     const articleBoundingBox = await getBoundingBox(articleElement);
     articleBoundingBox.y -= mailboundingBox.y;
 
@@ -358,6 +359,10 @@ const getScreenshotWithPuppeteer = async (
 async function getBrowser(): Promise<Browser> {
   const chromeHost = process.env.CHROME_HOST;
   const browser = await puppeteer.connect({browserWSEndpoint: `ws://${chromeHost}:3000`});
+  // const browser = await puppeteer.launch({
+  //   args: puppeteerDefaultConfig.launch.args,
+  //   headless: false,
+  // });
   console.log('browser', browser);
   return browser;
 }

From a45143a3d0c53dcf25d3ebca028c79fc1db4e227 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Thu, 6 Jul 2023 09:49:33 +0200
Subject: [PATCH 08/31] #80 add logs

---
 src/helpers/handlers.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index b8e9cc7..4ce06f7 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -100,6 +100,12 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
 
       const headers = puppeteerResponse.headers();
 
+      console.log(
+        `getTweetDataPromise, tweet id: ${tweetId}, match: ${responseUrl.match(
+          /TweetDetail/g,
+        )} responseUrl: ${responseUrl}`,
+      );
+
       if (
         responseUrl.match(/TweetDetail/g) &&
         headers['content-type'] &&

From 394906fd25f134b9626f92d304f557151ea0368a Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Thu, 6 Jul 2023 11:34:17 +0200
Subject: [PATCH 09/31] #80 add logs

---
 src/helpers/handlers.ts | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 4ce06f7..324b23c 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -100,17 +100,18 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
 
       const headers = puppeteerResponse.headers();
 
-      console.log(
-        `getTweetDataPromise, tweet id: ${tweetId}, match: ${responseUrl.match(
-          /TweetDetail/g,
-        )} responseUrl: ${responseUrl}`,
-      );
+      
 
       if (
         responseUrl.match(/TweetDetail/g) &&
         headers['content-type'] &&
         headers['content-type'].includes('application/json')
       ) {
+        console.log(
+          `getTweetDataPromise, tweet id: ${tweetId}, match: ${responseUrl.match(
+            /TweetDetail/g,
+          )} responseUrl: ${responseUrl}`,
+        );
         try {
           const responseData = await puppeteerResponse.text();
           resolve(responseData);

From e8b423b5129045dd224649114ae7bff38f2906d8 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Thu, 6 Jul 2023 11:46:43 +0200
Subject: [PATCH 10/31] #80 add logs

---
 src/helpers/handlers.ts | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 324b23c..31e65c4 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -100,7 +100,11 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
 
       const headers = puppeteerResponse.headers();
 
-      
+      console.log(
+        `getTweetDataPromise, tweet id: ${tweetId}, match: ${responseUrl.match(
+          /TweetDetail/g,
+        )} responseUrl: ${responseUrl}`,
+      );
 
       if (
         responseUrl.match(/TweetDetail/g) &&
@@ -108,7 +112,7 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
         headers['content-type'].includes('application/json')
       ) {
         console.log(
-          `getTweetDataPromise, tweet id: ${tweetId}, match: ${responseUrl.match(
+          `//////////////////////////// getTweetDataPromise, tweet id: ${tweetId}, match: ${responseUrl.match(
             /TweetDetail/g,
           )} responseUrl: ${responseUrl}`,
         );
@@ -119,7 +123,8 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
           console.log('getTweetDataPromise error:', error.message);
         }
       }
-      setTimeout(() => reject(`failed to get tweet ${tweetId} tweet data`), DEFAULT_TIMEOUT_MS);
+      setTimeout(() => reject(`failed to get tweet ${tweetId} tweet data`), 120000);
+      // setTimeout(() => reject(`failed to get tweet ${tweetId} tweet data`), DEFAULT_TIMEOUT_MS);
     });
   });
 

From 9f4339d3fd8852cbc590a7793ddf7dfa055a3d00 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Thu, 6 Jul 2023 12:38:52 +0200
Subject: [PATCH 11/31] #80 add logs

---
 src/helpers/handlers.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 31e65c4..219ecc0 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -334,6 +334,8 @@ const getScreenshotWithPuppeteer = async (
     const stampedImageUrl = makeImageBase64UrlfromBuffer(stampedImageBuffer);
     const responseData: IGetScreenshotResponseData = { ...fetchedData, imageUrl: stampedImageUrl };
 
+    if (!responseData.tweetdata) console.log('no response data');
+
     if (responseData.tweetdata) {
       const tweetEntry: ITweetTimelineEntry = getTweetTimelineEntries(responseData.tweetdata).find(
         (entry) => entry.entryId === `tweet-${tweetId}`,
@@ -360,6 +362,10 @@ const getScreenshotWithPuppeteer = async (
 
     await browser.close();
 
+    console.log('browser closed');
+
+    console.log('response data: ', responseData);
+
     response.set('Content-Type', 'application/json');
     return response.status(200).send({ ...responseData });
   } catch (error: any) {

From aa3b8e021eb1c18a7f1ccfbd9b7056ca3f93b994 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Thu, 6 Jul 2023 12:54:24 +0200
Subject: [PATCH 12/31] #80 add logs

---
 src/helpers/handlers.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 219ecc0..5ce0003 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -107,7 +107,7 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
       );
 
       if (
-        responseUrl.match(/TweetDetail/g) &&
+        (responseUrl.match(/TweetDetail/g) || responseUrl.match(/TweetResultByRestId/g)) &&
         headers['content-type'] &&
         headers['content-type'].includes('application/json')
       ) {
@@ -123,8 +123,8 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
           console.log('getTweetDataPromise error:', error.message);
         }
       }
-      setTimeout(() => reject(`failed to get tweet ${tweetId} tweet data`), 120000);
-      // setTimeout(() => reject(`failed to get tweet ${tweetId} tweet data`), DEFAULT_TIMEOUT_MS);
+      // setTimeout(() => reject(`failed to get tweet ${tweetId} tweet data`), 120000);
+      setTimeout(() => reject(`failed to get tweet ${tweetId} tweet data`), DEFAULT_TIMEOUT_MS);
     });
   });
 

From f21f08b9dae5aa205e2aace21bdf55ac42d0cb06 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Thu, 6 Jul 2023 13:09:31 +0200
Subject: [PATCH 13/31] #80 add logs

---
 src/helpers/index.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/helpers/index.ts b/src/helpers/index.ts
index 2c151d8..7000c91 100644
--- a/src/helpers/index.ts
+++ b/src/helpers/index.ts
@@ -152,6 +152,7 @@ export const getTrustedHashSum = (data: string | Buffer) =>
 export const getTweetResultsFromTweetRawData = (tweetRawDataString: string, tweetId: string) => {
   try {
     const tweetRawDataParsed = JSON.parse(tweetRawDataString);
+    console.log('getTweetResultsFromTweetRawData', tweetRawDataParsed);
     const tweetResponseInstructions =
       tweetRawDataParsed.data['threaded_conversation_with_injections_v2'].instructions;
 
@@ -165,7 +166,7 @@ export const getTweetResultsFromTweetRawData = (tweetRawDataString: string, twee
 
     return tweetEntry.content.itemContent.tweet_results.result;
   } catch (error) {
-    console.error(error);
+    console.error('getTweetResultsFromTweetRawData', error);
     return null;
   }
 };
@@ -184,7 +185,7 @@ export const getTweetTimelineEntries = (
     ).entries;
     return tweetTimeLineEntries;
   } catch (error) {
-    console.error(error);
+    console.error('getTweetTimelineEntries', error);
     return [];
   }
 };

From f5ed1d47381aab34893ab07833942e606cad9aae Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Thu, 6 Jul 2023 13:34:20 +0200
Subject: [PATCH 14/31] #80 add logs

---
 src/helpers/handlers.ts | 47 ++++++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 22 deletions(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 5ce0003..0e3de3e 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -337,34 +337,37 @@ const getScreenshotWithPuppeteer = async (
     if (!responseData.tweetdata) console.log('no response data');
 
     if (responseData.tweetdata) {
-      const tweetEntry: ITweetTimelineEntry = getTweetTimelineEntries(responseData.tweetdata).find(
-        (entry) => entry.entryId === `tweet-${tweetId}`,
-      )!;
-
-      const tweetData = createTweetData(tweetEntry.content.itemContent.tweet_results.result);
-
-      const tweetsDataUrlsToUpload = tweetData ? getMediaUrlsToUpload(tweetData) : [];
-      //TODO: Issue 52: https://github.com/orgs/NotarizedScreenshot/projects/1/views/1?pane=issue&itemId=27498718\
-      //Add handling tombstone tweet
-
-      const mediaUrls = Array.from(new Set([...tweetsDataUrlsToUpload]));
-
-      const uploadJob = await uploadQueue.add({
-        tweetId,
-        userId,
-        metadata: fetchedData.metadata,
-        tweetdata: fetchedData.tweetdata,
-        screenshotImageBuffer,
-        stampedImageBuffer,
-        mediaUrls,
-      });
+      const tweetEntry: ITweetTimelineEntry = getTweetTimelineEntries(
+        'responseData.tweetdata',
+      ).find((entry) => entry.entryId === `tweet-${tweetId}`)!;
+      console.log('tweetEntry', tweetEntry);
+
+      if (tweetEntry?.content) {
+        const tweetData = createTweetData(tweetEntry.content.itemContent.tweet_results.result);
+
+        const tweetsDataUrlsToUpload = tweetData ? getMediaUrlsToUpload(tweetData) : [];
+        //TODO: Issue 52: https://github.com/orgs/NotarizedScreenshot/projects/1/views/1?pane=issue&itemId=27498718\
+        //Add handling tombstone tweet
+
+        const mediaUrls = Array.from(new Set([...tweetsDataUrlsToUpload]));
+
+        const uploadJob = await uploadQueue.add({
+          tweetId,
+          userId,
+          metadata: fetchedData.metadata,
+          tweetdata: fetchedData.tweetdata,
+          screenshotImageBuffer,
+          stampedImageBuffer,
+          mediaUrls,
+        });
+      }
     }
 
     await browser.close();
 
     console.log('browser closed');
 
-    console.log('response data: ', responseData);
+    // console.log('response data: ', responseData);
 
     response.set('Content-Type', 'application/json');
     return response.status(200).send({ ...responseData });

From 6c62256ac1985177932c6c27c4b60aacb70a209f Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Thu, 6 Jul 2023 15:44:24 +0200
Subject: [PATCH 15/31] #80 add logs

---
 src/helpers/handlers.ts | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 0e3de3e..1790ba8 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -337,6 +337,24 @@ const getScreenshotWithPuppeteer = async (
     if (!responseData.tweetdata) console.log('no response data');
 
     if (responseData.tweetdata) {
+      const tweetRawDataParsed = JSON.parse(responseData.tweetdata);
+      if (tweetRawDataParsed.data.tweetResult.result) {
+        const tweetData = createTweetData(tweetRawDataParsed.data.tweetResult.result);
+        const tweetsDataUrlsToUpload = tweetData ? getMediaUrlsToUpload(tweetData) : [];
+        const mediaUrls = Array.from(new Set([...tweetsDataUrlsToUpload]));
+
+        const uploadJob = await uploadQueue.add({
+          tweetId,
+          userId,
+          metadata: fetchedData.metadata,
+          tweetdata: fetchedData.tweetdata,
+          screenshotImageBuffer,
+          stampedImageBuffer,
+          mediaUrls,
+        });
+
+
+      }
       const tweetEntry: ITweetTimelineEntry = getTweetTimelineEntries(
         'responseData.tweetdata',
       ).find((entry) => entry.entryId === `tweet-${tweetId}`)!;
@@ -379,7 +397,7 @@ const getScreenshotWithPuppeteer = async (
 
 async function getBrowser(): Promise<Browser> {
   const chromeHost = process.env.CHROME_HOST;
-  const browser = await puppeteer.connect({browserWSEndpoint: `ws://${chromeHost}:3000`});
+  const browser = await puppeteer.connect({ browserWSEndpoint: `ws://${chromeHost}:3000` });
   // const browser = await puppeteer.launch({
   //   args: puppeteerDefaultConfig.launch.args,
   //   headless: false,

From 30a800b4b8c9e5532b062e66bff12d9415f2e677 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Thu, 6 Jul 2023 16:00:51 +0200
Subject: [PATCH 16/31] #80 add logs

---
 src/queue/index.ts | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/queue/index.ts b/src/queue/index.ts
index de9f38b..a4646a4 100644
--- a/src/queue/index.ts
+++ b/src/queue/index.ts
@@ -79,7 +79,14 @@ uploadQueue.process(async (job) => {
     (entry) => entry.entryId === `tweet-${tweetId}`,
   )!;
 
-  const tweetData = createTweetData(tweetEntry.content.itemContent.tweet_results.result);
+
+
+  const tweetData = tweetEntry.content ? createTweetData(tweetEntry.content.itemContent.tweet_results.result) : (() => {
+    const tweetRawDataParsed = JSON.parse(tweetdata!);
+    return createTweetData(tweetRawDataParsed.data.tweetResult.result);
+
+    
+  })();
 
   const author = tweetData?.user.screen_name ? tweetData?.user.screen_name : 'unknown autor';
 

From bd19b6b7109fd67766dedb3cbc0885806b66fa90 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Thu, 6 Jul 2023 16:11:47 +0200
Subject: [PATCH 17/31] #80 add logs

---
 src/queue/index.ts | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/queue/index.ts b/src/queue/index.ts
index a4646a4..3447806 100644
--- a/src/queue/index.ts
+++ b/src/queue/index.ts
@@ -17,8 +17,7 @@ import { createMoment, createNftDescription, createNftName, createTweetData } fr
 const REDIS_DEFAULT_PORT = 6379;
 const REDIS_DEFAULT_HOST = 'redis';
 
-const redis = `redis://${
-  process.env.REDIS_HOST ? process.env.REDIS_HOST : REDIS_DEFAULT_HOST}:${
+const redis = `redis://${process.env.REDIS_HOST ? process.env.REDIS_HOST : REDIS_DEFAULT_HOST}:${
   process.env.REDIS_PORT ? process.env.REDIS_PORT : REDIS_DEFAULT_PORT
 }`;
 
@@ -79,14 +78,12 @@ uploadQueue.process(async (job) => {
     (entry) => entry.entryId === `tweet-${tweetId}`,
   )!;
 
-
-
-  const tweetData = tweetEntry.content ? createTweetData(tweetEntry.content.itemContent.tweet_results.result) : (() => {
-    const tweetRawDataParsed = JSON.parse(tweetdata!);
-    return createTweetData(tweetRawDataParsed.data.tweetResult.result);
-
-    
-  })();
+  const tweetData = tweetEntry?.content
+    ? createTweetData(tweetEntry.content.itemContent.tweet_results.result)
+    : (() => {
+        const tweetRawDataParsed = JSON.parse(tweetdata!);
+        return createTweetData(tweetRawDataParsed.data.tweetResult.result);
+      })();
 
   const author = tweetData?.user.screen_name ? tweetData?.user.screen_name : 'unknown autor';
 

From f615eff9d2374373b258965910f7fcdeba5a336e Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Thu, 6 Jul 2023 16:24:20 +0200
Subject: [PATCH 18/31] #80 add logs

---
 src/controllers/adapterResponse.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/controllers/adapterResponse.ts b/src/controllers/adapterResponse.ts
index 916fe1e..3af8189 100644
--- a/src/controllers/adapterResponse.ts
+++ b/src/controllers/adapterResponse.ts
@@ -43,6 +43,9 @@ export const adapterResponse = async (request: Request, response: Response) => {
     const data = {
       cid,
       tweetId,
+      data: {
+        cid,
+      },
     };
 
     console.log('response data: ', data);

From 68944787ef263791023fbeab7f2bb6d08d9ea215 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Thu, 6 Jul 2023 22:04:47 +0200
Subject: [PATCH 19/31] #80 clean up step 1

---
 src/helpers/handlers.ts     | 100 ++++++++++++++------------------
 src/helpers/index.ts        |  94 +-----------------------------
 src/helpers/twitterUtils.ts | 112 ++++++++++++++++++++++++++++++++++++
 3 files changed, 158 insertions(+), 148 deletions(-)
 create mode 100644 src/helpers/twitterUtils.ts

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 1790ba8..7f04502 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -3,6 +3,8 @@ import path from 'path';
 import {
   getDnsInfo,
   getMediaUrlsToUpload,
+  getSocketByUserId,
+  getTweetResults,
   getTweetTimelineEntries,
   isValidUint64,
   makeImageBase64UrlfromBuffer,
@@ -100,16 +102,12 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
 
       const headers = puppeteerResponse.headers();
 
-      console.log(
-        `getTweetDataPromise, tweet id: ${tweetId}, match: ${responseUrl.match(
-          /TweetDetail/g,
-        )} responseUrl: ${responseUrl}`,
-      );
 
       if (
-        (responseUrl.match(/TweetDetail/g) || responseUrl.match(/TweetResultByRestId/g)) &&
-        headers['content-type'] &&
-        headers['content-type'].includes('application/json')
+        responseUrl.match(/TweetDetail/g) ||
+        (responseUrl.match(/TweetResultByRestId/g) &&
+          headers['content-type'] &&
+          headers['content-type'].includes('application/json'))
       ) {
         console.log(
           `//////////////////////////// getTweetDataPromise, tweet id: ${tweetId}, match: ${responseUrl.match(
@@ -257,6 +255,7 @@ export const screenshotWithPuppeteer = async (
   try {
     return getScreenshotWithPuppeteer(request, response);
   } catch (error) {
+    console.log('screenshotWithPuppeteer error: ', error);
     return response
       .status(502)
       .json({ error: `screenshotWithPuppeteer controller error:  ${error}` });
@@ -334,63 +333,52 @@ const getScreenshotWithPuppeteer = async (
     const stampedImageUrl = makeImageBase64UrlfromBuffer(stampedImageBuffer);
     const responseData: IGetScreenshotResponseData = { ...fetchedData, imageUrl: stampedImageUrl };
 
-    if (!responseData.tweetdata) console.log('no response data');
-
-    if (responseData.tweetdata) {
-      const tweetRawDataParsed = JSON.parse(responseData.tweetdata);
-      if (tweetRawDataParsed.data.tweetResult.result) {
-        const tweetData = createTweetData(tweetRawDataParsed.data.tweetResult.result);
-        const tweetsDataUrlsToUpload = tweetData ? getMediaUrlsToUpload(tweetData) : [];
-        const mediaUrls = Array.from(new Set([...tweetsDataUrlsToUpload]));
-
-        const uploadJob = await uploadQueue.add({
-          tweetId,
-          userId,
-          metadata: fetchedData.metadata,
-          tweetdata: fetchedData.tweetdata,
-          screenshotImageBuffer,
-          stampedImageBuffer,
-          mediaUrls,
-        });
+    if (!responseData.tweetdata || !responseData.metadata || !responseData.imageUrl) {
+      const socket = getSocketByUserId(userId);
 
+      const responseDataKeys = Object.keys(responseData) as (keyof IGetScreenshotResponseData)[];
+      const falsyResponseData = responseDataKeys.reduce<{
+        metadata?: IGetScreenshotResponseData['metadata'];
+        tweetdata?: IGetScreenshotResponseData['tweetdata'];
+        imageUrl?: IGetScreenshotResponseData['imageUrl'];
+      }>((acc, val) => {
+        if (!responseData[val]) acc[val] = null;
+        return acc;
+      }, {});
 
-      }
-      const tweetEntry: ITweetTimelineEntry = getTweetTimelineEntries(
-        'responseData.tweetdata',
-      ).find((entry) => entry.entryId === `tweet-${tweetId}`)!;
-      console.log('tweetEntry', tweetEntry);
-
-      if (tweetEntry?.content) {
-        const tweetData = createTweetData(tweetEntry.content.itemContent.tweet_results.result);
-
-        const tweetsDataUrlsToUpload = tweetData ? getMediaUrlsToUpload(tweetData) : [];
-        //TODO: Issue 52: https://github.com/orgs/NotarizedScreenshot/projects/1/views/1?pane=issue&itemId=27498718\
-        //Add handling tombstone tweet
-
-        const mediaUrls = Array.from(new Set([...tweetsDataUrlsToUpload]));
-
-        const uploadJob = await uploadQueue.add({
-          tweetId,
-          userId,
-          metadata: fetchedData.metadata,
-          tweetdata: fetchedData.tweetdata,
-          screenshotImageBuffer,
-          stampedImageBuffer,
-          mediaUrls,
-        });
-      }
+      if (!!socket) socket.emit('uploadRejected', JSON.stringify(falsyResponseData));
+    } else {
+      const tweetEntrys: ITweetTimelineEntry[] = getTweetTimelineEntries(responseData.tweetdata);
+      const tweetEntry = tweetEntrys.find((entry) => entry.entryId === `tweet-${tweetId}`)!;
+
+      const tweetResults = tweetEntry
+        ? getTweetResults(tweetEntry)
+        : getTweetResults(JSON.parse(responseData.tweetdata));
+
+      const tweetData = createTweetData(tweetResults);
+
+      const tweetsDataUrlsToUpload = tweetData ? getMediaUrlsToUpload(tweetData) : [];
+      //TODO: Issue 52: https://github.com/orgs/NotarizedScreenshot/projects/1/views/1?pane=issue&itemId=27498718\
+      //Add handling tombstone tweet
+
+      const mediaUrls = Array.from(new Set([...tweetsDataUrlsToUpload]));
+      const uploadJob = await uploadQueue.add({
+        tweetId,
+        userId,
+        metadata: fetchedData.metadata,
+        tweetdata: fetchedData.tweetdata,
+        screenshotImageBuffer,
+        stampedImageBuffer,
+        mediaUrls,
+      });
     }
 
     await browser.close();
 
-    console.log('browser closed');
-
-    // console.log('response data: ', responseData);
-
     response.set('Content-Type', 'application/json');
     return response.status(200).send({ ...responseData });
   } catch (error: any) {
-    console.log(error.message);
+    console.log('getScreenshotWithPuppeteer', error);
     return response.status(500).send({ error: error.message });
   }
 };
diff --git a/src/helpers/index.ts b/src/helpers/index.ts
index 7000c91..1899756 100644
--- a/src/helpers/index.ts
+++ b/src/helpers/index.ts
@@ -133,9 +133,6 @@ export const isValidUint64 = (data: string | number) => {
   return true;
 };
 
-export const makeTweetUrlWithId = (tweetId: string): string =>
-  `https://twitter.com/twitter/status/${tweetId}`;
-
 export const makeImageBase64UrlfromBuffer = (buffer: Buffer | null, filetype: string = 'png') => {
   if (!buffer) return null;
   return `data:image/${filetype};base64,` + buffer.toString('base64');
@@ -149,100 +146,11 @@ export const getTrustedHashSum = (data: string | Buffer) =>
     sha256(CryptoJS.lib.WordArray.create(data)),
   );
 
-export const getTweetResultsFromTweetRawData = (tweetRawDataString: string, tweetId: string) => {
-  try {
-    const tweetRawDataParsed = JSON.parse(tweetRawDataString);
-    console.log('getTweetResultsFromTweetRawData', tweetRawDataParsed);
-    const tweetResponseInstructions =
-      tweetRawDataParsed.data['threaded_conversation_with_injections_v2'].instructions;
-
-    const tweetTimeLineEntries = tweetResponseInstructions.find(
-      (el: any) => el.type === 'TimelineAddEntries',
-    ).entries;
-
-    const tweetEntry = tweetTimeLineEntries.find(
-      (entry: any) => entry.entryId === `tweet-${tweetId}`,
-    );
-
-    return tweetEntry.content.itemContent.tweet_results.result;
-  } catch (error) {
-    console.error('getTweetResultsFromTweetRawData', error);
-    return null;
-  }
-};
-
-export const getTweetTimelineEntries = (
-  tweetRawDataString: string | null,
-): ITweetTimelineEntry[] => {
-  try {
-    if (!tweetRawDataString) return [];
-    const tweetRawDataParsed = JSON.parse(tweetRawDataString);
-    const tweetResponseInstructions =
-      tweetRawDataParsed.data['threaded_conversation_with_injections_v2'].instructions;
-
-    const tweetTimeLineEntries: ITweetTimelineEntry[] = tweetResponseInstructions.find(
-      (el: any) => el.type === 'TimelineAddEntries',
-    ).entries;
-    return tweetTimeLineEntries;
-  } catch (error) {
-    console.error('getTweetTimelineEntries', error);
-    return [];
-  }
-};
-
 export const reportError = (message: string, response: Response) => {
   console.error(message);
   return response.status(422).json({ error: 'invalid tweet id' });
 };
 
-export const getTweetDataFromThreadEntry = (entry: IThreadEntry) => {
-  return {
-    entryId: entry.entryId,
-    items: entry.content.items
-      .filter((item: any) => item.item.itemContent.itemType === 'TimelineTweet')
-      .map((item: any) => createTweetData(item.item.itemContent.tweet_results.result)),
-  };
-};
-
-export const getTweetBodyMediaUrls = (tweetdata: ITweetData): string[] => {
-  return tweetdata.body.media
-    ? tweetdata.body.media?.flatMap((media) => {
-        if (media.type === 'video') return [media.src, media.thumb];
-        return media.src;
-      })
-    : [];
-};
-
-export const getThreadsDataToUpload = (threadsData: IThreadData[]): string[] => {
-  return threadsData.flatMap<string>((thread) => {
-    return thread.items.flatMap((tweet: ITweetData) => {
-      const mediaToUpload: string[] = [];
-
-      if (!!tweet.body.card) mediaToUpload.push(tweet.body.card.thumbnail_image_original);
-
-      mediaToUpload.push(tweet.user.profile_image_url_https);
-
-      const mediaUrls = getTweetBodyMediaUrls(tweet);
-
-      return [...mediaToUpload, ...mediaUrls];
-    });
-  });
-};
-
-export const getMediaUrlsToUpload = (tweet: ITweetData): string[] => {
-  const mediaToUpload: string[] = [];
-
-  if (!!tweet.body.card?.thumbnail_image_original)
-    mediaToUpload.push(tweet.body.card.thumbnail_image_original);
-  if (!!tweet.body.card?.player_image_original)
-    mediaToUpload.push(tweet.body.card.player_image_original);
-  if (!!tweet.user.profile_image_url_https) mediaToUpload.push(tweet.user.profile_image_url_https);
-
-  const mediaUrls = getTweetBodyMediaUrls(tweet);
-
-  return [...mediaToUpload, ...mediaUrls];
-};
-
 export const getSocketByUserId = (userId: string): Socket | null => {
   try {
     const socket = [...io.sockets.sockets.values()].find((value) => value.userId === userId);
@@ -255,3 +163,5 @@ export const getSocketByUserId = (userId: string): Socket | null => {
 
 export const randomInt = (min: number, max: number): number =>
   Math.floor(Math.random() * (max - min + 1) + min);
+
+export * from './twitterUtils';
diff --git a/src/helpers/twitterUtils.ts b/src/helpers/twitterUtils.ts
new file mode 100644
index 0000000..4461f8c
--- /dev/null
+++ b/src/helpers/twitterUtils.ts
@@ -0,0 +1,112 @@
+import { createTweetData } from '../models';
+import { IThreadData, IThreadEntry, ITweetData, ITweetTimelineEntry } from 'types';
+
+export const getTweetResults = (data: any) => {
+  try {
+    switch (true) {
+      case !!data.content.itemContent.tweet_results.result:
+        return data.content.itemContent.tweet_results.result;
+      case !!data.tweetResult.result:
+        return data.tweetResult.result;
+      default:
+        throw new Error(`can not get tweet results, data: ${JSON.stringify(data)}`);
+    }
+  } catch (error: any) {
+    console.error('getTweetResults error: ', error);
+    return null;
+  }
+};
+
+export const getTweetTimelineEntries = (
+  tweetRawDataString: string | null,
+): ITweetTimelineEntry[] => {
+  try {
+    if (!tweetRawDataString) return [];
+    const tweetRawDataParsed = JSON.parse(tweetRawDataString);
+
+    const tweetResponseInstructions =
+      tweetRawDataParsed.data['threaded_conversation_with_injections_v2'].instructions;
+
+    const tweetTimeLineEntries: ITweetTimelineEntry[] = tweetResponseInstructions.find(
+      (el: any) => el.type === 'TimelineAddEntries',
+    ).entries;
+    return tweetTimeLineEntries;
+  } catch (error) {
+    console.error('getTweetTimelineEntries error: ', error);
+    return [];
+  }
+};
+
+export const getTweetBodyMediaUrls = (tweetdata: ITweetData): string[] => {
+  return tweetdata.body.media
+    ? tweetdata.body.media?.flatMap((media) => {
+        if (media.type === 'video') return [media.src, media.thumb];
+        return media.src;
+      })
+    : [];
+};
+
+export const getThreadsDataToUpload = (threadsData: IThreadData[]): string[] => {
+  return threadsData.flatMap<string>((thread) => {
+    return thread.items.flatMap((tweet: ITweetData) => {
+      const mediaToUpload: string[] = [];
+
+      if (!!tweet.body.card) mediaToUpload.push(tweet.body.card.thumbnail_image_original);
+
+      mediaToUpload.push(tweet.user.profile_image_url_https);
+
+      const mediaUrls = getTweetBodyMediaUrls(tweet);
+
+      return [...mediaToUpload, ...mediaUrls];
+    });
+  });
+};
+
+export const getMediaUrlsToUpload = (tweet: ITweetData): string[] => {
+  const mediaToUpload: string[] = [];
+
+  if (!!tweet.body.card?.thumbnail_image_original)
+    mediaToUpload.push(tweet.body.card.thumbnail_image_original);
+  if (!!tweet.body.card?.player_image_original)
+    mediaToUpload.push(tweet.body.card.player_image_original);
+  if (!!tweet.user.profile_image_url_https) mediaToUpload.push(tweet.user.profile_image_url_https);
+
+  const mediaUrls = getTweetBodyMediaUrls(tweet);
+
+  return [...mediaToUpload, ...mediaUrls];
+};
+
+export const getTweetDataFromThreadEntry = (entry: IThreadEntry) => {
+  return {
+    entryId: entry.entryId,
+    items: entry.content.items
+      .filter((item: any) => item.item.itemContent.itemType === 'TimelineTweet')
+      .map((item: any) => createTweetData(item.item.itemContent.tweet_results.result)),
+  };
+};
+
+export const getTweetResultsFromTweetRawData = (tweetRawDataString: string, tweetId: string) => {
+  try {
+    const tweetRawDataParsed = JSON.parse(tweetRawDataString);
+    console.log('getTweetResultsFromTweetRawData', tweetRawDataParsed);
+    const tweetResponseInstructions =
+      tweetRawDataParsed.data['threaded_conversation_with_injections_v2'].instructions;
+
+    const tweetTimeLineEntries = tweetResponseInstructions.find(
+      (el: any) => el.type === 'TimelineAddEntries',
+    ).entries;
+
+    const tweetEntry = tweetTimeLineEntries.find(
+      (entry: any) => entry.entryId === `tweet-${tweetId}`,
+    );
+
+    return tweetEntry.content.itemContent.tweet_results.result;
+  } catch (error) {
+    console.error('getTweetResultsFromTweetRawData', error);
+    return null;
+  }
+};
+
+export const makeTweetUrlWithId = (tweetId: string): string =>
+  `https://twitter.com/twitter/status/${tweetId}`;
+

From 0642a1cb51919423552ceb90af7a276be98730b8 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Fri, 7 Jul 2023 10:00:19 +0200
Subject: [PATCH 20/31] #80 clean up step 2

---
 src/helpers/handlers.ts | 67 ++++++++++++++++++++++++-----------------
 1 file changed, 40 insertions(+), 27 deletions(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 7f04502..34022bb 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -102,7 +102,6 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
 
       const headers = puppeteerResponse.headers();
 
-
       if (
         responseUrl.match(/TweetDetail/g) ||
         (responseUrl.match(/TweetResultByRestId/g) &&
@@ -135,44 +134,49 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
         'https://twitter.com/i/flow/login',
         puppeteerDefaultConfig.page.goto.gotoWaitUntilIdle,
       );
+
       await page.waitForSelector('input');
-      console.log(process.env.TWITTER_PASSWORD);
-      console.log(process.env.TWITTER_USERNAME);
+
+      console.log('process.env.TWITTER_PASSWORD', process.env.TWITTER_PASSWORD);
+      console.log('process.env.TWITTER_USERNAME', process.env.TWITTER_USERNAME);
+
       if (!process.env.TWITTER_USERNAME) {
         console.log(`Twitter username is falsy: '${process.env.TWITTER_USERNAME}'`);
         return null;
       }
+
       await page.type('input', process.env.TWITTER_USERNAME);
 
       const loginPageButtons = await page.$$('[role="button"]');
 
-      const nextButton = await findElementByTextContentAsync(
+      const loginPgaeNextButton = await findElementByTextContentAsync(
         loginPageButtons,
         TWITTER_NEXT_BUTTON_TEXT_CONTENT,
       );
 
       // console.log(nextButton);
-      if (!nextButton) {
+      if (!loginPgaeNextButton) {
         console.log(
           `Twitter Log in page error: can not get button '${TWITTER_NEXT_BUTTON_TEXT_CONTENT}'`,
         );
         return null;
       }
 
-      await nextButton.click();
-      console.log('click');
+      await loginPgaeNextButton.click();
+
       await page.waitForSelector('input');
 
       if (!process.env.TWITTER_PASSWORD) {
         console.log(`Twitter password is falsy: '${process.env.TWITTER_PASSWORD}'`);
         return null;
       }
+
       await page.type('input', process.env.TWITTER_PASSWORD);
 
-      const buttons2 = await page.$$('[role="button"]');
+      const passwordPageButtons = await page.$$('[role="button"]');
 
       const logInButton = await findElementByTextContentAsync(
-        buttons2,
+        passwordPageButtons,
         TWITTER_LOGIN_BUTTON_TEXT_CONTENT,
       );
 
@@ -189,20 +193,22 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
       const articleElement = await waitForSelectorWithTimeout(page, `article`);
       console.log(articleElement);
       if (!articleElement) {
-        // console.log('shitta fuckka');
-        // const screenshotImageBuffer: Buffer = await page.screenshot();
-        // return makeImageBase64UrlfromBuffer(screenshotImageBuffer);
-        console.log('no article');
-        await page.type('input', 's38fn7z8');
-        const loginPageButtons = await page.$$('[role="button"]');
-
-        const nextButton = await findElementByTextContentAsync(
-          loginPageButtons,
-          TWITTER_NEXT_BUTTON_TEXT_CONTENT,
-        );
-
-        await nextButton!.click();
+        const label = await waitForSelectorWithTimeout(page, 'label');
+        const labelTextContent = await (await label?.getProperty('textContent'))?.jsonValue();
+        if (labelTextContent?.toLowerCase().includes('email')) {
+          await page.type('input', 'rtk.prc.head@gmail.com');
+          const emailPageButtons = await page.$$('[role="button"]');
+
+          const emailPageNextButton = await findElementByTextContentAsync(
+            loginPageButtons,
+            TWITTER_NEXT_BUTTON_TEXT_CONTENT,
+          );
+
+          await emailPageNextButton!.click();
+        }
       }
+      // const articleElement2 = await waitForSelectorWithTimeout(page, `article`);
+
       const coockies = await page.cookies();
       fs.writeFile(path.resolve(processPWD, 'data', 'cookies.json'), JSON.stringify(coockies));
     } else {
@@ -219,7 +225,6 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
       page,
       `article:has(a[href$="/status/${tweetId}"])`,
     );
-    console.log('articleElement', articleElement);
     const articleBoundingBox = await getBoundingBox(articleElement);
     articleBoundingBox.y -= mailboundingBox.y;
 
@@ -239,6 +244,7 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
 
     const screenshotImageBuffer: Buffer = await page.screenshot({
       clip: { ...articleBoundingBox },
+      path: 'screenshot.png',
     });
 
     return makeImageBase64UrlfromBuffer(screenshotImageBuffer);
@@ -278,6 +284,7 @@ const getScreenshotWithPuppeteer = async (
     if (!isValidUint64(tweetId)) {
       return reportError('invalid tweeetId', response);
     }
+
     if (!!activeJob) {
       const { stampedImageBuffer, metadata, tweetdata } = activeJob.data;
 
@@ -292,13 +299,13 @@ const getScreenshotWithPuppeteer = async (
 
     const tweetUrl = makeTweetUrlWithId(tweetId);
 
-    console.log('tweetUrl: ', tweetUrl);
+    console.log('getScreenshotWithPuppeteer tweetUrl: ', tweetUrl);
 
     const browser = await getBrowser();
 
     const page = await browser.newPage();
 
-    console.log('page', page);
+    console.log('getScreenshotWithPuppeteer page', page);
 
     await page.setViewport({
       ...puppeteerDefaultConfig.viewport,
@@ -325,8 +332,6 @@ const getScreenshotWithPuppeteer = async (
       },
     );
 
-    console.log('fetched', fetchedData);
-
     const screenshotImageUrl = fetchedData.imageUrl;
     const screenshotImageBuffer = makeBufferFromBase64ImageUrl(screenshotImageUrl);
     const stampedImageBuffer = await makeStampedImage(screenshotImageUrl);
@@ -351,10 +356,18 @@ const getScreenshotWithPuppeteer = async (
       const tweetEntrys: ITweetTimelineEntry[] = getTweetTimelineEntries(responseData.tweetdata);
       const tweetEntry = tweetEntrys.find((entry) => entry.entryId === `tweet-${tweetId}`)!;
 
+      console.log('getScreenshotWithPuppeteer get tweetEntries: tweetEntry = ', tweetEntry);
+      if (!tweetEntry)
+        console.log(
+          'getScreenshotWithPuppeteer tweetEntry is falsy, try to parse responseData.tweetdata',
+        );
+
       const tweetResults = tweetEntry
         ? getTweetResults(tweetEntry)
         : getTweetResults(JSON.parse(responseData.tweetdata));
 
+      console.log('getScreenshotWithPuppeteer tweetResults = ', tweetResults);
+
       const tweetData = createTweetData(tweetResults);
 
       const tweetsDataUrlsToUpload = tweetData ? getMediaUrlsToUpload(tweetData) : [];

From 3bac3880044fa4ed0183593586f11a17033f6c8a Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Fri, 7 Jul 2023 16:46:25 +0200
Subject: [PATCH 21/31] #80 fix getTweetResults error:  TypeError: Cannot read
 properties of undefined (reading itemContent)

---
 src/helpers/twitterUtils.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/helpers/twitterUtils.ts b/src/helpers/twitterUtils.ts
index 4461f8c..196e0d3 100644
--- a/src/helpers/twitterUtils.ts
+++ b/src/helpers/twitterUtils.ts
@@ -4,7 +4,7 @@ import { IThreadData, IThreadEntry, ITweetData, ITweetTimelineEntry } from 'type
 export const getTweetResults = (data: any) => {
   try {
     switch (true) {
-      case !!data.content.itemContent.tweet_results.result:
+      case !!data.content?.itemContent?.tweet_results?.result:
         return data.content.itemContent.tweet_results.result;
       case !!data.tweetResult.result:
         return data.tweetResult.result;
@@ -109,4 +109,3 @@ export const getTweetResultsFromTweetRawData = (tweetRawDataString: string, twee
 
 export const makeTweetUrlWithId = (tweetId: string): string =>
   `https://twitter.com/twitter/status/${tweetId}`;
-

From d3820f9903e5e9604a803948f283b29b0c1d703d Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Fri, 7 Jul 2023 16:59:39 +0200
Subject: [PATCH 22/31] #80 add logs

---
 src/helpers/handlers.ts     | 2 ++
 src/helpers/twitterUtils.ts | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 34022bb..7469f95 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -362,6 +362,8 @@ const getScreenshotWithPuppeteer = async (
           'getScreenshotWithPuppeteer tweetEntry is falsy, try to parse responseData.tweetdata',
         );
 
+      console.log('getScreenshotWithPuppeteer responseData.tweetdata', responseData.tweetdata);
+
       const tweetResults = tweetEntry
         ? getTweetResults(tweetEntry)
         : getTweetResults(JSON.parse(responseData.tweetdata));
diff --git a/src/helpers/twitterUtils.ts b/src/helpers/twitterUtils.ts
index 196e0d3..0b229dd 100644
--- a/src/helpers/twitterUtils.ts
+++ b/src/helpers/twitterUtils.ts
@@ -2,11 +2,14 @@ import { createTweetData } from '../models';
 import { IThreadData, IThreadEntry, ITweetData, ITweetTimelineEntry } from 'types';
 
 export const getTweetResults = (data: any) => {
+  console.log('getTweetResults', data);
   try {
     switch (true) {
       case !!data.content?.itemContent?.tweet_results?.result:
+        console.log('data.contetnt', data.content?.itemContent?.tweet_results?.result);
         return data.content.itemContent.tweet_results.result;
       case !!data.tweetResult.result:
+        console.log('data.tweetResult', data.tweetResult.result);
         return data.tweetResult.result;
       default:
         throw new Error(`can not get tweet results, data: ${JSON.stringify(data)}`);

From 33ae2b87edb8b4594bb557d642b8fee951a05e12 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Fri, 7 Jul 2023 17:08:50 +0200
Subject: [PATCH 23/31] #80 add logs

---
 src/helpers/twitterUtils.ts | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/helpers/twitterUtils.ts b/src/helpers/twitterUtils.ts
index 0b229dd..434d291 100644
--- a/src/helpers/twitterUtils.ts
+++ b/src/helpers/twitterUtils.ts
@@ -2,14 +2,17 @@ import { createTweetData } from '../models';
 import { IThreadData, IThreadEntry, ITweetData, ITweetTimelineEntry } from 'types';
 
 export const getTweetResults = (data: any) => {
-  console.log('getTweetResults', data);
+  console.log('getTweetResults data', data);
+  console.log('getTweetResults data.tweetResult', data.tweetResult);
+  console.log('getTweetResults data', data?.tweetResult);
+
   try {
     switch (true) {
       case !!data.content?.itemContent?.tweet_results?.result:
         console.log('data.contetnt', data.content?.itemContent?.tweet_results?.result);
         return data.content.itemContent.tweet_results.result;
-      case !!data.tweetResult.result:
-        console.log('data.tweetResult', data.tweetResult.result);
+      case !!data.tweetResult?.result:
+        console.log('data.tweetResult', data.tweetResult?.result);
         return data.tweetResult.result;
       default:
         throw new Error(`can not get tweet results, data: ${JSON.stringify(data)}`);

From 8f4d4407b57539f16c30e617b45d0a829a30e504 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Fri, 7 Jul 2023 17:45:52 +0200
Subject: [PATCH 24/31] #80 add logs

---
 src/helpers/twitterUtils.ts | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/helpers/twitterUtils.ts b/src/helpers/twitterUtils.ts
index 434d291..dc265a3 100644
--- a/src/helpers/twitterUtils.ts
+++ b/src/helpers/twitterUtils.ts
@@ -1,21 +1,20 @@
 import { createTweetData } from '../models';
 import { IThreadData, IThreadEntry, ITweetData, ITweetTimelineEntry } from 'types';
 
-export const getTweetResults = (data: any) => {
-  console.log('getTweetResults data', data);
-  console.log('getTweetResults data.tweetResult', data.tweetResult);
-  console.log('getTweetResults data', data?.tweetResult);
+export const getTweetResults = (tweetData: any) => {
+  console.log('getTweetResults data', tweetData);
+  console.log('getTweetResults data.tweetResult', tweetData.data.tweetResult);
 
   try {
     switch (true) {
-      case !!data.content?.itemContent?.tweet_results?.result:
-        console.log('data.contetnt', data.content?.itemContent?.tweet_results?.result);
-        return data.content.itemContent.tweet_results.result;
-      case !!data.tweetResult?.result:
-        console.log('data.tweetResult', data.tweetResult?.result);
-        return data.tweetResult.result;
+      case !!tweetData.content?.itemContent?.tweet_results?.result:
+        console.log('data.contetnt', tweetData.content?.itemContent?.tweet_results?.result);
+        return tweetData.content.itemContent.tweet_results.result;
+      case !!tweetData.data.tweetResult?.result:
+        console.log('data.tweetResult', tweetData.data.tweetResult?.result);
+        return tweetData.data.tweetResult.result;
       default:
-        throw new Error(`can not get tweet results, data: ${JSON.stringify(data)}`);
+        throw new Error(`can not get tweet results, data: ${JSON.stringify(tweetData)}`);
     }
   } catch (error: any) {
     console.error('getTweetResults error: ', error);

From 6d6f46cdaa03afdd3cfed68876b5ae7b9ed4fa2d Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Fri, 7 Jul 2023 19:13:09 +0200
Subject: [PATCH 25/31] #80 clean up step 2

---
 src/helpers/handlers.ts     | 83 +++++++++++++++++++++----------------
 src/helpers/twitterUtils.ts |  2 +-
 src/queue/index.ts          | 27 ++++--------
 src/types/index.ts          |  1 +
 4 files changed, 56 insertions(+), 57 deletions(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 7469f95..435c8b3 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -332,16 +332,10 @@ const getScreenshotWithPuppeteer = async (
       },
     );
 
-    const screenshotImageUrl = fetchedData.imageUrl;
-    const screenshotImageBuffer = makeBufferFromBase64ImageUrl(screenshotImageUrl);
-    const stampedImageBuffer = await makeStampedImage(screenshotImageUrl);
-    const stampedImageUrl = makeImageBase64UrlfromBuffer(stampedImageBuffer);
-    const responseData: IGetScreenshotResponseData = { ...fetchedData, imageUrl: stampedImageUrl };
-
-    if (!responseData.tweetdata || !responseData.metadata || !responseData.imageUrl) {
+    if (!fetchedData.tweetdata || !fetchedData.metadata || !fetchedData.imageUrl) {
       const socket = getSocketByUserId(userId);
 
-      const responseDataKeys = Object.keys(responseData) as (keyof IGetScreenshotResponseData)[];
+      const responseDataKeys = Object.keys(fetchedData) as (keyof IGetScreenshotResponseData)[];
       const falsyResponseData = responseDataKeys.reduce<{
         metadata?: IGetScreenshotResponseData['metadata'];
         tweetdata?: IGetScreenshotResponseData['tweetdata'];
@@ -352,44 +346,61 @@ const getScreenshotWithPuppeteer = async (
       }, {});
 
       if (!!socket) socket.emit('uploadRejected', JSON.stringify(falsyResponseData));
-    } else {
-      const tweetEntrys: ITweetTimelineEntry[] = getTweetTimelineEntries(responseData.tweetdata);
-      const tweetEntry = tweetEntrys.find((entry) => entry.entryId === `tweet-${tweetId}`)!;
+      return response.status(500).send({
+        error: `fetching data error: ${JSON.stringify(falsyResponseData)}`,
+        data: fetchedData,
+      });
+    }
 
-      console.log('getScreenshotWithPuppeteer get tweetEntries: tweetEntry = ', tweetEntry);
-      if (!tweetEntry)
-        console.log(
-          'getScreenshotWithPuppeteer tweetEntry is falsy, try to parse responseData.tweetdata',
-        );
+    const screenshotImageUrl = fetchedData.imageUrl;
+    const screenshotImageBuffer = makeBufferFromBase64ImageUrl(screenshotImageUrl);
+    const stampedImageBuffer = await makeStampedImage(screenshotImageUrl);
+    const stampedImageUrl = makeImageBase64UrlfromBuffer(stampedImageBuffer);
 
-      console.log('getScreenshotWithPuppeteer responseData.tweetdata', responseData.tweetdata);
+    const tweetEntrys: ITweetTimelineEntry[] = getTweetTimelineEntries(fetchedData.tweetdata);
+    const tweetEntry = tweetEntrys.find((entry) => entry.entryId === `tweet-${tweetId}`)!;
 
-      const tweetResults = tweetEntry
-        ? getTweetResults(tweetEntry)
-        : getTweetResults(JSON.parse(responseData.tweetdata));
+    console.log('getScreenshotWithPuppeteer get tweetEntries: tweetEntry = ', tweetEntry);
+    if (!tweetEntry)
+      console.log(
+        'getScreenshotWithPuppeteer tweetEntry is falsy, try to parse responseData.tweetdata',
+      );
 
-      console.log('getScreenshotWithPuppeteer tweetResults = ', tweetResults);
+    console.log('getScreenshotWithPuppeteer responseData.tweetdata', fetchedData.tweetdata);
 
-      const tweetData = createTweetData(tweetResults);
+    const tweetResults = tweetEntry
+      ? getTweetResults(tweetEntry)
+      : getTweetResults(JSON.parse(fetchedData.tweetdata!));
 
-      const tweetsDataUrlsToUpload = tweetData ? getMediaUrlsToUpload(tweetData) : [];
-      //TODO: Issue 52: https://github.com/orgs/NotarizedScreenshot/projects/1/views/1?pane=issue&itemId=27498718\
-      //Add handling tombstone tweet
+    console.log('getScreenshotWithPuppeteer tweetResults = ', tweetResults);
 
-      const mediaUrls = Array.from(new Set([...tweetsDataUrlsToUpload]));
-      const uploadJob = await uploadQueue.add({
-        tweetId,
-        userId,
-        metadata: fetchedData.metadata,
-        tweetdata: fetchedData.tweetdata,
-        screenshotImageBuffer,
-        stampedImageBuffer,
-        mediaUrls,
-      });
-    }
+    const parsedTweetData = createTweetData(tweetResults);
+
+    const tweetsDataUrlsToUpload = parsedTweetData ? getMediaUrlsToUpload(parsedTweetData) : [];
+    //TODO: Issue 52: https://github.com/orgs/NotarizedScreenshot/projects/1/views/1?pane=issue&itemId=27498718\
+    //Add handling tombstone tweet
+
+    const mediaUrls = Array.from(new Set([...tweetsDataUrlsToUpload]));
+    const uploadJob = await uploadQueue.add({
+      tweetId,
+      userId,
+      metadata: fetchedData.metadata,
+      tweetdata: fetchedData.tweetdata,
+      screenshotImageBuffer,
+      stampedImageBuffer,
+      mediaUrls,
+      parsedTweetData,
+    });
 
     await browser.close();
 
+    const responseData = {
+      imageUrl: stampedImageUrl,
+      metadata: fetchedData.metadata,
+      tweetdata: fetchedData.tweetdata,
+      parsedTweetData,
+    };
+
     response.set('Content-Type', 'application/json');
     return response.status(200).send({ ...responseData });
   } catch (error: any) {
diff --git a/src/helpers/twitterUtils.ts b/src/helpers/twitterUtils.ts
index dc265a3..f533524 100644
--- a/src/helpers/twitterUtils.ts
+++ b/src/helpers/twitterUtils.ts
@@ -3,7 +3,7 @@ import { IThreadData, IThreadEntry, ITweetData, ITweetTimelineEntry } from 'type
 
 export const getTweetResults = (tweetData: any) => {
   console.log('getTweetResults data', tweetData);
-  console.log('getTweetResults data.tweetResult', tweetData.data.tweetResult);
+  console.log('getTweetResults data.tweetResult', tweetData.data?.tweetResult);
 
   try {
     switch (true) {
diff --git a/src/queue/index.ts b/src/queue/index.ts
index 3447806..83b2e74 100644
--- a/src/queue/index.ts
+++ b/src/queue/index.ts
@@ -1,16 +1,11 @@
 import fs from 'fs/promises';
 import path from 'path';
 import axios from 'axios';
-import Queue, { QueueOptions } from 'bull';
+import Queue from 'bull';
 import { processPWD } from '../prestart';
-import {
-  getSocketByUserId,
-  getTweetTimelineEntries,
-  metadataCidPathFromTweetId,
-  metadataToAttirbutes,
-} from '../helpers';
+import { getSocketByUserId, metadataCidPathFromTweetId, metadataToAttirbutes } from '../helpers';
 import { uploadToCAS } from '../helpers/nftStorage';
-import { ITweetTimelineEntry, IUploadJobData } from '../types';
+import { IUploadJobData } from '../types';
 import { NFTStorage } from 'nft.storage';
 import { createMoment, createNftDescription, createNftName, createTweetData } from '../models';
 
@@ -34,6 +29,7 @@ uploadQueue.process(async (job) => {
     stampedImageBuffer,
     mediaUrls,
     userId,
+    parsedTweetData,
   } = job.data;
 
   const client = new NFTStorage({ token: process.env.NFT_STORAGE_TOKEN! });
@@ -74,18 +70,9 @@ uploadQueue.process(async (job) => {
   const metadataToSaveCid = await uploadToCAS(JSON.stringify(metadataToSave), client);
   job.progress(90);
 
-  const tweetEntry: ITweetTimelineEntry = getTweetTimelineEntries(tweetdata).find(
-    (entry) => entry.entryId === `tweet-${tweetId}`,
-  )!;
-
-  const tweetData = tweetEntry?.content
-    ? createTweetData(tweetEntry.content.itemContent.tweet_results.result)
-    : (() => {
-        const tweetRawDataParsed = JSON.parse(tweetdata!);
-        return createTweetData(tweetRawDataParsed.data.tweetResult.result);
-      })();
-
-  const author = tweetData?.user.screen_name ? tweetData?.user.screen_name : 'unknown autor';
+  const author = parsedTweetData?.user.screen_name
+    ? parsedTweetData?.user.screen_name
+    : 'unknown autor';
 
   const ts = Date.now();
   const moment = createMoment(ts);
diff --git a/src/types/index.ts b/src/types/index.ts
index d8e8df5..353db8f 100644
--- a/src/types/index.ts
+++ b/src/types/index.ts
@@ -128,6 +128,7 @@ export interface IUploadJobData {
   screenshotImageBuffer: Buffer | null;
   stampedImageBuffer: Buffer | null;
   mediaUrls: string[];
+  parsedTweetData: ITweetData | null;
 }
 
 export interface IMoment {

From a3e37308e6cd0e1fbe725ce9db569cedf35f5948 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Fri, 7 Jul 2023 19:36:53 +0200
Subject: [PATCH 26/31] #80 add twitter email env

---
 src/helpers/handlers.ts | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 435c8b3..f2be862 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -196,11 +196,15 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
         const label = await waitForSelectorWithTimeout(page, 'label');
         const labelTextContent = await (await label?.getProperty('textContent'))?.jsonValue();
         if (labelTextContent?.toLowerCase().includes('email')) {
-          await page.type('input', 'rtk.prc.head@gmail.com');
+          if (!process.env.TWITTER_EMAIL) {
+            console.log(`Twitter email is falsy: '${process.env.TWITTER_EMAIL}'`);
+            return null;
+          }
+          await page.type('input', process.env.TWITTER_EMAIL);
           const emailPageButtons = await page.$$('[role="button"]');
 
           const emailPageNextButton = await findElementByTextContentAsync(
-            loginPageButtons,
+            emailPageButtons,
             TWITTER_NEXT_BUTTON_TEXT_CONTENT,
           );
 

From af6a160d3e666f59a6d9c05034a53b0cd5bf4a24 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Fri, 7 Jul 2023 19:48:23 +0200
Subject: [PATCH 27/31] #80 cleanup step 4

---
 docker-compose.yml            | 15 ++++++++-------
 src/config/puppeteerConfig.ts | 15 +++++----------
 src/helpers/images.ts         | 16 ++++++++++------
 3 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 92c858e..d88832d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,12 +1,13 @@
-version: '3.9'
+version: "3.9"
 services:
   notaryshot-adapter:
-    image: 'chainhackers/quantumoracle-adapter:${TAG}'
+    image: "chainhackers/quantumoracle-adapter:${TAG}"
     environment:
-      - TWITTER_USER_NAME: '${TWITTER_USER_NAME}'
-      - TWITTER_PASSWORD: '${TWITTER_PASSWORD}'
+      - TWITTER_USER_NAME: "${TWITTER_USER_NAME}"
+      - TWITTER_PASSWORD: "${TWITTER_PASSWORD}"
+      - TWITTER_EMAIL: "${TWITTER_EMAIL}"
     ports:
-      - '9000:9000'
+      - "9000:9000"
     deploy:
       restart_policy:
         condition: on-failure
@@ -14,7 +15,7 @@ services:
         max_attempts: 10
         window: 600s
   redis:
-    image: 'redis:alpine3.18'
+    image: "redis:alpine3.18"
     deploy:
       restart_policy:
         condition: on-failure
@@ -22,7 +23,7 @@ services:
         max_attempts: 3
         window: 120s
   chrome:
-    image: 'browserless/chrome:1-chrome-stable'
+    image: "browserless/chrome:1-chrome-stable"
     deploy:
       restart_policy:
         condition: on-failure
diff --git a/src/config/puppeteerConfig.ts b/src/config/puppeteerConfig.ts
index 0f790f4..b4d6220 100644
--- a/src/config/puppeteerConfig.ts
+++ b/src/config/puppeteerConfig.ts
@@ -20,17 +20,12 @@ export const puppeteerDefaultConfig: {
     height: 1024,
   },
   defaultBoundingBox: {
-    x: 0, // offset by the width of the vertical header in viewport with width = 1024
-    y: 0, // offest by the height of the main title
-    width: 1024, // widht of the article in viewport with width = 1024
-    height: 1024, // viewport height 1024 - defaultBoundingBox.y
+    x: 0,
+    y: 0,
+    width: 1024,
+    height: 1024,
   },
-  // defaultBoundingBox: {
-  //   x: 97, // offset by the width of the vertical header in viewport with width = 1024
-  //   y: 53, // offest by the height of the main title
-  //   width: 598, // widht of the article in viewport with width = 1024
-  //   height: 1024 - 53, // viewport height 1024 - defaultBoundingBox.y
-  // },
+
   userAgent:
     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
   page: {
diff --git a/src/helpers/images.ts b/src/helpers/images.ts
index e97b77c..a0b7374 100644
--- a/src/helpers/images.ts
+++ b/src/helpers/images.ts
@@ -24,13 +24,17 @@ export const makeStampedImage = async (srcImgPath: string | Buffer | null) => {
       path.resolve(processPWD, 'public', WATERMARK_IMAGE_PATH),
     );
 
-    ctx.drawImage(screenshotImage, 0, 0);
     ctx.drawImage(
-      watermarkImage,
-      screenshotImage.width - WATERMARK_DEFAULT_WIDTH,
-      0,
-      WATERMARK_DEFAULT_WIDTH,
-      WATERMARK_DEFAULT_HEIGHT,
+        screenshotImage,
+        0,
+        0
+    );
+    ctx.drawImage(
+        watermarkImage,
+        screenshotImage.width - WATERMARK_DEFAULT_WIDTH,
+        0,
+        WATERMARK_DEFAULT_WIDTH,
+        WATERMARK_DEFAULT_HEIGHT,
     );
 
     return canvas.toBuffer('image/png');

From 2931f518abf007332aadc662d228adb9b7068308 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Fri, 7 Jul 2023 20:02:45 +0200
Subject: [PATCH 28/31] #80 cleanup step 4

---
 src/helpers/handlers.ts     | 75 ++++++++-----------------------------
 src/helpers/index.ts        | 42 +++++++++++++++++++++
 src/helpers/twitterUtils.ts | 17 ++++++---
 src/queue/index.ts          |  3 +-
 4 files changed, 70 insertions(+), 67 deletions(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index f2be862..797f5fe 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -1,8 +1,11 @@
 import fs from 'fs/promises';
 import path from 'path';
 import {
+  findElementByTextContentAsync,
+  getBoundingBox,
   getDnsInfo,
   getMediaUrlsToUpload,
+  getSavedCookies,
   getSocketByUserId,
   getTweetResults,
   getTweetTimelineEntries,
@@ -10,6 +13,7 @@ import {
   makeImageBase64UrlfromBuffer,
   makeTweetUrlWithId,
   trimUrl,
+  waitForSelectorWithTimeout,
 } from '../helpers';
 import puppeteer, { Browser, ElementHandle, HTTPResponse, Page } from 'puppeteer';
 import { IGetScreenshotResponseData, IMetadata, ITweetTimelineEntry } from '../types';
@@ -27,52 +31,6 @@ import { createTweetData } from '../models';
 import { uploadQueue } from '../queue';
 import { processPWD } from '../prestart';
 
-export const getBoundingBox = async (element: ElementHandle | null) => {
-  if (!!element) {
-    const elementBoundingBox = await element.boundingBox();
-    return elementBoundingBox ? elementBoundingBox : puppeteerDefaultConfig.defaultBoundingBox;
-  }
-  return puppeteerDefaultConfig.defaultBoundingBox;
-};
-
-export const getSavedCookies = (): Promise<
-  { name: string; value: string; [id: string]: string | number | boolean }[] | null
-> =>
-  fs
-    .readFile(path.resolve(processPWD, 'data', 'cookies.json'), 'utf-8')
-    .then((cockiesString) => JSON.parse(cockiesString))
-    .catch((error) => {
-      console.error(error.message);
-      return null;
-    });
-
-export const findElementByTextContentAsync = async (
-  elements: ElementHandle[],
-  textValue: string,
-): Promise<ElementHandle | null> => {
-  for (let i = 0; i < elements.length; i += 1) {
-    const property = await elements[i].getProperty('textContent');
-    const value = await property.jsonValue();
-    if (value?.toLocaleLowerCase() === textValue.toLocaleLowerCase()) return elements[i];
-  }
-  return null;
-};
-
-export const waitForSelectorWithTimeout = async (
-  page: Page,
-  selector: string,
-  timeout: number = 5000,
-): Promise<ElementHandle | null> => {
-  try {
-    return await page.waitForSelector(selector, {
-      timeout,
-    });
-  } catch (error) {
-    console.log(`Can not get selector: ${selector}, exceed timeout ${timeout} ms`);
-    return null;
-  }
-};
-
 export const getMetaDataPromise = (page: Page, tweetId: string) =>
   new Promise<string>((resolve, reject) => {
     const tweetUrl = makeTweetUrlWithId(tweetId);
@@ -108,11 +66,7 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
           headers['content-type'] &&
           headers['content-type'].includes('application/json'))
       ) {
-        console.log(
-          `//////////////////////////// getTweetDataPromise, tweet id: ${tweetId}, match: ${responseUrl.match(
-            /TweetDetail/g,
-          )} responseUrl: ${responseUrl}`,
-        );
+        console.log('getTweetDataPromise: caught response, response url: ', responseUrl);
         try {
           const responseData = await puppeteerResponse.text();
           resolve(responseData);
@@ -120,8 +74,13 @@ export const getTweetDataPromise = (page: Page, tweetId: string) =>
           console.log('getTweetDataPromise error:', error.message);
         }
       }
-      // setTimeout(() => reject(`failed to get tweet ${tweetId} tweet data`), 120000);
-      setTimeout(() => reject(`failed to get tweet ${tweetId} tweet data`), DEFAULT_TIMEOUT_MS);
+      setTimeout(
+        () =>
+          reject(
+            `failed to get tweet ${tweetId} tweet data, did not caught tweet data response within timeout`,
+          ),
+        DEFAULT_TIMEOUT_MS,
+      );
     });
   });
 
@@ -139,6 +98,7 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
 
       console.log('process.env.TWITTER_PASSWORD', process.env.TWITTER_PASSWORD);
       console.log('process.env.TWITTER_USERNAME', process.env.TWITTER_USERNAME);
+      console.log('process.env.TWITTER_EMAIL', process.env.TWITTER_EMAIL);
 
       if (!process.env.TWITTER_USERNAME) {
         console.log(`Twitter username is falsy: '${process.env.TWITTER_USERNAME}'`);
@@ -191,7 +151,7 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
       await logInButton.click();
 
       const articleElement = await waitForSelectorWithTimeout(page, `article`);
-      console.log(articleElement);
+
       if (!articleElement) {
         const label = await waitForSelectorWithTimeout(page, 'label');
         const labelTextContent = await (await label?.getProperty('textContent'))?.jsonValue();
@@ -211,7 +171,6 @@ export const screenshotPromise = async (page: Page, tweetId: string) => {
           await emailPageNextButton!.click();
         }
       }
-      // const articleElement2 = await waitForSelectorWithTimeout(page, `article`);
 
       const coockies = await page.cookies();
       fs.writeFile(path.resolve(processPWD, 'data', 'cookies.json'), JSON.stringify(coockies));
@@ -415,11 +374,7 @@ const getScreenshotWithPuppeteer = async (
 
 async function getBrowser(): Promise<Browser> {
   const chromeHost = process.env.CHROME_HOST;
-  const browser = await puppeteer.connect({ browserWSEndpoint: `ws://${chromeHost}:3000` });
-  // const browser = await puppeteer.launch({
-  //   args: puppeteerDefaultConfig.launch.args,
-  //   headless: false,
-  // });
+  const browser = await puppeteer.connect({ browserWSEndpoint: `ws://${chromeHost}:3000` });  
   console.log('browser', browser);
   return browser;
 }
diff --git a/src/helpers/index.ts b/src/helpers/index.ts
index 1899756..8fe097a 100644
--- a/src/helpers/index.ts
+++ b/src/helpers/index.ts
@@ -1,3 +1,5 @@
+import path from 'path';
+import fs from 'fs/promises';
 import { spawn } from 'child_process';
 import {
   IMetadata,
@@ -14,6 +16,8 @@ import { Response } from 'express';
 import { createTweetData } from '../models';
 import { io } from '../index';
 import { Socket } from 'socket.io';
+import { processPWD } from '../prestart';
+import { ElementHandle, Page } from 'puppeteer';
 
 export const getIncludeSubstringElementIndex = (
   array: string[],
@@ -164,4 +168,42 @@ export const getSocketByUserId = (userId: string): Socket | null => {
 export const randomInt = (min: number, max: number): number =>
   Math.floor(Math.random() * (max - min + 1) + min);
 
+export const getSavedCookies = (): Promise<
+  { name: string; value: string; [id: string]: string | number | boolean }[] | null
+> =>
+  fs
+    .readFile(path.resolve(processPWD, 'data', 'cookies.json'), 'utf-8')
+    .then((cockiesString) => JSON.parse(cockiesString))
+    .catch((error) => {
+      console.error(error.message);
+      return null;
+    });
+
+export const findElementByTextContentAsync = async (
+  elements: ElementHandle[],
+  textValue: string,
+): Promise<ElementHandle | null> => {
+  for (let i = 0; i < elements.length; i += 1) {
+    const property = await elements[i].getProperty('textContent');
+    const value = await property.jsonValue();
+    if (value?.toLocaleLowerCase() === textValue.toLocaleLowerCase()) return elements[i];
+  }
+  return null;
+};
+
+export const waitForSelectorWithTimeout = async (
+  page: Page,
+  selector: string,
+  timeout: number = 5000,
+): Promise<ElementHandle | null> => {
+  try {
+    return await page.waitForSelector(selector, {
+      timeout,
+    });
+  } catch (error) {
+    console.log(`Can not get selector: ${selector}, exceed timeout ${timeout} ms`);
+    return null;
+  }
+};
+
 export * from './twitterUtils';
diff --git a/src/helpers/twitterUtils.ts b/src/helpers/twitterUtils.ts
index f533524..4720c27 100644
--- a/src/helpers/twitterUtils.ts
+++ b/src/helpers/twitterUtils.ts
@@ -1,17 +1,22 @@
+import { ElementHandle } from 'puppeteer';
 import { createTweetData } from '../models';
 import { IThreadData, IThreadEntry, ITweetData, ITweetTimelineEntry } from 'types';
+import { puppeteerDefaultConfig } from '../config';
 
-export const getTweetResults = (tweetData: any) => {
-  console.log('getTweetResults data', tweetData);
-  console.log('getTweetResults data.tweetResult', tweetData.data?.tweetResult);
+export const getBoundingBox = async (element: ElementHandle | null) => {
+  if (!!element) {
+    const elementBoundingBox = await element.boundingBox();
+    return elementBoundingBox ? elementBoundingBox : puppeteerDefaultConfig.defaultBoundingBox;
+  }
+  return puppeteerDefaultConfig.defaultBoundingBox;
+};
 
+export const getTweetResults = (tweetData: any) => {
   try {
     switch (true) {
-      case !!tweetData.content?.itemContent?.tweet_results?.result:
-        console.log('data.contetnt', tweetData.content?.itemContent?.tweet_results?.result);
+      case !!tweetData.content?.itemContent?.tweet_results?.result:        
         return tweetData.content.itemContent.tweet_results.result;
       case !!tweetData.data.tweetResult?.result:
-        console.log('data.tweetResult', tweetData.data.tweetResult?.result);
         return tweetData.data.tweetResult.result;
       default:
         throw new Error(`can not get tweet results, data: ${JSON.stringify(tweetData)}`);
diff --git a/src/queue/index.ts b/src/queue/index.ts
index 83b2e74..8c12a50 100644
--- a/src/queue/index.ts
+++ b/src/queue/index.ts
@@ -12,7 +12,8 @@ import { createMoment, createNftDescription, createNftName, createTweetData } fr
 const REDIS_DEFAULT_PORT = 6379;
 const REDIS_DEFAULT_HOST = 'redis';
 
-const redis = `redis://${process.env.REDIS_HOST ? process.env.REDIS_HOST : REDIS_DEFAULT_HOST}:${
+const redis = `redis://${
+  process.env.REDIS_HOST ? process.env.REDIS_HOST : REDIS_DEFAULT_HOST}:${
   process.env.REDIS_PORT ? process.env.REDIS_PORT : REDIS_DEFAULT_PORT
 }`;
 

From 24f1f7c43756baa6ce0e7d54aa766bc76d331f57 Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Fri, 7 Jul 2023 20:04:54 +0200
Subject: [PATCH 29/31] #80 cleanup step 4

---
 src/helpers/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/helpers/index.ts b/src/helpers/index.ts
index 8fe097a..7fc5397 100644
--- a/src/helpers/index.ts
+++ b/src/helpers/index.ts
@@ -175,7 +175,7 @@ export const getSavedCookies = (): Promise<
     .readFile(path.resolve(processPWD, 'data', 'cookies.json'), 'utf-8')
     .then((cockiesString) => JSON.parse(cockiesString))
     .catch((error) => {
-      console.error(error.message);
+      console.error('getSavedCookies error: ', error.message);
       return null;
     });
 

From 4dff69cff432e02a7403894df78d82983b2ca1ac Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Fri, 7 Jul 2023 23:54:21 +0200
Subject: [PATCH 30/31] #80 fixed nft image src - to be stamped

---
 src/queue/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/queue/index.ts b/src/queue/index.ts
index 8c12a50..aa56c3c 100644
--- a/src/queue/index.ts
+++ b/src/queue/index.ts
@@ -78,7 +78,7 @@ uploadQueue.process(async (job) => {
   const ts = Date.now();
   const moment = createMoment(ts);
   const name = createNftName(tweetId, moment);
-  const image = 'ipfs://' + screenshotCid;
+  const image = 'ipfs://' + stampedScreenShotCid;
   const time = new Date(ts).toUTCString();
 
   const description = createNftDescription(tweetId, author, moment);

From 218146627977a3b5062d5c65bbc30ff57309a48a Mon Sep 17 00:00:00 2001
From: poludnev <poludnev.roman@gmail.com>
Date: Sat, 15 Jul 2023 00:17:29 +0200
Subject: [PATCH 31/31] update active job response

---
 src/helpers/handlers.ts | 13 ++++++++++---
 src/types/index.ts      |  4 ++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/helpers/handlers.ts b/src/helpers/handlers.ts
index 797f5fe..94f4e3e 100644
--- a/src/helpers/handlers.ts
+++ b/src/helpers/handlers.ts
@@ -16,7 +16,13 @@ import {
   waitForSelectorWithTimeout,
 } from '../helpers';
 import puppeteer, { Browser, ElementHandle, HTTPResponse, Page } from 'puppeteer';
-import { IGetScreenshotResponseData, IMetadata, ITweetTimelineEntry } from '../types';
+import {
+  IGetScreenshotResponseData,
+  IMetadata,
+  IResponseData,
+  ITweetData,
+  ITweetTimelineEntry,
+} from '../types';
 import {
   DEFAULT_TIMEOUT_MS,
   TWITTER_LOGIN_BUTTON_TEXT_CONTENT,
@@ -249,12 +255,13 @@ const getScreenshotWithPuppeteer = async (
     }
 
     if (!!activeJob) {
-      const { stampedImageBuffer, metadata, tweetdata } = activeJob.data;
+      const { stampedImageBuffer, metadata, tweetdata, parsedTweetData } = activeJob.data;
 
-      const responseData: IGetScreenshotResponseData = {
+      const responseData: IResponseData = {
         imageUrl: makeImageBase64UrlfromBuffer(Buffer.from(stampedImageBuffer!)),
         metadata,
         tweetdata,
+        parsedTweetData,
       };
       response.set('Content-Type', 'application/json');
       return response.status(200).send({ ...responseData, jobId: activeJob.id });
diff --git a/src/types/index.ts b/src/types/index.ts
index 353db8f..a8afe93 100644
--- a/src/types/index.ts
+++ b/src/types/index.ts
@@ -76,6 +76,10 @@ export interface IGetScreenshotResponseData {
   tweetdata: string | null;
 }
 
+export interface IResponseData extends IGetScreenshotResponseData {
+  parsedTweetData: ITweetData | null;
+}
+
 export type ITweetPageMetaData = [IMetadata | null, ITweetRawData | null];
 
 export interface ITweetTimelineEntry {