feat(csi-1309): added notifications for failed tests only (#21)

* feat(csi-704): removed unneeded code

* feat(csi-1306): added notifications for failed tests

* chore(snapshot): 1.8.0-snapshot.0

* feat(csi-1306): added notifications for failed tests

* chore(snapshot): 1.8.0-snapshot.1

* feat(csi-1309): added notifications for failed tests only

* feat(csi-1309): code cleanup

* chore(snapshot): 1.8.0-snapshot.2

* fix: improve readability of brief summary

---------

Co-authored-by: Kalin Krustev <kalin.krustev@gmail.com>

Author: geka-evk

package-lock.json

@@ -1,7 +1,7 @@
 {
   "name": "@mojaloop/ml-testing-toolkit-client-lib",
   "description": "Testing Toolkit Client Library",
-  "version": "1.7.0",
+  "version": "1.8.0-snapshot.2",
   "license": "Apache-2.0",
   "author": "Vijaya Kumar Guthi, ModusBox Inc. ",
   "contributors": [

package.json

@@ -47,6 +47,7 @@ program
   .option('--report-auto-filename-enable <reportAutoFilenameEnable>', 'default: false, if true the file name will be generated by the backend')
   .option('--report-target <reportTarget>', 'default: "file://<file_name_genrated_by_backend>"  --- supported targets: "file://path_to_file", "s3://<bucket_name>[/<file_path>]"')
   .option('--slack-webhook-url <slackWebhookUrl>', 'default: "Disabled"  --- supported formats: "https://....."')
+  .option('--slack-webhook-url-for-failed <slackWebhookUrl>', 'default: "Disabled"  --- supported formats: "https://....."')
   .option('--extra-summary-information <Comma separated values in the format key:value>', 'default: none  --- example: "Testcase Name:something,Environment:Dev1"')
   .option('--brief-summary-prefix <Prefix to use for a brief summary in Slack>', 'default: none  --- example: "environment name, test name"')
   .on('--help', () => { // Extra information on help message

src/client.js

@@ -38,6 +38,10 @@ const millisecondsToTime = (milliseconds) => {
   return `${String(hours).padStart(2, '0')}:${String(minutes % 60).padStart(2, '0')}:${String(seconds % 60).padStart(2, '0')}`
 }
 
+/**
+ * @param {FinalReport} progress
+ * @param {string} reportURL - URL of the report
+ */
 const generateSlackBlocks = (progress, reportURL) => {
   const slackBlocks = []
   let totalAssertionsCount = 0
@@ -88,10 +92,10 @@ const generateSlackBlocks = (progress, reportURL) => {
         text: [
           `${totalAssertionsCount === totalPassedAssertionsCount ? '🟢' : '🔴'}`,
           reportURL ? `<${reportURL}|${config.briefSummaryPrefix}>` : `${config.briefSummaryPrefix}`,
+          `tests: \`${progress.test_cases.length}\`,`,
+          `requests: \`${totalRequestsCount}\`,`,
           `failed: \`${totalAssertionsCount - totalPassedAssertionsCount}/${totalAssertionsCount}`,
           `(${(100 * ((totalAssertionsCount - totalPassedAssertionsCount) / totalAssertionsCount)).toFixed(2)}%)\`,`,
-          `requests: \`${totalRequestsCount}\`,`,
-          `tests: \`${progress.test_cases.length}\`,`,
           `duration: \`${millisecondsToTime(progress.runtimeInformation.runDurationMs)}\``,
           top5FailedTestCases.length > 0 && '\nTop 5 failed test cases:\n',
           top5FailedTestCases.length > 0 && top5FailedTestCases.map(tc => `• ${tc.name}: \`${tc.failedAssertions}\``).join('\n')
@@ -170,25 +174,52 @@ const generateSlackBlocks = (progress, reportURL) => {
   return slackBlocks
 }
 
+/**
+ * @param {FinalReport} progress
+ * @param {string} reportURL - URL of the report
+ */
 const sendSlackNotification = async (progress, reportURL = 'http://localhost/') => {
-  if (config.slackWebhookUrl) {
-    const url = config.slackWebhookUrl
-    const webhook = new IncomingWebhook(url)
-    const blocks = generateSlackBlocks(progress, reportURL)
-
-    try {
-      // console.log(JSON.stringify(slackBlocks, null, 2))
-      await webhook.send({
-        text: 'Test Report',
-        blocks
-      })
-      console.log('Slack notification sent.')
-    } catch (err) {
-      console.log('ERROR: Sending slack notification failed. ', err.message)
-    }
+  const { slackWebhookUrl, slackWebhookUrlForFailed } = config
+
+  if (!slackWebhookUrl && !needToNotifyFailed(slackWebhookUrlForFailed, progress)) {
+    console.log('No Slack webhook URLs configured.')
+    return
+  }
+  const blocks = generateSlackBlocks(progress, reportURL)
+
+  if (slackWebhookUrl) {
+    await sendWebhook(slackWebhookUrl, 'Test Report', blocks)
+  }
+
+  if (needToNotifyFailed(slackWebhookUrlForFailed, progress)) {
+    await sendWebhook(slackWebhookUrlForFailed, 'Failed Tests Report', blocks)
+  }
+}
+
+const sendWebhook = async (url, text, blocks) => {
+  const webhook = new IncomingWebhook(url)
+  try {
+    await webhook.send({ text, blocks })
+    console.log('Slack notification sent.')
+  } catch (err) {
+    console.log('ERROR: Sending Slack notification failed. ', err.message)
   }
 }
 
+/**
+ * Determines if a notification for failed tests needs to be sent.
+ * @param {string | undefined} webhookUrl
+ * @param {FinalReport} progress
+ * @returns {boolean}
+ */
+const needToNotifyFailed = (webhookUrl, progress) => {
+  return webhookUrl && (!progress?.runtimeInformation?.totalAssertions
+    ? true
+    : progress.runtimeInformation.totalPassedAssertions !== progress.runtimeInformation.totalAssertions)
+}
+
 module.exports = {
-  sendSlackNotification
+  sendSlackNotification,
+  sendWebhook,
+  needToNotifyFailed
 }

src/extras/slack-broadcast.js

@@ -180,6 +180,36 @@ const sendTemplate = async (sessionId) => {
   }
 }
 
+/**
+ * Consolidated final report, created by generateFinalReport function (ml-testing-toolkit repo).
+ * @typedef {Object} FinalReport
+ * @property {RuntimeInformation} runtimeInformation - Provides metadata about the runtime environment or execution context.
+ * @property {Array<Object>} test_cases - An array of objects where each object represents a test case and its results.
+ * @property {string} status
+ * @property {Object} totalResult
+ * @property {Object} saveReportStatus
+ * @property {unknown} [otherFields] - see ml-testing-toolkit repo.
+ */
+
+/**
+ * @typedef {Object} RuntimeInformation
+ * @property {string} testReportId - Test report ID
+ * @property {string} completedTimeISO - Completed time in ISO format
+ * @property {string} startedTime - Started time in readable format
+ * @property {string} completedTime - Completed time in readable format
+ * @property {string} completedTimeUTC - Completed time in UTC format
+ * @property {number} startedTS - Started timestamp
+ * @property {number} completedTS - Completed timestamp
+ * @property {number} runDurationMs - Run duration in milliseconds
+ * @property {number} totalAssertions - Total number of assertions
+ * @property {number} totalPassedAssertions - Total number of passed assertions
+ */
+
+/**
+ * Handles incoming progress updates and processes them as needed.
+ * @param {FinalReport} progress
+ * @returns {Promise<void>}
+ */
 const handleIncomingProgress = async (progress) => {
   const config = objectStore.get('config')
   if (progress.status === 'FINISHED') {

src/modes/outbound.js

@@ -63,6 +63,7 @@ const cli = (commanderOptions) => {
     reportAutoFilenameEnable: commanderOptions.reportAutoFilenameEnable === 'true' || configFile.reportAutoFilenameEnable === true,
     reportTarget: commanderOptions.reportTarget || configFile.reportTarget,
     slackWebhookUrl: commanderOptions.slackWebhookUrl || configFile.slackWebhookUrl,
+    slackWebhookUrlForFailed: commanderOptions.slackWebhookUrlForFailed || configFile.slackWebhookUrlForFailed,
     slackPassedImage: configFile.slackPassedImage,
     slackFailedImage: configFile.slackFailedImage,
     baseURL: commanderOptions.baseUrl || configFile.baseURL,

src/router.js

@@ -134,4 +134,32 @@ describe('Cli client', () => {
       }))
     })
   })
+
+  describe('needToNotifyFailed Tests -->', () => {
+    it('should not notify if slackWebhookUrlForFailed is not configured', () => {
+      expect(slackBroadCast.needToNotifyFailed(undefined, {})).toBeFalsy()
+    })
+
+    it('should notify if webhookUrl is set, but no progress.runtimeInformation info', () => {
+      expect(slackBroadCast.needToNotifyFailed('url', {})).toBe(true)
+    })
+
+    it('should NOT notify success tests', () => {
+      expect(slackBroadCast.needToNotifyFailed('url', {
+        runtimeInformation: {
+          totalAssertions: 1,
+          totalPassedAssertions: 1
+        }
+      })).toBe(false)
+    })
+
+    it('should notify in case failed tests', () => {
+      expect(slackBroadCast.needToNotifyFailed('url', {
+        runtimeInformation: {
+          totalAssertions: 1,
+          totalPassedAssertions: 0
+        }
+      })).toBe(true)
+    })
+  })
 })

test/unit/extras/slack-broadcast.test.js

@@ -40,7 +40,7 @@ describe('Cli client', () => {
     it('when mode is monitoring should not throw an error', async () => {
       const config = {
         "mode": "monitoring"
-      }  
+      }
       spyExit.mockImplementationOnce(jest.fn())
       expect(() => {
         cli(config)