feat: enhance release report structure, implement brief summary option for Slack (#19)

Author: kalinkrustev

package-lock.json

@@ -63,13 +63,13 @@
     "snapshot": "npx standard-version --no-verify --skip.changelog --prerelease snapshot --releaseCommitMessageFormat 'chore(snapshot): {{currentTag}}'"
   },
   "dependencies": {
-    "@mojaloop/central-services-logger": "11.5.5",
+    "@mojaloop/central-services-logger": "11.6.2",
     "@mojaloop/ml-testing-toolkit-shared-lib": "14.0.4",
-    "@mojaloop/sdk-standard-components": "19.7.0",
-    "@slack/webhook": "7.0.4",
+    "@mojaloop/sdk-standard-components": "19.10.3",
+    "@slack/webhook": "7.0.5",
     "atob": "2.1.2",
     "aws-sdk": "2.1692.0",
-    "axios": "1.7.9",
+    "axios": "1.8.3",
     "cli-table3": "0.6.5",
     "commander": "13.1.0",
     "dotenv": "16.4.7",

package.json

@@ -48,6 +48,7 @@ program
   .option('--report-target <reportTarget>', 'default: "file://<file_name_genrated_by_backend>"  --- supported targets: "file://path_to_file", "s3://<bucket_name>[/<file_path>]"')
   .option('--slack-webhook-url <slackWebhookUrl>', 'default: "Disabled"  --- supported formats: "https://....."')
   .option('--extra-summary-information <Comma separated values in the format key:value>', 'default: none  --- example: "Testcase Name:something,Environment:Dev1"')
+  .option('--brief-summary-prefix <Prefix to use for a brief summary in Slack>', 'default: none  --- example: "environment name, test name"')
   .on('--help', () => { // Extra information on help message
     console.log('')
     console.log(' *** If the option report-target is set to use AWS S3 service, the variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION) should be passed in environment')

src/client.js

@@ -29,10 +29,21 @@
 const axios = require('axios').default
 const config = require('rc')('release_cd', {})
 
-module.exports = async function (name, result) {
+module.exports = async function (name, { runtimeInformation, test_cases: testCases = [] }) {
   if (!config.reportUrl) return
   const data = {
-    [`tests.${name}`]: result
+    [`tests.${name}`]: {
+      ...runtimeInformation,
+      assertions: Object.fromEntries(testCases.map(
+        testCase => testCase?.requests?.map(
+          request => request?.request?.tests?.assertions
+            .filter(assertion => assertion?.assertionId ?? assertion.id)
+            .map(
+              assertion => [`${testCase.testCaseId ?? testCase.name}.${request.request.requestId ?? request.request.id}.${assertion?.assertionId ?? assertion.id}`, assertion?.resultStatus?.status]
+            )
+        )
+      ).flat(2))
+    }
   }
   console.log(`Sending report to ${config.reportUrl}`, data)
   await axios({

src/extras/release-cd.js

@@ -31,24 +31,38 @@ const objectStore = require('../objectStore')
 
 const config = objectStore.get('config')
 
-const generateSlackBlocks = (progress) => {
+const millisecondsToTime = (milliseconds) => {
+  const seconds = Math.floor(milliseconds / 1000)
+  const minutes = Math.floor(seconds / 60)
+  const hours = Math.floor(minutes / 60)
+  return `${String(hours).padStart(2, '0')}:${String(minutes % 60).padStart(2, '0')}:${String(seconds % 60).padStart(2, '0')}`
+}
+
+const generateSlackBlocks = (progress, reportURL) => {
   const slackBlocks = []
   let totalAssertionsCount = 0
   let totalPassedAssertionsCount = 0
   let totalRequestsCount = 0
+  const failedTestCases = []
   progress.test_cases.forEach(testCase => {
     // console.log(fStr.yellow(testCase.name))
     totalRequestsCount += testCase.requests.length
-    // let testCaseAssertionsCount = 0
-    // let testCasePassedAssertionsCount = 0
+    let testCaseAssertionsCount = 0
+    let testCasePassedAssertionsCount = 0
     testCase.requests.forEach(req => {
       const passedAssertionsCount = req.request.tests && req.request.tests.passedAssertionsCount ? req.request.tests.passedAssertionsCount : 0
       const assertionsCount = req.request.tests && req.request.tests.assertions && req.request.tests.assertions.length ? req.request.tests.assertions.length : 0
       totalAssertionsCount += assertionsCount
       totalPassedAssertionsCount += passedAssertionsCount
-      // testCaseAssertionsCount += assertionsCount
-      // testCasePassedAssertionsCount += passedAssertionsCount
+      testCaseAssertionsCount += assertionsCount
+      testCasePassedAssertionsCount += passedAssertionsCount
     })
+    if (testCaseAssertionsCount !== testCasePassedAssertionsCount) {
+      failedTestCases.push({
+        name: testCase.name,
+        failedAssertions: testCaseAssertionsCount - testCasePassedAssertionsCount
+      })
+    }
     // const passed = testCasePassedAssertionsCount === testCaseAssertionsCount
     // // TODO: make sure this list should not be more than 40 because we can add only max 50 blocks in a slack message
     // if(!passed) {
@@ -61,6 +75,40 @@ const generateSlackBlocks = (progress) => {
     //   })
     // }
   })
+
+  // totalAssertionsCount = totalPassedAssertionsCount
+  // failedTestCases.length = 0
+
+  if (config.briefSummaryPrefix) {
+    const top5FailedTestCases = failedTestCases.sort((a, b) => b.failedAssertions - a.failedAssertions).slice(0, 5)
+    return [{
+      type: 'context',
+      elements: [{
+        type: 'mrkdwn',
+        text: [
+          `${totalAssertionsCount === totalPassedAssertionsCount ? '🟢' : '🔴'}`,
+          reportURL ? `<${reportURL}|${config.briefSummaryPrefix}>` : `${config.briefSummaryPrefix}`,
+          `failed: \`${totalAssertionsCount - totalPassedAssertionsCount}/${totalAssertionsCount}`,
+          `(${(100 * ((totalAssertionsCount - totalPassedAssertionsCount) / totalAssertionsCount)).toFixed(2)}%)\`,`,
+          `requests: \`${totalRequestsCount}\`,`,
+          `tests: \`${progress.test_cases.length}\`,`,
+          `duration: \`${millisecondsToTime(progress.runtimeInformation.runDurationMs)}\``,
+          top5FailedTestCases.length > 0 && '\nTop 5 failed test cases:\n',
+          top5FailedTestCases.length > 0 && top5FailedTestCases.map(tc => `• ${tc.name}: \`${tc.failedAssertions}\``).join('\n')
+        ].filter(Boolean).join(' ')
+      }]
+    }]
+  }
+
+  slackBlocks.push({
+    type: 'header',
+    text: {
+      type: 'plain_text',
+      text: 'Testing Toolkit Report',
+      emoji: true
+    }
+  })
+
   let summaryText = ''
 
   summaryText += '>Total assertions: *' + totalAssertionsCount + '*\n'
@@ -107,40 +155,32 @@ const generateSlackBlocks = (progress) => {
     },
     ...additionalParams
   })
+  if (reportURL) {
+    slackBlocks.push({
+      type: 'section',
+      text: {
+        type: 'mrkdwn',
+        text: '<' + reportURL + '|View Report>'
+      }
+    })
+  }
+  slackBlocks.push({
+    type: 'divider'
+  })
   return slackBlocks
 }
 
-const sendSlackNotification = async (progress, reportURL = null) => {
+const sendSlackNotification = async (progress, reportURL = 'http://localhost/') => {
   if (config.slackWebhookUrl) {
     const url = config.slackWebhookUrl
     const webhook = new IncomingWebhook(url)
-    let slackBlocks = []
-    slackBlocks.push({
-      type: 'header',
-      text: {
-        type: 'plain_text',
-        text: 'Testing Toolkit Report',
-        emoji: true
-      }
-    })
-    slackBlocks = slackBlocks.concat(generateSlackBlocks(progress))
-    if (reportURL) {
-      slackBlocks.push({
-        type: 'section',
-        text: {
-          type: 'mrkdwn',
-          text: '<' + reportURL + '|View Report>'
-        }
-      })
-    }
-    slackBlocks.push({
-      type: 'divider'
-    })
+    const blocks = generateSlackBlocks(progress, reportURL)
+
     try {
       // console.log(JSON.stringify(slackBlocks, null, 2))
       await webhook.send({
         text: 'Test Report',
-        blocks: slackBlocks
+        blocks
       })
       console.log('Slack notification sent.')
     } catch (err) {

src/extras/slack-broadcast.js

@@ -200,7 +200,7 @@ const handleIncomingProgress = async (progress) => {
         }
       }
       try {
-        await releaseCd(config.reportName, progress.totalResult.runtimeInformation)
+        await releaseCd(config.reportName, progress.totalResult)
       } catch (err) {
         console.error(err)
       }

src/modes/outbound.js

@@ -67,6 +67,7 @@ const cli = (commanderOptions) => {
     slackFailedImage: configFile.slackFailedImage,
     baseURL: commanderOptions.baseUrl || configFile.baseURL,
     extraSummaryInformation: commanderOptions.extraSummaryInformation || configFile.extraSummaryInformation,
+    briefSummaryPrefix: commanderOptions.briefSummaryPrefix || configFile.briefSummaryPrefix,
     labels: commanderOptions.labels || configFile.labels,
     batchSize: commanderOptions.batchSize || configFile.batchSize || parseInt(process.env.TESTCASES_BATCH_SIZE, 10)
   }

src/router.js

@@ -42,14 +42,41 @@ describe('Release CD', () => {
   describe('Post test results', () => {
     it('Posts test result to configured URL', async () => {
       const name = 'test'
-      const result = {}
+      const result = {
+        test_cases: [
+          {
+            name: 'test-case',
+            requests: [
+              {
+                request: {
+                  id: 'request-id',
+                  tests: {
+                    assertions: [
+                      {
+                        id: 'assertion-id',
+                        resultStatus: {
+                          status: 'SUCCESS'
+                        }
+                      }
+                    ]
+                  }
+                }
+              }
+            ]
+          }
+        ]
+      }
       config.reportUrl = 'http://example.com'
       await releaseCd(name, result)
       expect(axios.default).toHaveBeenCalledWith({
         method: 'post',
         url: 'http://example.com',
         data: {
-          [`tests.${name}`]: result
+          [`tests.${name}`]: {
+            assertions: {
+              'test-case.request-id.assertion-id': 'SUCCESS'
+            }
+          }
         }
       })
     })

test/unit/extras/release-cd.test.js

@@ -102,6 +102,13 @@ describe('Cli client', () => {
         text: expect.any(String),
         blocks: expect.any(Array)
       }))
+      SpySlackSend.mockResolvedValueOnce(null)
+      config.briefSummaryPrefix = 'brief'
+      await expect(slackBroadCast.sendSlackNotification(sampleProgress)).resolves.toBe(undefined)
+      expect(SpySlackSend).toHaveBeenCalledWith(expect.objectContaining({
+        text: expect.any(String),
+        blocks: expect.any(Array)
+      }))
     })
     it('When failed case, it should call slack send function', async () => {
       config.slackWebhookUrl = 'http://some_url'