Initial integration with slack notifications on remote failure (#344)

* Initial integration with slack notifications on remote failure

Author: filipecosta90

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.8.1"
+version = "0.8.2"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "README.md"
@@ -44,6 +44,9 @@ scikit-learn = "^0.22.2"
 Jinja2 = "^3.0.3"
 watchdog = "^2.1.6"
 redis = "4.2.2"
+slack-sdk = "^3.15.2"
+slack-bolt = "^1.13.0"
+certifi = "^2021.10.8"
 
 [tool.poetry.dev-dependencies]
 pytest = "^4.6"

pyproject.toml

@@ -52,6 +52,10 @@
 )
 
 BENCHMARK_REPETITIONS = int(os.getenv("BENCHMARK_REPETITIONS", 1))
+# circleci related info
+CIRCLE_BUILD_URL = os.getenv("CIRCLE_BUILD_URL", None)
+CIRCLE_JOB = os.getenv("CIRCLE_JOB", None)
+WH_TOKEN = os.getenv("PERFORMANCE_WH_TOKEN", None)
 REDIS_BINARY = os.getenv("REDIS_BINARY", "redis-server")
 
 

redisbench_admin/run/common.py

@@ -0,0 +1,44 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2022., Redis Labs Modules
+#  All rights reserved.
+#
+
+import logging
+
+
+def generate_failure_notification(
+    webhook_client,
+    job_name,
+    https_link,
+    failure_reason,
+    gh_org,
+    gh_repo,
+    branch=None,
+    tag=None,
+):
+
+    headline_test = "{}/{} FAILED job {} due to {}".format(
+        gh_org, gh_repo, job_name, failure_reason
+    )
+    extra_detail = ""
+    if branch is not None:
+        extra_detail = "This job was on a branch named {}\n".format(branch)
+    if tag is not None:
+        extra_detail = "This job was on a tag named {}\n".format(tag)
+    blocks = [
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "{}/{} job name {} failed due to *{}*.\n{}<{}|Check CI job details>\n".format(
+                    gh_org, gh_repo, job_name, failure_reason, extra_detail, https_link
+                ),
+            },
+        }
+    ]
+    response = webhook_client.send(text=headline_test, blocks=blocks)
+    if response.status_code != 200:
+        logging.error(
+            "Error while sending slack notification. Error message {}".response.body
+        )

redisbench_admin/run_remote/notifications.py

@@ -25,6 +25,9 @@
 from redisbench_admin.run.common import (
     get_start_time_vars,
     BENCHMARK_REPETITIONS,
+    CIRCLE_BUILD_URL,
+    CIRCLE_JOB,
+    WH_TOKEN,
     get_setup_type_and_primaries_count,
     common_properties_log,
     print_results_table_stdout,
@@ -41,6 +44,7 @@
 from redisbench_admin.run.ssh import ssh_pem_check
 from redisbench_admin.run_remote.args import TF_OVERRIDE_NAME
 from redisbench_admin.run_remote.consts import min_recommended_benchmark_duration
+from redisbench_admin.run_remote.notifications import generate_failure_notification
 from redisbench_admin.run_remote.remote_client import run_remote_client_tool
 from redisbench_admin.run_remote.remote_db import (
     remote_tmpdir_prune,
@@ -75,6 +79,7 @@
     EC2_REGION,
 )
 
+from slack_sdk.webhook import WebhookClient
 
 # 7 days expire
 STALL_INFO_DAYS = 7
@@ -116,6 +121,8 @@ def run_remote_command_logic(args, project_name, project_version):
     profilers_enabled = args.enable_profilers
     keep_env_and_topo = args.keep_env_and_topo
 
+    webhook_url = "https://hooks.slack.com/services/{}".format(WH_TOKEN)
+
     if args.skip_env_vars_verify is False:
         check_ec2_env()
 
@@ -144,14 +151,39 @@ def run_remote_command_logic(args, project_name, project_version):
         clusterconfig,
     ) = prepare_benchmark_definitions(args)
 
+    ci_job_link = CIRCLE_BUILD_URL
+    ci_job_name = CIRCLE_JOB
+    failure_reason = ""
+    webhook_notifications_active = False
+    webhook_client_slack = None
+    if ci_job_link is not None:
+        logging.info(
+            "Detected where in a CI flow named {}. Here's the reference link: {}".format(
+                ci_job_name, ci_job_link
+            )
+        )
+        webhook_notifications_active = True
+        webhook_client_slack = WebhookClient(webhook_url)
+
     return_code = 0
     if benchmark_defs_result is False:
         return_code = 1
         if args.fail_fast:
-            logging.critical(
-                "Detected errors while preparing benchmark definitions. Exiting right away!"
-            )
-            exit(1)
+            failure_reason = "Detected errors while preparing benchmark definitions"
+            logging.critical("{}. Exiting right away!".format(failure_reason))
+            if webhook_notifications_active:
+                generate_failure_notification(
+                    webhook_client_slack,
+                    ci_job_name,
+                    ci_job_link,
+                    failure_reason,
+                    tf_github_org,
+                    tf_github_repo,
+                    tf_github_branch,
+                    None,
+                )
+
+            exit(return_code)
 
     remote_envs = {}
     dirname = "."
@@ -872,10 +904,13 @@ def run_remote_command_logic(args, project_name, project_version):
                                         tsname_project_total_failures,
                                     )
                                     return_code |= 1
+                                    failure_reason = "Some unexpected exception was caught during remote work on test named {}".format(
+                                        test_name
+                                    )
                                     logging.critical(
-                                        "Some unexpected exception was caught "
-                                        "during remote work. Failing test...."
+                                        "{}. Failing test....".format(failure_reason)
                                     )
+
                                     logging.critical(sys.exc_info()[0])
                                     print("-" * 60)
                                     traceback.print_exc(file=sys.stdout)
@@ -940,6 +975,20 @@ def run_remote_command_logic(args, project_name, project_version):
                     EXPIRE_TIME_SECS_PROFILE_KEYS,
                     profile_markdown_str,
                 )
+
+    if return_code != 0 and webhook_notifications_active:
+        if failure_reason == "":
+            failure_reason = "Some unexpected exception was caught during remote work"
+        generate_failure_notification(
+            webhook_client_slack,
+            ci_job_name,
+            ci_job_link,
+            failure_reason,
+            tf_github_org,
+            tf_github_repo,
+            tf_github_branch,
+            None,
+        )
     exit(return_code)