Extended failure check/notification to module path and env vars on run-remote (#345)

Author: filipecosta90

redisbench_admin/run/modules.py

@@ -8,6 +8,8 @@
 
 
 def redis_modules_check(local_module_files):
+    status = True
+    error_message = ""
     if local_module_files is not None:
         logging.info(
             "Using the following module artifacts: {}".format(local_module_files)
@@ -17,15 +19,14 @@ def redis_modules_check(local_module_files):
                 "Checking if module artifact {} exists...".format(local_module_file)
             )
             if os.path.exists(local_module_file) is False:
-                logging.error(
-                    "Specified module artifact does not exist: {}".format(
-                        local_module_file
-                    )
+                error_message = "Specified module artifact does not exist: {}".format(
+                    local_module_file
                 )
-                exit(1)
+                logging.error(error_message)
             else:
                 logging.info(
                     "Confirmed that module artifact: '{}' exists!".format(
                         local_module_file
                     )
                 )
+    return status, error_message

redisbench_admin/run_remote/run_remote.py

@@ -121,12 +121,60 @@ def run_remote_command_logic(args, project_name, project_version):
     profilers_enabled = args.enable_profilers
     keep_env_and_topo = args.keep_env_and_topo
 
+    if WH_TOKEN is not None:
+        webhook_notifications_active = True
+
     webhook_url = "https://hooks.slack.com/services/{}".format(WH_TOKEN)
 
-    if args.skip_env_vars_verify is False:
-        check_ec2_env()
+    ci_job_link = CIRCLE_BUILD_URL
+    ci_job_name = CIRCLE_JOB
+    failure_reason = ""
+    webhook_client_slack = None
+    if ci_job_link is None:
+        webhook_notifications_active = False
+        logging.warning(
+            "Disabling webhook notificaitons given CIRCLE_BUILD_URL is None"
+        )
+
+    if webhook_notifications_active is True:
+        logging.info(
+            "Detected where in a CI flow named {}. Here's the reference link: {}".format(
+                ci_job_name, ci_job_link
+            )
+        )
+        webhook_client_slack = WebhookClient(webhook_url)
 
-    redis_modules_check(local_module_files)
+    if args.skip_env_vars_verify is False:
+        env_check_status, failure_reason = check_ec2_env()
+        if env_check_status is False:
+            if webhook_notifications_active:
+                generate_failure_notification(
+                    webhook_client_slack,
+                    ci_job_name,
+                    ci_job_link,
+                    failure_reason,
+                    tf_github_org,
+                    tf_github_repo,
+                    tf_github_branch,
+                    None,
+                )
+            exit(1)
+
+    module_check_status, error_message = redis_modules_check(local_module_files)
+    if module_check_status is False:
+        if webhook_notifications_active:
+            failure_reason = error_message
+            generate_failure_notification(
+                webhook_client_slack,
+                ci_job_name,
+                ci_job_link,
+                failure_reason,
+                tf_github_org,
+                tf_github_repo,
+                tf_github_branch,
+                None,
+            )
+        exit(1)
 
     common_properties_log(
         tf_bin_path,
@@ -151,20 +199,6 @@ def run_remote_command_logic(args, project_name, project_version):
         clusterconfig,
     ) = prepare_benchmark_definitions(args)
 
-    ci_job_link = CIRCLE_BUILD_URL
-    ci_job_name = CIRCLE_JOB
-    failure_reason = ""
-    webhook_notifications_active = False
-    webhook_client_slack = None
-    if ci_job_link is not None:
-        logging.info(
-            "Detected where in a CI flow named {}. Here's the reference link: {}".format(
-                ci_job_name, ci_job_link
-            )
-        )
-        webhook_notifications_active = True
-        webhook_client_slack = WebhookClient(webhook_url)
-
     return_code = 0
     if benchmark_defs_result is False:
         return_code = 1

redisbench_admin/utils/remote.py

@@ -1040,12 +1040,21 @@ def get_overall_dashboard_keynames(
 
 
 def check_ec2_env():
+    status = True
+    error_message = ""
     if EC2_ACCESS_KEY is None or EC2_ACCESS_KEY == "":
-        logging.error("missing required AWS_ACCESS_KEY_ID env variable")
-        exit(1)
+        error_message = "missing required AWS_ACCESS_KEY_ID env variable"
+        status = False
+
     if EC2_REGION is None or EC2_REGION == "":
-        logging.error("missing required AWS_DEFAULT_REGION env variable")
-        exit(1)
+        error_message = "missing required AWS_DEFAULT_REGION env variable"
+        status = False
+
     if EC2_SECRET_KEY is None or EC2_SECRET_KEY == "":
-        logging.error("missing required AWS_SECRET_ACCESS_KEY env variable")
-        exit(1)
+        error_message = "missing required AWS_SECRET_ACCESS_KEY env variable"
+        status = False
+
+    if status is False:
+        logging.error(error_message)
+
+    return status, error_message