Task/fix slowdown

app/job/rest.py

@@ -1,12 +1,15 @@
 import time
 import uuid
+from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 
 import dateutil
-from flask import Blueprint, current_app, jsonify, request
+from flask import Blueprint, copy_current_request_context, current_app, jsonify, request
+from notifications_utils.clients.redis.annual_limit import TOTAL_SMS_FISCAL_YEAR_TO_YESTERDAY
 from notifications_utils.recipients import RecipientCSV
 from notifications_utils.template import Template
 
+from app.annual_limit_utils import get_annual_limit_notifications_v2
 from app.aws.s3 import get_job_from_s3, get_job_metadata_from_s3
 from app.celery.tasks import process_job
 from app.config import QueueNames
@@ -52,6 +55,7 @@
     notifications_filter_schema,
     unarchived_template_schema,
 )
+from app.sms_fragment_utils import fetch_todays_requested_sms_count
 from app.utils import midnight_n_days_ago, pagination_links
 
 job_blueprint = Blueprint("job", __name__, url_prefix="/service/<uuid:service_id>/job")
@@ -194,29 +198,47 @@ def create_job(service_id):
 
         # calculate the number of simulated recipients
         t0 = time.time()
-        requested_recipients = [i["phone_number"].data for i in list(recipient_csv.get_rows())]
-        t1 = time.time()
-        current_app.logger.info("[create_job] built requested_recipients list in {:.3f}s".format(t1 - t0))
 
-        t0 = time.time()
-        has_simulated, has_real_recipients = csv_has_simulated_and_non_simulated_recipients(
-            requested_recipients, template.template_type
-        )
+        # Fetch limits in parallel
+        @copy_current_request_context
+        def get_sms_sent_today():
+            return fetch_todays_requested_sms_count(service.id)
+
+        @copy_current_request_context
+        def get_sms_sent_this_fiscal():
+            return get_annual_limit_notifications_v2(service.id)
+
+        with ThreadPoolExecutor() as executor:
+            future_sms_sent_today = executor.submit(get_sms_sent_today)
+            future_sms_sent_this_fiscal = executor.submit(get_sms_sent_this_fiscal)
+
+            # Use list comprehension but avoid intermediate list(recipient_csv.get_rows())
+            requested_recipients = [i["phone_number"].data for i in recipient_csv.get_rows()]
+
+            has_simulated, has_real_recipients = csv_has_simulated_and_non_simulated_recipients(
+                requested_recipients, template.template_type
+            )
+
+            sms_sent_today = future_sms_sent_today.result()
+            sms_sent_this_fiscal = future_sms_sent_this_fiscal.result()[TOTAL_SMS_FISCAL_YEAR_TO_YESTERDAY]
+
         t1 = time.time()
-        current_app.logger.info("[create_job] csv_has_simulated_and_non_simulated_recipients took {:.3f}s".format(t1 - t0))
+        current_app.logger.info("[create_job] built requested_recipients list and checked limits in {:.3f}s".format(t1 - t0))
 
         if has_simulated and has_real_recipients:
             raise InvalidRequest(message="Bulk sending to testing and non-testing numbers is not supported", status_code=400)
 
         # Check and track limits if we're not sending test notifications
         if has_real_recipients and not has_simulated:
             t0 = time.time()
-            check_sms_annual_limit(service, len(recipient_csv))
+            check_sms_annual_limit(
+                service, len(recipient_csv), sms_sent_today=sms_sent_today, sms_sent_this_fiscal=sms_sent_this_fiscal
+            )
             t1 = time.time()
             current_app.logger.info("[create_job] check_sms_annual_limit took {:.3f}s".format(t1 - t0))
 
             t0 = time.time()
-            check_sms_daily_limit(service, len(recipient_csv))
+            check_sms_daily_limit(service, len(recipient_csv), messages_sent=sms_sent_today)
             t1 = time.time()
             current_app.logger.info("[create_job] check_sms_daily_limit took {:.3f}s".format(t1 - t0))
 

app/notifications/validators.py

@@ -130,8 +130,9 @@ def check_service_over_daily_message_limit(key_type: ApiKeyType, service: Servic
     counter_name="rate_limit.live_service_daily_sms",
     exception=LiveServiceTooManySMSRequestsError,
 )
-def check_sms_daily_limit(service: Service, requested_sms=0):
-    messages_sent = fetch_todays_requested_sms_count(service.id)
+def check_sms_daily_limit(service: Service, requested_sms=0, messages_sent=None):
+    if messages_sent is None:
+        messages_sent = fetch_todays_requested_sms_count(service.id)
     over_sms_daily_limit = (messages_sent + requested_sms) > service.sms_daily_limit
 
     # Send a warning when reaching the daily message limit
@@ -238,10 +239,12 @@ def check_email_annual_limit(service: Service, requested_emails=0):
     counter_name="rate_limit.live_service_annual_sms",
     exception=LiveServiceRequestExceedsSMSAnnualLimitError,
 )
-def check_sms_annual_limit(service: Service, requested_sms=0):
+def check_sms_annual_limit(service: Service, requested_sms=0, sms_sent_today=None, sms_sent_this_fiscal=None):
     current_fiscal_year = get_fiscal_year(datetime.utcnow())
-    sms_sent_today = fetch_todays_requested_sms_count(service.id)
-    sms_sent_this_fiscal = get_annual_limit_notifications_v2(service.id)[TOTAL_SMS_FISCAL_YEAR_TO_YESTERDAY]
+    if sms_sent_today is None:
+        sms_sent_today = fetch_todays_requested_sms_count(service.id)
+    if sms_sent_this_fiscal is None:
+        sms_sent_this_fiscal = get_annual_limit_notifications_v2(service.id)[TOTAL_SMS_FISCAL_YEAR_TO_YESTERDAY]
     send_exceeds_annual_limit = (sms_sent_today + sms_sent_this_fiscal + requested_sms) > service.sms_annual_limit
     send_reaches_annual_limit = (sms_sent_today + sms_sent_this_fiscal + requested_sms) == service.sms_annual_limit
     is_near_annual_limit = (sms_sent_today + sms_sent_this_fiscal + requested_sms) >= (