diff --git a/app/job/rest.py b/app/job/rest.py index 856646b693..dd86bc77a7 100644 --- a/app/job/rest.py +++ b/app/job/rest.py @@ -1,12 +1,15 @@ import time import uuid +from concurrent.futures import ThreadPoolExecutor from datetime import datetime import dateutil -from flask import Blueprint, current_app, jsonify, request +from flask import Blueprint, copy_current_request_context, current_app, jsonify, request +from notifications_utils.clients.redis.annual_limit import TOTAL_SMS_FISCAL_YEAR_TO_YESTERDAY from notifications_utils.recipients import RecipientCSV from notifications_utils.template import Template +from app.annual_limit_utils import get_annual_limit_notifications_v2 from app.aws.s3 import get_job_from_s3, get_job_metadata_from_s3 from app.celery.tasks import process_job from app.config import QueueNames @@ -52,6 +55,7 @@ notifications_filter_schema, unarchived_template_schema, ) +from app.sms_fragment_utils import fetch_todays_requested_sms_count from app.utils import midnight_n_days_ago, pagination_links job_blueprint = Blueprint("job", __name__, url_prefix="/service//job") @@ -194,16 +198,32 @@ def create_job(service_id): # calculate the number of simulated recipients t0 = time.time() - requested_recipients = [i["phone_number"].data for i in list(recipient_csv.get_rows())] - t1 = time.time() - current_app.logger.info("[create_job] built requested_recipients list in {:.3f}s".format(t1 - t0)) - t0 = time.time() - has_simulated, has_real_recipients = csv_has_simulated_and_non_simulated_recipients( - requested_recipients, template.template_type - ) + # Fetch limits in parallel + @copy_current_request_context + def get_sms_sent_today(): + return fetch_todays_requested_sms_count(service.id) + + @copy_current_request_context + def get_sms_sent_this_fiscal(): + return get_annual_limit_notifications_v2(service.id) + + with ThreadPoolExecutor() as executor: + future_sms_sent_today = executor.submit(get_sms_sent_today) + future_sms_sent_this_fiscal = executor.submit(get_sms_sent_this_fiscal) + + # Use list comprehension but avoid intermediate list(recipient_csv.get_rows()) + requested_recipients = [i["phone_number"].data for i in recipient_csv.get_rows()] + + has_simulated, has_real_recipients = csv_has_simulated_and_non_simulated_recipients( + requested_recipients, template.template_type + ) + + sms_sent_today = future_sms_sent_today.result() + sms_sent_this_fiscal = future_sms_sent_this_fiscal.result()[TOTAL_SMS_FISCAL_YEAR_TO_YESTERDAY] + t1 = time.time() - current_app.logger.info("[create_job] csv_has_simulated_and_non_simulated_recipients took {:.3f}s".format(t1 - t0)) + current_app.logger.info("[create_job] built requested_recipients list and checked limits in {:.3f}s".format(t1 - t0)) if has_simulated and has_real_recipients: raise InvalidRequest(message="Bulk sending to testing and non-testing numbers is not supported", status_code=400) @@ -211,12 +231,14 @@ def create_job(service_id): # Check and track limits if we're not sending test notifications if has_real_recipients and not has_simulated: t0 = time.time() - check_sms_annual_limit(service, len(recipient_csv)) + check_sms_annual_limit( + service, len(recipient_csv), sms_sent_today=sms_sent_today, sms_sent_this_fiscal=sms_sent_this_fiscal + ) t1 = time.time() current_app.logger.info("[create_job] check_sms_annual_limit took {:.3f}s".format(t1 - t0)) t0 = time.time() - check_sms_daily_limit(service, len(recipient_csv)) + check_sms_daily_limit(service, len(recipient_csv), messages_sent=sms_sent_today) t1 = time.time() current_app.logger.info("[create_job] check_sms_daily_limit took {:.3f}s".format(t1 - t0)) diff --git a/app/notifications/validators.py b/app/notifications/validators.py index 40aef3c381..46b326bad3 100644 --- a/app/notifications/validators.py +++ b/app/notifications/validators.py @@ -130,8 +130,9 @@ def check_service_over_daily_message_limit(key_type: ApiKeyType, service: Servic counter_name="rate_limit.live_service_daily_sms", exception=LiveServiceTooManySMSRequestsError, ) -def check_sms_daily_limit(service: Service, requested_sms=0): - messages_sent = fetch_todays_requested_sms_count(service.id) +def check_sms_daily_limit(service: Service, requested_sms=0, messages_sent=None): + if messages_sent is None: + messages_sent = fetch_todays_requested_sms_count(service.id) over_sms_daily_limit = (messages_sent + requested_sms) > service.sms_daily_limit # Send a warning when reaching the daily message limit @@ -238,10 +239,12 @@ def check_email_annual_limit(service: Service, requested_emails=0): counter_name="rate_limit.live_service_annual_sms", exception=LiveServiceRequestExceedsSMSAnnualLimitError, ) -def check_sms_annual_limit(service: Service, requested_sms=0): +def check_sms_annual_limit(service: Service, requested_sms=0, sms_sent_today=None, sms_sent_this_fiscal=None): current_fiscal_year = get_fiscal_year(datetime.utcnow()) - sms_sent_today = fetch_todays_requested_sms_count(service.id) - sms_sent_this_fiscal = get_annual_limit_notifications_v2(service.id)[TOTAL_SMS_FISCAL_YEAR_TO_YESTERDAY] + if sms_sent_today is None: + sms_sent_today = fetch_todays_requested_sms_count(service.id) + if sms_sent_this_fiscal is None: + sms_sent_this_fiscal = get_annual_limit_notifications_v2(service.id)[TOTAL_SMS_FISCAL_YEAR_TO_YESTERDAY] send_exceeds_annual_limit = (sms_sent_today + sms_sent_this_fiscal + requested_sms) > service.sms_annual_limit send_reaches_annual_limit = (sms_sent_today + sms_sent_this_fiscal + requested_sms) == service.sms_annual_limit is_near_annual_limit = (sms_sent_today + sms_sent_this_fiscal + requested_sms) >= (