Change the get notifications for a month to call a new summary table

app/dao/fact_notification_status_dao.py

@@ -25,6 +25,7 @@
     SMS_TYPE,
     ApiKey,
     FactNotificationStatus,
+    MonthlyNotificationStatsSummary,
     Notification,
     NotificationHistory,
     Service,
@@ -174,7 +175,12 @@ def fetch_notification_status_for_service_by_month(start_date, end_date, service
 
 
 def fetch_delivered_notification_stats_by_month(filter_heartbeats=None):
-    query = (
+    """
+    Fetch delivered/sent notification stats by month from the summary table.
+    This is much faster than querying the 28M+ row ft_notification_status table.
+
+    This originally used to call ft_notification_status
+        query = (
         db.session.query(
             func.date_trunc("month", FactNotificationStatus.bst_date).cast(db.Text).label("month"),
             FactNotificationStatus.notification_type,
@@ -183,7 +189,7 @@ def fetch_delivered_notification_stats_by_month(filter_heartbeats=None):
         .filter(
             FactNotificationStatus.key_type != KEY_TYPE_TEST,
             FactNotificationStatus.notification_status.in_([NOTIFICATION_DELIVERED, NOTIFICATION_SENT]),
-            FactNotificationStatus.bst_date >= "2019-11-01",  # GC Notify start date
+            FactNotificationStatus.bst_date >= "2019-11-01",  # ~6 years of data
         )
         .group_by(
             func.date_trunc("month", FactNotificationStatus.bst_date),
@@ -196,14 +202,38 @@ def fetch_delivered_notification_stats_by_month(filter_heartbeats=None):
     )
     if filter_heartbeats:
         query = query.filter(
-            FactNotificationStatus.service_id.notin_(
+            FactNotificationStatus.service_id.notin_([...])  # Excludes 2 services
+        )
+    return query.all()
+
+    But now we store the results of this query in MonthlyNotificationStatsSummary. We only store
+    delivered, and sent notifications in this table, and we aggregate it as well. We also exclude any
+    TEST keys.
+
+    See the celery task in reporting_tasks.py called create_monthly_notification_status_summary
+    """
+    query = db.session.query(
+        MonthlyNotificationStatsSummary.month,
+        MonthlyNotificationStatsSummary.notification_type,
+        MonthlyNotificationStatsSummary.notification_count.label("count"),
+    ).filter(
+        MonthlyNotificationStatsSummary.month >= "2019-11-01",  # GC Notify start date
+    )
+
+    if filter_heartbeats:
+        query = query.filter(
+            MonthlyNotificationStatsSummary.service_id.notin_(
                 [
                     current_app.config["NOTIFY_SERVICE_ID"],
                     current_app.config["HEARTBEAT_SERVICE_ID"],
                 ]
             ),
         )
-    return query.all()
+
+    return query.order_by(
+        MonthlyNotificationStatsSummary.month.desc(),
+        MonthlyNotificationStatsSummary.notification_type,
+    ).all()
 
 
 def fetch_notification_stats_for_trial_services():

tests/app/dao/test_fact_notification_status_dao.py

@@ -48,6 +48,7 @@
     create_api_key,
     create_ft_notification_status,
     create_job,
+    create_monthly_notification_stats_summary,
     create_notification,
     create_notification_history,
     create_service,
@@ -813,138 +814,120 @@ def test_fetch_monthly_template_usage_for_service(sample_service):
     assert results[3].count == 6
 
 
-@freeze_time("2020-11-02 14:00")
-def test_fetch_delivered_notification_stats_by_month(sample_service):
-    sms_template = create_template(service=sample_service, template_type="sms", template_name="a")
-    email_template = create_template(service=sample_service, template_type="email", template_name="b")
-
-    # Not counted: before GC Notify started
-    create_ft_notification_status(
-        utc_date=date(2019, 10, 10),
-        service=sample_service,
-        template=email_template,
-        count=3,
-    )
-
-    create_ft_notification_status(
-        utc_date=date(2019, 12, 10),
-        service=sample_service,
-        template=email_template,
-        count=3,
-    )
-
-    create_ft_notification_status(
-        utc_date=date(2019, 12, 5),
-        service=sample_service,
-        template=sms_template,
-        notification_status=NOTIFICATION_DELIVERED,
-        count=6,
-    )
-
-    create_ft_notification_status(
-        utc_date=date(2020, 1, 1),
-        service=sample_service,
-        template=sms_template,
-        notification_status=NOTIFICATION_SENT,
-        count=4,
-    )
-
-    # Not counted: failed notifications
-    create_ft_notification_status(
-        utc_date=date(2020, 1, 1),
-        service=sample_service,
-        template=sms_template,
-        notification_status=NOTIFICATION_FAILED,
-        count=10,
-    )
-
-    create_ft_notification_status(
-        utc_date=date(2020, 3, 1),
-        service=sample_service,
-        template=email_template,
-        count=5,
-    )
-
-    results = fetch_delivered_notification_stats_by_month()
-
-    assert len(results) == 4
+class TestFetchDeliveredNotificationStatsbyMonth:
+    @freeze_time("2020-11-02 14:00")
+    def test_fetch_delivered_notification_stats_by_month(self, sample_service):
+        # Not counted: before GC Notify started (2019-10)
+        create_monthly_notification_stats_summary(
+            month="2019-10-01",
+            service=sample_service,
+            notification_type="email",
+            count=3,
+        )
 
-    assert results[0].keys() == ["month", "notification_type", "count"]
-    assert results[0].month.startswith("2020-03-01")
-    assert results[0].notification_type == "email"
-    assert results[0].count == 5
+        # December 2019 - email
+        create_monthly_notification_stats_summary(
+            month="2019-12-01",
+            service=sample_service,
+            notification_type="email",
+            count=3,
+        )
 
-    assert results[1].month.startswith("2020-01-01")
-    assert results[1].notification_type == "sms"
-    assert results[1].count == 4
+        # December 2019 - sms
+        create_monthly_notification_stats_summary(
+            month="2019-12-01",
+            service=sample_service,
+            notification_type="sms",
+            count=6,
+        )
 
-    assert results[2].month.startswith("2019-12-01")
-    assert results[2].notification_type == "email"
-    assert results[2].count == 3
+        # January 2020 - sms
+        create_monthly_notification_stats_summary(
+            month="2020-01-01",
+            service=sample_service,
+            notification_type="sms",
+            count=4,
+        )
 
-    assert results[3].month.startswith("2019-12-01")
-    assert results[3].notification_type == "sms"
-    assert results[3].count == 6
+        # March 2020 - email
+        create_monthly_notification_stats_summary(
+            month="2020-03-01",
+            service=sample_service,
+            notification_type="email",
+            count=5,
+        )
 
+        results = fetch_delivered_notification_stats_by_month()
 
-@freeze_time("2020-11-02 14:00")
-def test_fetch_delivered_notification_stats_by_month_filter_heartbeats(notify_api, sample_service):
-    sms_template = create_template(service=sample_service, template_type="sms", template_name="a")
-    email_template = create_template(service=sample_service, template_type="email", template_name="b")
+        assert len(results) == 4
 
-    # Not counted: before GC Notify started
-    create_ft_notification_status(
-        utc_date=date(2019, 10, 10),
-        service=sample_service,
-        template=email_template,
-        count=3,
-    )
+        assert results[0].keys() == ["month", "notification_type", "count"]
+        assert results[0].month.startswith("2020-03-01")
+        assert results[0].notification_type == "email"
+        assert results[0].count == 5
 
-    create_ft_notification_status(
-        utc_date=date(2019, 12, 10),
-        service=sample_service,
-        template=email_template,
-        count=3,
-    )
+        assert results[1].month.startswith("2020-01-01")
+        assert results[1].notification_type == "sms"
+        assert results[1].count == 4
+
+        assert results[2].month.startswith("2019-12-01")
+        assert results[2].notification_type == "email"
+        assert results[2].count == 3
+
+        assert results[3].month.startswith("2019-12-01")
+        assert results[3].notification_type == "sms"
+        assert results[3].count == 6
+
+    @freeze_time("2020-11-02 14:00")
+    def test_fetch_delivered_notification_stats_by_month_filter_heartbeats(self, notify_api, sample_service):
+        # Not counted: before GC Notify started (2019-10)
+        create_monthly_notification_stats_summary(
+            month="2019-10-01",
+            service=sample_service,
+            notification_type="email",
+            count=3,
+        )
 
-    create_ft_notification_status(
-        utc_date=date(2019, 12, 5),
-        service=sample_service,
-        template=sms_template,
-        notification_status=NOTIFICATION_DELIVERED,
-        count=6,
-    )
+        # December 2019 - email
+        create_monthly_notification_stats_summary(
+            month="2019-12-01",
+            service=sample_service,
+            notification_type="email",
+            count=3,
+        )
 
-    create_ft_notification_status(
-        utc_date=date(2020, 1, 1),
-        service=sample_service,
-        template=sms_template,
-        notification_status=NOTIFICATION_SENT,
-        count=4,
-    )
+        # December 2019 - sms
+        create_monthly_notification_stats_summary(
+            month="2019-12-01",
+            service=sample_service,
+            notification_type="sms",
+            count=6,
+        )
 
-    # Not counted: failed notifications
-    create_ft_notification_status(
-        utc_date=date(2020, 1, 1),
-        service=sample_service,
-        template=sms_template,
-        notification_status=NOTIFICATION_FAILED,
-        count=10,
-    )
+        # January 2020 - sms
+        create_monthly_notification_stats_summary(
+            month="2020-01-01",
+            service=sample_service,
+            notification_type="sms",
+            count=4,
+        )
 
-    create_ft_notification_status(
-        utc_date=date(2020, 3, 1),
-        service=sample_service,
-        template=email_template,
-        count=5,
-    )
-    with set_config(notify_api, "NOTIFY_SERVICE_ID", email_template.service_id):
-        results = fetch_delivered_notification_stats_by_month(filter_heartbeats=True)
-        assert len(results) == 0
+        # March 2020 - email
+        create_monthly_notification_stats_summary(
+            month="2020-03-01",
+            service=sample_service,
+            notification_type="email",
+            count=5,
+        )
 
+        # When filtering heartbeats and sample_service is configured as NOTIFY_SERVICE_ID,
+        # all its records should be filtered out
+        with set_config(notify_api, "NOTIFY_SERVICE_ID", sample_service.id):
+            results = fetch_delivered_notification_stats_by_month(filter_heartbeats=True)
+            assert len(results) == 0
 
-def test_fetch_delivered_notification_stats_by_month_empty():
-    assert fetch_delivered_notification_stats_by_month() == []
+    def test_fetch_delivered_notification_stats_by_month_empty(self):
+        assert fetch_delivered_notification_stats_by_month() == []
 
 
 def test_fetch_notification_stats_for_trial_services(sample_service):

tests/app/db.py

@@ -49,6 +49,7 @@
     Job,
     LetterBranding,
     LetterRate,
+    MonthlyNotificationStatsSummary,
     Notification,
     NotificationHistory,
     Organisation,
@@ -807,6 +808,42 @@ def create_ft_notification_status(
     return data
 
 
+def create_monthly_notification_stats_summary(
+    month,
+    service,
+    notification_type="sms",
+    count=1,
+):
+    """
+    Create a monthly notification stats summary record.
+    Month can be either:
+    - A date object (e.g., date(2019, 12, 1)) - will be converted to "YYYY-MM-01 00:00:00+00"
+    - A string (e.g., "2019-12-01") - will be converted to "YYYY-MM-01 00:00:00+00"
+    """
+    # Handle both date objects and strings
+    if isinstance(month, str):
+        # If it's already a string, ensure it has the right format
+        if " " not in month:
+            # String like "2019-12-01" -> "2019-12-01 00:00:00+00"
+            month_str = f"{month} 00:00:00+00"
+        else:
+            # Already has timestamp, use as-is
+            month_str = month
+    else:
+        # Convert date object to string format "YYYY-MM-01 00:00:00+00"
+        month_str = month.strftime("%Y-%m-01 00:00:00+00")
+
+    data = MonthlyNotificationStatsSummary(
+        month=month_str,
+        service_id=service.id,
+        notification_type=notification_type,
+        notification_count=count,
+    )
+    db.session.add(data)
+    db.session.commit()
+    return data
+
+
 def create_complaint(service=None, notification=None, created_at=None):
     if not service:
         service = create_service()