From ab08a33fcfc090d7cd740dd215fdda07763f642a Mon Sep 17 00:00:00 2001 From: Jay-Lalwani Date: Fri, 28 Feb 2025 14:39:04 -0500 Subject: [PATCH 1/7] prefetch related --- tcf_website/views/search.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/tcf_website/views/search.py b/tcf_website/views/search.py index 205b85038..57e0d8be2 100644 --- a/tcf_website/views/search.py +++ b/tcf_website/views/search.py @@ -14,18 +14,28 @@ def group_by_dept(courses): """Groups courses by their department and adds relevant data.""" + # Get all unique mnemonics from courses + mnemonics = set(course["mnemonic"] for course in courses) + + # Fetch all subdepartments in a single query + subdepts = { + subdept.mnemonic: subdept + for subdept in Subdepartment.objects.filter(mnemonic__in=mnemonics) + } + grouped_courses = {} for course in courses: course_dept = course["mnemonic"] if course_dept not in grouped_courses: - subdept = Subdepartment.objects.filter(mnemonic=course_dept).first() - # should only ever have one returned with that mnemonic - grouped_courses[course_dept] = { - "subdept_name": subdept.name, - "dept_id": subdept.department_id, - "courses": [], - } - grouped_courses[course_dept]["courses"].append(course) + subdept = subdepts.get(course_dept) + if subdept: + grouped_courses[course_dept] = { + "subdept_name": subdept.name, + "dept_id": subdept.department_id, + "courses": [], + } + if course_dept in grouped_courses: + grouped_courses[course_dept]["courses"].append(course) return grouped_courses @@ -155,10 +165,12 @@ def normalize_search_query(q: str) -> str: # Apply filters results = apply_filters(results, filters) - results = (results.filter(max_similarity__gte=similarity_threshold) - .filter(Q(number__isnull=True) | Q(number__regex=r"^\d{4}$")) - .exclude(semester_last_taught_id__lt=48) - .order_by("-max_similarity"))[:15] + results = ( + results.filter(max_similarity__gte=similarity_threshold) + .filter(Q(number__isnull=True) | Q(number__regex=r"^\d{4}$")) + .exclude(semester_last_taught_id__lt=48) + .order_by("-max_similarity") + )[:15] courses = [ { @@ -208,6 +220,7 @@ def filter_courses(filters): return courses + def apply_filters(results, filters): """Apply filters to course queryset.""" if filters.get("disciplines"): From d2562375af4ac69321a46f97f98bf94b27f3f050 Mon Sep 17 00:00:00 2001 From: Jay-Lalwani Date: Fri, 28 Feb 2025 15:48:21 -0500 Subject: [PATCH 2/7] code cleanup --- tcf_website/views/search.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tcf_website/views/search.py b/tcf_website/views/search.py index 57e0d8be2..502de811a 100644 --- a/tcf_website/views/search.py +++ b/tcf_website/views/search.py @@ -145,21 +145,15 @@ def normalize_search_query(q: str) -> str: Course.objects.select_related("subdepartment") .only("title", "number", "subdepartment__mnemonic", "description") .annotate( - mnemonic_similarity=TrigramSimilarity("combined_mnemonic_number", search_query), - title_similarity=TrigramSimilarity("title", search_query), - ) - # round results to two decimal places - .annotate( + mnemonic=F("subdepartment__mnemonic"), max_similarity=Round( Greatest( - F("mnemonic_similarity"), - F("title_similarity"), + TrigramSimilarity("combined_mnemonic_number", search_query), + TrigramSimilarity("title", search_query), ), 2, - ) + ), ) - # expose mnemonic to view - .annotate(mnemonic=F("subdepartment__mnemonic")) ) # Apply filters @@ -222,21 +216,27 @@ def filter_courses(filters): def apply_filters(results, filters): - """Apply filters to course queryset.""" + """Apply filters to course queryset using Q objects for more readable code.""" + filter_conditions = Q() + if filters.get("disciplines"): - results = results.filter(disciplines__name__in=filters.get("disciplines")) + filter_conditions &= Q(disciplines__name__in=filters.get("disciplines")) if filters.get("subdepartments"): - results = results.filter(subdepartment__mnemonic__in=filters.get("subdepartments")) + filter_conditions &= Q(subdepartment__mnemonic__in=filters.get("subdepartments")) if filters.get("instructors"): - results = results.filter(section__instructors__id__in=filters.get("instructors")) + filter_conditions &= Q(section__instructors__id__in=filters.get("instructors")) + + if filter_conditions: + results = results.filter(filter_conditions) + # Handle time filters weekdays = [day for day in filters.get("weekdays", []) if day] from_time = filters.get("from_time") to_time = filters.get("to_time") - if len(weekdays) != 5 and len(weekdays) != 0 or from_time or to_time: + if (len(weekdays) != 5 and len(weekdays) != 0) or from_time or to_time: time_filtered = Course.filter_by_time(days=weekdays, start_time=from_time, end_time=to_time) results = results.filter(id__in=time_filtered.values_list("id", flat=True)) From f4066d9549b6f2e74a22a6e955604fcb475a7d56 Mon Sep 17 00:00:00 2001 From: Jay-Lalwani Date: Fri, 28 Feb 2025 16:07:25 -0500 Subject: [PATCH 3/7] basic cache --- tcf_website/views/search.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tcf_website/views/search.py b/tcf_website/views/search.py index 502de811a..53f4873e2 100644 --- a/tcf_website/views/search.py +++ b/tcf_website/views/search.py @@ -8,6 +8,7 @@ from django.db.models import F, FloatField, Q from django.db.models.functions import Greatest, Round from django.shortcuts import render +from django.core.cache import cache from ..models import Course, Instructor, Subdepartment @@ -60,14 +61,24 @@ def search(request): # Save filters to session request.session["search_filters"] = filters - if query: + # Create a cache key from the query and filters + cache_key = f"search_{query}_{hash(frozenset(str(filters.items())))}" + cached_results = cache.get(cache_key) + + if cached_results: + courses, instructors, courses_first = cached_results + elif query: courses = fetch_courses(query, filters) instructors = fetch_instructors(query) courses_first = decide_order(courses, instructors) + # Cache the results for 10 minutes + cache.set(cache_key, (courses, instructors, courses_first), 600) else: courses = filter_courses(filters) instructors = [] courses_first = True + # Cache the results for 10 minutes + cache.set(cache_key, (courses, instructors, courses_first), 600) ctx = { "query": query[:30] + ("..." if len(query) > 30 else ""), From 66af2c44719db444af770fd1e1800b21d9724bac Mon Sep 17 00:00:00 2001 From: Jay-Lalwani Date: Sat, 15 Mar 2025 17:40:06 -0400 Subject: [PATCH 4/7] simplify caching to whole page --- tcf_website/views/search.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tcf_website/views/search.py b/tcf_website/views/search.py index 53f4873e2..728a216b5 100644 --- a/tcf_website/views/search.py +++ b/tcf_website/views/search.py @@ -9,6 +9,7 @@ from django.db.models.functions import Greatest, Round from django.shortcuts import render from django.core.cache import cache +from django.views.decorators.cache import cache_page from ..models import Course, Instructor, Subdepartment @@ -41,6 +42,7 @@ def group_by_dept(courses): return grouped_courses +@cache_page(60 * 15) # Cache for 15 minutes def search(request): """Search results view.""" @@ -61,24 +63,14 @@ def search(request): # Save filters to session request.session["search_filters"] = filters - # Create a cache key from the query and filters - cache_key = f"search_{query}_{hash(frozenset(str(filters.items())))}" - cached_results = cache.get(cache_key) - - if cached_results: - courses, instructors, courses_first = cached_results - elif query: + if query: courses = fetch_courses(query, filters) instructors = fetch_instructors(query) courses_first = decide_order(courses, instructors) - # Cache the results for 10 minutes - cache.set(cache_key, (courses, instructors, courses_first), 600) else: courses = filter_courses(filters) instructors = [] courses_first = True - # Cache the results for 10 minutes - cache.set(cache_key, (courses, instructors, courses_first), 600) ctx = { "query": query[:30] + ("..." if len(query) > 30 else ""), From 3798ed3765f3fbdc2e2ab3a9b454386873a346f8 Mon Sep 17 00:00:00 2001 From: Jay-Lalwani Date: Sat, 15 Mar 2025 17:54:10 -0400 Subject: [PATCH 5/7] control flow cleanup --- tcf_website/views/search.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tcf_website/views/search.py b/tcf_website/views/search.py index 728a216b5..1d690b7d2 100644 --- a/tcf_website/views/search.py +++ b/tcf_website/views/search.py @@ -28,16 +28,21 @@ def group_by_dept(courses): grouped_courses = {} for course in courses: course_dept = course["mnemonic"] + + # Skip courses without valid subdepartment + if course_dept not in subdepts: + continue + + subdept = subdepts[course_dept] + if course_dept not in grouped_courses: - subdept = subdepts.get(course_dept) - if subdept: - grouped_courses[course_dept] = { - "subdept_name": subdept.name, - "dept_id": subdept.department_id, - "courses": [], - } - if course_dept in grouped_courses: - grouped_courses[course_dept]["courses"].append(course) + grouped_courses[course_dept] = { + "subdept_name": subdept.name, + "dept_id": subdept.department_id, + "courses": [], + } + + grouped_courses[course_dept]["courses"].append(course) return grouped_courses From 115ff17a50cc31553e94f0cf98c172feb237539f Mon Sep 17 00:00:00 2001 From: Jay-Lalwani Date: Sat, 15 Mar 2025 17:59:21 -0400 Subject: [PATCH 6/7] simplify filtering conditions --- tcf_website/views/search.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tcf_website/views/search.py b/tcf_website/views/search.py index 1d690b7d2..c709ced8a 100644 --- a/tcf_website/views/search.py +++ b/tcf_website/views/search.py @@ -8,7 +8,6 @@ from django.db.models import F, FloatField, Q from django.db.models.functions import Greatest, Round from django.shortcuts import render -from django.core.cache import cache from django.views.decorators.cache import cache_page from ..models import Course, Instructor, Subdepartment @@ -244,7 +243,7 @@ def apply_filters(results, filters): from_time = filters.get("from_time") to_time = filters.get("to_time") - if (len(weekdays) != 5 and len(weekdays) != 0) or from_time or to_time: + if weekdays or from_time or to_time: time_filtered = Course.filter_by_time(days=weekdays, start_time=from_time, end_time=to_time) results = results.filter(id__in=time_filtered.values_list("id", flat=True)) From 65e576f1f260f9c289e676370058c39fbd9b7ae5 Mon Sep 17 00:00:00 2001 From: Jay-Lalwani Date: Sat, 15 Mar 2025 18:09:01 -0400 Subject: [PATCH 7/7] revert trigram similarity --- tcf_website/views/search.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tcf_website/views/search.py b/tcf_website/views/search.py index c709ced8a..2ad9dfeed 100644 --- a/tcf_website/views/search.py +++ b/tcf_website/views/search.py @@ -152,15 +152,21 @@ def normalize_search_query(q: str) -> str: Course.objects.select_related("subdepartment") .only("title", "number", "subdepartment__mnemonic", "description") .annotate( - mnemonic=F("subdepartment__mnemonic"), + mnemonic_similarity=TrigramSimilarity("combined_mnemonic_number", search_query), + title_similarity=TrigramSimilarity("title", search_query), + ) + # round results to two decimal places + .annotate( max_similarity=Round( Greatest( - TrigramSimilarity("combined_mnemonic_number", search_query), - TrigramSimilarity("title", search_query), + F("mnemonic_similarity"), + F("title_similarity"), ), 2, - ), + ) ) + # expose mnemonic to view + .annotate(mnemonic=F("subdepartment__mnemonic")) ) # Apply filters