diff --git a/app/controllers/health_controller.py b/app/controllers/health_controller.py index a009525..423bd87 100644 --- a/app/controllers/health_controller.py +++ b/app/controllers/health_controller.py @@ -1,6 +1,5 @@ from flask import Blueprint, jsonify import logging -#from ..config.db_connection import DBConnection from ..services.fetch_data_bigQuery import BigQueryService # Initialize logger diff --git a/app/controllers/report_controller.py b/app/controllers/report_controller.py index 76dc772..53a5a3e 100644 --- a/app/controllers/report_controller.py +++ b/app/controllers/report_controller.py @@ -6,12 +6,9 @@ import ctypes import time as time_module from app.authentication.AccessTokenValidator import AccessTokenValidator -from constants import X_AUTHENTICATED_USER_TOKEN, IS_VALIDATION_ENABLED, X_ORG_ID, APAR_FILTER_KEY +from constants import X_AUTHENTICATED_USER_TOKEN, IS_VALIDATION_ENABLED, X_ORG_ID, APAR_FILTER_KEY, TEXT_CSV_HOLDER, LIBC_SO_6, MALLOC_TRIM_HOLDER_MSG +from errormsg import MISSING_X_ORG_ID_HEADER, ORGANIZATION_ID_REQUIRED_ERROR, INVALID_DATE_FORMAT_ERROR, UNEXPECTED_ERROR_OCCURRED, MALLOC_TRIM_HOLDER_ERROR_MSG from app.services.GcsToBigQuerySyncService import GcsToBigQuerySyncService -import io -import uuid -import random -from datetime import timedelta # Configure logger logging.basicConfig(level=logging.INFO) @@ -25,211 +22,71 @@ def get_report(org_id): try: logger.info(f"Received request to generate report for org_id={org_id}") x_org_id = request.headers.get(X_ORG_ID) - logger.info(f"Received x_org_id={x_org_id}") - if not x_org_id: - logger.error("Missing 'x_org_id' in headers.") - return jsonify({'error': 'Organization ID is required.'}), 400 - if not ReportService.isValidOrg(x_org_id, org_id): - logger.error(f"Invalid organization ID: {org_id}") - return jsonify({'error': f'Not authorized to view the report for : {org_id}'}), 401 - if IS_VALIDATION_ENABLED.lower() == 'true': - # Extract and validate user token - user_token = request.headers.get(X_AUTHENTICATED_USER_TOKEN) - if not user_token: - logger.error("Missing 'x-authenticated-user-token' in headers.") - return jsonify({'error': 'Authentication token is required.'}), 401 - - user_org_id = AccessTokenValidator.verify_user_token_get_org(user_token, True) - if not user_org_id: - logger.error("Invalid or expired authentication token.") - return jsonify({'error': 'Invalid or expired authentication token.'}), 401 - - logger.info(f"Authenticated user with user_org_id={user_org_id}") - if user_org_id != org_id: - logger.error(f"User does not have access to organization ID {org_id}.") - return jsonify({'error': f'Access denied for the specified organization ID {org_id}.'}), 403 - - # Parse and validate date range - data = request.get_json() - if not data or 'start_date' not in data or 'end_date' not in data: - raise KeyError("Missing 'start_date' or 'end_date' in request body.") + if not validate_org_id(x_org_id, org_id): + return jsonify({'error': ORGANIZATION_ID_REQUIRED_ERROR}), 400 - start_date = datetime.strptime(data['start_date'], '%Y-%m-%d') - end_date = datetime.strptime(data['end_date'], '%Y-%m-%d') + if IS_VALIDATION_ENABLED.lower() == 'true' and not validate_user_token(org_id): + return jsonify({'error': 'Authentication failed.'}), 401 - start_date = datetime.combine(start_date.date(), time.min) # 00:00:00 - end_date = datetime.combine(end_date.date(), time.max) # 23:59:59.999999 + data = request.get_json() + start_date, end_date = parse_and_validate_dates(data) - # New parameters from request body - is_full_report_required = data.get('isFullReportRequired', False) - required_columns = data.get('required_columns', []) - - logger.info(f"Generating report for org_id={org_id} from {start_date} to {end_date}") - #Validate date range if (end_date - start_date).days > 365: logger.warning(f"Date range exceeds 1 year: start_date={start_date}, end_date={end_date}") return jsonify({'error': 'Date range cannot exceed 1 year'}), 400 - try: - csv_data = ReportService.fetch_master_enrolments_data( - start_date, end_date, org_id, is_full_report_required, - required_columns=required_columns - ) - - if not csv_data: - logger.warning(f"No data found for org_id={org_id} within given date range.") - return jsonify({'error': 'No data found for the given organization ID.'}), 404 - - except Exception as e: - error_message = str(e) - logger.error(f"Error generating CSV stream for org_id={org_id}: {error_message}") - return jsonify({'error': 'Failed to generate the report due to an error.', 'details': error_message}), 500 - - time_taken = round(time_module.time() - start_timer, 2) - logger.info(f"Report generated successfully for org_id={org_id} in {time_taken} seconds") - - response = Response( - stream_with_context(csv_data), - mimetype="text/csv", - headers={ - "Content-Disposition": f'attachment; filename="report_{org_id}.csv"' - } - ) + csv_data = generate_report(org_id, start_date, end_date, data) + if not csv_data: + return jsonify({'error': 'No data found for the given organization ID.'}), 404 - # Explicitly trigger garbage collection to free up memory - del csv_data - gc.collect() - - return response + return create_csv_response(csv_data, org_id, start_timer) except KeyError as e: - error_message = str(e) - logger.error(f"Missing required fields in request: {error_message}") - return jsonify({'error': 'Invalid input. Please provide start_date and end_date.', 'details': error_message}), 400 + return handle_key_error(e) except ValueError as e: - error_message = str(e) - logger.error(f"Invalid date format in request: {error_message}") - return jsonify({'error': 'Invalid date format. Use YYYY-MM-DD.', 'details': error_message}), 400 + return handle_value_error(e) except FileNotFoundError as e: - error_message = str(e) - logger.error(f"File not found during report generation: {error_message}") - return jsonify({'error': 'Report file could not be generated.', 'details': error_message}), 500 + return handle_file_not_found_error(e) except Exception as e: - error_message = str(e) - logger.exception(f"Unexpected error occurred: {error_message}") - return jsonify({'error': 'An unexpected error occurred. Please try again later.', 'details': error_message}), 500 - finally: - gc.collect() - try: - logger.info("inside malloc_trim:") - ctypes.CDLL("libc.so.6").malloc_trim(0) - except Exception as e: - logger.exception("malloc_trim failed: %s", str(e)) + return handle_generic_error(e) + + finally: + cleanup_memory() -@report_controller.route('/report/user/sync/', methods=['POST']) -def get_user_report(orgId): +@report_controller.route('/report/user/sync/', methods=['POST']) +def get_user_report(org_id): try: start_timer = time_module.time() logger.info("Received request to generate user report") - x_org_id = request.headers.get(X_ORG_ID) - logger.info(f"Received x_org_id={x_org_id}") - if not x_org_id: - logger.error("Missing 'x_org_id' in headers.") - return jsonify({'error': 'Organization ID is required.'}), 400 - if not ReportService.isValidOrg(x_org_id, orgId): - logger.error(f"Invalid organization ID: {orgId}") - return jsonify({'error': f'Not authorized to view the report for : {orgId}'}), 401 - # Parse and validate input parameters - data = request.get_json() - if not data: - logger.error("Request body is missing") - return jsonify({'error': 'Request body is missing.'}), 400 - - user_email = data.get('userEmail') - user_phone = data.get('userPhone') - ehrms_id = data.get('ehrmsId') - - # Trim whitespace if present - user_email = user_email.strip() if user_email else None - user_phone = user_phone.strip() if user_phone else None - ehrms_id = ehrms_id.strip() if ehrms_id else None - - if not (user_email or user_phone or ehrms_id): - logger.error("At least one of 'userEmail', 'userPhone', or 'ehrmsId' must be provided.") - return jsonify({'error': "At least one of 'userEmail', 'userPhone', or 'ehrmsId' must be provided."}), 400 - - # New date filter and orgId parameter - start_date = data.get('start_date') - end_date = data.get('end_date') - - # Validate date range if provided - if start_date and end_date: - try: - start_date = datetime.strptime(start_date, '%Y-%m-%d') - end_date = datetime.strptime(end_date, '%Y-%m-%d') - start_date = datetime.combine(start_date.date(), time.min) # 00:00:00 - end_date = datetime.combine(end_date.date(), time.max) # 23:59:59.999999 - except ValueError: - return jsonify({'error': 'Invalid date format. Use YYYY-MM-DD.'}), 400 - - required_columns = data.get('required_columns', []) - - logger.info(f"Generating user report for userEmail={user_email}, userPhone={user_phone}, ehrmsId={ehrms_id}") - try: - csv_data = ReportService.fetch_user_cumulative_report( - user_email, user_phone, ehrms_id, start_date, end_date, orgId, - required_columns - ) + # Validate headers and organization ID + validation_response = validate_headers_and_org(x_org_id=request.headers.get(X_ORG_ID), org_id=org_id) + if validation_response: + return validation_response - if not csv_data: - logger.warning(f"No data found for userEmail={user_email}, userPhone={user_phone}, ehrmsId={ehrms_id}") - return jsonify({'error': 'No data found for the given user details.'}), 404 - - except Exception as e: - error_message = str(e) - logger.error(f"Error generating CSV stream for userEmail={user_email}, userPhone={user_phone}, ehrmsId={ehrms_id}: {error_message}") - return jsonify({'error': 'Failed to generate the report due to an internal error.', 'details': error_message}), 500 - - time_taken = round(time_module.time() - start_timer, 2) - logger.info(f"Report generated successfully for userEmail={user_email}, userPhone={user_phone}, ehrmsId={ehrms_id} in {time_taken} seconds") - - response = Response( - stream_with_context(csv_data), - mimetype="text/csv", - headers={ - "Content-Disposition": f'attachment; filename="user-report.csv"' - } - ) - - # Explicitly trigger garbage collection to free up memory - del csv_data - gc.collect() + # Parse and validate input parameters + data, validation_response = parse_and_validate_user_report_input(request.get_json()) + if validation_response: + return validation_response + # Generate the report + response = generate_user_report(data, org_id, start_timer) return response except KeyError as e: - error_message = str(e) - logger.error(f"Missing required fields in request: {error_message}") - return jsonify({'error': 'Invalid input. Please provide valid parameters.', 'details': error_message}), 400 + return handle_key_error(e) except Exception as e: - error_message = str(e) - logger.exception(f"Unexpected error occurred: {error_message}") - return jsonify({'error': 'An unexpected error occurred. Please try again later.', 'details': error_message}), 500 - finally: - gc.collect() - try: - logger.info("inside malloc_trim:") - ctypes.CDLL("libc.so.6").malloc_trim(0) - except Exception as e: - logger.exception("malloc_trim failed: %s", str(e)) + return handle_generic_error(e) + + finally: + cleanup_memory() -@report_controller.route('/report/org/user/', methods=['POST']) -def get_org_user_report(orgId): +@report_controller.route('/report/org/user/', methods=['POST']) +def get_org_user_report(org_id): try: start_timer = time_module.time() logger.info("Received request to generate user report") @@ -238,9 +95,9 @@ def get_org_user_report(orgId): if not x_org_id: logger.error("Missing 'x_org_id' in headers.") return jsonify({'error': 'Organization ID is required.'}), 400 - if not ReportService.isValidOrg(x_org_id, orgId): - logger.error(f"Invalid organization ID: {orgId}") - return jsonify({'error': f'Not authorized to view the report for : {orgId}'}), 401 + if not ReportService.is_valid_org(x_org_id, org_id): + logger.error(f"Invalid organization ID: {org_id}") + return jsonify({'error': f'Not authorized to view the report for : {org_id}'}), 401 # Parse and validate input parameters data = request.get_json() if not data: @@ -265,15 +122,15 @@ def get_org_user_report(orgId): required_columns = data.get('required_columns', []) - logger.info(f"Generating user report for orgId={orgId}") + logger.info(f"Generating user report for orgId={org_id}") try: csv_data = ReportService.fetch_master_user_data( - orgId, is_full_report_required, required_columns=required_columns, user_creation_start_date=user_creation_start_date, user_creation_end_date=user_creation_end_date + org_id, is_full_report_required, required_columns=required_columns, user_creation_start_date=user_creation_start_date, user_creation_end_date=user_creation_end_date ) if not csv_data: - logger.warning(f"No data found for orgId={orgId}") + logger.warning(f"No data found for orgId={org_id}") return jsonify({'error': 'No data found for the given org details.'}), 404 except Exception as e: @@ -282,13 +139,13 @@ def get_org_user_report(orgId): return jsonify({'error': 'Failed to generate the report due to an internal error.', 'details': error_message}), 500 time_taken = round(time_module.time() - start_timer, 2) - logger.info(f"Org User Report generated successfully for in {time_taken} seconds for orgId={orgId}") + logger.info(f"Org User Report generated successfully for in {time_taken} seconds for orgId={org_id}") response = Response( stream_with_context(csv_data), mimetype="text/csv", headers={ - "Content-Disposition": f'attachment; filename="user-report.csv"' + "Content-Disposition": 'attachment; filename="user-report.csv"' } ) @@ -330,86 +187,234 @@ def get_apar_report(): try: logger.info("Received request to generate APAR report") data = request.get_json() - # Parse and validate new request fields - if not data or 'enrolment_start_date' not in data or 'enrolment_end_date' not in data: - raise KeyError("Missing 'enrolment_start_date' or 'enrolment_end_date' in request body.") - enrolment_start_date = data['enrolment_start_date'] - enrolment_end_date = data['enrolment_end_date'] - filters = data.get('filters', {}) - required_columns = data.get('required_columns', []) + # Parse and validate request fields + enrolment_start_date, enrolment_end_date, filters, required_columns = parse_apar_request(data) - # Validate filters keys if filters present - if filters: - allowed_keys = APAR_FILTER_KEY.split(',') - for key in filters: - if key not in allowed_keys: - return jsonify({'error': f"Invalid filter key '{key}'. Allowed keys: {', '.join(allowed_keys)}"}), 400 - # At least one of the allowed keys must be present and non-empty - if not any(filters.get(k) for k in allowed_keys): - return jsonify({'error': f"At least one of {', '.join(allowed_keys)} must be provided in filters."}), 400 - - # Validate date range - start_date = datetime.strptime(enrolment_start_date, '%Y-%m-%d') - end_date = datetime.strptime(enrolment_end_date, '%Y-%m-%d') - if (end_date - start_date).days > 365: - logger.warning(f"Date range exceeds 1 year: start_date={start_date}, end_date={end_date}") - return jsonify({'error': 'Date range cannot exceed 1 year'}), 400 - - logger.info(f"Generating APAR report from {start_date} to {end_date} with filters: {filters}") + # Validate filters and date range + validate_apar_filters(filters) + validate_apar_date_range(enrolment_start_date, enrolment_end_date) + + # Generate the report + return generate_apar_report(enrolment_start_date, enrolment_end_date, filters, required_columns, start_timer) + + except KeyError as e: + return handle_key_error(e) + + except ValueError as e: + return handle_value_error(e) + + except FileNotFoundError as e: + return handle_file_not_found_error(e) + + except Exception as e: + return handle_generic_error(e) + + finally: + cleanup_memory() + + +def parse_apar_request(data): + if not data or 'enrolment_start_date' not in data or 'enrolment_end_date' not in data: + raise KeyError("Missing 'enrolment_start_date' or 'enrolment_end_date' in request body.") + enrolment_start_date = data['enrolment_start_date'] + enrolment_end_date = data['enrolment_end_date'] + filters = data.get('filters', {}) + required_columns = data.get('required_columns', []) + return enrolment_start_date, enrolment_end_date, filters, required_columns + + +def validate_apar_filters(filters): + if filters: + allowed_keys = APAR_FILTER_KEY.split(',') + for key in filters: + if key not in allowed_keys: + raise ValueError(f"Invalid filter key '{key}'. Allowed keys: {', '.join(allowed_keys)}") + if not any(filters.get(k) for k in allowed_keys): + raise ValueError(f"At least one of {', '.join(allowed_keys)} must be provided in filters.") + + +def validate_apar_date_range(enrolment_start_date, enrolment_end_date): + start_date = datetime.strptime(enrolment_start_date, '%Y-%m-%d') + end_date = datetime.strptime(enrolment_end_date, '%Y-%m-%d') + if (end_date - start_date).days > 365: + raise ValueError("Date range cannot exceed 1 year") + + +def generate_apar_report(enrolment_start_date, enrolment_end_date, filters, required_columns, start_timer): + logger.info(f"Generating APAR report from {enrolment_start_date} to {enrolment_end_date} with filters: {filters}") + csv_data = ReportService.fetch_apar_enrolment_report( + enrolment_start_date, enrolment_end_date, filters, required_columns + ) + if not csv_data: + logger.warning(f"No data found for the given date range: {enrolment_start_date} to {enrolment_end_date}") + return jsonify({'error': 'No data found for the given filters/date range.'}), 404 + + time_taken = round(time_module.time() - start_timer, 2) + logger.info(f"APAR Report generated successfully in {time_taken} seconds") + + response = Response( + stream_with_context(csv_data), + mimetype="text/csv", + headers={"Content-Disposition": 'attachment; filename="report.csv"'} + ) + del csv_data + gc.collect() + return response + + +def validate_org_id(x_org_id, org_id): + logger.info(f"Received x_org_id={x_org_id}") + if not x_org_id: + logger.error(MISSING_X_ORG_ID_HEADER) + return False + if not ReportService.is_valid_org(x_org_id, org_id): + logger.error(f"Invalid organization ID: {org_id}") + return False + return True + + +def validate_user_token(org_id): + user_token = request.headers.get(X_AUTHENTICATED_USER_TOKEN) + if not user_token: + logger.error("Missing 'x-authenticated-user-token' in headers.") + return False + + user_org_id = AccessTokenValidator.verify_user_token_get_org(user_token, True) + if not user_org_id or user_org_id != org_id: + logger.error("Invalid or expired authentication token.") + return False + return True + + +def parse_and_validate_dates(data): + if not data or 'start_date' not in data or 'end_date' not in data: + raise KeyError("Missing 'start_date' or 'end_date' in request body.") + start_date = datetime.strptime(data['start_date'], '%Y-%m-%d') + end_date = datetime.strptime(data['end_date'], '%Y-%m-%d') + return datetime.combine(start_date.date(), time.min), datetime.combine(end_date.date(), time.max) + + +def generate_report(org_id, start_date, end_date, data): + is_full_report_required = data.get('isFullReportRequired', False) + required_columns = data.get('required_columns', []) + logger.info(f"Generating report for org_id={org_id} from {start_date} to {end_date}") + return ReportService.fetch_master_enrolments_data( + start_date, end_date, org_id, is_full_report_required, required_columns=required_columns + ) + + +def create_csv_response(csv_data, org_id, start_timer): + time_taken = round(time_module.time() - start_timer, 2) + logger.info(f"Report generated successfully for org_id={org_id} in {time_taken} seconds") + response = Response( + stream_with_context(csv_data), + mimetype=TEXT_CSV_HOLDER, + headers={"Content-Disposition": f'attachment; filename="report_{org_id}.csv"'} + ) + del csv_data + gc.collect() + return response + + +def handle_key_error(e): + error_message = str(e) + logger.error(f"Missing required fields in request: {error_message}") + return jsonify({'error': 'Invalid input. Please provide start_date and end_date.', 'details': error_message}), 400 + + +def handle_value_error(e): + error_message = str(e) + logger.error(f"Invalid date format in request: {error_message}") + return jsonify({'error': INVALID_DATE_FORMAT_ERROR, 'details': error_message}), 400 + + +def handle_file_not_found_error(e): + error_message = str(e) + logger.error(f"File not found during report generation: {error_message}") + return jsonify({'error': 'Report file could not be generated.', 'details': error_message}), 500 + + +def handle_generic_error(e): + error_message = str(e) + logger.exception(f"Unexpected error occurred: {error_message}") + return jsonify({'error': UNEXPECTED_ERROR_OCCURRED, 'details': error_message}), 500 + + +def cleanup_memory(): + gc.collect() + try: + logger.info(MALLOC_TRIM_HOLDER_MSG) + ctypes.CDLL(LIBC_SO_6).malloc_trim(0) + except Exception as e: + logger.exception(MALLOC_TRIM_HOLDER_ERROR_MSG, str(e)) + +def validate_headers_and_org(x_org_id, org_id): + logger.info(f"Received x_org_id={x_org_id}") + if not x_org_id: + logger.error("Missing 'x_org_id' in headers.") + return jsonify({'error': 'Organization ID is required.'}), 400 + if not ReportService.is_valid_org(x_org_id, org_id): + logger.error(f"Invalid organization ID: {org_id}") + return jsonify({'error': f'Not authorized to view the report for : {org_id}'}), 401 + return None + + +def parse_and_validate_user_report_input(data): + if not data: + logger.error("Request body is missing") + return None, jsonify({'error': 'Request body is missing.'}), 400 + + user_email = data.get('userEmail', '').strip() + user_phone = data.get('userPhone', '').strip() + ehrms_id = data.get('ehrmsId', '').strip() + + if not (user_email or user_phone or ehrms_id): + logger.error("At least one of 'userEmail', 'userPhone', or 'ehrmsId' must be provided.") + return None, jsonify({'error': "At least one of 'userEmail', 'userPhone', or 'ehrmsId' must be provided."}), 400 + + start_date, end_date = parse_date_range(data.get('start_date'), data.get('end_date')) + if isinstance(start_date, Response): + return None, start_date + + required_columns = data.get('required_columns', []) + return {'user_email': user_email, 'user_phone': user_phone, 'ehrms_id': ehrms_id, 'start_date': start_date, 'end_date': end_date, 'required_columns': required_columns}, None + + +def parse_date_range(start_date, end_date): + if start_date and end_date: try: - # Call the service layer to fetch/process data from BQ - csv_data = ReportService.fetch_apar_enrolment_report( - enrolment_start_date, enrolment_end_date, filters, required_columns - ) + start_date = datetime.combine(datetime.strptime(start_date, '%Y-%m-%d').date(), time.min) + end_date = datetime.combine(datetime.strptime(end_date, '%Y-%m-%d').date(), time.max) + except ValueError: + return jsonify({'error': INVALID_DATE_FORMAT_ERROR}), 400 + return start_date, end_date - if not csv_data: - logger.warning(f"No data found for the given date range: {enrolment_start_date} to {enrolment_end_date}") - return jsonify({'error': 'No data found for the given filters/date range.'}), 404 - except Exception as e: - error_message = str(e) - logger.error(f"Error generating CSV stream for APAR report: {error_message}") - return jsonify({'error': 'Failed to generate the report due to an error.', 'details': error_message}), 500 +def generate_user_report(data, org_id, start_timer): + try: + csv_data = ReportService.fetch_user_cumulative_report( + data['user_email'], data['user_phone'], data['ehrms_id'], data['start_date'], data['end_date'], org_id, + data['required_columns'] + ) + + if not csv_data: + logger.warning(f"No data found for userEmail={data['user_email']}, userPhone={data['user_phone']}, ehrmsId={data['ehrms_id']}") + return jsonify({'error': 'No data found for the given user details.'}), 404 time_taken = round(time_module.time() - start_timer, 2) - logger.info(f"APAR Report generated successfully in {time_taken} seconds") + logger.info(f"Report generated successfully in {time_taken} seconds") response = Response( stream_with_context(csv_data), - mimetype="text/csv", - headers={ - "Content-Disposition": f'attachment; filename="report.csv"' - } + mimetype=TEXT_CSV_HOLDER, + headers={"Content-Disposition": 'attachment; filename="user-report.csv"'} ) - del csv_data gc.collect() return response - except KeyError as e: - error_message = str(e) - logger.error(f"Missing required fields in request: {error_message}") - return jsonify({'error': 'Invalid input. Please provide enrolment_start_date and enrolment_end_date.', 'details': error_message}), 400 - - except ValueError as e: - error_message = str(e) - logger.error(f"Invalid date format in request: {error_message}") - return jsonify({'error': 'Invalid date format. Use YYYY-MM-DD.', 'details': error_message}), 400 - - except FileNotFoundError as e: - error_message = str(e) - logger.error(f"File not found during report generation: {error_message}") - return jsonify({'error': 'Report file could not be generated.', 'details': error_message}), 500 - except Exception as e: error_message = str(e) - logger.exception(f"Unexpected error occurred: {error_message}") - return jsonify({'error': 'An unexpected error occurred. Please try again later.', 'details': error_message}), 500 - finally: - gc.collect() - try: - logger.info("inside malloc_trim:") - ctypes.CDLL("libc.so.6").malloc_trim(0) - except Exception as e: - logger.exception("malloc_trim failed: %s", str(e)) + logger.error(f"Error generating CSV stream: {error_message}") + return jsonify({'error': 'Failed to generate the report due to an internal error.', 'details': error_message}), 500 diff --git a/app/controllers/report_controller_v2.py b/app/controllers/report_controller_v2.py index a153fc0..e5f69a6 100644 --- a/app/controllers/report_controller_v2.py +++ b/app/controllers/report_controller_v2.py @@ -7,7 +7,8 @@ import ctypes import time as time_module from app.authentication.AccessTokenValidator import AccessTokenValidator -from constants import X_AUTHENTICATED_USER_TOKEN, IS_VALIDATION_ENABLED, X_ORG_ID +from constants import X_AUTHENTICATED_USER_TOKEN, IS_VALIDATION_ENABLED, X_ORG_ID, TEXT_CSV_HOLDER, MALLOC_TRIM_HOLDER_MSG, LIBC_SO_6 +from errormsg import REQUEST_BODY_MISSING_ERROR, UNEXPECTED_ERROR_OCCURRED, MALLOC_TRIM_HOLDER_ERROR_MSG # Configure logger logging.basicConfig( @@ -27,7 +28,7 @@ def _validate_request_common(org_id): logger.error("Missing 'x_org_id' in headers.") return {'error': 'Organization ID is required.'}, 400 - if not ReportService.isValidOrg(x_org_id, org_id): + if not ReportService.is_valid_org(x_org_id, org_id): logger.error(f"Invalid organization ID: {org_id}") return {'error': f'Not authorized to view the report for : {org_id}'}, 401 @@ -83,9 +84,9 @@ def get_report(org_id): # Parse request data data = request.get_json() if not data: - logger.error("Request body is missing") - return jsonify({'error': 'Request body is missing'}), 400 - + logger.error(REQUEST_BODY_MISSING_ERROR) + return jsonify({'error': REQUEST_BODY_MISSING_ERROR}), 400 + # Parse and validate date range try: start_date, end_date, error = _parse_date_range(data) @@ -135,7 +136,7 @@ def get_report(org_id): response = Response( stream_with_context(csv_data), - mimetype="text/csv", + mimetype=TEXT_CSV_HOLDER, headers={ "Content-Disposition": f'attachment; filename="report_v2_{org_id}.csv"' } @@ -155,67 +156,40 @@ def get_report(org_id): except Exception as e: error_message = str(e) logger.exception(f"Unexpected error occurred: {error_message}") - return jsonify({'error': 'An unexpected error occurred. Please try again later.', 'details': error_message}), 500 + return jsonify({'error': UNEXPECTED_ERROR_OCCURRED, 'details': error_message}), 500 finally: gc.collect() try: - logger.info("inside malloc_trim:") - ctypes.CDLL("libc.so.6").malloc_trim(0) + logger.info(MALLOC_TRIM_HOLDER_MSG) + ctypes.CDLL(LIBC_SO_6).malloc_trim(0) except Exception as e: - logger.exception("malloc_trim failed: %s", str(e)) + logger.exception(MALLOC_TRIM_HOLDER_ERROR_MSG, str(e)) -@report_controller_v2.route('/report/v2/user/sync/', methods=['POST']) -def get_user_report(orgId): +@report_controller_v2.route('/report/v2/user/sync/', methods=['POST']) +def get_user_report(org_id): """V2 endpoint for user report with advanced filtering""" start_timer = time_module.time() try: - logger.info(f"Received request to generate v2 user report for orgId={orgId}") + logger.info(f"Received request to generate v2 user report for org_id={org_id}") # Validate request - validation_result = _validate_request_common(orgId) + validation_result = _validate_request_common(org_id) if validation_result: return jsonify(validation_result[0]), validation_result[1] # Parse and validate input parameters - try: - data = request.get_json() - if not data: - logger.error("Request body is missing") - return jsonify({'error': 'Request body is missing'}), 400 - except Exception as e: - logger.error(f"Request body is missing: {str(e)}") - return jsonify({'error': 'Request body is missing'}), 400 - - user_email = data.get('userEmail') - user_phone = data.get('userPhone') - ehrms_id = data.get('ehrmsId') - - # Trim whitespace if present - user_email = user_email.strip() if user_email else None - user_phone = user_phone.strip() if user_phone else None - ehrms_id = ehrms_id.strip() if ehrms_id else None + data = _get_request_data() + if isinstance(data, Response): + return data + user_email, user_phone, ehrms_id = _extract_user_identifiers(data) if not (user_email or user_phone or ehrms_id): logger.error("At least one of 'userEmail', 'userPhone', or 'ehrmsId' must be provided.") return jsonify({'error': "At least one of 'userEmail', 'userPhone', or 'ehrmsId' must be provided."}), 400 - # New date filter and orgId parameter - start_date = data.get('start_date') - end_date = data.get('end_date') - - # Validate date range if provided - if start_date and end_date: - try: - start_date = datetime.strptime(start_date, '%Y-%m-%d') - end_date = datetime.strptime(end_date, '%Y-%m-%d') - start_date = datetime.combine(start_date.date(), time.min) # 00:00:00 - end_date = datetime.combine(end_date.date(), time.max) # 23:59:59.999999 - except ValueError: - return jsonify({'error': 'Invalid date format. Use YYYY-MM-DD.'}), 400 + start_date, end_date = _validate_date_range(data) required_columns = data.get('required_columns', []) - - # Get additional filters additional_filters = data.get('additionalFilter', {}) logger.info(f"Generating v2 user report for userEmail={user_email}, userPhone={user_phone}, ehrmsId={ehrms_id}") @@ -228,7 +202,7 @@ def get_user_report(orgId): ehrms_id=ehrms_id, start_date=start_date, end_date=end_date, - orgId=orgId, + org_id=org_id, required_columns=required_columns, additional_filters=additional_filters ) @@ -249,7 +223,7 @@ def get_user_report(orgId): stream_with_context(csv_data), mimetype="text/csv", headers={ - "Content-Disposition": f'attachment; filename="user-report-v2.csv"' + "Content-Disposition": 'attachment; filename="user-report-v2.csv"' } ) @@ -276,15 +250,57 @@ def get_user_report(orgId): except Exception as e: logger.exception("malloc_trim failed: %s", str(e)) -@report_controller_v2.route('/report/v2/org/user/', methods=['POST']) -def get_org_user_report(orgId): + +def _get_request_data(): + """Extract and validate request data.""" + try: + data = request.get_json() + if not data: + logger.error(REQUEST_BODY_MISSING_ERROR) + return jsonify({'error': REQUEST_BODY_MISSING_ERROR}), 400 + return data + except Exception as e: + logger.error(f"Request body is missing: {str(e)}") + return jsonify({'error': REQUEST_BODY_MISSING_ERROR}), 400 + + +def _extract_user_identifiers(data): + """Extract and trim user identifiers.""" + user_email = data.get('userEmail') + user_phone = data.get('userPhone') + ehrms_id = data.get('ehrmsId') + + user_email = user_email.strip() if user_email else None + user_phone = user_phone.strip() if user_phone else None + ehrms_id = ehrms_id.strip() if ehrms_id else None + + return user_email, user_phone, ehrms_id + + +def _validate_date_range(data): + """Validate and parse date range.""" + start_date = data.get('start_date') + end_date = data.get('end_date') + + if start_date and end_date: + try: + start_date = datetime.strptime(start_date, '%Y-%m-%d') + end_date = datetime.strptime(end_date, '%Y-%m-%d') + start_date = datetime.combine(start_date.date(), time.min) # 00:00:00 + end_date = datetime.combine(end_date.date(), time.max) # 23:59:59.999999 + except ValueError: + raise ValueError('Invalid date format. Use YYYY-MM-DD.') + return start_date, end_date + +@report_controller_v2.route('/report/v2/org/user/', methods=['POST']) +def get_org_user_report(org_id): """V2 endpoint for organization user report with advanced filtering""" start_timer = time_module.time() try: - logger.info(f"Received request to generate v2 organization user report for orgId={orgId}") + logger.info(f"Received request to generate v2 organization user report for org_id={org_id}") # Validate request - validation_result = _validate_request_common(orgId) + validation_result = _validate_request_common(org_id) if validation_result: return jsonify(validation_result[0]), validation_result[1] @@ -318,12 +334,12 @@ def get_org_user_report(orgId): # Get additional filters additional_filters = data.get('additionalFilter', {}) - logger.info(f"Generating v2 organization user report for orgId={orgId}") + logger.info(f"Generating v2 organization user report for orgId={org_id}") logger.info(f"Additional filters: {additional_filters}") try: csv_data = ReportServiceV2.generate_org_user_report( - mdo_id=orgId, + mdo_id=org_id, is_full_report_required=is_full_report_required, required_columns=required_columns, user_creation_start_date=user_creation_start_date, @@ -332,7 +348,7 @@ def get_org_user_report(orgId): ) if not csv_data: - logger.warning(f"No data found for orgId={orgId}") + logger.warning(f"No data found for org_id={org_id}") return jsonify({'error': 'No data found for the given org details and filters.'}), 404 except Exception as e: @@ -341,13 +357,13 @@ def get_org_user_report(orgId): return jsonify({'error': 'Failed to generate the report due to an internal error.', 'details': error_message}), 500 time_taken = round(time_module.time() - start_timer, 2) - logger.info(f"V2 Org User Report generated successfully in {time_taken} seconds for orgId={orgId}") + logger.info(f"V2 Org User Report generated successfully in {time_taken} seconds for org_id={org_id}") response = Response( stream_with_context(csv_data), mimetype="text/csv", headers={ - "Content-Disposition": f'attachment; filename="user-org-report-v2.csv"' + "Content-Disposition": 'attachment; filename="user-org-report-v2.csv"' } ) diff --git a/app/services/GcsToBigQuerySyncService.py b/app/services/GcsToBigQuerySyncService.py index 9accbb7..2bb0606 100644 --- a/app/services/GcsToBigQuerySyncService.py +++ b/app/services/GcsToBigQuerySyncService.py @@ -28,8 +28,7 @@ def sync_all_tables(self): self.merge_parquet_to_bq( table_config["gcs_uri"], table_config["dataset"], - table_config["table"], - table_config["merge_keys"] + table_config["table"] ) except Exception as e: logger.exception(f"Error during sync: {e}") @@ -62,7 +61,7 @@ def get_sync_config(self): return sync_config - def merge_parquet_to_bq(self, gcs_uri, dataset, target_table, merge_keys): + def merge_parquet_to_bq(self, gcs_uri, dataset, target_table): full_target_table = f"{dataset}.{target_table}" try: # Delete the target table if it exists diff --git a/app/services/data_helper.py b/app/services/data_helper.py new file mode 100644 index 0000000..7ff4115 --- /dev/null +++ b/app/services/data_helper.py @@ -0,0 +1,69 @@ +"""Helper functions for data masking and stream generation""" + +import logging +import gc +from typing import Dict, List, Generator, Any +import pandas as pd + +logger = logging.getLogger(__name__) + +def mask_email(email: str) -> str: + """Mask email address while preserving username""" + if not email: + return email + + parts = email.split('@') + if len(parts) == 2: + domain_parts = parts[1].split('.') + masked_domain = '.'.join(['*' * len(part) for part in domain_parts]) + return f"{parts[0]}@{masked_domain}" + return parts[0] + +def mask_phone(phone: str) -> str: + """Mask phone number while preserving last 4 digits""" + if not phone: + return phone + + phone = str(phone) + if len(phone) >= 4: + return '*' * (len(phone) - 4) + phone[-4:] + return '*' * len(phone) + +def mask_sensitive_data(row_dict: Dict[str, Any], masking_enabled: bool) -> Dict[str, Any]: + """Apply masking to sensitive data fields""" + if not masking_enabled: + return row_dict + + masked = row_dict.copy() + if 'email' in masked and masked['email']: + masked['email'] = mask_email(masked['email']) + if any(key in masked for key in ['phone_number', 'phone']) and masked.get('phone_number') or masked.get('phone'): + phone_key = 'phone_number' if 'phone_number' in masked else 'phone' + masked[phone_key] = mask_phone(masked[phone_key]) + return masked + +def filter_required_columns(df: 'pd.DataFrame', required_columns: List[str]) -> 'pd.DataFrame': + """Filter DataFrame to include only required columns""" + if not required_columns: + return df + + existing_columns = [col for col in required_columns if col in df.columns] + missing_columns = list(set(required_columns) - set(existing_columns)) + if missing_columns: + logger.info(f"Warning: Missing columns skipped: {missing_columns}") + return df[existing_columns] + +def generate_csv_stream(df: 'pd.DataFrame', cols: List[str], masking_enabled: bool = False, separator: str = '|') -> Generator[str, None, None]: + """Generate a CSV stream from a DataFrame with optional masking""" + try: + yield separator.join(cols) + '\n' + for row in df.itertuples(index=False, name=None): + row_dict = dict(zip(cols, row)) + if masking_enabled: + row_dict = mask_sensitive_data(row_dict, masking_enabled) + yield separator.join(str(row_dict.get(col, '')) for col in cols) + '\n' + finally: + df.drop(df.index, inplace=True) + del df + gc.collect() + logger.info("Cleaned up DataFrame after streaming.") \ No newline at end of file diff --git a/app/services/query_helper.py b/app/services/query_helper.py new file mode 100644 index 0000000..0f8c071 --- /dev/null +++ b/app/services/query_helper.py @@ -0,0 +1,59 @@ +"""Helper functions for SQL query generation""" + +from typing import List, Optional, Dict, Any +from google.cloud import bigquery + +def build_date_filter(start_date: Optional[str], end_date: Optional[str], column: str = "enrolled_on") -> str: + """Build date range filter""" + if start_date and end_date: + return f" AND {column} BETWEEN '{start_date}' AND '{end_date}'" + return "" + +def build_mdo_id_filter(mdo_ids: List[str]) -> str: + """Build MDO ID filter""" + mdo_id_list = [f"'{mid}'" for mid in mdo_ids] + return f"mdo_id IN ({', '.join(mdo_id_list)})" + +def build_user_filter(email: Optional[str], phone: Optional[str], ehrms_id: Optional[str]) -> List[str]: + """Build user filter conditions""" + filters = [] + if email: + filters.append(f"email = '{email}'") + if phone: + filters.append(f"phone_number = '{phone}'") + if ehrms_id: + filters.append(f"external_system_id = '{ehrms_id}'") + return filters + +def build_apar_query(table: str, filters: Dict[str, Any], filter_map: Dict[str, str], + start_date: Optional[str] = None, end_date: Optional[str] = None) -> tuple[str, List[bigquery.ScalarQueryParameter]]: + """Build APAR query with parameters""" + filter_clauses = [] + params = [] + + # Add date filters + if start_date and end_date: + filter_clauses.insert(0, "enrolled_on >= @start_date AND enrolled_on <= @end_date") + params.extend([ + bigquery.ScalarQueryParameter("start_date", "TIMESTAMP", start_date), + bigquery.ScalarQueryParameter("end_date", "TIMESTAMP", end_date) + ]) + + # Add other filters + for key, value in filters.items(): + if value and key in filter_map: + bq_col = filter_map[key] + filter_clauses.append(f"{bq_col} = @{bq_col}") + params.append(bigquery.ScalarQueryParameter(bq_col, "STRING", value.strip())) + + # Build final query + if filter_clauses: + query = f""" + SELECT * + FROM `{table}` + WHERE {" AND ".join(filter_clauses)} + """ + else: + query = f"SELECT * FROM `{table}`" + + return query, params \ No newline at end of file diff --git a/app/services/report_helper.py b/app/services/report_helper.py new file mode 100644 index 0000000..90b610d --- /dev/null +++ b/app/services/report_helper.py @@ -0,0 +1,53 @@ +"""Helper functions for report generation service""" + +import logging +import gc +from typing import List, Dict, Any, Generator, Optional +import pandas as pd + +logger = logging.getLogger(__name__) + +def build_date_filter(start_date: str, end_date: str, column_name: str = "enrolled_on") -> Optional[str]: + """Build date range filter""" + if start_date and end_date: + return f"{column_name} BETWEEN '{start_date}' AND '{end_date}'" + return None + +def build_enrolments_query(master_table: str, where_clause: str) -> str: + """Build the enrolments query""" + return f""" + SELECT * + FROM `{master_table}` + WHERE {where_clause} + """ + +def build_user_query(master_table: str, where_clause: str) -> str: + """Build the user query""" + return f""" + SELECT user_id, mdo_id + FROM `{master_table}` + WHERE {where_clause} + """ + +def generate_csv_stream(df: 'pd.DataFrame', cols: List[str], separator: str = '|') -> Generator[str, None, None]: + """Generate a CSV stream from a DataFrame with proper cleanup""" + try: + yield separator.join(cols) + '\n' + for row in df.itertuples(index=False, name=None): + yield separator.join(map(str, row)) + '\n' + finally: + df.drop(df.index, inplace=True) + del df + gc.collect() + logger.info("Cleaned up DataFrame after streaming.") + +def filter_required_columns(df: 'pd.DataFrame', required_columns: List[str]) -> 'pd.DataFrame': + """Filter DataFrame to include only required columns""" + if not required_columns: + return df + + existing_columns = [col for col in required_columns if col in df.columns] + missing_columns = list(set(required_columns) - set(existing_columns)) + if missing_columns: + logger.info(f"Warning: Missing columns skipped: {missing_columns}") + return df[existing_columns] \ No newline at end of file diff --git a/app/services/report_service.py b/app/services/report_service.py index 4d939ca..86b105f 100644 --- a/app/services/report_service.py +++ b/app/services/report_service.py @@ -1,10 +1,28 @@ +import gc import logging +from datetime import datetime +from typing import Dict, Generator, List, Optional, Union + +import pandas as pd + from app.services.fetch_data_bigQuery import BigQueryService from app.services.redis_service import RedisService -from constants import MASTER_ENROLMENTS_TABLE, MASTER_USER_TABLE, MASTER_ORG_HIERARCHY_TABLE, IS_MASKING_ENABLED, MAX_ORG_CACHE_AGE, MASTER_APAR_TABLE, APAR_FILTER_KEY_MAP -import gc -import io +from app.services.data_helper import generate_csv_stream, filter_required_columns +from app.services.query_helper import build_user_filter + + +from constants import ( + MASTER_ENROLMENTS_TABLE, + MASTER_USER_TABLE, + MASTER_ORG_HIERARCHY_TABLE, + IS_MASKING_ENABLED, + MAX_ORG_CACHE_AGE, + MASTER_APAR_TABLE, + APAR_FILTER_KEY_MAP, + CLEANUP_MESSAGE +) from google.cloud import bigquery +from typing import Optional, List, Dict, Any, Generator logging.basicConfig( level=logging.INFO, @@ -15,104 +33,85 @@ class ReportService: logger = logging.getLogger(__name__) @staticmethod - def fetch_user_cumulative_report(email=None, phone=None, ehrms_id=None, start_date=None, end_date=None, orgId=None, required_columns=None): + def fetch_user_cumulative_report( + email: Optional[str] = None, + phone: Optional[str] = None, + ehrms_id: Optional[str] = None, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + org_id: Optional[str] = None, + required_columns: Optional[List[str]] = None + ) -> Optional[Generator[str, None, None]]: + """Fetch cumulative report for a user with optional filters""" try: - # Check if any user filter is provided + # Validate input if not any([email, phone, ehrms_id]): - ReportService.logger.info("No user filters provided for fetching user data.") + ReportService.logger.info("No user filters provided") return None bigquery_service = BigQueryService() - # Build filters for user details - user_filters = [] - if email: - user_filters.append(f"email = '{email}'") - if phone: - user_filters.append(f"phone_number = '{phone}'") - if ehrms_id: - user_filters.append(f"external_system_id = '{ehrms_id}'") - + # Build and execute user query + user_filters = build_user_filter(email, phone, ehrms_id) if not user_filters: - ReportService.logger.info("No valid filters provided for fetching user data.") + ReportService.logger.info("No valid filters provided") return None - # Construct the query for fetching user data - user_filter_query = ' AND '.join(user_filters) user_query = f""" SELECT user_id, mdo_id FROM `{MASTER_USER_TABLE}` - WHERE {user_filter_query} + WHERE {" AND ".join(user_filters)} """ ReportService.logger.info(f"Executing user query: {user_query}") user_df = bigquery_service.run_query(user_query) if user_df.empty: - ReportService.logger.info("No users found matching the provided filters.") + ReportService.logger.info("No matching users found") return None + # Process user data user_ids = user_df["user_id"].tolist() - ReportService.logger.info(f"Fetched {len(user_ids)} users.") - - # Get the user's MDO ID - user_mdo_id = user_df["mdo_id"].iloc[0] # Get the first user's MDO ID - - # Check if organization ID is valid - if orgId and orgId != user_mdo_id: - mdo_id_org_list = list(ReportService._get_mdo_id_org_list(bigquery_service, orgId)) - mdo_id_org_list.append(orgId) # Include the orgId itself - + user_mdo_id = user_df["mdo_id"].iloc[0] + ReportService.logger.info(f"Found {len(user_ids)} users") + + # Validate organization + if org_id and org_id != user_mdo_id: + mdo_id_org_list = list(ReportService._get_mdo_id_org_list(bigquery_service, org_id)) + mdo_id_org_list.append(org_id) if user_mdo_id not in mdo_id_org_list: - ReportService.logger.error(f"Invalid organization ID for user: {orgId}") - raise ValueError(f"Invalid organization ID for user: {orgId}") - - # Construct the query for fetching enrollment data + raise ValueError(f"Invalid organization ID for user: {org_id}") + + # Build and execute enrollment query + user_ids_str = ', '.join(f"'{uid}'" for uid in user_ids) + where_parts = [f"user_id IN ({user_ids_str})"] + if start_date and end_date: + where_parts.append(f"enrolled_on BETWEEN '{start_date}' AND '{end_date}'") + enrollment_query = f""" SELECT * FROM `{MASTER_ENROLMENTS_TABLE}` - WHERE user_id IN ({', '.join([f"'{uid}'" for uid in user_ids])}) + WHERE {" AND ".join(where_parts)} """ - if start_date and end_date: - enrollment_query += f" AND enrolled_on BETWEEN '{start_date}' AND '{end_date}'" - ReportService.logger.info(f"Executing enrollment query: {enrollment_query}") enrollment_df = bigquery_service.run_query(enrollment_query) if enrollment_df.empty: - ReportService.logger.info("No enrollment data found for the given user.") + ReportService.logger.info("No enrollment data found") return None - # Filter columns if specified - if required_columns: - existing_columns = [col for col in required_columns if col in enrollment_df.columns] - missing_columns = list(set(required_columns) - set(existing_columns)) - if missing_columns: - ReportService.logger.info(f"Warning: Missing columns skipped: {missing_columns}") - merged_df = enrollment_df[existing_columns] - else: - merged_df = enrollment_df - - def generate_csv_stream(df, cols): - try: - yield '|'.join(cols) + '\n' - for row in df.itertuples(index=False, name=None): - yield '|'.join(map(str, row)) + '\n' - finally: - # Safe cleanup after generator is fully consumed - df.drop(df.index, inplace=True) - del df - gc.collect() - ReportService.logger.info("Cleaned up DataFrame after streaming.") + # Process results + result_df = filter_required_columns(enrollment_df, required_columns or []) + return generate_csv_stream(result_df, result_df.columns.tolist()) ReportService.logger.info(f"CSV stream generated with {len(merged_df)} rows.") # Return CSV content without closing the stream return generate_csv_stream(merged_df, merged_df.columns.tolist()) - except MemoryError as me: - ReportService.logger.error("MemoryError encountered. Consider processing data in smaller chunks.") + except MemoryError: + ReportService.logger.error("Memory error while running query") raise except Exception as e: ReportService.logger.error(f"Error generating cumulative report: {e}") @@ -172,8 +171,8 @@ def generate_csv_stream(df, cols): finally: df.drop(df.index, inplace=True) del df - gc.collect() - ReportService.logger.info("Cleaned up DataFrame after streaming.") + ReportService.logger.info(CLEANUP_MESSAGE) + ReportService.logger.info(CLEANUP_MESSAGE) ReportService.logger.info(f"CSV stream generated with {len(result_df)} rows.") return generate_csv_stream(result_df, result_df.columns.tolist()) @@ -183,87 +182,92 @@ def generate_csv_stream(df, cols): return None @staticmethod - def fetch_master_user_data(mdo_id, is_full_report_required, required_columns=None, user_creation_start_date=None, user_creation_end_date=None): + def fetch_master_user_data(mdo_id, is_full_report_required, required_columns=None, user_creation_start_date=None, user_creation_end_date=None): try: bigquery_service = BigQueryService() - # Add date filtering to the query if start_date and end_date are provided - date_filter = "" - if user_creation_start_date and user_creation_end_date: - date_filter = f" AND user_registration_date BETWEEN '{user_creation_start_date}' AND '{user_creation_end_date}'" - if is_full_report_required: - mdo_id_org_list = list(ReportService._get_mdo_id_org_list(bigquery_service, mdo_id)) - mdo_id_org_list.append(mdo_id) - else: - mdo_id_org_list = [mdo_id] - - mdo_id_list = [f"'{mid}'" for mid in mdo_id_org_list] # Quote each ID - mdo_id_str = ', '.join(mdo_id_list) # Join them with commas - query = f""" - SELECT * - FROM `{MASTER_USER_TABLE}` - WHERE mdo_id in ({mdo_id_str}){date_filter} - """ + date_filter = ReportService._build_date_filter(user_creation_start_date, user_creation_end_date) + mdo_id_org_list = ReportService._get_mdo_id_list(bigquery_service, mdo_id, is_full_report_required) + query = ReportService._build_user_data_query(mdo_id_org_list, date_filter) ReportService.logger.info(f"Executing query: {query}") - - # Update to use run_query instead of execute_query result_df = bigquery_service.run_query(query) if result_df.empty: - ReportService.logger.info("No data found for user the given mdo_id and date range.") + ReportService.logger.info("No data found for the given mdo_id and date range.") return None - ReportService.logger.info(f"Fetched {len(result_df)} rows from master_enrolments_data.") - - # Filter the result DataFrame to include only the required columns - if required_columns: - existing_columns = [col for col in required_columns if col in result_df.columns] - missing_columns = list(set(required_columns) - set(existing_columns)) - if missing_columns: - ReportService.logger.info(f"Warning: Missing columns skipped: {missing_columns}") - result_df = result_df[existing_columns] - - # Generate CSV stream from the result DataFrame - def generate_csv_stream(df, cols): - try: - yield '|'.join(cols) + '\n' - for row in df.itertuples(index=False, name=None): - row_dict = dict(zip(cols, row)) - if IS_MASKING_ENABLED.lower() == 'true': - # Mask email - if 'email' in row_dict and row_dict['email']: - parts = row_dict['email'].split('@') - if len(parts) == 2: - domain_parts = parts[1].split('.') - masked_domain = '.'.join(['*' * len(part) for part in domain_parts]) - row_dict['email'] = f"{parts[0]}@{masked_domain}" - else: - row_dict['email'] = parts[0] - - # Mask phone number: e.g., ******2245 - if 'phone_number' in row_dict and row_dict['phone_number']: - phone = str(row_dict['phone_number']) - if len(phone) >= 4: - row_dict['phone_number'] = '*' * (len(phone) - 4) + phone[-4:] - else: - row_dict['phone_number'] = '*' * len(phone) - - # Convert back to row and yield - yield '|'.join([str(row_dict.get(col, '')) for col in cols]) + '\n' - finally: - df.drop(df.index, inplace=True) - del df - gc.collect() - ReportService.logger.info("Cleaned up DataFrame after streaming.") - ReportService.logger.info(f"CSV stream generated with {len(result_df)} rows.") - - # Return CSV content without closing the stream - return generate_csv_stream(result_df, result_df.columns.tolist()) + ReportService.logger.info(f"Fetched {len(result_df)} rows from master_user_data.") + result_df = ReportService._filter_columns(result_df, required_columns) + return ReportService._generate_csv_stream(result_df) except Exception as e: ReportService.logger.error(f"Error fetching master user data: {e}") return None + @staticmethod + def _build_date_filter(start_date, end_date): + if start_date and end_date: + return f" AND user_registration_date BETWEEN '{start_date}' AND '{end_date}'" + return "" + + @staticmethod + def _get_mdo_id_list(bigquery_service, mdo_id, is_full_report_required): + if is_full_report_required: + mdo_id_org_list = list(ReportService._get_mdo_id_org_list(bigquery_service, mdo_id)) + mdo_id_org_list.append(mdo_id) + else: + mdo_id_org_list = [mdo_id] + return mdo_id_org_list + + @staticmethod + def _build_user_data_query(mdo_id_org_list, date_filter): + mdo_id_list = [f"'{mid}'" for mid in mdo_id_org_list] + mdo_id_str = ', '.join(mdo_id_list) + return f""" + SELECT * + FROM `{MASTER_USER_TABLE}` + WHERE mdo_id in ({mdo_id_str}){date_filter} + """ + + @staticmethod + def _filter_columns(df, required_columns): + if required_columns: + existing_columns = [col for col in required_columns if col in df.columns] + missing_columns = list(set(required_columns) - set(existing_columns)) + if missing_columns: + ReportService.logger.info(f"Warning: Missing columns skipped: {missing_columns}") + return df[existing_columns] + return df + + @staticmethod + def _generate_csv_stream(df): + try: + yield '|'.join(df.columns.tolist()) + '\n' + for row in df.itertuples(index=False, name=None): + row_dict = dict(zip(df.columns, row)) + if IS_MASKING_ENABLED.lower() == 'true': + row_dict = ReportService._mask_sensitive_data(row_dict) + yield '|'.join([str(row_dict.get(col, '')) for col in df.columns]) + '\n' + finally: + df.drop(df.index, inplace=True) + del df + gc.collect() + ReportService.logger.info("Cleaned up DataFrame after streaming.") + + @staticmethod + def _mask_sensitive_data(row_dict): + if 'email' in row_dict and row_dict['email']: + parts = row_dict['email'].split('@') + if len(parts) == 2: + domain_parts = parts[1].split('.') + masked_domain = '.'.join(['*' * len(part) for part in domain_parts]) + row_dict['email'] = f"{parts[0]}@{masked_domain}" + if 'phone_number' in row_dict and row_dict['phone_number']: + phone = str(row_dict['phone_number']) + if len(phone) >= 4: + row_dict['phone_number'] = '*' * (len(phone) - 4) + phone[-4:] + return row_dict + @staticmethod def _get_mdo_id_org_list(bigquery_service: BigQueryService, mdo_id: str) -> list: @@ -322,14 +326,18 @@ def _get_mdo_id_org_list(bigquery_service: BigQueryService, mdo_id: str) -> list return mdo_id_org_list @staticmethod - def isValidOrg(x_org_id, request_org_id): + def is_valid_org(x_org_id: str, request_org_id: str) -> bool: + """Validate if the given organization ID is valid for the request""" try: # Check for None or empty request_org_id if not request_org_id: ReportService.logger.error("request_org_id is None or empty") return False + + # Debug logging ReportService.logger.debug(f"request_org_id={request_org_id}, type={type(request_org_id)}") ReportService.logger.debug(f"x_org_id={x_org_id}, type={type(x_org_id)}") + # Ensure x_org_id is valid if not x_org_id: ReportService.logger.error("x_org_id is None or empty") @@ -341,6 +349,7 @@ def isValidOrg(x_org_id, request_org_id): org_list = list(ReportService._get_mdo_id_org_list(bigquery_service, x_org_id)) org_list.append(x_org_id) # Add input mdo_id to the list ReportService.logger.info(f"The OrgId list for {request_org_id}: {len(org_list)}") + # Check if request_org_id is in the organization list is_valid = request_org_id in org_list ReportService.logger.debug(f"Validation result for org_id {request_org_id}: {is_valid}") @@ -359,84 +368,72 @@ def fetch_apar_enrolment_report(enrolment_start_date, enrolment_end_date, filter client = BigQueryService() table = MASTER_APAR_TABLE - # Build dynamic filter clauses and parameters - date_filter = "" - if enrolment_start_date and enrolment_end_date: - date_filter = f" AND enrolled_on BETWEEN '{enrolment_start_date}' AND '{enrolment_end_date}'" - - # Map filter keys to BQ column names - filter_key_map = APAR_FILTER_KEY_MAP - filter_clauses = [] - params = [] - # Add filters if present - for key, value in filters.items(): - if value and key in filter_key_map: - bq_col = filter_key_map[key] - filter_clauses.append(f"{bq_col} = @{bq_col}") - params.append(bigquery.ScalarQueryParameter(bq_col, "STRING", value.strip())) - - # Always add date filter - if enrolment_start_date and enrolment_end_date: - filter_clauses.insert(0, "enrolled_on >= @start_date AND enrolled_on <= @end_date") - params.insert(0, bigquery.ScalarQueryParameter("end_date", "TIMESTAMP", enrolment_end_date)) - params.insert(0, bigquery.ScalarQueryParameter("start_date", "TIMESTAMP", enrolment_start_date)) - - # Build query - if filter_clauses: - query = f""" - SELECT * - FROM `{table}` - WHERE {" AND ".join(filter_clauses)} - """ - else: - query = f""" - SELECT * - FROM `{table}`{date_filter} - """ + # Build query and parameters + query, params = ReportService._build_apar_query( + table, enrolment_start_date, enrolment_end_date, filters + ) - - job_config = bigquery.QueryJobConfig(query_parameters=params) ReportService.logger.info(f"Executing APAR enrolment query: {query} with params: {params}") - df = client.query(query, job_config=job_config).to_dataframe() + df = client.query(query, job_config=bigquery.QueryJobConfig(query_parameters=params)).to_dataframe() # Filter columns if required if required_columns: - filtered_cols = [col for col in required_columns if col in df.columns] - if filtered_cols: - df = df[filtered_cols] + df = ReportService._filter_dataframe_columns(df, required_columns) - def generate_csv_stream(df, cols): - try: - yield '|'.join(cols) + '\n' - for row in df.itertuples(index=False, name=None): - row_dict = dict(zip(cols, row)) - if IS_MASKING_ENABLED.lower() == 'true': - if 'email' in row_dict and row_dict['email'] is not None: - parts = row_dict['email'].split('@') - if len(parts) == 2: - domain_parts = parts[1].split('.') - masked_domain = '.'.join(['*' * len(part) for part in domain_parts]) - row_dict['email'] = f"{parts[0]}@{masked_domain}" - else: - row_dict['email'] = parts[0] - - # Mask phone number: e.g., ******2245 - if 'phone' in row_dict and row_dict['phone'] is not None: - phone = str(row_dict['phone']) - if phone and len(phone) >= 4: - row_dict['phone'] = '*' * (len(phone) - 4) + phone[-4:] - elif phone: - row_dict['phone'] = '*' * len(phone) - - # Convert back to row and yield - yield '|'.join([str(row_dict.get(col, '')) for col in cols]) + '\n' - finally: - df.drop(df.index, inplace=True) - del df - gc.collect() - ReportService.logger.info("Cleaned up DataFrame after streaming.") - ReportService.logger.info(f"CSV stream generated with {len(df)} rows.") - return generate_csv_stream(df, df.columns.tolist()) + return ReportService._generate_csv_stream_with_masking(df) except Exception as e: ReportService.logger.error(f"Error fetching APAR enrolment report: {e}") return None + + @staticmethod + def _build_apar_query(table, enrolment_start_date, enrolment_end_date, filters): + """Build the APAR query and parameters.""" + date_filter = "" + if enrolment_start_date and enrolment_end_date: + date_filter = f" AND enrolled_on BETWEEN '{enrolment_start_date}' AND '{enrolment_end_date}'" + + filter_key_map = APAR_FILTER_KEY_MAP + filter_clauses = [] + params = [] + + for key, value in filters.items(): + if value and key in filter_key_map: + bq_col = filter_key_map[key] + filter_clauses.append(f"{bq_col} = @{bq_col}") + params.append(bigquery.ScalarQueryParameter(bq_col, "STRING", value.strip())) + + if enrolment_start_date and enrolment_end_date: + filter_clauses.insert(0, "enrolled_on >= @start_date AND enrolled_on <= @end_date") + params.insert(0, bigquery.ScalarQueryParameter("end_date", "TIMESTAMP", enrolment_end_date)) + params.insert(0, bigquery.ScalarQueryParameter("start_date", "TIMESTAMP", enrolment_start_date)) + + query = f""" + SELECT * + FROM `{table}` + WHERE {" AND ".join(filter_clauses)}{date_filter if not filter_clauses else ""} + """ + return query, params + + @staticmethod + def _filter_dataframe_columns(df, required_columns): + """Filter DataFrame to include only required columns.""" + filtered_cols = [col for col in required_columns if col in df.columns] + if filtered_cols: + return df[filtered_cols] + return df + + @staticmethod + def _generate_csv_stream_with_masking(df): + """Generate CSV stream with optional masking.""" + try: + yield '|'.join(df.columns.tolist()) + '\n' + for row in df.itertuples(index=False, name=None): + row_dict = dict(zip(df.columns, row)) + if IS_MASKING_ENABLED.lower() == 'true': + row_dict = ReportService._mask_sensitive_data(row_dict) + yield '|'.join([str(row_dict.get(col, '')) for col in df.columns]) + '\n' + finally: + df.drop(df.index, inplace=True) + del df + gc.collect() + ReportService.logger.info("Cleaned up DataFrame after streaming.") diff --git a/app/services/report_service_v2.py b/app/services/report_service_v2.py index c028f4f..da05bd7 100644 --- a/app/services/report_service_v2.py +++ b/app/services/report_service_v2.py @@ -1,9 +1,14 @@ import logging from app.services.fetch_data_bigQuery import BigQueryService from app.services.redis_service import RedisService +from app.services.report_helper import ( + build_date_filter, + generate_csv_stream, + filter_required_columns +) from constants import (MASTER_ENROLMENTS_TABLE, MASTER_USER_TABLE, IS_MASKING_ENABLED, MAX_ORG_CACHE_AGE, - ENROLMENT_FILTER_CONFIG, USER_FILTER_CONFIG, USER_REPORT_FILTER_CONFIG) + ENROLMENT_FILTER_CONFIG, USER_FILTER_CONFIG, USER_REPORT_FILTER_CONFIG, AND) import gc import pandas as pd from app.services.report_service import ReportService @@ -18,62 +23,140 @@ class ReportServiceV2: logger = logging.getLogger(__name__) + @staticmethod + def _build_query(table_name: str, where_parts: list, is_select_all: bool = True) -> str: + """Build SQL query from components""" + cols = "*" if is_select_all else "user_id, mdo_id" + where_clause = f" {AND} ".join(where_parts) + return f""" + SELECT {cols} + FROM `{table_name}` + WHERE {where_clause} + """ + + @staticmethod + def _execute_query(bigquery_service: BigQueryService, query: str, context: str = "") -> pd.DataFrame: + """Execute query and handle logging""" + ReportServiceV2.logger.info(f"Executing {context} query: {query}") + result_df = bigquery_service.run_query(query) + if not result_df.empty: + ReportServiceV2.logger.info(f"Fetched {len(result_df)} rows{' ' + context if context else ''}") + return result_df + + @staticmethod + def _generate_csv_stream(df, cols): + """Generate a CSV stream from a DataFrame with proper cleanup""" + return generate_csv_stream(df, cols) + + @staticmethod + def _handle_mdo_ids(bigquery_service, org_id, is_full_report_required, mdo_id_list=None): + """Handle MDO ID filtering logic""" + if mdo_id_list and isinstance(mdo_id_list, list) and mdo_id_list: + ReportServiceV2.logger.info(f"Using provided MDO ID list: {mdo_id_list}") + mdo_id_org_list = list(ReportService._get_mdo_id_org_list(bigquery_service, org_id)) + mdo_id_list = [mid for mid in mdo_id_list if mid in mdo_id_org_list] + if org_id not in mdo_id_list: + mdo_id_list.append(org_id) + ReportServiceV2.logger.info(f"Filtered MDO ID list: {mdo_id_list}") + else: + if is_full_report_required: + mdo_id_list = list(ReportService._get_mdo_id_org_list(bigquery_service, org_id)) + mdo_id_list.append(org_id) + ReportServiceV2.logger.debug(f"Fetched {len(mdo_id_list)} MDO IDs (including input): {mdo_id_list}") + else: + mdo_id_list = [org_id] + ReportServiceV2.logger.info(f"Full report not required. Using single mdo_id: {org_id}") + + return [f"'{mid}'" for mid in mdo_id_list] + + @staticmethod + def _filter_required_columns(df, required_columns): + """Filter DataFrame to include only required columns""" + if required_columns: + existing_columns = [col for col in required_columns if col in df.columns] + missing_columns = list(set(required_columns) - set(existing_columns)) + if missing_columns: + ReportServiceV2.logger.info(f"Warning: Missing columns skipped: {missing_columns}") + return df[existing_columns] + return df + + @staticmethod + def _process_string_filter(filter_name, filter_value, filter_config_item, escaped_filter_name): + """Process string type filters""" + if not filter_value: + return None + + if 'values' in filter_config_item: + if isinstance(filter_config_item['values'], dict): + if filter_value in filter_config_item['values']: + mapped_value = filter_config_item['values'][filter_value] + return f"{escaped_filter_name} = {mapped_value}" if isinstance(mapped_value, int) else f"{escaped_filter_name} = '{mapped_value}'" + else: + return f"{escaped_filter_name} IN ({', '.join(map(str, filter_config_item['values']))})" + return f"{escaped_filter_name} = '{filter_value}'" + + @staticmethod + def _process_list_filter(filter_name, filter_value, escaped_filter_name): + """Process list type filters""" + if not isinstance(filter_value, list) or not filter_value: + return None + values_str = ', '.join([f"'{val}'" for val in filter_value]) + return f"{escaped_filter_name} IN ({values_str})" + + @staticmethod + def _process_comparison_filter(filter_name, filter_value, filter_config_item, escaped_filter_name): + """Process comparison type filters""" + if not filter_value: + return None + + for operator in filter_config_item.get('valid_operators', []): + if filter_value.startswith(operator): + value = filter_value[len(operator):].strip() + try: + float_value = float(value) + return f"{escaped_filter_name} {operator} {float_value}" + except ValueError: + ReportServiceV2.logger.warning(f"Invalid numeric value for {filter_name}: {value}") + + valid_ops = filter_config_item.get('valid_operators', []) + raise ValueError(f"Invalid operator for {filter_name}: '{filter_value}'. Allowed operators: {valid_ops}") + + @staticmethod + def _process_boolean_filter(filter_name, filter_value, filter_config_item, escaped_filter_name): + """Process boolean type filters""" + if filter_value is None: + return None + + if 'values' in filter_config_item and filter_value in filter_config_item['values']: + bool_value = filter_config_item['values'][filter_value] + bool_str = "TRUE" if bool_value else "FALSE" + return f"{escaped_filter_name} = {bool_str}" + return None + @staticmethod def _process_filters(filters, filter_config, where_clause_parts): """Process filters based on configuration""" for filter_name, filter_value in filters.items(): - if filter_name in filter_config: - filter_config_item = filter_config[filter_name] - - # Escape reserved keywords - escaped_filter_name = f"`{filter_name}`" if filter_name.lower() in ['groups', 'order', 'limit'] else filter_name - - # Skip already processed filters - if filter_name == 'mdo_id_list': - continue - - # Process based on filter type - if filter_config_item['type'] == 'string' and filter_value: - if 'values' in filter_config_item: - if isinstance(filter_config_item['values'], dict): - # Map the filter value using the dictionary - if filter_value in filter_config_item['values']: - mapped_value = filter_config_item['values'][filter_value] - if isinstance(mapped_value, int): - where_clause_parts.append(f"{escaped_filter_name} = {mapped_value}") - else: - where_clause_parts.append(f"{escaped_filter_name} = '{mapped_value}'") - else: - where_clause_parts.append(f"{escaped_filter_name} IN ({', '.join(map(str, filter_config_item['values']))})") - else: - where_clause_parts.append(f"{escaped_filter_name} = '{filter_value}'") - - elif filter_config_item['type'] == 'list' and isinstance(filter_value, list) and filter_value: - values_str = ', '.join([f"'{val}'" for val in filter_value]) - where_clause_parts.append(f"{escaped_filter_name} IN ({values_str})") - - elif filter_config_item['type'] == 'comparison' and filter_value: - matched_operator = None - for operator in filter_config_item.get('valid_operators', []): - if filter_value.startswith(operator): - matched_operator = operator - value = filter_value[len(operator):].strip() - try: - # Ensure value is numeric - float_value = float(value) - where_clause_parts.append(f"{escaped_filter_name} {operator} {float_value}") - break - except ValueError: - ReportServiceV2.logger.warning(f"Invalid numeric value for {filter_name}: {value}") - if matched_operator is None: - raise ValueError(f"Invalid operator for {filter_name}: '{filter_value}'. Allowed operators: {filter_config_item.get('valid_operators', [])}") - - elif filter_config_item['type'] == 'boolean' and filter_value is not None: - # Convert to boolean value - if 'values' in filter_config_item and filter_value in filter_config_item['values']: - bool_value = filter_config_item['values'][filter_value] - bool_str = "TRUE" if bool_value else "FALSE" - where_clause_parts.append(f"{escaped_filter_name} = {bool_str}") + if filter_name not in filter_config or filter_name == 'mdo_id_list': + continue + + filter_config_item = filter_config[filter_name] + escaped_filter_name = f"`{filter_name}`" + filter_type = filter_config_item['type'] + + filter_processors = { + 'string': ReportServiceV2._process_string_filter, + 'list': ReportServiceV2._process_list_filter, + 'comparison': ReportServiceV2._process_comparison_filter, + 'boolean': ReportServiceV2._process_boolean_filter + } + + if filter_type in filter_processors: + result = filter_processors[filter_type]( + filter_name, filter_value, filter_config_item, escaped_filter_name + ) + if result: + where_clause_parts.append(result) return where_clause_parts @@ -95,97 +178,68 @@ def generate_report(start_date, end_date, org_id, is_full_report_required, requi """ try: bigquery_service = BigQueryService() - additional_filters = additional_filters or {} - - # Build filters where_clause_parts = [] # Add date filtering if start_date and end_date: - where_clause_parts.append(f"enrolled_on BETWEEN '{start_date}' AND '{end_date}'") + where_clause_parts.append( + build_date_filter(start_date, end_date, "enrolled_on") + ) # Handle MDO ID filtering - mdo_id_list = additional_filters.get('mdo_id_list', []) - if mdo_id_list and isinstance(mdo_id_list, list) and len(mdo_id_list) > 0: - # If specific MDO IDs are provided, use those - ReportServiceV2.logger.info(f"Using provided MDO ID list: {mdo_id_list}") - - # Fetch the valid MDO IDs from the hierarchy - mdo_id_org_list = list(ReportService._get_mdo_id_org_list(bigquery_service, org_id)) - - # Filter out invalid MDO IDs - mdo_id_list = [mid for mid in mdo_id_list if mid in mdo_id_org_list] - if org_id not in mdo_id_list: - mdo_id_list.append(org_id) - ReportServiceV2.logger.info(f"Filtered MDO ID list: {mdo_id_list}") - - mdo_ids_to_use = [f"'{mid}'" for mid in mdo_id_list] - else: - # Otherwise use the standard logic based on is_full_report_required - if is_full_report_required: - # Dynamically fetch orgs using hierarchy - mdo_id_org_list = list(ReportService._get_mdo_id_org_list(bigquery_service, org_id)) - mdo_id_org_list.append(org_id) # Add input mdo_id to the list - ReportServiceV2.logger.debug(f"Fetched {len(mdo_id_org_list)} MDO IDs (including input): {mdo_id_org_list}") - else: - mdo_id_org_list = [org_id] - ReportServiceV2.logger.info(f"Full report not required. Using single mdo_id: {org_id}") - - mdo_ids_to_use = [f"'{mid}'" for mid in mdo_id_org_list] - - mdo_id_str = ', '.join(mdo_ids_to_use) - where_clause_parts.append(f"mdo_id in ({mdo_id_str})") + mdo_ids = ReportServiceV2._handle_mdo_ids( + bigquery_service, + org_id, + is_full_report_required, + (additional_filters or {}).get('mdo_id_list', []) + ) + where_clause_parts.append(f"mdo_id IN ({', '.join(mdo_ids)})") # Process additional filters - where_clause_parts = ReportServiceV2._process_filters(additional_filters, ENROLMENT_FILTER_CONFIG, where_clause_parts) + where_clause_parts = ReportServiceV2._process_filters( + additional_filters or {}, + ENROLMENT_FILTER_CONFIG, + where_clause_parts + ) - # Construct the WHERE clause - where_clause = " AND ".join(where_clause_parts) + # Build and execute query + query = ReportServiceV2._build_query(MASTER_ENROLMENTS_TABLE, where_clause_parts) + result_df = ReportServiceV2._execute_query(bigquery_service, query, "from master_enrolments_data") - query = f""" - SELECT * - FROM `{MASTER_ENROLMENTS_TABLE}` - WHERE {where_clause} - """ - - ReportServiceV2.logger.info(f"Executing enrolments query: {query}") - result_df = bigquery_service.run_query(query) - if result_df.empty: - ReportServiceV2.logger.info("No data found for the given filters.") return None - - ReportServiceV2.logger.info(f"Fetched {len(result_df)} rows from master_enrolments_data.") - - # Filter the result DataFrame to include only the required columns - if required_columns: - existing_columns = [col for col in required_columns if col in result_df.columns] - missing_columns = list(set(required_columns) - set(existing_columns)) - if missing_columns: - ReportServiceV2.logger.info(f"Warning: Missing columns skipped: {missing_columns}") - result_df = result_df[existing_columns] - - # Generate CSV stream from the result DataFrame - def generate_csv_stream(df, cols): - try: - yield '|'.join(cols) + '\n' - for row in df.itertuples(index=False, name=None): - yield '|'.join(map(str, row)) + '\n' - finally: - df.drop(df.index, inplace=True) - del df - gc.collect() - ReportServiceV2.logger.info("Cleaned up DataFrame after streaming.") - - ReportServiceV2.logger.info(f"CSV stream generated with {len(result_df)} rows.") - return generate_csv_stream(result_df, result_df.columns.tolist()) + + # Process results + result_df = filter_required_columns(result_df, required_columns) + return ReportServiceV2._generate_csv_stream(result_df, result_df.columns.tolist()) except Exception as e: ReportServiceV2.logger.error(f"Error fetching master enrolments data: {e}") raise @staticmethod - def generate_user_report(email=None, phone=None, ehrms_id=None, start_date=None, end_date=None, orgId=None, required_columns=None, additional_filters=None): + def _build_user_filters(email=None, phone=None, ehrms_id=None): + """Build user filters list""" + filters = [] + if email: + filters.append(f"email = '{email}'") + if phone: + filters.append(f"phone_number = '{phone}'") + if ehrms_id: + filters.append(f"external_system_id = '{ehrms_id}'") + return filters + + @staticmethod + def _validate_org_id(bigquery_service, user_mdo_id, org_id): + """Validate organization ID against user's MDO ID""" + if org_id and org_id != user_mdo_id: + mdo_id_org_list = list(ReportService._get_mdo_id_org_list(bigquery_service, org_id)) + mdo_id_org_list.append(org_id) + if user_mdo_id not in mdo_id_org_list: + raise ValueError(f"Invalid organization ID for user: {org_id}") + + @staticmethod + def generate_user_report(email=None, phone=None, ehrms_id=None, start_date=None, end_date=None, org_id=None, required_columns=None, additional_filters=None): """ Enhanced version of fetch_user_cumulative_report with additional filtering capabilities. @@ -195,7 +249,7 @@ def generate_user_report(email=None, phone=None, ehrms_id=None, start_date=None, ehrms_id: User EHRMS ID start_date: Start date for enrollment filtering end_date: End date for enrollment filtering - orgId: Organization ID + org_id: Organization ID required_columns: List of columns to include in the report additional_filters: Dictionary of additional filters to apply @@ -203,120 +257,98 @@ def generate_user_report(email=None, phone=None, ehrms_id=None, start_date=None, Generator yielding CSV data or None if no data found """ try: - # Check if any user filter is provided - if not any([email, phone, ehrms_id]): - ReportServiceV2.logger.info("No user filters provided for fetching user data.") - return None - - bigquery_service = BigQueryService() - additional_filters = additional_filters or {} - - # Build filters for user details - user_filters = [] - if email: - user_filters.append(f"email = '{email}'") - if phone: - user_filters.append(f"phone_number = '{phone}'") - if ehrms_id: - user_filters.append(f"external_system_id = '{ehrms_id}'") - + # Build and validate user filters + user_filters = ReportServiceV2._build_user_filters(email, phone, ehrms_id) if not user_filters: - ReportServiceV2.logger.info("No valid filters provided for fetching user data.") + ReportServiceV2.logger.info("No valid user filters provided.") return None - # Construct the query for fetching user data - user_filter_query = ' AND '.join(user_filters) - user_query = f""" - SELECT user_id, mdo_id - FROM `{MASTER_USER_TABLE}` - WHERE {user_filter_query} - """ - - ReportServiceV2.logger.info(f"Executing user query: {user_query}") - user_df = bigquery_service.run_query(user_query) + # Query user data + bigquery_service = BigQueryService() + user_query = ReportServiceV2._build_query( + MASTER_USER_TABLE, + user_filters, + is_select_all=False + ) + user_df = ReportServiceV2._execute_query(bigquery_service, user_query, "user") if user_df.empty: ReportServiceV2.logger.info("No users found matching the provided filters.") return None + # Validate organization ID + user_mdo_id = user_df["mdo_id"].iloc[0] + ReportServiceV2._validate_org_id(bigquery_service, user_mdo_id, org_id) + + # Build enrollment filters user_ids = user_df["user_id"].tolist() - ReportServiceV2.logger.info(f"Fetched {len(user_ids)} users.") - - # Get the user's MDO ID - user_mdo_id = user_df["mdo_id"].iloc[0] # Get the first user's MDO ID - - # Check if organization ID is valid - if orgId and orgId != user_mdo_id: - mdo_id_org_list = list(ReportService._get_mdo_id_org_list(bigquery_service, orgId)) - mdo_id_org_list.append(orgId) # Include the orgId itself - - if user_mdo_id not in mdo_id_org_list: - ReportServiceV2.logger.error(f"Invalid organization ID for user: {orgId}") - raise ValueError(f"Invalid organization ID for user: {orgId}") - - # Build filters for enrollment data - # Fix the string formatting issue by breaking it down into simpler steps user_ids_quoted = [f"'{uid}'" for uid in user_ids] - user_ids_str = ", ".join(user_ids_quoted) - where_clause_parts = [f"user_id IN ({user_ids_str})"] + where_clause_parts = [f"user_id IN ({', '.join(user_ids_quoted)})"] - # Add date filtering if start_date and end_date: - where_clause_parts.append(f"enrolled_on BETWEEN '{start_date}' AND '{end_date}'") + where_clause_parts.append(build_date_filter(start_date, end_date, "enrolled_on")) - # Process additional filters - where_clause_parts = ReportServiceV2._process_filters(additional_filters, USER_REPORT_FILTER_CONFIG, where_clause_parts) - - # Construct the WHERE clause - where_clause = " AND ".join(where_clause_parts) + # Add additional filters + where_clause_parts = ReportServiceV2._process_filters( + additional_filters or {}, + USER_REPORT_FILTER_CONFIG, + where_clause_parts + ) - enrollment_query = f""" - SELECT * - FROM `{MASTER_ENROLMENTS_TABLE}` - WHERE {where_clause} - """ - - ReportServiceV2.logger.info(f"Executing enrollment query: {enrollment_query}") - enrollment_df = bigquery_service.run_query(enrollment_query) + # Query enrollments + enrollment_query = ReportServiceV2._build_query( + MASTER_ENROLMENTS_TABLE, + where_clause_parts + ) + enrollment_df = ReportServiceV2._execute_query( + bigquery_service, + enrollment_query, + "enrollment" + ) if enrollment_df.empty: ReportServiceV2.logger.info("No enrollment data found for the given user and filters.") return None - # Filter columns if specified - if required_columns: - existing_columns = [col for col in required_columns if col in enrollment_df.columns] - missing_columns = list(set(required_columns) - set(existing_columns)) - if missing_columns: - ReportServiceV2.logger.info(f"Warning: Missing columns skipped: {missing_columns}") - merged_df = enrollment_df[existing_columns] - else: - merged_df = enrollment_df - - def generate_csv_stream(df, cols): - try: - yield '|'.join(cols) + '\n' - for row in df.itertuples(index=False, name=None): - yield '|'.join(map(str, row)) + '\n' - finally: - # Safe cleanup after generator is fully consumed - df.drop(df.index, inplace=True) - del df - gc.collect() - ReportServiceV2.logger.info("Cleaned up DataFrame after streaming.") - - ReportServiceV2.logger.info(f"CSV stream generated with {len(merged_df)} rows.") - - # Return CSV content without closing the stream - return generate_csv_stream(merged_df, merged_df.columns.tolist()) + # Process and return results + result_df = filter_required_columns(enrollment_df, required_columns) + return ReportServiceV2._generate_csv_stream(result_df, result_df.columns.tolist()) - except MemoryError as me: + except MemoryError: ReportServiceV2.logger.error("MemoryError encountered. Consider processing data in smaller chunks.") raise except Exception as e: ReportServiceV2.logger.error(f"Error generating user report: {e}") raise + @staticmethod + def _mask_sensitive_data(row_dict): + """Mask sensitive data in user reports""" + if IS_MASKING_ENABLED.lower() != 'true': + return row_dict + + masked_dict = row_dict.copy() + + # Mask email + if 'email' in masked_dict and masked_dict['email']: + parts = masked_dict['email'].split('@') + if len(parts) == 2: + domain_parts = parts[1].split('.') + masked_domain = '.'.join(['*' * len(part) for part in domain_parts]) + masked_dict['email'] = f"{parts[0]}@{masked_domain}" + else: + masked_dict['email'] = parts[0] + + # Mask phone number + if 'phone_number' in masked_dict and masked_dict['phone_number']: + phone = str(masked_dict['phone_number']) + if len(phone) >= 4: + masked_dict['phone_number'] = '*' * (len(phone) - 4) + phone[-4:] + else: + masked_dict['phone_number'] = '*' * len(phone) + + return masked_dict + @staticmethod def generate_org_user_report(mdo_id, is_full_report_required, required_columns=None, user_creation_start_date=None, user_creation_end_date=None, additional_filters=None): """ @@ -335,110 +367,62 @@ def generate_org_user_report(mdo_id, is_full_report_required, required_columns=N """ try: bigquery_service = BigQueryService() - additional_filters = additional_filters or {} - - # Build filters where_clause_parts = [] # Add date filtering if user_creation_start_date and user_creation_end_date: - where_clause_parts.append(f"user_registration_date BETWEEN '{user_creation_start_date}' AND '{user_creation_end_date}'") + where_clause_parts.append( + build_date_filter(user_creation_start_date, user_creation_end_date, "user_registration_date") + ) # Handle MDO ID filtering - mdo_id_list = additional_filters.get('mdo_id_list', []) - if mdo_id_list and isinstance(mdo_id_list, list) and len(mdo_id_list) > 0: - # If specific MDO IDs are provided, use those - ReportServiceV2.logger.info(f"Using provided MDO ID list: {mdo_id_list}") - # Fetch the valid MDO IDs from the hierarchy - mdo_id_org_list = list(ReportService._get_mdo_id_org_list(bigquery_service, mdo_id)) - - # Filter out invalid MDO IDs - mdo_id_list = [mid for mid in mdo_id_list if mid in mdo_id_org_list] - ReportServiceV2.logger.info(f"Filtered MDO ID list: {mdo_id_list}") - if mdo_id not in mdo_id_list: - mdo_id_list.append(mdo_id) - mdo_ids_to_use = [f"'{mid}'" for mid in mdo_id_list] - else: - # Otherwise use the standard logic based on is_full_report_required - if is_full_report_required: - # Dynamically fetch orgs using hierarchy - mdo_id_org_list = list(ReportService._get_mdo_id_org_list(bigquery_service, mdo_id)) - mdo_id_org_list.append(mdo_id) # Add input mdo_id to the set - ReportServiceV2.logger.debug(f"Fetched {len(mdo_id_org_list)} MDO IDs (including input): {mdo_id_org_list}") - else: - mdo_id_org_list = [mdo_id] - ReportServiceV2.logger.info(f"Full report not required. Using single mdo_id: {mdo_id}") - - mdo_ids_to_use = [f"'{mid}'" for mid in mdo_id_org_list] - - mdo_id_str = ', '.join(mdo_ids_to_use) - where_clause_parts.append(f"mdo_id in ({mdo_id_str})") + mdo_ids = ReportServiceV2._handle_mdo_ids( + bigquery_service, + mdo_id, + is_full_report_required, + (additional_filters or {}).get('mdo_id_list', []) + ) + where_clause_parts.append(f"mdo_id IN ({', '.join(mdo_ids)})") + # Remove date filters from additional filters to avoid conflicts if user_creation_start_date and user_creation_end_date and additional_filters: - additional_filters = {key: value for key, value in additional_filters.items() if not key.startswith("user_registration_date")} + additional_filters = { + key: value for key, value in additional_filters.items() + if not key.startswith("user_registration_date") + } # Process additional filters - where_clause_parts = ReportServiceV2._process_filters(additional_filters, USER_FILTER_CONFIG, where_clause_parts) + where_clause_parts = ReportServiceV2._process_filters( + additional_filters or {}, + USER_FILTER_CONFIG, + where_clause_parts + ) - # Construct the WHERE clause - where_clause = " AND ".join(where_clause_parts) + # Build and execute query + query = ReportServiceV2._build_query(MASTER_USER_TABLE, where_clause_parts) + result_df = ReportServiceV2._execute_query(bigquery_service, query, "from master_user_data") - query = f""" - SELECT * - FROM `{MASTER_USER_TABLE}` - WHERE {where_clause} - """ - - ReportServiceV2.logger.info(f"Executing user query: {query}") - result_df = bigquery_service.run_query(query) - if result_df.empty: - ReportServiceV2.logger.info("No data found for the given filters.") return None - - ReportServiceV2.logger.info(f"Fetched {len(result_df)} rows from master_user_data.") - - # Filter the result DataFrame to include only the required columns - if required_columns: - existing_columns = [col for col in required_columns if col in result_df.columns] - missing_columns = list(set(required_columns) - set(existing_columns)) - if missing_columns: - ReportServiceV2.logger.info(f"Warning: Missing columns skipped: {missing_columns}") - result_df = result_df[existing_columns] - - # Generate CSV stream from the result DataFrame - def generate_csv_stream(df, cols): + + # Process results with masking + result_df = filter_required_columns(result_df, required_columns) + + def generate_masked_csv_stream(df, cols): try: yield '|'.join(cols) + '\n' for row in df.itertuples(index=False, name=None): row_dict = dict(zip(cols, row)) - if IS_MASKING_ENABLED.lower() == 'true': - # Mask email - if 'email' in row_dict and row_dict['email']: - parts = row_dict['email'].split('@') - if len(parts) == 2: - domain_parts = parts[1].split('.') - masked_domain = '.'.join(['*' * len(part) for part in domain_parts]) - row_dict['email'] = f"{parts[0]}@{masked_domain}" - else: - row_dict['email'] = parts[0] - - # Mask phone number: e.g., ******2245 - if 'phone_number' in row_dict and row_dict['phone_number']: - phone = str(row_dict['phone_number']) - if len(phone) >= 4: - row_dict['phone_number'] = '*' * (len(phone) - 4) + phone[-4:] - else: - row_dict['phone_number'] = '*' * len(phone) - - # Convert back to row and yield - yield '|'.join([str(row_dict.get(col, '')) for col in cols]) + '\n' + masked_dict = ReportServiceV2._mask_sensitive_data(row_dict) + yield '|'.join([str(masked_dict.get(col, '')) for col in cols]) + '\n' finally: df.drop(df.index, inplace=True) del df gc.collect() ReportServiceV2.logger.info("Cleaned up DataFrame after streaming.") + return generate_masked_csv_stream(result_df, result_df.columns.tolist()) + ReportServiceV2.logger.info(f"CSV stream generated with {len(result_df)} rows.") return generate_csv_stream(result_df, result_df.columns.tolist()) diff --git a/constants.py b/constants.py index a4a7135..a47cf9d 100644 --- a/constants.py +++ b/constants.py @@ -83,4 +83,9 @@ "external_system_id": "external_system_id" } """)) -APAR_FILTER_KEY = os.environ.get("APAR_FILTER_KEY", "user_email,mobile_no,parichay_id,external_system_id") \ No newline at end of file +APAR_FILTER_KEY = os.environ.get("APAR_FILTER_KEY", "user_email,mobile_no,parichay_id,external_system_id") +TEXT_CSV_HOLDER = 'text/csv' +MALLOC_TRIM_HOLDER_MSG = 'inside malloc_trim' +LIBC_SO_6 = 'libc.so.6' +AND = 'AND' +CLEANUP_MESSAGE = "Cleaned up DataFrame after streaming." \ No newline at end of file diff --git a/errormsg.py b/errormsg.py new file mode 100644 index 0000000..d7c67ae --- /dev/null +++ b/errormsg.py @@ -0,0 +1,6 @@ +REQUEST_BODY_MISSING_ERROR = "Request body is missing" +UNEXPECTED_ERROR_OCCURRED = "An unexpected error occurred. Please try again later." +MALLOC_TRIM_HOLDER_ERROR_MSG = "malloc_trim failed: %s" +MISSING_X_ORG_ID_HEADER = "Missing 'x_org_id' in headers." +ORGANIZATION_ID_REQUIRED_ERROR = "Organization ID is required." +INVALID_DATE_FORMAT_ERROR = "Invalid date format. Use YYYY-MM-DD." diff --git a/tests/controllers/test_report_controller.py b/tests/controllers/test_report_controller.py index 39263d8..7908c6d 100644 --- a/tests/controllers/test_report_controller.py +++ b/tests/controllers/test_report_controller.py @@ -28,7 +28,7 @@ def test_get_report_success(mock_report_service, client): """Test successful report generation.""" # Setup mock_report_service.fetch_master_enrolments_data.return_value = iter(["header\n", "data1\n", "data2\n"]) - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -47,8 +47,8 @@ def test_get_report_success(mock_report_service, client): assert response.mimetype == 'text/csv' assert 'attachment; filename="report_org123.csv"' in response.headers['Content-Disposition'] - # Check that isValidOrg was called with correct parameters - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + # Check that is_valid_org was called with correct parameters + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') # Check that the service was called with correct parameters mock_report_service.fetch_master_enrolments_data.assert_called_once() @@ -63,7 +63,7 @@ def test_get_report_success(mock_report_service, client): def test_get_report_unauthorized_org_id(mock_report_service, client): """Test report generation with unauthorized organization ID.""" # Setup - mock_report_service.isValidOrg.return_value = False + mock_report_service.is_valid_org.return_value = False # Execute response = client.post( @@ -80,7 +80,7 @@ def test_get_report_unauthorized_org_id(mock_report_service, client): assert response.status_code == 401 data = response.get_json() assert 'Not authorized to view the report for' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') # Ensure the service method was not called mock_report_service.fetch_master_enrolments_data.assert_not_called() @@ -105,7 +105,7 @@ def test_get_report_missing_x_org_id(mock_report_service, client): data = response.get_json() assert 'Organization ID is required' in data['error'] # Ensure the service methods were not called - mock_report_service.isValidOrg.assert_not_called() + mock_report_service.is_valid_org.assert_not_called() mock_report_service.fetch_master_enrolments_data.assert_not_called() @@ -117,7 +117,7 @@ def test_get_report_with_auth_success(mock_validator, mock_report_service, clien # Setup mock_validator.verify_user_token_get_org.return_value = "org123" mock_report_service.fetch_master_enrolments_data.return_value = iter(["header\n", "data1\n", "data2\n"]) - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -135,7 +135,7 @@ def test_get_report_with_auth_success(mock_validator, mock_report_service, clien # Verify assert response.status_code == 200 mock_validator.verify_user_token_get_org.assert_called_once_with('valid-token', True) - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.IS_VALIDATION_ENABLED", "true") @@ -143,7 +143,7 @@ def test_get_report_with_auth_success(mock_validator, mock_report_service, clien def test_get_report_missing_auth_token(mock_report_service, client): """Test report generation with missing auth token.""" # Setup - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -159,7 +159,7 @@ def test_get_report_missing_auth_token(mock_report_service, client): assert response.status_code == 401 data = response.get_json() assert 'Authentication token is required' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.IS_VALIDATION_ENABLED", "true") @@ -169,7 +169,7 @@ def test_get_report_invalid_auth_token(mock_report_service, mock_validator, clie """Test report generation with invalid auth token.""" # Setup mock_validator.verify_user_token_get_org.return_value = "" - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -188,7 +188,7 @@ def test_get_report_invalid_auth_token(mock_report_service, mock_validator, clie assert response.status_code == 401 data = response.get_json() assert 'Invalid or expired authentication token' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.IS_VALIDATION_ENABLED", "true") @@ -198,7 +198,7 @@ def test_get_report_unauthorized_org(mock_report_service, mock_validator, client """Test report generation with unauthorized organization.""" # Setup mock_validator.verify_user_token_get_org.return_value = "org456" - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -217,14 +217,14 @@ def test_get_report_unauthorized_org(mock_report_service, mock_validator, client assert response.status_code == 403 data = response.get_json() assert 'Access denied for the specified organization ID' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org789', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org789', 'org123') @patch("app.controllers.report_controller.ReportService") def test_get_report_missing_dates(mock_report_service, client): """Test report generation with missing date parameters.""" # Setup - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post('/report/org/enrolment/org123', json={}, headers={'x-org-id': 'org456'}) @@ -233,14 +233,14 @@ def test_get_report_missing_dates(mock_report_service, client): assert response.status_code == 400 data = response.get_json() assert 'Invalid input' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") def test_get_report_invalid_date_format(mock_report_service, client): """Test report generation with invalid date format.""" # Setup - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -256,14 +256,14 @@ def test_get_report_invalid_date_format(mock_report_service, client): assert response.status_code == 400 data = response.get_json() assert 'Invalid date format' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") def test_get_report_date_range_too_long(mock_report_service, client): """Test report generation with date range exceeding 1 year.""" # Setup - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -279,7 +279,7 @@ def test_get_report_date_range_too_long(mock_report_service, client): assert response.status_code == 400 data = response.get_json() assert 'Date range cannot exceed 1 year' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") @@ -288,7 +288,7 @@ def test_get_report_no_data(mock_report_service, client): """Test report generation with no data found.""" # Setup mock_report_service.fetch_master_enrolments_data.return_value = None - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -304,7 +304,7 @@ def test_get_report_no_data(mock_report_service, client): assert response.status_code == 404 data = response.get_json() assert 'No data found' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") @@ -313,7 +313,7 @@ def test_get_report_service_error(mock_report_service, client): """Test report generation with service error.""" # Setup mock_report_service.fetch_master_enrolments_data.side_effect = Exception("Service error") - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -329,7 +329,7 @@ def test_get_report_service_error(mock_report_service, client): assert response.status_code == 500 data = response.get_json() assert 'Failed to generate the report' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") @@ -337,7 +337,7 @@ def test_get_user_report_success(mock_report_service, client): """Test successful user report generation.""" # Setup mock_report_service.fetch_user_cumulative_report.return_value = iter(["header\n", "data1\n", "data2\n"]) - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -356,8 +356,8 @@ def test_get_user_report_success(mock_report_service, client): assert response.mimetype == 'text/csv' assert 'attachment; filename="user-report.csv"' in response.headers['Content-Disposition'] - # Check that isValidOrg was called with correct parameters - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + # Check that is_valid_org was called with correct parameters + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') # Check that the service was called with correct parameters mock_report_service.fetch_user_cumulative_report.assert_called_once() @@ -370,7 +370,7 @@ def test_get_user_report_success(mock_report_service, client): def test_get_user_report_unauthorized_org_id(mock_report_service, client): """Test user report generation with unauthorized organization ID.""" # Setup - mock_report_service.isValidOrg.return_value = False + mock_report_service.is_valid_org.return_value = False # Execute response = client.post( @@ -387,7 +387,7 @@ def test_get_user_report_unauthorized_org_id(mock_report_service, client): assert response.status_code == 401 data = response.get_json() assert 'Not authorized to view the report for' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') # Ensure the service method was not called mock_report_service.fetch_user_cumulative_report.assert_not_called() @@ -396,7 +396,7 @@ def test_get_user_report_unauthorized_org_id(mock_report_service, client): def test_get_user_report_missing_user_identifiers(mock_report_service, client): """Test user report generation with missing user identifiers.""" # Setup - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -412,28 +412,28 @@ def test_get_user_report_missing_user_identifiers(mock_report_service, client): assert response.status_code == 400 data = response.get_json() assert 'At least one of' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") def test_get_user_report_missing_body(mock_report_service, client): """Test user report generation with missing request body.""" # Setup - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post('/report/user/sync/org123', json=None, headers={'x-org-id': 'org456'}) # Verify assert response.status_code == 500 - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") def test_get_user_report_invalid_date_format(mock_report_service, client): """Test user report generation with invalid date format.""" # Setup - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -450,7 +450,7 @@ def test_get_user_report_invalid_date_format(mock_report_service, client): assert response.status_code == 400 data = response.get_json() assert 'Invalid date format' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") @@ -458,7 +458,7 @@ def test_get_user_report_no_data(mock_report_service, client): """Test user report generation with no data found.""" # Setup mock_report_service.fetch_user_cumulative_report.return_value = None - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -473,7 +473,7 @@ def test_get_user_report_no_data(mock_report_service, client): assert response.status_code == 404 data = response.get_json() assert 'No data found' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") @@ -481,7 +481,7 @@ def test_get_user_report_with_phone(mock_report_service, client): """Test user report generation with phone number.""" # Setup mock_report_service.fetch_user_cumulative_report.return_value = iter(["header\n", "data1\n", "data2\n"]) - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -496,7 +496,7 @@ def test_get_user_report_with_phone(mock_report_service, client): # Verify assert response.status_code == 200 - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') mock_report_service.fetch_user_cumulative_report.assert_called_once() args, kwargs = mock_report_service.fetch_user_cumulative_report.call_args assert args[1] == '1234567890' # phone @@ -507,7 +507,7 @@ def test_get_user_report_with_ehrms_id(mock_report_service, client): """Test user report generation with EHRMS ID.""" # Setup mock_report_service.fetch_user_cumulative_report.return_value = iter(["header\n", "data1\n", "data2\n"]) - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -522,7 +522,7 @@ def test_get_user_report_with_ehrms_id(mock_report_service, client): # Verify assert response.status_code == 200 - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') mock_report_service.fetch_user_cumulative_report.assert_called_once() args, kwargs = mock_report_service.fetch_user_cumulative_report.call_args assert args[2] == 'EMP123' # ehrmsId @@ -533,7 +533,7 @@ def test_get_user_report_with_whitespace(mock_report_service, client): """Test user report generation with whitespace in identifiers.""" # Setup mock_report_service.fetch_user_cumulative_report.return_value = iter(["header\n", "data1\n", "data2\n"]) - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -548,7 +548,7 @@ def test_get_user_report_with_whitespace(mock_report_service, client): # Verify assert response.status_code == 200 - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') mock_report_service.fetch_user_cumulative_report.assert_called_once() args, kwargs = mock_report_service.fetch_user_cumulative_report.call_args assert args[0] == 'test@example.com' # email (trimmed) @@ -561,7 +561,7 @@ def test_get_user_report_service_error(mock_report_service, client): """Test user report generation with service error.""" # Setup mock_report_service.fetch_user_cumulative_report.side_effect = Exception("Service error") - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -576,7 +576,7 @@ def test_get_user_report_service_error(mock_report_service, client): assert response.status_code == 500 data = response.get_json() assert 'Failed to generate the report' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") @@ -584,7 +584,7 @@ def test_get_org_user_report_success(mock_report_service, client): """Test successful organization user report generation.""" # Setup mock_report_service.fetch_master_user_data.return_value = iter(["header\n", "data1\n", "data2\n"]) - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -603,8 +603,8 @@ def test_get_org_user_report_success(mock_report_service, client): assert response.mimetype == 'text/csv' assert 'attachment; filename="user-report.csv"' in response.headers['Content-Disposition'] - # Check that isValidOrg was called with correct parameters - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + # Check that is_valid_org was called with correct parameters + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') # Check that the service was called with correct parameters mock_report_service.fetch_master_user_data.assert_called_once() @@ -620,7 +620,7 @@ def test_get_org_user_report_success(mock_report_service, client): def test_get_org_user_report_unauthorized_org_id(mock_report_service, client): """Test organization user report generation with unauthorized organization ID.""" # Setup - mock_report_service.isValidOrg.return_value = False + mock_report_service.is_valid_org.return_value = False # Execute response = client.post( @@ -637,7 +637,7 @@ def test_get_org_user_report_unauthorized_org_id(mock_report_service, client): assert response.status_code == 401 data = response.get_json() assert 'Not authorized to view the report for' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') # Ensure the service method was not called mock_report_service.fetch_master_user_data.assert_not_called() @@ -647,7 +647,7 @@ def test_get_org_user_report_no_data(mock_report_service, client): """Test organization user report generation with no data found.""" # Setup mock_report_service.fetch_master_user_data.return_value = None - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -660,7 +660,7 @@ def test_get_org_user_report_no_data(mock_report_service, client): assert response.status_code == 404 data = response.get_json() assert 'No data found' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") @@ -668,7 +668,7 @@ def test_get_org_user_report_service_error(mock_report_service, client): """Test organization user report generation with service error.""" # Setup mock_report_service.fetch_master_user_data.side_effect = Exception("Service error") - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -681,28 +681,28 @@ def test_get_org_user_report_service_error(mock_report_service, client): assert response.status_code == 500 data = response.get_json() assert 'Failed to generate the report' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") def test_get_org_user_report_missing_body(mock_report_service, client): """Test organization user report generation with missing request body.""" # Setup - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post('/report/org/user/org123', json=None, headers={'x-org-id': 'org456'}) # Verify assert response.status_code == 500 - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") def test_get_org_user_report_invalid_date_format(mock_report_service, client): """Test organization user report generation with invalid date format.""" # Setup - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -718,7 +718,7 @@ def test_get_org_user_report_invalid_date_format(mock_report_service, client): assert response.status_code == 400 data = response.get_json() assert 'Invalid date format' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.GcsToBigQuerySyncService") @@ -763,7 +763,7 @@ def test_get_report_with_malloc_trim(mock_report_service, mock_cdll, client): """Test report generation with malloc_trim call.""" # Setup mock_report_service.fetch_master_enrolments_data.return_value = iter(["header\n", "data1\n", "data2\n"]) - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True mock_libc = MagicMock() mock_cdll.return_value = mock_libc @@ -779,7 +779,7 @@ def test_get_report_with_malloc_trim(mock_report_service, mock_cdll, client): # Verify assert response.status_code == 200 - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') mock_cdll.assert_called_with("libc.so.6") mock_libc.malloc_trim.assert_called_with(0) @@ -790,7 +790,7 @@ def test_get_user_report_with_malloc_trim(mock_report_service, mock_cdll, client """Test user report generation with malloc_trim call.""" # Setup mock_report_service.fetch_user_cumulative_report.return_value = iter(["header\n", "data1\n", "data2\n"]) - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True mock_libc = MagicMock() mock_cdll.return_value = mock_libc @@ -805,7 +805,7 @@ def test_get_user_report_with_malloc_trim(mock_report_service, mock_cdll, client # Verify assert response.status_code == 200 - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') mock_cdll.assert_called_with("libc.so.6") mock_libc.malloc_trim.assert_called_with(0) @@ -816,7 +816,7 @@ def test_get_org_user_report_with_malloc_trim(mock_report_service, mock_cdll, cl """Test organization user report generation with malloc_trim call.""" # Setup mock_report_service.fetch_master_user_data.return_value = iter(["header\n", "data1\n", "data2\n"]) - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True mock_libc = MagicMock() mock_cdll.return_value = mock_libc @@ -829,7 +829,7 @@ def test_get_org_user_report_with_malloc_trim(mock_report_service, mock_cdll, cl # Verify assert response.status_code == 200 - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') mock_cdll.assert_called_with("libc.so.6") mock_libc.malloc_trim.assert_called_with(0) @@ -840,7 +840,7 @@ def test_malloc_trim_exception(mock_report_service, mock_cdll, client): """Test handling of malloc_trim exception.""" # Setup mock_report_service.fetch_master_enrolments_data.return_value = iter(["header\n", "data1\n", "data2\n"]) - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True mock_cdll.side_effect = Exception("malloc_trim error") # Execute @@ -855,7 +855,7 @@ def test_malloc_trim_exception(mock_report_service, mock_cdll, client): # Verify - should still return 200 as the malloc_trim exception is caught assert response.status_code == 200 - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") @@ -1029,7 +1029,7 @@ def test_get_report_file_not_found_error(mock_report_service, client): """Test report generation with FileNotFoundError.""" # Setup mock_report_service.fetch_master_enrolments_data.side_effect = FileNotFoundError("File not found") - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -1045,7 +1045,7 @@ def test_get_report_file_not_found_error(mock_report_service, client): assert response.status_code == 500 data = response.get_json() assert 'Failed to generate the report due to an error' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") @@ -1053,7 +1053,7 @@ def test_get_user_report_key_error(mock_report_service, client): """Test user report generation with KeyError.""" # Setup mock_report_service.fetch_user_cumulative_report.side_effect = KeyError("Missing key") - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -1068,7 +1068,7 @@ def test_get_user_report_key_error(mock_report_service, client): assert response.status_code == 500 data = response.get_json() assert 'Failed to generate the report due to an internal error' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") @@ -1076,7 +1076,7 @@ def test_get_org_user_report_key_error(mock_report_service, client): """Test organization user report generation with KeyError.""" # Setup mock_report_service.fetch_master_user_data.side_effect = KeyError("Missing key") - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -1089,7 +1089,7 @@ def test_get_org_user_report_key_error(mock_report_service, client): assert response.status_code == 500 data = response.get_json() assert 'Failed to generate the report due to an internal error' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") @@ -1119,7 +1119,7 @@ def test_get_apar_report_file_not_found_error(mock_report_service, client): def test_get_user_report_with_all_identifiers_empty_strings(mock_report_service, client): """Test user report generation with all identifiers as empty strings after trimming.""" # Setup - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -1136,14 +1136,14 @@ def test_get_user_report_with_all_identifiers_empty_strings(mock_report_service, assert response.status_code == 400 data = response.get_json() assert 'At least one of' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') @patch("app.controllers.report_controller.ReportService") def test_get_report_with_json_none(mock_report_service, client): """Test report generation with None JSON body.""" # Setup - mock_report_service.isValidOrg.return_value = True + mock_report_service.is_valid_org.return_value = True # Execute response = client.post( @@ -1157,4 +1157,4 @@ def test_get_report_with_json_none(mock_report_service, client): assert response.status_code == 500 data = response.get_json() assert 'An unexpected error occurred' in data['error'] - mock_report_service.isValidOrg.assert_called_once_with('org456', 'org123') \ No newline at end of file + mock_report_service.is_valid_org.assert_called_once_with('org456', 'org123') \ No newline at end of file diff --git a/tests/controllers/test_report_controller_v2.py b/tests/controllers/test_report_controller_v2.py index a32b614..8f39803 100644 --- a/tests/controllers/test_report_controller_v2.py +++ b/tests/controllers/test_report_controller_v2.py @@ -39,7 +39,7 @@ def test_get_report_success(mock_report_service, mock_report_service_v2, client) @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_report_unauthorized_org_id(mock_is_valid_org, mock_report_service_v2, client): """Dummy test for unauthorized organization ID.""" @@ -50,7 +50,7 @@ def test_get_report_unauthorized_org_id(mock_is_valid_org, mock_report_service_v from unittest.mock import patch @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", new="false") def test_get_report_missing_x_org_id(mock_is_valid_org, mock_report_service_v2, client): """Test report generation with missing x-org-id header.""" @@ -84,7 +84,7 @@ def test_get_report_missing_x_org_id(mock_is_valid_org, mock_report_service_v2, @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "true") @patch("app.controllers.report_controller_v2.AccessTokenValidator") def test_get_report_with_auth_success(mock_validator, mock_is_valid_org, mock_report_service_v2, client): @@ -114,7 +114,7 @@ def test_get_report_with_auth_success(mock_validator, mock_is_valid_org, mock_re @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "true") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") def test_get_report_missing_auth_token(mock_is_valid_org, client): """Test report generation with missing auth token.""" # Setup @@ -139,7 +139,7 @@ def test_get_report_missing_auth_token(mock_is_valid_org, client): @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "true") @patch("app.controllers.report_controller_v2.AccessTokenValidator") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") def test_get_report_invalid_auth_token(mock_is_valid_org, mock_validator, client): """Test report generation with invalid auth token.""" # Setup @@ -168,7 +168,7 @@ def test_get_report_invalid_auth_token(mock_is_valid_org, mock_validator, client @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "true") @patch("app.controllers.report_controller_v2.AccessTokenValidator") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") def test_get_report_unauthorized_org(mock_is_valid_org, mock_validator, client): """Test report generation with unauthorized organization.""" # Setup @@ -195,7 +195,7 @@ def test_get_report_unauthorized_org(mock_is_valid_org, mock_validator, client): mock_is_valid_org.assert_called_once_with('org789', 'org123') -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") def test_get_report_missing_dates(mock_is_valid_org, client): """Test report generation with missing date parameters.""" # Setup @@ -210,7 +210,7 @@ def test_get_report_missing_dates(mock_is_valid_org, client): assert 'Request body is missing' in data['error'] -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") def test_get_report_invalid_date_format(mock_is_valid_org, client): """Test report generation with invalid date format.""" # Setup @@ -232,7 +232,7 @@ def test_get_report_invalid_date_format(mock_is_valid_org, client): assert 'Invalid Request.' in data['error'] -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") def test_get_report_missing_required_fields(mock_is_valid_org, client): """Test report generation with missing required fields.""" # Setup @@ -253,7 +253,7 @@ def test_get_report_missing_required_fields(mock_is_valid_org, client): assert 'Invalid input' in data['error'] -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_report_date_range_exceeds_limit(mock_is_valid_org, client): """Test report generation with date range exceeding 1 year.""" @@ -279,7 +279,7 @@ def test_get_report_date_range_exceeds_limit(mock_is_valid_org, client): @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_report_no_data_found(mock_is_valid_org, mock_report_service_v2, client): """Test report generation when no data is found.""" @@ -305,7 +305,7 @@ def test_get_report_no_data_found(mock_is_valid_org, mock_report_service_v2, cli @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_report_service_exception(mock_is_valid_org, mock_report_service_v2, client): """Test report generation when service throws an exception.""" @@ -331,7 +331,7 @@ def test_get_report_service_exception(mock_is_valid_org, mock_report_service_v2, @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_report_file_not_found(mock_is_valid_org, mock_report_service_v2, client): """Test report generation when file is not found.""" @@ -358,7 +358,7 @@ def test_get_report_file_not_found(mock_is_valid_org, mock_report_service_v2, cl # Tests for user report endpoint @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_user_report_success(mock_is_valid_org, mock_report_service_v2, client): """Test successful user report generation.""" @@ -390,11 +390,11 @@ def test_get_user_report_success(mock_is_valid_org, mock_report_service_v2, clie mock_report_service_v2.generate_user_report.assert_called_once() args, kwargs = mock_report_service_v2.generate_user_report.call_args assert kwargs['email'] == 'user@example.com' - assert kwargs['orgId'] == 'org123' + assert kwargs['org_id'] == 'org123' @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_user_report_with_phone(mock_is_valid_org, mock_report_service_v2, client): """Test user report generation with phone number.""" @@ -421,7 +421,7 @@ def test_get_user_report_with_phone(mock_is_valid_org, mock_report_service_v2, c @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_user_report_with_ehrms_id(mock_is_valid_org, mock_report_service_v2, client): """Test user report generation with EHRMS ID.""" @@ -448,7 +448,7 @@ def test_get_user_report_with_ehrms_id(mock_is_valid_org, mock_report_service_v2 @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_user_report_missing_user_identifiers(mock_is_valid_org, mock_report_service_v2, client): """Test user report generation with missing user identifiers.""" @@ -472,7 +472,7 @@ def test_get_user_report_missing_user_identifiers(mock_is_valid_org, mock_report @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_user_report_invalid_date_format(mock_is_valid_org, mock_report_service_v2, client): """Test user report generation with invalid date format.""" @@ -498,7 +498,7 @@ def test_get_user_report_invalid_date_format(mock_is_valid_org, mock_report_serv # Tests for org user report endpoint @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_org_user_report_success(mock_is_valid_org, mock_report_service_v2, client): """Test successful organization user report generation.""" @@ -536,7 +536,7 @@ def test_get_org_user_report_success(mock_is_valid_org, mock_report_service_v2, @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_org_user_report_invalid_date_format(mock_is_valid_org, mock_report_service_v2, client): """Test org user report generation with invalid date format.""" @@ -585,7 +585,7 @@ def test_get_org_user_report_no_data_found(mock_is_valid_org, mock_report_servic @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_org_user_report_service_exception(mock_is_valid_org, mock_report_service_v2, client): """Test org user report generation when service throws an exception.""" @@ -609,7 +609,7 @@ def test_get_org_user_report_service_exception(mock_is_valid_org, mock_report_se assert 'Failed to generate the report' in data['error'] @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_report_general_exception(mock_is_valid_org, mock_report_service_v2, client): """Test report generation when a general exception occurs.""" @@ -635,7 +635,7 @@ def test_get_report_general_exception(mock_is_valid_org, mock_report_service_v2, @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") @patch("app.controllers.report_controller_v2.ctypes.CDLL") def test_get_report_malloc_trim_exception(mock_cdll, mock_is_valid_org, mock_report_service_v2, client): @@ -664,7 +664,7 @@ def test_get_report_malloc_trim_exception(mock_cdll, mock_is_valid_org, mock_rep @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_user_report_service_exception(mock_is_valid_org, mock_report_service_v2, client): """Dummy test for user report service exception.""" @@ -672,14 +672,14 @@ def test_get_user_report_service_exception(mock_is_valid_org, mock_report_servic @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_org_user_report_no_request_body(mock_is_valid_org, mock_report_service_v2, client): assert True # Dummy assertion @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") @patch("app.controllers.report_controller_v2.ctypes.CDLL") def test_get_org_user_report_malloc_trim_exception(mock_cdll, mock_is_valid_org, mock_report_service_v2, client): @@ -707,7 +707,7 @@ def test_get_org_user_report_malloc_trim_exception(mock_cdll, mock_is_valid_org, @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_org_user_report_key_error(mock_is_valid_org, mock_report_service_v2, client): """Dummy test for org user report key error.""" @@ -715,7 +715,7 @@ def test_get_org_user_report_key_error(mock_is_valid_org, mock_report_service_v2 @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_report_with_invalid_json(mock_is_valid_org, mock_report_service_v2, client): """Test report generation with invalid JSON.""" @@ -738,7 +738,7 @@ def test_get_report_with_invalid_json(mock_is_valid_org, mock_report_service_v2, @patch("app.controllers.report_controller_v2.ReportServiceV2") -@patch("app.services.report_service.ReportService.isValidOrg") +@patch("app.services.report_service.ReportService.is_valid_org") @patch("app.controllers.report_controller_v2.IS_VALIDATION_ENABLED", "false") def test_get_report_with_empty_data(mock_is_valid_org, mock_report_service_v2, client): """Test report generation with empty data.""" diff --git a/tests/services/test_report_redis_integration.py b/tests/services/test_report_redis_integration.py index fc6dcb4..3d5e483 100644 --- a/tests/services/test_report_redis_integration.py +++ b/tests/services/test_report_redis_integration.py @@ -104,9 +104,9 @@ def test_get_mdo_id_org_list_empty_result(self): self.mock_bigquery_instance.run_query.assert_called_once() self.mock_redis_client.set.assert_called_once() - def test_isValidOrg_with_cache(self): + def test_is_valid_org_with_cache(self): """ - Test isValidOrg method using Redis cache + Test is_valid_org method using Redis cache """ # Arrange x_org_id = "parent_org" @@ -117,7 +117,7 @@ def test_isValidOrg_with_cache(self): self.mock_redis_client.get.return_value = '["child_org", "other_org"]' # Act - result = ReportService.isValidOrg(x_org_id, request_org_id) + result = ReportService.is_valid_org(x_org_id, request_org_id) # Assert self.assertTrue(result) @@ -125,9 +125,9 @@ def test_isValidOrg_with_cache(self): # BigQuery should not be called on cache hit self.mock_bigquery_instance.run_query.assert_not_called() - def test_isValidOrg_invalid_org(self): + def test_is_valid_org_invalid_org(self): """ - Test isValidOrg method with invalid organization + Test is_valid_org method with invalid organization """ # Arrange x_org_id = "parent_org" @@ -138,7 +138,7 @@ def test_isValidOrg_invalid_org(self): self.mock_redis_client.get.return_value = '["child_org", "other_org"]' # Act - result = ReportService.isValidOrg(x_org_id, request_org_id) + result = ReportService.is_valid_org(x_org_id, request_org_id) # Assert self.assertFalse(result) @@ -146,32 +146,32 @@ def test_isValidOrg_invalid_org(self): # BigQuery should not be called on cache hit self.mock_bigquery_instance.run_query.assert_not_called() - def test_isValidOrg_empty_request_org_id(self): + def test_is_valid_org_empty_request_org_id(self): """ - Test isValidOrg method with empty request_org_id + Test is_valid_org method with empty request_org_id """ # Arrange x_org_id = "parent_org" request_org_id = "" # Act - result = ReportService.isValidOrg(x_org_id, request_org_id) + result = ReportService.is_valid_org(x_org_id, request_org_id) # Assert self.assertFalse(result) # Redis should not be called with empty request_org_id self.mock_redis_client.get.assert_not_called() - def test_isValidOrg_empty_x_org_id(self): + def test_is_valid_org_empty_x_org_id(self): """ - Test isValidOrg method with empty x_org_id + Test is_valid_org method with empty x_org_id """ # Arrange x_org_id = "" request_org_id = "child_org" # Act - result = ReportService.isValidOrg(x_org_id, request_org_id) + result = ReportService.is_valid_org(x_org_id, request_org_id) # Assert self.assertFalse(result) diff --git a/tests/services/test_report_service.py b/tests/services/test_report_service.py index 22b2376..2d98f95 100644 --- a/tests/services/test_report_service.py +++ b/tests/services/test_report_service.py @@ -34,7 +34,7 @@ def test_fetch_user_cumulative_report_success(mock_bigquery_service): # Execute result = ReportService.fetch_user_cumulative_report( email='test@example.com', - orgId='org1' + org_id='org1' ) # Verify @@ -80,7 +80,7 @@ def test_fetch_user_cumulative_report_with_filters(mock_bigquery_service): ehrms_id='EMP123', start_date=start_date, end_date=end_date, - orgId='org1', + org_id='org1', required_columns=['user_id', 'course_id'] ) @@ -485,11 +485,11 @@ def test_get_mdo_id_org_list(mock_redis_service, mock_bigquery_service): @patch('app.services.report_service.ReportService._get_mdo_id_org_list') -def test_isValidOrg_success(mock_get_mdo_list, mock_bigquery_service): +def test_is_valid_org_success(mock_get_mdo_list, mock_bigquery_service): """ - Test that isValidOrg returns True when request_org_id is in the org_list. + Test that is_valid_org returns True when request_org_id is in the org_list. - This test verifies that the isValidOrg method correctly identifies + This test verifies that the is_valid_org method correctly identifies when a given request_org_id is present in the list of organizations associated with the x_org_id, including the x_org_id itself. """ @@ -502,7 +502,7 @@ def test_isValidOrg_success(mock_get_mdo_list, mock_bigquery_service): mock_get_mdo_list.return_value = mock_org_list # Execute - result = ReportService.isValidOrg(x_org_id, request_org_id) + result = ReportService.is_valid_org(x_org_id, request_org_id) # Verify assert result is True @@ -510,9 +510,9 @@ def test_isValidOrg_success(mock_get_mdo_list, mock_bigquery_service): @patch('app.services.report_service.ReportService._get_mdo_id_org_list') -def test_isValidOrg_failure(mock_get_mdo_list, mock_bigquery_service): +def test_is_valid_org_failure(mock_get_mdo_list, mock_bigquery_service): """ - Test that isValidOrg returns False when request_org_id is not in the org_list. + Test that is_valid_org returns False when request_org_id is not in the org_list. """ # Setup x_org_id = "org1" @@ -523,7 +523,7 @@ def test_isValidOrg_failure(mock_get_mdo_list, mock_bigquery_service): mock_get_mdo_list.return_value = mock_org_list # Execute - result = ReportService.isValidOrg(x_org_id, request_org_id) + result = ReportService.is_valid_org(x_org_id, request_org_id) # Verify assert result is False @@ -531,9 +531,9 @@ def test_isValidOrg_failure(mock_get_mdo_list, mock_bigquery_service): @patch('app.services.report_service.ReportService._get_mdo_id_org_list') -def test_isValidOrg_with_x_org_id(mock_get_mdo_list, mock_bigquery_service): +def test_is_valid_org_with_x_org_id(mock_get_mdo_list, mock_bigquery_service): """ - Test that isValidOrg returns True when request_org_id is the same as x_org_id. + Test that is_valid_org returns True when request_org_id is the same as x_org_id. """ # Setup x_org_id = "org1" @@ -544,7 +544,7 @@ def test_isValidOrg_with_x_org_id(mock_get_mdo_list, mock_bigquery_service): mock_get_mdo_list.return_value = mock_org_list # Execute - result = ReportService.isValidOrg(x_org_id, request_org_id) + result = ReportService.is_valid_org(x_org_id, request_org_id) # Verify assert result is True @@ -553,49 +553,49 @@ def test_isValidOrg_with_x_org_id(mock_get_mdo_list, mock_bigquery_service): @patch('app.services.report_service.BigQueryService') -def test_isValidOrg_exception_handling(mock_bigquery_service_class): +def test_is_valid_org_exception_handling(mock_bigquery_service_class): """ - Test that isValidOrg handles exceptions and returns False when an error occurs. + Test that is_valid_org handles exceptions and returns False when an error occurs. """ # Setup mock_bigquery_service_class.side_effect = Exception("Test exception") # Execute - result = ReportService.isValidOrg("org1", "org2") + result = ReportService.is_valid_org("org1", "org2") # Verify assert result is False mock_bigquery_service_class.assert_called_once() -def test_isValidOrg_missing_org_id(): +def test_is_valid_org_missing_org_id(): """ - Test that isValidOrg returns False when request_org_id is None or empty. + Test that is_valid_org returns False when request_org_id is None or empty. """ with patch('app.services.report_service.BigQueryService') as mock_bigquery: # Test with None - result1 = ReportService.isValidOrg("org1", None) + result1 = ReportService.is_valid_org("org1", None) assert result1 is False # Test with empty string - result2 = ReportService.isValidOrg("org1", "") + result2 = ReportService.is_valid_org("org1", "") assert result2 is False # Verify that BigQueryService was not called mock_bigquery.assert_not_called() -def test_isValidOrg_missing_x_org_id(): +def test_is_valid_org_missing_x_org_id(): """ - Test that isValidOrg returns False when x_org_id is None or empty. + Test that is_valid_org returns False when x_org_id is None or empty. """ with patch('app.services.report_service.BigQueryService') as mock_bigquery: # Test with None x_org_id - result1 = ReportService.isValidOrg(None, "org1") + result1 = ReportService.is_valid_org(None, "org1") assert result1 is False # Test with empty x_org_id - result2 = ReportService.isValidOrg("", "org1") + result2 = ReportService.is_valid_org("", "org1") assert result2 is False # Verify that BigQueryService was not called @@ -643,7 +643,7 @@ def test_fetch_user_cumulative_report_invalid_org_id(mock_bigquery_service): with pytest.raises(ValueError, match="Invalid organization ID for user"): ReportService.fetch_user_cumulative_report( email='test@example.com', - orgId='invalid_org' + org_id='invalid_org' ) @@ -719,7 +719,7 @@ def test_fetch_user_cumulative_report_1(mock_bigquery_service): ehrms_id = None start_date = None end_date = None - orgId = None + org_id = None required_columns = None # Execute @@ -729,7 +729,7 @@ def test_fetch_user_cumulative_report_1(mock_bigquery_service): ehrms_id=ehrms_id, start_date=start_date, end_date=end_date, - orgId=orgId, + org_id=org_id, required_columns=required_columns ) @@ -747,7 +747,7 @@ def test_fetch_user_cumulative_report_3(mock_bigquery_service): ehrms_id = "12345" start_date = "2023-01-01" end_date = "2023-12-31" - orgId = "org1" + org_id = "org1" required_columns = ["user_id", "course_id", "progress"] # Configure mock to return an empty DataFrame @@ -760,7 +760,7 @@ def test_fetch_user_cumulative_report_3(mock_bigquery_service): ehrms_id=ehrms_id, start_date=start_date, end_date=end_date, - orgId=orgId, + org_id=org_id, required_columns=required_columns ) @@ -785,7 +785,7 @@ def test_fetch_user_cumulative_report_5(mock_bigquery_service): ehrms_id='EMP123', start_date='2023-01-01', end_date='2023-12-31', - orgId='ORG1', + org_id='ORG1', required_columns=['user_id', 'course_id'] ) @@ -811,7 +811,7 @@ def test_fetch_user_cumulative_report_7(mock_bigquery_service): ehrms_id='EHRMS123', start_date='2023-01-01', end_date='2023-12-31', - orgId='org2', + org_id='org2', required_columns=['column1', 'column2'] ) @@ -836,7 +836,7 @@ def test_fetch_user_cumulative_report_8(mock_bigquery_service): ehrms_id="EMP123", start_date="2023-01-01", end_date="2023-12-31", - orgId="ORG1" + org_id="ORG1" ) # Verify @@ -848,7 +848,7 @@ def test_fetch_user_cumulative_report_9(mock_bigquery_service): Test fetch_user_cumulative_report with specific conditions: - email, phone, and ehrms_id are provided - user is found - - orgId is not in user_mdo_id + - org_id is not in user_mdo_id - user_mdo_id is in mdo_id_org_list - start_date and end_date are not provided - enrollment data is empty @@ -865,7 +865,7 @@ def test_fetch_user_cumulative_report_9(mock_bigquery_service): email='user@example.com', phone='1234567890', ehrms_id='EHRMS123', - orgId='mdo2' + org_id='mdo2' ) # Verify diff --git a/tests/services/test_report_service_v2.py b/tests/services/test_report_service_v2.py index 7c749dd..5a91730 100644 --- a/tests/services/test_report_service_v2.py +++ b/tests/services/test_report_service_v2.py @@ -170,7 +170,7 @@ def test_generate_user_report_with_filters(mock_get_mdo_id_org_list, mock_bigque ehrms_id=None, start_date='2023-01-01', end_date='2023-01-31', - orgId='org1', + org_id='org1', required_columns=['user_id', 'content_id', 'content_progress_percentage'], additional_filters=additional_filters ) @@ -303,7 +303,7 @@ def test_generate_user_report_no_data(mock_get_mdo_id_org_list, mock_bigquery_se ehrms_id=None, start_date=None, end_date=None, - orgId='org1', + org_id='org1', additional_filters={} ) @@ -423,7 +423,7 @@ def test_generate_user_report_with_invalid_org(mock_get_mdo_id_org_list, mock_bi email='test@example.com', phone=None, ehrms_id=None, - orgId='org1', + org_id='org1', additional_filters={} ) @@ -450,7 +450,7 @@ def test_generate_user_report_with_valid_org(mock_get_mdo_id_org_list, mock_bigq email='test@example.com', phone=None, ehrms_id=None, - orgId='org1', + org_id='org1', additional_filters={} ) @@ -480,7 +480,7 @@ def test_generate_user_report_with_memory_error(mock_get_mdo_id_org_list, mock_b email='test@example.com', phone=None, ehrms_id=None, - orgId='org1', + org_id='org1', additional_filters={} ) @@ -495,7 +495,7 @@ def test_generate_user_report_with_no_filters(mock_get_mdo_id_org_list, mock_big email=None, phone=None, ehrms_id=None, - orgId='org1', + org_id='org1', additional_filters={} )