diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 8aed5b6..bae521c 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -5,7 +5,7 @@ on: branches-ignore: - '**' pull_request: - branches: [main, dev, stage] + branches: [ main, dev, stage ] jobs: UnitTest: @@ -15,28 +15,61 @@ jobs: DATABASE_NAME: test_database steps: - - name: Checkout code - uses: actions/checkout@v2 + - name: Checkout code + uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt - - name: Run tests with coverage - run: | - coverage run --source=src -m unittest discover -s tests/unit_tests - coverage xml - - - name: Check coverage - run: | - coverage report --fail-under=85 + - name: Determine output folder + id: set_output_folder + run: | + if [[ $GITHUB_EVENT_NAME == "pull_request" ]]; then + branch_name=$GITHUB_BASE_REF + else + branch_name=$GITHUB_REF_NAME + fi + + if [[ $branch_name == "main" ]]; then + echo "output_folder=prod" >> $GITHUB_ENV + elif [[ $branch_name == "stage" ]]; then + echo "output_folder=stage" >> $GITHUB_ENV + elif [[ $branch_name == "dev" ]]; then + echo "output_folder=dev" >> $GITHUB_ENV + else + echo "Unknown branch: $branch_name" + exit 1 + fi + - name: Run tests with coverage + run: | + timestamp=$(date '+%Y-%m-%d_%H-%M-%S') + mkdir -p test_results + log_file="test_results/${timestamp}_report.log" + echo -e "\nTest Cases Report Report\n" >> $log_file + # Run the tests and append output to the log file + python -m coverage run --source=src -m unittest discover -s tests/unit_tests >> $log_file 2>&1 + echo -e "\nCoverage Report\n" >> $log_file + coverage report >> $log_file + - name: Check coverage + run: | + coverage report --fail-under=85 + - name: Upload report to Azure + uses: LanceMcCarthy/Action-AzureBlobUpload@v2 + with: + source_folder: 'test_results' + destination_folder: '${{ env.output_folder }}' + connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} + container_name: 'osw-validation-service ' + clean_destination_folder: false + delete_if_exists: false diff --git a/.gitignore b/.gitignore index 08f9708..6c7d076 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,6 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ reports/ +test_results +download.py +test.py \ No newline at end of file diff --git a/README.md b/README.md index 60f6bfe..a2b29ac 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,16 @@ The project is built on Python with FastAPI framework. All the regular nuances f - Connecting this to cloud will need the following in the `.env` file ```bash +PROVIDER=Azure QUEUECONNECTION=xxxx STORAGECONNECTION=xxxx VALIDATION_REQ_TOPIC=xxxx VALIDATION_REQ_SUB=xxxx VALIDATION_RES_TOPIC=xxxx CONTAINER_NAME=xxxx -AUTH_PERMISSION_URL=xxx -MAX_CONCURRENT_MESSAGES=xxx - +AUTH_PERMISSION_URL=xxx # This is the URL to get the token +MAX_CONCURRENT_MESSAGES=xxx # Optional if not provided defaults to 2 +AUTH_SIMULATE=xxx # Optional if not provided defaults to False ``` The application connect with the `STORAGECONNECTION` string provided in `.env` file and validates downloaded zipfile using `python-osw-validation` package. diff --git a/requirements.txt b/requirements.txt index 6a41b03..13ed1dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ fastapi==0.88.0 pydantic==1.10.4 -python-ms-core==0.0.22 +python-ms-core==0.0.23 uvicorn==0.20.0 html_testRunner==1.2.1 geopandas==0.14.4 -python-osw-validation==0.2.7 \ No newline at end of file +python-osw-validation==0.2.9 \ No newline at end of file diff --git a/src/osw_validator.py b/src/osw_validator.py index 23c2aa0..cfcc5f7 100644 --- a/src/osw_validator.py +++ b/src/osw_validator.py @@ -9,6 +9,7 @@ from .models.queue_message_content import Upload, ValidationResult from .config import Settings import threading +import python_osw_validation logging.basicConfig() logger = logging.getLogger('OSW_VALIDATOR') @@ -77,11 +78,16 @@ def validate(self, received_message: Upload): def send_status(self, result: ValidationResult, upload_message: Upload): upload_message.data.success = result.is_valid upload_message.data.message = result.validation_message + resp_data = upload_message.data.to_json() + resp_data['package'] = { + 'python-ms-core': Core.__version__, + 'python-osw-validation': python_osw_validation.__version__ + } data = QueueMessage.data_from({ 'messageId': upload_message.message_id, 'messageType': upload_message.message_type, - 'data': upload_message.data.to_json() + 'data': resp_data }) try: self.core.get_topic(topic_name=self._settings.event_bus.validation_topic).publish(data=data) diff --git a/test_report.py b/test_report.py index bd27d0e..38c3f39 100644 --- a/test_report.py +++ b/test_report.py @@ -2,7 +2,7 @@ import HtmlTestRunner # Define your test cases -from tests.unit_tests.test_queue_message_content import TestUpload, TestUploadData, TestToJson, TestValidationResult +from tests.unit_tests.models.test_queue_message_content import TestUpload, TestUploadData, TestToJson, TestValidationResult from tests.unit_tests.test_validation import TestOtherValidation, TestValidation from tests.unit_tests.test_osw_validator import TestOSWValidator from tests.unit_tests.test_main import TestApp diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit_tests/interface/__init__.py b/tests/unit_tests/interface/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit_tests/interface/test_validator_abstract.py b/tests/unit_tests/interface/test_validator_abstract.py new file mode 100644 index 0000000..5f95071 --- /dev/null +++ b/tests/unit_tests/interface/test_validator_abstract.py @@ -0,0 +1,42 @@ +import unittest +from abc import ABC +from unittest.mock import MagicMock +from python_ms_core.core.queue.models.queue_message import QueueMessage +from src.interface.validator_abstract import ValidatorAbstract + + +# A concrete implementation of ValidatorAbstract for testing +class ConcreteValidator(ValidatorAbstract): + def validate(self, message: QueueMessage) -> None: + # Example implementation: Simply pass for testing purposes + pass + + +class TestValidatorAbstract(unittest.TestCase): + + def test_abstract_method_enforcement(self): + # Ensure that ValidatorAbstract cannot be instantiated directly + with self.assertRaises(TypeError): + ValidatorAbstract() + + def test_concrete_validator_instance(self): + # Ensure a concrete class can be instantiated and implements `validate` + validator = ConcreteValidator() + self.assertIsInstance(validator, ValidatorAbstract) + + def test_validate_method_called(self): + # Mock a QueueMessage object + message = MagicMock(spec=QueueMessage) + + # Create an instance of the concrete validator + validator = ConcreteValidator() + + # Call the validate method and ensure it executes without error + validator.validate(message) + + # Assert that the mocked message object is a valid argument + self.assertTrue(hasattr(message, '__class__')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit_tests/models/__init__.py b/tests/unit_tests/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit_tests/test_queue_message_content.py b/tests/unit_tests/models/test_queue_message_content.py similarity index 99% rename from tests/unit_tests/test_queue_message_content.py rename to tests/unit_tests/models/test_queue_message_content.py index 904ad9e..4ac1e2c 100644 --- a/tests/unit_tests/test_queue_message_content.py +++ b/tests/unit_tests/models/test_queue_message_content.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock from src.models.queue_message_content import ValidationResult, Upload, UploadData, to_json -current_dir = os.path.dirname(os.path.abspath(os.path.join(__file__, '../'))) +current_dir = os.path.dirname(os.path.abspath(os.path.join(__file__, '../../'))) parent_dir = os.path.dirname(current_dir) TEST_JSON_FILE = os.path.join(parent_dir, 'src/assets/osw-upload.json') diff --git a/tests/unit_tests/test_config.py b/tests/unit_tests/test_config.py new file mode 100644 index 0000000..886199f --- /dev/null +++ b/tests/unit_tests/test_config.py @@ -0,0 +1,47 @@ +import os +import unittest +from unittest.mock import patch +from src.config import Settings + + +class TestSettings(unittest.TestCase): + + @patch.dict(os.environ, { + 'AUTH_PERMISSION_URL': 'http://auth-url.com', + 'MAX_CONCURRENT_MESSAGES': '5', + 'AUTH_SIMULATE': 'True' + }, clear=True) + def test_settings_with_simulated_auth(self): + settings = Settings() + self.assertEqual(settings.app_name, 'python-osw-validation') + self.assertEqual(settings.auth_permission_url, 'http://auth-url.com') + self.assertEqual(settings.max_concurrent_messages, 5) + self.assertEqual(settings.auth_provider, 'Simulated') + + @patch.dict(os.environ, { + 'AUTH_PERMISSION_URL': 'http://auth-url.com', + 'MAX_CONCURRENT_MESSAGES': '10', + 'AUTH_SIMULATE': 'False' + }, clear=True) + def test_settings_with_hosted_auth(self): + settings = Settings() + self.assertEqual(settings.auth_provider, 'Hosted') + + @patch.dict(os.environ, { + 'AUTH_SIMULATE': 'invalid_value' + }, clear=True) + def test_settings_with_invalid_auth_simulate(self): + settings = Settings() + self.assertEqual(settings.auth_provider, 'Hosted') + + @patch.dict(os.environ, {}, clear=True) + def test_default_settings(self): + settings = Settings() + self.assertEqual(settings.app_name, 'python-osw-validation') + self.assertEqual(settings.event_bus.container_name, 'osw') + self.assertIsNone(settings.auth_permission_url) + self.assertEqual(settings.max_concurrent_messages, 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit_tests/test_service.py b/tests/unit_tests/test_service.py index 408fca5..1c16397 100644 --- a/tests/unit_tests/test_service.py +++ b/tests/unit_tests/test_service.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch, MagicMock from src.osw_validator import OSWValidator +from src.models.queue_message_content import Upload from src.models.queue_message_content import ValidationResult @@ -25,7 +26,7 @@ def setUp(self, mock_core, mock_settings): self.service = OSWValidator() self.service.storage_client = MagicMock() self.service.container_name = 'test_container' - self.auth = MagicMock() + self.service.auth = MagicMock() # Define a sample message with proper strings self.sample_message = { @@ -115,7 +116,8 @@ def test_validate_with_no_file_upload_path(self, mock_validation_result): # Assert that the result indicates failure self.assertFalse(actual_result.is_valid) - self.assertEqual(actual_result.validation_message, 'Error occurred while validating OSW request Request does not have valid file path specified.') + self.assertEqual(actual_result.validation_message, + 'Error occurred while validating OSW request Request does not have valid file path specified.') # Ensure the upload_message is the expected object self.assertEqual(actual_upload_message, mock_request_message) @@ -162,6 +164,185 @@ def test_validate_with_validation_only_in_message_type(self, mock_has_permission self.assertTrue(actual_result.is_valid) self.assertEqual(actual_upload_message, mock_request_message) + @patch('src.osw_validator.threading.Thread') + def test_stop_listening(self, mock_thread): + # Arrange + mock_thread_instance = MagicMock() + mock_thread.return_value = mock_thread_instance + + self.service.listener_thread = mock_thread_instance + + # Act + result = self.service.stop_listening() + + # Assert + mock_thread_instance.join.assert_called_once_with(timeout=0) + self.assertIsNone(result) + + def test_has_permission_success(self): + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + self.service.auth.has_permission = MagicMock() + self.service.auth.has_permission.return_value = True + + result = self.service.has_permission( + roles=['tdei-admin', 'poc', 'osw_data_generator'], + queue_message=mock_message + ) + self.assertTrue(result) + self.service.auth.has_permission.assert_called_once() + + def test_has_permission_failure(self): + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + self.service.auth.has_permission = MagicMock() + self.service.auth.has_permission.return_value = False + + result = self.service.has_permission( + roles=[], + queue_message=mock_message + ) + self.assertFalse(result) + self.service.auth.has_permission.assert_called_once() + + def test_has_permission_exception(self): + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + self.service.auth.has_permission = MagicMock() + self.service.auth.has_permission.side_effect = Exception('Mock exception') + + result = self.service.has_permission( + roles=[], + queue_message=mock_message + ) + self.assertFalse(result) + self.service.auth.has_permission.assert_called_once() + + def test_send_status_success(self): + validation_result = ValidationResult() + validation_result.is_valid = True + validation_result.validation_message = '' + + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + mock_topic = self.service.core.get_topic.return_value + mock_publish = mock_topic.publish + + self.service.send_status(result=validation_result, upload_message=mock_message) + + mock_publish.assert_called_once() + + def test_send_status_failure(self): + validation_result = ValidationResult() + validation_result.is_valid = False + validation_result.validation_message = 'Failed' + + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + mock_topic = self.service.core.get_topic.return_value + mock_publish = mock_topic.publish + + self.service.send_status(result=validation_result, upload_message=mock_message) + + mock_publish.assert_called_once() + + def test_send_status_exception(self): + validation_result = ValidationResult() + validation_result.is_valid = False + validation_result.validation_message = 'Failed' + + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + mock_topic = self.service.core.get_topic.return_value + mock_publish = mock_topic.publish + mock_publish.side_effect = Exception('Mock Exception') + + self.service.send_status(result=validation_result, upload_message=mock_message) + + mock_publish.assert_called_once() + + def test_validate_with_unauthorized(self): + # Arrange + self.service.has_permission = MagicMock() + self.service.has_permission.return_value = None + + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444', + 'file_upload_path': 'mock_file_upload_path' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + # Mock the send_status method + self.service.send_status = MagicMock() + + # Act + self.service.validate(mock_message) + + # Assert + + # Ensure send_status was called with a successful validation result + self.service.send_status.assert_called_once() + + # Extract the actual result and upload message + actual_result = self.service.send_status.call_args[1]['result'] + actual_upload_message = self.service.send_status.call_args[1]['upload_message'] + + # Assert the result is valid and the upload_message is the mock_request_message + self.assertFalse(actual_result.is_valid) + self.service.has_permission.assert_called_once() if __name__ == '__main__':