From 9342f2f38ec48efb62c2ad55778ff4f2c02bcb95 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Wed, 20 Nov 2024 23:37:44 +0530 Subject: [PATCH 1/5] Fixed task-1467 - Added unit test cases - Added unit test cases pipeline - Added support to upload test cases results on Azure blob --- .github/workflows/unit_tests.yaml | 71 +++++-- download.py | 54 +++++ test.py | 59 ++++++ test_report.py | 2 +- tests/__init__.py | 0 tests/unit_tests/interface/__init__.py | 0 .../interface/test_validator_abstract.py | 42 ++++ tests/unit_tests/models/__init__.py | 0 .../test_queue_message_content.py | 2 +- tests/unit_tests/test_config.py | 47 +++++ tests/unit_tests/test_service.py | 185 +++++++++++++++++- 11 files changed, 439 insertions(+), 23 deletions(-) create mode 100644 download.py create mode 100644 test.py create mode 100644 tests/__init__.py create mode 100644 tests/unit_tests/interface/__init__.py create mode 100644 tests/unit_tests/interface/test_validator_abstract.py create mode 100644 tests/unit_tests/models/__init__.py rename tests/unit_tests/{ => models}/test_queue_message_content.py (99%) create mode 100644 tests/unit_tests/test_config.py diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 8aed5b6..bae521c 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -5,7 +5,7 @@ on: branches-ignore: - '**' pull_request: - branches: [main, dev, stage] + branches: [ main, dev, stage ] jobs: UnitTest: @@ -15,28 +15,61 @@ jobs: DATABASE_NAME: test_database steps: - - name: Checkout code - uses: actions/checkout@v2 + - name: Checkout code + uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt - - name: Run tests with coverage - run: | - coverage run --source=src -m unittest discover -s tests/unit_tests - coverage xml - - - name: Check coverage - run: | - coverage report --fail-under=85 + - name: Determine output folder + id: set_output_folder + run: | + if [[ $GITHUB_EVENT_NAME == "pull_request" ]]; then + branch_name=$GITHUB_BASE_REF + else + branch_name=$GITHUB_REF_NAME + fi + + if [[ $branch_name == "main" ]]; then + echo "output_folder=prod" >> $GITHUB_ENV + elif [[ $branch_name == "stage" ]]; then + echo "output_folder=stage" >> $GITHUB_ENV + elif [[ $branch_name == "dev" ]]; then + echo "output_folder=dev" >> $GITHUB_ENV + else + echo "Unknown branch: $branch_name" + exit 1 + fi + - name: Run tests with coverage + run: | + timestamp=$(date '+%Y-%m-%d_%H-%M-%S') + mkdir -p test_results + log_file="test_results/${timestamp}_report.log" + echo -e "\nTest Cases Report Report\n" >> $log_file + # Run the tests and append output to the log file + python -m coverage run --source=src -m unittest discover -s tests/unit_tests >> $log_file 2>&1 + echo -e "\nCoverage Report\n" >> $log_file + coverage report >> $log_file + - name: Check coverage + run: | + coverage report --fail-under=85 + - name: Upload report to Azure + uses: LanceMcCarthy/Action-AzureBlobUpload@v2 + with: + source_folder: 'test_results' + destination_folder: '${{ env.output_folder }}' + connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} + container_name: 'osw-validation-service ' + clean_destination_folder: false + delete_if_exists: false diff --git a/download.py b/download.py new file mode 100644 index 0000000..50c5e44 --- /dev/null +++ b/download.py @@ -0,0 +1,54 @@ +import os +import requests + +FILES_TO_UPLOAD = [ + 'graph-10.xml', + 'graph-30.xml', + 'graph-50.xml', + 'graph-70.xml', + 'graph-90.xml', + 'graph-100.xml', + 'graph-200.xml', + 'graph-250.xml', + 'kingCounty-130.xml', + 'seattle.new.xml', + 'tacoma.xml', + 'vancouver.xml', +] + + +def download_file(url: str, download_folder: str = 'downloads'): + # Create the download folder if it doesn't exist + if not os.path.exists(download_folder): + os.makedirs(download_folder) + + # Download the file + print(f"Downloading file from {url}...") + response = requests.get(url, stream=True) + + if response.status_code == 200: + # Try to get the file name from the Content-Disposition header + content_disposition = response.headers.get('content-disposition') + if content_disposition: + filename = content_disposition.split('filename=')[-1].strip('"') + else: + # Fall back to using the last part of the URL as the file name + filename = url.split('/')[-1] + + file_path = os.path.join(download_folder, filename) + + # Write the file + with open(file_path, 'wb') as file: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + file.write(chunk) + + print(f"File downloaded successfully and saved to {file_path}") + else: + print(f"Failed to download file. Status code: {response.status_code}") + +# Example usage + +for i in FILES_TO_UPLOAD: + url = f'https://tdeisamplestorage.blob.core.windows.net/tdei-storage-test/input/{i}' + download_file(url, '/Users/anuj/Work/Gaussian/TDEI-python-osw-validation/xml') diff --git a/test.py b/test.py new file mode 100644 index 0000000..b5e076d --- /dev/null +++ b/test.py @@ -0,0 +1,59 @@ +import uuid +import time +from python_ms_core import Core +from python_ms_core.core.queue.models.queue_message import QueueMessage + +core = Core() + +FILES_TO_UPLOAD = [ + # 'graph-10.zip', + # 'graph-30.zip', + # 'graph-50.zip', + # 'graph-70.zip', + # 'graph-90.zip', + # 'graph-100.zip', + # 'graph-200.zip', + # 'graph-250.zip', + # 'kingCounty-130.zip', + # 'seattle.new.zip', + # 'tacoma.zip', + 'vancouver.zip', +] + + +def run(publishing_topic): + # for i in range(0, 50): + # messageId = str(i) + # message = QueueMessage.data_from({ + # 'messageId': f'{messageId}', + # 'messageType': 'Formatter', + # 'data': { + # "tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb", + # 'file_upload_path': f'https://tdeisamplestorage.blob.core.windows.net/tdei-storage-test/input/graph-10.zip', + # 'user_id': '285f0ceb - 2816 - 451d - 8140 - 1e8ed7399993', + # }, + # }) + # + # publishing_topic.publish(data=message) + # print(f'Sent message: {messageId}') + # time.sleep(1) + + for i in FILES_TO_UPLOAD: + messageId = uuid.uuid1().hex[0:24] + message = QueueMessage.data_from({ + 'messageId': uuid.uuid1().hex[0:24], + 'messageType': 'Formatter', + 'data': { + "tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb", + 'file_upload_path': f'https://tdeisamplestorage.blob.core.windows.net/tdei-storage-test/input/{i}', + 'user_id': '285f0ceb - 2816 - 451d - 8140 - 1e8ed7399993', + }, + }) + + publishing_topic.publish(data=message) + print(f'Sent message: {messageId}') + + + +topic = core.get_topic(topic_name='temp-anuj') +run(topic) diff --git a/test_report.py b/test_report.py index bd27d0e..38c3f39 100644 --- a/test_report.py +++ b/test_report.py @@ -2,7 +2,7 @@ import HtmlTestRunner # Define your test cases -from tests.unit_tests.test_queue_message_content import TestUpload, TestUploadData, TestToJson, TestValidationResult +from tests.unit_tests.models.test_queue_message_content import TestUpload, TestUploadData, TestToJson, TestValidationResult from tests.unit_tests.test_validation import TestOtherValidation, TestValidation from tests.unit_tests.test_osw_validator import TestOSWValidator from tests.unit_tests.test_main import TestApp diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit_tests/interface/__init__.py b/tests/unit_tests/interface/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit_tests/interface/test_validator_abstract.py b/tests/unit_tests/interface/test_validator_abstract.py new file mode 100644 index 0000000..5f95071 --- /dev/null +++ b/tests/unit_tests/interface/test_validator_abstract.py @@ -0,0 +1,42 @@ +import unittest +from abc import ABC +from unittest.mock import MagicMock +from python_ms_core.core.queue.models.queue_message import QueueMessage +from src.interface.validator_abstract import ValidatorAbstract + + +# A concrete implementation of ValidatorAbstract for testing +class ConcreteValidator(ValidatorAbstract): + def validate(self, message: QueueMessage) -> None: + # Example implementation: Simply pass for testing purposes + pass + + +class TestValidatorAbstract(unittest.TestCase): + + def test_abstract_method_enforcement(self): + # Ensure that ValidatorAbstract cannot be instantiated directly + with self.assertRaises(TypeError): + ValidatorAbstract() + + def test_concrete_validator_instance(self): + # Ensure a concrete class can be instantiated and implements `validate` + validator = ConcreteValidator() + self.assertIsInstance(validator, ValidatorAbstract) + + def test_validate_method_called(self): + # Mock a QueueMessage object + message = MagicMock(spec=QueueMessage) + + # Create an instance of the concrete validator + validator = ConcreteValidator() + + # Call the validate method and ensure it executes without error + validator.validate(message) + + # Assert that the mocked message object is a valid argument + self.assertTrue(hasattr(message, '__class__')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit_tests/models/__init__.py b/tests/unit_tests/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit_tests/test_queue_message_content.py b/tests/unit_tests/models/test_queue_message_content.py similarity index 99% rename from tests/unit_tests/test_queue_message_content.py rename to tests/unit_tests/models/test_queue_message_content.py index 904ad9e..4ac1e2c 100644 --- a/tests/unit_tests/test_queue_message_content.py +++ b/tests/unit_tests/models/test_queue_message_content.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock from src.models.queue_message_content import ValidationResult, Upload, UploadData, to_json -current_dir = os.path.dirname(os.path.abspath(os.path.join(__file__, '../'))) +current_dir = os.path.dirname(os.path.abspath(os.path.join(__file__, '../../'))) parent_dir = os.path.dirname(current_dir) TEST_JSON_FILE = os.path.join(parent_dir, 'src/assets/osw-upload.json') diff --git a/tests/unit_tests/test_config.py b/tests/unit_tests/test_config.py new file mode 100644 index 0000000..886199f --- /dev/null +++ b/tests/unit_tests/test_config.py @@ -0,0 +1,47 @@ +import os +import unittest +from unittest.mock import patch +from src.config import Settings + + +class TestSettings(unittest.TestCase): + + @patch.dict(os.environ, { + 'AUTH_PERMISSION_URL': 'http://auth-url.com', + 'MAX_CONCURRENT_MESSAGES': '5', + 'AUTH_SIMULATE': 'True' + }, clear=True) + def test_settings_with_simulated_auth(self): + settings = Settings() + self.assertEqual(settings.app_name, 'python-osw-validation') + self.assertEqual(settings.auth_permission_url, 'http://auth-url.com') + self.assertEqual(settings.max_concurrent_messages, 5) + self.assertEqual(settings.auth_provider, 'Simulated') + + @patch.dict(os.environ, { + 'AUTH_PERMISSION_URL': 'http://auth-url.com', + 'MAX_CONCURRENT_MESSAGES': '10', + 'AUTH_SIMULATE': 'False' + }, clear=True) + def test_settings_with_hosted_auth(self): + settings = Settings() + self.assertEqual(settings.auth_provider, 'Hosted') + + @patch.dict(os.environ, { + 'AUTH_SIMULATE': 'invalid_value' + }, clear=True) + def test_settings_with_invalid_auth_simulate(self): + settings = Settings() + self.assertEqual(settings.auth_provider, 'Hosted') + + @patch.dict(os.environ, {}, clear=True) + def test_default_settings(self): + settings = Settings() + self.assertEqual(settings.app_name, 'python-osw-validation') + self.assertEqual(settings.event_bus.container_name, 'osw') + self.assertIsNone(settings.auth_permission_url) + self.assertEqual(settings.max_concurrent_messages, 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit_tests/test_service.py b/tests/unit_tests/test_service.py index 408fca5..1c16397 100644 --- a/tests/unit_tests/test_service.py +++ b/tests/unit_tests/test_service.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch, MagicMock from src.osw_validator import OSWValidator +from src.models.queue_message_content import Upload from src.models.queue_message_content import ValidationResult @@ -25,7 +26,7 @@ def setUp(self, mock_core, mock_settings): self.service = OSWValidator() self.service.storage_client = MagicMock() self.service.container_name = 'test_container' - self.auth = MagicMock() + self.service.auth = MagicMock() # Define a sample message with proper strings self.sample_message = { @@ -115,7 +116,8 @@ def test_validate_with_no_file_upload_path(self, mock_validation_result): # Assert that the result indicates failure self.assertFalse(actual_result.is_valid) - self.assertEqual(actual_result.validation_message, 'Error occurred while validating OSW request Request does not have valid file path specified.') + self.assertEqual(actual_result.validation_message, + 'Error occurred while validating OSW request Request does not have valid file path specified.') # Ensure the upload_message is the expected object self.assertEqual(actual_upload_message, mock_request_message) @@ -162,6 +164,185 @@ def test_validate_with_validation_only_in_message_type(self, mock_has_permission self.assertTrue(actual_result.is_valid) self.assertEqual(actual_upload_message, mock_request_message) + @patch('src.osw_validator.threading.Thread') + def test_stop_listening(self, mock_thread): + # Arrange + mock_thread_instance = MagicMock() + mock_thread.return_value = mock_thread_instance + + self.service.listener_thread = mock_thread_instance + + # Act + result = self.service.stop_listening() + + # Assert + mock_thread_instance.join.assert_called_once_with(timeout=0) + self.assertIsNone(result) + + def test_has_permission_success(self): + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + self.service.auth.has_permission = MagicMock() + self.service.auth.has_permission.return_value = True + + result = self.service.has_permission( + roles=['tdei-admin', 'poc', 'osw_data_generator'], + queue_message=mock_message + ) + self.assertTrue(result) + self.service.auth.has_permission.assert_called_once() + + def test_has_permission_failure(self): + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + self.service.auth.has_permission = MagicMock() + self.service.auth.has_permission.return_value = False + + result = self.service.has_permission( + roles=[], + queue_message=mock_message + ) + self.assertFalse(result) + self.service.auth.has_permission.assert_called_once() + + def test_has_permission_exception(self): + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + self.service.auth.has_permission = MagicMock() + self.service.auth.has_permission.side_effect = Exception('Mock exception') + + result = self.service.has_permission( + roles=[], + queue_message=mock_message + ) + self.assertFalse(result) + self.service.auth.has_permission.assert_called_once() + + def test_send_status_success(self): + validation_result = ValidationResult() + validation_result.is_valid = True + validation_result.validation_message = '' + + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + mock_topic = self.service.core.get_topic.return_value + mock_publish = mock_topic.publish + + self.service.send_status(result=validation_result, upload_message=mock_message) + + mock_publish.assert_called_once() + + def test_send_status_failure(self): + validation_result = ValidationResult() + validation_result.is_valid = False + validation_result.validation_message = 'Failed' + + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + mock_topic = self.service.core.get_topic.return_value + mock_publish = mock_topic.publish + + self.service.send_status(result=validation_result, upload_message=mock_message) + + mock_publish.assert_called_once() + + def test_send_status_exception(self): + validation_result = ValidationResult() + validation_result.is_valid = False + validation_result.validation_message = 'Failed' + + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + mock_topic = self.service.core.get_topic.return_value + mock_publish = mock_topic.publish + mock_publish.side_effect = Exception('Mock Exception') + + self.service.send_status(result=validation_result, upload_message=mock_message) + + mock_publish.assert_called_once() + + def test_validate_with_unauthorized(self): + # Arrange + self.service.has_permission = MagicMock() + self.service.has_permission.return_value = None + + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444', + 'file_upload_path': 'mock_file_upload_path' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + # Mock the send_status method + self.service.send_status = MagicMock() + + # Act + self.service.validate(mock_message) + + # Assert + + # Ensure send_status was called with a successful validation result + self.service.send_status.assert_called_once() + + # Extract the actual result and upload message + actual_result = self.service.send_status.call_args[1]['result'] + actual_upload_message = self.service.send_status.call_args[1]['upload_message'] + + # Assert the result is valid and the upload_message is the mock_request_message + self.assertFalse(actual_result.is_valid) + self.service.has_permission.assert_called_once() if __name__ == '__main__': From e98640ae322d201039ea5d9132d4b5abab7a3de5 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Wed, 20 Nov 2024 23:37:59 +0530 Subject: [PATCH 2/5] Fixed task-1467 - Added unit test cases - Added unit test cases pipeline - Added support to upload test cases results on Azure blob --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 08f9708..bc63360 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,4 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ reports/ +test_results \ No newline at end of file From c2584aa4f8554286b87e3fa7514ca1aa0641b435 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Thu, 28 Nov 2024 01:09:00 +0530 Subject: [PATCH 3/5] Removed unnecessary files --- .gitignore | 4 +++- download.py | 54 -------------------------------------------- requirements.txt | 4 ++-- test.py | 59 ------------------------------------------------ 4 files changed, 5 insertions(+), 116 deletions(-) delete mode 100644 download.py delete mode 100644 test.py diff --git a/.gitignore b/.gitignore index bc63360..6c7d076 100644 --- a/.gitignore +++ b/.gitignore @@ -159,4 +159,6 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ reports/ -test_results \ No newline at end of file +test_results +download.py +test.py \ No newline at end of file diff --git a/download.py b/download.py deleted file mode 100644 index 50c5e44..0000000 --- a/download.py +++ /dev/null @@ -1,54 +0,0 @@ -import os -import requests - -FILES_TO_UPLOAD = [ - 'graph-10.xml', - 'graph-30.xml', - 'graph-50.xml', - 'graph-70.xml', - 'graph-90.xml', - 'graph-100.xml', - 'graph-200.xml', - 'graph-250.xml', - 'kingCounty-130.xml', - 'seattle.new.xml', - 'tacoma.xml', - 'vancouver.xml', -] - - -def download_file(url: str, download_folder: str = 'downloads'): - # Create the download folder if it doesn't exist - if not os.path.exists(download_folder): - os.makedirs(download_folder) - - # Download the file - print(f"Downloading file from {url}...") - response = requests.get(url, stream=True) - - if response.status_code == 200: - # Try to get the file name from the Content-Disposition header - content_disposition = response.headers.get('content-disposition') - if content_disposition: - filename = content_disposition.split('filename=')[-1].strip('"') - else: - # Fall back to using the last part of the URL as the file name - filename = url.split('/')[-1] - - file_path = os.path.join(download_folder, filename) - - # Write the file - with open(file_path, 'wb') as file: - for chunk in response.iter_content(chunk_size=1024): - if chunk: - file.write(chunk) - - print(f"File downloaded successfully and saved to {file_path}") - else: - print(f"Failed to download file. Status code: {response.status_code}") - -# Example usage - -for i in FILES_TO_UPLOAD: - url = f'https://tdeisamplestorage.blob.core.windows.net/tdei-storage-test/input/{i}' - download_file(url, '/Users/anuj/Work/Gaussian/TDEI-python-osw-validation/xml') diff --git a/requirements.txt b/requirements.txt index 6a41b03..7e93426 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ fastapi==0.88.0 pydantic==1.10.4 -python-ms-core==0.0.22 +python-ms-core==0.0.23 uvicorn==0.20.0 html_testRunner==1.2.1 geopandas==0.14.4 -python-osw-validation==0.2.7 \ No newline at end of file +python-osw-validation==0.2.8 \ No newline at end of file diff --git a/test.py b/test.py deleted file mode 100644 index b5e076d..0000000 --- a/test.py +++ /dev/null @@ -1,59 +0,0 @@ -import uuid -import time -from python_ms_core import Core -from python_ms_core.core.queue.models.queue_message import QueueMessage - -core = Core() - -FILES_TO_UPLOAD = [ - # 'graph-10.zip', - # 'graph-30.zip', - # 'graph-50.zip', - # 'graph-70.zip', - # 'graph-90.zip', - # 'graph-100.zip', - # 'graph-200.zip', - # 'graph-250.zip', - # 'kingCounty-130.zip', - # 'seattle.new.zip', - # 'tacoma.zip', - 'vancouver.zip', -] - - -def run(publishing_topic): - # for i in range(0, 50): - # messageId = str(i) - # message = QueueMessage.data_from({ - # 'messageId': f'{messageId}', - # 'messageType': 'Formatter', - # 'data': { - # "tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb", - # 'file_upload_path': f'https://tdeisamplestorage.blob.core.windows.net/tdei-storage-test/input/graph-10.zip', - # 'user_id': '285f0ceb - 2816 - 451d - 8140 - 1e8ed7399993', - # }, - # }) - # - # publishing_topic.publish(data=message) - # print(f'Sent message: {messageId}') - # time.sleep(1) - - for i in FILES_TO_UPLOAD: - messageId = uuid.uuid1().hex[0:24] - message = QueueMessage.data_from({ - 'messageId': uuid.uuid1().hex[0:24], - 'messageType': 'Formatter', - 'data': { - "tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb", - 'file_upload_path': f'https://tdeisamplestorage.blob.core.windows.net/tdei-storage-test/input/{i}', - 'user_id': '285f0ceb - 2816 - 451d - 8140 - 1e8ed7399993', - }, - }) - - publishing_topic.publish(data=message) - print(f'Sent message: {messageId}') - - - -topic = core.get_topic(topic_name='temp-anuj') -run(topic) From 631fe220b44be6a9b5a8a28e087811e27e63d5b9 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Thu, 28 Nov 2024 16:40:22 +0530 Subject: [PATCH 4/5] Updated package --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7e93426..13ed1dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ python-ms-core==0.0.23 uvicorn==0.20.0 html_testRunner==1.2.1 geopandas==0.14.4 -python-osw-validation==0.2.8 \ No newline at end of file +python-osw-validation==0.2.9 \ No newline at end of file From cf408b4c44518cf28a020a610dbf0850df157238 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Mon, 23 Dec 2024 14:17:39 +0530 Subject: [PATCH 5/5] Fixed Task 1564 and 1566 - Task [1564](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/1564/) - Added a package object in response which contains custom package name and version - Task [1566](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/1566/) - Updated readme for env variable details --- README.md | 7 ++++--- src/osw_validator.py | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 60f6bfe..a2b29ac 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,16 @@ The project is built on Python with FastAPI framework. All the regular nuances f - Connecting this to cloud will need the following in the `.env` file ```bash +PROVIDER=Azure QUEUECONNECTION=xxxx STORAGECONNECTION=xxxx VALIDATION_REQ_TOPIC=xxxx VALIDATION_REQ_SUB=xxxx VALIDATION_RES_TOPIC=xxxx CONTAINER_NAME=xxxx -AUTH_PERMISSION_URL=xxx -MAX_CONCURRENT_MESSAGES=xxx - +AUTH_PERMISSION_URL=xxx # This is the URL to get the token +MAX_CONCURRENT_MESSAGES=xxx # Optional if not provided defaults to 2 +AUTH_SIMULATE=xxx # Optional if not provided defaults to False ``` The application connect with the `STORAGECONNECTION` string provided in `.env` file and validates downloaded zipfile using `python-osw-validation` package. diff --git a/src/osw_validator.py b/src/osw_validator.py index 23c2aa0..cfcc5f7 100644 --- a/src/osw_validator.py +++ b/src/osw_validator.py @@ -9,6 +9,7 @@ from .models.queue_message_content import Upload, ValidationResult from .config import Settings import threading +import python_osw_validation logging.basicConfig() logger = logging.getLogger('OSW_VALIDATOR') @@ -77,11 +78,16 @@ def validate(self, received_message: Upload): def send_status(self, result: ValidationResult, upload_message: Upload): upload_message.data.success = result.is_valid upload_message.data.message = result.validation_message + resp_data = upload_message.data.to_json() + resp_data['package'] = { + 'python-ms-core': Core.__version__, + 'python-osw-validation': python_osw_validation.__version__ + } data = QueueMessage.data_from({ 'messageId': upload_message.message_id, 'messageType': upload_message.message_type, - 'data': upload_message.data.to_json() + 'data': resp_data }) try: self.core.get_topic(topic_name=self._settings.event_bus.validation_topic).publish(data=data)