Stage to Prod

.github/workflows/unit_tests.yaml

@@ -5,7 +5,7 @@
     branches-ignore:
       - '**'
   pull_request:
-    branches: [main, dev, stage]
+    branches: [ main, dev, stage ]
 
 jobs:
   UnitTest:
@@ -15,28 +15,61 @@
       DATABASE_NAME: test_database
 
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v2
+      - name: Checkout code
+        uses: actions/checkout@v2
 
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.10'
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
 
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
 
-    - name: Run tests with coverage
-      run: |
-        coverage run --source=src -m unittest discover -s tests/unit_tests
-        coverage xml  
-
-    - name: Check coverage
-      run: |
-        coverage report --fail-under=85
+      - name: Determine output folder
+        id: set_output_folder
+        run: |
+          if [[ $GITHUB_EVENT_NAME == "pull_request" ]]; then
+            branch_name=$GITHUB_BASE_REF
+          else
+            branch_name=$GITHUB_REF_NAME
+          fi
+          
+          if [[ $branch_name == "main" ]]; then
+            echo "output_folder=prod" >> $GITHUB_ENV
+          elif [[ $branch_name == "stage" ]]; then
+            echo "output_folder=stage" >> $GITHUB_ENV
+          elif [[ $branch_name == "dev" ]]; then
+            echo "output_folder=dev" >> $GITHUB_ENV
+          else
+            echo "Unknown branch: $branch_name"
+            exit 1
+          fi
 
+      - name: Run tests with coverage
+        run: |
+          timestamp=$(date '+%Y-%m-%d_%H-%M-%S')
+          mkdir -p test_results
+          log_file="test_results/${timestamp}_report.log"
+          echo -e "\nTest Cases Report Report\n" >> $log_file
+          # Run the tests and append output to the log file
+          python -m coverage run --source=src -m unittest discover -s tests/unit_tests >> $log_file 2>&1
+          echo -e "\nCoverage Report\n" >> $log_file
+          coverage report >> $log_file
 
+      - name: Check coverage
+        run: |
+          coverage report --fail-under=85
 
+      - name: Upload report to Azure
+        uses: LanceMcCarthy/Action-AzureBlobUpload@v2
+        with:
+          source_folder: 'test_results'
+          destination_folder: '${{ env.output_folder }}'
+          connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
+          container_name: 'osw-validation-service '
+          clean_destination_folder: false
+          delete_if_exists: false
 

.gitignore

@@ -159,3 +159,6 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
 reports/
+test_results
+download.py
+test.py

README.md

@@ -23,15 +23,16 @@ The project is built on Python with FastAPI framework. All the regular nuances f
 - Connecting this to cloud will need the following in the `.env` file
 
 ```bash
+PROVIDER=Azure
 QUEUECONNECTION=xxxx
 STORAGECONNECTION=xxxx
 VALIDATION_REQ_TOPIC=xxxx
 VALIDATION_REQ_SUB=xxxx
 VALIDATION_RES_TOPIC=xxxx
 CONTAINER_NAME=xxxx
-AUTH_PERMISSION_URL=xxx
-MAX_CONCURRENT_MESSAGES=xxx
-
+AUTH_PERMISSION_URL=xxx # This is the URL to get the token
+MAX_CONCURRENT_MESSAGES=xxx # Optional if not provided defaults to 2
+AUTH_SIMULATE=xxx # Optional if not provided defaults to False
 ```
 
 The application connect with the `STORAGECONNECTION` string provided in `.env` file and validates downloaded zipfile using `python-osw-validation` package.

requirements.txt

@@ -1,7 +1,7 @@
 fastapi==0.88.0
 pydantic==1.10.4
-python-ms-core==0.0.22
+python-ms-core==0.0.23
 uvicorn==0.20.0
 html_testRunner==1.2.1
 geopandas==0.14.4
-python-osw-validation==0.2.7
+python-osw-validation==0.2.9

src/osw_validator.py

@@ -9,6 +9,7 @@
 from .models.queue_message_content import Upload, ValidationResult
 from .config import Settings
 import threading
+import python_osw_validation
 
 logging.basicConfig()
 logger = logging.getLogger('OSW_VALIDATOR')
@@ -77,11 +78,16 @@ def validate(self, received_message: Upload):
     def send_status(self, result: ValidationResult, upload_message: Upload):
         upload_message.data.success = result.is_valid
         upload_message.data.message = result.validation_message
+        resp_data = upload_message.data.to_json()
+        resp_data['package'] = {
+            'python-ms-core': Core.__version__,
+            'python-osw-validation': python_osw_validation.__version__
+        }
 
         data = QueueMessage.data_from({
             'messageId': upload_message.message_id,
             'messageType': upload_message.message_type,
-            'data': upload_message.data.to_json()
+            'data': resp_data
         })
         try:
             self.core.get_topic(topic_name=self._settings.event_bus.validation_topic).publish(data=data)

test_report.py

@@ -2,7 +2,7 @@
 import HtmlTestRunner
 
 # Define your test cases
-from tests.unit_tests.test_queue_message_content import TestUpload, TestUploadData, TestToJson, TestValidationResult
+from tests.unit_tests.models.test_queue_message_content import TestUpload, TestUploadData, TestToJson, TestValidationResult
 from tests.unit_tests.test_validation import TestOtherValidation, TestValidation
 from tests.unit_tests.test_osw_validator import TestOSWValidator
 from tests.unit_tests.test_main import TestApp

tests/unit_tests/interface/test_validator_abstract.py

@@ -0,0 +1,42 @@
+import unittest
+from abc import ABC
+from unittest.mock import MagicMock
+from python_ms_core.core.queue.models.queue_message import QueueMessage
+from src.interface.validator_abstract import ValidatorAbstract
+
+
+# A concrete implementation of ValidatorAbstract for testing
+class ConcreteValidator(ValidatorAbstract):
+    def validate(self, message: QueueMessage) -> None:
+        # Example implementation: Simply pass for testing purposes
+        pass
+
+
+class TestValidatorAbstract(unittest.TestCase):
+
+    def test_abstract_method_enforcement(self):
+        # Ensure that ValidatorAbstract cannot be instantiated directly
+        with self.assertRaises(TypeError):
+            ValidatorAbstract()
+
+    def test_concrete_validator_instance(self):
+        # Ensure a concrete class can be instantiated and implements `validate`
+        validator = ConcreteValidator()
+        self.assertIsInstance(validator, ValidatorAbstract)
+
+    def test_validate_method_called(self):
+        # Mock a QueueMessage object
+        message = MagicMock(spec=QueueMessage)
+
+        # Create an instance of the concrete validator
+        validator = ConcreteValidator()
+
+        # Call the validate method and ensure it executes without error
+        validator.validate(message)
+
+        # Assert that the mocked message object is a valid argument
+        self.assertTrue(hasattr(message, '__class__'))
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/unit_tests/test_queue_message_content.py → tests/unit_tests/models/test_queue_message_content.py

@@ -4,7 +4,7 @@
 from unittest.mock import MagicMock
 from src.models.queue_message_content import ValidationResult, Upload, UploadData, to_json
 
-current_dir = os.path.dirname(os.path.abspath(os.path.join(__file__, '../')))
+current_dir = os.path.dirname(os.path.abspath(os.path.join(__file__, '../../')))
 parent_dir = os.path.dirname(current_dir)
 
 TEST_JSON_FILE = os.path.join(parent_dir, 'src/assets/osw-upload.json')

tests/unit_tests/test_config.py

@@ -0,0 +1,47 @@
+import os
+import unittest
+from unittest.mock import patch
+from src.config import Settings
+
+
+class TestSettings(unittest.TestCase):
+
+    @patch.dict(os.environ, {
+        'AUTH_PERMISSION_URL': 'http://auth-url.com',
+        'MAX_CONCURRENT_MESSAGES': '5',
+        'AUTH_SIMULATE': 'True'
+    }, clear=True)
+    def test_settings_with_simulated_auth(self):
+        settings = Settings()
+        self.assertEqual(settings.app_name, 'python-osw-validation')
+        self.assertEqual(settings.auth_permission_url, 'http://auth-url.com')
+        self.assertEqual(settings.max_concurrent_messages, 5)
+        self.assertEqual(settings.auth_provider, 'Simulated')
+
+    @patch.dict(os.environ, {
+        'AUTH_PERMISSION_URL': 'http://auth-url.com',
+        'MAX_CONCURRENT_MESSAGES': '10',
+        'AUTH_SIMULATE': 'False'
+    }, clear=True)
+    def test_settings_with_hosted_auth(self):
+        settings = Settings()
+        self.assertEqual(settings.auth_provider, 'Hosted')
+
+    @patch.dict(os.environ, {
+        'AUTH_SIMULATE': 'invalid_value'
+    }, clear=True)
+    def test_settings_with_invalid_auth_simulate(self):
+        settings = Settings()
+        self.assertEqual(settings.auth_provider, 'Hosted')
+
+    @patch.dict(os.environ, {}, clear=True)
+    def test_default_settings(self):
+        settings = Settings()
+        self.assertEqual(settings.app_name, 'python-osw-validation')
+        self.assertEqual(settings.event_bus.container_name, 'osw')
+        self.assertIsNone(settings.auth_permission_url)
+        self.assertEqual(settings.max_concurrent_messages, 2)
+
+
+if __name__ == '__main__':
+    unittest.main()