diff --git a/requirements.txt b/requirements.txt index c8de9a2..b5d176e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ pydantic==1.10.4 python-ms-core==0.0.22 uvicorn==0.20.0 html_testRunner==1.2.1 -python-osw-validation==0.2.4 \ No newline at end of file +python-osw-validation==0.2.6 \ No newline at end of file diff --git a/src/osw_validator.py b/src/osw_validator.py index d5467b0..c7229dc 100644 --- a/src/osw_validator.py +++ b/src/osw_validator.py @@ -1,6 +1,4 @@ -import uuid import logging -import datetime import urllib.parse from typing import List from python_ms_core import Core @@ -90,7 +88,6 @@ def send_status(self, result: ValidationResult, upload_message: Upload): except Exception as e: logger.error(f'Error occurred while publishing message for : {upload_message.message_id} with error: {e}') - def has_permission(self, roles: List[str], queue_message: Upload) -> bool: try: permission_request = PermissionRequest( diff --git a/src/validation.py b/src/validation.py index dfc3e45..12f623c 100644 --- a/src/validation.py +++ b/src/validation.py @@ -1,4 +1,6 @@ import os +import uuid +import time import shutil import logging import traceback @@ -6,11 +8,10 @@ from .config import Settings from python_osw_validation import OSWValidation from .models.queue_message_content import ValidationResult -import uuid ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) # Path used for download file generation. -DOWNLOAD_FILE_PATH = f'{Path.cwd()}/downloads' +DOWNLOAD_DIR = f'{Path.cwd()}/downloads' logging.basicConfig() logger = logging.getLogger('OSW_VALIDATION') @@ -25,11 +26,22 @@ def __init__(self, file_path=None, storage_client=None): self.file_path = file_path self.file_relative_path = file_path.split('/')[-1] self.client = self.storage_client.get_container(container_name=self.container_name) + is_exists = os.path.exists(DOWNLOAD_DIR) + unique_id = self.get_unique_id() + if not is_exists: + os.makedirs(DOWNLOAD_DIR) + self.unique_dir_path = os.path.join(DOWNLOAD_DIR, unique_id) + if not os.path.exists(self.unique_dir_path): + os.makedirs(self.unique_dir_path) def validate(self, max_errors=20) -> ValidationResult: - return self.is_osw_valid(max_errors) + try: + return self.is_osw_valid(max_errors) + finally: + Validation.clean_up(self.unique_dir_path) def is_osw_valid(self, max_errors) -> ValidationResult: + start_time = time.time() result = ValidationResult() result.is_valid = False result.validation_message = '' @@ -47,24 +59,19 @@ def is_osw_valid(self, max_errors) -> ValidationResult: else: result.validation_message = 'Failed to validate because unknown file format' logger.error(f' Failed to validate because unknown file format') - + end_time = time.time() + time_taken = end_time - start_time + logger.info(f'Validation completed in {time_taken} seconds') return result # Downloads the single file into a unique directory def download_single_file(self, file_upload_path=None) -> str: - is_exists = os.path.exists(DOWNLOAD_FILE_PATH) - unique_id = self.get_unique_id() - if not is_exists: - os.makedirs(DOWNLOAD_FILE_PATH) - unique_directory = os.path.join(DOWNLOAD_FILE_PATH,unique_id) - if not os.path.exists(unique_directory): - os.makedirs(unique_directory) - + file = self.storage_client.get_file_from_url(self.container_name, file_upload_path) try: if file.file_path: file_path = os.path.basename(file.file_path) - local_download_path = os.path.join(unique_directory,file_path) + local_download_path = os.path.join(self.unique_dir_path, file_path) with open(local_download_path, 'wb') as blob: blob.write(file.get_stream()) logger.info(f' File downloaded to location: {local_download_path}') @@ -80,14 +87,11 @@ def get_unique_id(self) -> str: unique_id = uuid.uuid1().hex[0:24] return unique_id - - @staticmethod def clean_up(path): if os.path.isfile(path): logger.info(f' Removing File: {path}') os.remove(path) else: - # folder = os.path.join(DOWNLOAD_FILE_PATH, path) logger.info(f' Removing Folder: {path}') shutil.rmtree(path, ignore_errors=False)