From b3a96e773d0bdb9250404e9698ce16b59b58738c Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 1 Oct 2024 14:05:02 +0530 Subject: [PATCH 1/2] Updated validator package - Updated validator package - Removed the folder once the processing is complete --- requirements.txt | 2 +- src/osw_validator.py | 3 --- src/validation.py | 28 ++++++++++++++-------------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/requirements.txt b/requirements.txt index c8de9a2..b5d176e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ pydantic==1.10.4 python-ms-core==0.0.22 uvicorn==0.20.0 html_testRunner==1.2.1 -python-osw-validation==0.2.4 \ No newline at end of file +python-osw-validation==0.2.6 \ No newline at end of file diff --git a/src/osw_validator.py b/src/osw_validator.py index d5467b0..c7229dc 100644 --- a/src/osw_validator.py +++ b/src/osw_validator.py @@ -1,6 +1,4 @@ -import uuid import logging -import datetime import urllib.parse from typing import List from python_ms_core import Core @@ -90,7 +88,6 @@ def send_status(self, result: ValidationResult, upload_message: Upload): except Exception as e: logger.error(f'Error occurred while publishing message for : {upload_message.message_id} with error: {e}') - def has_permission(self, roles: List[str], queue_message: Upload) -> bool: try: permission_request = PermissionRequest( diff --git a/src/validation.py b/src/validation.py index dfc3e45..97e208b 100644 --- a/src/validation.py +++ b/src/validation.py @@ -10,7 +10,7 @@ ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) # Path used for download file generation. -DOWNLOAD_FILE_PATH = f'{Path.cwd()}/downloads' +DOWNLOAD_DIR = f'{Path.cwd()}/downloads' logging.basicConfig() logger = logging.getLogger('OSW_VALIDATION') @@ -25,9 +25,19 @@ def __init__(self, file_path=None, storage_client=None): self.file_path = file_path self.file_relative_path = file_path.split('/')[-1] self.client = self.storage_client.get_container(container_name=self.container_name) + is_exists = os.path.exists(DOWNLOAD_DIR) + unique_id = self.get_unique_id() + if not is_exists: + os.makedirs(DOWNLOAD_DIR) + self.unique_dir_path = os.path.join(DOWNLOAD_DIR, unique_id) + if not os.path.exists(self.unique_dir_path): + os.makedirs(self.unique_dir_path) def validate(self, max_errors=20) -> ValidationResult: - return self.is_osw_valid(max_errors) + try: + return self.is_osw_valid(max_errors) + finally: + Validation.clean_up(self.unique_dir_path) def is_osw_valid(self, max_errors) -> ValidationResult: result = ValidationResult() @@ -52,19 +62,12 @@ def is_osw_valid(self, max_errors) -> ValidationResult: # Downloads the single file into a unique directory def download_single_file(self, file_upload_path=None) -> str: - is_exists = os.path.exists(DOWNLOAD_FILE_PATH) - unique_id = self.get_unique_id() - if not is_exists: - os.makedirs(DOWNLOAD_FILE_PATH) - unique_directory = os.path.join(DOWNLOAD_FILE_PATH,unique_id) - if not os.path.exists(unique_directory): - os.makedirs(unique_directory) - + file = self.storage_client.get_file_from_url(self.container_name, file_upload_path) try: if file.file_path: file_path = os.path.basename(file.file_path) - local_download_path = os.path.join(unique_directory,file_path) + local_download_path = os.path.join(self.unique_dir_path, file_path) with open(local_download_path, 'wb') as blob: blob.write(file.get_stream()) logger.info(f' File downloaded to location: {local_download_path}') @@ -80,14 +83,11 @@ def get_unique_id(self) -> str: unique_id = uuid.uuid1().hex[0:24] return unique_id - - @staticmethod def clean_up(path): if os.path.isfile(path): logger.info(f' Removing File: {path}') os.remove(path) else: - # folder = os.path.join(DOWNLOAD_FILE_PATH, path) logger.info(f' Removing Folder: {path}') shutil.rmtree(path, ignore_errors=False) From 283e6301559a0b37831ef050f5d6c355034afeea Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 1 Oct 2024 14:22:42 +0530 Subject: [PATCH 2/2] Added time taken --- src/validation.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/validation.py b/src/validation.py index 97e208b..12f623c 100644 --- a/src/validation.py +++ b/src/validation.py @@ -1,4 +1,6 @@ import os +import uuid +import time import shutil import logging import traceback @@ -6,7 +8,6 @@ from .config import Settings from python_osw_validation import OSWValidation from .models.queue_message_content import ValidationResult -import uuid ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) # Path used for download file generation. @@ -40,6 +41,7 @@ def validate(self, max_errors=20) -> ValidationResult: Validation.clean_up(self.unique_dir_path) def is_osw_valid(self, max_errors) -> ValidationResult: + start_time = time.time() result = ValidationResult() result.is_valid = False result.validation_message = '' @@ -57,7 +59,9 @@ def is_osw_valid(self, max_errors) -> ValidationResult: else: result.validation_message = 'Failed to validate because unknown file format' logger.error(f' Failed to validate because unknown file format') - + end_time = time.time() + time_taken = end_time - start_time + logger.info(f'Validation completed in {time_taken} seconds') return result # Downloads the single file into a unique directory