diff --git a/Exercises/Exercise-1/.gitignore b/Exercises/Exercise-1/.gitignore new file mode 100644 index 00000000..10a638a6 --- /dev/null +++ b/Exercises/Exercise-1/.gitignore @@ -0,0 +1 @@ +.virtual diff --git a/Exercises/Exercise-1/docker-compose.yml b/Exercises/Exercise-1/docker-compose.yml index 0323fa55..26b4549a 100644 --- a/Exercises/Exercise-1/docker-compose.yml +++ b/Exercises/Exercise-1/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.9" services: test: image: "exercise-1" @@ -9,4 +8,4 @@ services: image: "exercise-1" volumes: - .:/app - command: python3 main.py \ No newline at end of file + command: python3 main.py diff --git a/Exercises/Exercise-1/main.py b/Exercises/Exercise-1/main.py index 95eff512..35720b6e 100644 --- a/Exercises/Exercise-1/main.py +++ b/Exercises/Exercise-1/main.py @@ -1,4 +1,10 @@ -import requests +import os +import re +import zipfile as zp +from io import BytesIO +from typing import List + +import requests as r download_uris = [ "https://divvy-tripdata.s3.amazonaws.com/Divvy_Trips_2018_Q4.zip", @@ -10,10 +16,69 @@ "https://divvy-tripdata.s3.amazonaws.com/Divvy_Trips_2220_Q1.zip", ] +dir_path = '' + def main(): - # your code here - pass + ''' main ''' + global dir_path + dir_path = _create_downloads_dir() + + _process_uris(download_uris) + + return + + +def _process_uris(uri_set: List) -> None: + ''' High level helper that calls all uri processing functions. + params: + uri_set: List - uris to process. + ''' + for uri in uri_set: + file_name = _extract_file_name(uri) + _make_file(uri, file_name) + return + + +def _create_downloads_dir() -> str: + ''' Creates the downloads directory if not currently present. + We use os.getcwd and path.join to have a path agnostic of + system. + params: + None + ''' + current_path, new_dir = os.getcwd(), 'downloads' + new_path = os.path.join(current_path, new_dir) + if not os.path.isdir(new_path): + os.mkdir(new_path) + return new_path + + +def _extract_file_name(uri: str) -> str: + ''' Creates the full file name with csv extension from list + of uris. + params: + uri: string - uri used for file name extraction + ''' + match = re.search(r'([^\/]+)(?=\.zip$)', uri) + if match: + file_name = match.group(1) + return ''.join([file_name, '.csv']) + return '' + + +def _make_file(uri: str, file_name: str) -> None: + ''' Makes the GET request, and given valid response, we proceed + with writing the file to disk. + params: + uri: string - uri used for request + file_name: string - created by way of extracting from uri + ''' + res = r.get(uri) + if res.ok: + _ = (zp.ZipFile(BytesIO(res.content)) + .extract(member=file_name, path=dir_path)) # went this way as opposed to context wrapping an open file to write + return if __name__ == "__main__": diff --git a/Exercises/Exercise-1/requirements.txt b/Exercises/Exercise-1/requirements.txt index 9cdfca0c..b88770cd 100644 --- a/Exercises/Exercise-1/requirements.txt +++ b/Exercises/Exercise-1/requirements.txt @@ -1 +1,3 @@ -requests==2.27.1 \ No newline at end of file +pandas +pytest +requests==2.27.1 diff --git a/Exercises/Exercise-1/tests/__pycache__/test_main.cpython-313-pytest-8.3.4.pyc b/Exercises/Exercise-1/tests/__pycache__/test_main.cpython-313-pytest-8.3.4.pyc new file mode 100644 index 00000000..5469895f Binary files /dev/null and b/Exercises/Exercise-1/tests/__pycache__/test_main.cpython-313-pytest-8.3.4.pyc differ diff --git a/Exercises/Exercise-1/tests/test_main.py b/Exercises/Exercise-1/tests/test_main.py new file mode 100644 index 00000000..eb020cbb --- /dev/null +++ b/Exercises/Exercise-1/tests/test_main.py @@ -0,0 +1,90 @@ +from os import path +from unittest.mock import patch + +import pytest + +from main import _create_downloads_dir, _extract_file_name, _make_file, _process_uris + + +@pytest.mark.parametrize( + 'uris, test_fname', + [ + (['uri.one'], 'one'), + (['uri.two'], 'two'), + (['uri.three'], 'three'), + (['uri.four'], 'four'), + (['uri.five'], 'five') + ] +) +@patch('main._extract_file_name') +@patch('main._make_file') +def test_process_uris(mock_make_file, mock_extract_file_name, uris, test_fname): + mock_extract_file_name.return_value = test_fname + _process_uris(uris) + mock_extract_file_name.assert_called_once_with(uris[-1]) + mock_make_file.assert_called_once_with(uris[-1], test_fname) + + +@patch('main.os.getcwd', return_value='/current/path') +@patch('main.os.path.isdir', return_value=False) +@patch('main.os.mkdir') +def test_create_downloads_dir(mock_mkdir, mock_isdir, mock_getcwd): + res = _create_downloads_dir() + + assert res == '/current/path/downloads' + mock_mkdir.assert_called_once_with(res) + + +@pytest.mark.parametrize( + 'uri, expected', + [ + ('webbitywebsite.com/file_one.zip', 'file_one.csv'), + ('webbitywebsite.com/file_two.zip', 'file_two.csv'), + ('webbitywebsite.com/resource/file_three.zip', 'file_three.csv'), + ('webbitywebsite.com/file_four.csv', ''), + ('webbitywebsite.com/file_five.zzip', '') + ] +) +def test_extract_file_name(uri, expected): + result = _extract_file_name(uri) + assert result == expected + + +@pytest.mark.parametrize( # possibly future refactor to: pytest.generate_tests() + 'uri, test_fname, test_response_ok, test_path', + [ + ('uri.one', 'one', True, ''), + ('uri.two', 'two', False, ''), + ('uri.three', 'three', False, ''), + ('uri.four', 'four', True, ''), + ('uri.five', 'five', True, '') + ] +) +@patch('main.r') +@patch('main.zp.ZipFile') +@patch('main.BytesIO') +def test_make_file(mock_bytesio, + mock_zipfile, + mock_requests, + uri, + test_fname, + test_response_ok, + test_path): + + mock_response = mock_requests.Response() + mock_response.ok = test_response_ok + mock_requests.get.return_value = mock_response + + _make_file(uri, test_fname) + + mock_requests.get.assert_called_once_with(uri) + + if test_response_ok: + mock_bytesio.assert_called_once() + mock_zipfile.assert_called_once() + mock_zipfile.return_value.extract.assert_called_once_with(member=test_fname, + path=test_path) + else: + mock_bytesio.assert_not_called() + mock_zipfile.assert_not_called() + mock_zipfile.return_value.extract.assert_not_called()