From 2275fccb38ee806d3841ab10f236dffee5d7ef21 Mon Sep 17 00:00:00 2001 From: aycz Date: Mon, 17 Feb 2025 15:48:36 -0600 Subject: [PATCH 1/6] Adds all required functionality...i think. --- Exercises/Exercise-1/.gitignore | 1 + Exercises/Exercise-1/main.py | 48 +++++++++++++++++++++++++-- Exercises/Exercise-1/requirements.txt | 3 +- 3 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 Exercises/Exercise-1/.gitignore diff --git a/Exercises/Exercise-1/.gitignore b/Exercises/Exercise-1/.gitignore new file mode 100644 index 00000000..10a638a6 --- /dev/null +++ b/Exercises/Exercise-1/.gitignore @@ -0,0 +1 @@ +.virtual diff --git a/Exercises/Exercise-1/main.py b/Exercises/Exercise-1/main.py index 95eff512..f9448deb 100644 --- a/Exercises/Exercise-1/main.py +++ b/Exercises/Exercise-1/main.py @@ -1,4 +1,10 @@ -import requests +from io import BytesIO +import os +import re +import requests as r +import zipfile as zp + +import pandas as pd download_uris = [ "https://divvy-tripdata.s3.amazonaws.com/Divvy_Trips_2018_Q4.zip", @@ -10,10 +16,46 @@ "https://divvy-tripdata.s3.amazonaws.com/Divvy_Trips_2220_Q1.zip", ] +dir_path = '' + def main(): - # your code here - pass + ''' docstring''' + # TO DO: + # download files + global dir_path + dir_path = _create_downloads_dir() + + for uri in download_uris: + file_name = _extract_file_name(uri) + _make_file(uri, file_name) + + return + + +def _create_downloads_dir(): + ''' docstring''' + current_path, new_dir = os.getcwd(), 'downloads' + new_path = os.path.join(current_path, new_dir) + if not os.path.isdir(new_path): + os.mkdir(new_path) + return new_path + + +def _extract_file_name(uri: str): + file_name = re.search(r'([^\/]+)(?=\.zip$)', uri).group(1) + if file_name: + return ''.join([file_name, '.csv']) + return '' + + +def _make_file(uri: str, file_name: str): + ''' docstring''' + res = r.get(uri) + if res.ok: + _ = (zp.ZipFile(BytesIO(res.content)) + .extract(member=file_name, path=dir_path)) + return if __name__ == "__main__": diff --git a/Exercises/Exercise-1/requirements.txt b/Exercises/Exercise-1/requirements.txt index 9cdfca0c..08f1190e 100644 --- a/Exercises/Exercise-1/requirements.txt +++ b/Exercises/Exercise-1/requirements.txt @@ -1 +1,2 @@ -requests==2.27.1 \ No newline at end of file +pandas +requests==2.27.1 From 5dc81ad7e776f9de85e19cd7ddd0b01f450e7910 Mon Sep 17 00:00:00 2001 From: aycz Date: Wed, 19 Feb 2025 12:41:26 -0600 Subject: [PATCH 2/6] refactor --- Exercises/Exercise-1/main.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/Exercises/Exercise-1/main.py b/Exercises/Exercise-1/main.py index f9448deb..869762ac 100644 --- a/Exercises/Exercise-1/main.py +++ b/Exercises/Exercise-1/main.py @@ -1,6 +1,7 @@ from io import BytesIO import os import re +from typing import List import requests as r import zipfile as zp @@ -20,20 +21,24 @@ def main(): - ''' docstring''' - # TO DO: - # download files + ''' ''' global dir_path dir_path = _create_downloads_dir() - for uri in download_uris: + _process_uris(download_uris) + + return + + +def _process_uris(uri_set: List) -> None: + ''' docstring ''' + for uri in uri_set: file_name = _extract_file_name(uri) _make_file(uri, file_name) - return -def _create_downloads_dir(): +def _create_downloads_dir() -> str: ''' docstring''' current_path, new_dir = os.getcwd(), 'downloads' new_path = os.path.join(current_path, new_dir) @@ -42,17 +47,21 @@ def _create_downloads_dir(): return new_path -def _extract_file_name(uri: str): - file_name = re.search(r'([^\/]+)(?=\.zip$)', uri).group(1) - if file_name: +def _extract_file_name(uri: str) -> str: + ''' docstring ''' + match = re.search(r'([^\/]+)(?=\.zip$)', uri) + if match: + file_name = match.group(1) return ''.join([file_name, '.csv']) return '' -def _make_file(uri: str, file_name: str): +def _make_file(uri: str, file_name: str) -> None: ''' docstring''' res = r.get(uri) if res.ok: + print('#' * 50) + print(uri, file_name) _ = (zp.ZipFile(BytesIO(res.content)) .extract(member=file_name, path=dir_path)) return From a0d05cc416d8144feaf1218078343b3342266a45 Mon Sep 17 00:00:00 2001 From: aycz Date: Wed, 19 Feb 2025 12:42:09 -0600 Subject: [PATCH 3/6] Adds unit testing for all logic. --- .../test_main.cpython-313-pytest-8.3.4.pyc | Bin 0 -> 4966 bytes Exercises/Exercise-1/tests/test_main.py | 90 ++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 Exercises/Exercise-1/tests/__pycache__/test_main.cpython-313-pytest-8.3.4.pyc create mode 100644 Exercises/Exercise-1/tests/test_main.py diff --git a/Exercises/Exercise-1/tests/__pycache__/test_main.cpython-313-pytest-8.3.4.pyc b/Exercises/Exercise-1/tests/__pycache__/test_main.cpython-313-pytest-8.3.4.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5469895fb2c0d9394d2730bdbd44755ebb9ace0c GIT binary patch literal 4966 zcmcIn&2JmW6`$Q5l1oygtxw95Wl>7}v64*LGUYgRWW)(#G&16#ToYBJ00c#@Z9){6 zoh2=s1zHs?P#t|S(4$eHfPHgOpy;80K#yW1m+Z7f4WwwBn;hljOZ(pJ2bYrK#6UZc z-pu=&c{?A!_qZ2}MF_OTlfS$D6^uSd#%*C|ut#M=9ul4C(sjZl(aVd9B`3)uWq~AW z`0P)8p$Nx#X5cFjn3E| z=>a`BE9s%MtgFz6p^rcxg}w*+81!-IkEG?jV1W+p-uB=?QNvyfZHwc#V0+V4f45c|_#n zZnT0+c5xFfr%!fP{Mua^ye(;Oe1Us*Lz6hI=4*GL4?thIqftJXD=;HpGjc_9y;3&w zMJrb}AZY{rR2mz1fs?01HmX)islGQkhbS7CdEO0>M zmm!d(%!~3e0WTi~Nm3R8AtV<~N*^yV-Wfk{I zB*|@}$v3|E4>V2TJFm9Q-<^cU6SMhhbrv<2pfDS3WA#4o71sVEWXHh?k;A$KWP|LE zkm>mydit-l_raSxG|{FbEjrhvb5GO1mw$6fX7-FNB*kA7vg|+$AsPSua4Z0BB>?% zEzdg^I$aJr1|)-LwXaQj>s|&;akOg{0nXhHlLYRC6SEszeyGcy^ih8{@CXWhG6Udjs#n$+k!d8T^*oMt5#7Lqu;pjljDhNa0|0R@r^jkx_r zguy%-RPWDg5r$8aN8YX87k4?&c&~UxIc8K@!Cb56nZbKNom;7w%C%C(;=!^B@66)E zIqQyDFBgSr?snN+$(L=23+Jr-EdyuZ@vGuxmdwfrjG2u0=Wdy7Enfo=s+RMWd<`@V z&W~s+kJwKYudFi`*O^C!+p(WhS1O4nw<8l9??3OG-lEf6ebde8bc;HFpV8^f3m#_=o^{EZ9S#)W zzRwz1Flp0xi=JxIQ;#gLVq>s&BL=uMlZ4$e3N@ps1vI%azHY3nlxp{(vr0811;;ci z@(bjMRHIaF^mtssy89{zNFzQ6u&s{nL? zuMfJE|2cGdNz#q&TS1rS1<2++*|js@Xj*2^`ndT&H#lVFYjH`j%FX&d#@T9{!@VkeQDLUI}jem&EW zOd**@G6O^las?hC!;H?}LUJC6A6zc@nRyE*7ck@C%#gsD10@H$0(Q8(^&=nvGD9Su ze57p0-rTs>j`lXAV=e0ZeMZMNFM6Ckc-AFrc5$}FpXk_wCC>{ie4HoVreikH%m|GLFwpr~G$QHG*q^y;tFc zmcC#1^h@lpYNqVfOw0J0qAjI66)CK62#{oO1NJq4KF3jvCa9#K@R;9V-24hK00^AwK> z9W#Dd$C{sA#$`r;@T1~+_?`q03kg06&&z%vxUfU#{uxTN z)q$3Jx~ZPtQMCi(iKaTyQnjY4wbZLk_39r-+8<_i6%u|=`jSZDo6>HOga=#dsit~r zM?E9_{bRrX<)(W1nYtiwLmT3%TN(-+<;QY%F`2Tr9SXb@Gu9bYDm?6oc*v0n6ozxP z#9{MTy;7>-KBe#s7``zcwCCd#pT8(rZTuF;uc8HL*eZ`gu?jz$Ypm2T#1Ao1#tE1e zd;Nrni0$G}eP_)q*2~5f_5+v!(ZIR^bXSoi>FJKe)T4p6HBrqrpIMFA$dH z7Xw5NZH1EC(!`&o;8tjOTRQearqWeuPa*OMlF7Y6Fx(wR;fuI1PdqZ4(naAg`uJp1 zItRmzpKMDBbe%z`VI=2~ytNwy*)BMCMcx0U+=)S!PS}eKz-TwJ&vedj3hQs&``Ne` eEUx_HZE5I*JS<(o^%6)#;Jsqb7bBn$`~6>_HXLmL literal 0 HcmV?d00001 diff --git a/Exercises/Exercise-1/tests/test_main.py b/Exercises/Exercise-1/tests/test_main.py new file mode 100644 index 00000000..eb020cbb --- /dev/null +++ b/Exercises/Exercise-1/tests/test_main.py @@ -0,0 +1,90 @@ +from os import path +from unittest.mock import patch + +import pytest + +from main import _create_downloads_dir, _extract_file_name, _make_file, _process_uris + + +@pytest.mark.parametrize( + 'uris, test_fname', + [ + (['uri.one'], 'one'), + (['uri.two'], 'two'), + (['uri.three'], 'three'), + (['uri.four'], 'four'), + (['uri.five'], 'five') + ] +) +@patch('main._extract_file_name') +@patch('main._make_file') +def test_process_uris(mock_make_file, mock_extract_file_name, uris, test_fname): + mock_extract_file_name.return_value = test_fname + _process_uris(uris) + mock_extract_file_name.assert_called_once_with(uris[-1]) + mock_make_file.assert_called_once_with(uris[-1], test_fname) + + +@patch('main.os.getcwd', return_value='/current/path') +@patch('main.os.path.isdir', return_value=False) +@patch('main.os.mkdir') +def test_create_downloads_dir(mock_mkdir, mock_isdir, mock_getcwd): + res = _create_downloads_dir() + + assert res == '/current/path/downloads' + mock_mkdir.assert_called_once_with(res) + + +@pytest.mark.parametrize( + 'uri, expected', + [ + ('webbitywebsite.com/file_one.zip', 'file_one.csv'), + ('webbitywebsite.com/file_two.zip', 'file_two.csv'), + ('webbitywebsite.com/resource/file_three.zip', 'file_three.csv'), + ('webbitywebsite.com/file_four.csv', ''), + ('webbitywebsite.com/file_five.zzip', '') + ] +) +def test_extract_file_name(uri, expected): + result = _extract_file_name(uri) + assert result == expected + + +@pytest.mark.parametrize( # possibly future refactor to: pytest.generate_tests() + 'uri, test_fname, test_response_ok, test_path', + [ + ('uri.one', 'one', True, ''), + ('uri.two', 'two', False, ''), + ('uri.three', 'three', False, ''), + ('uri.four', 'four', True, ''), + ('uri.five', 'five', True, '') + ] +) +@patch('main.r') +@patch('main.zp.ZipFile') +@patch('main.BytesIO') +def test_make_file(mock_bytesio, + mock_zipfile, + mock_requests, + uri, + test_fname, + test_response_ok, + test_path): + + mock_response = mock_requests.Response() + mock_response.ok = test_response_ok + mock_requests.get.return_value = mock_response + + _make_file(uri, test_fname) + + mock_requests.get.assert_called_once_with(uri) + + if test_response_ok: + mock_bytesio.assert_called_once() + mock_zipfile.assert_called_once() + mock_zipfile.return_value.extract.assert_called_once_with(member=test_fname, + path=test_path) + else: + mock_bytesio.assert_not_called() + mock_zipfile.assert_not_called() + mock_zipfile.return_value.extract.assert_not_called() From 00e130e30eeaf98b77c46452b9dd4746ab2e7074 Mon Sep 17 00:00:00 2001 From: aycz Date: Wed, 19 Feb 2025 12:43:04 -0600 Subject: [PATCH 4/6] Updates dependencies and yaml. --- Exercises/Exercise-1/docker-compose.yml | 3 +-- Exercises/Exercise-1/requirements.txt | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Exercises/Exercise-1/docker-compose.yml b/Exercises/Exercise-1/docker-compose.yml index 0323fa55..26b4549a 100644 --- a/Exercises/Exercise-1/docker-compose.yml +++ b/Exercises/Exercise-1/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.9" services: test: image: "exercise-1" @@ -9,4 +8,4 @@ services: image: "exercise-1" volumes: - .:/app - command: python3 main.py \ No newline at end of file + command: python3 main.py diff --git a/Exercises/Exercise-1/requirements.txt b/Exercises/Exercise-1/requirements.txt index 08f1190e..b88770cd 100644 --- a/Exercises/Exercise-1/requirements.txt +++ b/Exercises/Exercise-1/requirements.txt @@ -1,2 +1,3 @@ pandas +pytest requests==2.27.1 From ccf265b51e74546c4600d03cdd578509f19fb66a Mon Sep 17 00:00:00 2001 From: aycz Date: Wed, 19 Feb 2025 12:51:43 -0600 Subject: [PATCH 5/6] Adds full docstrings. --- Exercises/Exercise-1/main.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/Exercises/Exercise-1/main.py b/Exercises/Exercise-1/main.py index 869762ac..23f7628e 100644 --- a/Exercises/Exercise-1/main.py +++ b/Exercises/Exercise-1/main.py @@ -1,11 +1,10 @@ -from io import BytesIO import os import re -from typing import List -import requests as r import zipfile as zp +from io import BytesIO +from typing import List -import pandas as pd +import requests as r download_uris = [ "https://divvy-tripdata.s3.amazonaws.com/Divvy_Trips_2018_Q4.zip", @@ -21,7 +20,7 @@ def main(): - ''' ''' + ''' main ''' global dir_path dir_path = _create_downloads_dir() @@ -31,7 +30,10 @@ def main(): def _process_uris(uri_set: List) -> None: - ''' docstring ''' + ''' High level helper that calls all uri processing functions. + params: + uri_set: List - uris to process. + ''' for uri in uri_set: file_name = _extract_file_name(uri) _make_file(uri, file_name) @@ -39,7 +41,12 @@ def _process_uris(uri_set: List) -> None: def _create_downloads_dir() -> str: - ''' docstring''' + ''' Creates the downloads directory if not currently present. + We use os.getcwd and path.join to have a path agnostic of + system. + params: + None + ''' current_path, new_dir = os.getcwd(), 'downloads' new_path = os.path.join(current_path, new_dir) if not os.path.isdir(new_path): @@ -48,7 +55,11 @@ def _create_downloads_dir() -> str: def _extract_file_name(uri: str) -> str: - ''' docstring ''' + ''' Creates the full file name with csv extension from list + of uris. + params: + uri: string - uri used for file name extraction + ''' match = re.search(r'([^\/]+)(?=\.zip$)', uri) if match: file_name = match.group(1) @@ -57,13 +68,18 @@ def _extract_file_name(uri: str) -> str: def _make_file(uri: str, file_name: str) -> None: - ''' docstring''' + ''' Makes the GET request, and given valid response, we proceed + with writing the file to disk. + params: + uri: string - uri used for request + file_name: string - created by way of extracting from uri + ''' res = r.get(uri) if res.ok: print('#' * 50) print(uri, file_name) _ = (zp.ZipFile(BytesIO(res.content)) - .extract(member=file_name, path=dir_path)) + .extract(member=file_name, path=dir_path)) # went this way as opposed to context wrapping an open file to write return From ba79583847e1fb3fbf02e8e4c23113d81727e771 Mon Sep 17 00:00:00 2001 From: aycz Date: Wed, 19 Feb 2025 12:56:48 -0600 Subject: [PATCH 6/6] Removes lingering print statements. --- Exercises/Exercise-1/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Exercises/Exercise-1/main.py b/Exercises/Exercise-1/main.py index 23f7628e..35720b6e 100644 --- a/Exercises/Exercise-1/main.py +++ b/Exercises/Exercise-1/main.py @@ -76,8 +76,6 @@ def _make_file(uri: str, file_name: str) -> None: ''' res = r.get(uri) if res.ok: - print('#' * 50) - print(uri, file_name) _ = (zp.ZipFile(BytesIO(res.content)) .extract(member=file_name, path=dir_path)) # went this way as opposed to context wrapping an open file to write return