From 855a1920ead6c795998beee8af2905640ff0cf87 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 8 May 2019 16:53:19 -0300 Subject: [PATCH 1/7] py3 --- .travis.yml | 10 +++++++++- packtrack/__init__.py | 6 +++--- packtrack/bs.py | 9 +++++++++ packtrack/correios.py | 4 ++-- packtrack/royal.py | 3 ++- packtrack/scraping.py | 26 ++++++++++++++++++-------- setup.py | 25 ++++++++++++++++++------- tests/correios_api_test.py | 6 +++++- tests/correios_test.py | 8 ++++++-- tests/scraping_test.py | 11 +++++++---- 10 files changed, 79 insertions(+), 29 deletions(-) create mode 100644 packtrack/bs.py diff --git a/.travis.yml b/.travis.yml index 8e1e41c..11e99bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,13 @@ language: python +python: + - "2.7" + - "3.4" + - "3.5" + - "3.6" + - "3.7" + - "3.8" + before_install: pip install -r requirements.txt -script: nosetests \ No newline at end of file +script: nosetests diff --git a/packtrack/__init__.py b/packtrack/__init__.py index 80d4852..969f14b 100755 --- a/packtrack/__init__.py +++ b/packtrack/__init__.py @@ -1,6 +1,6 @@ -from correios import EncomendaRepository -from royal import RoyalMail -from dhl_gm import DhlGmTracker +from .correios import EncomendaRepository +from .royal import RoyalMail +from .dhl_gm import DhlGmTracker class Correios(object): diff --git a/packtrack/bs.py b/packtrack/bs.py new file mode 100644 index 0000000..23e1023 --- /dev/null +++ b/packtrack/bs.py @@ -0,0 +1,9 @@ +import functools + +try: + from bs4 import BeautifulSoup +except ImportError: + from BeautifulSoup import BeautifulSoup + + +BeautifulSoup = functools.partial(BeautifulSoup, features="lxml") diff --git a/packtrack/correios.py b/packtrack/correios.py index 4514751..72a4f3c 100644 --- a/packtrack/correios.py +++ b/packtrack/correios.py @@ -15,7 +15,7 @@ def get(self, numero, auth=None): return func(numero, **kwargs) def _init_scraper(self, backend): - from scraping import CorreiosWebsiteScraper, CorreiosRastroService + from .scraping import CorreiosWebsiteScraper, CorreiosRastroService if backend is None: backend = 'www2' @@ -37,7 +37,7 @@ def adicionar_status(self, status): d = datetime self.status.append(status) t_format = self.validar_data(status.data) - self.status.sort(lambda x, y: 1 if d.strptime(x.data, t_format) > d.strptime(y.data, t_format) else -1) + self.status.sort(key=lambda x: d.strptime(x.data, t_format)) def validar_data(self, data): if re.match('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$', data): diff --git a/packtrack/royal.py b/packtrack/royal.py index 38d2617..5032854 100644 --- a/packtrack/royal.py +++ b/packtrack/royal.py @@ -1,5 +1,6 @@ import requests -from bs4 import BeautifulSoup + +from .bs import BeautifulSoup class RoyalMail(object): diff --git a/packtrack/scraping.py b/packtrack/scraping.py index fc0c3b0..92babdb 100755 --- a/packtrack/scraping.py +++ b/packtrack/scraping.py @@ -1,15 +1,21 @@ import os import re -from HTMLParser import HTMLParser -from BeautifulSoup import BeautifulSoup +try: + from html.parser import HTMLParser + from html import unescape +except ImportError: + from HTMLParser import HTMLParser + unescape = None + import requests from requests.exceptions import RequestException from zeep import Client as Zeep from zeep.cache import InMemoryCache from zeep.transports import Transport -from correios import Encomenda, Status +from .bs import BeautifulSoup +from .correios import Encomenda, Status class CorreiosWebsiteScraper(object): @@ -65,6 +71,8 @@ def _text(self, value): def _get_all_status_from_html(self, html): status = [] html_parser = HTMLParser() + _unescape = unescape or html_parser.unescape + if ").*', html, re.S) @@ -80,15 +88,17 @@ def _get_all_status_from_html(self, html): except AttributeError: continue for td in tds: - content = td.renderContents().replace('\r', ' ') \ - .split('
') - class_ = td['class'] + content = td.renderContents().decode() + content = content.replace('\r', ' ') + content = content.replace('\xa0', ' ') + content = content.split('
') + class_ = td['class'] and td['class'][0] if class_ == 'sroDtEvent': data = '%s %s' % (content[0].strip(), content[1].strip()) local = '/'.join(self._text(content[2]).rsplit(' / ', 1)).upper() elif class_ == 'sroLbEvent': - situacao = html_parser.unescape(self._text(content[0])) - detalhes = html_parser.unescape(self._text(content[1])) + situacao = _unescape(self._text(content[0])) + detalhes = _unescape(self._text(content[1])) if detalhes: detalhes = u'%s %s' % (situacao, detalhes) status.append(Status(data=data, local=local, diff --git a/setup.py b/setup.py index dfbab09..9e2dad5 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,21 @@ # coding: UTF-8 from setuptools import setup +import sys + +install_requires = [ + 'requests >= 0.14.2', + 'lxml >= 3.0.0', + 'zeep >= 1.6.0', +] +if sys.version_info >= (3, 0): + install_requires.append('beautifulsoup4 >= 4.3.2') +else: + install_requires.append('BeautifulSoup >= 3.1.0') + +tests_require = [ + 'mockito', +] + setup( name='packtrack', @@ -12,11 +28,6 @@ keywords='encomendas track api', url='https://github.com/aleborba/packtrack', long_description='API Python para obter informacoes de encomendas. Para mais detalhes veja a documentacao no Github: https://github.com/aleborba/packtrack/blob/master/README.textile', - install_requires=[ - 'BeautifulSoup >= 3.1.0', - 'requests >= 0.14.2', - 'beautifulsoup4 >= 4.3.2', - 'lxml >= 2.3.5', - 'zeep >= 1.6.0', - ], + install_requires=install_requires, + tests_require=tests_require, ) diff --git a/tests/correios_api_test.py b/tests/correios_api_test.py index 6777ec3..94e4f53 100644 --- a/tests/correios_api_test.py +++ b/tests/correios_api_test.py @@ -1,6 +1,10 @@ import unittest -from mock import Mock +try: + from unittest.mock import Mock +except ImportError: + from mock import Mock + from mockito import when from packtrack import Correios diff --git a/tests/correios_test.py b/tests/correios_test.py index d24712e..26c9d4f 100644 --- a/tests/correios_test.py +++ b/tests/correios_test.py @@ -1,7 +1,11 @@ import unittest -from mock import Mock -from mockito import * +try: + from unittest.mock import Mock +except ImportError: + from mock import Mock + +from mockito import when from packtrack.correios import Encomenda, Status, EncomendaRepository diff --git a/tests/scraping_test.py b/tests/scraping_test.py index 547c605..7aff6f3 100644 --- a/tests/scraping_test.py +++ b/tests/scraping_test.py @@ -2,7 +2,10 @@ import os import unittest -import mock +try: + from unittest import mock +except ImportError: + import mock from packtrack.scraping import CorreiosWebsiteScraper from packtrack.dhl_gm import DhlGmTracker @@ -17,9 +20,9 @@ def _assert_status(self, status, data, local, situacao, detalhes): self.assertEqual(detalhes, status.detalhes) def test_should_get_data_from_correios_website(self): - example_file = open('%s/tests/correios_website/exemplo_rastreamento_correios1.html' % os.getcwd()) - sample_html = example_file.read() - example_file.close() + filename = '%s/tests/correios_website/exemplo_rastreamento_correios1.html' % os.getcwd() + with open(filename, "rb") as f: + sample_html = f.read() http_client_mock = mock.Mock() response_mock = mock.Mock() From 639e3899d0b38ad51dd4f1fabbf5d45cdd5b474a Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 8 May 2019 16:58:37 -0300 Subject: [PATCH 2/7] fix python versions --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 11e99bc..f541d62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,7 @@ python: - "3.4" - "3.5" - "3.6" - - "3.7" - - "3.8" + - "3.7-dev" before_install: pip install -r requirements.txt From 88e4001eb36cbb5ec01976c41bb6da82643d1f47 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 8 May 2019 17:17:18 -0300 Subject: [PATCH 3/7] remove mockito --- requirements.txt | 8 -------- setup.py | 2 +- tests/correios_api_test.py | 10 ++++------ tests/correios_test.py | 6 ++---- 4 files changed, 7 insertions(+), 19 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5e0d12c..0000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -BeautifulSoup==3.2.1 -argparse==1.2.1 -beautifulsoup4==4.3.2 -lxml==2.3.5 -mockito==0.5.1 -requests==2.18.1 -wsgiref==0.1.2 -zeep==1.6.0 diff --git a/setup.py b/setup.py index 9e2dad5..9e2c60c 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,6 @@ install_requires.append('BeautifulSoup >= 3.1.0') tests_require = [ - 'mockito', ] @@ -29,5 +28,6 @@ url='https://github.com/aleborba/packtrack', long_description='API Python para obter informacoes de encomendas. Para mais detalhes veja a documentacao no Github: https://github.com/aleborba/packtrack/blob/master/README.textile', install_requires=install_requires, + test_suite="tests", tests_require=tests_require, ) diff --git a/tests/correios_api_test.py b/tests/correios_api_test.py index 94e4f53..f5b62c4 100644 --- a/tests/correios_api_test.py +++ b/tests/correios_api_test.py @@ -5,8 +5,6 @@ except ImportError: from mock import Mock -from mockito import when - from packtrack import Correios @@ -14,20 +12,20 @@ class CorreiosTest(unittest.TestCase): def test_should_use_repository_to_get_encomenda(self): encomenda_repository_mock = Mock() - when(encomenda_repository_mock).get('123', auth=None) \ - .thenReturn('encomenda123') + encomenda_repository_mock.get.return_value = "encomenda123" Correios._backends[None] = encomenda_repository_mock assert Correios.track('123') == 'encomenda123' + encomenda_repository_mock.get.assert_called_with("123", auth=None) def test_service_should_receive_auth(self): auth = ('mi', 'mimi') encomenda_repository_mock = Mock() - when(encomenda_repository_mock).get('123', auth=auth) \ - .thenReturn('encomenda123') + encomenda_repository_mock.get.return_value = "encomenda123" Correios._backends['service'] = encomenda_repository_mock assert Correios.track( '123', backend='service', auth=auth) == 'encomenda123' + encomenda_repository_mock.get.assert_called_with("123", auth=auth) diff --git a/tests/correios_test.py b/tests/correios_test.py index 26c9d4f..06b8abe 100644 --- a/tests/correios_test.py +++ b/tests/correios_test.py @@ -5,8 +5,6 @@ except ImportError: from mock import Mock -from mockito import when - from packtrack.correios import Encomenda, Status, EncomendaRepository class EncomendaRepositoryTest(unittest.TestCase): @@ -15,8 +13,8 @@ def test_should_get_encomenda_by_numero(self): encomenda_123 = Status(data='2009-01-28 17:49:00') correios_website_scraper_mock = Mock() - when(correios_website_scraper_mock).get_encomenda_info('123', auth=None).thenReturn(encomenda_123) - + correios_website_scraper_mock.get_encomenda_info.return_value = encomenda_123 + repository = EncomendaRepository() repository.correios_website_scraper = correios_website_scraper_mock encomenda = repository.get('123') From 4eca6d2c9a7740b9889d69aceb7d81edb2f9da7f Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 8 May 2019 17:22:47 -0300 Subject: [PATCH 4/7] setup test --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f541d62..72c2f56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,4 @@ python: - "3.6" - "3.7-dev" -before_install: pip install -r requirements.txt - -script: nosetests +script: python setup.py test From 6ca110646eb128bd6c03aa3cff476cac34f8151e Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 8 May 2019 17:26:12 -0300 Subject: [PATCH 5/7] bs4 lxml --- packtrack/bs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packtrack/bs.py b/packtrack/bs.py index 23e1023..8c3b9ae 100644 --- a/packtrack/bs.py +++ b/packtrack/bs.py @@ -4,6 +4,6 @@ from bs4 import BeautifulSoup except ImportError: from BeautifulSoup import BeautifulSoup + BeautifulSoup = functools.partial(BeautifulSoup, features="lxml") - -BeautifulSoup = functools.partial(BeautifulSoup, features="lxml") +BeautifulSoup = BeautifulSoup From 12914bb8d843b558ed857a9f7e577646dd319457 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 8 May 2019 17:27:07 -0300 Subject: [PATCH 6/7] . --- packtrack/bs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packtrack/bs.py b/packtrack/bs.py index 8c3b9ae..59f2676 100644 --- a/packtrack/bs.py +++ b/packtrack/bs.py @@ -2,8 +2,8 @@ try: from bs4 import BeautifulSoup + BeautifulSoup = functools.partial(BeautifulSoup, features="lxml") except ImportError: from BeautifulSoup import BeautifulSoup - BeautifulSoup = functools.partial(BeautifulSoup, features="lxml") BeautifulSoup = BeautifulSoup From d09bfd03412c8601081458aa36b7b308531d0fb3 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 8 May 2019 17:48:52 -0300 Subject: [PATCH 7/7] normalize weird bs behaviour --- packtrack/scraping.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/packtrack/scraping.py b/packtrack/scraping.py index 92babdb..61f6f5d 100755 --- a/packtrack/scraping.py +++ b/packtrack/scraping.py @@ -88,11 +88,16 @@ def _get_all_status_from_html(self, html): except AttributeError: continue for td in tds: - content = td.renderContents().decode() - content = content.replace('\r', ' ') - content = content.replace('\xa0', ' ') - content = content.split('
') - class_ = td['class'] and td['class'][0] + content = td.renderContents() + content = content.replace(b'\r', b' ') + content = content.replace(b'\xa0', b' ') + content = content.replace(b'
', b'
') + content = content.split(b'
') + + # bs4 return a list, bs3 return a string, join normalize + # this behaviour + class_ = "".join(td['class']) + if class_ == 'sroDtEvent': data = '%s %s' % (content[0].strip(), content[1].strip()) local = '/'.join(self._text(content[2]).rsplit(' / ', 1)).upper()