From ed676db3b467fdac6a922b1478b7b5a2a15208fd Mon Sep 17 00:00:00 2001 From: pdvass Date: Tue, 21 Mar 2023 04:32:23 +0200 Subject: [PATCH 1/3] Add cache functionality and unittest --- crossref/restful.py | 35 ++++++++++++++++++++++++++++++++++- tests/test_restful.py | 9 +++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/crossref/restful.py b/crossref/restful.py index b837965..319c0fa 100644 --- a/crossref/restful.py +++ b/crossref/restful.py @@ -1,10 +1,12 @@ # coding: utf-8 import requests +import requests_cache from time import sleep from crossref import validators, VERSION + LIMIT = 100 MAXOFFSET = 10000 FACETS_MAX_LIMIT = 1000 @@ -134,6 +136,7 @@ class Endpoint: def __init__( self, + backend=None, request_url=None, request_params=None, context=None, @@ -141,7 +144,10 @@ def __init__( throttle=True, crossref_plus_token=None, timeout=30, + ): + if backend: + requests_cache.install_cache(cache_name='crossref_cache', backend=backend) self.do_http_request = HTTPRequest(throttle=throttle).do_http_request self.etiquette = etiquette or Etiquette() self.custom_header = {"user-agent": str(self.etiquette)} @@ -575,6 +581,10 @@ class Works(Endpoint): "update-type": None, } + def __init__(self, backend=None, request_url=None, request_params=None, context=None, etiquette=None, throttle=True, crossref_plus_token=None, timeout=30): + super().__init__(backend, request_url, request_params, context, etiquette, throttle, crossref_plus_token, timeout) + self.backend = backend + def order(self, order="asc"): """ This method retrieve an iterable object that implements the method @@ -628,6 +638,7 @@ def order(self, order="asc"): request_params["order"] = order return self.__class__( + backend=self.backend, request_url=request_url, request_params=request_params, context=context, @@ -726,6 +737,7 @@ def select(self, *args): ) return self.__class__( + backend=self.backend, request_url=request_url, request_params=request_params, context=context, @@ -786,6 +798,7 @@ def sort(self, sort="score"): request_params["sort"] = sort return self.__class__( + backend=self.backend, request_url=request_url, request_params=request_params, context=context, @@ -838,6 +851,7 @@ def filter(self, **kwargs): request_params["filter"] += "," + decoded_fltr + ":" + str(value) return self.__class__( + backend=self.backend, request_url=request_url, request_params=request_params, context=context, @@ -924,7 +938,8 @@ def query(self, *args, **kwargs): ) request_params["query.%s" % field.replace("_", "-")] = value - return self.__class__(request_url=request_url, + return self.__class__(backend=self.backend, + request_url=request_url, request_params=request_params, context=context, etiquette=self.etiquette, @@ -968,6 +983,7 @@ def sample(self, sample_size=20): request_params["sample"] = sample_size return self.__class__( + backend=self.backend, request_url=request_url, request_params=request_params, context=context, @@ -1114,6 +1130,10 @@ class Funders(Endpoint): "location": None, } + def __init__(self, backend=None, request_url=None, request_params=None, context=None, etiquette=None, throttle=True, crossref_plus_token=None, timeout=30): + super().__init__(backend, request_url, request_params, context, etiquette, throttle, crossref_plus_token, timeout) + self.backend = backend + def query(self, *args): """ This method retrieve an iterable object that implements the method @@ -1142,6 +1162,7 @@ def query(self, *args): request_params["query"] = " ".join([str(i) for i in args]) return self.__class__( + backend=self.backend, request_url=request_url, request_params=request_params, etiquette=self.etiquette, @@ -1194,6 +1215,7 @@ def filter(self, **kwargs): request_params["filter"] += "," + decoded_fltr + ":" + str(value) return self.__class__( + backend=self.backend, request_url=request_url, request_params=request_params, context=context, @@ -1300,6 +1322,10 @@ class Members(Endpoint): "current-doi-count": validators.is_integer, } + def __init__(self, backend=None, request_url=None, request_params=None, context=None, etiquette=None, throttle=True, crossref_plus_token=None, timeout=30): + super().__init__(backend, request_url, request_params, context, etiquette, throttle, crossref_plus_token, timeout) + self.backend = backend + def query(self, *args): """ This method retrieve an iterable object that implements the method @@ -1346,6 +1372,7 @@ def query(self, *args): request_params["query"] = " ".join([str(i) for i in args]) return self.__class__( + backend=self.backend, request_url=request_url, request_params=request_params, context=context, @@ -1398,6 +1425,7 @@ def filter(self, **kwargs): request_params["filter"] += "," + decoded_fltr + ":" + str(value) return self.__class__( + backend=self.backend, request_url=request_url, request_params=request_params, context=context, @@ -1694,6 +1722,10 @@ class Journals(Endpoint): ENDPOINT = "journals" + def __init__(self, backend=None, request_url=None, request_params=None, context=None, etiquette=None, throttle=True, crossref_plus_token=None, timeout=30): + super().__init__(backend, request_url, request_params, context, etiquette, throttle, crossref_plus_token, timeout) + self.backend=backend + def query(self, *args): """ This method retrieve an iterable object that implements the method @@ -1723,6 +1755,7 @@ def query(self, *args): request_params["query"] = " ".join([str(i) for i in args]) return self.__class__( + backend=self.backend, request_url=request_url, request_params=request_params, context=context, diff --git a/tests/test_restful.py b/tests/test_restful.py index 95619db..0a497fe 100644 --- a/tests/test_restful.py +++ b/tests/test_restful.py @@ -1,6 +1,7 @@ # coding: utf-8 import unittest +import pathlib as pl from crossref import restful from crossref import VERSION @@ -67,6 +68,14 @@ def test_work_with_sample_and_filters(self): self.assertEqual(result, 'https://api.crossref.org/works?filter=type%3Ajournal-article&sample=5') + def test_work_with_backend(self): + _ = restful.Works(backend='filesystem') + path = 'crossref_cache' + + if not pl.Path(path).resolve().is_dir(): + raise AssertionError("File does not exist: %s" % str(path)) + + def test_members_filters(self): result = restful.Members(etiquette=self.etiquette).filter(has_public_references="true").url From 65be863d2aa4f5ea4b34155219516a33fb89b5e3 Mon Sep 17 00:00:00 2001 From: Pantelis Vassilopoulos Date: Tue, 21 Mar 2023 20:05:32 +0200 Subject: [PATCH 2/3] Add requests-cache to setup.py --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2e623c3..dd52f21 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,8 @@ from crossref import VERSION install_requires = [ - 'requests>=2.11.1' + 'requests>=2.11.1', + 'requests-cache>=1.0.0', ] tests_require = [] From 4153439b15fc740423448494e864565ab4c45b9e Mon Sep 17 00:00:00 2001 From: pdvass Date: Thu, 20 Apr 2023 17:34:27 +0300 Subject: [PATCH 3/3] Put backend at the end of arguent list --- crossref/restful.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/crossref/restful.py b/crossref/restful.py index 319c0fa..10e4223 100644 --- a/crossref/restful.py +++ b/crossref/restful.py @@ -581,8 +581,8 @@ class Works(Endpoint): "update-type": None, } - def __init__(self, backend=None, request_url=None, request_params=None, context=None, etiquette=None, throttle=True, crossref_plus_token=None, timeout=30): - super().__init__(backend, request_url, request_params, context, etiquette, throttle, crossref_plus_token, timeout) + def __init__(self, request_url=None, request_params=None, context=None, etiquette=None, throttle=True, crossref_plus_token=None, timeout=30, backend=None): + super().__init__(request_url, request_params, context, etiquette, throttle, crossref_plus_token, timeout, backend) self.backend = backend def order(self, order="asc"): @@ -638,12 +638,12 @@ def order(self, order="asc"): request_params["order"] = order return self.__class__( - backend=self.backend, request_url=request_url, request_params=request_params, context=context, etiquette=self.etiquette, timeout=self.timeout, + backend=self.backend, ) def select(self, *args): @@ -737,12 +737,12 @@ def select(self, *args): ) return self.__class__( - backend=self.backend, request_url=request_url, request_params=request_params, context=context, etiquette=self.etiquette, timeout=self.timeout, + backend=self.backend, ) def sort(self, sort="score"): @@ -798,12 +798,12 @@ def sort(self, sort="score"): request_params["sort"] = sort return self.__class__( - backend=self.backend, request_url=request_url, request_params=request_params, context=context, etiquette=self.etiquette, timeout=self.timeout, + backend=self.backend, ) def filter(self, **kwargs): @@ -851,12 +851,12 @@ def filter(self, **kwargs): request_params["filter"] += "," + decoded_fltr + ":" + str(value) return self.__class__( - backend=self.backend, request_url=request_url, request_params=request_params, context=context, etiquette=self.etiquette, timeout=self.timeout, + backend=self.backend, ) def facet(self, facet_name, facet_count=100): @@ -938,12 +938,12 @@ def query(self, *args, **kwargs): ) request_params["query.%s" % field.replace("_", "-")] = value - return self.__class__(backend=self.backend, - request_url=request_url, + return self.__class__(request_url=request_url, request_params=request_params, context=context, etiquette=self.etiquette, - timeout=self.timeout) + timeout=self.timeout, + backend=self.backend,) def sample(self, sample_size=20): """ @@ -983,12 +983,12 @@ def sample(self, sample_size=20): request_params["sample"] = sample_size return self.__class__( - backend=self.backend, request_url=request_url, request_params=request_params, context=context, etiquette=self.etiquette, timeout=self.timeout, + backend=self.backend, ) def doi(self, doi, only_message=True): @@ -1130,8 +1130,8 @@ class Funders(Endpoint): "location": None, } - def __init__(self, backend=None, request_url=None, request_params=None, context=None, etiquette=None, throttle=True, crossref_plus_token=None, timeout=30): - super().__init__(backend, request_url, request_params, context, etiquette, throttle, crossref_plus_token, timeout) + def __init__(self, request_url=None, request_params=None, context=None, etiquette=None, throttle=True, crossref_plus_token=None, timeout=30, backend=None,): + super().__init__(request_url, request_params, context, etiquette, throttle, crossref_plus_token, timeout, backend,) self.backend = backend def query(self, *args): @@ -1162,11 +1162,11 @@ def query(self, *args): request_params["query"] = " ".join([str(i) for i in args]) return self.__class__( - backend=self.backend, request_url=request_url, request_params=request_params, etiquette=self.etiquette, timeout=self.timeout, + backend=self.backend, ) def filter(self, **kwargs): @@ -1215,12 +1215,12 @@ def filter(self, **kwargs): request_params["filter"] += "," + decoded_fltr + ":" + str(value) return self.__class__( - backend=self.backend, request_url=request_url, request_params=request_params, context=context, etiquette=self.etiquette, timeout=self.timeout, + backend=self.backend, ) def funder(self, funder_id, only_message=True): @@ -1322,8 +1322,8 @@ class Members(Endpoint): "current-doi-count": validators.is_integer, } - def __init__(self, backend=None, request_url=None, request_params=None, context=None, etiquette=None, throttle=True, crossref_plus_token=None, timeout=30): - super().__init__(backend, request_url, request_params, context, etiquette, throttle, crossref_plus_token, timeout) + def __init__(self, request_url=None, request_params=None, context=None, etiquette=None, throttle=True, crossref_plus_token=None, timeout=30, backend=None,): + super().__init__(request_url, request_params, context, etiquette, throttle, crossref_plus_token, timeout, backend) self.backend = backend def query(self, *args): @@ -1372,12 +1372,12 @@ def query(self, *args): request_params["query"] = " ".join([str(i) for i in args]) return self.__class__( - backend=self.backend, request_url=request_url, request_params=request_params, context=context, etiquette=self.etiquette, timeout=self.timeout, + backend=self.backend, ) def filter(self, **kwargs): @@ -1425,12 +1425,12 @@ def filter(self, **kwargs): request_params["filter"] += "," + decoded_fltr + ":" + str(value) return self.__class__( - backend=self.backend, request_url=request_url, request_params=request_params, context=context, etiquette=self.etiquette, timeout=self.timeout, + backend=self.backend, ) def member(self, member_id, only_message=True): @@ -1722,8 +1722,8 @@ class Journals(Endpoint): ENDPOINT = "journals" - def __init__(self, backend=None, request_url=None, request_params=None, context=None, etiquette=None, throttle=True, crossref_plus_token=None, timeout=30): - super().__init__(backend, request_url, request_params, context, etiquette, throttle, crossref_plus_token, timeout) + def __init__(self, request_url=None, request_params=None, context=None, etiquette=None, throttle=True, crossref_plus_token=None, timeout=30, backend=None): + super().__init__(request_url, request_params, context, etiquette, throttle, crossref_plus_token, timeout, backend) self.backend=backend def query(self, *args): @@ -1755,12 +1755,12 @@ def query(self, *args): request_params["query"] = " ".join([str(i) for i in args]) return self.__class__( - backend=self.backend, request_url=request_url, request_params=request_params, context=context, etiquette=self.etiquette, timeout=self.timeout, + backend=self.backend, ) def journal(self, issn, only_message=True):