From 7cd944524fd514c121c793eb0cc38f54da469ea0 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Mon, 8 Feb 2021 08:43:54 +0300 Subject: [PATCH 01/24] HH-123745 add weight support --- consul/base.py | 21 +++++++++++++++++++++ tests/test_std.py | 21 +++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/consul/base.py b/consul/base.py index 9d27dbd..2b412be 100755 --- a/consul/base.py +++ b/consul/base.py @@ -284,6 +284,20 @@ def cb(response): return cb +# +# Convenience to define weight + +class Weight(object): + """ + There object for set weights parameters like this + {'passing': 100, 'warning': 100} + """ + + @classmethod + def weights(cls, passing, warning): + return {'passing': passing, 'warning': warning} + + class HTTPClient(six.with_metaclass(abc.ABCMeta, object)): def __init__(self, host='127.0.0.1', port=8500, scheme='http', verify=True, cert=None, timeout=None): @@ -1364,6 +1378,7 @@ def register( check=None, token=None, meta=None, + weights=None, # *deprecated* use check parameter script=None, interval=None, @@ -1396,6 +1411,10 @@ def register( *meta* specifies arbitrary KV metadata linked to the service formatted as {k1:v1, k2:v2}. + *weights* specifies weights for the service. + If this field is not provided weights + will default to {"Passing": 1, "Warning": 1}. + *script*, *interval*, *ttl*, *http*, and *timeout* arguments are deprecated. use *check* instead. @@ -1425,6 +1444,8 @@ def register( payload['meta'] = meta if check: payload['check'] = check + if weights: + payload['weights'] = weights else: payload.update(Check._compat( diff --git a/tests/test_std.py b/tests/test_std.py index 2ff1099..68712fa 100644 --- a/tests/test_std.py +++ b/tests/test_std.py @@ -8,6 +8,7 @@ import consul import consul.std +from consul.base import Weight Check = consul.Check @@ -348,6 +349,26 @@ def test_agent_register_enable_tag_override(self, consul_port): # Cleanup tasks c.agent.check.deregister('foo') + def test_agent_register_enable_weights(self, consul_port): + c = consul.Consul(port=consul_port) + index, nodes = c.health.service("foo1") + assert nodes == [] + + c.agent.service.register('foo', weights=Weight.weights(10, 10)) + assert c.agent.services()['foo']['Weights'] == {"Passing": 10, "Warning": 10} + # Cleanup tasks + c.agent.check.deregister('foo') + + def test_agent_register_disable_weights(self, consul_port): + c = consul.Consul(port=consul_port) + index, nodes = c.health.service("foo1") + assert nodes == [] + + c.agent.service.register('foo') + assert c.agent.services()['foo']['Weights'] == {"Passing": 1, "Warning": 1} + # Cleanup tasks + c.agent.check.deregister('foo') + def test_agent_service_maintenance(self, consul_port): c = consul.Consul(port=consul_port) From ff384a88c0aee70438683d2be9ec4fbdf23d7f0f Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Tue, 9 Feb 2021 11:47:11 +0300 Subject: [PATCH 02/24] =?UTF-8?q?Bump=20version:=200.1.4=20=E2=86=92=200.1?= =?UTF-8?q?.5-dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- sonar-project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e0380b6..45001bd 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.1.4 +current_version = 0.1.5-dev parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index cabfc76..82f939e 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.1.4' +__version__ = '0.1.5-dev' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/sonar-project.properties b/sonar-project.properties index 373175e..2205923 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 sonar.projectName=Python Consul2 -sonar.projectVersion=0.1.4 +sonar.projectVersion=0.1.5-dev # Comma-separated paths to directories with sources (required) sonar.sources=consul From 3a224f25619b1292812b77ab955b7dbc91b5f2cc Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Tue, 9 Feb 2021 11:47:27 +0300 Subject: [PATCH 03/24] =?UTF-8?q?Bump=20version:=200.1.5-dev=20=E2=86=92?= =?UTF-8?q?=200.1.6-dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- sonar-project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 45001bd..e19f5b0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.1.5-dev +current_version = 0.1.6-dev parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index 82f939e..d205487 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.1.5-dev' +__version__ = '0.1.6-dev' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/sonar-project.properties b/sonar-project.properties index 2205923..a1c62c3 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 sonar.projectName=Python Consul2 -sonar.projectVersion=0.1.5-dev +sonar.projectVersion=0.1.6-dev # Comma-separated paths to directories with sources (required) sonar.sources=consul From 117060dbb6b521fcb9bb55c71dd396f8ec20c3e3 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Tue, 9 Feb 2021 11:47:53 +0300 Subject: [PATCH 04/24] =?UTF-8?q?Bump=20version:=200.1.6-dev=20=E2=86=92?= =?UTF-8?q?=200.1.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- sonar-project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e19f5b0..10671cb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.1.6-dev +current_version = 0.1.6 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index d205487..d5ed6b9 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.1.6-dev' +__version__ = '0.1.6' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/sonar-project.properties b/sonar-project.properties index a1c62c3..ffa6398 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 sonar.projectName=Python Consul2 -sonar.projectVersion=0.1.6-dev +sonar.projectVersion=0.1.6 # Comma-separated paths to directories with sources (required) sonar.sources=consul From a71a5972b73bdbb337b0603b4643657f8d774d65 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Tue, 9 Feb 2021 17:38:45 +0300 Subject: [PATCH 05/24] HH-123895 add total_timeout for long requests --- consul/aio.py | 9 ++++---- consul/base.py | 23 +++++++++++++++++--- consul/std.py | 2 +- consul/tornado.py | 5 +++-- consul/twisted.py | 5 +++-- tests/test_base.py | 14 ++++++------ tests/test_std.py | 48 +++++++++++++++++++++++++++++++++++++++-- tests/test_std_token.py | 2 +- 8 files changed, 86 insertions(+), 22 deletions(-) diff --git a/consul/aio.py b/consul/aio.py index e782f62..b22d2bc 100644 --- a/consul/aio.py +++ b/consul/aio.py @@ -5,6 +5,7 @@ import warnings import aiohttp +from aiohttp import ClientTimeout from consul import base @@ -20,10 +21,10 @@ def __init__(self, *args, loop=None, **kwargs): self._session = None self._loop = loop or asyncio.get_event_loop() - async def _request(self, callback, method, uri, data=None, headers=None): + async def _request(self, callback, method, uri, data=None, headers=None, total_timeout=None): connector = aiohttp.TCPConnector(loop=self._loop, verify_ssl=self.verify) - async with aiohttp.ClientSession(connector=connector) as session: + async with aiohttp.ClientSession(connector=connector, timeout=ClientTimeout(total=total_timeout)) as session: self._session = session resp = await session.request(method=method, url=uri, @@ -47,9 +48,9 @@ def __del__(self): ResourceWarning) asyncio.ensure_future(self.close()) - async def get(self, callback, path, params=None, headers=None): + async def get(self, callback, path, params=None, headers=None, total_timeout=None): uri = self.uri(path, params) - return await self._request(callback, 'GET', uri, headers=headers) + return await self._request(callback, 'GET', uri, headers=headers, total_timeout=total_timeout) async def put(self, callback, path, params=None, data='', headers=None): uri = self.uri(path, params) diff --git a/consul/base.py b/consul/base.py index 2b412be..7159fb6 100755 --- a/consul/base.py +++ b/consul/base.py @@ -4,6 +4,7 @@ import json import logging import os +import re import warnings import six @@ -316,7 +317,7 @@ def uri(self, path, params=None): return uri @abc.abstractmethod - def get(self, callback, path, params=None, headers=None): + def get(self, callback, path, params=None, headers=None, total_timeout=None): raise NotImplementedError @abc.abstractmethod @@ -2955,7 +2956,8 @@ def get( consistency=None, keys=False, separator=None, - dc=None): + dc=None, + total_timeout=None): """ Returns a tuple of (*index*, *value[s]*) @@ -3002,6 +3004,10 @@ def get( if index: params.append(('index', index)) if wait: + assert total_timeout, \ + 'total_timeout should be setted' + assert not self._convert_wait_to_seconds(wait) >= total_timeout, \ + f'wait: {wait} should be less than total_timeout: {total_timeout}s' params.append(('wait', wait)) if recurse: params.append(('recurse', '1')) @@ -3030,7 +3036,18 @@ def get( CB.json(index=True, decode=decode, one=one, map=lambda x: x if x else None), path='/v1/kv/%s' % key, - params=params, headers=headers) + params=params, headers=headers, total_timeout=total_timeout) + + def _convert_wait_to_seconds(self, wait): + unit_to_seconds_multiplier = { + 'ms': 0.001, + 's': 1, + 'm': 60 + } + wait_digit = int(re.search(r'\d+', wait).group()) + multiplier = unit_to_seconds_multiplier[re.search(r'ms|s|m', wait).group()] + + return wait_digit * multiplier def put( self, diff --git a/consul/std.py b/consul/std.py index 224b6fe..feb54ea 100644 --- a/consul/std.py +++ b/consul/std.py @@ -19,7 +19,7 @@ def response(response): response.text, response.content) - def get(self, callback, path, params=None, headers=None): + def get(self, callback, path, params=None, headers=None, total_timeout=None): uri = self.uri(path, params) return callback(self.response( self.session.get(uri, diff --git a/consul/tornado.py b/consul/tornado.py index 8294885..d90e610 100644 --- a/consul/tornado.py +++ b/consul/tornado.py @@ -31,12 +31,13 @@ def _request(self, callback, request): response = e.response raise gen.Return(callback(self.response(response))) - def get(self, callback, path, params=None, headers=None): + def get(self, callback, path, params=None, headers=None, total_timeout=None): uri = self.uri(path, params) request = httpclient.HTTPRequest(uri, method='GET', validate_cert=self.verify, - headers=headers) + headers=headers, + connect_timeout=total_timeout) return self._request(callback, request) def put(self, callback, path, params=None, data='', headers=None): diff --git a/consul/twisted.py b/consul/twisted.py index 900ea75..02c37e3 100644 --- a/consul/twisted.py +++ b/consul/twisted.py @@ -100,13 +100,14 @@ def request(self, callback, method, url, **kwargs): 'Request incomplete: {} {}'.format(method.upper(), url)) @inlineCallbacks - def get(self, callback, path, params=None, headers=None): + def get(self, callback, path, params=None, headers=None, total_timeout=None): uri = self.uri(path, params) response = yield self.request(callback, 'get', uri, params=params, - headers=headers) + headers=headers, + total_timeout=total_timeout) returnValue(response) @inlineCallbacks diff --git a/tests/test_base.py b/tests/test_base.py index a80227f..5d815dc 100755 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -11,7 +11,7 @@ Response = consul.base.Response Request = collections.namedtuple( - 'Request', ['method', 'path', 'params', 'data', 'headers']) + 'Request', ['method', 'path', 'params', 'data', 'headers', 'total_timeout']) class HTTPClient(object): @@ -19,14 +19,14 @@ def __init__(self, host=None, port=None, scheme=None, verify=True, cert=None, timeout=None, headers=None): pass - def get(self, callback, path, params=None, headers=None): - return Request('get', path, params, None, headers) + def get(self, callback, path, params=None, headers=None, total_timeout=None): + return Request('get', path, params, None, headers, total_timeout) - def put(self, callback, path, params=None, data='', headers=None): - return Request('put', path, params, data, headers) + def put(self, callback, path, params=None, data='', headers=None, total_timeout=None): + return Request('put', path, params, data, headers, total_timeout) - def delete(self, callback, path, params=None, headers=None): - return Request('delete', path, params, None, headers) + def delete(self, callback, path, params=None, headers=None, total_timeout=None): + return Request('delete', path, params, None, headers, total_timeout) class Consul(consul.base.Consul): diff --git a/tests/test_std.py b/tests/test_std.py index 68712fa..7475b99 100644 --- a/tests/test_std.py +++ b/tests/test_std.py @@ -31,13 +31,57 @@ def test_kv(self, consul_port): index, data = c.kv.get('foo') assert data['Value'] == six.b('bar') - def test_kv_wait(self, consul_port): + def test_kv_wait_ms(self, consul_port): c = consul.Consul(port=consul_port) assert c.kv.put('foo', 'bar') is True index, data = c.kv.get('foo') - check, data = c.kv.get('foo', index=index, wait='20ms') + check, data = c.kv.get('foo', index=index, wait='20ms', total_timeout=30) assert index == check + def test_kv_wait_s(self, consul_port): + c = consul.Consul(port=consul_port) + assert c.kv.put('foo', 'bar') is True + index, data = c.kv.get('foo') + check, data = c.kv.get('foo', index=index, wait='20s', total_timeout=30) + assert index == check + + def test_kv_wait_m(self, consul_port): + c = consul.Consul(port=consul_port) + assert c.kv.put('foo', 'bar') is True + index, data = c.kv.get('foo') + check, data = c.kv.get('foo', index=index, wait='1m', total_timeout=61) + assert index == check + + def test_kv_wait_more_timeout_ms(self, consul_port): + c = consul.Consul(port=consul_port) + assert c.kv.put('foo', 'bar') is True + index, data = c.kv.get('foo') + pytest.raises( + AssertionError, + c.kv.get, + 'foo', index=index, wait='20ms', total_timeout=0 + ) + + def test_kv_wait_more_timeout_s(self, consul_port): + c = consul.Consul(port=consul_port) + assert c.kv.put('foo', 'bar') is True + index, data = c.kv.get('foo') + pytest.raises( + AssertionError, + c.kv.get, + 'foo', index=index, wait='20s', total_timeout=19 + ) + + def test_kv_wait_more_timeout_m(self, consul_port): + c = consul.Consul(port=consul_port) + assert c.kv.put('foo', 'bar') is True + index, data = c.kv.get('foo') + pytest.raises( + AssertionError, + c.kv.get, + 'foo', index=index, wait='1m', total_timeout=59 + ) + def test_kv_encoding(self, consul_port): c = consul.Consul(port=consul_port) diff --git a/tests/test_std_token.py b/tests/test_std_token.py index 86b60c9..b5852ad 100644 --- a/tests/test_std_token.py +++ b/tests/test_std_token.py @@ -24,7 +24,7 @@ def test_kv_wait(self, acl_consul): c = consul.Consul(port=acl_consul.port, token=acl_consul.token) assert c.kv.put('foo', 'bar') is True index, data = c.kv.get('foo') - check, data = c.kv.get('foo', index=index, wait='20ms') + check, data = c.kv.get('foo', index=index, wait='20ms', total_timeout=30) assert index == check def test_kv_encoding(self, acl_consul): From dac26840eff1e4fd489cc70e1523a37b0607e0ee Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 10 Feb 2021 17:39:15 +0300 Subject: [PATCH 06/24] =?UTF-8?q?Bump=20version:=200.1.6=20=E2=86=92=200.2?= =?UTF-8?q?.0-dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- sonar-project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 10671cb..d73be91 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.1.6 +current_version = 0.2.0-dev parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index d5ed6b9..7ef6b57 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.1.6' +__version__ = '0.2.0-dev' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/sonar-project.properties b/sonar-project.properties index ffa6398..5faf36c 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 sonar.projectName=Python Consul2 -sonar.projectVersion=0.1.6 +sonar.projectVersion=0.2.0-dev # Comma-separated paths to directories with sources (required) sonar.sources=consul From e5f44b88547b1354140ca79c545142f8fc095fd1 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 10 Feb 2021 17:39:32 +0300 Subject: [PATCH 07/24] =?UTF-8?q?Bump=20version:=200.2.0-dev=20=E2=86=92?= =?UTF-8?q?=200.2.0-hh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- sonar-project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d73be91..344ac4c 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.2.0-dev +current_version = 0.2.0-hh parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index 7ef6b57..59259a3 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.2.0-dev' +__version__ = '0.2.0-hh' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/sonar-project.properties b/sonar-project.properties index 5faf36c..5b38d60 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 sonar.projectName=Python Consul2 -sonar.projectVersion=0.2.0-dev +sonar.projectVersion=0.2.0-hh # Comma-separated paths to directories with sources (required) sonar.sources=consul From c9b0123373c02bdecc94538ae0a3dded36bbdf53 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Mon, 15 Feb 2021 14:26:49 +0300 Subject: [PATCH 08/24] HH-124106 add sync cached based on consul blocking queries --- consul/base.py | 190 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) diff --git a/consul/base.py b/consul/base.py index 7159fb6..f69a4f6 100755 --- a/consul/base.py +++ b/consul/base.py @@ -1,10 +1,14 @@ +from __future__ import annotations + import abc import base64 import collections +import enum import json import logging import os import re +import threading import warnings import six @@ -299,6 +303,192 @@ def weights(cls, passing, warning): return {'passing': passing, 'warning': warning} +class ConsistencyMode(enum.Enum): + """ + Most of the read query endpoints support multiple levels of consistency. + Since no policy will suit all clients' needs, + these consistency modes allow the user to have the ultimate say in how + to balance the trade-offs inherent in a distributed system. + The three read modes are: + + *DEFAULT* - If not specified, the default is strongly consistent in almost all cases. + However, there is a small window in which a new leader may be elected during which + he old leader may service stale values. The trade-off is fast reads but potentially stale values. + The condition resulting in stale reads is hard to trigger, and most clients should + not need to worry about this case. Also, note that this race condition only applies to reads, not writes. + + *CONSISTENT* - This mode is strongly consistent without caveats. + It requires that a leader verify with a quorum of peers that it is still leader. + This introduces an additional round-trip to all server nodes. The trade-off is increased + latency due to an extra round trip. Most clients should not use + this unless they cannot tolerate a stale read. + + *STALE* - This mode allows any server to service the read regardless of whether it is the leader. + This means reads can be arbitrarily stale; however, results are generally + consistent to within 50 milliseconds of the leader. The trade-off is very fast and scalable reads + with a higher likelihood of stale values. Since this mode allows reads without a leader, a cluster + that is unavailable will still be able to respond to queries. + """ + + DEFAULT = 'default' + CONSISTENT = 'consistent' + STALE = 'stale' + + +class ConsulCacheBase(metaclass=abc.ABCMeta): + """ + Base consul cache implements, that support blocking query. + + *cache* is a dict that consist cache values by key. + + *callbacks* is a list of methods, that will be invoked when cache updated + + *watch_seconds* is the maximum duration for the blocking request. + + *index* is the current Consul index, suitable for making subsequent + calls to wait for changes since this query was last run. + """ + + def __init__(self, watch_seconds: str): + self.cache = dict() + self.callbacks = [] + self.watch_seconds = watch_seconds + self.index = None + self._running = True + self._cache_thread = threading.Thread( + target=self._update_cache, + name='update_consul_cache_thread', + daemon=True) + + def start(self): + self._cache_thread.start() + + def stop(self): + self._running = False + + def add_listener(self, callback, trigger_current=False): + self.callbacks.append(callback) + log.debug(f'Registered callback: {self.callbacks}') + if trigger_current: + for key, value in self.cache.items(): + callback(key, value) + + @abc.abstractmethod + def _update_cache(cls): + pass + + +class HealthCache(ConsulCacheBase): + """ + Consul health service cache. + + *service* is a service name for getting healths info. + + *passing* specifies that the server should return only nodes + with all checks in the passing state. This can be used to avoid + additional filtering on the client side. + """ + + def __init__(self, + health_client: Consul.Health, + watch_seconds: str, + service: str, + passing: bool): + super().__init__(watch_seconds) + self.service = service + self.health_client = health_client + self.passing = passing + + def _update_cache(self): + while self._running: + try: + params = { + 'service': self.service, + 'passing': self.passing, + 'index': self.index, + 'wait': self.watch_seconds + } + log.debug(f'Param for health query: {params}') + self.index, values = self.health_client.service(**params) + old_cache = self.cache + self.cache = {self.service: values} + if self.callbacks and self._running: + for key, old_value in old_cache.items(): + new_value = self.cache.get(key, None) + for callback in self.callbacks: + callback(key, new_value) + except ConsulException as e: + log.error(f'Some problem with update consul cache: {e}') + + +class KVCache(ConsulCacheBase): + """ + Consul key-value cache. + + *path* is a key for getting value + + *consistency_mode* sets the consistency mode to use by default for all reads + that support the consistency option. It's still possible to override + this by passing explicitly for a given request. *consistency* can be + either 'default', 'consistent' or 'stale'. + + *total_timeout* is a ttl of HTTP session. Should be more than *watch_seconds* + + *cache_initial_warmup_timeout* is a ttl of HTTP session for initialize cache. + May be None, will use *total_timeout* insted + """ + + def __init__(self, + kv_client: Consul.KV, + watch_seconds: str, + path: str, + total_timeout: int, + consistency_mode: ConsistencyMode, + cache_initial_warmup_timeout=None): + super().__init__(watch_seconds) + self.kv_client = kv_client + self.path = path + self.consistency_mode = consistency_mode.value + self.total_timeout = total_timeout + self.cache_initial_warmup_timeout = cache_initial_warmup_timeout + self.cache = {self.path: kv_client.get( + key=path, + total_timeout=self._get_warmup_timeout() + )[1]} + + def _get_warmup_timeout(self): + if self.cache_initial_warmup_timeout: + return self.cache_initial_warmup_timeout + return self.total_timeout + + def get_value(self): + return self.cache.get(self.path, None) + + def _update_cache(self): + while self._running: + try: + params = { + 'key': self.path, + 'index': self.index, + 'wait': self.watch_seconds, + 'total_timeout': self.total_timeout, + 'consistency': self.consistency_mode + } + log.debug(f'Param for kv query: {params}') + self.index, values = self.kv_client.get(**params) + old_cache = self.cache + self.cache = {self.path: values} + if self.callbacks and self._running: + for key, new_value in self.cache.items(): + old_value = old_cache.get(key, None) + if old_value != new_value: + log.debug(f'Value was changed for key={key}. old: {old_value} new: {new_value}') + for callback in self.callbacks: + callback(key, new_value) + except ConsulException as e: + log.error(f'Some problem with update consul cache: {e}') + + class HTTPClient(six.with_metaclass(abc.ABCMeta, object)): def __init__(self, host='127.0.0.1', port=8500, scheme='http', verify=True, cert=None, timeout=None): From 850ec9096b5044da277cff29cab9d73010aa2efa Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 24 Feb 2021 12:57:41 +0300 Subject: [PATCH 09/24] =?UTF-8?q?Bump=20version:=200.2.0-hh=20=E2=86=92=20?= =?UTF-8?q?0.2.1-hh-dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- sonar-project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 344ac4c..116a528 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.2.0-hh +current_version = 0.2.1-hh-dev parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index 59259a3..65d3165 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.2.0-hh' +__version__ = '0.2.1-hh-dev' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/sonar-project.properties b/sonar-project.properties index 5b38d60..710c738 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 sonar.projectName=Python Consul2 -sonar.projectVersion=0.2.0-hh +sonar.projectVersion=0.2.1-hh-dev # Comma-separated paths to directories with sources (required) sonar.sources=consul From 31384c918b4de4208be288689daebae862d608a7 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 24 Feb 2021 13:06:28 +0300 Subject: [PATCH 10/24] =?UTF-8?q?Bump=20version:=200.2.1-hh-dev=20?= =?UTF-8?q?=E2=86=92=200.2.1-hh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- sonar-project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 116a528..fdc7fc1 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.2.1-hh-dev +current_version = 0.2.1-hh parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index 65d3165..c7c9abe 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.2.1-hh-dev' +__version__ = '0.2.1-hh' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/sonar-project.properties b/sonar-project.properties index 710c738..4844021 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 sonar.projectName=Python Consul2 -sonar.projectVersion=0.2.1-hh-dev +sonar.projectVersion=0.2.1-hh # Comma-separated paths to directories with sources (required) sonar.sources=consul From 3dbb43cf8be44fff31ed68f92b8a13817d10f4b1 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Mon, 22 Mar 2021 13:57:01 +0300 Subject: [PATCH 11/24] HH-125740 added dc, recurse and cold-start health cache --- consul/base.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/consul/base.py b/consul/base.py index f69a4f6..313d0d5 100755 --- a/consul/base.py +++ b/consul/base.py @@ -393,11 +393,19 @@ def __init__(self, health_client: Consul.Health, watch_seconds: str, service: str, - passing: bool): + passing: bool, + dc: str): super().__init__(watch_seconds) self.service = service self.health_client = health_client self.passing = passing + self.dc = dc + self.index, service_health = health_client.service( + service=service, + passing=passing, + dc=dc, + ) + self.cache = {self.service: service_health} def _update_cache(self): while self._running: @@ -406,7 +414,8 @@ def _update_cache(self): 'service': self.service, 'passing': self.passing, 'index': self.index, - 'wait': self.watch_seconds + 'wait': self.watch_seconds, + 'dc': self.dc } log.debug(f'Param for health query: {params}') self.index, values = self.health_client.service(**params) @@ -443,18 +452,22 @@ def __init__(self, watch_seconds: str, path: str, total_timeout: int, + recurse: bool, consistency_mode: ConsistencyMode, cache_initial_warmup_timeout=None): super().__init__(watch_seconds) self.kv_client = kv_client self.path = path + self.recurse = recurse self.consistency_mode = consistency_mode.value self.total_timeout = total_timeout self.cache_initial_warmup_timeout = cache_initial_warmup_timeout - self.cache = {self.path: kv_client.get( + self.index, kv = kv_client.get( key=path, + recurse=recurse, total_timeout=self._get_warmup_timeout() - )[1]} + ) + self.cache = {self.path: kv} def _get_warmup_timeout(self): if self.cache_initial_warmup_timeout: @@ -472,7 +485,8 @@ def _update_cache(self): 'index': self.index, 'wait': self.watch_seconds, 'total_timeout': self.total_timeout, - 'consistency': self.consistency_mode + 'consistency': self.consistency_mode, + 'recurse': self.recurse } log.debug(f'Param for kv query: {params}') self.index, values = self.kv_client.get(**params) From bca13e16577ab51ebb66b5065c9920712bf20157 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 14 Apr 2021 18:33:42 +0300 Subject: [PATCH 12/24] =?UTF-8?q?Bump=20version:=200.2.1-hh=20=E2=86=92=20?= =?UTF-8?q?0.2.2-hh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- sonar-project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index fdc7fc1..369d517 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.2.1-hh +current_version = 0.2.2-hh parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index c7c9abe..4e5d8b2 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.2.1-hh' +__version__ = '0.2.2-hh' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/sonar-project.properties b/sonar-project.properties index 4844021..45375bc 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 sonar.projectName=Python Consul2 -sonar.projectVersion=0.2.1-hh +sonar.projectVersion=0.2.2-hh # Comma-separated paths to directories with sources (required) sonar.sources=consul From 26a553774e0e6e6b4e6f8c18a45bcc49bed79f09 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 12 May 2021 17:51:55 +0300 Subject: [PATCH 13/24] HH-129203 fix lower-case in HealthCache --- consul/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/consul/base.py b/consul/base.py index 313d0d5..2e37cab 100755 --- a/consul/base.py +++ b/consul/base.py @@ -399,11 +399,11 @@ def __init__(self, self.service = service self.health_client = health_client self.passing = passing - self.dc = dc + self.dc = dc.lower() self.index, service_health = health_client.service( - service=service, - passing=passing, - dc=dc, + service=self.service, + passing=self.passing, + dc=self.dc, ) self.cache = {self.service: service_health} From ee27c1a123e7be02609d75ffe8e7ad5282820766 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 12 May 2021 18:30:03 +0300 Subject: [PATCH 14/24] =?UTF-8?q?Bump=20version:=200.2.2-hh=20=E2=86=92=20?= =?UTF-8?q?0.2.3-hh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- sonar-project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 369d517..70ab901 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.2.2-hh +current_version = 0.2.3-hh parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index 4e5d8b2..4e68f08 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.2.2-hh' +__version__ = '0.2.3-hh' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/sonar-project.properties b/sonar-project.properties index 45375bc..4e5e390 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 sonar.projectName=Python Consul2 -sonar.projectVersion=0.2.2-hh +sonar.projectVersion=0.2.3-hh # Comma-separated paths to directories with sources (required) sonar.sources=consul From 76a93de8655f0279ebd8b7409e0f414bba5f743f Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Fri, 14 May 2021 12:55:36 +0300 Subject: [PATCH 15/24] HH-129345 rename lib to python-consul2-hh --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- setup.py | 2 +- sonar-project.properties | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 70ab901..687ba6d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.2.3-hh +current_version = 0.2.3 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index 4e68f08..7ccf793 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.2.3-hh' +__version__ = '0.2.3' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/setup.py b/setup.py index 92dc99a..4452195 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ def run_tests(self): setup( - name='python-consul2', + name='python-consul2-hh', version=metadata['version'], author='yan.gao', author_email='373251686@qq.com', diff --git a/sonar-project.properties b/sonar-project.properties index 4e5e390..af936bc 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 -sonar.projectName=Python Consul2 -sonar.projectVersion=0.2.3-hh +sonar.projectName=Python Consul2 HH +sonar.projectVersion=0.2.3 # Comma-separated paths to directories with sources (required) sonar.sources=consul From c6c99fb07285f6edc0ef2499dea9eab270ed6021 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Tue, 25 May 2021 18:59:18 +0300 Subject: [PATCH 16/24] HH-130008 extend exceptions for caches and add backoff delay --- consul/base.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/consul/base.py b/consul/base.py index 2e37cab..2b26630 100755 --- a/consul/base.py +++ b/consul/base.py @@ -9,6 +9,7 @@ import os import re import threading +import time import warnings import six @@ -349,10 +350,11 @@ class ConsulCacheBase(metaclass=abc.ABCMeta): calls to wait for changes since this query was last run. """ - def __init__(self, watch_seconds: str): + def __init__(self, watch_seconds: str, backoff_delay_seconds: int): self.cache = dict() self.callbacks = [] self.watch_seconds = watch_seconds + self.backoff_delay_seconds = backoff_delay_seconds self.index = None self._running = True self._cache_thread = threading.Thread( @@ -392,10 +394,11 @@ class HealthCache(ConsulCacheBase): def __init__(self, health_client: Consul.Health, watch_seconds: str, + backoff_delay_seconds: int, service: str, passing: bool, dc: str): - super().__init__(watch_seconds) + super().__init__(watch_seconds, backoff_delay_seconds) self.service = service self.health_client = health_client self.passing = passing @@ -426,8 +429,9 @@ def _update_cache(self): new_value = self.cache.get(key, None) for callback in self.callbacks: callback(key, new_value) - except ConsulException as e: - log.error(f'Some problem with update consul cache: {e}') + except Exception as e: + log.error(f'Some problem with update consul cache: {e}. Will retry in {self.backoff_delay_seconds}s') + time.sleep(self.backoff_delay_seconds) class KVCache(ConsulCacheBase): @@ -450,12 +454,13 @@ class KVCache(ConsulCacheBase): def __init__(self, kv_client: Consul.KV, watch_seconds: str, + backoff_delay_seconds: int, path: str, total_timeout: int, recurse: bool, consistency_mode: ConsistencyMode, cache_initial_warmup_timeout=None): - super().__init__(watch_seconds) + super().__init__(watch_seconds, backoff_delay_seconds) self.kv_client = kv_client self.path = path self.recurse = recurse @@ -499,8 +504,9 @@ def _update_cache(self): log.debug(f'Value was changed for key={key}. old: {old_value} new: {new_value}') for callback in self.callbacks: callback(key, new_value) - except ConsulException as e: - log.error(f'Some problem with update consul cache: {e}') + except Exception as e: + log.error(f'Some problem with update consul cache: {e}. Will retry in {self.backoff_delay_seconds}s') + time.sleep(self.backoff_delay_seconds) class HTTPClient(six.with_metaclass(abc.ABCMeta, object)): From 24962dffd01fdf7372ab2ea9d09c2c408de4180c Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 26 May 2021 12:24:45 +0300 Subject: [PATCH 17/24] =?UTF-8?q?Bump=20version:=200.2.3=20=E2=86=92=200.2?= =?UTF-8?q?.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- sonar-project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 687ba6d..40fc694 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.2.3 +current_version = 0.2.4 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index 7ccf793..8fa5210 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.2.3' +__version__ = '0.2.4' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/sonar-project.properties b/sonar-project.properties index af936bc..95fa0a3 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 sonar.projectName=Python Consul2 HH -sonar.projectVersion=0.2.3 +sonar.projectVersion=0.2.4 # Comma-separated paths to directories with sources (required) sonar.sources=consul From 0dc1e3cf84e89ed3e04c7d3575359c0adad16c23 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 26 May 2021 13:18:09 +0300 Subject: [PATCH 18/24] add aiohttp --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 5e03435..51fb41d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests six>=1.4 +aiohttp == 3.7.4 From 5e906729e553334f153174b05ac8095b41376e1c Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 26 May 2021 13:19:52 +0300 Subject: [PATCH 19/24] =?UTF-8?q?Bump=20version:=200.2.4=20=E2=86=92=200.2?= =?UTF-8?q?.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- sonar-project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 40fc694..cbbd29c 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.2.4 +current_version = 0.2.5 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index 8fa5210..95b1666 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.2.4' +__version__ = '0.2.5' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/sonar-project.properties b/sonar-project.properties index 95fa0a3..dd2b926 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 sonar.projectName=Python Consul2 HH -sonar.projectVersion=0.2.4 +sonar.projectVersion=0.2.5 # Comma-separated paths to directories with sources (required) sonar.sources=consul From ecba4304f683c38153b125b731acc60efaebc647 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 29 Sep 2021 12:50:22 +0300 Subject: [PATCH 20/24] HH-137449 add 'caller' as a param for request Consul --- consul/base.py | 53 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/consul/base.py b/consul/base.py index 2b26630..147f4db 100755 --- a/consul/base.py +++ b/consul/base.py @@ -350,9 +350,10 @@ class ConsulCacheBase(metaclass=abc.ABCMeta): calls to wait for changes since this query was last run. """ - def __init__(self, watch_seconds: str, backoff_delay_seconds: int): + def __init__(self, watch_seconds: str, backoff_delay_seconds: int, caller: str): self.cache = dict() self.callbacks = [] + self.caller = caller self.watch_seconds = watch_seconds self.backoff_delay_seconds = backoff_delay_seconds self.index = None @@ -397,8 +398,9 @@ def __init__(self, backoff_delay_seconds: int, service: str, passing: bool, - dc: str): - super().__init__(watch_seconds, backoff_delay_seconds) + dc: str, + caller: str): + super().__init__(watch_seconds, backoff_delay_seconds, caller) self.service = service self.health_client = health_client self.passing = passing @@ -407,6 +409,7 @@ def __init__(self, service=self.service, passing=self.passing, dc=self.dc, + caller=self.caller ) self.cache = {self.service: service_health} @@ -418,7 +421,8 @@ def _update_cache(self): 'passing': self.passing, 'index': self.index, 'wait': self.watch_seconds, - 'dc': self.dc + 'dc': self.dc, + 'caller': self.caller } log.debug(f'Param for health query: {params}') self.index, values = self.health_client.service(**params) @@ -459,8 +463,9 @@ def __init__(self, total_timeout: int, recurse: bool, consistency_mode: ConsistencyMode, + caller: str, cache_initial_warmup_timeout=None): - super().__init__(watch_seconds, backoff_delay_seconds) + super().__init__(watch_seconds, backoff_delay_seconds, caller) self.kv_client = kv_client self.path = path self.recurse = recurse @@ -470,7 +475,8 @@ def __init__(self, self.index, kv = kv_client.get( key=path, recurse=recurse, - total_timeout=self._get_warmup_timeout() + total_timeout=self._get_warmup_timeout(), + caller=self.caller ) self.cache = {self.path: kv} @@ -491,7 +497,8 @@ def _update_cache(self): 'wait': self.watch_seconds, 'total_timeout': self.total_timeout, 'consistency': self.consistency_mode, - 'recurse': self.recurse + 'recurse': self.recurse, + 'caller': self.caller } log.debug(f'Param for kv query: {params}') self.index, values = self.kv_client.get(**params) @@ -1596,6 +1603,7 @@ def register( ttl=None, http=None, timeout=None, + caller=None, enable_tag_override=False): """ Add a new service to the local agent. There is more @@ -1629,6 +1637,8 @@ def register( *script*, *interval*, *ttl*, *http*, and *timeout* arguments are deprecated. use *check* instead. + *caller* is a name of caller service. + *enable_tag_override* is an optional bool that enable you to modify a service tags from servers(consul agent role server) Default is set to False. @@ -1637,7 +1647,7 @@ def register( for more information https://www.consul.io/docs/agent/services.html """ - + params = [] payload = {} payload['name'] = name @@ -1658,6 +1668,9 @@ def register( if weights: payload['weights'] = weights + if caller: + params.append(('caller', caller)) + else: payload.update(Check._compat( script=script, @@ -1675,22 +1688,28 @@ def register( CB.bool(), path='/v1/agent/service/register', headers=headers, + params=params, data=json.dumps(payload)) - def deregister(self, service_id, token=None): + def deregister(self, service_id, caller=None, token=None): """ Used to remove a service from the local agent. The agent will take care of deregistering the service with the Catalog. If there is an associated check, that is also deregistered. """ + + params = [] headers = {} token = token or self.agent.token if token: headers['X-Consul-Token'] = token + if caller: + params.append(('caller', caller)) return self.agent.http.put( CB.bool(), path='/v1/agent/service/deregister/%s' % service_id, - headers=headers + headers=headers, + params=params ) def maintenance(self, service_id, enable, reason=None, token=None): @@ -2935,7 +2954,8 @@ def service(self, dc=None, near=None, token=None, - node_meta=None): + node_meta=None, + caller=None): """ Returns a tuple of (*index*, *nodes*) @@ -2963,6 +2983,8 @@ def service(self, *node_meta* is an optional meta data used for filtering, a dictionary formatted as {k1:v1, k2:v2}. + + *caller* is a name of caller service. """ params = [] headers = {} @@ -2987,6 +3009,8 @@ def service(self, for nodemeta_name, nodemeta_value in node_meta.items(): params.append(('node-meta', '{0}:{1}'. format(nodemeta_name, nodemeta_value))) + if caller: + params.append(('caller', caller)) return self.agent.http.get( CB.json(index=True), path='/v1/health/service/%s' % service, @@ -3167,7 +3191,8 @@ def get( keys=False, separator=None, dc=None, - total_timeout=None): + total_timeout=None, + caller=None): """ Returns a tuple of (*index*, *value[s]*) @@ -3203,6 +3228,8 @@ def get( "Session": "adf4238a-882b-9ddc-4a9d-5b6758e4159e" } + *caller* is a name of caller service. + Note, if the requested key does not exists *(index, None)* is returned. It's then possible to long poll on the index for when the key is created. @@ -3234,6 +3261,8 @@ def get( consistency = consistency or self.agent.consistency if consistency in ('consistent', 'stale'): params.append((consistency, '1')) + if caller: + params.append(('caller', caller)) one = False decode = False From c7128e2e11bad8ef720e0d64873c1d144ed1f772 Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 29 Sep 2021 18:14:57 +0300 Subject: [PATCH 21/24] =?UTF-8?q?Bump=20version:=200.2.5=20=E2=86=92=200.2?= =?UTF-8?q?.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- consul/__init__.py | 2 +- sonar-project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cbbd29c..274b67c 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 0.2.5 +current_version = 0.2.6 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/consul/__init__.py b/consul/__init__.py index 95b1666..43a3e56 100644 --- a/consul/__init__.py +++ b/consul/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.2.5' +__version__ = '0.2.6' from consul.base import ACLDisabled # noqa from consul.base import ACLPermissionDenied # noqa diff --git a/sonar-project.properties b/sonar-project.properties index dd2b926..979d314 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # Required metadata sonar.projectKey=com.github:poppyred:python-consul2 sonar.projectName=Python Consul2 HH -sonar.projectVersion=0.2.5 +sonar.projectVersion=0.2.6 # Comma-separated paths to directories with sources (required) sonar.sources=consul From 9d8971d65b0f6738da083e07ffc6df05a89ab0ff Mon Sep 17 00:00:00 2001 From: "al.kazantsev" Date: Wed, 16 Feb 2022 15:07:29 +0300 Subject: [PATCH 22/24] HH-145802 optional for import annotations --- consul/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/consul/base.py b/consul/base.py index 147f4db..f5846d7 100755 --- a/consul/base.py +++ b/consul/base.py @@ -1,4 +1,7 @@ -from __future__ import annotations +try: + from __future__ import annotations +except ImportError: + pass import abc import base64 From a810998483567210a69ae105ee46239c3b580533 Mon Sep 17 00:00:00 2001 From: "al.kazantsev" Date: Wed, 16 Feb 2022 16:52:53 +0300 Subject: [PATCH 23/24] HH-145802 remove import --- consul/base.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/consul/base.py b/consul/base.py index f5846d7..a600d21 100755 --- a/consul/base.py +++ b/consul/base.py @@ -1,8 +1,3 @@ -try: - from __future__ import annotations -except ImportError: - pass - import abc import base64 import collections From cb3460aa6647ed9ef2707622419fdd9b04faa6fc Mon Sep 17 00:00:00 2001 From: Nikolay Gribanov Date: Wed, 16 Feb 2022 17:20:36 +0300 Subject: [PATCH 24/24] HH-145802 remove specific types --- consul/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consul/base.py b/consul/base.py index a600d21..c862204 100755 --- a/consul/base.py +++ b/consul/base.py @@ -391,7 +391,7 @@ class HealthCache(ConsulCacheBase): """ def __init__(self, - health_client: Consul.Health, + health_client, watch_seconds: str, backoff_delay_seconds: int, service: str, @@ -454,7 +454,7 @@ class KVCache(ConsulCacheBase): """ def __init__(self, - kv_client: Consul.KV, + kv_client, watch_seconds: str, backoff_delay_seconds: int, path: str,