From 170b05c9784ec326613c96a17553b6f14e9da3a4 Mon Sep 17 00:00:00 2001 From: Jelle Smet Date: Wed, 22 Jun 2016 10:52:21 +0200 Subject: [PATCH 1/7] Raise an exception when no data is received. At least we should get an actual empty list when our query does not return any matches --- mk_livestatus/livestatus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk_livestatus/livestatus.py b/mk_livestatus/livestatus.py index 25ccdf9..37c63b1 100644 --- a/mk_livestatus/livestatus.py +++ b/mk_livestatus/livestatus.py @@ -63,7 +63,7 @@ def call(self, request): s.shutdown(socket.SHUT_WR) rawdata = s.makefile().read() if not rawdata: - return [] + raise Exception("Livestatus returned no data.") data = json.loads(rawdata) return [dict(zip(data[0], value)) for value in data[1:]] finally: From 8c1f2bebe999c0dc45e94b243b4dc835e6d85b01 Mon Sep 17 00:00:00 2001 From: Jelle Smet Date: Wed, 22 Jun 2016 12:12:32 +0200 Subject: [PATCH 2/7] Improve error handling. --- mk_livestatus/livestatus.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/mk_livestatus/livestatus.py b/mk_livestatus/livestatus.py index 37c63b1..7e84f02 100644 --- a/mk_livestatus/livestatus.py +++ b/mk_livestatus/livestatus.py @@ -27,7 +27,7 @@ def call(self): __call__ = call def __str__(self): - request = 'GET %s' % (self._resource) + request = 'GET %s\nResponseHeader: fixed16' % (self._resource) if self._columns and any(self._columns): request += '\nColumns: %s' % (' '.join(self._columns)) if self._filters: @@ -63,8 +63,17 @@ def call(self, request): s.shutdown(socket.SHUT_WR) rawdata = s.makefile().read() if not rawdata: - raise Exception("Livestatus returned no data.") - data = json.loads(rawdata) + raise Exception("Livestatus service returned no data.") + data = self.validateHeader(rawdata) + data = json.loads(data) return [dict(zip(data[0], value)) for value in data[1:]] finally: s.close() + + def validateHeader(self, rawdata): + header = rawdata[0:16] + if header[0:3] == "200": + data = rawdata[16:] + return data + else: + raise Exception("The Livestatus query contained an error. Reason: %s" % (rawdata[16:])) From 01362176dc0958f2ae86e944bf09b23f22beaea6 Mon Sep 17 00:00:00 2001 From: Jelle Smet Date: Thu, 23 Jun 2016 12:25:15 +0200 Subject: [PATCH 3/7] Make use of livestatus headers to raise meaningful exceptions to user differentiating between syntax errors and network connectivity --- mk_livestatus/errors.py | 6 ++++++ mk_livestatus/livestatus.py | 8 ++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 mk_livestatus/errors.py diff --git a/mk_livestatus/errors.py b/mk_livestatus/errors.py new file mode 100644 index 0000000..fb048e1 --- /dev/null +++ b/mk_livestatus/errors.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + +class LivestatusError(Exception): + pass diff --git a/mk_livestatus/livestatus.py b/mk_livestatus/livestatus.py index 7e84f02..26db58f 100644 --- a/mk_livestatus/livestatus.py +++ b/mk_livestatus/livestatus.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +from .errors import LivestatusError import socket import json @@ -62,8 +63,11 @@ def call(self, request): s.send(request) s.shutdown(socket.SHUT_WR) rawdata = s.makefile().read() + except Exception as err: + raise LivestatusError("Failed to connect to Livestatus. Reason: %s" % (err)) + else: if not rawdata: - raise Exception("Livestatus service returned no data.") + raise LivestatusError("Livestatus service returned no data.") data = self.validateHeader(rawdata) data = json.loads(data) return [dict(zip(data[0], value)) for value in data[1:]] @@ -76,4 +80,4 @@ def validateHeader(self, rawdata): data = rawdata[16:] return data else: - raise Exception("The Livestatus query contained an error. Reason: %s" % (rawdata[16:])) + raise LivestatusError("The Livestatus query contained an error. Reason: %s" % (rawdata[16:])) From 469ba59e8b8bfcc901c9f21ddd854c633394487e Mon Sep 17 00:00:00 2001 From: Jelle Smet Date: Wed, 10 Aug 2016 11:12:17 +0200 Subject: [PATCH 4/7] Set REUSEADDR on socket to recycle sockets in fin_wait --- mk_livestatus/livestatus.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mk_livestatus/livestatus.py b/mk_livestatus/livestatus.py index 26db58f..30e15e3 100644 --- a/mk_livestatus/livestatus.py +++ b/mk_livestatus/livestatus.py @@ -57,6 +57,7 @@ def call(self, request): try: if len(self.peer) == 2: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) else: s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) s.connect(self.peer) From 64edc0361cd417a9b378c7a616a3f3854dc48328 Mon Sep 17 00:00:00 2001 From: Jelle Smet Date: Wed, 10 Aug 2016 15:57:15 +0200 Subject: [PATCH 5/7] Fixed bug where socket wasn't closed when livestatus side closed the connection resulting into many FIN_WAIT2 connections leading to exhausting file descriptors --- mk_livestatus/livestatus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mk_livestatus/livestatus.py b/mk_livestatus/livestatus.py index 30e15e3..0235b15 100644 --- a/mk_livestatus/livestatus.py +++ b/mk_livestatus/livestatus.py @@ -64,7 +64,9 @@ def call(self, request): s.send(request) s.shutdown(socket.SHUT_WR) rawdata = s.makefile().read() + s.close() except Exception as err: + s.close() raise LivestatusError("Failed to connect to Livestatus. Reason: %s" % (err)) else: if not rawdata: @@ -72,8 +74,6 @@ def call(self, request): data = self.validateHeader(rawdata) data = json.loads(data) return [dict(zip(data[0], value)) for value in data[1:]] - finally: - s.close() def validateHeader(self, rawdata): header = rawdata[0:16] From d15f90248ba9019eb57a56f9aeefa4ec2c5f6974 Mon Sep 17 00:00:00 2001 From: Jelle Smet Date: Fri, 12 Aug 2016 11:37:46 +0200 Subject: [PATCH 6/7] - Explicitly set keepalive to socket to prevent time_wait2 lingering sockets. - Wrap socket cleanup in function --- mk_livestatus/livestatus.py | 40 ++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/mk_livestatus/livestatus.py b/mk_livestatus/livestatus.py index 0235b15..fb63de4 100644 --- a/mk_livestatus/livestatus.py +++ b/mk_livestatus/livestatus.py @@ -45,28 +45,50 @@ def filter(self, filter_str): self._filters.append(filter_str) return self + def close(self): + pass + class Socket(object): def __init__(self, peer): self.peer = peer + self.socket = None + self.fd = None def __getattr__(self, name): return Query(self, name) + def cleanup(self): + + try: + self.fd.close() + del(self.fd) + except Exception: + pass + try: + self.socket.close() + del(self.socket) + except Exception: + pass + def call(self, request): try: if len(self.peer) == 2: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 1) + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 3) + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 5) else: - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - s.connect(self.peer) - s.send(request) - s.shutdown(socket.SHUT_WR) - rawdata = s.makefile().read() - s.close() + self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.socket.connect(self.peer) + self.socket.send(request) + self.socket.shutdown(socket.SHUT_WR) + self.fd = self.socket.makefile() + rawdata = self.fd.read() + self.cleanup() except Exception as err: - s.close() + self.cleanup() raise LivestatusError("Failed to connect to Livestatus. Reason: %s" % (err)) else: if not rawdata: From f33b55020d2d4afa05b6422c21bc265d06bcbb36 Mon Sep 17 00:00:00 2001 From: Jelle Smet Date: Fri, 6 Oct 2017 10:37:57 +0200 Subject: [PATCH 7/7] handle encoding --- mk_livestatus/livestatus.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mk_livestatus/livestatus.py b/mk_livestatus/livestatus.py index fb63de4..a1f39eb 100644 --- a/mk_livestatus/livestatus.py +++ b/mk_livestatus/livestatus.py @@ -50,10 +50,11 @@ def close(self): class Socket(object): - def __init__(self, peer): + def __init__(self, peer, encoding="latin-1"): self.peer = peer self.socket = None self.fd = None + self.encoding = encoding def __getattr__(self, name): return Query(self, name) @@ -94,7 +95,7 @@ def call(self, request): if not rawdata: raise LivestatusError("Livestatus service returned no data.") data = self.validateHeader(rawdata) - data = json.loads(data) + data = json.loads(data, encoding=self.encoding) return [dict(zip(data[0], value)) for value in data[1:]] def validateHeader(self, rawdata):