diff --git a/admin/packaging.py b/admin/packaging.py index 7448680cdd..7ba8641cfe 100644 --- a/admin/packaging.py +++ b/admin/packaging.py @@ -1004,6 +1004,8 @@ def omnibus_package_builder( flocker_node_path), (FilePath('/opt/flocker/bin/flocker-node-era'), flocker_node_path), + (FilePath('/opt/flocker/bin/flocker-zk'), + flocker_node_path), ] ), BuildPackage( @@ -1116,10 +1118,13 @@ class DockerBuild(object): to build. """ def run(self): - check_call( - ['docker', 'build', - '--pull', '--tag', self.tag, - self.build_directory.path]) + cmd = ['docker', 'build'] + for env_key in ('http_proxy', 'https_proxy'): + env_val = os.environ.get(env_key) + if env_val is not None: + cmd.extend(['--build-arg', '='.join((env_key, env_val))]) + cmd.extend(['--pull', '--tag', self.tag, self.build_directory.path]) + check_call(cmd) @attributes(['tag', 'volumes', 'command']) @@ -1138,9 +1143,14 @@ def run(self): for container, host in self.volumes.iteritems(): volume_options.extend( ['--volume', '%s:%s' % (host.path, container.path)]) - + proxy_options = [] + for env_key in ('http_proxy', 'https_proxy'): + env_val = os.environ.get(env_key) + if env_val is not None: + proxy_options.extend(['-e', '='.join((env_key, env_val))]) result = call( ['docker', 'run', '--rm'] + + proxy_options + volume_options + [self.tag] + self.command) if result: raise SystemExit(result) diff --git a/docs/flocker-features/configuration-stores.rst b/docs/flocker-features/configuration-stores.rst new file mode 100644 index 0000000000..060102d138 --- /dev/null +++ b/docs/flocker-features/configuration-stores.rst @@ -0,0 +1,53 @@ +==================== +Configuration Stores +==================== + +The :ref:`Flocker control service ` maintains information about the state of the cluster and must store it across restarts. The default persistence mechanism for control service data is to save it as a file on the control node. + +By introducing a configuration store plugin architecture, Flocker opens up the possibility of using a variety of persistence mechanisms. Using a network-based configuration store is an important step toward building a highly available Flocker control service. + + +Listing and activating configuration stores +=========================================== + +Configuration store plugins are added to Flocker as a community effort. Run ``flocker-control --help`` for details on which configuration stores are supported by your Flocker distribution, and how to use them. + +Activating a specific configuration store requires changing the command line arguments in the flocker-control startup script. For example when running flocker-control under systemd on CentOS 7: + +#. Modify ``/usr/lib/systemd/system/flocker-control.service`` and add the ``--configuration-store-plugin`` parameter:: + + ExecStart=/usr/sbin/flocker-control --configuration-store-plugin=myplugin --extra-pluginvar=foo + +#. Execute:: + + systemctl daemon-reload + systemctl restart flocker-control + + +Directory configuration store +============================= + +The default. Saves the cluster state as a flat file in a configurable location, by default in ``/var/lib/flocker``. Example:: + + + flocker-control --data-path=/path/to/share + + +ZooKeeper configuration store +============================= + +Saves the configuration in `ZooKeeper `_ , a highly available key-value store designed for maintaining configuration information. Example:: + + flocker-control --configuration-store-plugin=zookeeper --zookeeper-hosts=192.168.0.1:2181,192.168.0.2:2181,192.168.0.3:2181 + + +Helper script +------------- + +A bundled tool, `flocker-zk`, helps migrate and examine configuration data held in ZooKeeper. + + +Log entries +----------- + +The log message type `flocker:control:store:zookeeper` identifies entries originating from this plugin. diff --git a/docs/flocker-features/index.rst b/docs/flocker-features/index.rst index f1d7b53090..cb520f5787 100644 --- a/docs/flocker-features/index.rst +++ b/docs/flocker-features/index.rst @@ -12,3 +12,4 @@ Flocker Features security storage-backends storage-profiles + configuration-stores diff --git a/docs/gettinginvolved/infrastructure/packaging.rst b/docs/gettinginvolved/infrastructure/packaging.rst index 1f1efe062e..5eda347164 100644 --- a/docs/gettinginvolved/infrastructure/packaging.rst +++ b/docs/gettinginvolved/infrastructure/packaging.rst @@ -187,7 +187,7 @@ To run the ``flocker-control`` container: clusterhqci/flocker-control:master flocker-docker-plugin --------------------- +--------------------- The ``flocker-docker-plugin`` Docker image is built using the same ``docker build ...`` command line as for ``flocker-dataset-agent`` but substituting the ``docker-plugin``. diff --git a/flocker/cli/zookeeper_script.py b/flocker/cli/zookeeper_script.py new file mode 100644 index 0000000000..0171136879 --- /dev/null +++ b/flocker/cli/zookeeper_script.py @@ -0,0 +1,127 @@ +# Copyright (C) 2017 Nokia Corporation and/or its subsidiary(-ies). +# See LICENSE file for details. + +""" +Command line tool for manipulating ZooKeeper-held Flocker control data. +""" + +from argparse import ArgumentParser, RawDescriptionHelpFormatter +from contextlib import contextmanager +import logging +import datetime +import sys + +from kazoo.client import KazooClient +from kazoo.exceptions import NoNodeError + +ZK_CONFIG_PATH = '/flocker/configuration' +ZOOKEEPER_HOSTS = 'localhost:2181' +MIGRATE_FROM = '/var/lib/flocker/current_configuration.json' +EPILOG = '''get: + Print ZooKeeper node contents to stdout. + +put: + Pipe from stdin to ZooKeeper. + +migrate: + Store %s in ZooKeeper. +''' % MIGRATE_FROM + + +class ScriptError(RuntimeError): + '''Known errors we want to catch''' + + +@contextmanager +def zk_client(zkhosts): + '''Minimal ZooKeeper client''' + _client = KazooClient( + hosts=zkhosts, + connection_retry={'max_tries': 4, 'delay': 1}, + command_retry={'max_tries': 4, 'delay': 1} + ) + try: + _client.start() + yield _client + finally: + _client.stop() + + +def get(zkhosts, node): + '''Get node contents from ZooKeeper''' + with zk_client(zkhosts) as client: + try: + content, stat = client.get(node) + except NoNodeError: + raise ScriptError('No such node %s' % node) + mtime = datetime.datetime.fromtimestamp(stat.mtime / 1000).isoformat() + logging.info( + 'Fetched node. Version:%s, mtime:%s, size:%s', + stat.version, mtime, len(content)) + return content + + +def put(zkhosts, node, content): + '''Store something to a ZooKeeper node''' + with zk_client(zkhosts) as client: + client.ensure_path(node) + stat = client.set(node, content) + mtime = datetime.datetime.fromtimestamp(stat.mtime / 1000).isoformat() + logging.info( + 'Stored node. Version:%s, mtime:%s, size:%s', + stat.version, mtime, len(content)) + return stat.version + + +def main(): + '''Entrypoint for the command line tool''' + parser = ArgumentParser(description=__doc__, epilog=EPILOG, + formatter_class=RawDescriptionHelpFormatter) + parser.add_argument('action', + nargs='?', + choices=['put', 'get', 'migrate'], + default='get', + help='default: get; see below for details') + parser.add_argument('--zookeeper-hosts', '-z', + help='default: %s' % ZOOKEEPER_HOSTS) + parser.add_argument('--node', + '-n', + help='default: %s' % ZK_CONFIG_PATH) + parser.add_argument('-v', + dest='verbose', + action='count', + default=0, + help='increase verbosity') + parser.add_argument('-q', + dest='quiet', + action='count', + default=0, + help='decrease verbosity') + parser.set_defaults(zookeeper_hosts=ZOOKEEPER_HOSTS, node=ZK_CONFIG_PATH) + args = parser.parse_args() + log_level = logging.WARNING - (args.verbose * 10) + (args.quiet * 10) + logging.basicConfig( + level=log_level, + format='%(message)s', + handlers=[logging.StreamHandler(sys.stderr)]) + try: + if args.action == 'get': + sys.stdout.write(get(args.zookeeper_hosts, args.node)) + elif args.action == 'put': + print put(args.zookeeper_hosts, args.node, sys.stdin.read()) + elif args.action == 'migrate': + with open(MIGRATE_FROM, 'rb') as dir_cfg_store: + content = dir_cfg_store.read() + print put(args.zookeeper_hosts, args.node, content) + except ScriptError, exc: + logging.error(str(exc)) + sys.exit(2) + except Exception, exc: # pylint: disable=broad-except + if log_level <= logging.DEBUG: + raise + else: + logging.error(str(exc)) + sys.exit(3) + +if __name__ == '__main__': + main() diff --git a/flocker/control/configuration_store/test/test_zookeeper.py b/flocker/control/configuration_store/test/test_zookeeper.py new file mode 100644 index 0000000000..b368494056 --- /dev/null +++ b/flocker/control/configuration_store/test/test_zookeeper.py @@ -0,0 +1,42 @@ +# Copyright (C) 2017 Nokia Corporation and/or its subsidiary(-ies). +# See LICENSE file for details. + +""" +Tests for ``flocker.control.configuration_store.zookeeper``. +""" +import subprocess +from time import sleep + +from ....testtools import AsyncTestCase +from ..testtools import IConfigurationStoreTestsMixin +from ..zookeeper import ZooKeeperConfigurationStore + + +class ZooKeeperConfigurationStoreInterfaceTests(IConfigurationStoreTestsMixin, + AsyncTestCase): + """ + Tests for ``ZooKeeperConfigurationStore``. + """ + def _wait_for_zk(self, maxtries=15): + cmd = ['docker', 'logs', self.docker_id] + for _ in range(maxtries): + subpr = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, _stderr = subpr.communicate() + if 'binding to port' in stdout: + return + sleep(1) + raise RuntimeError('Timed out waiting for Zookeeper container') + + def setUp(self): + super(ZooKeeperConfigurationStoreInterfaceTests, self).setUp() + self.docker_id = subprocess.check_output( + ['docker', 'run', '-d', '--network=host', 'zookeeper:3.4']).strip() + self._wait_for_zk() + self.store = ZooKeeperConfigurationStore( + zookeeper_hosts='localhost:2181' + ) + + def tearDown(self): + super(ZooKeeperConfigurationStoreInterfaceTests, self).tearDown() + subprocess.check_output(['docker', 'rm', '-f', self.docker_id]) diff --git a/flocker/control/configuration_store/zookeeper.py b/flocker/control/configuration_store/zookeeper.py new file mode 100644 index 0000000000..915e5ddb5e --- /dev/null +++ b/flocker/control/configuration_store/zookeeper.py @@ -0,0 +1,87 @@ +# Copyright (C) 2017 Nokia Corporation and/or its subsidiary(-ies). +# See LICENSE file for details. + +""" +Persistence of cluster configuration to ZooKeeper. +""" + +from contextlib import contextmanager +from datetime import datetime +import logging + +from eliot import Message +from kazoo.client import KazooClient +from pyrsistent import PClass, field +from twisted.internet.defer import succeed + +from zope.interface import implementer + +from .interface import IConfigurationStore + + +ZK_CONFIG_PATH = '/flocker/configuration' + + +# produce Eliot logs from standard logger +class EliotLogHandler(logging.Handler): + def emit(self, record): + Message.new( + message_type=u'flocker:control:store:zookeeper', + message=record.getMessage() + ).write() + + +logger = logging.getLogger("kazoo") +logger.setLevel(logging.INFO) +logger.addHandler(EliotLogHandler()) + + +@implementer(IConfigurationStore) +class ZooKeeperConfigurationStore(PClass): + zookeeper_hosts = field(mandatory=True, type=str) + + @property + @contextmanager + def client(self): + _client = KazooClient( + hosts=self.zookeeper_hosts, + connection_retry={'max_tries': 8, 'delay': 1}, + command_retry={'max_tries': 8, 'delay': 1} + ) + try: + _client.start() + yield _client + finally: + _client.stop() + + def initialize(self): + with self.client as client: + client.ensure_path(ZK_CONFIG_PATH) + return succeed(None) + + def get_content_sync(self): + with self.client as client: + out, stat = client.get(ZK_CONFIG_PATH) + mtime = datetime.fromtimestamp(stat.mtime / 1000).isoformat() + logger.info( + 'Fetched configuration version: %s, mtime: %s, size: %s', + stat.version, mtime, len(out)) + return out + + def get_content(self): + return succeed(self.get_content_sync()) + + def set_content(self, content): + with self.client as client: + stat = client.set(ZK_CONFIG_PATH, content) + mtime = datetime.fromtimestamp(stat.mtime / 1000).isoformat() + logger.info( + 'Stored configuration version: %s, mtime: %s, size: %s', + stat.version, mtime, len(content)) + return succeed(None) + + +def zookeeper_store_from_options(options): + return ZooKeeperConfigurationStore( + zookeeper_hosts=options["zookeeper-hosts"] + ) diff --git a/flocker/control/script.py b/flocker/control/script.py index 2e2f007262..6e8197e2f5 100644 --- a/flocker/control/script.py +++ b/flocker/control/script.py @@ -22,6 +22,7 @@ from ._persistence import ConfigurationPersistenceService from ._clusterstate import ClusterStateService from .configuration_store.directory import directory_store_from_options +from .configuration_store.zookeeper import zookeeper_store_from_options from ..common.script import ( flocker_standard_options, FlockerScriptRunner, main_for_service, enable_profiling, disable_profiling) @@ -70,6 +71,14 @@ def __unicode__(self): ]], ), + ConfigurationStorePlugin( + name=u'zookeeper', + factory=zookeeper_store_from_options, + options=[[ + "zookeeper-hosts", "z", b"localhost:2181", + "The directory where data will be persisted.", str + ]], + ) ] CONFIGURATION_STORE_PLUGINS_BY_NAME = { p.name: p for p in CONFIGURATION_STORE_PLUGINS diff --git a/flocker/control/test/test_script.py b/flocker/control/test/test_script.py index 1794cf36a3..e5c3399270 100644 --- a/flocker/control/test/test_script.py +++ b/flocker/control/test/test_script.py @@ -76,6 +76,43 @@ def test_path(self): options.parseOptions([b"--data-path", b"/var/xxx"]) self.assertEqual(options["data-path"], FilePath(b"/var/xxx")) + def test_configuration_store_plugin_zookeeper(self): + """ + ``--configuration-store-plugin=zookeeper`` sets ZooKeeper config store + plugin + """ + options = ControlOptions() + options.parseOptions(["--configuration-store-plugin=zookeeper"]) + self.assertEqual( + u"zookeeper", + options["configuration-store-plugin"].name + ) + + def test_configuration_zookeeper_default_hosts(self): + """ + ``--configuration-store-plugin=zookeeper`` sets default host string for + Kazoo + """ + options = ControlOptions() + options.parseOptions(["--configuration-store-plugin=zookeeper"]) + self.assertEqual( + u"localhost:2181", + options["zookeeper-hosts"] + ) + + def test_configuration_zookeeper_hosts(self): + """ + ``--zookeeper-hosts`` sets host string for Kazoo + """ + options = ControlOptions() + options.parseOptions(["--configuration-store-plugin=zookeeper", + "--zookeeper-hosts", + "example.com:2000,example.org:3000"]) + self.assertEqual( + "example.com:2000,example.org:3000", + options["zookeeper-hosts"] + ) + def test_default_agent_port(self): """ The default AMP port configured by ``ControlOptions`` is 4524. diff --git a/flocker/node/agents/functional/test_cinder.py b/flocker/node/agents/functional/test_cinder.py index b809a0ce68..26f8b8860b 100644 --- a/flocker/node/agents/functional/test_cinder.py +++ b/flocker/node/agents/functional/test_cinder.py @@ -6,6 +6,7 @@ """ import json +import os from unittest import skipIf from urlparse import urlsplit from uuid import uuid4 @@ -1158,7 +1159,7 @@ def webserver_for_test(test, url_path, response_content): def _respond(request): return response_content factory = Site(app.resource()) - endpoint = serverFromString(reactor, b"tcp:0") + endpoint = serverFromString(reactor, b"tcp:0:interface=127.0.0.1") listening = endpoint.listen(factory) def stop_port(port): @@ -1172,6 +1173,13 @@ class MetadataFromServiceTests(AsyncTestCase): """ Tests for ``metadata_from_service``. """ + def setUp(self): + super(MetadataFromServiceTests, self).setUp() + if 'no_proxy' in os.environ: + no_proxy = set(os.environ.get('no_proxy').split(',')) + no_proxy.update(('10.0.0.0', '127.0.0.1')) + os.environ['no_proxy'] = ','.join(no_proxy) + def test_success(self): """ The metadata is downloaded, decoded and returned. diff --git a/requirements/admin.txt b/requirements/admin.txt index 41eb00b19b..1540b8503b 100644 --- a/requirements/admin.txt +++ b/requirements/admin.txt @@ -1,3 +1,4 @@ +kazoo==2.2.1 apache-libcloud==1.4.0 # The `aws` command is required by the ``admin.installer``, # `admin.test.test_installer` and in the release process. diff --git a/requirements/flocker.txt b/requirements/flocker.txt index 7a7a22d923..35ce51cd7e 100644 --- a/requirements/flocker.txt +++ b/requirements/flocker.txt @@ -18,6 +18,7 @@ google-compute-engine==2.3.1 hypothesis==3.6.0 ipaddr==2.1.11 jsonschema==2.5.1 +kazoo==2.2.1 keystoneauth1==2.16.0 # FLOC-4410 keystoneclient 3.0.0 moves some exceptions around. python-keystoneclient==2.3.1 diff --git a/setup.py b/setup.py index 1ea932dd5d..5ee8494cba 100644 --- a/setup.py +++ b/setup.py @@ -131,6 +131,7 @@ def requirements_list_from_file(requirements_file, dependency_links): 'flocker-benchmark = ' + 'flocker.node.benchmark:flocker_benchmark_main', 'flocker-node-era = flocker.common._era:era_main', + 'flocker-zk = flocker.cli.zookeeper_script:main' ], },