From 4580ff0ef80168a5a0246c69f4d3d0455008d2b6 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Tue, 4 Oct 2016 12:49:42 -0500 Subject: [PATCH 01/23] Sample script to make a cluster --- examples/make-cluster.py | 181 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 examples/make-cluster.py diff --git a/examples/make-cluster.py b/examples/make-cluster.py new file mode 100644 index 0000000..b06661d --- /dev/null +++ b/examples/make-cluster.py @@ -0,0 +1,181 @@ +#!/usr/bin/python3 + +import sys +import re +import os +import pwd +import logging +import argparse +from requests.auth import HTTPDigestAuth +from marklogic import MarkLogic +from marklogic.models.host import Host +from marklogic.connection import Connection +from marklogic.exceptions import * + +""" +This app joins several MarkLogic instances together into a cluster. +It can also couple that cluster to another existing cluster. +""" + +class App: + def __init__(self): + self.marklogic = None + self.uname = pwd.getpwuid(os.getuid()).pw_name + self.adminuser = "admin" + self.adminpass = "admin" + self.realm = "public" + self.host = None + self.boothost = None + self.couple = None + self.couple_user = None + self.couple_pass = None + self.name = None + + def set_credentials(self, user, password): + self.adminuser = user + self.adminpass = password + + def set_realm(self, realm): + self.realm = realm + + def set_boot_host(self, host): + self.boothost = host + + def set_host(self, name): + self.host = name + + def set_couple(self, couple): + self.couple = couple + + def set_couple_credentials(self, user, password): + self.couple_user = user + self.couple_pass = password + + def set_name(self, name): + self.name = name + + def setup_cluster(self): + if self.couple is not None and self.couple_pass is None: + self.couple_pass = self.adminpass + self.couple_user = self.adminuser + + if self.boothost is None: + self.boothost = self.host[0] + self.host.remove(self.boothost) + + if self.ml_init(self.boothost): + self.ml_security(self.boothost) + + conn = Connection(self.boothost, HTTPDigestAuth(self.adminuser, self.adminpass)) + self.marklogic = MarkLogic(conn) + + for hostname in self.host: + self.ml_init(hostname) + self.ml_join(self.boothost, hostname) + + if self.name is not None: + print("{0}: rename cluster...".format(self.boothost)) + cluster = self.marklogic.cluster() + cluster.set_cluster_name(self.name) + cluster.update() + + if self.couple is not None: + for couple in self.couple: + print("{0}: couple with {1}...".format(self.boothost, couple)) + altconn = Connection(couple, + HTTPDigestAuth(self.couple_user, + self.couple_pass)) + altml = MarkLogic(altconn) + altcluster = altml.cluster() + cluster = self.marklogic.cluster() + cluster.couple(altcluster) + + print("Finished") + + def ml_init(self, hostname): + print("{0}: initialize host...".format(hostname)) + try: + host = MarkLogic.instance_init(hostname) + except UnauthorizedAPIRequest: + # Assume that this happened because the host is already initialized + host = Host(hostname) + + return host.just_initialized() + + def ml_join(self, boothost, hostname): + print("{0}: join cluster with {1}...".format(hostname, boothost)) + cluster = self.marklogic.cluster() + host = Host(hostname) + cluster.add_host(host) + + def ml_security(self, hostname): + print("{0}: initialize security...".format(hostname)) + MarkLogic.instance_admin(hostname, self.realm, self.adminuser, self.adminpass) + +def main(): + parser = argparse.ArgumentParser( + description="Join MarkLogic server instances into a cluster") + + parser.add_argument('--credentials', default='admin:admin', + metavar='USER:PASS', + help='Admin user:pass for new cluster') + parser.add_argument('--realm', default='public', + help='Security realm for new cluster') + parser.add_argument('--host', nargs="+", + metavar='HOST', + required=True, + help='Hostnames of instances to join') + parser.add_argument('--boot', + metavar='HOST', + help='Select the bootstrap host for new cluster') + parser.add_argument('--name', + help='Set the cluster name for the new cluster') + parser.add_argument('--couple', nargs="+", + metavar='BOOTHOST', + help='Bootstrap host of cluster with which to couple') + parser.add_argument('--couple-credentials', + metavar='USER:PASS', + help='Admin user:pass for cluster with which to couple') + parser.add_argument('--debug', action='store_true', + help='Enable debug logging') + + args = vars(parser.parse_args()) + + app = App() + + for opt in args: + arg = args[opt] + if opt == "credentials" and arg is not None: + try: + adminuser, adminpass = re.split(":", arg) + app.set_credentials(adminuser, adminpass) + except ValueError: + print ("--credentials value must be 'user:password':", arg) + sys.exit(1) + if opt == "realm": + app.set_realm(arg) + if opt == "boot": + app.set_boot_host(arg) + if opt == "host": + app.set_host(arg) + if opt == "name": + app.set_name(arg) + if opt == "couple": + app.set_couple(arg) + if opt == "couple_credentials" and arg is not None: + try: + adminuser, adminpass = re.split(":", arg) + app.set_couple_credentials(adminuser, adminpass) + except ValueError: + print ("--couple-credentials value must be 'user:password':", arg) + sys.exit(1) + + if args['debug']: + logging.basicConfig(level=logging.WARNING) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("marklogic").setLevel(logging.DEBUG) + + app.setup_cluster() + +if __name__ == '__main__': + main() From 93d1fad5976a3e593842f2642bc0231596f393be Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 16 Nov 2016 16:24:26 -0600 Subject: [PATCH 02/23] Support new/renamed MarkLogic 9 properties; small fixes, mostly python lint related --- marklogic/cli/manager/database.py | 7 +++++-- marklogic/models/database/__init__.py | 7 ++++--- marklogic/models/forest/__init__.py | 29 +++++++++++++-------------- marklogic/models/group/__init__.py | 14 ++++++------- 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/marklogic/cli/manager/database.py b/marklogic/cli/manager/database.py index a8bc695..4b485b0 100644 --- a/marklogic/cli/manager/database.py +++ b/marklogic/cli/manager/database.py @@ -33,7 +33,7 @@ class DatabaseManager(Manager): The DatabaseManager performs operations on databases. """ def __init__(self): - pass + self.forests = [] def list(self, args, config, connection): names = Database.list(connection) @@ -70,7 +70,10 @@ def modify(self, args, config, connection): sys.exit(1) if args['json'] is not None: - database = self._read(None, args['json'], + rname = name + if 'name' in args['json']: + rname = None + database = self._read(rname, args['json'], connection=connection) database.name = name diff --git a/marklogic/models/database/__init__.py b/marklogic/models/database/__init__.py index 39f0c83..93b077c 100644 --- a/marklogic/models/database/__init__.py +++ b/marklogic/models/database/__init__.py @@ -3283,9 +3283,9 @@ def unmarshal(cls, config, hostname=None, # atomic values or lists of atomic values. atomic = {'attribute-value-positions', 'collection-lexicon', - 'database-name', 'directory-creation', + 'data-encryption', 'database-name', 'directory-creation', 'element-value-positions', 'element-word-positions', - 'enabled', 'expunge-locks', + 'enabled', 'encryption-key-id', 'expunge-locks', 'fast-case-sensitive-searches', 'fast-diacritic-sensitive-searches', 'fast-element-character-searches', @@ -3294,7 +3294,8 @@ def unmarshal(cls, config, hostname=None, 'fast-element-word-searches', 'fast-phrase-searches', 'fast-reverse-searches', 'field-value-positions', 'field-value-searches', - 'forest', 'format-compatibility', 'in-memory-limit', + 'forest', 'format-compatibility', + 'in-memory-geospatial-region-index-size', 'in-memory-limit', 'in-memory-list-size', 'in-memory-range-index-size', 'in-memory-reverse-index-size', 'in-memory-tree-size', diff --git a/marklogic/models/forest/__init__.py b/marklogic/models/forest/__init__.py index c6c3d4a..d7d3c6f 100644 --- a/marklogic/models/forest/__init__.py +++ b/marklogic/models/forest/__init__.py @@ -22,9 +22,10 @@ # Paul Hoehne 03/01/2015 Initial development # -import json, logging -import marklogic.exceptions -from marklogic.utilities.validators import * +import json +import logging +from marklogic.exceptions import UnexpectedManagementAPIResponse, UnsupportedOperation +from marklogic.utilities.validators import validate_forest_availability, validate_boolean from marklogic.models.model import Model from marklogic.models.forest.scheduledbackup import ScheduledForestBackup from marklogic.models.forest.replica import ForestReplica @@ -121,10 +122,10 @@ def set_database(self,name): return self._set_config_property('database', name) def database_replication(self): - raise UnsupportedOperation("Not implemented yet") + raise UnsupportedOperation("Not implemented yet") def set_database_replication(self,value): - raise UnsupportedOperation("Not implemented yet") + raise UnsupportedOperation("Not implemented yet") def enabled(self): return self._get_config_property('enabled') @@ -330,8 +331,6 @@ def lookup(cls, connection, name): :param connection: The connection to a MarkLogic server :return: The Forest object """ - logger = logging.getLogger("marklogic") - uri = connection.uri("forests", name) response = connection.get(uri) @@ -374,11 +373,11 @@ def unmarshal(cls, config, connection=None, save_connection=True): atomic = {'availability', 'data-directory', 'database-replication', 'enabled', - 'failover-enable', 'fast-data-directory', + 'failover-enable', 'fast-data-directory', 'fast-data-max-size', 'forest-name', 'host', 'large-data-directory', 'range', 'rebalancer-enable', 'updates-allowed', 'database' - } + } for key in result._config: olist = [] @@ -389,34 +388,34 @@ def unmarshal(cls, config, connection=None, save_connection=True): for backup in result._config['forest-backup']: #logger.debug(backup) temp = None - if (backup['backup-type'] == 'minutely'): + if backup['backup-type'] == 'minutely': temp = ScheduledForestBackup.minutely( backup['backup-directory'], backup['backup-period']) - elif (backup['backup-type'] == 'hourly'): + elif backup['backup-type'] == 'hourly': minute = int(backup['backup-start-time'].split(':')[1]) temp = ScheduledForestBackup.hourly( backup['backup-directory'], backup['backup-period'], minute) - elif (backup['backup-type'] == 'daily'): + elif backup['backup-type'] == 'daily': temp = ScheduledForestBackup.daily( backup['backup-directory'], backup['backup-period'], backup['backup-start-time']) - elif (backup['backup-type'] == 'weekly'): + elif backup['backup-type'] == 'weekly': temp = ScheduledForestBackup.weekly( backup['backup-directory'], backup['backup-period'], backup['backup-day'], backup['backup-start-time']) - elif (backup['backup-type'] == 'monthly'): + elif backup['backup-type'] == 'monthly': temp = ScheduledForestBackup.monthly( backup['backup-directory'], backup['backup-period'], backup['backup-month-day'], backup['backup-start-time']) - elif (backup['backup-type'] == 'once'): + elif backup['backup-type'] == 'once': temp = ScheduledForestBackup.once( backup['backup-directory'], backup['backup-start-date'], diff --git a/marklogic/models/group/__init__.py b/marklogic/models/group/__init__.py index 66340b6..0bdbbdd 100644 --- a/marklogic/models/group/__init__.py +++ b/marklogic/models/group/__init__.py @@ -34,7 +34,7 @@ from marklogic.models.group.audit import Audit, AuditEvent, AuditRestriction from marklogic.models.group.schema import Schema -class Group(Model,PropertyLists): +class Group(Model, PropertyLists): """ The Group class encapsulates a MarkLogic group. It provides methods to set/get group attributes. The use of methods will @@ -69,10 +69,10 @@ def marshal(self): :return: A hash of the keys in this object and their values, recursively. """ - struct = { } + struct = {} for key in self._config: if key == 'audit': - substruct = { } + substruct = {} audit = self._config[key]._config for prop in audit: if prop == 'audit-event': @@ -94,7 +94,7 @@ def marshal(self): olist.append(schema._config) struct[key] = olist else: - struct[key] = self._config[key]; + struct[key] = self._config[key] return struct @@ -152,9 +152,9 @@ def unmarshal(cls, config, 'xdqp-ssl-ciphers', 'xdqp-ssl-enabled', 'xdqp-timeout', 'opsdirector-config', 'opsdirector-log-level', 'opsdirector-metering', - 'opsdirector-session-uri', 'telemetry-config', + 'opsdirector-session-endpoint', 'telemetry-config', 'telemetry-log-level', 'telemetry-metering', - 'telemetry-session-uri' + 'telemetry-session-endpoint' } for key in result._config: @@ -267,7 +267,7 @@ def update(self, connection=None): self.name = self._config['group-name'] if 'etag' in response.headers: - self.etag = response.headers['etag'] + self.etag = response.headers['etag'] return self From f016dfc1a18b6ea1bc9581a582718c12dece1a76 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 16 Nov 2016 16:31:42 -0600 Subject: [PATCH 03/23] MarkLogic Python API version 0.0.13 released --- marklogic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marklogic/__init__.py b/marklogic/__init__.py index ba01f03..0c6fa31 100644 --- a/marklogic/__init__.py +++ b/marklogic/__init__.py @@ -36,7 +36,7 @@ from marklogic.models.server import OdbcServer, XdbcServer from marklogic.exceptions import InvalidAPIRequest, UnexpectedManagementAPIResponse -__version__ = "0.0.12" +__version__ = "0.0.13" class MarkLogic: """ From 810b88b2e823491490146ad5ccd07836ea90d374 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Thu, 17 Nov 2016 16:52:58 -0600 Subject: [PATCH 04/23] Added getters/setters for new properties --- marklogic/models/database/__init__.py | 53 ++++++ marklogic/models/forest/__init__.py | 11 ++ marklogic/models/group/__init__.py | 243 +++++++++++++++++++++----- 3 files changed, 259 insertions(+), 48 deletions(-) diff --git a/marklogic/models/database/__init__.py b/marklogic/models/database/__init__.py index 93b077c..d97d003 100644 --- a/marklogic/models/database/__init__.py +++ b/marklogic/models/database/__init__.py @@ -1755,6 +1755,24 @@ def in_memory_triple_index_size(self): """ return self._get_config_property('in-memory-triple-index-size') + def set_in_memory_geospatial_region_index_size(self, value): + """ + Set the in-memory geospatial region index size. + + :param value: The value (1-64 megabytes) + :return: The database object + """ + self._validate(value, {'max': 64, 'min': 1}) + return self._set_config_property('in-memory-geospatial-region-index-size', value) + + def in_memory_geospatial_region_index_size(self, value): + """ + The in-memory geospatial region index size. + + :return: The size in megabytes + """ + return self._get_config_property('in-memory-geospatial-region-index-size') + def set_large_size_threshold(self, limit=1024): """ Sets size threshold for large objects, in kilobytes. @@ -2660,6 +2678,41 @@ def assignment_policy(self): """ return self._get_config_property('assignment-policy') + def data_encryption(self): + """ + Encryption at rest for this database. + + :return: The encryption setting. + """ + return self._get_config_property('data-encryption') + + def set_data_encryption(self, value): + """ + Encryption at rest for this database. + + :param value: The encryption setting. + :return: The database object. + """ + self._validate(value, ['on', 'off', 'default-cluster']) + return self._set_config_property('data-encryption', value) + + def encryption_key_id(self): + """ + Data encryption key id. + + :return: The key id. + """ + return self._get_config_property('encryption-key-id') + + def set_encryption_key_id(self, value): + """ + Set the data encryption key id. + + :param value: The key id. + :return: The database object. + """ + return self._set_config_property('encryption-key-id', value) + def path_namespaces(self): """ Return the path namespaces defined or None, if no path namespaces diff --git a/marklogic/models/forest/__init__.py b/marklogic/models/forest/__init__.py index d7d3c6f..f2c1513 100644 --- a/marklogic/models/forest/__init__.py +++ b/marklogic/models/forest/__init__.py @@ -111,6 +111,17 @@ def fast_data_directory(self): def set_fast_data_directory(self, path): return self._set_config_property('fast-data-directory', path) + def fast_data_max_size(self): + """ + Return the fast data max size for the forest. + + :return:The fast data maxs ize + """ + return self._get_config_property('fast-data-max-size') + + def set_fast_data_max_size(self, size): + return self._set_config_property('fast-data-max-size', size) + def database(self): return self._get_config_property('database') diff --git a/marklogic/models/group/__init__.py b/marklogic/models/group/__init__.py index 0bdbbdd..1c02a7c 100644 --- a/marklogic/models/group/__init__.py +++ b/marklogic/models/group/__init__.py @@ -336,7 +336,7 @@ def compressed_tree_read_size(self): """ return self._get_config_property('compressed-tree-read-size') - def set_compressed_tree_read_size(self,value): + def set_compressed_tree_read_size(self, value): """ Set the compressed-tree-read-size. @@ -354,7 +354,7 @@ def performance_metering_retain_hourly(self): """ return self._get_config_property('performance-metering-retain-hourly') - def set_performance_metering_retain_hourly(self,value): + def set_performance_metering_retain_hourly(self, value): """ Set the performance-metering-retain-hourly. @@ -372,7 +372,7 @@ def expanded_tree_cache_size(self): """ return self._get_config_property('expanded-tree-cache-size') - def set_expanded_tree_cache_size(self,value): + def set_expanded_tree_cache_size(self, value): """ Set the expanded-tree-cache-size. @@ -386,13 +386,13 @@ def metering_enabled(self): """ true or false - :return: The metering-enabled. + :return: The metering-enabled value. """ return self._get_config_property('metering-enabled') - def set_metering_enabled(self,value=True): + def set_metering_enabled(self, value=True): """ - Set the metering-enabled. + Set metering-enabled. :param value: The metering-enabled. :return: The object with the mutated property value. @@ -400,6 +400,154 @@ def set_metering_enabled(self,value=True): self._validate(value, 'boolean') return self._set_config_property('metering-enabled', value) + def opsdirector_config(self): + """ + The OpsDirector config level: disabled, frequent, or infrequent. + + :return: The config level. + """ + return self._get_config_property("opsdirector-config") + + def set_opsdirector_config(self, value): + """ + Set the OpsDirector config level: disabled, frequent, or infrequent. + + :param value: The config level. + :return: The object with the mutated property value. + """ + self._validate(value, ['disabled', 'frequent', 'infrequent']) + return self._set_config_property('opsdirector-config', value) + + def opsdirector_log_level(self): + """ + The OpsDirector log level. + + :return: The log level. + """ + return self._get_config_property("opsdirector-log-level") + + def set_opsdirector_log_level(self, value): + """ + Set the OpsDirector log level. + + :param value: The log level. + :return: The object with the mutated property value. + """ + self._validate(value, ['disabled', 'finest', 'finer', 'fine', + 'debug', 'config', 'info', 'notice', + 'warning', 'error', 'critical', 'alert', 'emergency']) + return self._set_config_property("opsdirector-log-level", value) + + def opsdirector_metering(self): + """ + The OpsDirector metering level. + + :return: The metering level. + """ + return self._get_config_property("opsdirector_metering") + + def set_opsdirector_metering(self, value): + """ + Set the OpsDirector metering level. + + :param value: The metering level. + :return: The object with the mutated property value. + """ + self._validate(value, ['disabled', 'full', 'aggregates', 'usage-only']) + return self._set_config_property("opsdirector-metering", value) + + def opsdirector_session_endpoint(self): + """ + The OpsDirector session endpoint. + + :return: The endpoint. + """ + return self._get_config_property("opsdirector-session-endpoint") + + def set_opsdirector_session_endpoint(self, value): + """ + Set the OpsDirector session endpoint. + + :param value: The endpoint. + :return: The object with the mutated property value. + """ + # FIXME: Should I test that this is a reasonable http(s) URI? + return self._set_config_property("opsdirector-session-endpoint", value) + + def telemetry_config(self): + """ + The Telemetry config level: disabled, frequent, or infrequent. + + :return: The config level. + """ + return self._get_config_property("telemetry-config") + + def set_telemetry_config(self, value): + """ + Set the Telemetry config level: disabled, frequent, or infrequent. + + :param value: The config level. + :return: The object with the mutated property value. + """ + self._validate(value, ['disabled', 'frequent', 'infrequent']) + return self._set_config_property('telemetry-config', value) + + def telemetry_log_level(self): + """ + The Telemetry log level. + + :return: The log level. + """ + return self._get_config_property("telemetry-log-level") + + def set_telemetry_log_level(self, value): + """ + Set the Telemetry log level. + + :param value: The log level. + :return: The object with the mutated property value. + """ + self._validate(value, ['disabled', 'finest', 'finer', 'fine', + 'debug', 'config', 'info', 'notice', + 'warning', 'error', 'critical', 'alert', 'emergency']) + return self._set_config_property("telemetry-log-level", value) + + def telemetry_metering(self): + """ + The Telemetry metering level. + + :return: The metering level. + """ + return self._get_config_property("telemetry_metering") + + def set_telemetry_metering(self, value): + """ + Set the Telemetry metering level. + + :param value: The metering level. + :return: The object with the mutated property value. + """ + self._validate(value, ['disabled', 'full', 'aggregates', 'usage-only']) + return self._set_config_property("telemetry-metering", value) + + def telemetry_session_endpoint(self): + """ + The Telemetry session endpoint. + + :return: The endpoint. + """ + return self._get_config_property("telemetry-session-endpoint") + + def set_telemetry_session_endpoint(self, value): + """ + Set the Telemetry session endpoint. + + :param value: The endpoint. + :return: The object with the mutated property value. + """ + # FIXME: Should I test that this is a reasonable http(s) URI? + return self._set_config_property("telemetry-session-endpoint", value) + def triple_cache_timeout(self): """ An integer number of seconds, min 0, max 4294967295. @@ -408,7 +556,7 @@ def triple_cache_timeout(self): """ return self._get_config_property('triple-cache-timeout') - def set_triple_cache_timeout(self,value): + def set_triple_cache_timeout(self, value): """ Set the triple-cache-timeout. @@ -426,7 +574,7 @@ def security_database(self): """ return self._get_config_property('security-database') - def set_security_database(self,value): + def set_security_database(self, value): """ Set the security-database. @@ -444,7 +592,7 @@ def module_cache_timeout(self): """ return self._get_config_property('module-cache-timeout') - def set_module_cache_timeout(self,value): + def set_module_cache_timeout(self, value): """ Set the module-cache-timeout. @@ -462,7 +610,7 @@ def performance_metering_retain_raw(self): """ return self._get_config_property('performance-metering-retain-raw') - def set_performance_metering_retain_raw(self,value): + def set_performance_metering_retain_raw(self, value): """ Set the performance-metering-retain-raw. @@ -480,7 +628,7 @@ def performance_metering_period(self): """ return self._get_config_property('performance-metering-period') - def set_performance_metering_period(self,value): + def set_performance_metering_period(self, value): """ Set the performance-metering-period. @@ -498,7 +646,7 @@ def background_io_limit(self): """ return self._get_config_property('background-io-limit') - def set_background_io_limit(self,value): + def set_background_io_limit(self, value): """ Set the background-io-limit. @@ -516,7 +664,7 @@ def file_log_level(self): """ return self._get_config_property('file-log-level') - def set_file_log_level(self,value): + def set_file_log_level(self, value): """ Set the file-log-level. @@ -534,7 +682,7 @@ def meters_database(self): """ return self._get_config_property('meters-database') - def set_meters_database(self,value): + def set_meters_database(self, value): """ Set the meters-database. @@ -552,7 +700,7 @@ def list_cache_partitions(self): """ return self._get_config_property('list-cache-partitions') - def set_list_cache_partitions(self,value): + def set_list_cache_partitions(self, value): """ Set the list-cache-partitions. @@ -570,7 +718,7 @@ def s3_protocol(self): """ return self._get_config_property('s3-protocol') - def set_s3_protocol(self,value): + def set_s3_protocol(self, value): """ Set the s3-protocol. @@ -588,7 +736,7 @@ def xdqp_ssl_enabled(self): """ return self._get_config_property('xdqp-ssl-enabled') - def set_xdqp_ssl_enabled(self,value=True): + def set_xdqp_ssl_enabled(self, value=True): """ Set the xdqp-ssl-enabled. @@ -606,7 +754,7 @@ def performance_metering_retain_daily(self): """ return self._get_config_property('performance-metering-retain-daily') - def set_performance_metering_retain_daily(self,value): + def set_performance_metering_retain_daily(self, value): """ Set the performance-metering-retain-daily. @@ -624,7 +772,7 @@ def http_user_agent(self): """ return self._get_config_property('http-user-agent') - def set_http_user_agent(self,value): + def set_http_user_agent(self, value): """ Set the http-user-agent. @@ -642,7 +790,7 @@ def compressed_tree_cache_size(self): """ return self._get_config_property('compressed-tree-cache-size') - def set_compressed_tree_cache_size(self,value): + def set_compressed_tree_cache_size(self, value): """ Set the compressed-tree-cache-size. @@ -660,7 +808,7 @@ def rotate_log_files(self): """ return self._get_config_property('rotate-log-files') - def set_rotate_log_files(self,value): + def set_rotate_log_files(self, value): """ Set the rotate-log-files. @@ -678,7 +826,7 @@ def host_timeout(self): """ return self._get_config_property('host-timeout') - def set_host_timeout(self,value): + def set_host_timeout(self, value): """ Set the host-timeout. @@ -696,7 +844,7 @@ def group_name(self): """ return self._get_config_property('group-name') - def set_group_name(self,value): + def set_group_name(self, value): """ Set the group-name. @@ -714,7 +862,7 @@ def triple_cache_partitions(self): """ return self._get_config_property('triple-cache-partitions') - def set_triple_cache_partitions(self,value): + def set_triple_cache_partitions(self, value): """ Set the triple-cache-partitions. @@ -732,7 +880,7 @@ def performance_metering_enabled(self): """ return self._get_config_property('performance-metering-enabled') - def set_performance_metering_enabled(self,value=True): + def set_performance_metering_enabled(self, value=True): """ Set the performance-metering-enabled. @@ -750,7 +898,7 @@ def triple_value_cache_partitions(self): """ return self._get_config_property('triple-value-cache-partitions') - def set_triple_value_cache_partitions(self,value): + def set_triple_value_cache_partitions(self, value): """ Set the triple-value-cache-partitions. @@ -768,7 +916,7 @@ def xdqp_ssl_allow_sslv3(self): """ return self._get_config_property('xdqp-ssl-allow-sslv3') - def set_xdqp_ssl_allow_sslv3(self,value=True): + def set_xdqp_ssl_allow_sslv3(self, value=True): """ Set the xdqp-ssl-allow-sslv3. @@ -786,7 +934,7 @@ def compressed_tree_cache_partitions(self): """ return self._get_config_property('compressed-tree-cache-partitions') - def set_compressed_tree_cache_partitions(self,value): + def set_compressed_tree_cache_partitions(self, value): """ Set the compressed-tree-cache-partitions. @@ -804,7 +952,7 @@ def xdqp_ssl_ciphers(self): """ return self._get_config_property('xdqp-ssl-ciphers') - def set_xdqp_ssl_ciphers(self,value): + def set_xdqp_ssl_ciphers(self, value): """ Set the xdqp-ssl-ciphers. @@ -822,7 +970,7 @@ def events_activated(self): """ return self._get_config_property('events-activated') - def set_events_activated(self,value=True): + def set_events_activated(self, value=True): """ Set the events-activated. @@ -840,7 +988,7 @@ def expanded_tree_cache_partitions(self): """ return self._get_config_property('expanded-tree-cache-partitions') - def set_expanded_tree_cache_partitions(self,value): + def set_expanded_tree_cache_partitions(self, value): """ Set the expanded-tree-cache-partitions. @@ -858,7 +1006,7 @@ def keep_log_files(self): """ return self._get_config_property('keep-log-files') - def set_keep_log_files(self,value): + def set_keep_log_files(self, value): """ Set the keep-log-files. @@ -876,7 +1024,7 @@ def smtp_relay(self): """ return self._get_config_property('smtp-relay') - def set_smtp_relay(self,value): + def set_smtp_relay(self, value): """ Set the smtp-relay. @@ -894,7 +1042,7 @@ def http_timeout(self): """ return self._get_config_property('http-timeout') - def set_http_timeout(self,value): + def set_http_timeout(self, value): """ Set the http-timeout. @@ -912,7 +1060,7 @@ def triple_value_cache_timeout(self): """ return self._get_config_property('triple-value-cache-timeout') - def set_triple_value_cache_timeout(self,value): + def set_triple_value_cache_timeout(self, value): """ Set the triple-value-cache-timeout. @@ -930,7 +1078,7 @@ def s3_domain(self): """ return self._get_config_property('s3-domain') - def set_s3_domain(self,value): + def set_s3_domain(self, value): """ Set the s3-domain. @@ -948,7 +1096,7 @@ def triple_cache_size(self): """ return self._get_config_property('triple-cache-size') - def set_triple_cache_size(self,value): + def set_triple_cache_size(self, value): """ Set the triple-cache-size. @@ -966,7 +1114,7 @@ def system_log_level(self): """ return self._get_config_property('system-log-level') - def set_system_log_level(self,value): + def set_system_log_level(self, value): """ Set the system-log-level. @@ -984,7 +1132,7 @@ def s3_server_side_encryption(self): """ return self._get_config_property('s3-server-side-encryption') - def set_s3_server_side_encryption(self,value): + def set_s3_server_side_encryption(self, value): """ Set the s3-server-side-encryption. @@ -1002,7 +1150,7 @@ def host_initial_timeout(self): """ return self._get_config_property('host-initial-timeout') - def set_host_initial_timeout(self,value): + def set_host_initial_timeout(self, value): """ Set the host-initial-timeout. @@ -1020,7 +1168,7 @@ def list_cache_size(self): """ return self._get_config_property('list-cache-size') - def set_list_cache_size(self,value): + def set_list_cache_size(self, value): """ Set the list-cache-size. @@ -1038,7 +1186,7 @@ def xdqp_timeout(self): """ return self._get_config_property('xdqp-timeout') - def set_xdqp_timeout(self,value): + def set_xdqp_timeout(self, value): """ Set the xdqp-timeout. @@ -1056,7 +1204,7 @@ def failover_enable(self): """ return self._get_config_property('failover-enable') - def set_failover_enable(self,value=True): + def set_failover_enable(self, value=True): """ Set the failover-enable. @@ -1074,7 +1222,7 @@ def triple_value_cache_size(self): """ return self._get_config_property('triple-value-cache-size') - def set_triple_value_cache_size(self,value): + def set_triple_value_cache_size(self, value): """ Set the triple-value-cache-size. @@ -1092,7 +1240,7 @@ def xdqp_ssl_allow_tls(self): """ return self._get_config_property('xdqp-ssl-allow-tls') - def set_xdqp_ssl_allow_tls(self,value=True): + def set_xdqp_ssl_allow_tls(self, value=True): """ Set the xdqp-ssl-allow-tls. @@ -1110,7 +1258,7 @@ def smtp_timeout(self): """ return self._get_config_property('smtp-timeout') - def set_smtp_timeout(self,value): + def set_smtp_timeout(self, value): """ Set the smtp-timeout. @@ -1128,7 +1276,7 @@ def retry_timeout(self): """ return self._get_config_property('retry-timeout') - def set_retry_timeout(self,value): + def set_retry_timeout(self, value): """ Set the retry-timeout. @@ -1137,4 +1285,3 @@ def set_retry_timeout(self,value): """ self._validate(value, {'max': 4294967295, 'min': 0}) return self._set_config_property('retry-timeout', value) - From 7c7b743863a7396f2a7a9f687142dbd558140ea5 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Thu, 17 Nov 2016 16:56:40 -0600 Subject: [PATCH 05/23] MarkLogic Python API version 0.0.14 released --- marklogic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marklogic/__init__.py b/marklogic/__init__.py index 0c6fa31..ed3de89 100644 --- a/marklogic/__init__.py +++ b/marklogic/__init__.py @@ -36,7 +36,7 @@ from marklogic.models.server import OdbcServer, XdbcServer from marklogic.exceptions import InvalidAPIRequest, UnexpectedManagementAPIResponse -__version__ = "0.0.13" +__version__ = "0.0.14" class MarkLogic: """ From 9bf50f4d1cb739052fab53083c6f084b34cea8af Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 1 Mar 2017 11:31:04 -0600 Subject: [PATCH 06/23] Fixed message formatting --- marklogic/utilities/validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/marklogic/utilities/validators.py b/marklogic/utilities/validators.py index 1994734..b51c910 100644 --- a/marklogic/utilities/validators.py +++ b/marklogic/utilities/validators.py @@ -244,8 +244,8 @@ def validate_collation(index_type, collation): return if collation is None or collation == "": return - raise ValidationError('Collation cannot be {0} for an index of type {1}' \ - .format(index_type, collation)) + raise ValidationError('Invalid collation for index of type {0}' \ + .format(index_type), repr(collation)) def validate_type(raw_val, cls): """ From 4e3ebb2829f54de557ec5d71a85ab851a575530e Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 1 Mar 2017 11:45:36 -0600 Subject: [PATCH 07/23] First, crude (insecure) attempt at https support --- marklogic/__init__.py | 2 +- marklogic/cli/template.py | 2 ++ marklogic/connection.py | 36 +++++++++++++++++++++++++----------- marklogic/mma.py | 16 ++++++++++++---- 4 files changed, 40 insertions(+), 16 deletions(-) diff --git a/marklogic/__init__.py b/marklogic/__init__.py index ed3de89..617dfca 100644 --- a/marklogic/__init__.py +++ b/marklogic/__init__.py @@ -36,7 +36,7 @@ from marklogic.models.server import OdbcServer, XdbcServer from marklogic.exceptions import InvalidAPIRequest, UnexpectedManagementAPIResponse -__version__ = "0.0.14" +__version__ = "0.0.15" class MarkLogic: """ diff --git a/marklogic/cli/template.py b/marklogic/cli/template.py index 2894e5d..ea647ea 100644 --- a/marklogic/cli/template.py +++ b/marklogic/cli/template.py @@ -638,6 +638,8 @@ def _make_parser(self, command, artifact, description=""): help='Host on which to issue the request') parser.add_argument('--credentials', default='admin:admin', help='Login credentials for request') + parser.add_argument('--https', action='store_true', + help='Enable https') parser.add_argument('--debug', action='store_true', help='Enable debug logging') return parser diff --git a/marklogic/connection.py b/marklogic/connection.py index 6b7dca4..e359f30 100644 --- a/marklogic/connection.py +++ b/marklogic/connection.py @@ -31,6 +31,7 @@ from requests.exceptions import ReadTimeout from requests.packages.urllib3.exceptions import ProtocolError from requests.packages.urllib3.exceptions import ReadTimeoutError +from requests.packages import urllib3 """ Connection related classes and method to connect to MarkLogic. @@ -56,6 +57,9 @@ def __init__(self, host, auth, self.logger = logging.getLogger("marklogic.connection") self.payload_logger = logging.getLogger("marklogic.connection.payloads") + self.verify = False # Danger, Will Robinson! + urllib3.disable_warnings() + # You'd expect parameters to be a dictionary, but then it couldn't # have repeated keys, so it's an array. def uri(self, relation, name=None, @@ -104,7 +108,7 @@ def client_uri(self, path, protocol=None, host=None, port=None, version=None): def head(self, uri, accept="application/json"): self.logger.debug("HEAD {0}...".format(uri)) - self.response = requests.head(uri, auth=self.auth) + self.response = requests.head(uri, auth=self.auth, verify=self.verify) return self._response() def get(self, uri, accept="application/json", headers=None): @@ -117,7 +121,8 @@ def get(self, uri, accept="application/json", headers=None): self.payload_logger.debug("Headers:") self.payload_logger.debug(json.dumps(headers, indent=2)) - self.response = requests.get(uri, auth=self.auth, headers=headers) + self.response = requests.get(uri, auth=self.auth, headers=headers, + verify=self.verify) return self._response() def post(self, uri, payload=None, etag=None, headers=None, @@ -143,14 +148,17 @@ def post(self, uri, payload=None, etag=None, headers=None, self.payload_logger.debug(payload) if payload is None: - self.response = requests.post(uri, auth=self.auth, headers=headers) + self.response = requests.post(uri, auth=self.auth, headers=headers, + verify=self.verify) else: if content_type == "application/json": self.response = requests.post(uri, json=payload, - auth=self.auth, headers=headers) + auth=self.auth, headers=headers, + verify=self.verify) else: self.response = requests.post(uri, data=payload, - auth=self.auth, headers=headers) + auth=self.auth, headers=headers, + verify=self.verify) return self._response() @@ -173,14 +181,17 @@ def put(self, uri, payload=None, etag=None, self.payload_logger.debug(payload) if payload is None: - self.response = requests.put(uri, auth=self.auth, headers=headers) + self.response = requests.put(uri, auth=self.auth, headers=headers, + verify=self.verify) else: if content_type == "application/json": self.response = requests.put(uri, json=payload, - auth=self.auth, headers=headers) + auth=self.auth, headers=headers, + verify=self.verify) else: self.response = requests.put(uri, data=payload, - auth=self.auth, headers=headers) + auth=self.auth, headers=headers, + verify=self.verify) return self._response() @@ -203,10 +214,12 @@ def delete(self, uri, payload=None, etag=None, self.payload_logger.debug(payload) if payload is None: - self.response = requests.delete(uri, auth=self.auth, headers=headers) + self.response = requests.delete(uri, auth=self.auth, headers=headers, + verify=self.verify) else: self.response = requests.delete(uri, json=payload, - auth=self.auth, headers=headers) + auth=self.auth, headers=headers, + verify=self.verify) return self._response() @@ -250,7 +263,8 @@ def wait_for_restart(self, last_startup, timestamp_uri="/admin/v1/timestamp"): self.logger.debug("Waiting for restart of {0}" .format(self.host)) response = requests.get(uri, auth=self.auth, - headers={'accept': 'application/json'}) + headers={'accept': 'application/json'}, + verify=self.verify) done = (response.status_code == 200 and response.text != last_startup) except TypeError: diff --git a/marklogic/mma.py b/marklogic/mma.py index 8a4bec4..0db3f29 100644 --- a/marklogic/mma.py +++ b/marklogic/mma.py @@ -7,6 +7,7 @@ import shlex import sys from requests.auth import HTTPDigestAuth +from requests.auth import HTTPBasicAuth from marklogic.connection import Connection from marklogic.cli.template import Template @@ -57,7 +58,7 @@ def run(self, argv): optarg = False elif tok.startswith("-"): options.append(tok) - if tok != "--debug": + if tok != "--debug" and tok != "--https": optarg = True elif "=" in tok: params.append(tok) @@ -152,9 +153,16 @@ def run(self, argv): mgmt_port = args['hostname'].split(":")[1] except IndexError: mgmt_port = 8002 - self.connection = Connection(host, - HTTPDigestAuth(username, password), - management_port=mgmt_port) + + if args['https']: + self.connection = Connection(host, + HTTPBasicAuth(username, password), + protocol="https", + management_port=mgmt_port) + else: + self.connection = Connection(host, + HTTPDigestAuth(username, password), + management_port=mgmt_port) # do it! if command == 'run': From cf2865b80d784404099fee05a6c38d1b133368a1 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Fri, 24 Mar 2017 10:09:15 -0500 Subject: [PATCH 08/23] On start/stop/restart attempt to wait until operation finishes --- marklogic/__init__.py | 2 +- marklogic/cli/manager/marklogic.py | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/marklogic/__init__.py b/marklogic/__init__.py index 617dfca..cb133a3 100644 --- a/marklogic/__init__.py +++ b/marklogic/__init__.py @@ -36,7 +36,7 @@ from marklogic.models.server import OdbcServer, XdbcServer from marklogic.exceptions import InvalidAPIRequest, UnexpectedManagementAPIResponse -__version__ = "0.0.15" +__version__ = "0.0.16" class MarkLogic: """ diff --git a/marklogic/cli/manager/marklogic.py b/marklogic/cli/manager/marklogic.py index a163fb2..455af67 100644 --- a/marklogic/cli/manager/marklogic.py +++ b/marklogic/cli/manager/marklogic.py @@ -152,7 +152,11 @@ def restart(self, args, config, connection): cluster = LocalCluster(connection=connection).read() print("Restarting cluster...") cluster.restart() - + # Make sure it's back up + status = self.status(args,config,connection,internal=True) + while status != 'up': + time.sleep(2) + status = self.status(args,config,connection,internal=True) else: hostname = connection.host if hostname == 'localhost': @@ -160,6 +164,11 @@ def restart(self, args, config, connection): host = Host(hostname,connection=connection).read() print("Restarting host...") host.restart() + # Make sure it's back up + status = self.status(args,config,connection,internal=True) + while status != 'up': + time.sleep(2) + status = self.status(args,config,connection,internal=True) def stop(self, args, config, connection): status = self.status(args, config, connection, internal=True) @@ -174,6 +183,11 @@ def stop(self, args, config, connection): cluster = LocalCluster(connection=connection).read() print("Shutting down cluster...") cluster.shutdown() + # Make sure it's all the way down + status = self.status(args,config,connection,internal=True) + while status != 'down': + time.sleep(2) + status = self.status(args,config,connection,internal=True) else: hostname = connection.host if hostname == 'localhost': @@ -190,6 +204,11 @@ def stop(self, args, config, connection): print("Shutting down host: " + host.host_name()) host.shutdown() + # Make sure it's all the way down + status = self.status(args,config,connection,internal=True) + while status != 'down': + time.sleep(2) + status = self.status(args,config,connection,internal=True) status = self.status(args,config,connection,internal=True) while status == 'up': From 4eaba7fa3b31d92a1d02ae03da787832103b53ee Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 2 May 2018 07:48:40 -0500 Subject: [PATCH 09/23] Support reading credentials from config file --- examples/mldbmirror.py | 47 +++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/examples/mldbmirror.py b/examples/mldbmirror.py index fe38c64..131cdce 100644 --- a/examples/mldbmirror.py +++ b/examples/mldbmirror.py @@ -67,6 +67,27 @@ def connect(self, args): self.path = os.path.abspath(args['path']) self.loadconfig(self.path) + if args['hostname'] is None: + if 'host' in self.config: + self.hostname = self.config['host'] + if 'port' in self.config: + self.port = self.config['port'] + else: + self.port = 8000 + if 'management-port' in self.config: + self.management_port = self.config['management-port'] + else: + self.management_port = 8002 + else: + parts = args['hostname'].split(":") + self.hostname = parts.pop(0) + self.management_port = 8002 + self.port = 8000 + if parts: + self.management_port = parts.pop(0) + if parts: + self.port = parts.pop(0) + if args['credentials'] is not None: cred = args['credentials'] else: @@ -74,6 +95,11 @@ def connect(self, args): cred = self.config['user'] + ":" + self.config['pass'] else: cred = None + key = self.hostname + ":" + str(self.management_port) + if key in self.config: + obj = self.config[key] + if 'user' in obj and 'pass' in obj: + cred = obj['user'] + ":" + obj['pass'] try: adminuser, adminpass = re.split(":", cred) @@ -102,27 +128,6 @@ def connect(self, args): if self.root.endswith("/"): self.root = self.root[0:len(self.root)-1] - if args['hostname'] is None: - if 'host' in self.config: - self.hostname = self.config['host'] - if 'port' in self.config: - self.port = self.config['port'] - else: - self.port = 8000 - if 'management-port' in self.config: - self.management_port = self.config['management-port'] - else: - self.management_port = 8002 - else: - parts = args['hostname'].split(":") - self.hostname = parts.pop(0) - self.management_port = 8002 - self.port = 8000 - if parts: - self.management_port = parts.pop(0) - if parts: - self.port = parts.pop(0) - self.connection \ = Connection(self.hostname, HTTPDigestAuth(adminuser, adminpass), \ port=self.port, management_port=self.management_port) From 4286eee939129e5d873f80101cbbaa37e1035677 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 2 May 2018 09:34:30 -0500 Subject: [PATCH 10/23] Script to read all properties --- examples/read-everything.py | 111 ++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 examples/read-everything.py diff --git a/examples/read-everything.py b/examples/read-everything.py new file mode 100644 index 0000000..dfb490b --- /dev/null +++ b/examples/read-everything.py @@ -0,0 +1,111 @@ +#!/usr/bin/python3 +# +# Copyright 2015 MarkLogic Corporation +# +# This script attempts to read all of the resource types on the cluster. +# The point of this script is to make sure that we catch any new properties +# that have been added by the server. + +__author__ = 'ndw' + +import argparse +import logging +import json +import logging +import sys +from requests.auth import HTTPDigestAuth +from marklogic.connection import Connection +from marklogic.models.cluster import LocalCluster +from marklogic.models.group import Group +from marklogic.models.host import Host +from marklogic.models.database import Database +from marklogic.models.permission import Permission +from marklogic.models.privilege import Privilege +from marklogic.models.role import Role +from marklogic.models.forest import Forest +from marklogic.models.server import Server +from marklogic.models.user import User + +class ReadEverything: + def __init__(self, connection): + self.databases = {} + self.forests = {} + self.servers = {} + self.users = {} + self.roles = {} + self.privileges = {} + self.connection = connection + pass + + def readClass(self, kind, klass, max_read=sys.maxsize): + names = klass.list(self.connection) + for name in names: + if max_read > 0: + if name.find("|") > 0: + parts = name.split("|") + rsrc = klass.lookup(self.connection, parts[0], parts[1]) + else: + rsrc = klass.lookup(self.connection, name) + max_read = max_read - 1 + print("{}: {}".format(kind, len(names))) + + def readPrivileges(self): + names = Privilege.list(self.connection) + max_read = { "execute": 5, "uri": 5 } + counts = { "execute": 0, "uri": 0 } + for name in names: + parts = name.split("|") + kind = parts[0] + pname = parts[1] + + counts[kind] = counts[kind] + 1 + + if max_read[kind] > 0: + rsrc = Privilege.lookup(self.connection, pname, kind) + max_read[kind] = max_read[kind] - 1 + + print("Execute privileges: {}".format(counts["execute"])) + print("URI privileges: {}".format(counts["uri"])) + + def read(self): + conn = self.connection + cluster = LocalCluster(connection=conn).read() + print("Read local cluster: {}".format(cluster.cluster_name())) + + self.readClass("Groups", Group, max_read=5) + self.readClass("Hosts", Host, max_read=5) + self.readClass("Databases", Database, max_read=5) + self.readClass("Forests", Forest, max_read=5) + self.readClass("Servers", Server) + self.readClass("Roles", Role, max_read=5) + self.readClass("Users", User, max_read=5) + self.readPrivileges() + + return + +logging.basicConfig(level=logging.INFO) + +parser = argparse.ArgumentParser() +parser.add_argument("--host", action='store', default="localhost", + help="Management API host") +parser.add_argument("--username", action='store', default="admin", + help="User name") +parser.add_argument("--password", action='store', default="admin", + help="Password") +parser.add_argument('--debug', action='store_true', + help='Enable debug logging') +args = parser.parse_args() + +if args.debug: + logging.basicConfig(level=logging.WARNING) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("marklogic").setLevel(logging.DEBUG) + +conn = Connection(args.host, HTTPDigestAuth(args.username, args.password)) +read_everything = ReadEverything(conn) + +print("Reading all resources from {}".format(args.host)) + +read_everything.read() + +print("Finished") From cf2d4ab75a1b10e86fd9676debbb9211b6575065 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 2 May 2018 09:34:46 -0500 Subject: [PATCH 11/23] Test groups --- tests/test_group.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/test_group.py diff --git a/tests/test_group.py b/tests/test_group.py new file mode 100644 index 0000000..31850f3 --- /dev/null +++ b/tests/test_group.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2015 MarkLogic Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0# +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# File History +# ------------ +# +# Paul Hoehne 03/26/2015 Initial development +# Norman Walsh 05/02/2018 Adapted from test_host.py +# + +from mlconfig import MLConfig +from marklogic.models.group import Group + +class TestGroup(MLConfig): + def group_list(self): + return Group.list(self.connection) + + def test_list_groups(self): + groups = self.group_list() + assert len(groups) > 0 + assert groups + + def test_load(self): + group_name = self.group_list()[0] + group = Group(group_name) + assert group.read(self.connection) is not None + assert group.host_timeout() > 0 + From 95f628b008511fc7a1b7deff4adb07950c00b131 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 2 May 2018 09:35:20 -0500 Subject: [PATCH 12/23] Support new V9 xdqp group properties --- marklogic/models/group/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/marklogic/models/group/__init__.py b/marklogic/models/group/__init__.py index 1c02a7c..623103e 100644 --- a/marklogic/models/group/__init__.py +++ b/marklogic/models/group/__init__.py @@ -154,7 +154,9 @@ def unmarshal(cls, config, 'opsdirector-log-level', 'opsdirector-metering', 'opsdirector-session-endpoint', 'telemetry-config', 'telemetry-log-level', 'telemetry-metering', - 'telemetry-session-endpoint' + 'telemetry-session-endpoint', + 'xdqp-ssl-disable-sslv3', 'xdqp-ssl-disable-tlsv1', + 'xdqp-ssl-disable-tlsv1-1', 'xdqp-ssl-disable-tlsv1-2' } for key in result._config: From 6a90f4ccbff1770c1535146ed43a18d2449fd486 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 2 May 2018 09:35:43 -0500 Subject: [PATCH 13/23] Ingore .pytest_cache directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a5103ff..e66520c 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ htmlcov/ nosetests.xml coverage.xml *,cover +.pytest_cache/ # Translations *.mo From 260cd085a02db4bfb6a8f8829d209f7a8902734f Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 2 May 2018 09:37:00 -0500 Subject: [PATCH 14/23] Bump version number to 0.0.17 --- marklogic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marklogic/__init__.py b/marklogic/__init__.py index cb133a3..ea44c57 100644 --- a/marklogic/__init__.py +++ b/marklogic/__init__.py @@ -36,7 +36,7 @@ from marklogic.models.server import OdbcServer, XdbcServer from marklogic.exceptions import InvalidAPIRequest, UnexpectedManagementAPIResponse -__version__ = "0.0.16" +__version__ = "0.0.17" class MarkLogic: """ From 8e14e04b802b0f9c3b24cac0a76a4dd7af801468 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 13 Mar 2019 08:42:09 -0500 Subject: [PATCH 15/23] New example script --- examples/init-server.py | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 examples/init-server.py diff --git a/examples/init-server.py b/examples/init-server.py new file mode 100644 index 0000000..0c9b5ac --- /dev/null +++ b/examples/init-server.py @@ -0,0 +1,43 @@ +#!/usr/bin/python3 +# +# Copyright 2018 MarkLogic Corporation +# + +__author__ = 'ndw' + +import argparse +import logging +import json +import logging +from marklogic import MarkLogic + +class InitServer: + def __init__(self): + pass + +#logging.basicConfig(level=logging.INFO) + +parser = argparse.ArgumentParser() +parser.add_argument("--host", action='store', default="localhost", + help="Management API host") +parser.add_argument("--username", action='store', default="admin", + help="User name") +parser.add_argument("--password", action='store', default="admin", + help="Password") +parser.add_argument("--wallet", action='store', default="admin", + help="Wallet password") +parser.add_argument('--debug', action='store_true', + help='Enable debug logging') +args = parser.parse_args() + +if args.debug: + logging.basicConfig(level=logging.WARNING) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("marklogic").setLevel(logging.DEBUG) + +print("Initialize host {}".format(args.host)) +MarkLogic.instance_init(args.host) +print("Initialize admin {}".format(args.host)) +MarkLogic.instance_admin(args.host, "public", args.username, args.password, args.wallet) + +print("finished") From c4a4ab8f2e9642005099d7819060acc152787359 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 13 Mar 2019 08:42:51 -0500 Subject: [PATCH 16/23] Notice and skip docs with unsavable names --- examples/mldbmirror.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/mldbmirror.py b/examples/mldbmirror.py index 131cdce..6bc79f0 100644 --- a/examples/mldbmirror.py +++ b/examples/mldbmirror.py @@ -491,15 +491,17 @@ def _download_directory(self, trans): down_map = {} skip_list = [] for uri in uris: - if not self.can_store_on_filesystem(uri): - raise RuntimeError("Cannot save URI:", uri) - localfile = self.path + uri skip = False - if uri in stamps and os.path.exists(localfile): - statinfo = os.stat(localfile) - stamp = self._convert_timestamp(stamps[uri]) - skip = statinfo.st_mtime >= stamp.timestamp() + + if not self.can_store_on_filesystem(uri): + print("Skipping " + uri + ": cannot store on filesystem") + skip = True + else: + if uri in stamps and os.path.exists(localfile): + statinfo = os.stat(localfile) + stamp = self._convert_timestamp(stamps[uri]) + skip = statinfo.st_mtime >= stamp.timestamp() if skip: skip_list.append(localfile) @@ -714,7 +716,7 @@ def can_store_on_filesystem(self, filename): filesystem because if it's ever uploaded, it'll get a leading /. """ if (not filename.startswith("/")) or ("//" in filename) \ - or (":" in filename) or ('"' in filename) or ('"' in filename) \ + or (":" in filename) or ('"' in filename) or ("'" in filename) \ or ("\\" in filename): return False else: From 4fc4303da29af6bfc7e1598a6ac56595cb11136c Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 13 Mar 2019 08:43:09 -0500 Subject: [PATCH 17/23] Bump dependencies --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9ee37a6..a37b3a5 100644 --- a/setup.py +++ b/setup.py @@ -25,8 +25,8 @@ def read(*filenames, **kwargs): long_description=read('README.rst'), packages=find_packages(), install_requires=[ - 'requests>=2.8.0', - 'requests_toolbelt>=0.6.0' + 'requests>=2.21.0', + 'requests_toolbelt>=0.9.1' ], include_package_data=True, platforms='any', From a82ea1c414e757f61a96071e6a91e77a61a8b483 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 13 Mar 2019 08:43:33 -0500 Subject: [PATCH 18/23] Add support for wallet password --- marklogic/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/marklogic/__init__.py b/marklogic/__init__.py index ea44c57..740a1ec 100644 --- a/marklogic/__init__.py +++ b/marklogic/__init__.py @@ -446,7 +446,7 @@ def instance_init(cls, host): return Host(host)._set_just_initialized() @classmethod - def instance_admin(cls,host,realm,admin,password): + def instance_admin(cls,host,realm,admin,password,wallet_password=None): """ Initializes the security database of a newly initialized server. @@ -463,6 +463,9 @@ def instance_admin(cls,host,realm,admin,password): 'realm': realm } + if wallet_password is not None: + payload["wallet-password"] = wallet_password + uri = "{0}://{1}:8001/admin/v1/instance-admin".format( conn.protocol, conn.host) From 23c09bc72f0e795d59835308cf29d1a1d20eecd5 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 13 Mar 2019 08:43:52 -0500 Subject: [PATCH 19/23] Use logger.warning instead of logger.warn --- marklogic/models/database/__init__.py | 2 +- marklogic/models/forest/__init__.py | 2 +- marklogic/models/group/__init__.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/marklogic/models/database/__init__.py b/marklogic/models/database/__init__.py index d97d003..689dddd 100644 --- a/marklogic/models/database/__init__.py +++ b/marklogic/models/database/__init__.py @@ -3755,7 +3755,7 @@ def unmarshal(cls, config, hostname=None, olist.append(temp) result._config['range-path-index'] = olist else: - logger.warn("Unexpected database property: " + key) + logger.warning("Unexpected database property: " + key) return result diff --git a/marklogic/models/forest/__init__.py b/marklogic/models/forest/__init__.py index f2c1513..f21c6b3 100644 --- a/marklogic/models/forest/__init__.py +++ b/marklogic/models/forest/__init__.py @@ -445,7 +445,7 @@ def unmarshal(cls, config, connection=None, save_connection=True): olist.append(temp) result._config['forest-replica'] = olist else: - logger.warn("Unexpected forest property: " + key) + logger.warning("Unexpected forest property: " + key) return result diff --git a/marklogic/models/group/__init__.py b/marklogic/models/group/__init__.py index 623103e..f215e7e 100644 --- a/marklogic/models/group/__init__.py +++ b/marklogic/models/group/__init__.py @@ -190,7 +190,7 @@ def unmarshal(cls, config, r['audit-restriction-items']) restrictions.append(rest) else: - logger.warn("Unexpected audit property: " + prop) + logger.warning("Unexpected audit property: " + prop) audit = Audit(enabled, keep, rotate, events, restrictions) result._config[key] = audit elif key == 'event': @@ -202,7 +202,7 @@ def unmarshal(cls, config, schemas.append(schema) result._config[key] = schemas else: - logger.warn("Unexpected group property: " + key) + logger.warning("Unexpected group property: " + key) return result From dfc6a2f1f3e8b8eb529f189f2c3ca83d73f75071 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 13 Mar 2019 09:04:12 -0500 Subject: [PATCH 20/23] Nudge python version --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4af490d..7accaa2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python python: -- '3.4' +- '3.7' sudo: true before_install: - pip install requests From 6141e94662f45f69927e443eef87469a3ba5f8bc Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 13 Mar 2019 09:04:21 -0500 Subject: [PATCH 21/23] Fix MarkLogic URI --- shared/dev-tasks/travis-install-ml.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/dev-tasks/travis-install-ml.sh b/shared/dev-tasks/travis-install-ml.sh index b008590..be5f35b 100755 --- a/shared/dev-tasks/travis-install-ml.sh +++ b/shared/dev-tasks/travis-install-ml.sh @@ -49,7 +49,7 @@ else # if the user passed a day string as a param then use it instead test $1 && day=$1 # make a version number out of the date - ver="8.0-$day" + ver="9.0-$day" echo "********* Downloading MarkLogic nightly $ver" From 5d086b3787791f73c0293773d9883d72bba43c67 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 13 Mar 2019 09:08:10 -0500 Subject: [PATCH 22/23] Apparently 3.7 isn't available at Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7accaa2..4af490d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python python: -- '3.7' +- '3.4' sudo: true before_install: - pip install requests From d49b73727a6cfa2bb1c135b1dc581e23f2184e25 Mon Sep 17 00:00:00 2001 From: Norman Walsh Date: Wed, 13 Mar 2019 09:20:08 -0500 Subject: [PATCH 23/23] Attempt to fix ML install --- .travis.yml | 6 ++---- shared/dev-tasks/travis-install-ml.sh | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4af490d..3a94266 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,10 +7,8 @@ before_install: - echo 'America/Los_Angeles' | sudo tee /etc/timezone - sudo dpkg-reconfigure --frontend noninteractive tzdata install: -- if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then ./shared/dev-tasks/travis-install-ml.sh - release ; else (exit 0) ; fi -- if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then ./shared/dev-tasks/setup-marklogic.sh - ; else (exit 0) ; fi +- if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then ./shared/dev-tasks/travis-install-ml.sh ; else (exit 0) ; fi +- if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then ./shared/dev-tasks/setup-marklogic.sh ; else (exit 0) ; fi script: - python setup.py test env: diff --git a/shared/dev-tasks/travis-install-ml.sh b/shared/dev-tasks/travis-install-ml.sh index be5f35b..bd3fee0 100755 --- a/shared/dev-tasks/travis-install-ml.sh +++ b/shared/dev-tasks/travis-install-ml.sh @@ -60,7 +60,7 @@ else suff="_amd64.deb" fnamedeb=$fnamedeb$suff - url="https://root.marklogic.com/nightly/builds/linux64/rh6-intel64-80-test-1.marklogic.com/b8_0/pkgs.$day/$fname" + url="https://root.marklogic.com/nightly/builds/linux64-rh7/rh7v-intel64-90-test-build.marklogic.com/b9_0/pkgs.20190313/$fname" status=$(curl -k --anyauth -u $MLBUILD_USER:$MLBUILD_PASSWORD --head --write-out %{http_code} --silent --output /dev/null $url) if [[ $status = 200 ]]; then