From 79a648648ee9266eb2f4e82cb3305183d73e7f9d Mon Sep 17 00:00:00 2001 From: Tymoteusz Jankowski Date: Thu, 26 Sep 2013 11:01:18 +0200 Subject: [PATCH 1/7] added choose data importer --- choose_importer.py | 117 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 choose_importer.py diff --git a/choose_importer.py b/choose_importer.py new file mode 100644 index 0000000..260d597 --- /dev/null +++ b/choose_importer.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# coding: utf-8 + +import logging + +from urlparse import urljoin + +import requests + +from django.contrib.auth import get_user_model + +from choose.compliance import models + +logging.basicConfig(level=logging.DEBUG) + +logger = logging.getLogger(__name__) + +''' +one-time script for importing Services and Countries to choose + +usage: + put it in choos-e dir + type in user and password keys in config variable + import and run main function from choos-e shell (command line:choose shell) +''' + + +config = { + 'user': , + 'password': , +} + + +def main(): + #data = get_data() + s = requests.Session() + base_url = 'http://jiraf.office:8080' + rest_path = '/rest/api/2/search?maxResults=9999&jql=project%20%3D%20%22INT%22%20AND%20issuetype%20%3D%20%22International%20service%22%20ORDER%20BY%20summary%20ASC' + rest_url = urljoin(base_url, rest_path) + + resp = s.get(rest_url, params={ + 'os_authType': 'basic', + 'os_username': config['user'], + 'os_password': config['password'], + 'start-index': 0, + }) + resp.raise_for_status() + rest_data = resp.json() + errors, success = 0, 0 + total = len(rest_data['issues']) + paths = { + 'country_name': 'fields,customfield_10004,value', + 'country_code': 'fields,customfield_10004,value', + 'company_name': 'fields,customfield_10012,value', + 'business_line_string': 'fields,customfield_10013,value', + 'city_name': 'fields,customfield_10004,child,value', + 'service_name': 'fields,customfield_10003', + } + for issue in rest_data['issues']: + data = {} + for key, path in paths.items(): + value = issue + logger.debug(u'processing path: {}'.format(path)) + for step in path.split(','): + try: + value = value[step] + except Exception as e: + logger.debug(u'missed step: {}'.format(step)) + value = 'empty' + continue + logger.debug(u'got value: {}'.format(value)) + data[key] = value + try: + logger.debug('fill with data: {}'.format(data)) + fill_db(**data) + success += 1 + except Exception as e: + errors += 1 + logger.debug(u'inserting failed: {}:{}'.format( + data['country_code'], data['service_name']) + ) + logger.error(e) + logger.info(u'failed inserts: {}'.format(errors)) + logger.info(u'successful inserts: {}'.format(success)) + logger.info(u'total: {}'.format(total)) + + +def fill_db(country_name, country_code, company_name, business_line_string, + city_name, service_name): + country = models.Country.objects.get_or_create( + name=country_name, code=country_code)[0] + + company = models.Company.objects.get_or_create(name=company_name)[0] + user = get_user_model().objects.all()[0] + business_line = models.BusinessLine.objects.get_or_create( + name=business_line_string + )[0] + city = models.City.objects.get_or_create( + name=city_name, country=country)[0] + it_location = models.ITLocation.objects.get_or_create(location=city)[0] + service_data = { + 'name': service_name, + #country + 'company': company, + 'contact': user, + 'business_line': business_line, + 'it_location': it_location, + 'slug': '', + } + logger.debug('service_data: {}'.format(service_data)) + + service = models.Service.objects.get_or_create(**service_data)[0] + service.countries.add(country) + service.save() + +if __name__ == '__main__': + main() From 9c99c318da6d3b7cc9af4cf0ae266ddbe5438c01 Mon Sep 17 00:00:00 2001 From: Tymoteusz Jankowski Date: Mon, 30 Sep 2013 11:00:47 +0200 Subject: [PATCH 2/7] added script deleting keys from basilisk db --- _clean_basilisk_db.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 _clean_basilisk_db.py diff --git a/_clean_basilisk_db.py b/_clean_basilisk_db.py new file mode 100644 index 0000000..438887e --- /dev/null +++ b/_clean_basilisk_db.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# coding: utf-8 + +import logging + +import redis + +from dashboard import settings + +""" +usage: + copy file to dashboard dir: + basilisk/dashboard + run: + %(env) python _clean_db +""" + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +def main(): + redis_conf = settings.REDIS + redis_conf = dict(settings.REDIS) + redis_conf.update({'db': redis_conf['selected_db']}) + del redis_conf['selected_db'] + logger.info('redis connection params: {}'.format(redis_conf)) + client = redis.StrictRedis(**redis_conf) + keys_path = 'dashboard:tile:*' + logger.info('itering path: {}'.format(keys_path)) + for key in client.keys(keys_path): + client.delete(key) + logger.info('deleted key: {}'.format(key)) + +if __name__ == '__main__': + main() From 0ebb4ac19614bb43cae63178da5b57a71f870e0a Mon Sep 17 00:00:00 2001 From: Tymoteusz Jankowski Date: Mon, 30 Sep 2013 11:42:28 +0200 Subject: [PATCH 3/7] add quick commented getter line --- _clean_basilisk_db.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/_clean_basilisk_db.py b/_clean_basilisk_db.py index 438887e..a577837 100644 --- a/_clean_basilisk_db.py +++ b/_clean_basilisk_db.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # coding: utf-8 +import json import logging import redis @@ -28,8 +29,11 @@ def main(): keys_path = 'dashboard:tile:*' logger.info('itering path: {}'.format(keys_path)) for key in client.keys(keys_path): + # mv to sep. fn. *list* or something + #logger.info(json.loads(client.get(key))) client.delete(key) logger.info('deleted key: {}'.format(key)) + if __name__ == '__main__': main() From 5c54e11f218e5bd65dee5e86b15da36469c4ffcd Mon Sep 17 00:00:00 2001 From: Michal Jastrzebski Date: Wed, 2 Oct 2013 15:15:50 +0200 Subject: [PATCH 4/7] shrink db --- shrink_db.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 shrink_db.py diff --git a/shrink_db.py b/shrink_db.py new file mode 100644 index 0000000..14ee472 --- /dev/null +++ b/shrink_db.py @@ -0,0 +1,50 @@ +import sys, os + +source_name = sys.argv[1] + +cmd = ('bzegrep -i -v "' +'^INSERT INTO \`activitylog_[a-z]|' +'^INSERT INTO \`celery_taskmeta|' +'^INSERT INTO \`celery_tasksetmeta|' +'^INSERT INTO \`cmdb_archivedcichangecmdbhistory|' +'^INSERT INTO \`cmdb_archivedcichangegit|' +'^INSERT INTO \`cmdb_archivedcichangepuppet|' +'^INSERT INTO \`cmdb_archivedcichangezabbixtrigger|' +'^INSERT INTO \`cmdb_archivedcichange|' +'^INSERT INTO \`cmdb_archivedpuppetlog|' +'^INSERT INTO \`cmdb_cichangecmdbhistory|' +'^INSERT INTO \`cmdb_cichangegit|' +'^INSERT INTO \`cmdb_cichangepuppet|' +'^INSERT INTO \`cmdb_cichangezabbixtrigger|' +'^INSERT INTO \`cmdb_cichange|' +'^INSERT INTO \`cmdb_ciincident|' +'^INSERT INTO \`cmdb_jirachanges|' +'^INSERT INTO \`cmdb_puppetlog|' +'^INSERT INTO \`discovery_discoverywarning|' +#'^INSERT INTO \`discovery_historycost|' +#'^INSERT INTO \`discovery_historychange|' +#'^INSERT INTO \`discovery_software|' +'^INSERT INTO \`discovery_splunkusage|' +'^INSERT INTO \`django_admin_log|' +'^INSERT INTO \`django_session|' +'^INSERT INTO \`resource_tags|' +'^INSERT INTO \`djcelery_[a-z]|' +'^INSERT INTO \`dnsedit_dnshistory' +#'^INSERT INTO \`ralph_assets_.*|' +#'^INSERT INTO \`ralph_pricing_.*|/' +#'^INSERT INTO \`fact_names|/' +#'^INSERT INTO \`fact_values|/' +#'^INSERT INTO \`fact_values_copy|/' +#'^INSERT INTO \`hosts|/' +#'^INSERT INTO \`inventory_facts|/' +#'^INSERT INTO \`inventory_nodes|/' +#'^INSERT INTO \`param_names|/' +#'^INSERT INTO \`param_values|/' +#'^INSERT INTO \`puppet_tags|/' +#'^INSERT INTO \`resource_tags|/' +#'^INSERT INTO \`resources|/' +#'^INSERT INTO \`source_files +'" {} > dd.sql').format(source_name) + +print cmd +# os.system(cmd) \ No newline at end of file From aeb0eb5cfbe3ced930bf2220db55fcac346d002b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szymon=20Py=C5=BCalski?= Date: Thu, 10 Oct 2013 13:44:35 +0200 Subject: [PATCH 5/7] Corrected import script --- asset_update_from_csv.py | 42 ---------------- entry_points.ini | 2 + ralph_scripts/__init__.py | 0 ralph_scripts/asset_update_from_csv.py | 50 +++++++++++++++++++ .../autoci_performance_tests.py | 0 .../device_list_with_profit_center.py | 0 .../ralph_share_history.py | 0 ...vice_prices_per_venture_all_data_to_csv.py | 0 .../reports_device_prices_per_venture.py | 0 .../reports_ventures.py | 0 setup.py | 17 +++++++ 11 files changed, 69 insertions(+), 42 deletions(-) delete mode 100644 asset_update_from_csv.py create mode 100644 entry_points.ini create mode 100644 ralph_scripts/__init__.py create mode 100644 ralph_scripts/asset_update_from_csv.py rename autoci_performance_tests.py => ralph_scripts/autoci_performance_tests.py (100%) rename device_list_with_profit_center.py => ralph_scripts/device_list_with_profit_center.py (100%) rename ralph_share_history.py => ralph_scripts/ralph_share_history.py (100%) rename report_device_prices_per_venture_all_data_to_csv.py => ralph_scripts/report_device_prices_per_venture_all_data_to_csv.py (100%) rename reports_device_prices_per_venture.py => ralph_scripts/reports_device_prices_per_venture.py (100%) rename reports_ventures.py => ralph_scripts/reports_ventures.py (100%) create mode 100644 setup.py diff --git a/asset_update_from_csv.py b/asset_update_from_csv.py deleted file mode 100644 index f2c4720..0000000 --- a/asset_update_from_csv.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import csv - -from ralph_assets.models_assets import Asset - -def csv_to_dicts(filename): - with open(filename, 'rU') as file: - reader = csv.reader(file) - data = [] - for i, row in enumerate(reader): - if i == 0: - fields = row[0].split(';') - else: - field = {} - for key, item in enumerate(row[0].split(';')): - field[fields[key]] = item - data.append(field) - return data, fields - - -def asset_update(filename): - csv_data, db_fields = csv_to_dicts(filename) - for item in csv_data: - try: - asset = Asset.objects.get(id=item.get('id')) - except Asset.DoesNotExist: - print(item.get('id')) - continue - for field in db_fields: - old_value = getattr(asset, field) - new_value = item.get(field) - if not old_value == new_value: - setattr(asset, field, new_value) - print(asset.id, field, old_value, new_value) - asset.save() diff --git a/entry_points.ini b/entry_points.ini new file mode 100644 index 0000000..91bad5a --- /dev/null +++ b/entry_points.ini @@ -0,0 +1,2 @@ +[console_scripts] +asset_update_from_csv = ralph_scripts.asset_update_from_csv:asset_update_from_csv diff --git a/ralph_scripts/__init__.py b/ralph_scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ralph_scripts/asset_update_from_csv.py b/ralph_scripts/asset_update_from_csv.py new file mode 100644 index 0000000..55b79e5 --- /dev/null +++ b/ralph_scripts/asset_update_from_csv.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import csv +import sys +from collections import namedtuple + +from ralph_assets.models_assets import Asset + +def csv_to_tuples(filename): + with open(filename, 'rU') as file: + reader = csv.reader(file, delimiter=b',', quotechar=b'"') + Row = namedtuple('Row', next(reader)) + for i, row in enumerate(reader): + yield Row._make(row) + + +def asset_update(filename): + for item in csv_to_tuples(filename): + id = item.id + try: + asset = Asset.objects.get(id=item.id) + except Asset.DoesNotExist: + print('Asset missing: {0}'.format(item.id)) + continue + for field in item._fields: + if field == 'id': + continue + old_value = getattr(asset, field) + new_value = getattr(item, field) + if not old_value == new_value: + setattr(asset, field, new_value) + print( + ('Asset {id} field {field} changed from' + '{old_value} to {new_value}.').format(**locals())) + asset.save() + +def asset_update_from_csv(): + if len(sys.argv) != 2: + print('USAGE: asset_update_from_csv filename.csv') + else: + asset_update(sys.argv[1]) + +if __name__ == '__main__': + asset_update_from_csv() diff --git a/autoci_performance_tests.py b/ralph_scripts/autoci_performance_tests.py similarity index 100% rename from autoci_performance_tests.py rename to ralph_scripts/autoci_performance_tests.py diff --git a/device_list_with_profit_center.py b/ralph_scripts/device_list_with_profit_center.py similarity index 100% rename from device_list_with_profit_center.py rename to ralph_scripts/device_list_with_profit_center.py diff --git a/ralph_share_history.py b/ralph_scripts/ralph_share_history.py similarity index 100% rename from ralph_share_history.py rename to ralph_scripts/ralph_share_history.py diff --git a/report_device_prices_per_venture_all_data_to_csv.py b/ralph_scripts/report_device_prices_per_venture_all_data_to_csv.py similarity index 100% rename from report_device_prices_per_venture_all_data_to_csv.py rename to ralph_scripts/report_device_prices_per_venture_all_data_to_csv.py diff --git a/reports_device_prices_per_venture.py b/ralph_scripts/reports_device_prices_per_venture.py similarity index 100% rename from reports_device_prices_per_venture.py rename to ralph_scripts/reports_device_prices_per_venture.py diff --git a/reports_ventures.py b/ralph_scripts/reports_ventures.py similarity index 100% rename from reports_ventures.py rename to ralph_scripts/reports_ventures.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..3dda843 --- /dev/null +++ b/setup.py @@ -0,0 +1,17 @@ +# -*- encoding: utf-8 -*- + +from setuptools import setup, find_packages + +with open('entry_points.ini') as f: + entry_points = f.read() + +setup ( + name = 'ralph_scripts', + description = 'Useful scripts for ralph', + author = 'Grupa Allegro Sp. z o.o. and Contributors', + author_email = 'it-ralph-dev@allegro.pl', + install_requires = [ + 'ralph>=1.2.6', + ], + entry_points = entry_points, +) From 5dd3e294ac386ba15e6f9c30bee285beb1241533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szymon=20Py=C5=BCalski?= Date: Thu, 10 Oct 2013 13:59:41 +0200 Subject: [PATCH 6/7] Moved scripts from stash to package --- _clean_basilisk_db.py => ralph_scripts/_clean_basilisk_db.py | 0 choose_importer.py => ralph_scripts/choose_importer.py | 0 shrink_db.py => ralph_scripts/shrink_db.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename _clean_basilisk_db.py => ralph_scripts/_clean_basilisk_db.py (100%) rename choose_importer.py => ralph_scripts/choose_importer.py (100%) rename shrink_db.py => ralph_scripts/shrink_db.py (100%) diff --git a/_clean_basilisk_db.py b/ralph_scripts/_clean_basilisk_db.py similarity index 100% rename from _clean_basilisk_db.py rename to ralph_scripts/_clean_basilisk_db.py diff --git a/choose_importer.py b/ralph_scripts/choose_importer.py similarity index 100% rename from choose_importer.py rename to ralph_scripts/choose_importer.py diff --git a/shrink_db.py b/ralph_scripts/shrink_db.py similarity index 100% rename from shrink_db.py rename to ralph_scripts/shrink_db.py From 0301e46392813d32b0a0b17ed17b005f6a7044cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szymon=20Py=C5=BCalski?= Date: Thu, 10 Oct 2013 14:02:26 +0200 Subject: [PATCH 7/7] Renamed the module --- entry_points.ini | 2 +- {ralph_scripts => pylabs_tools}/__init__.py | 0 {ralph_scripts => pylabs_tools}/_clean_basilisk_db.py | 0 {ralph_scripts => pylabs_tools}/asset_update_from_csv.py | 0 .../autoci_performance_tests.py | 0 {ralph_scripts => pylabs_tools}/choose_importer.py | 0 .../device_list_with_profit_center.py | 0 {ralph_scripts => pylabs_tools}/ralph_share_history.py | 0 .../report_device_prices_per_venture_all_data_to_csv.py | 0 .../reports_device_prices_per_venture.py | 0 {ralph_scripts => pylabs_tools}/reports_ventures.py | 0 {ralph_scripts => pylabs_tools}/shrink_db.py | 0 setup.py | 8 +++----- 13 files changed, 4 insertions(+), 6 deletions(-) rename {ralph_scripts => pylabs_tools}/__init__.py (100%) rename {ralph_scripts => pylabs_tools}/_clean_basilisk_db.py (100%) rename {ralph_scripts => pylabs_tools}/asset_update_from_csv.py (100%) rename {ralph_scripts => pylabs_tools}/autoci_performance_tests.py (100%) rename {ralph_scripts => pylabs_tools}/choose_importer.py (100%) rename {ralph_scripts => pylabs_tools}/device_list_with_profit_center.py (100%) rename {ralph_scripts => pylabs_tools}/ralph_share_history.py (100%) rename {ralph_scripts => pylabs_tools}/report_device_prices_per_venture_all_data_to_csv.py (100%) rename {ralph_scripts => pylabs_tools}/reports_device_prices_per_venture.py (100%) rename {ralph_scripts => pylabs_tools}/reports_ventures.py (100%) rename {ralph_scripts => pylabs_tools}/shrink_db.py (100%) diff --git a/entry_points.ini b/entry_points.ini index 91bad5a..682e781 100644 --- a/entry_points.ini +++ b/entry_points.ini @@ -1,2 +1,2 @@ [console_scripts] -asset_update_from_csv = ralph_scripts.asset_update_from_csv:asset_update_from_csv +asset_update_from_csv = pylabs_tools.asset_update_from_csv:asset_update_from_csv diff --git a/ralph_scripts/__init__.py b/pylabs_tools/__init__.py similarity index 100% rename from ralph_scripts/__init__.py rename to pylabs_tools/__init__.py diff --git a/ralph_scripts/_clean_basilisk_db.py b/pylabs_tools/_clean_basilisk_db.py similarity index 100% rename from ralph_scripts/_clean_basilisk_db.py rename to pylabs_tools/_clean_basilisk_db.py diff --git a/ralph_scripts/asset_update_from_csv.py b/pylabs_tools/asset_update_from_csv.py similarity index 100% rename from ralph_scripts/asset_update_from_csv.py rename to pylabs_tools/asset_update_from_csv.py diff --git a/ralph_scripts/autoci_performance_tests.py b/pylabs_tools/autoci_performance_tests.py similarity index 100% rename from ralph_scripts/autoci_performance_tests.py rename to pylabs_tools/autoci_performance_tests.py diff --git a/ralph_scripts/choose_importer.py b/pylabs_tools/choose_importer.py similarity index 100% rename from ralph_scripts/choose_importer.py rename to pylabs_tools/choose_importer.py diff --git a/ralph_scripts/device_list_with_profit_center.py b/pylabs_tools/device_list_with_profit_center.py similarity index 100% rename from ralph_scripts/device_list_with_profit_center.py rename to pylabs_tools/device_list_with_profit_center.py diff --git a/ralph_scripts/ralph_share_history.py b/pylabs_tools/ralph_share_history.py similarity index 100% rename from ralph_scripts/ralph_share_history.py rename to pylabs_tools/ralph_share_history.py diff --git a/ralph_scripts/report_device_prices_per_venture_all_data_to_csv.py b/pylabs_tools/report_device_prices_per_venture_all_data_to_csv.py similarity index 100% rename from ralph_scripts/report_device_prices_per_venture_all_data_to_csv.py rename to pylabs_tools/report_device_prices_per_venture_all_data_to_csv.py diff --git a/ralph_scripts/reports_device_prices_per_venture.py b/pylabs_tools/reports_device_prices_per_venture.py similarity index 100% rename from ralph_scripts/reports_device_prices_per_venture.py rename to pylabs_tools/reports_device_prices_per_venture.py diff --git a/ralph_scripts/reports_ventures.py b/pylabs_tools/reports_ventures.py similarity index 100% rename from ralph_scripts/reports_ventures.py rename to pylabs_tools/reports_ventures.py diff --git a/ralph_scripts/shrink_db.py b/pylabs_tools/shrink_db.py similarity index 100% rename from ralph_scripts/shrink_db.py rename to pylabs_tools/shrink_db.py diff --git a/setup.py b/setup.py index 3dda843..fc06398 100644 --- a/setup.py +++ b/setup.py @@ -6,12 +6,10 @@ entry_points = f.read() setup ( - name = 'ralph_scripts', - description = 'Useful scripts for ralph', + name = 'pylabs_tools', + description = 'Useful scripts', author = 'Grupa Allegro Sp. z o.o. and Contributors', author_email = 'it-ralph-dev@allegro.pl', - install_requires = [ - 'ralph>=1.2.6', - ], entry_points = entry_points, + packages = find_packages(), )