From 238babe6616749511a6691f67a19b47423947b5c Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Tue, 12 Feb 2019 17:11:30 +0200 Subject: [PATCH 01/13] Move api.chronograph planning functionality to databridge --- .../auction/bridge_utils/__init__.py | 0 .../auction/bridge_utils/constants.py | 38 +++ .../auction/bridge_utils/design.py | 43 ++++ .../auction/bridge_utils/managers.py | 236 ++++++++++++++++++ openprocurement/auction/bridge_utils/utils.py | 190 ++++++++++++++ openprocurement/auction/databridge.py | 30 ++- openprocurement/auction/plannings.py | 37 ++- 7 files changed, 572 insertions(+), 2 deletions(-) create mode 100644 openprocurement/auction/bridge_utils/__init__.py create mode 100644 openprocurement/auction/bridge_utils/constants.py create mode 100644 openprocurement/auction/bridge_utils/design.py create mode 100644 openprocurement/auction/bridge_utils/managers.py create mode 100644 openprocurement/auction/bridge_utils/utils.py diff --git a/openprocurement/auction/bridge_utils/__init__.py b/openprocurement/auction/bridge_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/openprocurement/auction/bridge_utils/constants.py b/openprocurement/auction/bridge_utils/constants.py new file mode 100644 index 0000000..9e93dc4 --- /dev/null +++ b/openprocurement/auction/bridge_utils/constants.py @@ -0,0 +1,38 @@ +import os +from datetime import time +from pytz import timezone + +from datetime import timedelta, datetime + + +STREAMS_ID = 'streams' +WORKING_DAY_START = time(11, 0) +INSIDER_WORKING_DAY_START = time(9, 30) +TEXAS_WORKING_DAY_START = time(10, 0) +WORKING_DAY_END = time(16, 0) +INSIDER_WORKING_DAY_DURATION = timedelta(minutes=480) +TEXAS_WORKING_DAY_DURATION = timedelta(hours=7) # Does not affect anything +DEFAULT_STREAMS_DOC = { + '_id': STREAMS_ID, + 'streams': 10, + 'dutch_streams': 15, + 'texas_streams': 20 +} + + +CALENDAR_ID = 'calendar' +STREAMS_ID = 'streams' +ROUNDING = timedelta(minutes=29) +MIN_PAUSE = timedelta(minutes=3) +BIDDER_TIME = timedelta(minutes=6) +SERVICE_TIME = timedelta(minutes=9) +STAND_STILL_TIME = timedelta(days=1) +SMOOTHING_MIN = 10 +SMOOTHING_REMIN = 60 +# value should be greater than SMOOTHING_MIN and SMOOTHING_REMIN +SMOOTHING_MAX = 300 +NOT_CLASSIC_AUCTIONS = ['dgfInsider', 'sellout.insider'] +STREAMS_KEYS = ['streams', 'dutch_streams', 'texas_streams'] + + +TZ = timezone(os.environ['TZ'] if 'TZ' in os.environ else 'Europe/Kiev') diff --git a/openprocurement/auction/bridge_utils/design.py b/openprocurement/auction/bridge_utils/design.py new file mode 100644 index 0000000..44b18e6 --- /dev/null +++ b/openprocurement/auction/bridge_utils/design.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +from couchdb.design import ViewDefinition +from openprocurement.auction.bridge_utils.constants import ( + INSIDER_WORKING_DAY_START, TEXAS_WORKING_DAY_START +) + + +def sync_design(db): + views = [j for i, j in globals().items() if "_view" in i] + ViewDefinition.sync_many(db, views) + + +plan_auctions_view = ViewDefinition('plan', 'auctions', '''function(doc) { + if(doc.streams || doc.dutch_streams || doc.texas_streams) { + for (var i in doc) { + if (i.indexOf('stream_') == 0) { + for (var t in doc[i]) { + if (doc[i][t]) { + var x = doc[i][t].split('_') + if (x.length == 2) { + emit(x, doc._id.split('_')[1] + 'T' + t); + } else { + emit([x[0], null], + doc._id.split('_')[1] + 'T' + t); + } + } + } + } + if (i.indexOf('dutch_streams') == 0) { + for (var aid in doc[i]) { + emit([doc[i][aid], null], + doc._id.split('_')[1] + 'T' + '%s'); + } + } + if (i.indexOf('texas_streams') == 0) { + for (var aid in doc[i]) { + emit([doc[i][aid], null], + doc._id.split('_')[1] + 'T' + '%s'); + } + } + }; + } +}''' % (INSIDER_WORKING_DAY_START.isoformat(), TEXAS_WORKING_DAY_START.isoformat())) diff --git a/openprocurement/auction/bridge_utils/managers.py b/openprocurement/auction/bridge_utils/managers.py new file mode 100644 index 0000000..2b26855 --- /dev/null +++ b/openprocurement/auction/bridge_utils/managers.py @@ -0,0 +1,236 @@ +# -*- coding: utf-8 -*- +import logging +import requests +from copy import deepcopy + +from couchdb import ResourceConflict +from datetime import timedelta, datetime +from iso8601 import parse_date + +from gevent.pool import Pool + +from openprocurement.auction.bridge_utils.constants import ( + DEFAULT_STREAMS_DOC, + TEXAS_WORKING_DAY_DURATION, + TEXAS_WORKING_DAY_START, + INSIDER_WORKING_DAY_DURATION, + INSIDER_WORKING_DAY_START, + WORKING_DAY_START, + WORKING_DAY_END, + STREAMS_ID, + TZ +) +from openprocurement.auction.bridge_utils.utils import find_free_slot + +POOL = Pool(1) + +LOGGER = logging.getLogger('Openprocurement Auction') + +ADAPTER = requests.adapters.HTTPAdapter(pool_connections=3, pool_maxsize=3) +SESSION = requests.Session() +SESSION.mount('http://', ADAPTER) +SESSION.mount('https://', ADAPTER) + + +class BaseAuctionsManager(object): + working_day_start = None + streams_key = None + + def get_date(self, db, mode, date): + """ + + Get hour of auction start and stream in which auction will run for + passed date. + + :param db: chronograph database with plan docs + :type db: couchdb.Database + :param mode: value from auction procedure, indicates mode of auction, used in plan doc id + :type mode: str + :param date: preplanned date for current auction + :type date: datetime.date + + :returns : (time of auction start, stream for auction, plan document) + :rtype: (datetime.time, int, couchdb.client.Document) + """ + plan_id = 'plan{}_{}'.format(mode, date.isoformat()) + plan = db.get(plan_id, {'_id': plan_id}) + plan_date_end, stream = self._get_hours_and_stream(plan) + plan_date = parse_date(date.isoformat() + 'T' + plan_date_end, None) + plan_date = plan_date.astimezone(TZ) if plan_date.tzinfo else TZ.localize(plan_date) + return plan_date.time(), stream, plan + + def get_streams(self, db, streams_id=STREAMS_ID): + """ + + Get allowed amount of streams for auction of particular type per day. + + :param db: chronograph database + :type db: couchdb.Database + :param streams_id: id of document with defined stream amounts + :type streams_id: str + + :return: amount of streams for auction per day + :rtype: int + """ + + streams = db.get(streams_id, deepcopy(DEFAULT_STREAMS_DOC)) + return streams.get(self.streams_key, DEFAULT_STREAMS_DOC[self.streams_key]) + + def _get_hours_and_stream(self, plan): + """ + Return time of auction start and first available stream from plan doc + + :param plan: document for planned auctions + :type plan: couchdb.client.Document + + :return: (time of auction start, first available stream) + :rtype: (str, int) + """ + raise NotImplementedError + + def set_end_of_auction(self, *args, **kwargs): + """ + Try to find end of auction for passed date, time and plan doc if such + action is possible and return it's value for further auction planning + """ + raise NotImplementedError + + def set_date(self, *args, **kwargs): + """ + Actually plan auction for particular date, time, stream and slot and + save it to chronograph database. + """ + raise NotImplementedError + + def free_slot(self, *args, **kwargs): + """ + Remove particular auction from stream slot in particular plan document. + """ + raise NotImplementedError + + +class ClassicAuctionsManager(BaseAuctionsManager): + working_day_start = WORKING_DAY_START + working_day_end = WORKING_DAY_END + streams_key = 'streams' + + def _get_hours_and_stream(self, plan): + plan_date_end = plan.get('time', self.working_day_start.isoformat()) + stream = plan.get(self.streams_key, 1) + return plan_date_end, stream + + def set_date(self, db, plan, auction_id, end_time, + cur_stream, start_time, new_slot=True): + if new_slot: + plan['time'] = end_time.isoformat() + plan[self.streams_key] = cur_stream + stream_id = 'stream_{}'.format(cur_stream) + stream = plan.get(stream_id, {}) + stream[start_time.isoformat()] = auction_id + plan[stream_id] = stream + db.save(plan) + + def free_slot(self, db, plan_id, auction_id, plan_time): + slot = plan_time.time().isoformat() + done = False + while not done: + try: + plan = db.get(plan_id) + streams = plan[self.streams_key] + for cur_stream in range(1, streams + 1): + stream_id = 'stream_{}'.format(cur_stream) + if plan[stream_id].get(slot) == auction_id: + plan[stream_id][slot] = None + db.save(plan) + done = True + except ResourceConflict: + done = False + except: + done = True + + def set_end_of_auction(self, stream, streams, nextDate, dayStart, plan): + freeSlot = find_free_slot(plan) + if freeSlot: + startDate, stream = freeSlot + start, end, dayStart, new_slot = startDate, startDate, startDate.time(), False + return start, end, dayStart, stream, new_slot + if dayStart >= self.working_day_end and stream < streams: + stream += 1 + dayStart = self.working_day_start + + start = TZ.localize(datetime.combine(nextDate, dayStart)) + end = start + timedelta(minutes=30) + + # end = calc_auction_end_time(auction.get('numberOfBids', len(auction.get('bids', []))), start) + + # TODO: redundant check, which was used with previous end calculation logic: + # if dayStart == self.working_day_start and end > TZ.localize( + # datetime.combine(nextDate, self.working_day_end) + # ) and stream <= streams: + # return start, end, dayStart, stream, True + + if end <= TZ.localize(datetime.combine(nextDate, self.working_day_end)) and stream <= streams: + return start, end, dayStart, stream, True + + +class NonClassicAuctionsManager(BaseAuctionsManager): + working_day_start = None + working_day_duration = None + streams_key = None + + def _get_hours_and_stream(self, plan): + plan_date_end = self.working_day_start.isoformat() + stream = len(plan.get(self.streams_key, [])) + return plan_date_end, stream + + def set_date(self, db, plan, auction_id, *args, **kwargs): + streams = plan.get(self.streams_key, []) + streams.append(auction_id) + plan[self.streams_key] = streams + db.save(plan) + + def free_slot(self, db, plan_id, auction_id, *args, **kwargs): + done = False + while not done: + try: + plan = db.get(plan_id) + slots = plan.get(self.streams_key, []) + pops = [] + for i in xrange(0, len(slots)): + if slots[i] == auction_id: + pops.append(i) + pops.sort(reverse=True) + for p in pops: + slots.pop(p) + plan[self.streams_key] = slots + db.save(plan) + done = True + except ResourceConflict: + done = False + except: + done = True + + def set_end_of_auction(self, stream, streams, nextDate, dayStart, *args, **kwargs): + if stream < streams: + start = TZ.localize(datetime.combine(nextDate, dayStart)) + end = start + self.working_day_duration + return start, end, dayStart, stream, False + + +class TexasAuctionsManager(NonClassicAuctionsManager): + working_day_start = TEXAS_WORKING_DAY_START + working_day_duration = TEXAS_WORKING_DAY_DURATION + streams_key = 'texas_streams' + + +class InsiderAuctionsManager(NonClassicAuctionsManager): + working_day_start = INSIDER_WORKING_DAY_START + working_day_duration = INSIDER_WORKING_DAY_DURATION + streams_key = 'dutch_streams' + + +MANAGERS_MAPPING = { + 'texas': TexasAuctionsManager, + 'dutch': InsiderAuctionsManager, + 'english': ClassicAuctionsManager +} diff --git a/openprocurement/auction/bridge_utils/utils.py b/openprocurement/auction/bridge_utils/utils.py new file mode 100644 index 0000000..ee23f68 --- /dev/null +++ b/openprocurement/auction/bridge_utils/utils.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +import os +import logging +import requests +from random import randint + +from couchdb import ResourceConflict +from datetime import timedelta, datetime +from iso8601 import parse_date + + +from openprocurement.auction.bridge_utils.constants import ( + TZ, + BIDDER_TIME, + SERVICE_TIME, + MIN_PAUSE, + ROUNDING, + WORKING_DAY_START, + CALENDAR_ID +) + + +LOGGER = logging.getLogger('Openprocurement Auction') + +ADAPTER = requests.adapters.HTTPAdapter(pool_connections=3, pool_maxsize=3) +SESSION = requests.Session() +SESSION.mount('http://', ADAPTER) +SESSION.mount('https://', ADAPTER) + + +def find_free_slot(plan): + streams = plan.get('streams', 0) + for cur_stream in range(1, streams + 1): + stream_id = 'stream_{}'.format(cur_stream) + for slot in plan[stream_id]: + if plan[stream_id].get(slot) is None: + plan_date = parse_date(plan['_id'].split('_')[1] + 'T' + slot, None) + plan_date = plan_date.astimezone(TZ) if plan_date.tzinfo else TZ.localize(plan_date) + return plan_date, cur_stream + + +def update_logging_context(request, params): + if not request.__dict__.get('logging_context'): + request.logging_context = {} + + for x, j in params.items(): + request.logging_context[x.upper()] = j + + +def context_unpack(request, msg, params=None): + if params: + update_logging_context(request, params) + logging_context = request.logging_context + journal_context = msg + for key, value in logging_context.items(): + journal_context["JOURNAL_" + key] = value + return journal_context + + +def get_now(): + return TZ.localize(datetime.now()) + + +def randomize(dt): + return dt + timedelta(seconds=randint(0, 1799)) + + +def get_manager_for_auction(auction, mapper): + default_manager = mapper['types'].get('english', None) + + auction_type = auction.get('auctionParameters', {}).get('type', None) + if auction_type: + return mapper['types'].get(auction_type, default_manager) + else: + pmt = auction.get('procurementMethodType') + return mapper['pmts'].get(pmt, default_manager) + + +def calc_auction_end_time(bids, start): + end = start + bids * BIDDER_TIME + SERVICE_TIME + MIN_PAUSE + seconds = (end - TZ.localize(datetime.combine(end, WORKING_DAY_START))).seconds + roundTo = ROUNDING.seconds + rounding = (seconds + roundTo - 1) // roundTo * roundTo + return (end + timedelta(0, rounding - seconds, -end.microsecond)).astimezone(TZ) + + +def get_calendar(db, calendar_id=CALENDAR_ID): + return db.get(calendar_id, {'_id': calendar_id}) + + +def skipped_days(days): + days_str = '' + if days: + days_str = ' Skipped {} full days.'.format(days) + return days_str + + +def planning_auction(auction, mapper, start, db, quick=False, lot_id=None): + # Need to move all + tid = auction.get('id', '') + mode = auction.get('mode', '') + manager = get_manager_for_auction(auction, mapper) + skipped_days = 0 + if quick: + quick_start = calc_auction_end_time(0, start) + return quick_start, 0, skipped_days + calendar = get_calendar(db) + streams = manager.get_streams(db) + start += timedelta(hours=1) + if start.time() > manager.working_day_start: + nextDate = start.date() + timedelta(days=1) + else: + nextDate = start.date() + while True: + # skip Saturday and Sunday + if calendar.get(nextDate.isoformat()) or nextDate.weekday() in [5, 6]: + nextDate += timedelta(days=1) + continue + dayStart, stream, plan = manager.get_date(db, mode, nextDate) + result = manager.set_end_of_auction(stream, streams, nextDate, dayStart, plan) + if result: + start, end, dayStart, stream, new_slot = result + break + nextDate += timedelta(days=1) + skipped_days += 1 + manager.set_date(db, plan, "_".join([tid, lot_id]) if lot_id else tid, end.time(), stream, dayStart, new_slot) + return start, stream, skipped_days + + +def check_auction(auction, db, mapper): + # Need to move all + now = get_now() + quick = os.environ.get('SANDBOX_MODE', False) and u'quick' in auction.get('submissionMethodDetails', '') + if not auction.get('lots') and 'shouldStartAfter' in auction.get('auctionPeriod', {}) and auction['auctionPeriod']['shouldStartAfter'] > auction['auctionPeriod'].get('startDate'): + period = auction.get('auctionPeriod') + shouldStartAfter = max(parse_date(period.get('shouldStartAfter'), TZ).astimezone(TZ), now) + planned = False + while not planned: + try: + auctionPeriod, stream, skip_days = planning_auction(auction, mapper, shouldStartAfter, db, quick) + planned = True + except ResourceConflict: + planned = False + auctionPeriod = randomize(auctionPeriod).isoformat() + planned = 'replanned' if period.get('startDate') else 'planned' + LOGGER.info( + '{} auction for auction {} to {}. Stream {}.{}'.format( + planned.title(), auction['id'], auctionPeriod, stream, skipped_days(skip_days) + ), + extra={ + 'MESSAGE_ID': '{}_auction_auction'.format(planned), + 'PLANNED_DATE': auctionPeriod, + 'PLANNED_STREAM': stream, + 'PLANNED_DAYS_SKIPPED': skip_days} + ) + return {'auctionPeriod': {'startDate': auctionPeriod}} + elif auction.get('lots'): + lots = [] + for lot in auction.get('lots', []): + if lot['status'] != 'active' or 'shouldStartAfter' not in lot.get('auctionPeriod', {}) or lot['auctionPeriod']['shouldStartAfter'] < lot['auctionPeriod'].get('startDate'): + lots.append({}) + continue + period = lot.get('auctionPeriod') + shouldStartAfter = max(parse_date(period.get('shouldStartAfter'), TZ).astimezone(TZ), now) + lot_id = lot['id'] + planned = False + while not planned: + try: + auctionPeriod, stream, skip_days = planning_auction(auction, mapper, shouldStartAfter, db, quick, lot_id) + planned = True + except ResourceConflict: + planned = False + auctionPeriod = randomize(auctionPeriod).isoformat() + planned = 'replanned' if period.get('startDate') else 'planned' + lots.append({'auctionPeriod': {'startDate': auctionPeriod}}) + LOGGER.info( + '{} auction for lot {} of auction {} to {}. Stream {}.{}'.format( + planned.title(), lot_id, auction['id'], auctionPeriod, stream, skipped_days(skip_days) + ), + extra={ + 'MESSAGE_ID': '{}_auction_lot'.format(planned), + 'PLANNED_DATE': auctionPeriod, + 'PLANNED_STREAM': stream, + 'PLANNED_DAYS_SKIPPED': skip_days, + 'LOT_ID': lot_id + } + ) + if any(lots): + return {'lots': lots} + return None diff --git a/openprocurement/auction/databridge.py b/openprocurement/auction/databridge.py index 2339a73..c9cc71e 100644 --- a/openprocurement/auction/databridge.py +++ b/openprocurement/auction/databridge.py @@ -14,11 +14,14 @@ from urlparse import urljoin from zope.interface import implementer from yaml import load -from couchdb import Database, Session +from couchdb import Database, Session, Server from dateutil.tz import tzlocal from openprocurement_client.sync import ResourceFeeder from openprocurement.auction.interfaces import\ IAuctionDatabridge, IAuctionsManager +from openprocurement.auction.bridge_utils.design import sync_design +from openprocurement.auction.bridge_utils.managers import MANAGERS_MAPPING + from openprocurement.auction.core import components from openprocurement.auction.utils import FeedItem, check_workers @@ -67,6 +70,30 @@ def __init__(self, config, re_planning=False, debug=False): retrievers_params=DEFAULT_RETRIEVERS_PARAMS ) + # Stream DB configurations + db_name = os.environ.get('DB_NAME', self.config['main']['stream_db']) + couch_server = Server(self.config_get('couch_url'), session=Session(retry_delays=range(60))) + + if db_name not in couch_server: + couch_server.create(db_name) + + db_for_streams = urljoin( + self.config_get('couch_url'), + db_name + ) + + self.stream_db = Database(db_for_streams, session=Session(retry_delays=range(10))) + + sync_design(self.stream_db) + + # Managers Mapping + self.manager_mapper = {'types': {}, 'pmts': {}} + for name, plugin in self.config_get('plugins').items(): + auction_manager = MANAGERS_MAPPING[name]() + self.manager_mapper['types'][name] = auction_manager + if plugin.get('procurement_method_types', []): + self.manager_mapper['pmts'].update({pmt: auction_manager for pmt in plugin.get('procurement_method_types')}) + def config_get(self, name): return self.config.get('main').get(name) @@ -81,6 +108,7 @@ def run(self): planning = self.mapper(feed) if not planning: continue + planning.add_auction_period() for cmd, item_id, lot_id in planning: if lot_id: LOGGER.info('Lot {} of tender {} selected for {}'.format( diff --git a/openprocurement/auction/plannings.py b/openprocurement/auction/plannings.py index 1b9a5dc..9321787 100644 --- a/openprocurement/auction/plannings.py +++ b/openprocurement/auction/plannings.py @@ -1,7 +1,9 @@ import logging import iso8601 +from json import dumps +from urlparse import urljoin -from datetime import datetime +from datetime import datetime, timedelta, time from time import mktime, time from gevent.subprocess import check_call @@ -21,6 +23,9 @@ from openprocurement.auction.utils import ( do_until_success, prepare_auction_worker_cmd ) +from openprocurement.auction.bridge_utils.utils import check_auction, context_unpack, SESSION +from openprocurement.auction.bridge_utils.managers import MANAGERS_MAPPING + SIMPLE_AUCTION_TYPE = 0 @@ -38,6 +43,36 @@ def __init__(self, bridge, item): def next(self): return self + def add_auction_period(self): + url = urljoin( + self.bridge.config_get('resource_api_server'), + '/'.join( + [ + 'api', + self.bridge.config_get('resource_api_version'), + self.bridge.config_get('resource_name'), + self.item.id + ], + ) + ) + LOGGER.info(url) + api_token = self.bridge.config_get('resource_api_token') + db = self.bridge.stream_db + changes = check_auction(self.item, db, self.bridge.manager_mapper) + if changes: + data = dumps({'data': changes}) + r = SESSION.patch(url, + data=data, + headers={'Content-Type': 'application/json'}, + auth=(api_token, '')) + if r.status_code != 200: + LOGGER.error( + "Error {} on updating auction '{}' with '{}': {}".format(r.status_code, url, data, r.text), + extra=context_unpack(r, {'MESSAGE_ID': 'error_patch_auction'}, + {'ERROR_STATUS': r.status_code})) + else: + LOGGER.info("Successfully updated auction '{}' with '{}'".format(r.status_code, url, data)) + def __iter__(self): if self.item['status'] == "active.auction": if 'lots' not in self.item and 'auctionPeriod' in self.item \ From fc53ddf19f399ecfb46c643cdef7d168df5984c5 Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Fri, 15 Feb 2019 16:26:32 +0200 Subject: [PATCH 02/13] Move find_free_slots to managers.py --- .../auction/bridge_utils/managers.py | 20 ++++++++++--------- openprocurement/auction/bridge_utils/utils.py | 13 ------------ 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/openprocurement/auction/bridge_utils/managers.py b/openprocurement/auction/bridge_utils/managers.py index 2b26855..c6f6a5f 100644 --- a/openprocurement/auction/bridge_utils/managers.py +++ b/openprocurement/auction/bridge_utils/managers.py @@ -20,7 +20,6 @@ STREAMS_ID, TZ ) -from openprocurement.auction.bridge_utils.utils import find_free_slot POOL = Pool(1) @@ -32,6 +31,17 @@ SESSION.mount('https://', ADAPTER) +def find_free_slot(plan): + streams = plan.get('streams', 0) + for cur_stream in range(1, streams + 1): + stream_id = 'stream_{}'.format(cur_stream) + for slot in plan[stream_id]: + if plan[stream_id].get(slot) is None: + plan_date = parse_date(plan['_id'].split('_')[1] + 'T' + slot, None) + plan_date = plan_date.astimezone(TZ) if plan_date.tzinfo else TZ.localize(plan_date) + return plan_date, cur_stream + + class BaseAuctionsManager(object): working_day_start = None streams_key = None @@ -161,14 +171,6 @@ def set_end_of_auction(self, stream, streams, nextDate, dayStart, plan): start = TZ.localize(datetime.combine(nextDate, dayStart)) end = start + timedelta(minutes=30) - # end = calc_auction_end_time(auction.get('numberOfBids', len(auction.get('bids', []))), start) - - # TODO: redundant check, which was used with previous end calculation logic: - # if dayStart == self.working_day_start and end > TZ.localize( - # datetime.combine(nextDate, self.working_day_end) - # ) and stream <= streams: - # return start, end, dayStart, stream, True - if end <= TZ.localize(datetime.combine(nextDate, self.working_day_end)) and stream <= streams: return start, end, dayStart, stream, True diff --git a/openprocurement/auction/bridge_utils/utils.py b/openprocurement/auction/bridge_utils/utils.py index ee23f68..4bc0072 100644 --- a/openprocurement/auction/bridge_utils/utils.py +++ b/openprocurement/auction/bridge_utils/utils.py @@ -28,17 +28,6 @@ SESSION.mount('https://', ADAPTER) -def find_free_slot(plan): - streams = plan.get('streams', 0) - for cur_stream in range(1, streams + 1): - stream_id = 'stream_{}'.format(cur_stream) - for slot in plan[stream_id]: - if plan[stream_id].get(slot) is None: - plan_date = parse_date(plan['_id'].split('_')[1] + 'T' + slot, None) - plan_date = plan_date.astimezone(TZ) if plan_date.tzinfo else TZ.localize(plan_date) - return plan_date, cur_stream - - def update_logging_context(request, params): if not request.__dict__.get('logging_context'): request.logging_context = {} @@ -96,7 +85,6 @@ def skipped_days(days): def planning_auction(auction, mapper, start, db, quick=False, lot_id=None): - # Need to move all tid = auction.get('id', '') mode = auction.get('mode', '') manager = get_manager_for_auction(auction, mapper) @@ -128,7 +116,6 @@ def planning_auction(auction, mapper, start, db, quick=False, lot_id=None): def check_auction(auction, db, mapper): - # Need to move all now = get_now() quick = os.environ.get('SANDBOX_MODE', False) and u'quick' in auction.get('submissionMethodDetails', '') if not auction.get('lots') and 'shouldStartAfter' in auction.get('auctionPeriod', {}) and auction['auctionPeriod']['shouldStartAfter'] > auction['auctionPeriod'].get('startDate'): From fc760bb12574c324fcab578947d67283aff169fc Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Fri, 15 Feb 2019 16:27:24 +0200 Subject: [PATCH 03/13] Add holidays to calendar in couchdb --- .../auction/bridge_utils/constants.py | 14 ++++ .../auction/bridge_utils/working_days.json | 64 +++++++++++++++++++ openprocurement/auction/databridge.py | 16 +++-- 3 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 openprocurement/auction/bridge_utils/working_days.json diff --git a/openprocurement/auction/bridge_utils/constants.py b/openprocurement/auction/bridge_utils/constants.py index 9e93dc4..c320f04 100644 --- a/openprocurement/auction/bridge_utils/constants.py +++ b/openprocurement/auction/bridge_utils/constants.py @@ -5,6 +5,18 @@ from datetime import timedelta, datetime +def read_json(name): + import inspect + import os.path + from json import loads + caller_file = inspect.stack()[1][1] + caller_dir = os.path.dirname(os.path.realpath(caller_file)) + file_path = os.path.join(caller_dir, name) + with open(file_path) as lang_file: + data = lang_file.read() + return loads(data) + + STREAMS_ID = 'streams' WORKING_DAY_START = time(11, 0) INSIDER_WORKING_DAY_START = time(9, 30) @@ -36,3 +48,5 @@ TZ = timezone(os.environ['TZ'] if 'TZ' in os.environ else 'Europe/Kiev') +# True means holiday +WORKING_DAYS = read_json('working_days.json') diff --git a/openprocurement/auction/bridge_utils/working_days.json b/openprocurement/auction/bridge_utils/working_days.json new file mode 100644 index 0000000..654eee2 --- /dev/null +++ b/openprocurement/auction/bridge_utils/working_days.json @@ -0,0 +1,64 @@ +{ + "2015-05-01": true, + "2015-05-04": true, + "2015-05-11": true, + "2015-06-01": true, + "2015-06-29": true, + "2015-08-24": true, + "2015-10-14": true, + "2016-01-01": true, + "2016-01-07": true, + "2016-01-08": true, + "2016-03-07": true, + "2016-03-08": true, + "2016-05-02": true, + "2016-05-03": true, + "2016-05-09": true, + "2016-06-20": true, + "2016-06-27": true, + "2016-06-28": true, + "2016-08-24": true, + "2016-10-14": true, + "2017-01-02": true, + "2017-01-09": true, + "2017-03-08": true, + "2017-04-17": true, + "2017-05-01": true, + "2017-05-02": true, + "2017-05-08": true, + "2017-05-09": true, + "2017-06-05": true, + "2017-06-28": true, + "2017-08-24": true, + "2017-08-25": true, + "2017-10-16": true, + "2018-01-01": true, + "2018-01-08": true, + "2018-03-08": true, + "2018-04-09": true, + "2018-05-01": true, + "2018-05-08": true, + "2018-05-09": true, + "2018-05-28": true, + "2018-06-28": true, + "2018-06-29": true, + "2018-08-24": true, + "2018-10-15": true, + "2018-12-24": true, + "2018-12-25": true, + "2018-12-31": true, + "2019-01-01": true, + "2019-01-07": true, + "2019-03-08": true, + "2019-04-29": true, + "2019-04-30": true, + "2019-05-01": true, + "2019-05-09": true, + "2019-06-17": true, + "2019-06-28": true, + "2019-08-26": true, + "2019-10-14": true, + "2019-12-25": true, + "2019-12-30": true, + "2019-12-31": true +} diff --git a/openprocurement/auction/databridge.py b/openprocurement/auction/databridge.py index c9cc71e..5c61da0 100644 --- a/openprocurement/auction/databridge.py +++ b/openprocurement/auction/databridge.py @@ -1,8 +1,10 @@ from gevent import monkey + monkey.patch_all() try: import urllib3.contrib.pyopenssl + urllib3.contrib.pyopenssl.inject_into_urllib3() except ImportError: pass @@ -17,15 +19,15 @@ from couchdb import Database, Session, Server from dateutil.tz import tzlocal from openprocurement_client.sync import ResourceFeeder -from openprocurement.auction.interfaces import\ +from openprocurement.auction.interfaces import \ IAuctionDatabridge, IAuctionsManager from openprocurement.auction.bridge_utils.design import sync_design from openprocurement.auction.bridge_utils.managers import MANAGERS_MAPPING +from openprocurement.auction.bridge_utils.constants import WORKING_DAYS, CALENDAR_ID from openprocurement.auction.core import components from openprocurement.auction.utils import FeedItem, check_workers - LOGGER = logging.getLogger(__name__) API_EXTRA = {'opt_fields': 'status,auctionPeriod,lots,procurementMethodType', 'mode': '_all_'} @@ -41,7 +43,6 @@ @implementer(IAuctionDatabridge) class AuctionsDataBridge(object): - """Auctions Data Bridge""" def __init__(self, config, re_planning=False, debug=False): @@ -83,7 +84,7 @@ def __init__(self, config, re_planning=False, debug=False): ) self.stream_db = Database(db_for_streams, session=Session(retry_delays=range(10))) - + self._set_holidays() sync_design(self.stream_db) # Managers Mapping @@ -94,6 +95,13 @@ def __init__(self, config, re_planning=False, debug=False): if plugin.get('procurement_method_types', []): self.manager_mapper['pmts'].update({pmt: auction_manager for pmt in plugin.get('procurement_method_types')}) + def _set_holidays(self): + calendar = {'_id': CALENDAR_ID} + calendar.update(WORKING_DAYS) + if CALENDAR_ID in self.stream_db: + del self.stream_db[CALENDAR_ID] + self.stream_db.save(calendar) + def config_get(self, name): return self.config.get('main').get(name) From 23243cc26c01db2654f0e407228fc02e26470df5 Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Fri, 15 Feb 2019 17:08:18 +0200 Subject: [PATCH 04/13] Expand tests --- .../auction/tests/bridge_utils/__init__.py | 0 .../bridge_utils/auctions_data_bridge.yaml | 63 +++ .../auction/tests/bridge_utils/base.py | 34 ++ .../auction/tests/bridge_utils/data.py | 190 +++++++++ .../tests/bridge_utils/test_managers.py | 389 ++++++++++++++++++ .../auction/tests/bridge_utils/test_utils.py | 304 ++++++++++++++ 6 files changed, 980 insertions(+) create mode 100644 openprocurement/auction/tests/bridge_utils/__init__.py create mode 100644 openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml create mode 100644 openprocurement/auction/tests/bridge_utils/base.py create mode 100644 openprocurement/auction/tests/bridge_utils/data.py create mode 100644 openprocurement/auction/tests/bridge_utils/test_managers.py create mode 100644 openprocurement/auction/tests/bridge_utils/test_utils.py diff --git a/openprocurement/auction/tests/bridge_utils/__init__.py b/openprocurement/auction/tests/bridge_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml b/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml new file mode 100644 index 0000000..7c2c92b --- /dev/null +++ b/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml @@ -0,0 +1,63 @@ +main: + resource_api_server: http://127.0.0.1:6543 + resource_api_version: "2.3" + resource_api_token: auction + resource_name: auctions + couch_url: http://admin:zaq1xsw2@0.0.0.0:9000/ + auctions_db: database + timezone: Europe/Kiev + stream_db : test_stream_db + plugins: + english: + auction_worker_config: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/etc/auction_worker_defaults.yaml + procurement_method_types: [DGFOtherAssets, dgfFinancialAssets] + auction_worker: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/bin/auction_worker + dutch: + auction_worker_config: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/etc/auction_worker_insider.yaml + procurement_method_types: [dgfInsider] + auction_worker: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/bin/auction_insider + texas: + auction_worker_config: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/etc/auction_worker_texas.yaml + procurement_method_types: [landLease] + auction_worker: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/bin/auction_texas + retrievers_params: + down_requests_sleep: 0.1 + up_requests_sleep: 0.1 + up_wait_sleep: 30 + up_wait_sleep_min: 5 + queue_size: 501 +version: 1 +disable_existing_loggers: false + +formatters: + simple: + format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + +handlers: + console: + class: logging.StreamHandler + level: ERROR + formatter: simple + stream: ext://sys.stdout + + journal: + class: ExtendedJournalHandler.ExtendedJournalHandler + level: INFO + SYSLOG_IDENTIFIER: AUCTIONS_DATA_BRIDGE + TENDERS_API_VERSION: "0" + TENDERS_API_URL: https://lb.api-sandbox.ea.openprocurement.org/ + +loggers: + openprocurement.auction.databridge: + handlers: [console, journal] + propagate: no + level: INFO + + openprocurement_client.sync: + handlers: [console, journal] + propagate: no + level: INFO + + "": + handlers: [console, journal] + level: INFO \ No newline at end of file diff --git a/openprocurement/auction/tests/bridge_utils/base.py b/openprocurement/auction/tests/bridge_utils/base.py new file mode 100644 index 0000000..64815fd --- /dev/null +++ b/openprocurement/auction/tests/bridge_utils/base.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +import os +import unittest + +from yaml import load +from couchdb import Server + +from openprocurement.auction.databridge import AuctionsDataBridge + + +class BaseWebTest(unittest.TestCase): + """Base Web Test to test openprocurement.api. + + It setups the database before each test and delete it after. + """ + + def setUp(self): + dir_path = os.path.dirname(os.path.realpath(__file__)) + + with open(dir_path + '/auctions_data_bridge.yaml') as config_file_obj: + config = load(config_file_obj.read()) + + bridge = AuctionsDataBridge(config) + + self.couchdb_server = Server(config['main'].get('couch_url')) + self.db = bridge.stream_db + self.mapper = bridge.manager_mapper + + def tearDown(self): + try: + del self.couchdb_server[self.db.name] + except: + pass diff --git a/openprocurement/auction/tests/bridge_utils/data.py b/openprocurement/auction/tests/bridge_utils/data.py new file mode 100644 index 0000000..cf00f0f --- /dev/null +++ b/openprocurement/auction/tests/bridge_utils/data.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +from datetime import timedelta + +from openprocurement.auction.bridge_utils.utils import get_now + +now = get_now() + +test_auction_data = { + "auctionID": "UA-EA-{}-000001".format(now.strftime('%Y-%m-%d')), + "awardCriteria": "highestCost", + "date": now.isoformat(), + "dateModified": now.isoformat(), + "id": "f547ece35436484e8656a2988fb52a44", + "status": "active.enquiries", + "submissionMethod": "electronicAuction", + "title": u"футляри до державних нагород", + "procuringEntity": { + "name": u"Державне управління справами", + "identifier": { + "scheme": u"UA-EDR", + "id": u"00037256", + "uri": u"http://www.dus.gov.ua/" + }, + "address": { + "countryName": u"Україна", + "postalCode": u"01220", + "region": u"м. Київ", + "locality": u"м. Київ", + "streetAddress": u"вул. Банкова, 11, корпус 1" + }, + "contactPoint": { + "name": u"Державне управління справами", + "telephone": u"0440000000" + } + }, + "next_check": (now + timedelta(days=7)).isoformat(), + "owner": "test", + "procurementMethod": "open", + "value": { + "amount": 500, + "currency": u"UAH", + "valueAddedTaxIncluded": True + }, + "minimalStep": { + "amount": 35, + "currency": u"UAH", + "valueAddedTaxIncluded": True + }, + "items": [ + { + "description": u"футляри до державних нагород", + "classification": { + "scheme": u"CAV", + "id": u"70122000-2", + "description": u"Cartons" + }, + "additionalClassifications": [ + { + "scheme": u"ДКПП", + "id": u"17.21.1", + "description": u"папір і картон гофровані, паперова й картонна тара" + } + ], + "unit": { + "name": u"item", + "code": u"44617100-9" + }, + "quantity": 5, + "id": "181f08dbe3944325a1bb272e756f04cb" + + } + ], + "enquiryPeriod": { + "endDate": (now + timedelta(days=7)).isoformat(), + "startDate": now.isoformat() + }, + "tenderPeriod": { + "startDate": (now + timedelta(days=7)).isoformat(), + "endDate": (now + timedelta(days=14)).isoformat() + } +} + +test_lots = [ + { + "title": "lot title", + "description": "lot description", + "value": test_auction_data["value"], + "minimalStep": test_auction_data["minimalStep"], + "id": "20e0201d81af4aaeb33546f04744d493", + "status": "active" + } +] + +test_bids = [ + { + "id": "1d1caac9baa7445f90ca1ba06a1c605c", + "owner": "test", + "status": "active", + "tenderers": [ + test_auction_data["procuringEntity"] + ], + "value": { + "amount": 469, + "currency": "UAH", + "valueAddedTaxIncluded": True + } + }, + { + "id": "cc15a3d3ad204143abe1ea0dd69a108e", + "owner": "test", + "status": "active", + "tenderers": [ + test_auction_data["procuringEntity"] + ], + "value": { + "amount": 479, + "currency": "UAH", + "valueAddedTaxIncluded": True + } + } +] + +plantest = { + "_id": "plantest_2017-10-03", + "dutch_streams": [ + "01fa8a7dc4b8eac3b5820747efc6fe36", + "442000d99bf203ddfd62b3be58350383", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "da8a28ed2bdf73ee1d373e4cadfed4c5", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "442000d99bf203ddfd62b3be58350383", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "da8a28ed2bdf73ee1d373e4cadfed4c5", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "442000d99bf203ddfd62b3be58350383", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "da8a28ed2bdf73ee1d373e4cadfed4c5", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "01fa8a7dc4b8eac3b5820747efc6fe36" + ], + "texas_streams": [ + "01fa8a7dc4b8eac3b5820747efc6fe36", + "442000d99bf203ddfd62b3be58350383", + "dc3d950743304d05adaa1cd5b0475075", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "da8a28ed2bdf73ee1d373e4cadfed4c5", + "dc3d950743304d05adaa1cd5b0475075", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "442000d99bf203ddfd62b3be58350383", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "dc3d950743304d05adaa1cd5b0475075", + "da8a28ed2bdf73ee1d373e4cadfed4c5", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "442000d99bf203ddfd62b3be58350383", + "dc3d950743304d05adaa1cd5b0475075", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "da8a28ed2bdf73ee1d373e4cadfed4c5", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "dc3d950743304d05adaa1cd5b0475075", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "01fa8a7dc4b8eac3b5820747efc6fe36" + "dc3d950743304d05adaa1cd5b0475075" + ], + "stream_1": { + "12:00:00": "01fa8a7dc4b8eac3b5820747efc6fe36", + "12:30:00": "da8a28ed2bdf73ee1d373e4cadfed4c5_1c2fb1e496b317b2b87e197e2332da77", + "13:00:00": "e51508cddc2c490005eaecb73c006b72", + "13:30:00": "e51508cddc2c490005eaecb73c006b72", + "11:30:00": "e51508cddc2c490005eaecb73c006b72", + "11:00:00": "01fa8a7dc4b8eac3b5820747efc6fe36", + "15:30:00": "e51508cddc2c490005eaecb73c006b72", + "15:00:00": "da8a28ed2bdf73ee1d373e4cadfed4c5_1c2fb1e496b317b2b87e197e2332da77", + "14:00:00": "da8a28ed2bdf73ee1d373e4cadfed4c5_b10f9f7f26157ae2f349be8dc2106d6e", + "14:30:00": "01fa8a7dc4b8eac3b5820747efc6fe36" + }, + "stream_2": { + "12:00:00": "01fa8a7dc4b8eac3b5820747efc6fe36", + "12:30:00": "da8a28ed2bdf73ee1d373e4cadfed4c5_1c2fb1e496b317b2b87e197e2332da77", + "13:00:00": "e51508cddc2c490005eaecb73c006b72", + "13:30:00": "01fa8a7dc4b8eac3b5820747efc6fe36", + "11:30:00": "da8a28ed2bdf73ee1d373e4cadfed4c5_b10f9f7f26157ae2f349be8dc2106d6e", + "11:00:00": "e51508cddc2c490005eaecb73c006b72", + "15:30:00": "e51508cddc2c490005eaecb73c006b72", + "15:00:00": "01fa8a7dc4b8eac3b5820747efc6fe36", + "14:00:00": "da8a28ed2bdf73ee1d373e4cadfed4c5_1c2fb1e496b317b2b87e197e2332da77", + "14:30:00": "01fa8a7dc4b8eac3b5820747efc6fe36" + }, + "streams": 2 +} diff --git a/openprocurement/auction/tests/bridge_utils/test_managers.py b/openprocurement/auction/tests/bridge_utils/test_managers.py new file mode 100644 index 0000000..518dc4f --- /dev/null +++ b/openprocurement/auction/tests/bridge_utils/test_managers.py @@ -0,0 +1,389 @@ +# -*- coding: utf-8 -*- +import mock +import unittest + +from copy import deepcopy +from datetime import datetime, timedelta, time +from iso8601 import parse_date + +from openprocurement.auction.bridge_utils.constants import TZ +from openprocurement.auction.bridge_utils.managers import ( + find_free_slot, + ClassicAuctionsManager, + InsiderAuctionsManager, + TexasAuctionsManager +) +from openprocurement.auction.tests.bridge_utils.base import BaseWebTest +from openprocurement.auction.tests.bridge_utils.data import plantest + + +class FindFreeSlotTest(BaseWebTest): + + def test_find_free_slot(self): + plan = deepcopy(plantest) + plan['streams'] = 1 + plan['stream_1']['12:00:00'] = None + plan_date, cur_stream = find_free_slot(plan) + + expected_date = parse_date('2017-10-03T12:00:00', None) + expected_date = expected_date.astimezone(TZ) if expected_date.tzinfo else TZ.localize(expected_date) + + self.assertEqual(plan_date, expected_date) + self.assertEqual(cur_stream, 1) + + def test_no_slot_founded(self): + plan = deepcopy(plantest) + plan['streams'] = 1 + plan['stream_1']['12:00:00'] = None + result = find_free_slot(plantest) + self.assertIsNone(result) + + +class TestClassicManager(BaseWebTest): + + def setUp(self): + super(TestClassicManager, self).setUp() + self.manager = ClassicAuctionsManager() + + def test_get_date(self): + date = datetime.now().date() + plan_date = parse_date(date.isoformat() + 'T' + self.manager.working_day_start.isoformat(), None) + plan_date = plan_date.astimezone(TZ) if plan_date.tzinfo else TZ.localize(plan_date) + + db = mock.MagicMock() + + db.get.return_value = plantest + now = datetime.now() + + plan_time, stream, plan = self.manager.get_date(db, '', now.date()) + + self.assertEqual(plan, plantest) + self.assertEqual(plan_time, plan_date.time()) + self.assertEqual(stream, plantest['streams']) + + def test_get_hours_and_stream(self): + plan_date_end, streams = self.manager._get_hours_and_stream(plantest) + self.assertEqual(plan_date_end, self.manager.working_day_start.isoformat()) + self.assertEqual(streams, plantest['streams']) + + def test_set_date(self): + db = mock.MagicMock() + old_plan = {'time': 'old', 'streams': 3} + plan = deepcopy(old_plan) + auction_id = '1' * 32 + end_time = datetime.now() + cur_stream = 1 + start_time = datetime.now() + stream_id = 'stream_{}'.format(cur_stream) + + self.manager.set_date( + db, plan, auction_id, end_time, + cur_stream, start_time, False + ) + self.assertIn(stream_id, plan) + stream = plan[stream_id] + self.assertIn(start_time.isoformat(), stream) + self.assertEqual(stream[start_time.isoformat()], auction_id) + self.assertEqual(plan['time'], old_plan['time']) + self.assertEqual(plan['streams'], old_plan['streams']) + + db.save.assert_called_with(plan) + + def test_date_with_new_slot(self): + db = mock.MagicMock() + old_plan = {'time': 'old', 'streams': 3} + plan = deepcopy(old_plan) + auction_id = '1' * 32 + end_time = datetime.now() + timedelta(days=1) + cur_stream = 1 + start_time = datetime.now() + stream_id = 'stream_{}'.format(cur_stream) + + self.manager.set_date( + db, plan, auction_id, end_time, + cur_stream, start_time, True + ) + self.assertIn(stream_id, plan) + stream = plan[stream_id] + self.assertIn(start_time.isoformat(), stream) + self.assertEqual(stream[start_time.isoformat()], auction_id) + self.assertEqual(plan['time'], end_time.isoformat()) + self.assertEqual(plan['streams'], cur_stream) + + db.save.assert_called_with(plan) + + def test_free_slot(self): + db = mock.MagicMock() + auction_id = '1' * 32 + plan_time = datetime.now().replace(hour=12, minute=0, second=0, microsecond=0) + + plan = { + 'stream_1': { + plan_time.time().isoformat(): auction_id + }, + 'streams': 1 + } + plan_id = 'plan_id' + db.get.return_value = plan + + self.manager.free_slot(db, plan_id, auction_id, plan_time) + + self.assertIsNone(plan['stream_1'][plan_time.time().isoformat()]) + db.get.assert_called_with(plan_id) + db.save.assert_called_with(plan) + + def test_free_slot_for_wrong_auction_id(self): + db = mock.MagicMock() + auction_id = '1' * 32 + plan_time = datetime.now().replace(hour=12, minute=0, second=0, microsecond=0) + wrong_auction_id = '2' * 32 + + plan = { + 'stream_1': { + plan_time.time().isoformat(): auction_id + }, + 'streams': 1 + } + plan_id = 'plan_id' + db.get.return_value = plan + + self.manager.free_slot(db, plan_id, wrong_auction_id, plan_time) + + self.assertEqual(plan['stream_1'][plan_time.time().isoformat()], auction_id) + db.get.assert_called_with(plan_id) + db.save.assert_called_with(plan) + + def test_set_end_of_auction_with_free_slot(self): + stream = 1 + streams = 3 + next_date = datetime.now() + day_start = time(10, 0) + + plan_time = datetime.now().replace(hour=12, minute=0, second=0, microsecond=0) + plan = { + '_id': 'plantest_2017-10-03', + 'stream_1': { + plan_time.time().isoformat(): None + }, + 'stream_2': {}, + 'stream_3': {}, + 'streams': 3 + } + expected_date = parse_date('2017-10-03T12:00:00', None) + expected_date = expected_date.astimezone(TZ) if expected_date.tzinfo else TZ.localize(expected_date) + + start, end, auction_day_start, auction_stream, new_slot = self.manager.set_end_of_auction( + stream, streams, next_date, day_start, plan + ) + self.assertEqual(start, expected_date) + self.assertEqual(end, expected_date) + self.assertEqual(auction_day_start, expected_date.time()) + self.assertEqual(auction_stream, stream) + self.assertFalse(new_slot) + + def test_set_end_of_auction_without_free_slot(self): + stream = 1 + streams = 3 + next_date = datetime.now().date() + day_start = time(8, 0) + + plan = { + '_id': 'plantest_2017-10-03', + 'stream_1': {}, + 'stream_2': {}, + 'stream_3': {}, + 'streams': 3 + } + + start, end, auction_day_start, auction_stream, new_slot = self.manager.set_end_of_auction( + stream, streams, next_date, day_start, plan + ) + + expected_start = TZ.localize(datetime.combine(next_date, day_start)) + + self.assertEqual(start, expected_start) + self.assertEqual(end, expected_start + timedelta(minutes=30)) + self.assertEqual(auction_day_start, day_start) + self.assertEqual(auction_stream, stream) + self.assertTrue(new_slot) + + def test_set_end_of_auction_without_free_slot_and_wrong_day_start(self): + stream = 1 + streams = 3 + next_date = datetime.now().date() + day_start = time(16, 0) + + plan = { + '_id': 'plantest_2017-10-03', + 'stream_1': {}, + 'stream_2': {}, + 'stream_3': {}, + 'streams': 3 + } + + start, end, auction_day_start, auction_stream, new_slot = self.manager.set_end_of_auction( + stream, streams, next_date, day_start, plan + ) + + expected_start = TZ.localize(datetime.combine(next_date, self.manager.working_day_start)) + + self.assertEqual(start, expected_start) + self.assertEqual(end, expected_start + timedelta(minutes=30)) + self.assertEqual(auction_day_start, self.manager.working_day_start) + self.assertEqual(auction_stream, stream + 1) + self.assertTrue(new_slot) + + def test_set_end_of_auction_without_free_slot_and_limit_stream(self): + stream = 3 + streams = 3 + next_date = datetime.now().date() + day_start = time(8, 0) + + plan = { + '_id': 'plantest_2017-10-03', + 'stream_1': {}, + 'stream_2': {}, + 'stream_3': {}, + 'streams': 3 + } + + start, end, auction_day_start, auction_stream, new_slot = self.manager.set_end_of_auction( + stream, streams, next_date, day_start, plan + ) + + expected_start = TZ.localize(datetime.combine(next_date, day_start)) + + self.assertEqual(start, expected_start) + self.assertEqual(end, expected_start + timedelta(minutes=30)) + self.assertEqual(auction_day_start, day_start) + self.assertEqual(auction_stream, stream) + self.assertTrue(new_slot) + + def test_set_end_of_auction_without_free_slot_limit_stream_and_wrong_day_start(self): + stream = 3 + streams = 3 + next_date = datetime.now().date() + day_start = time(16, 0) + + plan = { + '_id': 'plantest_2017-10-03', + 'stream_1': {}, + 'stream_2': {}, + 'stream_3': {}, + 'streams': 3 + } + + result = self.manager.set_end_of_auction( + stream, streams, next_date, day_start, plan + ) + self.assertIsNone(result) + + +class NonClassicManagerTestMixin(object): + manager = None + streams_key = '' + + def test_get_date(self): + date = datetime.now().date() + plan_date = parse_date(date.isoformat() + 'T' + self.manager.working_day_start.isoformat(), None) + plan_date = plan_date.astimezone(TZ) if plan_date.tzinfo else TZ.localize(plan_date) + + db = mock.MagicMock() + + db.get.return_value = plantest + + plan_time, stream, plan = self.manager.get_date(db, '', date) + + self.assertEqual(plan, plantest) + self.assertEqual(plan_time, plan_date.time()) + self.assertEqual(stream, len(plantest[self.streams_key])) + + def test_get_hours_and_stream(self): + plan = { + self.streams_key: [1, 2, 3, 4, 5] + } + + plan_date, streams = self.manager._get_hours_and_stream(plan) + self.assertEqual(plan_date, self.manager.working_day_start.isoformat()) + self.assertEqual(streams, len(plan[self.streams_key])) + + def test_set_date(self): + db = mock.MagicMock() + plan = { + self.streams_key: [] + } + auction_id = '1' * 32 + + self.manager.set_date(db, plan, auction_id) + self.assertIn(auction_id, plan[self.streams_key]) + self.assertEqual(len(plan[self.streams_key]), 1) + + db.save.assert_called_with(plan) + + def test_free_slot(self): + plan = { + self.streams_key: [ + '1' * 32, + '2' * 32, + '3' * 32, + '1' * 32, + ] + } + db = mock.MagicMock() + plan_id = 'plan_id' + db.get.return_value = plan + auction_id = '1' * 32 + + self.manager.free_slot(db, plan_id, auction_id) + + self.assertEqual(len(plan[self.streams_key]), 2) + self.assertNotIn('1' * 32, plan[self.streams_key]) + + db.get.assert_called_with(plan_id) + db.save.assert_called_with(plan) + + def set_end_of_auction(self): + stream = 1 + streams = 3 + next_date = datetime.now().date() + day_start = time(15, 0) + + start, end, auction_day_start, auction_stream, new_slot = self.manager.set_end_of_auction( + stream, streams, next_date, day_start + ) + + expected_start = TZ.localize(datetime.combine(next_date, day_start)) + + self.assertEqual(start, expected_start) + self.assertEqual(end, expected_start + self.manager.working_day_duration) + self.assertEqual(auction_day_start, day_start) + self.assertEqual(auction_stream, stream) + self.assertFalse(new_slot) + + def set_end_of_auction_with_wrong_stream(self): + stream = 10 + streams = 3 + next_date = datetime.now().date() + day_start = time(15, 0) + + result = self.manager.set_end_of_auction( + stream, streams, next_date, day_start + ) + + self.assertIsNone(result) + + +class TestInsiderAuctionManager(BaseWebTest, NonClassicManagerTestMixin): + streams_key = 'dutch_streams' + + def setUp(self): + super(TestInsiderAuctionManager, self).setUp() + self.manager = InsiderAuctionsManager() + + +class TestTexasAuctionManager(BaseWebTest, NonClassicManagerTestMixin): + streams_key = 'texas_streams' + + def setUp(self): + super(TestTexasAuctionManager, self).setUp() + self.manager = TexasAuctionsManager() diff --git a/openprocurement/auction/tests/bridge_utils/test_utils.py b/openprocurement/auction/tests/bridge_utils/test_utils.py new file mode 100644 index 0000000..df4ba4e --- /dev/null +++ b/openprocurement/auction/tests/bridge_utils/test_utils.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- +import mock +from copy import deepcopy +from datetime import datetime, timedelta + + +from openprocurement.auction.bridge_utils.constants import TZ +from openprocurement.auction.bridge_utils.managers import InsiderAuctionsManager +from openprocurement.auction.bridge_utils.utils import ( + planning_auction, + get_manager_for_auction, + check_auction +) +from openprocurement.auction.tests.bridge_utils.base import BaseWebTest +from openprocurement.auction.tests.bridge_utils.data import test_auction_data + + +test_auction_data_quick = deepcopy(test_auction_data) +test_auction_data_quick.update({ + "enquiryPeriod": { + 'startDate': datetime.now(TZ).isoformat(), + "endDate": datetime.now(TZ).isoformat() + }, + 'tenderPeriod': { + 'startDate': datetime.now(TZ).isoformat(), + "endDate": datetime.now(TZ).isoformat() + } +}) +test_auction_data_test_quick = deepcopy(test_auction_data_quick) +test_auction_data_test_quick['mode'] = 'test' + + +class CheckAuction(BaseWebTest): + + def test_check_aution(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + my_test_auction['auctionPeriod'] = {} + my_test_auction['auctionPeriod']['startDate'] = now.isoformat() + my_test_auction['auctionPeriod']['shouldStartAfter'] = (now + timedelta(days=10)).isoformat() + my_test_auction['procurementMethodType'] = 'dgfInsider' + + auction_period = check_auction(my_test_auction, self.db, self.mapper) + self.assertIn('auctionPeriod', auction_period) + self.assertIn('startDate', auction_period['auctionPeriod']) + + def test_check_auction_without_should_start(self): + my_test_auction = deepcopy(test_auction_data) + my_test_auction.pop('auctionPeriod', None) + + auction_period = check_auction(my_test_auction, self.db, self.mapper) + self.assertIsNone(auction_period) + + def test_check_auction_start_more_should_start(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + + my_test_auction['auctionPeriod'] = {} + my_test_auction['auctionPeriod']['startDate'] = now.isoformat() + my_test_auction['auctionPeriod']['shouldStartAfter'] = (now - timedelta(days=10)).isoformat() + + auction_period = check_auction(my_test_auction, self.db, self.mapper) + self.assertIsNone(auction_period) + + def test_check_auction_with_lots(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + auction_period = { + 'startDate': now.isoformat(), + 'shouldStartAfter': (now + timedelta(days=10)).isoformat() + } + lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} + my_test_auction['lots'] = [lot, lot] + + lots = check_auction(my_test_auction, self.db, self.mapper) + self.assertIn('lots', lots) + self.assertEqual(len(lots['lots']), 2) + + def test_check_auction_with_not_active_lot(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + auction_period = { + 'startDate': now.isoformat(), + 'shouldStartAfter': (now + timedelta(days=10)).isoformat() + } + lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} + not_active = deepcopy(lot) + not_active['status'] = 'pending' + my_test_auction['lots'] = [lot, not_active] + + lots = check_auction(my_test_auction, self.db, self.mapper) + self.assertIn('lots', lots) + self.assertEqual(len(lots['lots'][1].keys()), 0) + + def test_should_start_before_auction_start(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + auction_period = { + 'startDate': now.isoformat(), + 'shouldStartAfter': (now + timedelta(days=10)).isoformat() + } + lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} + should_start_before = deepcopy(lot) + should_start_before['auctionPeriod']['startDate'] = now.isoformat() + should_start_before['auctionPeriod']['shouldStartAfter'] = (now - timedelta(days=10)).isoformat() + my_test_auction['lots'] = [lot, should_start_before] + + lots = check_auction(my_test_auction, self.db, self.mapper) + self.assertIn('lots', lots) + self.assertEqual(len(lots['lots'][1].keys()), 0) + + def test_should_start_absent(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + auction_period = { + 'startDate': now.isoformat(), + 'shouldStartAfter': (now + timedelta(days=10)).isoformat() + } + lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} + no_should_start = deepcopy(lot) + no_should_start['auctionPeriod'].pop('shouldStartAfter') + my_test_auction['lots'] = [lot, no_should_start] + + lots = check_auction(my_test_auction, self.db, self.mapper) + self.assertIn('lots', lots) + self.assertEqual(len(lots['lots'][1].keys()), 0) + + def test_check_auction_with_invalid_lots(self): + now = datetime.now(TZ) + now = now.replace(year=2018, month=8, day=25) + + my_test_auction = deepcopy(test_auction_data) + auction_period = { + 'startDate': now.isoformat(), + 'shouldStartAfter': (now + timedelta(days=10)).isoformat() + } + lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} + not_active = deepcopy(lot) + not_active['status'] = 'pending' + my_test_auction['lots'] = [not_active, not_active] + + lots = check_auction(my_test_auction, self.db, self.mapper) + self.assertIsNone(lots) + + # should start after before auction + my_test_auction = deepcopy(test_auction_data) + should_start_before = deepcopy(lot) + should_start_before['auctionPeriod']['startDate'] = now.isoformat() + should_start_before['auctionPeriod']['shouldStartAfter'] = (now - timedelta(days=10)).isoformat() + my_test_auction['lots'] = [should_start_before, should_start_before] + + lots = check_auction(my_test_auction, self.db, self.mapper) + self.assertIsNone(lots) + + # should start absent + my_test_auction = deepcopy(test_auction_data) + no_should_start = deepcopy(lot) + no_should_start['auctionPeriod'].pop('shouldStartAfter') + my_test_auction['lots'] = [no_should_start, no_should_start] + + lots = check_auction(my_test_auction, self.db, self.mapper) + self.assertIsNone(lots) + + +class AuctionPlanning(BaseWebTest): + + def test_auction_quick_planning(self): + now = datetime.now(TZ) + auctionPeriodstartDate = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db, True)[0] + self.assertTrue(now < auctionPeriodstartDate < now + timedelta(hours=1)) + + def test_auction_quick_planning_insider(self): + now = datetime.now(TZ) + my_test_auction = deepcopy(test_auction_data_test_quick) + my_test_auction['procurementMethodType'] = 'dgfInsider' + auctionPeriodstartDate = planning_auction( + my_test_auction, self.mapper, now, self.db, True + )[0] + self.assertTrue( + now < auctionPeriodstartDate < now + timedelta(hours=1) + ) + + def test_auction_planning_overlow_insider(self): + now = datetime.now(TZ) + my_test_auction = deepcopy(test_auction_data_test_quick) + my_test_auction['procurementMethodType'] = 'dgfInsider' + res = planning_auction(my_test_auction, self.mapper, now, self.db)[0] + startDate = res.date() + count = 0 + while startDate == res.date(): + count += 1 + res = planning_auction(my_test_auction, self.mapper, now, self.db)[0] + self.assertEqual(count, 15) + + def test_auction_planning_overlow(self): + now = datetime.now(TZ) + res = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db)[0] + startDate = res.date() + count = 0 + while startDate == res.date(): + count += 1 + res = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db)[0] + self.assertEqual(count, 100) + + def test_auction_planning_free(self): + now = datetime.now(TZ) + test_auction_data_test_quick.pop("id") + res = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db)[0] + startDate, startTime = res.date(), res.time() + manager = get_manager_for_auction(test_auction_data, self.mapper) + manager.free_slot(self.db, "plantest_{}".format(startDate.isoformat()), "", res) + res = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db)[0] + self.assertEqual(res.time(), startTime) + + def test_auction_planning_buffer(self): + some_date = datetime(2015, 9, 21, 6, 30) + date = some_date.date() + ndate = (some_date + timedelta(days=1)).date() + res = planning_auction(test_auction_data_test_quick, self.mapper, some_date, self.db)[0] + self.assertEqual(res.date(), date) + some_date = some_date.replace(hour=10) + res = planning_auction(test_auction_data_test_quick, self.mapper, some_date, self.db)[0] + self.assertNotEqual(res.date(), date) + self.assertEqual(res.date(), ndate) + some_date = some_date.replace(hour=16) + res = planning_auction(test_auction_data_test_quick, self.mapper, some_date, self.db)[0] + self.assertNotEqual(res.date(), date) + self.assertEqual(res.date(), ndate) + + def test_skipping_holidays(self): + now = datetime.now(TZ) + now = now.replace(year=2018, month=8, day=25, hour=6) + # Set saturday + while now.weekday() != 5: + now = now + timedelta(days=1) + my_test_auction = deepcopy(test_auction_data) + my_test_auction['procurementMethodType'] = 'dgfInsider' + res, _, skipped_days = planning_auction(my_test_auction, self.mapper, now, self.db) + self.assertEqual(res.date(), (now + timedelta(days=2)).date()) + self.assertEqual(skipped_days, 0) + + def test_skipping_holiday_in_calendar(self): + now = datetime.now(TZ) + + # Holiday in 2018 year + now = now.replace(year=2018, month=12, day=25, hour=6) + self.assertNotIn(now.weekday(), [5, 6]) + + my_test_auction = deepcopy(test_auction_data) + my_test_auction['procurementMethodType'] = 'dgfInsider' + res, _, skipped_days = planning_auction(my_test_auction, self.mapper, now, self.db) + self.assertEqual(res.date(), (now + timedelta(days=1)).date()) + self.assertEqual(skipped_days, 0) + + def test_skipping_day_if_time_after_working_day_start(self): + now = datetime.now(TZ) + now = now.replace(year=2018, month=8, day=25, hour=12) + + while now.weekday() != 1: + now = now + timedelta(days=1) + + my_test_auction = deepcopy(test_auction_data) + my_test_auction['procurementMethodType'] = 'dgfInsider' + res, _, skipped_days = planning_auction(my_test_auction, self.mapper, now, self.db) + self.assertEqual(res.date(), (now + timedelta(days=1)).date()) + self.assertEqual(skipped_days, 0) + + def test_result_set_end_none(self): + my_test_auction = deepcopy(test_auction_data) + + now = datetime.now(TZ) + now = now.replace(year=2018, month=8, day=25, hour=6) + + # Set friday + while now.weekday() != 1: + now = now + timedelta(days=1) + + # set_end_of_auction return none once + with mock.patch('openprocurement.auction.bridge_utils.utils.get_manager_for_auction') as get_manager_mock: + mocked_manager = mock.MagicMock() + mocked_manager.working_day_start = InsiderAuctionsManager.working_day_start + + mocked_manager.get_date.side_effect = [ + ('1', '2', '3'), + ('1', '2', '3'), + ('1', '2', '3'), + ] + mocked_manager.set_end_of_auction.side_effect = [ + None, + None, + ('1', mock.MagicMock(), '3', '4', '5') + ] + + get_manager_mock.return_value = mocked_manager + + start, _, skipped_days = planning_auction(my_test_auction, self.mapper, now, self.db) + + self.assertEqual(skipped_days, 2) From ae656584f7104b8788f8172b554702bf2703d5d9 Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Mon, 18 Feb 2019 12:20:10 +0200 Subject: [PATCH 05/13] Add check_inner_auction function --- openprocurement/auction/bridge_utils/utils.py | 27 +++++++++++++++++++ openprocurement/auction/databridge.py | 1 + openprocurement/auction/plannings.py | 12 ++++++--- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/openprocurement/auction/bridge_utils/utils.py b/openprocurement/auction/bridge_utils/utils.py index 4bc0072..591423b 100644 --- a/openprocurement/auction/bridge_utils/utils.py +++ b/openprocurement/auction/bridge_utils/utils.py @@ -18,6 +18,7 @@ WORKING_DAY_START, CALENDAR_ID ) +from openprocurement.auction.bridge_utils.design import plan_auctions_view LOGGER = logging.getLogger('Openprocurement Auction') @@ -175,3 +176,29 @@ def check_auction(auction, db, mapper): if any(lots): return {'lots': lots} return None + + +def check_inner_auction(db, auction, mapper): + manager = get_manager_for_auction(auction, mapper) + + auction_time = auction.get('auctionPeriod', {}).get('startDate') and \ + parse_date(auction.get('auctionPeriod', {}).get('startDate')) + lots = dict([ + (i['id'], parse_date(i.get('auctionPeriod', {}).get('startDate'))) + for i in auction.get('lots', []) + if i.get('auctionPeriod', {}).get('startDate') + ]) + auc_list = [ + (x.key[1], TZ.localize(parse_date(x.value, None)), x.id) + for x in plan_auctions_view(db, startkey=[auction['id'], None], + endkey=[auction['id'], 32 * "f"]) + ] + for key, plan_time, plan_doc in auc_list: + if not key and (not auction_time or not + plan_time < auction_time < plan_time + + timedelta(minutes=30)): + manager.free_slot(db, plan_doc, auction['id'], plan_time) + elif key and (not lots.get(key) or lots.get(key) and not + plan_time < lots.get(key) < plan_time + + timedelta(minutes=30)): + manager.free_slot(db, plan_doc, "_".join([auction['id'], key]), plan_time) diff --git a/openprocurement/auction/databridge.py b/openprocurement/auction/databridge.py index 5c61da0..679f252 100644 --- a/openprocurement/auction/databridge.py +++ b/openprocurement/auction/databridge.py @@ -117,6 +117,7 @@ def run(self): if not planning: continue planning.add_auction_period() + planning.check_to_free_slot() for cmd, item_id, lot_id in planning: if lot_id: LOGGER.info('Lot {} of tender {} selected for {}'.format( diff --git a/openprocurement/auction/plannings.py b/openprocurement/auction/plannings.py index 9321787..f5abdc0 100644 --- a/openprocurement/auction/plannings.py +++ b/openprocurement/auction/plannings.py @@ -23,9 +23,12 @@ from openprocurement.auction.utils import ( do_until_success, prepare_auction_worker_cmd ) -from openprocurement.auction.bridge_utils.utils import check_auction, context_unpack, SESSION -from openprocurement.auction.bridge_utils.managers import MANAGERS_MAPPING - +from openprocurement.auction.bridge_utils.utils import ( + check_auction, + context_unpack, + SESSION, + check_inner_auction +) SIMPLE_AUCTION_TYPE = 0 @@ -73,6 +76,9 @@ def add_auction_period(self): else: LOGGER.info("Successfully updated auction '{}' with '{}'".format(r.status_code, url, data)) + def check_to_free_slot(self): + check_inner_auction(self.bridge.stream_db, self.item, self.bridge.manager_mapper) + def __iter__(self): if self.item['status'] == "active.auction": if 'lots' not in self.item and 'auctionPeriod' in self.item \ From c4950e244e6058ee4e69a33a14546cbfe12106cc Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Mon, 18 Feb 2019 12:20:27 +0200 Subject: [PATCH 06/13] Expand tests --- .../auction/tests/bridge_utils/test_utils.py | 127 +++++++++++++++++- 1 file changed, 124 insertions(+), 3 deletions(-) diff --git a/openprocurement/auction/tests/bridge_utils/test_utils.py b/openprocurement/auction/tests/bridge_utils/test_utils.py index df4ba4e..3fba845 100644 --- a/openprocurement/auction/tests/bridge_utils/test_utils.py +++ b/openprocurement/auction/tests/bridge_utils/test_utils.py @@ -3,16 +3,18 @@ from copy import deepcopy from datetime import datetime, timedelta +from iso8601 import parse_date from openprocurement.auction.bridge_utils.constants import TZ -from openprocurement.auction.bridge_utils.managers import InsiderAuctionsManager +from openprocurement.auction.bridge_utils.managers import InsiderAuctionsManager, MANAGERS_MAPPING from openprocurement.auction.bridge_utils.utils import ( planning_auction, get_manager_for_auction, - check_auction + check_auction, + check_inner_auction ) from openprocurement.auction.tests.bridge_utils.base import BaseWebTest -from openprocurement.auction.tests.bridge_utils.data import test_auction_data +from openprocurement.auction.tests.bridge_utils.data import test_auction_data, plantest test_auction_data_quick = deepcopy(test_auction_data) @@ -302,3 +304,122 @@ def test_result_set_end_none(self): start, _, skipped_days = planning_auction(my_test_auction, self.mapper, now, self.db) self.assertEqual(skipped_days, 2) + + +class TestCheckInnerAuction(BaseWebTest): + + def setUp(self): + super(TestCheckInnerAuction, self).setUp() + plantest['_id'] = 'plantest_{}'.format( + datetime.now().date().isoformat()) + plantest_from_db = self.db.get(plantest['_id'], {}) + plantest_from_db.update(plantest) + + self.db.save(plantest_from_db) + + def test_check_inner_auction(self): + insider_auction_id = '01fa8a7dc4b8eac3b5820747efc6fe36' + texas_auction_id = 'dc3d950743304d05adaa1cd5b0475075' + classic_auction_with_lots = 'da8a28ed2bdf73ee1d373e4cadfed4c5' + classic_auction_without_lots = 'e51508cddc2c490005eaecb73c006b72' + lots_ids = ['1c2fb1e496b317b2b87e197e2332da77', + 'b10f9f7f26157ae2f349be8dc2106d6e'] + + today = datetime.now().date().isoformat() + time = '12:15:00' # actually, can be any time between 12:00:00 and 12:30:00 due to existing asserts + raw_time = ''.join([today, 'T', time]) + + # datetime.datetime object prepared in the way scheduler actually does it: + test_time = TZ.localize(parse_date(raw_time, None)).isoformat() + + auction = { + 'id': insider_auction_id, + 'procurementMethodType': 'dgfInsider', + 'auctionPeriod': { + 'startDate': test_time + } + } + mapper = { + 'pmts': { + 'dgfInsider': MANAGERS_MAPPING['dutch'](), + 'landLease': MANAGERS_MAPPING['texas']() + }, + 'types': {'english': MANAGERS_MAPPING['english']()} + } + + plantest = self.db.get('plantest_{}'.format(today)) + + # Test dutch + self.assertEqual(len(plantest.get('dutch_streams', [])), 15) + self.assertIn(insider_auction_id, plantest.get('dutch_streams')) + + check_inner_auction(self.db, auction, mapper) + plantest = self.db.get('plantest_{}'.format(today)) + self.assertEqual(len(plantest.get('dutch_streams', [])), 6) + self.assertNotIn(insider_auction_id, plantest.get('dutch_streams')) + + # Test texas + auction['id'] = texas_auction_id + auction['procurementMethodType'] = 'landLease' + + self.assertEqual(len(plantest.get('texas_streams', [])), 20) + self.assertIn(texas_auction_id, plantest.get('texas_streams')) + + check_inner_auction(self.db, auction, mapper) + plantest = self.db.get('plantest_{}'.format(today)) + self.assertEqual(len(plantest.get('texas_streams', [])), 15) + self.assertNotIn(texas_auction_id, plantest.get('texas_streams')) + + # Test classic with lots + auction['procurementMethodType'] = 'classic' + auction['id'] = classic_auction_with_lots + auction['lots'] = [ + { + 'id': lots_ids[0], + 'auctionPeriod': {'startDate': test_time} + }, + { + 'id': lots_ids[1], + 'auctionPeriod': {'startDate': test_time} + } + ] + self.assertEqual(len(plantest.get('stream_1')), 10) + self.assertEqual(len(plantest.get('stream_2')), 10) + stream_1_none_count = len( + [v for k, v in plantest.get('stream_1').items() if v is None]) + stream_2_none_count = len( + [v for k, v in plantest.get('stream_2').items() if v is None]) + self.assertEqual(stream_1_none_count, 0) + self.assertEqual(stream_2_none_count, 0) + check_inner_auction(self.db, auction, mapper) + plantest = self.db.get('plantest_{}'.format(today)) + self.assertEqual(len(plantest.get('stream_1')), 10) + self.assertEqual(len(plantest.get('stream_2')), 10) + stream_1_none_count = len( + [v for k, v in plantest.get('stream_1').items() if v is None]) + stream_2_none_count = len( + [v for k, v in plantest.get('stream_2').items() if v is None]) + self.assertEqual(stream_1_none_count, 3) + self.assertEqual(stream_2_none_count, 3) + self.assertNotIn(classic_auction_with_lots, + plantest.get('stream_1', {}).values()) + self.assertNotIn(classic_auction_with_lots, + plantest.get('stream_2', {}).values()) + + # Test classic without lots + del auction['lots'] + auction['id'] = classic_auction_without_lots + check_inner_auction(self.db, auction, mapper) + plantest = self.db.get('plantest_{}'.format(today)) + self.assertEqual(len(plantest.get('stream_1')), 10) + self.assertEqual(len(plantest.get('stream_2')), 10) + stream_1_none_count = len( + [v for k, v in plantest.get('stream_1').items() if v is None]) + stream_2_none_count = len( + [v for k, v in plantest.get('stream_2').items() if v is None]) + self.assertEqual(stream_1_none_count, 7) + self.assertEqual(stream_2_none_count, 6) + self.assertNotIn(classic_auction_without_lots, + plantest.get('stream_1', {}).values()) + self.assertNotIn(classic_auction_without_lots, + plantest.get('stream_2', {}).values()) From e99c897b4ee7fc72ebd61f00e35e67efe9bbc0d3 Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Mon, 18 Feb 2019 13:29:06 +0200 Subject: [PATCH 07/13] Configurate streams amount from bridge config --- openprocurement/auction/bridge_utils/managers.py | 1 - openprocurement/auction/databridge.py | 12 +++++++++++- .../tests/bridge_utils/auctions_data_bridge.yaml | 4 ++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/openprocurement/auction/bridge_utils/managers.py b/openprocurement/auction/bridge_utils/managers.py index c6f6a5f..73553f3 100644 --- a/openprocurement/auction/bridge_utils/managers.py +++ b/openprocurement/auction/bridge_utils/managers.py @@ -82,7 +82,6 @@ def get_streams(self, db, streams_id=STREAMS_ID): :return: amount of streams for auction per day :rtype: int """ - streams = db.get(streams_id, deepcopy(DEFAULT_STREAMS_DOC)) return streams.get(self.streams_key, DEFAULT_STREAMS_DOC[self.streams_key]) diff --git a/openprocurement/auction/databridge.py b/openprocurement/auction/databridge.py index 679f252..002e596 100644 --- a/openprocurement/auction/databridge.py +++ b/openprocurement/auction/databridge.py @@ -23,7 +23,7 @@ IAuctionDatabridge, IAuctionsManager from openprocurement.auction.bridge_utils.design import sync_design from openprocurement.auction.bridge_utils.managers import MANAGERS_MAPPING -from openprocurement.auction.bridge_utils.constants import WORKING_DAYS, CALENDAR_ID +from openprocurement.auction.bridge_utils.constants import WORKING_DAYS, CALENDAR_ID, STREAMS_ID from openprocurement.auction.core import components from openprocurement.auction.utils import FeedItem, check_workers @@ -85,6 +85,7 @@ def __init__(self, config, re_planning=False, debug=False): self.stream_db = Database(db_for_streams, session=Session(retry_delays=range(10))) self._set_holidays() + self._set_streams_limits() sync_design(self.stream_db) # Managers Mapping @@ -102,6 +103,15 @@ def _set_holidays(self): del self.stream_db[CALENDAR_ID] self.stream_db.save(calendar) + def _set_streams_limits(self): + streams = self.config.get('main').get('streams', {}) + + stream_amount = {'_id': STREAMS_ID} + stream_amount.update(streams) + if STREAMS_ID in self.stream_db: + del self.stream_db[STREAMS_ID] + self.stream_db.save(stream_amount) + def config_get(self, name): return self.config.get('main').get(name) diff --git a/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml b/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml index 7c2c92b..09c6d38 100644 --- a/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml +++ b/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml @@ -7,6 +7,10 @@ main: auctions_db: database timezone: Europe/Kiev stream_db : test_stream_db + streams: + streams: 10 + dutch_streams: 15 + texas_streams: 20 plugins: english: auction_worker_config: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/etc/auction_worker_defaults.yaml From ff026a392b47eb3c7ba80af768cb009f64c33262 Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Thu, 21 Feb 2019 10:47:17 +0200 Subject: [PATCH 08/13] Replace path in test config with abstract path --- .../tests/bridge_utils/auctions_data_bridge.yaml | 12 ++++++------ openprocurement/auction/tests/bridge_utils/base.py | 4 +++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml b/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml index 09c6d38..cd36d0a 100644 --- a/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml +++ b/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml @@ -13,17 +13,17 @@ main: texas_streams: 20 plugins: english: - auction_worker_config: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/etc/auction_worker_defaults.yaml + auction_worker_config: /path/to/auction_worker_defaults.yaml procurement_method_types: [DGFOtherAssets, dgfFinancialAssets] - auction_worker: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/bin/auction_worker + auction_worker: /path/to/auction_worker dutch: - auction_worker_config: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/etc/auction_worker_insider.yaml + auction_worker_config: /path/to/auction_worker_insider.yaml procurement_method_types: [dgfInsider] - auction_worker: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/bin/auction_insider + auction_worker: /path/to/auction_insider texas: - auction_worker_config: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/etc/auction_worker_texas.yaml + auction_worker_config: /path/to/auction_worker_texas.yaml procurement_method_types: [landLease] - auction_worker: /home/oleksiy.veretiuk/QuintaGroup/projects/kadastral/openprocurement.auction.buildout/bin/auction_texas + auction_worker: /path/to/auction_texas retrievers_params: down_requests_sleep: 0.1 up_requests_sleep: 0.1 diff --git a/openprocurement/auction/tests/bridge_utils/base.py b/openprocurement/auction/tests/bridge_utils/base.py index 64815fd..f8ad6e9 100644 --- a/openprocurement/auction/tests/bridge_utils/base.py +++ b/openprocurement/auction/tests/bridge_utils/base.py @@ -2,6 +2,7 @@ from __future__ import print_function import os import unittest +import mock from yaml import load from couchdb import Server @@ -21,7 +22,8 @@ def setUp(self): with open(dir_path + '/auctions_data_bridge.yaml') as config_file_obj: config = load(config_file_obj.read()) - bridge = AuctionsDataBridge(config) + with mock.patch('openprocurement.auction.databridge.check_workers'): + bridge = AuctionsDataBridge(config) self.couchdb_server = Server(config['main'].get('couch_url')) self.db = bridge.stream_db From 75fe8d0e6249df5ab61b9c99a01e9541b0984504 Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Thu, 21 Feb 2019 18:37:31 +0200 Subject: [PATCH 09/13] Compare datetimes not strings --- openprocurement/auction/bridge_utils/utils.py | 74 +++++++++++++------ 1 file changed, 50 insertions(+), 24 deletions(-) diff --git a/openprocurement/auction/bridge_utils/utils.py b/openprocurement/auction/bridge_utils/utils.py index 591423b..8d0aa3f 100644 --- a/openprocurement/auction/bridge_utils/utils.py +++ b/openprocurement/auction/bridge_utils/utils.py @@ -119,37 +119,63 @@ def planning_auction(auction, mapper, start, db, quick=False, lot_id=None): def check_auction(auction, db, mapper): now = get_now() quick = os.environ.get('SANDBOX_MODE', False) and u'quick' in auction.get('submissionMethodDetails', '') - if not auction.get('lots') and 'shouldStartAfter' in auction.get('auctionPeriod', {}) and auction['auctionPeriod']['shouldStartAfter'] > auction['auctionPeriod'].get('startDate'): - period = auction.get('auctionPeriod') - shouldStartAfter = max(parse_date(period.get('shouldStartAfter'), TZ).astimezone(TZ), now) - planned = False - while not planned: - try: - auctionPeriod, stream, skip_days = planning_auction(auction, mapper, shouldStartAfter, db, quick) - planned = True - except ResourceConflict: - planned = False - auctionPeriod = randomize(auctionPeriod).isoformat() - planned = 'replanned' if period.get('startDate') else 'planned' - LOGGER.info( - '{} auction for auction {} to {}. Stream {}.{}'.format( - planned.title(), auction['id'], auctionPeriod, stream, skipped_days(skip_days) - ), - extra={ - 'MESSAGE_ID': '{}_auction_auction'.format(planned), - 'PLANNED_DATE': auctionPeriod, - 'PLANNED_STREAM': stream, - 'PLANNED_DAYS_SKIPPED': skip_days} + + if not auction.get('lots'): + period = auction.get('auctionPeriod', {}) + shouldStartAfter = parse_date(period['shouldStartAfter'], TZ).astimezone(TZ) if period.get('shouldStartAfter') else None + start_date = parse_date(period['startDate'], TZ).astimezone(TZ) if period.get('startDate') else None + + is_needed_to_add_auction_period = bool( + shouldStartAfter and + ( + start_date is None or + shouldStartAfter > start_date + ) ) - return {'auctionPeriod': {'startDate': auctionPeriod}} + + if shouldStartAfter and is_needed_to_add_auction_period: + shouldStartAfter = max(shouldStartAfter, now) + planned = False + while not planned: + try: + auctionPeriod, stream, skip_days = planning_auction(auction, mapper, shouldStartAfter, db, quick) + planned = True + except ResourceConflict: + planned = False + auctionPeriod = randomize(auctionPeriod).isoformat() + planned = 'replanned' if period.get('startDate') else 'planned' + LOGGER.info( + '{} auction for auction {} to {}. Stream {}.{}'.format( + planned.title(), auction['id'], auctionPeriod, stream, skipped_days(skip_days) + ), + extra={ + 'MESSAGE_ID': '{}_auction_auction'.format(planned), + 'PLANNED_DATE': auctionPeriod, + 'PLANNED_STREAM': stream, + 'PLANNED_DAYS_SKIPPED': skip_days} + ) + return {'auctionPeriod': {'startDate': auctionPeriod}} + elif auction.get('lots'): lots = [] for lot in auction.get('lots', []): - if lot['status'] != 'active' or 'shouldStartAfter' not in lot.get('auctionPeriod', {}) or lot['auctionPeriod']['shouldStartAfter'] < lot['auctionPeriod'].get('startDate'): + period = lot.get('auctionPeriod', {}) + shouldStartAfter = parse_date(period['shouldStartAfter'], TZ).astimezone(TZ) if period.get( + 'shouldStartAfter') else None + start_date = parse_date(period['startDate'], TZ).astimezone(TZ) if period.get('startDate') else None + + skip_lot = bool( + shouldStartAfter is None or ( + start_date is not None and + start_date > shouldStartAfter + ) + ) + + if lot['status'] != 'active' or skip_lot: lots.append({}) continue period = lot.get('auctionPeriod') - shouldStartAfter = max(parse_date(period.get('shouldStartAfter'), TZ).astimezone(TZ), now) + shouldStartAfter = max(shouldStartAfter, now) lot_id = lot['id'] planned = False while not planned: From f6f3fdac5c7a14c1c55396f5311c6f220f8e317b Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Thu, 21 Feb 2019 18:37:45 +0200 Subject: [PATCH 10/13] Expand tests --- .../auction/tests/bridge_utils/test_utils.py | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/openprocurement/auction/tests/bridge_utils/test_utils.py b/openprocurement/auction/tests/bridge_utils/test_utils.py index 3fba845..18b5dfb 100644 --- a/openprocurement/auction/tests/bridge_utils/test_utils.py +++ b/openprocurement/auction/tests/bridge_utils/test_utils.py @@ -54,7 +54,19 @@ def test_check_auction_without_should_start(self): auction_period = check_auction(my_test_auction, self.db, self.mapper) self.assertIsNone(auction_period) - def test_check_auction_start_more_should_start(self): + def test_check_auction_without_auction_start_date(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + + my_test_auction['auctionPeriod'] = {} + my_test_auction['auctionPeriod']['shouldStartAfter'] = (now - timedelta(days=10)).isoformat() + + auction_period = check_auction(my_test_auction, self.db, self.mapper) + self.assertIn('auctionPeriod', auction_period) + self.assertIn('startDate', auction_period['auctionPeriod']) + + def test_check_auction_start_after_should_start(self): now = datetime.now(TZ) my_test_auction = deepcopy(test_auction_data) @@ -133,10 +145,25 @@ def test_should_start_absent(self): self.assertIn('lots', lots) self.assertEqual(len(lots['lots'][1].keys()), 0) - def test_check_auction_with_invalid_lots(self): + def test_auction_start_absent(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + auction_period = { + 'shouldStartAfter': (now + timedelta(days=10)).isoformat() + } + lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} + my_test_auction['lots'] = [lot, lot] + + lots = check_auction(my_test_auction, self.db, self.mapper) + self.assertIn('lots', lots) + self.assertEqual(len(lots['lots']), 2) + + def test_check_auction_with_all_invalid_lots(self): now = datetime.now(TZ) now = now.replace(year=2018, month=8, day=25) + # no active lots my_test_auction = deepcopy(test_auction_data) auction_period = { 'startDate': now.isoformat(), @@ -150,7 +177,7 @@ def test_check_auction_with_invalid_lots(self): lots = check_auction(my_test_auction, self.db, self.mapper) self.assertIsNone(lots) - # should start after before auction + # should_start_after before auction my_test_auction = deepcopy(test_auction_data) should_start_before = deepcopy(lot) should_start_before['auctionPeriod']['startDate'] = now.isoformat() From 0311954846b3b0f95cdbff1b2fafc64fa495dbbc Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Wed, 27 Feb 2019 13:49:55 +0200 Subject: [PATCH 11/13] Divide check_auction in two function to create period for auction with lots or without --- openprocurement/auction/bridge_utils/utils.py | 166 ++++++++++-------- 1 file changed, 90 insertions(+), 76 deletions(-) diff --git a/openprocurement/auction/bridge_utils/utils.py b/openprocurement/auction/bridge_utils/utils.py index 8d0aa3f..56e9464 100644 --- a/openprocurement/auction/bridge_utils/utils.py +++ b/openprocurement/auction/bridge_utils/utils.py @@ -116,94 +116,108 @@ def planning_auction(auction, mapper, start, db, quick=False, lot_id=None): return start, stream, skipped_days -def check_auction(auction, db, mapper): +def create_auction_period(auction, db, mapper, quick): now = get_now() - quick = os.environ.get('SANDBOX_MODE', False) and u'quick' in auction.get('submissionMethodDetails', '') - if not auction.get('lots'): - period = auction.get('auctionPeriod', {}) + period = auction.get('auctionPeriod', {}) + shouldStartAfter = parse_date(period['shouldStartAfter'], TZ).astimezone(TZ) if period.get('shouldStartAfter') else None + start_date = parse_date(period['startDate'], TZ).astimezone(TZ) if period.get('startDate') else None + + is_needed_to_add_auction_period = bool( + shouldStartAfter and + ( + start_date is None or + shouldStartAfter > start_date + ) + ) + + if shouldStartAfter and is_needed_to_add_auction_period: + shouldStartAfter = max(shouldStartAfter, now) + planned = False + while not planned: + try: + auctionPeriod, stream, skip_days = planning_auction(auction, mapper, shouldStartAfter, db, quick) + planned = True + except ResourceConflict: + planned = False + auctionPeriod = randomize(auctionPeriod).isoformat() + planned = 'replanned' if period.get('startDate') else 'planned' + LOGGER.info( + '{} auction for auction {} to {}. Stream {}.{}'.format( + planned.title(), auction['id'], auctionPeriod, stream, skipped_days(skip_days) + ), + extra={ + 'MESSAGE_ID': '{}_auction_auction'.format(planned), + 'PLANNED_DATE': auctionPeriod, + 'PLANNED_STREAM': stream, + 'PLANNED_DAYS_SKIPPED': skip_days} + ) + return {'auctionPeriod': {'startDate': auctionPeriod}} + + return None + + +def create_auction_period_for_lots(auction, db, mapper, quick): + now = get_now() + + lots = [] + for lot in auction.get('lots', []): + period = lot.get('auctionPeriod', {}) shouldStartAfter = parse_date(period['shouldStartAfter'], TZ).astimezone(TZ) if period.get('shouldStartAfter') else None start_date = parse_date(period['startDate'], TZ).astimezone(TZ) if period.get('startDate') else None - is_needed_to_add_auction_period = bool( - shouldStartAfter and - ( - start_date is None or - shouldStartAfter > start_date + skip_lot = bool( + shouldStartAfter is None or ( + start_date is not None and + start_date > shouldStartAfter ) ) - if shouldStartAfter and is_needed_to_add_auction_period: - shouldStartAfter = max(shouldStartAfter, now) - planned = False - while not planned: - try: - auctionPeriod, stream, skip_days = planning_auction(auction, mapper, shouldStartAfter, db, quick) - planned = True - except ResourceConflict: - planned = False - auctionPeriod = randomize(auctionPeriod).isoformat() - planned = 'replanned' if period.get('startDate') else 'planned' - LOGGER.info( - '{} auction for auction {} to {}. Stream {}.{}'.format( - planned.title(), auction['id'], auctionPeriod, stream, skipped_days(skip_days) - ), - extra={ - 'MESSAGE_ID': '{}_auction_auction'.format(planned), - 'PLANNED_DATE': auctionPeriod, - 'PLANNED_STREAM': stream, - 'PLANNED_DAYS_SKIPPED': skip_days} - ) - return {'auctionPeriod': {'startDate': auctionPeriod}} - - elif auction.get('lots'): - lots = [] - for lot in auction.get('lots', []): - period = lot.get('auctionPeriod', {}) - shouldStartAfter = parse_date(period['shouldStartAfter'], TZ).astimezone(TZ) if period.get( - 'shouldStartAfter') else None - start_date = parse_date(period['startDate'], TZ).astimezone(TZ) if period.get('startDate') else None - - skip_lot = bool( - shouldStartAfter is None or ( - start_date is not None and - start_date > shouldStartAfter - ) - ) + if lot['status'] != 'active' or skip_lot: + lots.append({}) + continue + period = lot.get('auctionPeriod') + shouldStartAfter = max(shouldStartAfter, now) + lot_id = lot['id'] + planned = False + while not planned: + try: + auctionPeriod, stream, skip_days = planning_auction(auction, mapper, shouldStartAfter, db, quick, + lot_id) + planned = True + except ResourceConflict: + planned = False + auctionPeriod = randomize(auctionPeriod).isoformat() + planned = 'replanned' if period.get('startDate') else 'planned' + lots.append({'auctionPeriod': {'startDate': auctionPeriod}}) + LOGGER.info( + '{} auction for lot {} of auction {} to {}. Stream {}.{}'.format( + planned.title(), lot_id, auction['id'], auctionPeriod, stream, skipped_days(skip_days) + ), + extra={ + 'MESSAGE_ID': '{}_auction_lot'.format(planned), + 'PLANNED_DATE': auctionPeriod, + 'PLANNED_STREAM': stream, + 'PLANNED_DAYS_SKIPPED': skip_days, + 'LOT_ID': lot_id + } + ) + if any(lots): + return {'lots': lots} - if lot['status'] != 'active' or skip_lot: - lots.append({}) - continue - period = lot.get('auctionPeriod') - shouldStartAfter = max(shouldStartAfter, now) - lot_id = lot['id'] - planned = False - while not planned: - try: - auctionPeriod, stream, skip_days = planning_auction(auction, mapper, shouldStartAfter, db, quick, lot_id) - planned = True - except ResourceConflict: - planned = False - auctionPeriod = randomize(auctionPeriod).isoformat() - planned = 'replanned' if period.get('startDate') else 'planned' - lots.append({'auctionPeriod': {'startDate': auctionPeriod}}) - LOGGER.info( - '{} auction for lot {} of auction {} to {}. Stream {}.{}'.format( - planned.title(), lot_id, auction['id'], auctionPeriod, stream, skipped_days(skip_days) - ), - extra={ - 'MESSAGE_ID': '{}_auction_lot'.format(planned), - 'PLANNED_DATE': auctionPeriod, - 'PLANNED_STREAM': stream, - 'PLANNED_DAYS_SKIPPED': skip_days, - 'LOT_ID': lot_id - } - ) - if any(lots): - return {'lots': lots} return None +def check_auction(auction, db, mapper): + quick = os.environ.get('SANDBOX_MODE', False) and u'quick' in auction.get('submissionMethodDetails', '') + + if not auction.get('lots'): + return create_auction_period(auction, db, mapper, quick) + + elif auction.get('lots'): + return create_auction_period_for_lots(auction, db, mapper, quick) + + def check_inner_auction(db, auction, mapper): manager = get_manager_for_auction(auction, mapper) From 4f7a13e5c8b51f8d534698f49a9180254f8960b0 Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Wed, 27 Feb 2019 13:50:18 +0200 Subject: [PATCH 12/13] Expand tests --- .../auction/tests/bridge_utils/test_utils.py | 58 ++++++++++++++----- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/openprocurement/auction/tests/bridge_utils/test_utils.py b/openprocurement/auction/tests/bridge_utils/test_utils.py index 18b5dfb..8fa4063 100644 --- a/openprocurement/auction/tests/bridge_utils/test_utils.py +++ b/openprocurement/auction/tests/bridge_utils/test_utils.py @@ -11,7 +11,9 @@ planning_auction, get_manager_for_auction, check_auction, - check_inner_auction + check_inner_auction, + create_auction_period_for_lots, + create_auction_period ) from openprocurement.auction.tests.bridge_utils.base import BaseWebTest from openprocurement.auction.tests.bridge_utils.data import test_auction_data, plantest @@ -32,7 +34,32 @@ test_auction_data_test_quick['mode'] = 'test' -class CheckAuction(BaseWebTest): +class CheckAuctionTest(BaseWebTest): + + def test_auction_without_lots(self): + my_test_auction = deepcopy(test_auction_data) + my_test_auction.pop('lots', None) + + with mock.patch('openprocurement.auction.bridge_utils.utils.create_auction_period') as create_auction_period_mock: + expected_result = 'auction period for auction' + create_auction_period_mock.return_value = expected_result + result = check_auction(my_test_auction, self.db, self.mapper) + self.assertEqual(result, expected_result) + self.assertEqual(create_auction_period_mock.call_count, 1) + + def test_auction_with_lots(self): + my_test_auction = deepcopy(test_auction_data) + my_test_auction['lots'] = ['1', '2'] + + with mock.patch('openprocurement.auction.bridge_utils.utils.create_auction_period_for_lots') as create_auction_period_for_lots_mock: + expected_result = 'auction period for auction lots' + create_auction_period_for_lots_mock.return_value = expected_result + result = check_auction(my_test_auction, self.db, self.mapper) + self.assertEqual(result, expected_result) + self.assertEqual(create_auction_period_for_lots_mock.call_count, 1) + + +class CreateAuctionPeriodTest(BaseWebTest): def test_check_aution(self): now = datetime.now(TZ) @@ -43,7 +70,7 @@ def test_check_aution(self): my_test_auction['auctionPeriod']['shouldStartAfter'] = (now + timedelta(days=10)).isoformat() my_test_auction['procurementMethodType'] = 'dgfInsider' - auction_period = check_auction(my_test_auction, self.db, self.mapper) + auction_period = create_auction_period(my_test_auction, self.db, self.mapper, 'mode') self.assertIn('auctionPeriod', auction_period) self.assertIn('startDate', auction_period['auctionPeriod']) @@ -51,7 +78,7 @@ def test_check_auction_without_should_start(self): my_test_auction = deepcopy(test_auction_data) my_test_auction.pop('auctionPeriod', None) - auction_period = check_auction(my_test_auction, self.db, self.mapper) + auction_period = create_auction_period(my_test_auction, self.db, self.mapper, 'mode') self.assertIsNone(auction_period) def test_check_auction_without_auction_start_date(self): @@ -62,7 +89,7 @@ def test_check_auction_without_auction_start_date(self): my_test_auction['auctionPeriod'] = {} my_test_auction['auctionPeriod']['shouldStartAfter'] = (now - timedelta(days=10)).isoformat() - auction_period = check_auction(my_test_auction, self.db, self.mapper) + auction_period = create_auction_period(my_test_auction, self.db, self.mapper, 'mode') self.assertIn('auctionPeriod', auction_period) self.assertIn('startDate', auction_period['auctionPeriod']) @@ -75,9 +102,12 @@ def test_check_auction_start_after_should_start(self): my_test_auction['auctionPeriod']['startDate'] = now.isoformat() my_test_auction['auctionPeriod']['shouldStartAfter'] = (now - timedelta(days=10)).isoformat() - auction_period = check_auction(my_test_auction, self.db, self.mapper) + auction_period = create_auction_period(my_test_auction, self.db, self.mapper, 'mode') self.assertIsNone(auction_period) + +class CreateAuctionPeriodForLotsTest(BaseWebTest): + def test_check_auction_with_lots(self): now = datetime.now(TZ) @@ -89,7 +119,7 @@ def test_check_auction_with_lots(self): lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} my_test_auction['lots'] = [lot, lot] - lots = check_auction(my_test_auction, self.db, self.mapper) + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') self.assertIn('lots', lots) self.assertEqual(len(lots['lots']), 2) @@ -106,7 +136,7 @@ def test_check_auction_with_not_active_lot(self): not_active['status'] = 'pending' my_test_auction['lots'] = [lot, not_active] - lots = check_auction(my_test_auction, self.db, self.mapper) + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') self.assertIn('lots', lots) self.assertEqual(len(lots['lots'][1].keys()), 0) @@ -124,7 +154,7 @@ def test_should_start_before_auction_start(self): should_start_before['auctionPeriod']['shouldStartAfter'] = (now - timedelta(days=10)).isoformat() my_test_auction['lots'] = [lot, should_start_before] - lots = check_auction(my_test_auction, self.db, self.mapper) + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') self.assertIn('lots', lots) self.assertEqual(len(lots['lots'][1].keys()), 0) @@ -141,7 +171,7 @@ def test_should_start_absent(self): no_should_start['auctionPeriod'].pop('shouldStartAfter') my_test_auction['lots'] = [lot, no_should_start] - lots = check_auction(my_test_auction, self.db, self.mapper) + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') self.assertIn('lots', lots) self.assertEqual(len(lots['lots'][1].keys()), 0) @@ -155,7 +185,7 @@ def test_auction_start_absent(self): lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} my_test_auction['lots'] = [lot, lot] - lots = check_auction(my_test_auction, self.db, self.mapper) + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') self.assertIn('lots', lots) self.assertEqual(len(lots['lots']), 2) @@ -174,7 +204,7 @@ def test_check_auction_with_all_invalid_lots(self): not_active['status'] = 'pending' my_test_auction['lots'] = [not_active, not_active] - lots = check_auction(my_test_auction, self.db, self.mapper) + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') self.assertIsNone(lots) # should_start_after before auction @@ -184,7 +214,7 @@ def test_check_auction_with_all_invalid_lots(self): should_start_before['auctionPeriod']['shouldStartAfter'] = (now - timedelta(days=10)).isoformat() my_test_auction['lots'] = [should_start_before, should_start_before] - lots = check_auction(my_test_auction, self.db, self.mapper) + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') self.assertIsNone(lots) # should start absent @@ -193,7 +223,7 @@ def test_check_auction_with_all_invalid_lots(self): no_should_start['auctionPeriod'].pop('shouldStartAfter') my_test_auction['lots'] = [no_should_start, no_should_start] - lots = check_auction(my_test_auction, self.db, self.mapper) + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') self.assertIsNone(lots) From 79d816135cbe115c66e7a8051a6055075e8b45a1 Mon Sep 17 00:00:00 2001 From: Oleksiy Veretiuk Date: Thu, 4 Apr 2019 12:17:47 +0300 Subject: [PATCH 13/13] Add opportunity to disable patching auction period --- openprocurement/auction/databridge.py | 48 +++++++++++-------- .../bridge_utils/auctions_data_bridge.yaml | 1 + 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/openprocurement/auction/databridge.py b/openprocurement/auction/databridge.py index 002e596..c13ac0b 100644 --- a/openprocurement/auction/databridge.py +++ b/openprocurement/auction/databridge.py @@ -72,29 +72,32 @@ def __init__(self, config, re_planning=False, debug=False): ) # Stream DB configurations - db_name = os.environ.get('DB_NAME', self.config['main']['stream_db']) - couch_server = Server(self.config_get('couch_url'), session=Session(retry_delays=range(60))) + self.period_patching_enabled = self.config['main'].get('patchPeriodEnabled', False) - if db_name not in couch_server: - couch_server.create(db_name) + if self.period_patching_enabled: + couch_server = Server(self.config_get('couch_url'), session=Session(retry_delays=range(60))) + db_name = os.environ.get('DB_NAME', self.config['main']['stream_db']) - db_for_streams = urljoin( - self.config_get('couch_url'), - db_name - ) + if db_name not in couch_server: + couch_server.create(db_name) + + db_for_streams = urljoin( + self.config_get('couch_url'), + db_name + ) - self.stream_db = Database(db_for_streams, session=Session(retry_delays=range(10))) - self._set_holidays() - self._set_streams_limits() - sync_design(self.stream_db) + self.stream_db = Database(db_for_streams, session=Session(retry_delays=range(10))) + self._set_holidays() + self._set_streams_limits() + sync_design(self.stream_db) - # Managers Mapping - self.manager_mapper = {'types': {}, 'pmts': {}} - for name, plugin in self.config_get('plugins').items(): - auction_manager = MANAGERS_MAPPING[name]() - self.manager_mapper['types'][name] = auction_manager - if plugin.get('procurement_method_types', []): - self.manager_mapper['pmts'].update({pmt: auction_manager for pmt in plugin.get('procurement_method_types')}) + # Managers Mapping + self.manager_mapper = {'types': {}, 'pmts': {}} + for name, plugin in self.config_get('plugins').items(): + auction_manager = MANAGERS_MAPPING[name]() + self.manager_mapper['types'][name] = auction_manager + if plugin.get('procurement_method_types', []): + self.manager_mapper['pmts'].update({pmt: auction_manager for pmt in plugin.get('procurement_method_types')}) def _set_holidays(self): calendar = {'_id': CALENDAR_ID} @@ -126,8 +129,11 @@ def run(self): planning = self.mapper(feed) if not planning: continue - planning.add_auction_period() - planning.check_to_free_slot() + + if self.period_patching_enabled: + planning.add_auction_period() + planning.check_to_free_slot() + for cmd, item_id, lot_id in planning: if lot_id: LOGGER.info('Lot {} of tender {} selected for {}'.format( diff --git a/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml b/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml index cd36d0a..7be8e72 100644 --- a/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml +++ b/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml @@ -7,6 +7,7 @@ main: auctions_db: database timezone: Europe/Kiev stream_db : test_stream_db + patchPeriodEnabled: True streams: streams: 10 dutch_streams: 15