diff --git a/openprocurement/auction/bridge_utils/__init__.py b/openprocurement/auction/bridge_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/openprocurement/auction/bridge_utils/constants.py b/openprocurement/auction/bridge_utils/constants.py new file mode 100644 index 0000000..c320f04 --- /dev/null +++ b/openprocurement/auction/bridge_utils/constants.py @@ -0,0 +1,52 @@ +import os +from datetime import time +from pytz import timezone + +from datetime import timedelta, datetime + + +def read_json(name): + import inspect + import os.path + from json import loads + caller_file = inspect.stack()[1][1] + caller_dir = os.path.dirname(os.path.realpath(caller_file)) + file_path = os.path.join(caller_dir, name) + with open(file_path) as lang_file: + data = lang_file.read() + return loads(data) + + +STREAMS_ID = 'streams' +WORKING_DAY_START = time(11, 0) +INSIDER_WORKING_DAY_START = time(9, 30) +TEXAS_WORKING_DAY_START = time(10, 0) +WORKING_DAY_END = time(16, 0) +INSIDER_WORKING_DAY_DURATION = timedelta(minutes=480) +TEXAS_WORKING_DAY_DURATION = timedelta(hours=7) # Does not affect anything +DEFAULT_STREAMS_DOC = { + '_id': STREAMS_ID, + 'streams': 10, + 'dutch_streams': 15, + 'texas_streams': 20 +} + + +CALENDAR_ID = 'calendar' +STREAMS_ID = 'streams' +ROUNDING = timedelta(minutes=29) +MIN_PAUSE = timedelta(minutes=3) +BIDDER_TIME = timedelta(minutes=6) +SERVICE_TIME = timedelta(minutes=9) +STAND_STILL_TIME = timedelta(days=1) +SMOOTHING_MIN = 10 +SMOOTHING_REMIN = 60 +# value should be greater than SMOOTHING_MIN and SMOOTHING_REMIN +SMOOTHING_MAX = 300 +NOT_CLASSIC_AUCTIONS = ['dgfInsider', 'sellout.insider'] +STREAMS_KEYS = ['streams', 'dutch_streams', 'texas_streams'] + + +TZ = timezone(os.environ['TZ'] if 'TZ' in os.environ else 'Europe/Kiev') +# True means holiday +WORKING_DAYS = read_json('working_days.json') diff --git a/openprocurement/auction/bridge_utils/design.py b/openprocurement/auction/bridge_utils/design.py new file mode 100644 index 0000000..44b18e6 --- /dev/null +++ b/openprocurement/auction/bridge_utils/design.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +from couchdb.design import ViewDefinition +from openprocurement.auction.bridge_utils.constants import ( + INSIDER_WORKING_DAY_START, TEXAS_WORKING_DAY_START +) + + +def sync_design(db): + views = [j for i, j in globals().items() if "_view" in i] + ViewDefinition.sync_many(db, views) + + +plan_auctions_view = ViewDefinition('plan', 'auctions', '''function(doc) { + if(doc.streams || doc.dutch_streams || doc.texas_streams) { + for (var i in doc) { + if (i.indexOf('stream_') == 0) { + for (var t in doc[i]) { + if (doc[i][t]) { + var x = doc[i][t].split('_') + if (x.length == 2) { + emit(x, doc._id.split('_')[1] + 'T' + t); + } else { + emit([x[0], null], + doc._id.split('_')[1] + 'T' + t); + } + } + } + } + if (i.indexOf('dutch_streams') == 0) { + for (var aid in doc[i]) { + emit([doc[i][aid], null], + doc._id.split('_')[1] + 'T' + '%s'); + } + } + if (i.indexOf('texas_streams') == 0) { + for (var aid in doc[i]) { + emit([doc[i][aid], null], + doc._id.split('_')[1] + 'T' + '%s'); + } + } + }; + } +}''' % (INSIDER_WORKING_DAY_START.isoformat(), TEXAS_WORKING_DAY_START.isoformat())) diff --git a/openprocurement/auction/bridge_utils/managers.py b/openprocurement/auction/bridge_utils/managers.py new file mode 100644 index 0000000..73553f3 --- /dev/null +++ b/openprocurement/auction/bridge_utils/managers.py @@ -0,0 +1,237 @@ +# -*- coding: utf-8 -*- +import logging +import requests +from copy import deepcopy + +from couchdb import ResourceConflict +from datetime import timedelta, datetime +from iso8601 import parse_date + +from gevent.pool import Pool + +from openprocurement.auction.bridge_utils.constants import ( + DEFAULT_STREAMS_DOC, + TEXAS_WORKING_DAY_DURATION, + TEXAS_WORKING_DAY_START, + INSIDER_WORKING_DAY_DURATION, + INSIDER_WORKING_DAY_START, + WORKING_DAY_START, + WORKING_DAY_END, + STREAMS_ID, + TZ +) + +POOL = Pool(1) + +LOGGER = logging.getLogger('Openprocurement Auction') + +ADAPTER = requests.adapters.HTTPAdapter(pool_connections=3, pool_maxsize=3) +SESSION = requests.Session() +SESSION.mount('http://', ADAPTER) +SESSION.mount('https://', ADAPTER) + + +def find_free_slot(plan): + streams = plan.get('streams', 0) + for cur_stream in range(1, streams + 1): + stream_id = 'stream_{}'.format(cur_stream) + for slot in plan[stream_id]: + if plan[stream_id].get(slot) is None: + plan_date = parse_date(plan['_id'].split('_')[1] + 'T' + slot, None) + plan_date = plan_date.astimezone(TZ) if plan_date.tzinfo else TZ.localize(plan_date) + return plan_date, cur_stream + + +class BaseAuctionsManager(object): + working_day_start = None + streams_key = None + + def get_date(self, db, mode, date): + """ + + Get hour of auction start and stream in which auction will run for + passed date. + + :param db: chronograph database with plan docs + :type db: couchdb.Database + :param mode: value from auction procedure, indicates mode of auction, used in plan doc id + :type mode: str + :param date: preplanned date for current auction + :type date: datetime.date + + :returns : (time of auction start, stream for auction, plan document) + :rtype: (datetime.time, int, couchdb.client.Document) + """ + plan_id = 'plan{}_{}'.format(mode, date.isoformat()) + plan = db.get(plan_id, {'_id': plan_id}) + plan_date_end, stream = self._get_hours_and_stream(plan) + plan_date = parse_date(date.isoformat() + 'T' + plan_date_end, None) + plan_date = plan_date.astimezone(TZ) if plan_date.tzinfo else TZ.localize(plan_date) + return plan_date.time(), stream, plan + + def get_streams(self, db, streams_id=STREAMS_ID): + """ + + Get allowed amount of streams for auction of particular type per day. + + :param db: chronograph database + :type db: couchdb.Database + :param streams_id: id of document with defined stream amounts + :type streams_id: str + + :return: amount of streams for auction per day + :rtype: int + """ + streams = db.get(streams_id, deepcopy(DEFAULT_STREAMS_DOC)) + return streams.get(self.streams_key, DEFAULT_STREAMS_DOC[self.streams_key]) + + def _get_hours_and_stream(self, plan): + """ + Return time of auction start and first available stream from plan doc + + :param plan: document for planned auctions + :type plan: couchdb.client.Document + + :return: (time of auction start, first available stream) + :rtype: (str, int) + """ + raise NotImplementedError + + def set_end_of_auction(self, *args, **kwargs): + """ + Try to find end of auction for passed date, time and plan doc if such + action is possible and return it's value for further auction planning + """ + raise NotImplementedError + + def set_date(self, *args, **kwargs): + """ + Actually plan auction for particular date, time, stream and slot and + save it to chronograph database. + """ + raise NotImplementedError + + def free_slot(self, *args, **kwargs): + """ + Remove particular auction from stream slot in particular plan document. + """ + raise NotImplementedError + + +class ClassicAuctionsManager(BaseAuctionsManager): + working_day_start = WORKING_DAY_START + working_day_end = WORKING_DAY_END + streams_key = 'streams' + + def _get_hours_and_stream(self, plan): + plan_date_end = plan.get('time', self.working_day_start.isoformat()) + stream = plan.get(self.streams_key, 1) + return plan_date_end, stream + + def set_date(self, db, plan, auction_id, end_time, + cur_stream, start_time, new_slot=True): + if new_slot: + plan['time'] = end_time.isoformat() + plan[self.streams_key] = cur_stream + stream_id = 'stream_{}'.format(cur_stream) + stream = plan.get(stream_id, {}) + stream[start_time.isoformat()] = auction_id + plan[stream_id] = stream + db.save(plan) + + def free_slot(self, db, plan_id, auction_id, plan_time): + slot = plan_time.time().isoformat() + done = False + while not done: + try: + plan = db.get(plan_id) + streams = plan[self.streams_key] + for cur_stream in range(1, streams + 1): + stream_id = 'stream_{}'.format(cur_stream) + if plan[stream_id].get(slot) == auction_id: + plan[stream_id][slot] = None + db.save(plan) + done = True + except ResourceConflict: + done = False + except: + done = True + + def set_end_of_auction(self, stream, streams, nextDate, dayStart, plan): + freeSlot = find_free_slot(plan) + if freeSlot: + startDate, stream = freeSlot + start, end, dayStart, new_slot = startDate, startDate, startDate.time(), False + return start, end, dayStart, stream, new_slot + if dayStart >= self.working_day_end and stream < streams: + stream += 1 + dayStart = self.working_day_start + + start = TZ.localize(datetime.combine(nextDate, dayStart)) + end = start + timedelta(minutes=30) + + if end <= TZ.localize(datetime.combine(nextDate, self.working_day_end)) and stream <= streams: + return start, end, dayStart, stream, True + + +class NonClassicAuctionsManager(BaseAuctionsManager): + working_day_start = None + working_day_duration = None + streams_key = None + + def _get_hours_and_stream(self, plan): + plan_date_end = self.working_day_start.isoformat() + stream = len(plan.get(self.streams_key, [])) + return plan_date_end, stream + + def set_date(self, db, plan, auction_id, *args, **kwargs): + streams = plan.get(self.streams_key, []) + streams.append(auction_id) + plan[self.streams_key] = streams + db.save(plan) + + def free_slot(self, db, plan_id, auction_id, *args, **kwargs): + done = False + while not done: + try: + plan = db.get(plan_id) + slots = plan.get(self.streams_key, []) + pops = [] + for i in xrange(0, len(slots)): + if slots[i] == auction_id: + pops.append(i) + pops.sort(reverse=True) + for p in pops: + slots.pop(p) + plan[self.streams_key] = slots + db.save(plan) + done = True + except ResourceConflict: + done = False + except: + done = True + + def set_end_of_auction(self, stream, streams, nextDate, dayStart, *args, **kwargs): + if stream < streams: + start = TZ.localize(datetime.combine(nextDate, dayStart)) + end = start + self.working_day_duration + return start, end, dayStart, stream, False + + +class TexasAuctionsManager(NonClassicAuctionsManager): + working_day_start = TEXAS_WORKING_DAY_START + working_day_duration = TEXAS_WORKING_DAY_DURATION + streams_key = 'texas_streams' + + +class InsiderAuctionsManager(NonClassicAuctionsManager): + working_day_start = INSIDER_WORKING_DAY_START + working_day_duration = INSIDER_WORKING_DAY_DURATION + streams_key = 'dutch_streams' + + +MANAGERS_MAPPING = { + 'texas': TexasAuctionsManager, + 'dutch': InsiderAuctionsManager, + 'english': ClassicAuctionsManager +} diff --git a/openprocurement/auction/bridge_utils/utils.py b/openprocurement/auction/bridge_utils/utils.py new file mode 100644 index 0000000..56e9464 --- /dev/null +++ b/openprocurement/auction/bridge_utils/utils.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +import os +import logging +import requests +from random import randint + +from couchdb import ResourceConflict +from datetime import timedelta, datetime +from iso8601 import parse_date + + +from openprocurement.auction.bridge_utils.constants import ( + TZ, + BIDDER_TIME, + SERVICE_TIME, + MIN_PAUSE, + ROUNDING, + WORKING_DAY_START, + CALENDAR_ID +) +from openprocurement.auction.bridge_utils.design import plan_auctions_view + + +LOGGER = logging.getLogger('Openprocurement Auction') + +ADAPTER = requests.adapters.HTTPAdapter(pool_connections=3, pool_maxsize=3) +SESSION = requests.Session() +SESSION.mount('http://', ADAPTER) +SESSION.mount('https://', ADAPTER) + + +def update_logging_context(request, params): + if not request.__dict__.get('logging_context'): + request.logging_context = {} + + for x, j in params.items(): + request.logging_context[x.upper()] = j + + +def context_unpack(request, msg, params=None): + if params: + update_logging_context(request, params) + logging_context = request.logging_context + journal_context = msg + for key, value in logging_context.items(): + journal_context["JOURNAL_" + key] = value + return journal_context + + +def get_now(): + return TZ.localize(datetime.now()) + + +def randomize(dt): + return dt + timedelta(seconds=randint(0, 1799)) + + +def get_manager_for_auction(auction, mapper): + default_manager = mapper['types'].get('english', None) + + auction_type = auction.get('auctionParameters', {}).get('type', None) + if auction_type: + return mapper['types'].get(auction_type, default_manager) + else: + pmt = auction.get('procurementMethodType') + return mapper['pmts'].get(pmt, default_manager) + + +def calc_auction_end_time(bids, start): + end = start + bids * BIDDER_TIME + SERVICE_TIME + MIN_PAUSE + seconds = (end - TZ.localize(datetime.combine(end, WORKING_DAY_START))).seconds + roundTo = ROUNDING.seconds + rounding = (seconds + roundTo - 1) // roundTo * roundTo + return (end + timedelta(0, rounding - seconds, -end.microsecond)).astimezone(TZ) + + +def get_calendar(db, calendar_id=CALENDAR_ID): + return db.get(calendar_id, {'_id': calendar_id}) + + +def skipped_days(days): + days_str = '' + if days: + days_str = ' Skipped {} full days.'.format(days) + return days_str + + +def planning_auction(auction, mapper, start, db, quick=False, lot_id=None): + tid = auction.get('id', '') + mode = auction.get('mode', '') + manager = get_manager_for_auction(auction, mapper) + skipped_days = 0 + if quick: + quick_start = calc_auction_end_time(0, start) + return quick_start, 0, skipped_days + calendar = get_calendar(db) + streams = manager.get_streams(db) + start += timedelta(hours=1) + if start.time() > manager.working_day_start: + nextDate = start.date() + timedelta(days=1) + else: + nextDate = start.date() + while True: + # skip Saturday and Sunday + if calendar.get(nextDate.isoformat()) or nextDate.weekday() in [5, 6]: + nextDate += timedelta(days=1) + continue + dayStart, stream, plan = manager.get_date(db, mode, nextDate) + result = manager.set_end_of_auction(stream, streams, nextDate, dayStart, plan) + if result: + start, end, dayStart, stream, new_slot = result + break + nextDate += timedelta(days=1) + skipped_days += 1 + manager.set_date(db, plan, "_".join([tid, lot_id]) if lot_id else tid, end.time(), stream, dayStart, new_slot) + return start, stream, skipped_days + + +def create_auction_period(auction, db, mapper, quick): + now = get_now() + + period = auction.get('auctionPeriod', {}) + shouldStartAfter = parse_date(period['shouldStartAfter'], TZ).astimezone(TZ) if period.get('shouldStartAfter') else None + start_date = parse_date(period['startDate'], TZ).astimezone(TZ) if period.get('startDate') else None + + is_needed_to_add_auction_period = bool( + shouldStartAfter and + ( + start_date is None or + shouldStartAfter > start_date + ) + ) + + if shouldStartAfter and is_needed_to_add_auction_period: + shouldStartAfter = max(shouldStartAfter, now) + planned = False + while not planned: + try: + auctionPeriod, stream, skip_days = planning_auction(auction, mapper, shouldStartAfter, db, quick) + planned = True + except ResourceConflict: + planned = False + auctionPeriod = randomize(auctionPeriod).isoformat() + planned = 'replanned' if period.get('startDate') else 'planned' + LOGGER.info( + '{} auction for auction {} to {}. Stream {}.{}'.format( + planned.title(), auction['id'], auctionPeriod, stream, skipped_days(skip_days) + ), + extra={ + 'MESSAGE_ID': '{}_auction_auction'.format(planned), + 'PLANNED_DATE': auctionPeriod, + 'PLANNED_STREAM': stream, + 'PLANNED_DAYS_SKIPPED': skip_days} + ) + return {'auctionPeriod': {'startDate': auctionPeriod}} + + return None + + +def create_auction_period_for_lots(auction, db, mapper, quick): + now = get_now() + + lots = [] + for lot in auction.get('lots', []): + period = lot.get('auctionPeriod', {}) + shouldStartAfter = parse_date(period['shouldStartAfter'], TZ).astimezone(TZ) if period.get('shouldStartAfter') else None + start_date = parse_date(period['startDate'], TZ).astimezone(TZ) if period.get('startDate') else None + + skip_lot = bool( + shouldStartAfter is None or ( + start_date is not None and + start_date > shouldStartAfter + ) + ) + + if lot['status'] != 'active' or skip_lot: + lots.append({}) + continue + period = lot.get('auctionPeriod') + shouldStartAfter = max(shouldStartAfter, now) + lot_id = lot['id'] + planned = False + while not planned: + try: + auctionPeriod, stream, skip_days = planning_auction(auction, mapper, shouldStartAfter, db, quick, + lot_id) + planned = True + except ResourceConflict: + planned = False + auctionPeriod = randomize(auctionPeriod).isoformat() + planned = 'replanned' if period.get('startDate') else 'planned' + lots.append({'auctionPeriod': {'startDate': auctionPeriod}}) + LOGGER.info( + '{} auction for lot {} of auction {} to {}. Stream {}.{}'.format( + planned.title(), lot_id, auction['id'], auctionPeriod, stream, skipped_days(skip_days) + ), + extra={ + 'MESSAGE_ID': '{}_auction_lot'.format(planned), + 'PLANNED_DATE': auctionPeriod, + 'PLANNED_STREAM': stream, + 'PLANNED_DAYS_SKIPPED': skip_days, + 'LOT_ID': lot_id + } + ) + if any(lots): + return {'lots': lots} + + return None + + +def check_auction(auction, db, mapper): + quick = os.environ.get('SANDBOX_MODE', False) and u'quick' in auction.get('submissionMethodDetails', '') + + if not auction.get('lots'): + return create_auction_period(auction, db, mapper, quick) + + elif auction.get('lots'): + return create_auction_period_for_lots(auction, db, mapper, quick) + + +def check_inner_auction(db, auction, mapper): + manager = get_manager_for_auction(auction, mapper) + + auction_time = auction.get('auctionPeriod', {}).get('startDate') and \ + parse_date(auction.get('auctionPeriod', {}).get('startDate')) + lots = dict([ + (i['id'], parse_date(i.get('auctionPeriod', {}).get('startDate'))) + for i in auction.get('lots', []) + if i.get('auctionPeriod', {}).get('startDate') + ]) + auc_list = [ + (x.key[1], TZ.localize(parse_date(x.value, None)), x.id) + for x in plan_auctions_view(db, startkey=[auction['id'], None], + endkey=[auction['id'], 32 * "f"]) + ] + for key, plan_time, plan_doc in auc_list: + if not key and (not auction_time or not + plan_time < auction_time < plan_time + + timedelta(minutes=30)): + manager.free_slot(db, plan_doc, auction['id'], plan_time) + elif key and (not lots.get(key) or lots.get(key) and not + plan_time < lots.get(key) < plan_time + + timedelta(minutes=30)): + manager.free_slot(db, plan_doc, "_".join([auction['id'], key]), plan_time) diff --git a/openprocurement/auction/bridge_utils/working_days.json b/openprocurement/auction/bridge_utils/working_days.json new file mode 100644 index 0000000..654eee2 --- /dev/null +++ b/openprocurement/auction/bridge_utils/working_days.json @@ -0,0 +1,64 @@ +{ + "2015-05-01": true, + "2015-05-04": true, + "2015-05-11": true, + "2015-06-01": true, + "2015-06-29": true, + "2015-08-24": true, + "2015-10-14": true, + "2016-01-01": true, + "2016-01-07": true, + "2016-01-08": true, + "2016-03-07": true, + "2016-03-08": true, + "2016-05-02": true, + "2016-05-03": true, + "2016-05-09": true, + "2016-06-20": true, + "2016-06-27": true, + "2016-06-28": true, + "2016-08-24": true, + "2016-10-14": true, + "2017-01-02": true, + "2017-01-09": true, + "2017-03-08": true, + "2017-04-17": true, + "2017-05-01": true, + "2017-05-02": true, + "2017-05-08": true, + "2017-05-09": true, + "2017-06-05": true, + "2017-06-28": true, + "2017-08-24": true, + "2017-08-25": true, + "2017-10-16": true, + "2018-01-01": true, + "2018-01-08": true, + "2018-03-08": true, + "2018-04-09": true, + "2018-05-01": true, + "2018-05-08": true, + "2018-05-09": true, + "2018-05-28": true, + "2018-06-28": true, + "2018-06-29": true, + "2018-08-24": true, + "2018-10-15": true, + "2018-12-24": true, + "2018-12-25": true, + "2018-12-31": true, + "2019-01-01": true, + "2019-01-07": true, + "2019-03-08": true, + "2019-04-29": true, + "2019-04-30": true, + "2019-05-01": true, + "2019-05-09": true, + "2019-06-17": true, + "2019-06-28": true, + "2019-08-26": true, + "2019-10-14": true, + "2019-12-25": true, + "2019-12-30": true, + "2019-12-31": true +} diff --git a/openprocurement/auction/databridge.py b/openprocurement/auction/databridge.py index 2339a73..c13ac0b 100644 --- a/openprocurement/auction/databridge.py +++ b/openprocurement/auction/databridge.py @@ -1,8 +1,10 @@ from gevent import monkey + monkey.patch_all() try: import urllib3.contrib.pyopenssl + urllib3.contrib.pyopenssl.inject_into_urllib3() except ImportError: pass @@ -14,15 +16,18 @@ from urlparse import urljoin from zope.interface import implementer from yaml import load -from couchdb import Database, Session +from couchdb import Database, Session, Server from dateutil.tz import tzlocal from openprocurement_client.sync import ResourceFeeder -from openprocurement.auction.interfaces import\ +from openprocurement.auction.interfaces import \ IAuctionDatabridge, IAuctionsManager +from openprocurement.auction.bridge_utils.design import sync_design +from openprocurement.auction.bridge_utils.managers import MANAGERS_MAPPING +from openprocurement.auction.bridge_utils.constants import WORKING_DAYS, CALENDAR_ID, STREAMS_ID + from openprocurement.auction.core import components from openprocurement.auction.utils import FeedItem, check_workers - LOGGER = logging.getLogger(__name__) API_EXTRA = {'opt_fields': 'status,auctionPeriod,lots,procurementMethodType', 'mode': '_all_'} @@ -38,7 +43,6 @@ @implementer(IAuctionDatabridge) class AuctionsDataBridge(object): - """Auctions Data Bridge""" def __init__(self, config, re_planning=False, debug=False): @@ -67,6 +71,50 @@ def __init__(self, config, re_planning=False, debug=False): retrievers_params=DEFAULT_RETRIEVERS_PARAMS ) + # Stream DB configurations + self.period_patching_enabled = self.config['main'].get('patchPeriodEnabled', False) + + if self.period_patching_enabled: + couch_server = Server(self.config_get('couch_url'), session=Session(retry_delays=range(60))) + db_name = os.environ.get('DB_NAME', self.config['main']['stream_db']) + + if db_name not in couch_server: + couch_server.create(db_name) + + db_for_streams = urljoin( + self.config_get('couch_url'), + db_name + ) + + self.stream_db = Database(db_for_streams, session=Session(retry_delays=range(10))) + self._set_holidays() + self._set_streams_limits() + sync_design(self.stream_db) + + # Managers Mapping + self.manager_mapper = {'types': {}, 'pmts': {}} + for name, plugin in self.config_get('plugins').items(): + auction_manager = MANAGERS_MAPPING[name]() + self.manager_mapper['types'][name] = auction_manager + if plugin.get('procurement_method_types', []): + self.manager_mapper['pmts'].update({pmt: auction_manager for pmt in plugin.get('procurement_method_types')}) + + def _set_holidays(self): + calendar = {'_id': CALENDAR_ID} + calendar.update(WORKING_DAYS) + if CALENDAR_ID in self.stream_db: + del self.stream_db[CALENDAR_ID] + self.stream_db.save(calendar) + + def _set_streams_limits(self): + streams = self.config.get('main').get('streams', {}) + + stream_amount = {'_id': STREAMS_ID} + stream_amount.update(streams) + if STREAMS_ID in self.stream_db: + del self.stream_db[STREAMS_ID] + self.stream_db.save(stream_amount) + def config_get(self, name): return self.config.get('main').get(name) @@ -81,6 +129,11 @@ def run(self): planning = self.mapper(feed) if not planning: continue + + if self.period_patching_enabled: + planning.add_auction_period() + planning.check_to_free_slot() + for cmd, item_id, lot_id in planning: if lot_id: LOGGER.info('Lot {} of tender {} selected for {}'.format( diff --git a/openprocurement/auction/plannings.py b/openprocurement/auction/plannings.py index 1b9a5dc..f5abdc0 100644 --- a/openprocurement/auction/plannings.py +++ b/openprocurement/auction/plannings.py @@ -1,7 +1,9 @@ import logging import iso8601 +from json import dumps +from urlparse import urljoin -from datetime import datetime +from datetime import datetime, timedelta, time from time import mktime, time from gevent.subprocess import check_call @@ -21,6 +23,12 @@ from openprocurement.auction.utils import ( do_until_success, prepare_auction_worker_cmd ) +from openprocurement.auction.bridge_utils.utils import ( + check_auction, + context_unpack, + SESSION, + check_inner_auction +) SIMPLE_AUCTION_TYPE = 0 @@ -38,6 +46,39 @@ def __init__(self, bridge, item): def next(self): return self + def add_auction_period(self): + url = urljoin( + self.bridge.config_get('resource_api_server'), + '/'.join( + [ + 'api', + self.bridge.config_get('resource_api_version'), + self.bridge.config_get('resource_name'), + self.item.id + ], + ) + ) + LOGGER.info(url) + api_token = self.bridge.config_get('resource_api_token') + db = self.bridge.stream_db + changes = check_auction(self.item, db, self.bridge.manager_mapper) + if changes: + data = dumps({'data': changes}) + r = SESSION.patch(url, + data=data, + headers={'Content-Type': 'application/json'}, + auth=(api_token, '')) + if r.status_code != 200: + LOGGER.error( + "Error {} on updating auction '{}' with '{}': {}".format(r.status_code, url, data, r.text), + extra=context_unpack(r, {'MESSAGE_ID': 'error_patch_auction'}, + {'ERROR_STATUS': r.status_code})) + else: + LOGGER.info("Successfully updated auction '{}' with '{}'".format(r.status_code, url, data)) + + def check_to_free_slot(self): + check_inner_auction(self.bridge.stream_db, self.item, self.bridge.manager_mapper) + def __iter__(self): if self.item['status'] == "active.auction": if 'lots' not in self.item and 'auctionPeriod' in self.item \ diff --git a/openprocurement/auction/tests/bridge_utils/__init__.py b/openprocurement/auction/tests/bridge_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml b/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml new file mode 100644 index 0000000..7be8e72 --- /dev/null +++ b/openprocurement/auction/tests/bridge_utils/auctions_data_bridge.yaml @@ -0,0 +1,68 @@ +main: + resource_api_server: http://127.0.0.1:6543 + resource_api_version: "2.3" + resource_api_token: auction + resource_name: auctions + couch_url: http://admin:zaq1xsw2@0.0.0.0:9000/ + auctions_db: database + timezone: Europe/Kiev + stream_db : test_stream_db + patchPeriodEnabled: True + streams: + streams: 10 + dutch_streams: 15 + texas_streams: 20 + plugins: + english: + auction_worker_config: /path/to/auction_worker_defaults.yaml + procurement_method_types: [DGFOtherAssets, dgfFinancialAssets] + auction_worker: /path/to/auction_worker + dutch: + auction_worker_config: /path/to/auction_worker_insider.yaml + procurement_method_types: [dgfInsider] + auction_worker: /path/to/auction_insider + texas: + auction_worker_config: /path/to/auction_worker_texas.yaml + procurement_method_types: [landLease] + auction_worker: /path/to/auction_texas + retrievers_params: + down_requests_sleep: 0.1 + up_requests_sleep: 0.1 + up_wait_sleep: 30 + up_wait_sleep_min: 5 + queue_size: 501 +version: 1 +disable_existing_loggers: false + +formatters: + simple: + format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + +handlers: + console: + class: logging.StreamHandler + level: ERROR + formatter: simple + stream: ext://sys.stdout + + journal: + class: ExtendedJournalHandler.ExtendedJournalHandler + level: INFO + SYSLOG_IDENTIFIER: AUCTIONS_DATA_BRIDGE + TENDERS_API_VERSION: "0" + TENDERS_API_URL: https://lb.api-sandbox.ea.openprocurement.org/ + +loggers: + openprocurement.auction.databridge: + handlers: [console, journal] + propagate: no + level: INFO + + openprocurement_client.sync: + handlers: [console, journal] + propagate: no + level: INFO + + "": + handlers: [console, journal] + level: INFO \ No newline at end of file diff --git a/openprocurement/auction/tests/bridge_utils/base.py b/openprocurement/auction/tests/bridge_utils/base.py new file mode 100644 index 0000000..f8ad6e9 --- /dev/null +++ b/openprocurement/auction/tests/bridge_utils/base.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +import os +import unittest +import mock + +from yaml import load +from couchdb import Server + +from openprocurement.auction.databridge import AuctionsDataBridge + + +class BaseWebTest(unittest.TestCase): + """Base Web Test to test openprocurement.api. + + It setups the database before each test and delete it after. + """ + + def setUp(self): + dir_path = os.path.dirname(os.path.realpath(__file__)) + + with open(dir_path + '/auctions_data_bridge.yaml') as config_file_obj: + config = load(config_file_obj.read()) + + with mock.patch('openprocurement.auction.databridge.check_workers'): + bridge = AuctionsDataBridge(config) + + self.couchdb_server = Server(config['main'].get('couch_url')) + self.db = bridge.stream_db + self.mapper = bridge.manager_mapper + + def tearDown(self): + try: + del self.couchdb_server[self.db.name] + except: + pass diff --git a/openprocurement/auction/tests/bridge_utils/data.py b/openprocurement/auction/tests/bridge_utils/data.py new file mode 100644 index 0000000..cf00f0f --- /dev/null +++ b/openprocurement/auction/tests/bridge_utils/data.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +from datetime import timedelta + +from openprocurement.auction.bridge_utils.utils import get_now + +now = get_now() + +test_auction_data = { + "auctionID": "UA-EA-{}-000001".format(now.strftime('%Y-%m-%d')), + "awardCriteria": "highestCost", + "date": now.isoformat(), + "dateModified": now.isoformat(), + "id": "f547ece35436484e8656a2988fb52a44", + "status": "active.enquiries", + "submissionMethod": "electronicAuction", + "title": u"футляри до державних нагород", + "procuringEntity": { + "name": u"Державне управління справами", + "identifier": { + "scheme": u"UA-EDR", + "id": u"00037256", + "uri": u"http://www.dus.gov.ua/" + }, + "address": { + "countryName": u"Україна", + "postalCode": u"01220", + "region": u"м. Київ", + "locality": u"м. Київ", + "streetAddress": u"вул. Банкова, 11, корпус 1" + }, + "contactPoint": { + "name": u"Державне управління справами", + "telephone": u"0440000000" + } + }, + "next_check": (now + timedelta(days=7)).isoformat(), + "owner": "test", + "procurementMethod": "open", + "value": { + "amount": 500, + "currency": u"UAH", + "valueAddedTaxIncluded": True + }, + "minimalStep": { + "amount": 35, + "currency": u"UAH", + "valueAddedTaxIncluded": True + }, + "items": [ + { + "description": u"футляри до державних нагород", + "classification": { + "scheme": u"CAV", + "id": u"70122000-2", + "description": u"Cartons" + }, + "additionalClassifications": [ + { + "scheme": u"ДКПП", + "id": u"17.21.1", + "description": u"папір і картон гофровані, паперова й картонна тара" + } + ], + "unit": { + "name": u"item", + "code": u"44617100-9" + }, + "quantity": 5, + "id": "181f08dbe3944325a1bb272e756f04cb" + + } + ], + "enquiryPeriod": { + "endDate": (now + timedelta(days=7)).isoformat(), + "startDate": now.isoformat() + }, + "tenderPeriod": { + "startDate": (now + timedelta(days=7)).isoformat(), + "endDate": (now + timedelta(days=14)).isoformat() + } +} + +test_lots = [ + { + "title": "lot title", + "description": "lot description", + "value": test_auction_data["value"], + "minimalStep": test_auction_data["minimalStep"], + "id": "20e0201d81af4aaeb33546f04744d493", + "status": "active" + } +] + +test_bids = [ + { + "id": "1d1caac9baa7445f90ca1ba06a1c605c", + "owner": "test", + "status": "active", + "tenderers": [ + test_auction_data["procuringEntity"] + ], + "value": { + "amount": 469, + "currency": "UAH", + "valueAddedTaxIncluded": True + } + }, + { + "id": "cc15a3d3ad204143abe1ea0dd69a108e", + "owner": "test", + "status": "active", + "tenderers": [ + test_auction_data["procuringEntity"] + ], + "value": { + "amount": 479, + "currency": "UAH", + "valueAddedTaxIncluded": True + } + } +] + +plantest = { + "_id": "plantest_2017-10-03", + "dutch_streams": [ + "01fa8a7dc4b8eac3b5820747efc6fe36", + "442000d99bf203ddfd62b3be58350383", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "da8a28ed2bdf73ee1d373e4cadfed4c5", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "442000d99bf203ddfd62b3be58350383", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "da8a28ed2bdf73ee1d373e4cadfed4c5", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "442000d99bf203ddfd62b3be58350383", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "da8a28ed2bdf73ee1d373e4cadfed4c5", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "01fa8a7dc4b8eac3b5820747efc6fe36" + ], + "texas_streams": [ + "01fa8a7dc4b8eac3b5820747efc6fe36", + "442000d99bf203ddfd62b3be58350383", + "dc3d950743304d05adaa1cd5b0475075", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "da8a28ed2bdf73ee1d373e4cadfed4c5", + "dc3d950743304d05adaa1cd5b0475075", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "442000d99bf203ddfd62b3be58350383", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "dc3d950743304d05adaa1cd5b0475075", + "da8a28ed2bdf73ee1d373e4cadfed4c5", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "442000d99bf203ddfd62b3be58350383", + "dc3d950743304d05adaa1cd5b0475075", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "da8a28ed2bdf73ee1d373e4cadfed4c5", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "dc3d950743304d05adaa1cd5b0475075", + "01fa8a7dc4b8eac3b5820747efc6fe36", + "01fa8a7dc4b8eac3b5820747efc6fe36" + "dc3d950743304d05adaa1cd5b0475075" + ], + "stream_1": { + "12:00:00": "01fa8a7dc4b8eac3b5820747efc6fe36", + "12:30:00": "da8a28ed2bdf73ee1d373e4cadfed4c5_1c2fb1e496b317b2b87e197e2332da77", + "13:00:00": "e51508cddc2c490005eaecb73c006b72", + "13:30:00": "e51508cddc2c490005eaecb73c006b72", + "11:30:00": "e51508cddc2c490005eaecb73c006b72", + "11:00:00": "01fa8a7dc4b8eac3b5820747efc6fe36", + "15:30:00": "e51508cddc2c490005eaecb73c006b72", + "15:00:00": "da8a28ed2bdf73ee1d373e4cadfed4c5_1c2fb1e496b317b2b87e197e2332da77", + "14:00:00": "da8a28ed2bdf73ee1d373e4cadfed4c5_b10f9f7f26157ae2f349be8dc2106d6e", + "14:30:00": "01fa8a7dc4b8eac3b5820747efc6fe36" + }, + "stream_2": { + "12:00:00": "01fa8a7dc4b8eac3b5820747efc6fe36", + "12:30:00": "da8a28ed2bdf73ee1d373e4cadfed4c5_1c2fb1e496b317b2b87e197e2332da77", + "13:00:00": "e51508cddc2c490005eaecb73c006b72", + "13:30:00": "01fa8a7dc4b8eac3b5820747efc6fe36", + "11:30:00": "da8a28ed2bdf73ee1d373e4cadfed4c5_b10f9f7f26157ae2f349be8dc2106d6e", + "11:00:00": "e51508cddc2c490005eaecb73c006b72", + "15:30:00": "e51508cddc2c490005eaecb73c006b72", + "15:00:00": "01fa8a7dc4b8eac3b5820747efc6fe36", + "14:00:00": "da8a28ed2bdf73ee1d373e4cadfed4c5_1c2fb1e496b317b2b87e197e2332da77", + "14:30:00": "01fa8a7dc4b8eac3b5820747efc6fe36" + }, + "streams": 2 +} diff --git a/openprocurement/auction/tests/bridge_utils/test_managers.py b/openprocurement/auction/tests/bridge_utils/test_managers.py new file mode 100644 index 0000000..518dc4f --- /dev/null +++ b/openprocurement/auction/tests/bridge_utils/test_managers.py @@ -0,0 +1,389 @@ +# -*- coding: utf-8 -*- +import mock +import unittest + +from copy import deepcopy +from datetime import datetime, timedelta, time +from iso8601 import parse_date + +from openprocurement.auction.bridge_utils.constants import TZ +from openprocurement.auction.bridge_utils.managers import ( + find_free_slot, + ClassicAuctionsManager, + InsiderAuctionsManager, + TexasAuctionsManager +) +from openprocurement.auction.tests.bridge_utils.base import BaseWebTest +from openprocurement.auction.tests.bridge_utils.data import plantest + + +class FindFreeSlotTest(BaseWebTest): + + def test_find_free_slot(self): + plan = deepcopy(plantest) + plan['streams'] = 1 + plan['stream_1']['12:00:00'] = None + plan_date, cur_stream = find_free_slot(plan) + + expected_date = parse_date('2017-10-03T12:00:00', None) + expected_date = expected_date.astimezone(TZ) if expected_date.tzinfo else TZ.localize(expected_date) + + self.assertEqual(plan_date, expected_date) + self.assertEqual(cur_stream, 1) + + def test_no_slot_founded(self): + plan = deepcopy(plantest) + plan['streams'] = 1 + plan['stream_1']['12:00:00'] = None + result = find_free_slot(plantest) + self.assertIsNone(result) + + +class TestClassicManager(BaseWebTest): + + def setUp(self): + super(TestClassicManager, self).setUp() + self.manager = ClassicAuctionsManager() + + def test_get_date(self): + date = datetime.now().date() + plan_date = parse_date(date.isoformat() + 'T' + self.manager.working_day_start.isoformat(), None) + plan_date = plan_date.astimezone(TZ) if plan_date.tzinfo else TZ.localize(plan_date) + + db = mock.MagicMock() + + db.get.return_value = plantest + now = datetime.now() + + plan_time, stream, plan = self.manager.get_date(db, '', now.date()) + + self.assertEqual(plan, plantest) + self.assertEqual(plan_time, plan_date.time()) + self.assertEqual(stream, plantest['streams']) + + def test_get_hours_and_stream(self): + plan_date_end, streams = self.manager._get_hours_and_stream(plantest) + self.assertEqual(plan_date_end, self.manager.working_day_start.isoformat()) + self.assertEqual(streams, plantest['streams']) + + def test_set_date(self): + db = mock.MagicMock() + old_plan = {'time': 'old', 'streams': 3} + plan = deepcopy(old_plan) + auction_id = '1' * 32 + end_time = datetime.now() + cur_stream = 1 + start_time = datetime.now() + stream_id = 'stream_{}'.format(cur_stream) + + self.manager.set_date( + db, plan, auction_id, end_time, + cur_stream, start_time, False + ) + self.assertIn(stream_id, plan) + stream = plan[stream_id] + self.assertIn(start_time.isoformat(), stream) + self.assertEqual(stream[start_time.isoformat()], auction_id) + self.assertEqual(plan['time'], old_plan['time']) + self.assertEqual(plan['streams'], old_plan['streams']) + + db.save.assert_called_with(plan) + + def test_date_with_new_slot(self): + db = mock.MagicMock() + old_plan = {'time': 'old', 'streams': 3} + plan = deepcopy(old_plan) + auction_id = '1' * 32 + end_time = datetime.now() + timedelta(days=1) + cur_stream = 1 + start_time = datetime.now() + stream_id = 'stream_{}'.format(cur_stream) + + self.manager.set_date( + db, plan, auction_id, end_time, + cur_stream, start_time, True + ) + self.assertIn(stream_id, plan) + stream = plan[stream_id] + self.assertIn(start_time.isoformat(), stream) + self.assertEqual(stream[start_time.isoformat()], auction_id) + self.assertEqual(plan['time'], end_time.isoformat()) + self.assertEqual(plan['streams'], cur_stream) + + db.save.assert_called_with(plan) + + def test_free_slot(self): + db = mock.MagicMock() + auction_id = '1' * 32 + plan_time = datetime.now().replace(hour=12, minute=0, second=0, microsecond=0) + + plan = { + 'stream_1': { + plan_time.time().isoformat(): auction_id + }, + 'streams': 1 + } + plan_id = 'plan_id' + db.get.return_value = plan + + self.manager.free_slot(db, plan_id, auction_id, plan_time) + + self.assertIsNone(plan['stream_1'][plan_time.time().isoformat()]) + db.get.assert_called_with(plan_id) + db.save.assert_called_with(plan) + + def test_free_slot_for_wrong_auction_id(self): + db = mock.MagicMock() + auction_id = '1' * 32 + plan_time = datetime.now().replace(hour=12, minute=0, second=0, microsecond=0) + wrong_auction_id = '2' * 32 + + plan = { + 'stream_1': { + plan_time.time().isoformat(): auction_id + }, + 'streams': 1 + } + plan_id = 'plan_id' + db.get.return_value = plan + + self.manager.free_slot(db, plan_id, wrong_auction_id, plan_time) + + self.assertEqual(plan['stream_1'][plan_time.time().isoformat()], auction_id) + db.get.assert_called_with(plan_id) + db.save.assert_called_with(plan) + + def test_set_end_of_auction_with_free_slot(self): + stream = 1 + streams = 3 + next_date = datetime.now() + day_start = time(10, 0) + + plan_time = datetime.now().replace(hour=12, minute=0, second=0, microsecond=0) + plan = { + '_id': 'plantest_2017-10-03', + 'stream_1': { + plan_time.time().isoformat(): None + }, + 'stream_2': {}, + 'stream_3': {}, + 'streams': 3 + } + expected_date = parse_date('2017-10-03T12:00:00', None) + expected_date = expected_date.astimezone(TZ) if expected_date.tzinfo else TZ.localize(expected_date) + + start, end, auction_day_start, auction_stream, new_slot = self.manager.set_end_of_auction( + stream, streams, next_date, day_start, plan + ) + self.assertEqual(start, expected_date) + self.assertEqual(end, expected_date) + self.assertEqual(auction_day_start, expected_date.time()) + self.assertEqual(auction_stream, stream) + self.assertFalse(new_slot) + + def test_set_end_of_auction_without_free_slot(self): + stream = 1 + streams = 3 + next_date = datetime.now().date() + day_start = time(8, 0) + + plan = { + '_id': 'plantest_2017-10-03', + 'stream_1': {}, + 'stream_2': {}, + 'stream_3': {}, + 'streams': 3 + } + + start, end, auction_day_start, auction_stream, new_slot = self.manager.set_end_of_auction( + stream, streams, next_date, day_start, plan + ) + + expected_start = TZ.localize(datetime.combine(next_date, day_start)) + + self.assertEqual(start, expected_start) + self.assertEqual(end, expected_start + timedelta(minutes=30)) + self.assertEqual(auction_day_start, day_start) + self.assertEqual(auction_stream, stream) + self.assertTrue(new_slot) + + def test_set_end_of_auction_without_free_slot_and_wrong_day_start(self): + stream = 1 + streams = 3 + next_date = datetime.now().date() + day_start = time(16, 0) + + plan = { + '_id': 'plantest_2017-10-03', + 'stream_1': {}, + 'stream_2': {}, + 'stream_3': {}, + 'streams': 3 + } + + start, end, auction_day_start, auction_stream, new_slot = self.manager.set_end_of_auction( + stream, streams, next_date, day_start, plan + ) + + expected_start = TZ.localize(datetime.combine(next_date, self.manager.working_day_start)) + + self.assertEqual(start, expected_start) + self.assertEqual(end, expected_start + timedelta(minutes=30)) + self.assertEqual(auction_day_start, self.manager.working_day_start) + self.assertEqual(auction_stream, stream + 1) + self.assertTrue(new_slot) + + def test_set_end_of_auction_without_free_slot_and_limit_stream(self): + stream = 3 + streams = 3 + next_date = datetime.now().date() + day_start = time(8, 0) + + plan = { + '_id': 'plantest_2017-10-03', + 'stream_1': {}, + 'stream_2': {}, + 'stream_3': {}, + 'streams': 3 + } + + start, end, auction_day_start, auction_stream, new_slot = self.manager.set_end_of_auction( + stream, streams, next_date, day_start, plan + ) + + expected_start = TZ.localize(datetime.combine(next_date, day_start)) + + self.assertEqual(start, expected_start) + self.assertEqual(end, expected_start + timedelta(minutes=30)) + self.assertEqual(auction_day_start, day_start) + self.assertEqual(auction_stream, stream) + self.assertTrue(new_slot) + + def test_set_end_of_auction_without_free_slot_limit_stream_and_wrong_day_start(self): + stream = 3 + streams = 3 + next_date = datetime.now().date() + day_start = time(16, 0) + + plan = { + '_id': 'plantest_2017-10-03', + 'stream_1': {}, + 'stream_2': {}, + 'stream_3': {}, + 'streams': 3 + } + + result = self.manager.set_end_of_auction( + stream, streams, next_date, day_start, plan + ) + self.assertIsNone(result) + + +class NonClassicManagerTestMixin(object): + manager = None + streams_key = '' + + def test_get_date(self): + date = datetime.now().date() + plan_date = parse_date(date.isoformat() + 'T' + self.manager.working_day_start.isoformat(), None) + plan_date = plan_date.astimezone(TZ) if plan_date.tzinfo else TZ.localize(plan_date) + + db = mock.MagicMock() + + db.get.return_value = plantest + + plan_time, stream, plan = self.manager.get_date(db, '', date) + + self.assertEqual(plan, plantest) + self.assertEqual(plan_time, plan_date.time()) + self.assertEqual(stream, len(plantest[self.streams_key])) + + def test_get_hours_and_stream(self): + plan = { + self.streams_key: [1, 2, 3, 4, 5] + } + + plan_date, streams = self.manager._get_hours_and_stream(plan) + self.assertEqual(plan_date, self.manager.working_day_start.isoformat()) + self.assertEqual(streams, len(plan[self.streams_key])) + + def test_set_date(self): + db = mock.MagicMock() + plan = { + self.streams_key: [] + } + auction_id = '1' * 32 + + self.manager.set_date(db, plan, auction_id) + self.assertIn(auction_id, plan[self.streams_key]) + self.assertEqual(len(plan[self.streams_key]), 1) + + db.save.assert_called_with(plan) + + def test_free_slot(self): + plan = { + self.streams_key: [ + '1' * 32, + '2' * 32, + '3' * 32, + '1' * 32, + ] + } + db = mock.MagicMock() + plan_id = 'plan_id' + db.get.return_value = plan + auction_id = '1' * 32 + + self.manager.free_slot(db, plan_id, auction_id) + + self.assertEqual(len(plan[self.streams_key]), 2) + self.assertNotIn('1' * 32, plan[self.streams_key]) + + db.get.assert_called_with(plan_id) + db.save.assert_called_with(plan) + + def set_end_of_auction(self): + stream = 1 + streams = 3 + next_date = datetime.now().date() + day_start = time(15, 0) + + start, end, auction_day_start, auction_stream, new_slot = self.manager.set_end_of_auction( + stream, streams, next_date, day_start + ) + + expected_start = TZ.localize(datetime.combine(next_date, day_start)) + + self.assertEqual(start, expected_start) + self.assertEqual(end, expected_start + self.manager.working_day_duration) + self.assertEqual(auction_day_start, day_start) + self.assertEqual(auction_stream, stream) + self.assertFalse(new_slot) + + def set_end_of_auction_with_wrong_stream(self): + stream = 10 + streams = 3 + next_date = datetime.now().date() + day_start = time(15, 0) + + result = self.manager.set_end_of_auction( + stream, streams, next_date, day_start + ) + + self.assertIsNone(result) + + +class TestInsiderAuctionManager(BaseWebTest, NonClassicManagerTestMixin): + streams_key = 'dutch_streams' + + def setUp(self): + super(TestInsiderAuctionManager, self).setUp() + self.manager = InsiderAuctionsManager() + + +class TestTexasAuctionManager(BaseWebTest, NonClassicManagerTestMixin): + streams_key = 'texas_streams' + + def setUp(self): + super(TestTexasAuctionManager, self).setUp() + self.manager = TexasAuctionsManager() diff --git a/openprocurement/auction/tests/bridge_utils/test_utils.py b/openprocurement/auction/tests/bridge_utils/test_utils.py new file mode 100644 index 0000000..8fa4063 --- /dev/null +++ b/openprocurement/auction/tests/bridge_utils/test_utils.py @@ -0,0 +1,482 @@ +# -*- coding: utf-8 -*- +import mock +from copy import deepcopy +from datetime import datetime, timedelta + +from iso8601 import parse_date + +from openprocurement.auction.bridge_utils.constants import TZ +from openprocurement.auction.bridge_utils.managers import InsiderAuctionsManager, MANAGERS_MAPPING +from openprocurement.auction.bridge_utils.utils import ( + planning_auction, + get_manager_for_auction, + check_auction, + check_inner_auction, + create_auction_period_for_lots, + create_auction_period +) +from openprocurement.auction.tests.bridge_utils.base import BaseWebTest +from openprocurement.auction.tests.bridge_utils.data import test_auction_data, plantest + + +test_auction_data_quick = deepcopy(test_auction_data) +test_auction_data_quick.update({ + "enquiryPeriod": { + 'startDate': datetime.now(TZ).isoformat(), + "endDate": datetime.now(TZ).isoformat() + }, + 'tenderPeriod': { + 'startDate': datetime.now(TZ).isoformat(), + "endDate": datetime.now(TZ).isoformat() + } +}) +test_auction_data_test_quick = deepcopy(test_auction_data_quick) +test_auction_data_test_quick['mode'] = 'test' + + +class CheckAuctionTest(BaseWebTest): + + def test_auction_without_lots(self): + my_test_auction = deepcopy(test_auction_data) + my_test_auction.pop('lots', None) + + with mock.patch('openprocurement.auction.bridge_utils.utils.create_auction_period') as create_auction_period_mock: + expected_result = 'auction period for auction' + create_auction_period_mock.return_value = expected_result + result = check_auction(my_test_auction, self.db, self.mapper) + self.assertEqual(result, expected_result) + self.assertEqual(create_auction_period_mock.call_count, 1) + + def test_auction_with_lots(self): + my_test_auction = deepcopy(test_auction_data) + my_test_auction['lots'] = ['1', '2'] + + with mock.patch('openprocurement.auction.bridge_utils.utils.create_auction_period_for_lots') as create_auction_period_for_lots_mock: + expected_result = 'auction period for auction lots' + create_auction_period_for_lots_mock.return_value = expected_result + result = check_auction(my_test_auction, self.db, self.mapper) + self.assertEqual(result, expected_result) + self.assertEqual(create_auction_period_for_lots_mock.call_count, 1) + + +class CreateAuctionPeriodTest(BaseWebTest): + + def test_check_aution(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + my_test_auction['auctionPeriod'] = {} + my_test_auction['auctionPeriod']['startDate'] = now.isoformat() + my_test_auction['auctionPeriod']['shouldStartAfter'] = (now + timedelta(days=10)).isoformat() + my_test_auction['procurementMethodType'] = 'dgfInsider' + + auction_period = create_auction_period(my_test_auction, self.db, self.mapper, 'mode') + self.assertIn('auctionPeriod', auction_period) + self.assertIn('startDate', auction_period['auctionPeriod']) + + def test_check_auction_without_should_start(self): + my_test_auction = deepcopy(test_auction_data) + my_test_auction.pop('auctionPeriod', None) + + auction_period = create_auction_period(my_test_auction, self.db, self.mapper, 'mode') + self.assertIsNone(auction_period) + + def test_check_auction_without_auction_start_date(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + + my_test_auction['auctionPeriod'] = {} + my_test_auction['auctionPeriod']['shouldStartAfter'] = (now - timedelta(days=10)).isoformat() + + auction_period = create_auction_period(my_test_auction, self.db, self.mapper, 'mode') + self.assertIn('auctionPeriod', auction_period) + self.assertIn('startDate', auction_period['auctionPeriod']) + + def test_check_auction_start_after_should_start(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + + my_test_auction['auctionPeriod'] = {} + my_test_auction['auctionPeriod']['startDate'] = now.isoformat() + my_test_auction['auctionPeriod']['shouldStartAfter'] = (now - timedelta(days=10)).isoformat() + + auction_period = create_auction_period(my_test_auction, self.db, self.mapper, 'mode') + self.assertIsNone(auction_period) + + +class CreateAuctionPeriodForLotsTest(BaseWebTest): + + def test_check_auction_with_lots(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + auction_period = { + 'startDate': now.isoformat(), + 'shouldStartAfter': (now + timedelta(days=10)).isoformat() + } + lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} + my_test_auction['lots'] = [lot, lot] + + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') + self.assertIn('lots', lots) + self.assertEqual(len(lots['lots']), 2) + + def test_check_auction_with_not_active_lot(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + auction_period = { + 'startDate': now.isoformat(), + 'shouldStartAfter': (now + timedelta(days=10)).isoformat() + } + lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} + not_active = deepcopy(lot) + not_active['status'] = 'pending' + my_test_auction['lots'] = [lot, not_active] + + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') + self.assertIn('lots', lots) + self.assertEqual(len(lots['lots'][1].keys()), 0) + + def test_should_start_before_auction_start(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + auction_period = { + 'startDate': now.isoformat(), + 'shouldStartAfter': (now + timedelta(days=10)).isoformat() + } + lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} + should_start_before = deepcopy(lot) + should_start_before['auctionPeriod']['startDate'] = now.isoformat() + should_start_before['auctionPeriod']['shouldStartAfter'] = (now - timedelta(days=10)).isoformat() + my_test_auction['lots'] = [lot, should_start_before] + + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') + self.assertIn('lots', lots) + self.assertEqual(len(lots['lots'][1].keys()), 0) + + def test_should_start_absent(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + auction_period = { + 'startDate': now.isoformat(), + 'shouldStartAfter': (now + timedelta(days=10)).isoformat() + } + lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} + no_should_start = deepcopy(lot) + no_should_start['auctionPeriod'].pop('shouldStartAfter') + my_test_auction['lots'] = [lot, no_should_start] + + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') + self.assertIn('lots', lots) + self.assertEqual(len(lots['lots'][1].keys()), 0) + + def test_auction_start_absent(self): + now = datetime.now(TZ) + + my_test_auction = deepcopy(test_auction_data) + auction_period = { + 'shouldStartAfter': (now + timedelta(days=10)).isoformat() + } + lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} + my_test_auction['lots'] = [lot, lot] + + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') + self.assertIn('lots', lots) + self.assertEqual(len(lots['lots']), 2) + + def test_check_auction_with_all_invalid_lots(self): + now = datetime.now(TZ) + now = now.replace(year=2018, month=8, day=25) + + # no active lots + my_test_auction = deepcopy(test_auction_data) + auction_period = { + 'startDate': now.isoformat(), + 'shouldStartAfter': (now + timedelta(days=10)).isoformat() + } + lot = {'status': 'active', 'auctionPeriod': auction_period, 'id': '1' * 32} + not_active = deepcopy(lot) + not_active['status'] = 'pending' + my_test_auction['lots'] = [not_active, not_active] + + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') + self.assertIsNone(lots) + + # should_start_after before auction + my_test_auction = deepcopy(test_auction_data) + should_start_before = deepcopy(lot) + should_start_before['auctionPeriod']['startDate'] = now.isoformat() + should_start_before['auctionPeriod']['shouldStartAfter'] = (now - timedelta(days=10)).isoformat() + my_test_auction['lots'] = [should_start_before, should_start_before] + + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') + self.assertIsNone(lots) + + # should start absent + my_test_auction = deepcopy(test_auction_data) + no_should_start = deepcopy(lot) + no_should_start['auctionPeriod'].pop('shouldStartAfter') + my_test_auction['lots'] = [no_should_start, no_should_start] + + lots = create_auction_period_for_lots(my_test_auction, self.db, self.mapper, 'mode') + self.assertIsNone(lots) + + +class AuctionPlanning(BaseWebTest): + + def test_auction_quick_planning(self): + now = datetime.now(TZ) + auctionPeriodstartDate = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db, True)[0] + self.assertTrue(now < auctionPeriodstartDate < now + timedelta(hours=1)) + + def test_auction_quick_planning_insider(self): + now = datetime.now(TZ) + my_test_auction = deepcopy(test_auction_data_test_quick) + my_test_auction['procurementMethodType'] = 'dgfInsider' + auctionPeriodstartDate = planning_auction( + my_test_auction, self.mapper, now, self.db, True + )[0] + self.assertTrue( + now < auctionPeriodstartDate < now + timedelta(hours=1) + ) + + def test_auction_planning_overlow_insider(self): + now = datetime.now(TZ) + my_test_auction = deepcopy(test_auction_data_test_quick) + my_test_auction['procurementMethodType'] = 'dgfInsider' + res = planning_auction(my_test_auction, self.mapper, now, self.db)[0] + startDate = res.date() + count = 0 + while startDate == res.date(): + count += 1 + res = planning_auction(my_test_auction, self.mapper, now, self.db)[0] + self.assertEqual(count, 15) + + def test_auction_planning_overlow(self): + now = datetime.now(TZ) + res = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db)[0] + startDate = res.date() + count = 0 + while startDate == res.date(): + count += 1 + res = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db)[0] + self.assertEqual(count, 100) + + def test_auction_planning_free(self): + now = datetime.now(TZ) + test_auction_data_test_quick.pop("id") + res = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db)[0] + startDate, startTime = res.date(), res.time() + manager = get_manager_for_auction(test_auction_data, self.mapper) + manager.free_slot(self.db, "plantest_{}".format(startDate.isoformat()), "", res) + res = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db)[0] + self.assertEqual(res.time(), startTime) + + def test_auction_planning_buffer(self): + some_date = datetime(2015, 9, 21, 6, 30) + date = some_date.date() + ndate = (some_date + timedelta(days=1)).date() + res = planning_auction(test_auction_data_test_quick, self.mapper, some_date, self.db)[0] + self.assertEqual(res.date(), date) + some_date = some_date.replace(hour=10) + res = planning_auction(test_auction_data_test_quick, self.mapper, some_date, self.db)[0] + self.assertNotEqual(res.date(), date) + self.assertEqual(res.date(), ndate) + some_date = some_date.replace(hour=16) + res = planning_auction(test_auction_data_test_quick, self.mapper, some_date, self.db)[0] + self.assertNotEqual(res.date(), date) + self.assertEqual(res.date(), ndate) + + def test_skipping_holidays(self): + now = datetime.now(TZ) + now = now.replace(year=2018, month=8, day=25, hour=6) + # Set saturday + while now.weekday() != 5: + now = now + timedelta(days=1) + my_test_auction = deepcopy(test_auction_data) + my_test_auction['procurementMethodType'] = 'dgfInsider' + res, _, skipped_days = planning_auction(my_test_auction, self.mapper, now, self.db) + self.assertEqual(res.date(), (now + timedelta(days=2)).date()) + self.assertEqual(skipped_days, 0) + + def test_skipping_holiday_in_calendar(self): + now = datetime.now(TZ) + + # Holiday in 2018 year + now = now.replace(year=2018, month=12, day=25, hour=6) + self.assertNotIn(now.weekday(), [5, 6]) + + my_test_auction = deepcopy(test_auction_data) + my_test_auction['procurementMethodType'] = 'dgfInsider' + res, _, skipped_days = planning_auction(my_test_auction, self.mapper, now, self.db) + self.assertEqual(res.date(), (now + timedelta(days=1)).date()) + self.assertEqual(skipped_days, 0) + + def test_skipping_day_if_time_after_working_day_start(self): + now = datetime.now(TZ) + now = now.replace(year=2018, month=8, day=25, hour=12) + + while now.weekday() != 1: + now = now + timedelta(days=1) + + my_test_auction = deepcopy(test_auction_data) + my_test_auction['procurementMethodType'] = 'dgfInsider' + res, _, skipped_days = planning_auction(my_test_auction, self.mapper, now, self.db) + self.assertEqual(res.date(), (now + timedelta(days=1)).date()) + self.assertEqual(skipped_days, 0) + + def test_result_set_end_none(self): + my_test_auction = deepcopy(test_auction_data) + + now = datetime.now(TZ) + now = now.replace(year=2018, month=8, day=25, hour=6) + + # Set friday + while now.weekday() != 1: + now = now + timedelta(days=1) + + # set_end_of_auction return none once + with mock.patch('openprocurement.auction.bridge_utils.utils.get_manager_for_auction') as get_manager_mock: + mocked_manager = mock.MagicMock() + mocked_manager.working_day_start = InsiderAuctionsManager.working_day_start + + mocked_manager.get_date.side_effect = [ + ('1', '2', '3'), + ('1', '2', '3'), + ('1', '2', '3'), + ] + mocked_manager.set_end_of_auction.side_effect = [ + None, + None, + ('1', mock.MagicMock(), '3', '4', '5') + ] + + get_manager_mock.return_value = mocked_manager + + start, _, skipped_days = planning_auction(my_test_auction, self.mapper, now, self.db) + + self.assertEqual(skipped_days, 2) + + +class TestCheckInnerAuction(BaseWebTest): + + def setUp(self): + super(TestCheckInnerAuction, self).setUp() + plantest['_id'] = 'plantest_{}'.format( + datetime.now().date().isoformat()) + plantest_from_db = self.db.get(plantest['_id'], {}) + plantest_from_db.update(plantest) + + self.db.save(plantest_from_db) + + def test_check_inner_auction(self): + insider_auction_id = '01fa8a7dc4b8eac3b5820747efc6fe36' + texas_auction_id = 'dc3d950743304d05adaa1cd5b0475075' + classic_auction_with_lots = 'da8a28ed2bdf73ee1d373e4cadfed4c5' + classic_auction_without_lots = 'e51508cddc2c490005eaecb73c006b72' + lots_ids = ['1c2fb1e496b317b2b87e197e2332da77', + 'b10f9f7f26157ae2f349be8dc2106d6e'] + + today = datetime.now().date().isoformat() + time = '12:15:00' # actually, can be any time between 12:00:00 and 12:30:00 due to existing asserts + raw_time = ''.join([today, 'T', time]) + + # datetime.datetime object prepared in the way scheduler actually does it: + test_time = TZ.localize(parse_date(raw_time, None)).isoformat() + + auction = { + 'id': insider_auction_id, + 'procurementMethodType': 'dgfInsider', + 'auctionPeriod': { + 'startDate': test_time + } + } + mapper = { + 'pmts': { + 'dgfInsider': MANAGERS_MAPPING['dutch'](), + 'landLease': MANAGERS_MAPPING['texas']() + }, + 'types': {'english': MANAGERS_MAPPING['english']()} + } + + plantest = self.db.get('plantest_{}'.format(today)) + + # Test dutch + self.assertEqual(len(plantest.get('dutch_streams', [])), 15) + self.assertIn(insider_auction_id, plantest.get('dutch_streams')) + + check_inner_auction(self.db, auction, mapper) + plantest = self.db.get('plantest_{}'.format(today)) + self.assertEqual(len(plantest.get('dutch_streams', [])), 6) + self.assertNotIn(insider_auction_id, plantest.get('dutch_streams')) + + # Test texas + auction['id'] = texas_auction_id + auction['procurementMethodType'] = 'landLease' + + self.assertEqual(len(plantest.get('texas_streams', [])), 20) + self.assertIn(texas_auction_id, plantest.get('texas_streams')) + + check_inner_auction(self.db, auction, mapper) + plantest = self.db.get('plantest_{}'.format(today)) + self.assertEqual(len(plantest.get('texas_streams', [])), 15) + self.assertNotIn(texas_auction_id, plantest.get('texas_streams')) + + # Test classic with lots + auction['procurementMethodType'] = 'classic' + auction['id'] = classic_auction_with_lots + auction['lots'] = [ + { + 'id': lots_ids[0], + 'auctionPeriod': {'startDate': test_time} + }, + { + 'id': lots_ids[1], + 'auctionPeriod': {'startDate': test_time} + } + ] + self.assertEqual(len(plantest.get('stream_1')), 10) + self.assertEqual(len(plantest.get('stream_2')), 10) + stream_1_none_count = len( + [v for k, v in plantest.get('stream_1').items() if v is None]) + stream_2_none_count = len( + [v for k, v in plantest.get('stream_2').items() if v is None]) + self.assertEqual(stream_1_none_count, 0) + self.assertEqual(stream_2_none_count, 0) + check_inner_auction(self.db, auction, mapper) + plantest = self.db.get('plantest_{}'.format(today)) + self.assertEqual(len(plantest.get('stream_1')), 10) + self.assertEqual(len(plantest.get('stream_2')), 10) + stream_1_none_count = len( + [v for k, v in plantest.get('stream_1').items() if v is None]) + stream_2_none_count = len( + [v for k, v in plantest.get('stream_2').items() if v is None]) + self.assertEqual(stream_1_none_count, 3) + self.assertEqual(stream_2_none_count, 3) + self.assertNotIn(classic_auction_with_lots, + plantest.get('stream_1', {}).values()) + self.assertNotIn(classic_auction_with_lots, + plantest.get('stream_2', {}).values()) + + # Test classic without lots + del auction['lots'] + auction['id'] = classic_auction_without_lots + check_inner_auction(self.db, auction, mapper) + plantest = self.db.get('plantest_{}'.format(today)) + self.assertEqual(len(plantest.get('stream_1')), 10) + self.assertEqual(len(plantest.get('stream_2')), 10) + stream_1_none_count = len( + [v for k, v in plantest.get('stream_1').items() if v is None]) + stream_2_none_count = len( + [v for k, v in plantest.get('stream_2').items() if v is None]) + self.assertEqual(stream_1_none_count, 7) + self.assertEqual(stream_2_none_count, 6) + self.assertNotIn(classic_auction_without_lots, + plantest.get('stream_1', {}).values()) + self.assertNotIn(classic_auction_without_lots, + plantest.get('stream_2', {}).values())