Merge pull request #108 from datosgobar/102-editable-synchronizers

Hago Synchronizers editables desde el admin

Author: lucaslavandeira

django_datajsonar/admin.py

@@ -2,7 +2,9 @@
 from __future__ import unicode_literals
 
 from django import forms
+from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
+from django.core.exceptions import ValidationError
 from django.forms.models import formset_factory
 from django.contrib import admin, messages
 from django.contrib.admin import helpers, SimpleListFilter
@@ -14,13 +16,14 @@
 from scheduler.models import RepeatableJob
 from scheduler.admin import RepeatableJobAdmin
 
+from django_datajsonar.synchronizer import create_or_update_synchro
 from .views import config_csv
-from .actions import process_node_register_file_action, confirm_delete
+from .actions import process_node_register_file_action
 from .utils import download_config_csv, generate_stages
 from .tasks import bulk_whitelist, read_datajson
 from .models import DatasetIndexingFile, NodeRegisterFile, Node, NodeMetadata, ReadDataJsonTask, Metadata, Synchronizer, Stage
 from .models import Catalog, Dataset, Distribution, Field, Jurisdiction
-from .forms import ScheduleJobForm, SynchroForm, StageFormset, StageForm
+from .forms import ScheduleJobForm, SynchroForm, StageForm
 
 
 class EnhancedMetaAdmin(GenericTabularInline):
@@ -346,7 +349,6 @@ def process_register_file(self, _, queryset):
 
 
 class CustomRepeatableJobAdmin(RepeatableJobAdmin):
-
     actions = ['delete_and_unschedule']
 
     def delete_model(self, request, obj):
@@ -357,6 +359,7 @@ def delete_and_unschedule(self, _, queryset):
         for job in queryset:
             job.unschedule()
         queryset.delete()
+
     delete_and_unschedule.short_description = 'Delete and unschedule job'
 
     def get_actions(self, request):
@@ -366,34 +369,89 @@ def get_actions(self, request):
 
 
 class SynchronizerAdmin(admin.ModelAdmin):
+    StageFormset = formset_factory(StageForm, extra=0)
 
     def add_view(self, request, form_url='', extra_context=None):
-        synchro_form = SynchroForm()
+        return self._synchro_view(request)
+
+    def change_view(self, request, object_id, form_url='', extra_context=None):
+        synchro = Synchronizer.objects.get(id=object_id)
+        return self._synchro_view(request, synchro)
+
+    def _synchro_view(self, request, synchro=None):
+        if request.method == 'POST':
+            return self.post_synchro_edit(request, synchro.id if synchro else None)
+
+        if synchro is not None:
+            synchro_form = SynchroForm({
+                'name': synchro.name,
+                'frequency': synchro.frequency,
+                'scheduled_time': synchro.scheduled_time
+            })
+        else:
+            synchro_form = SynchroForm()
 
+        context = self.add_synchro_context(request, synchro_form, synchro)
+
+        return render(request, 'synchronizer.html', context)
+
+    def add_synchro_context(self, request, synchro_form, synchro=None):
         context = {
-            'title': 'Define new process',
-            'app_label': self.model._meta.app_label,
             'opts': self.model._meta,
-            'has_change_permission': self.has_change_permission(request)
+            'has_change_permission': self.has_change_permission(request),
+            'synchro_form': self.admin_synchro_form(request, synchro_form),
+            'stages_form': self.get_stages_formset(synchro),
+            'object': synchro,
+        }
+        return context
+
+    def get_stages_formset(self, model=None):
+        stages_data = []
+        next_stage = model.start_stage if model else None
+        while next_stage is not None:
+            stages_data.append({
+                'task': self.get_stage_choice(next_stage.name)
+            })
+            next_stage = next_stage.next_stage
+
+        if not stages_data:
+            stages_data.append({})
+
+        return self.StageFormset(initial=stages_data)
+
+    def admin_synchro_form(self, request, synchro_form):
+        return helpers.AdminForm(synchro_form,
+                                 list([(None, {'fields': synchro_form.base_fields})]),
+                                 self.get_prepopulated_fields(request))
+
+    def post_synchro_edit(self, request, object_id=None):
+        synchro = Synchronizer.objects.get(id=object_id) if object_id else None
+        synchro_form = SynchroForm(request.POST)
+        stages_formset = self.StageFormset(request.POST)
+
+        if not stages_formset.is_valid() or not synchro_form.is_valid():
+            synchro = Synchronizer.objects.get(id=object_id) if object_id else None
+            return render(request, 'synchronizer.html', self.add_synchro_context(request, synchro_form, synchro))
+
+        synchro_name = synchro_form.cleaned_data['name']
+        stages = generate_stages(stages_formset.forms, synchro_name)
+        if not stages:
+            messages.error(request, "No hay stages definidos")
+            return render(request, 'synchronizer.html', self.add_synchro_context(request, synchro_form, synchro))
+
+        data = {
+            'name': synchro_name,
+            'frequency': synchro_form.cleaned_data['frequency'],
+            'scheduled_time': synchro_form.cleaned_data['scheduled_time'],
         }
+        create_or_update_synchro(object_id, stages, data)
 
-        if request.method == 'POST':
-            synchro_form = SynchroForm(request.POST)
-            stages_formset = formset_factory(form=StageForm,
-                                             formset=StageFormset)(request.POST)
-
-            if stages_formset.is_valid() and synchro_form.is_valid():
-                synchro_name = synchro_form.cleaned_data['name']
-                stages = generate_stages(stages_formset.forms, synchro_name)
-                synchro_form.create_synchronizer(start_stage=stages[0])
-                return redirect('admin:django_datajsonar_synchronizer_changelist')
-
-        context['synchro_form'] = helpers.AdminForm(synchro_form, list([(None, {'fields': synchro_form.base_fields})]),
-                                                    self.get_prepopulated_fields(request))
-        stages_formset = formset_factory(form=StageForm,
-                                         formset=StageFormset)()
-        context['stages_form'] = stages_formset
-        return render(request, 'synchronizer.html', context)
+        return redirect('admin:django_datajsonar_synchronizer_changelist')
+
+    def get_stage_choice(self, stage_name):
+        for name in settings.DATAJSONAR_STAGES.keys():
+            if name in stage_name:
+                return name
 
 
 class EnhancedMetaFilter(SimpleListFilter):

django_datajsonar/forms.py

@@ -59,10 +59,6 @@ def clean(self):
 
 
 class SynchroForm(forms.Form):
-    class Meta:
-        model = Synchronizer
-        exclude = ['start_stage', 'actual_stage', 'status']
-
     name = forms.CharField(max_length=50)
     WEEK_DAYS = 'week days'
     DAILY = 'every day'
@@ -81,13 +77,6 @@ def clean(self):
         if cleaned_data['frequency'] == self.WEEK_DAYS:
             self.add_error('frequency', 'week days not yet implemented')
 
-    def create_synchronizer(self, start_stage):
-        scheduled_time = self.cleaned_data['scheduled_time']
-        cron_string = "{} {} * * *".format(scheduled_time.minute, scheduled_time.hour)
-        return Synchronizer.objects.create(name=self.cleaned_data['name'],
-                                           start_stage=start_stage,
-                                           cron_string=cron_string)
-
 
 class StageForm(forms.Form):
     task = forms.ChoiceField(choices=[(x, x) for x in settings.DATAJSONAR_STAGES.keys()],
@@ -102,9 +91,3 @@ def get_stage(self, name):
 
         data = settings.DATAJSONAR_STAGES[task]
         return Stage.objects.update_or_create(name=name, defaults=data)[0]
-
-
-class StageFormset(forms.BaseFormSet):
-    def __init__(self, *args, **kwargs):
-        super(StageFormset, self).__init__(*args, **kwargs)
-        self.queryset = Stage.objects.none()

django_datajsonar/frequency.py

@@ -0,0 +1,14 @@
+from datetime import datetime
+
+from croniter import croniter
+
+from django_datajsonar.strings import SYNCHRO_DAILY_FREQUENCY
+
+
+def get_next_run_date(start_time, scheduled_time, frequency):
+    if frequency != SYNCHRO_DAILY_FREQUENCY:
+        raise NotImplementedError
+
+    cron_string = "{} {} * * *".format(scheduled_time.minute, scheduled_time.hour)
+
+    return croniter(cron_string, start_time=start_time).get_next(datetime)

django_datajsonar/migrations/0001_auto_20190201_1344.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.15 on 2019-02-01 16:44
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('django_datajsonar', 'ensure_upkeep_job_exists'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='synchronizer',
+            name='cron_string',
+        ),
+        migrations.AddField(
+            model_name='synchronizer',
+            name='frequency',
+            field=models.CharField(choices=[('every day', 'every day'), ('week days', 'week days')], default='every day', max_length=16),
+        ),
+        migrations.AddField(
+            model_name='synchronizer',
+            name='scheduled_time',
+            field=models.TimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+    ]

django_datajsonar/migrations/ensure_upkeep_job_exists.py

@@ -11,7 +11,7 @@ def create_upkeep_repeatable_job(apps, schema_editor):
     db_alias = schema_editor.connection.alias
 
     upkeep_jobs_count = RepeatableJob.objects.using(db_alias).\
-        filter(callable='django_datajsonar.synchronizer_tasks.upkeep').count()
+        filter(callable='django_datajsonar.synchronizer.upkeep').count()
 
     if not upkeep_jobs_count:
         RepeatableJob.objects.using(db_alias).create(name='upkeep',

django_datajsonar/models.py

@@ -1,10 +1,8 @@
 #! coding: utf-8
 from __future__ import unicode_literals
 
-from datetime import datetime
 from importlib import import_module
 
-from croniter import croniter, CroniterBadCronError
 from django.contrib.auth.models import User
 from django.core.exceptions import ImproperlyConfigured, ValidationError
 from django.conf import settings
@@ -14,6 +12,8 @@
 from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
 from django.contrib.contenttypes.models import ContentType
 
+from django_datajsonar.frequency import get_next_run_date
+from django_datajsonar.strings import SYNCHRO_DAILY_FREQUENCY, SYNCHRO_WEEK_DAYS_FREQUENCY
 from django_datajsonar.utils import pending_or_running_jobs, import_string
 
 
@@ -458,14 +458,16 @@ class Synchronizer(models.Model):
     actual_stage = models.ForeignKey(to=Stage, related_name='running_synchronizer', null=True, blank=True,
                                      on_delete=models.PROTECT)
 
-    cron_string = models.CharField(max_length=64)
-    last_time_ran = models.DateTimeField(auto_now_add=True)
+    WEEK_DAYS = SYNCHRO_WEEK_DAYS_FREQUENCY
+    DAILY = SYNCHRO_DAILY_FREQUENCY
+    FREQUENCY_CHOICES = (
+        (DAILY, DAILY),
+        (WEEK_DAYS, WEEK_DAYS),
+    )
+    frequency = models.CharField(choices=FREQUENCY_CHOICES, max_length=16, default=DAILY)
+    scheduled_time = models.TimeField(auto_now_add=True)
 
-    def clean(self):
-        try:
-            self.next_start_date()
-        except CroniterBadCronError:
-            raise ValidationError({'cron_string': "Invalid cron string: {}".format(self.cron_string)})
+    last_time_ran = models.DateTimeField(auto_now_add=True)
 
     def begin_stage(self, stage=None):
         if self.status == self.RUNNING and stage is None:
@@ -495,10 +497,18 @@ def next_stage(self):
 
     def next_start_date(self):
         localtime = self.last_time_ran.astimezone(timezone.get_current_timezone())
-        return croniter(self.cron_string, start_time=localtime).get_next(datetime)
+        return get_next_run_date(localtime, self.scheduled_time, self.frequency)
 
     def __unicode__(self):
         return self.name
 
     def __str__(self):
         return self.__unicode__()
+
+    def get_stages(self):
+        stages = []
+        current_stage = self.start_stage
+        while current_stage is not None:
+            stages.append(current_stage)
+            current_stage = current_stage.next_stage
+        return stages

django_datajsonar/strings.py

@@ -2,3 +2,6 @@
 CATALOG_STATUS = u"Catalogo {}, status: {}"
 DATASET_STATUS = u"Dataset ({}, {}) status: {}"
 FILE_READ_ERROR = u"Error en la lectura del archivo de entrada"
+
+SYNCHRO_DAILY_FREQUENCY = 'every day'
+SYNCHRO_WEEK_DAYS_FREQUENCY = 'week days'

django_datajsonar/synchronizer_tasks.py renamed to django_datajsonar/synchronizer.py

@@ -3,7 +3,7 @@
 
 from django.utils import timezone
 
-from .models import Synchronizer
+from .models import Synchronizer, Stage
 
 
 def upkeep():
@@ -25,3 +25,25 @@ def start_synchros():
     for synchro in synchronizers:
         if now > synchro.next_start_date():
             synchro.begin_stage()
+
+
+def create_or_update_synchro(synchro_id, stages, data=None):
+    if data is None:
+        data = {}
+
+    if synchro_id is None:
+        return Synchronizer.objects.create(start_stage=stages[0], **data)
+
+    synchro_queryset = Synchronizer.objects.filter(id=synchro_id)
+    synchro_queryset.update(start_stage=stages[0], **data)
+
+    synchro = synchro_queryset.first()
+    old_stages = synchro.get_stages()
+
+    Stage.objects \
+        .filter(id__in=[stage.id for stage in old_stages]) \
+        .exclude(id__in=[stage.id for stage in stages]) \
+        .delete()
+
+    synchro.refresh_from_db()
+    return synchro