From 2ec2821d664259ae3898aa99ea226e38630216d2 Mon Sep 17 00:00:00 2001 From: Nils Rokita <0rokita@informatik.uni-hamburg.de> Date: Sun, 9 Feb 2020 22:23:35 +0100 Subject: [PATCH 1/2] slowtiles module This adds a module to keep trak of slow to generate tiles The tileserver can get the list and pregenerate them --- requirements.txt | 3 ++ tileservermapping/manage.py | 15 ------- tileservermapping/slowtiles/admin.py | 6 +++ .../slowtiles/migrations/0001_initial.py | 32 +++++++++++++++ .../slowtiles/migrations/__init__.py | 0 tileservermapping/slowtiles/models.py | 39 +++++++++++++++++++ tileservermapping/slowtiles/urls.py | 7 ++++ tileservermapping/slowtiles/views.py | 20 ++++++++++ tileservermapping/urls.py | 1 + 9 files changed, 108 insertions(+), 15 deletions(-) create mode 100644 requirements.txt delete mode 100755 tileservermapping/manage.py create mode 100644 tileservermapping/slowtiles/admin.py create mode 100644 tileservermapping/slowtiles/migrations/0001_initial.py create mode 100644 tileservermapping/slowtiles/migrations/__init__.py create mode 100644 tileservermapping/slowtiles/models.py create mode 100644 tileservermapping/slowtiles/urls.py create mode 100644 tileservermapping/slowtiles/views.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f38fb48 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +django +mercantile +geojson diff --git a/tileservermapping/manage.py b/tileservermapping/manage.py deleted file mode 100755 index 9ef148e..0000000 --- a/tileservermapping/manage.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tileservermapping.settings") - try: - from django.core.management import execute_from_command_line - except ImportError as exc: - raise ImportError( - "Couldn't import Django. Are you sure it's installed and " - "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" - ) from exc - execute_from_command_line(sys.argv) diff --git a/tileservermapping/slowtiles/admin.py b/tileservermapping/slowtiles/admin.py new file mode 100644 index 0000000..d492e63 --- /dev/null +++ b/tileservermapping/slowtiles/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin + +# Register your models here. +from .models import SlowTiles + +admin.site.register(SlowTiles) diff --git a/tileservermapping/slowtiles/migrations/0001_initial.py b/tileservermapping/slowtiles/migrations/0001_initial.py new file mode 100644 index 0000000..b6ee9e4 --- /dev/null +++ b/tileservermapping/slowtiles/migrations/0001_initial.py @@ -0,0 +1,32 @@ +# Generated by Django 2.2.1 on 2020-02-09 20:16 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('mapping', '0006_auto_20181226_1555'), + ] + + operations = [ + migrations.CreateModel( + name='SlowTiles', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('z', models.IntegerField()), + ('x', models.IntegerField()), + ('y', models.IntegerField()), + ('count', models.IntegerField(default=1)), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('server', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mapping.Server')), + ], + options={ + 'unique_together': {('z', 'x', 'y')}, + }, + ), + ] diff --git a/tileservermapping/slowtiles/migrations/__init__.py b/tileservermapping/slowtiles/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tileservermapping/slowtiles/models.py b/tileservermapping/slowtiles/models.py new file mode 100644 index 0000000..6b33e2e --- /dev/null +++ b/tileservermapping/slowtiles/models.py @@ -0,0 +1,39 @@ +from django.db import models, transaction +from tileservermapping.mapping.models import Server + + +class SlowTiles(models.Model): + z = models.IntegerField(null=False) + x = models.IntegerField(null=False) + y = models.IntegerField(null=False) + + server = models.ForeignKey(Server, on_delete=models.CASCADE) + + count = models.IntegerField(default=1) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = ['z', 'x', 'y'] + + def __str__(self): + return "".format(self.z, self.x, self.y, self.count) + + @staticmethod + def insert(z, x, y): + """ + inserts a tile z/x/y into the slow tile list, or increase the counter on an existing tile + :return: the current counter on this tile, or -1 if no tile server is responsible + """ + server = Server.get_server(z, x, y) + if server == Server.objects.get(z=0, x=0, y=0): + # we can not do anything on the external instance + return -1 + with transaction.atomic(): + try: + tile = SlowTiles.objects.get(z=z, x=x, y=y) + tile.count += 1 + except models.ObjectDoesNotExist: + tile = SlowTiles(z=z, x=x, y=y, server=server) + tile.save() + return tile.count diff --git a/tileservermapping/slowtiles/urls.py b/tileservermapping/slowtiles/urls.py new file mode 100644 index 0000000..2f462ff --- /dev/null +++ b/tileservermapping/slowtiles/urls.py @@ -0,0 +1,7 @@ +from django.urls import path +from .views import insert_tile, get_server_list + +urlpatterns = [ + path('add///', insert_tile, name="insert_tile"), + path('server_list///', get_server_list, name="server_list") +] diff --git a/tileservermapping/slowtiles/views.py b/tileservermapping/slowtiles/views.py new file mode 100644 index 0000000..d57ea57 --- /dev/null +++ b/tileservermapping/slowtiles/views.py @@ -0,0 +1,20 @@ +from django.http import HttpResponse + +from tileservermapping.mapping.models import Server +from tileservermapping.slowtiles.models import SlowTiles + + +def insert_tile(request, z, x, y): + tile_count = SlowTiles.insert(z, x, y) + if tile_count < 0: + return HttpResponse("No server responsible", content_type='text', status=404) + return HttpResponse("Added: {}/{}/{} Count: {}\n".format(z, x, y, tile_count), content_type='text') + + +def get_server_list(request, z, x, y): + server = Server.get_server(z, x, y) + tiles = SlowTiles.objects.filter(server=server, count__gte=5) + answer = "" + for tile in tiles: + answer += "{}/{}/{}\n".format(tile.z, tile.x, tile.y) + return HttpResponse(answer, content_type='text') diff --git a/tileservermapping/urls.py b/tileservermapping/urls.py index a6e0db8..5582826 100644 --- a/tileservermapping/urls.py +++ b/tileservermapping/urls.py @@ -20,4 +20,5 @@ urlpatterns = [ path('admin/', admin.site.urls), url(r'^mappings/', include('tileservermapping.mapping.urls')), + path('slow_tiles/', include('tileservermapping.slowtiles.urls')), ] From 59e5a52c378fc9d9ff7c527dcc1ac4abd0570153 Mon Sep 17 00:00:00 2001 From: Nils Rokita <0rokita@informatik.uni-hamburg.de> Date: Mon, 10 Feb 2020 23:02:21 +0100 Subject: [PATCH 2/2] add upload of generated tiles --- .gitignore | 2 +- tileservermapping/slowtiles/urls.py | 5 +++-- tileservermapping/slowtiles/views.py | 26 ++++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 45e47dc..7215a9d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # custom files db.sqlite settings.py - +.idea # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/tileservermapping/slowtiles/urls.py b/tileservermapping/slowtiles/urls.py index 2f462ff..1c108aa 100644 --- a/tileservermapping/slowtiles/urls.py +++ b/tileservermapping/slowtiles/urls.py @@ -1,7 +1,8 @@ from django.urls import path -from .views import insert_tile, get_server_list +from .views import insert_tile, get_server_list, upload_tile urlpatterns = [ path('add///', insert_tile, name="insert_tile"), - path('server_list///', get_server_list, name="server_list") + path('server_list///', get_server_list, name="server_list"), + path('upload_tile///.', upload_tile, name='upload_tile'), ] diff --git a/tileservermapping/slowtiles/views.py b/tileservermapping/slowtiles/views.py index d57ea57..0c967f5 100644 --- a/tileservermapping/slowtiles/views.py +++ b/tileservermapping/slowtiles/views.py @@ -1,4 +1,9 @@ +from pathlib import Path + +from django.conf import settings from django.http import HttpResponse +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST from tileservermapping.mapping.models import Server from tileservermapping.slowtiles.models import SlowTiles @@ -18,3 +23,24 @@ def get_server_list(request, z, x, y): for tile in tiles: answer += "{}/{}/{}\n".format(tile.z, tile.x, tile.y) return HttpResponse(answer, content_type='text') + + +def handle_uploaded_file(path, f): + with open(path, 'wb') as destination: + for chunk in f.chunks(): + destination.write(chunk) + +@require_POST +@csrf_exempt +def upload_tile(request, z, x, y, format): + # TODO auth oth server + # TODO check if tile in server + if not SlowTiles.objects.filter(x=x, y=y, z=z, count__gte=5).count(): + return HttpResponse('Tile should not be static\n', content_type="text", status=403) + path = Path(settings.TILE_WEBSERVER_DIR) / str(z) / str(x) + if not path.exists(): + path.mkdir(parents=True) + file = path / "{}.{}".format(y, format) + existed = file.exists() + handle_uploaded_file(file, request.FILES['tile']) + return HttpResponse('Updated\n' if existed else 'OK\n', content_type='text')