diff --git a/mapit/admin.py b/mapit/admin.py index 34acbb3d..119ceab4 100644 --- a/mapit/admin.py +++ b/mapit/admin.py @@ -26,7 +26,7 @@ def geometries_link(self, obj): class GeometryAdmin(admin.OSMGeoAdmin): - raw_id_fields = ('area',) + raw_id_fields = ('areas',) class GenerationAdmin(admin.OSMGeoAdmin): diff --git a/mapit/management/command_utils.py b/mapit/management/command_utils.py index 23103288..25814418 100644 --- a/mapit/management/command_utils.py +++ b/mapit/management/command_utils.py @@ -6,6 +6,7 @@ import shapely.ops import shapely.wkt from django.contrib.gis.geos import GEOSGeometry, MultiPolygon +from mapit.models import Geometry class KML(ContentHandler): @@ -42,7 +43,7 @@ def save_polygons(lookup): sys.stdout.write(".") sys.stdout.flush() # g = OGRGeometry(OGRGeomType('MultiPolygon')) - m.polygons.all().delete() + m.polygons.clear() for p in poly: if p.geom_name == 'POLYGON': shapes = [p] @@ -56,7 +57,11 @@ def save_polygons(lookup): continue # Make sure it is two-dimensional g.coord_dim = 2 - m.polygons.create(polygon=g.wkb) + try: + existing = Geometry.objects.get(polygon__equals=g.wkb) + m.polygons.add(existing) + except Geometry.DoesNotExist: + m.polygons.create(polygon=g.wkb) # m.polygon = g.wkt # m.save() # Clear the polygon's list, so that if it has both an ons_code and unit_id, it's not processed twice diff --git a/mapit/management/commands/mapit_NO_import_area_unions.py b/mapit/management/commands/mapit_NO_import_area_unions.py new file mode 100644 index 00000000..5c775c58 --- /dev/null +++ b/mapit/management/commands/mapit_NO_import_area_unions.py @@ -0,0 +1,55 @@ +# import_area_unions.py: +# This script is used to import regions (combinations of existing +# areas into a new area) into MaPit. +# +# Copyright (c) 2011 Petter Reinholdtsen. Some rights reserved using +# the GPL. Based on import_norway_osm.py by Matthew Somerville + +import csv +from django.utils.six import Iterator +from mapit.models import Country +from mapit.management.commands.mapit_create_area_unions import Command + + +# CSV format is +# ID;code;name;area1,area2,...;email;categories + +# Copied from +# http://www.mfasold.net/blog/2010/02/python-recipe-read-csvtsv-textfiles-and-ignore-comment-lines/ +class CommentedFile(Iterator): + def __init__(self, f, commentstring="#"): + self.f = f + self.commentstring = commentstring + + def __next__(self): + line = next(self.f) + while line.startswith(self.commentstring): + line = next(self.f) + return line + + def __iter__(self): + return self + + +class Command(Command): + help = 'Import region data' + label = '' + country = Country.objects.get(code='O') + option_defaults = { 'region-name-field': 1, 'region-id-field': 2, 'area-type-field': 3, 'unionagg': True } + + def process(self, filename, options): + print("Loading file %s" % filename) + region_line = csv.reader(CommentedFile(open(filename, "rb")), + delimiter=';') + + for regionid, area_type, regionname, area_names, email, categories in region_line: + print("Building region '%s'" % regionname) + if (-2147483648 > int(regionid) or 2147483647 < int(regionid)): + raise Exception("Region ID %d is outside range of 32-bit integer" % regionid) + + if not area_names: + raise Exception("No area names found for region with name %s!" % regionname) + + row = [ regionname, regionid, area_type ] + row.extend(area_names.split(',')) + self.handle_row( row, options ) diff --git a/mapit/management/commands/mapit_create_area_unions.py b/mapit/management/commands/mapit_create_area_unions.py new file mode 100644 index 00000000..a26cf060 --- /dev/null +++ b/mapit/management/commands/mapit_create_area_unions.py @@ -0,0 +1,198 @@ +# create_area_unions.py: +# This script is used to create regions which are combinations of existing +# areas into MapIt. It can do so either by using the many-many relationship +# between Areas and Geometries, or by using unionagg() to actually create new, +# larger Areas. + +import csv +import re +import sys +from optparse import make_option +from django.core.management.base import LabelCommand +from mapit.models import Area, Geometry, Generation, Type, Country +from mapit.management.command_utils import save_polygons + +class Command(LabelCommand): + help = 'Create areas from existing areas' + args = '' + country = None + option_defaults = {} + option_list = LabelCommand.option_list + ( + make_option( + '--commit', + action='store_true', + dest='commit', + help='Actually update the database' + ), + make_option( + '--generation-id', + action="store", + dest='generation-id', + help='Which generation ID should be used', + ), + make_option( + '--area-type-code', + action="store", + dest='area-type-code', + help='Which area type should be used (specify using code)', + ), + make_option( + '--header-row', + action = 'store_true', + dest = 'header-row', + default = False, + help = 'Set if the CSV file has a header row' + ), + make_option( + '--region-name-field', + action = 'store', + dest = 'region-name-field', + help = 'Set to the column of the CSV with the union name if present' + ), + make_option( + '--region-id-field', + action = 'store', + dest = 'region-id-field', + help = 'Set to the column of the CSV with the union ID if present' + ), + make_option( + '--area-type-field', + action = 'store', + dest = 'area-type-field', + help = 'Set to the column of the CSV with the area type code, if present' + ), + make_option( + '--country-code', + action="store", + dest='country-code', + default = None, + help = 'Set if you want to specify country of the regions' + ), + make_option( + '--unionagg', + action = 'store_true', + dest = 'unionagg', + default = False, + help = 'Set if you wish to actually create new geometries, rather than use existing ones' + ), + ) + + def find_area(self, name): + m = re.match('[0-9]+', name) + if m: + try: + return Area.objects.get(id=int(m.group())) + except Area.DoesNotExist: + pass + try: + return Area.objects.get(name__iexact=name, + generation_low__lte=self.current_generation, + generation_high__gte=self.new_generation + ) + except Area.MultipleObjectsReturned: + raise Exception, "More than one Area named %s, use area ID as well" % name + except Area.DoesNotExist: + raise Exception, "Area with name %s was not found!" % name + + def handle_label(self, filename, **options): + options.update(self.option_defaults) + + self.current_generation = Generation.objects.current() + if not self.current_generation: + self.current_generation = Generation.objects.new() + if options['generation-id']: + self.new_generation = Generation.objects.get(id=options['generation-id']) + else: + self.new_generation = Generation.objects.new() + if not self.new_generation: + raise Exception, "No new generation to be used for import!" + + area_type_code = options['area-type-code'] + if area_type_code: + if len(area_type_code)>3: + print "Area type code must be 3 letters or fewer, sorry" + sys.exit(1) + try: + self.area_type = Type.objects.get(code=area_type_code) + except: + type_desc = raw_input('Please give a description for area type code %s: ' % area_type_code) + self.area_type = Type(code=area_type_code, description=type_desc) + if options['commit']: + self.area_type.save() + + country_code = options['country-code'] + if country_code: + try: + self.country = Country.objects.get(code=country_code) + except: + country_name = raw_input('Please give the name for country code %s: ' % country_code) + self.country = Country(code=country_code, name=country_name) + if options['commit']: self.country.save() + + self.process(filename, options) + + def process(self, filename, options): + print 'Loading file %s' % filename + reader = csv.reader(open(filename)) + if options['header-row']: next(reader) + for row in reader: + self.handle_row(row, options) + + def handle_row(self, row, options): + region_id = None + region_name = None + if options['region-name-field']: + region_name = row[int(options['region-name-field'])-1] + if options['region-id-field']: + region_id = row[int(options['region-id-field'])-1] + + if options['area-type-field']: + area_type = Type.objects.get(code=row[int(options['area-type-field'])-1]) + else: + area_type = self.area_type + + areas = [] + for pos in range(0, len(row)): + if (options['region-name-field'] and pos == int(options['region-name-field'])-1) \ + or (options['region-id-field'] and pos == int(options['region-id-field'])-1) \ + or (options['area-type-field'] and pos == int(options['area-type-field'])-1): + continue + areas.append( self.find_area(row[pos]) ) + + if region_name is None: + region_name = ' / '.join( [ area.name for area in areas ] ) + + geometry = Geometry.objects.filter(areas__in=areas) + if options['unionagg']: + geometry = geometry.unionagg() + geometry = [ geometry.ogr ] + + try: + if region_id: + area = Area.objects.get(id=int(region_id), type=area_type) + else: + area = Area.objects.get(name=region_name, type=area_type) + except Area.DoesNotExist: + area = Area( + name = region_name, + type = area_type, + generation_low = self.new_generation, + generation_high = self.new_generation, + ) + if region_id: area.id = int(region_id) + if self.country: area.country = self.country + + # check that we are not about to skip a generation + if area.generation_high and self.current_generation and area.generation_high.id < self.current_generation.id: + raise Exception, "Area %s found, but not in current generation %s" % (area, self.current_generation) + area.generation_high = self.new_generation + + if options['commit']: + area.save() + if options['unionagg']: + save_polygons({ area.id : (area, geometry) }) + else: + area.polygons.clear() + for polygon in geometry: + area.polygons.add(polygon) + diff --git a/mapit/management/commands/mapit_import_area_unions.py b/mapit/management/commands/mapit_import_area_unions.py deleted file mode 100644 index 35b1d536..00000000 --- a/mapit/management/commands/mapit_import_area_unions.py +++ /dev/null @@ -1,133 +0,0 @@ -# import_area_unions.py: -# This script is used to import regions (combinations of existing -# areas into a new area) into MaPit. -# -# Copyright (c) 2011 Petter Reinholdtsen. Some rights reserved using -# the GPL. Based on import_norway_osm.py by Matthew Somerville - -import csv - -from django.core.management.base import LabelCommand -from django.contrib.gis.geos import GEOSGeometry -from django.contrib.gis.db.models import Union -from django.utils.six import Iterator - -from mapit.models import Area, Generation, Geometry, Country, Type -from mapit.management.command_utils import save_polygons - - -# CSV format is -# ID;code;name;area1,area2,...;email;categories - -# Copied from -# http://www.mfasold.net/blog/2010/02/python-recipe-read-csvtsv-textfiles-and-ignore-comment-lines/ -class CommentedFile(Iterator): - def __init__(self, f, commentstring="#"): - self.f = f - self.commentstring = commentstring - - def __next__(self): - line = next(self.f) - while line.startswith(self.commentstring): - line = next(self.f) - return line - - def __iter__(self): - return self - - -class Command(LabelCommand): - help = 'Import region data' - label = '' - - def add_arguments(self, parser): - super(Command, self).add_arguments(parser) - parser.add_argument('--commit', action='store_true', dest='commit', help='Actually update the database') - - def handle_label(self, filename, **options): - current_generation = Generation.objects.current() - new_generation = Generation.objects.new() - if not new_generation: - print("Using current generation %d" % current_generation.id) - new_generation = current_generation - else: - print("Using new generation %d" % new_generation.id) - - print("Loading file %s" % filename) - region_line = csv.reader(CommentedFile(open(filename, "rb")), - delimiter=';') - - for regionid, area_type, regionname, area_names, email, categories in region_line: - print("Building region '%s'" % regionname) - if (-2147483648 > int(regionid) or 2147483647 < int(regionid)): - raise Exception("Region ID %d is outside range of 32-bit integer" % regionid) - - if area_names: - # Look up areas using the names, find their geometry - # and build a geometric union to set as the geometry - # of the region. - geometry = None - for name in area_names.split(','): - name.strip() - name.lstrip() - - try: - # Use this to allow '123 Name' in area definition - areaidnum = int(name.split()[0]) - print("Looking up ID '%d'" % areaidnum) - args = { - 'id__exact': areaidnum, - 'generation_low__lte': current_generation, - 'generation_high__gte': new_generation, - } - except (ValueError, IndexError): - print("Looking up name '%s'" % name) - args = { - 'name__iexact': name, - 'generation_low__lte': current_generation, - 'generation_high__gte': new_generation, - } - area_id = Area.objects.filter(**args).only('id') - if 1 < len(area_id): - raise Exception("More than one Area named %s, use area ID as well" % name) - try: - print("ID: %d" % area_id[0].id) - args = { - 'area__exact': area_id[0].id, - } - if geometry: - geometry = geometry | Geometry.objects.filter(**args) - else: - geometry = Geometry.objects.filter(**args) - except: - raise Exception("Area or geometry with name %s was not found!" % name) - unionoutline = geometry.aggregate(Union('polygon'))['polygon__union'] - - def update_or_create(): - try: - m = Area.objects.get(id=int(regionid)) - print("Updating area %s with id %d" % (regionname, int(regionid))) - except Area.DoesNotExist: - print("Creating new area %s with id %d" % (regionname, int(regionid))) - m = Area( - id=int(regionid), - name=regionname, - type=Type.objects.get(code=area_type), - country=Country.objects.get(code='O'), - generation_low=new_generation, - generation_high=new_generation, - ) - - if m.generation_high and current_generation \ - and m.generation_high.id < current_generation.id: - raise Exception("Area %s found, but not in current generation %s" % (m, current_generation)) - m.generation_high = new_generation - - poly = [GEOSGeometry(unionoutline).ogr] - if options['commit']: - m.save() - save_polygons({regionid: (m, poly)}) - - update_or_create() - else: - raise Exception("No area names found for region with name %s!" % regionname) diff --git a/mapit/migrations/0005_add_areas_m2m.py b/mapit/migrations/0005_add_areas_m2m.py new file mode 100644 index 00000000..aec2977d --- /dev/null +++ b/mapit/migrations/0005_add_areas_m2m.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.8 on 2018-01-11 12:52 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('mapit', '0004_add_country_m2m'), + ] + + operations = [ + migrations.AddField( + model_name='geometry', + name='areas', + field=models.ManyToManyField(related_name='polygons', to='mapit.Area'), + ), + migrations.AlterField( + model_name='geometry', + name='area', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='polygons_old', to='mapit.Area', null=True), + ), + ] diff --git a/mapit/migrations/0006_move_geometry_area_ids.py b/mapit/migrations/0006_move_geometry_area_ids.py new file mode 100644 index 00000000..179d8e2a --- /dev/null +++ b/mapit/migrations/0006_move_geometry_area_ids.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.8 on 2018-01-11 12:52 +from __future__ import unicode_literals + +from django.db import migrations +from django.contrib.gis.db.models import Union + + +def move_ids_forward(apps, schema_editor): + Geometry = apps.get_model('mapit', 'Geometry') + for g in Geometry.objects.all().iterator(): + g.areas.add(g.area) + + +def move_ids_backward(apps, schema_editor): + Area = apps.get_model('mapit', 'Area') + Geometry = apps.get_model('mapit', 'Geometry') + # Going backwards, we're moving from the situation where a + # geometry can be in multiple areas, to only being in a single + # area. Please note that this isn't guaranteed to recreate + # exactly the same areas and geometries after going forwards + # and backwards through this migration, but for most purposes + # it'll be functionally the same. + for a in Area.objects.all().iterator(): + # Find any Geometry where area was set to this area, and + # set its area to NULL. This won't be necessary if you've + # just migrated backwards through 0007, but there might be + # some if you've just migrated forward to 0006: + a.polygons_old.clear() + # Do a union of any matching polygons to simplify the number of + # polygons used, and so we can duplicate any geometries that are shared + unioned = a.polygons.aggregate(Union('polygon'))['polygon__union'] + if unioned.geom_type == 'Polygon': + unioned = [unioned] + for polygon in unioned: + a.polygons_old.create(polygon=polygon) + # Now remove any geometries that have area_id set to NULL - + # these will be those only associated with areas via the old + # join table. + Geometry.objects.filter(area=None).delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('mapit', '0005_add_areas_m2m'), + ] + + operations = [ + migrations.RunPython(move_ids_forward, move_ids_backward), + ] diff --git a/mapit/migrations/0007_remove_geometry_area.py b/mapit/migrations/0007_remove_geometry_area.py new file mode 100644 index 00000000..8d4fc6d4 --- /dev/null +++ b/mapit/migrations/0007_remove_geometry_area.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.8 on 2018-01-11 12:56 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('mapit', '0006_move_geometry_area_ids'), + ] + + operations = [ + migrations.RemoveField( + model_name='geometry', + name='area', + ), + ] diff --git a/mapit/models.py b/mapit/models.py index 153c46cd..a482425b 100644 --- a/mapit/models.py +++ b/mapit/models.py @@ -172,11 +172,12 @@ def intersect(self, query_type, area, types, generation): query = ''' WITH - target AS ( SELECT ST_collect(polygon) polygon FROM mapit_geometry WHERE area_id=%%s ), + target AS ( SELECT ST_collect(polygon) polygon FROM mapit_geometry, mapit_geometry_areas WHERE mapit_geometry.id=geometry_id AND area_id=%%s ), geometry AS ( - SELECT mapit_geometry.* - FROM mapit_geometry, mapit_area, target - WHERE mapit_geometry.area_id = mapit_area.id + SELECT DISTINCT mapit_geometry.* + FROM mapit_geometry, mapit_geometry_areas, mapit_area, target + WHERE area_id = mapit_area.id + AND geometry_id = mapit_geometry.id AND mapit_geometry.polygon && target.polygon AND mapit_area.id != %%s AND mapit_area.generation_low_id <= %%s @@ -184,8 +185,8 @@ def intersect(self, query_type, area, types, generation): %s ) SELECT DISTINCT mapit_area.* - FROM mapit_area, geometry, target - WHERE geometry.area_id = mapit_area.id AND (%s) + FROM mapit_area, mapit_geometry_areas, geometry, target + WHERE geometry.id = geometry_id and area_id = mapit_area.id AND (%s) ''' % (query_area_type, query_geo) return RawQuerySet(raw_query=query, model=self.model, params=params, using=self._db) @@ -321,14 +322,15 @@ def export(self, @python_2_unicode_compatible class Geometry(models.Model): - area = models.ForeignKey(Area, related_name='polygons', on_delete=models.CASCADE) + areas = models.ManyToManyField(Area, related_name='polygons') polygon = models.PolygonField(srid=settings.MAPIT_AREA_SRID) class Meta: verbose_name_plural = 'geometries' def __str__(self): - return '%s, polygon %d' % (smart_text(self.area), self.id) + areas = ', '.join(map(smart_text, self.areas.all())) + return u'#%d (%s)' % (self.id, areas) @python_2_unicode_compatible diff --git a/mapit_gb/management/commands/mapit_UK_ni_consolidate_boundaries.py b/mapit_gb/management/commands/mapit_UK_ni_consolidate_boundaries.py index f8a71dc8..74c33c62 100644 --- a/mapit_gb/management/commands/mapit_UK_ni_consolidate_boundaries.py +++ b/mapit_gb/management/commands/mapit_UK_ni_consolidate_boundaries.py @@ -24,13 +24,17 @@ def save_polygons(area, **args): geometry = Geometry.objects.filter(**args) p = geometry.aggregate(Union('polygon'))['polygon__union'] if options['commit']: - area.polygons.all().delete() + area.polygons.clear() if p.geom_type == 'Polygon': shapes = [p] else: shapes = p for g in shapes: - area.polygons.create(polygon=g) + try: + existing = Geometry.objects.get(polygon__equals=g) + area.polygons.add(existing) + except Geometry.DoesNotExist: + area.polygons.create(polygon=g) done.append(area.id) print('done')