Added support for geo_point and geo_shape to the SQLAlchemy dialect

This should address #273

Author: autophagy

CHANGES.txt

@@ -2,6 +2,12 @@
 Changes for crate
 =================
 
+Unreleased
+==========
+
+- Added support for ``geo_point`` and ``geo_json`` types to the SQLAlchemy
+  dialect.
+
 2020/05/27 0.24.0
 =================
 

docs/appendices/data-types.rst

@@ -127,6 +127,8 @@ CrateDB           SQLAlchemy
 `array`__         `ARRAY`__
 `object`__        :ref:`object` |nbsp| (extension type)
 `array(object)`__ :ref:`objectarray` |nbsp| (extension type)
+`geo_point`__     :ref:`geopoint` |nbsp| (extension type)
+`geo_shape`__     :ref:`geoshape` |nbsp| (extension type)
 ================= =========================================
 
 __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#boolean
@@ -151,6 +153,8 @@ __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#a
 __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.ARRAY
 __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#object
 __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#array
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#geo-point
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#geo-shape
 
 .. _json: https://docs.python.org/3/library/json.html
 .. _HTTP endpoint: https://crate.io/docs/crate/reference/en/latest/interfaces/http.html

docs/sqlalchemy.rst

@@ -284,6 +284,59 @@ The resulting object will look like this::
     change, the `UPDATE`_ statement sent to CrateDB will include all of the
     ``ObjectArray`` data.
 
+.. _geopoint:
+.. _geoshape:
+
+``Geopoint`` and ``Geoshape``
+.............................
+
+The CrateDB SQLAlchemy dialect provides two geospatial types:
+
+- ``Geopoint``, which represents a longitude and latitude coordinate
+- ``Geoshape``, which is used to store geometric `GeoJSON geometry objects`_
+
+To use these types, you can create columns, like so::
+
+    >>> class City(Base):
+    ...
+    ...    __tablename__ = 'cities'
+    ...    name = sa.Column(sa.String, primary_key=True)
+    ...    coordinate = sa.Column(types.Geopoint)
+    ...    area = sa.Column(types.Geoshape)
+
+There are multiple ways of creating a geopoint. Firstly, you can define it as
+a tuple of ``(longitude, latitude)``::
+
+    >>> point = (139.76, 35.68)
+
+Secondly, you can define it as a geojson ``Point`` object::
+
+    >>> from geojson import Point
+    >>> point = Point(coordinates=(139.76, 35.68))
+
+To create a geoshape, you can use a geojson shape object, such as a ``Polygon``::
+
+    >>> from geojson import Point, Polygon
+    >>> area = Polygon(
+    ...     [
+    ...         [
+    ...             (139.806, 35.515),
+    ...             (139.919, 35.703),
+    ...             (139.768, 35.817),
+    ...             (139.575, 35.760),
+    ...             (139.584, 35.619),
+    ...             (139.806, 35.515),
+    ...         ]
+    ...     ]
+    ... )
+
+You can then set the values of the ``Geopoint`` and ``Geoshape`` columns::
+
+    >>> tokyo = City(name="Tokyo", coordinate=point, area=area)
+    >>> session.add(tokyo)
+    >>> session.commit()
+
+
 Querying
 ========
 
@@ -535,3 +588,4 @@ column on the ``Character`` class.
 .. _score: https://crate.io/docs/crate/reference/en/latest/general/dql/fulltext.html#usage
 .. _working with tables: http://docs.sqlalchemy.org/en/latest/core/metadata.html
 .. _UUIDs: https://docs.python.org/3/library/uuid.html
+.. _geojson geometry objects: https://tools.ietf.org/html/rfc7946#section-3.1

setup.py

@@ -69,7 +69,7 @@ def read(path):
     extras_require=dict(
         test=['zope.testing',
               'zc.customdoctests>=1.0.1'],
-        sqlalchemy=['sqlalchemy>=1.0,<1.4']
+        sqlalchemy=['sqlalchemy>=1.0,<1.4', 'geojson>=2.5.0']
     ),
     python_requires='>=3.4',
     install_requires=requirements,

src/crate/client/doctests/sqlalchemy.txt

@@ -182,6 +182,53 @@ all values of that field of all objects in that object array::
     >>> query.all()
     [([1, 2, 3],), (None,)]
 
+
+Geospatial Types
+================
+
+Geospatial types, such as ``geo_point`` and ``geo_area`` can also be used as
+part of a sqlalchemy schema::
+
+    >>> from crate.client.sqlalchemy.types import Geopoint, Geoshape
+
+    >>> class City(Base):
+    ...    __tablename__ = 'cities'
+    ...    name = sa.Column(sa.String, primary_key=True)
+    ...    coordinate = sa.Column(Geopoint)
+    ...    area = sa.Column(Geoshape)
+
+One way of inserting these types is using the Geojson library, to create
+points or shapes::
+
+    >>> from geojson import Point, Polygon
+    >>> area = Polygon(
+    ...     [
+    ...         [
+    ...             (139.806, 35.515),
+    ...             (139.919, 35.703),
+    ...             (139.768, 35.817),
+    ...             (139.575, 35.760),
+    ...             (139.584, 35.619),
+    ...             (139.806, 35.515),
+    ...         ]
+    ...     ]
+    ... )
+    >>> point = Point(coordinates=(139.76, 35.68))
+
+These two objects can then be added to an sqlalchemy model and added to the
+session::
+
+    >>> tokyo = City(coordinate=point, area=area, name='Tokyo')
+    >>> session.add(tokyo)
+    >>> session.commit()
+
+When retrieved, they are retrieved as the corresponding geojson objects::
+
+    >>> refresh("cities")
+    >>> query = session.query(City.name, City.coordinate, City.area)
+    >>> query.all()
+     [('Tokyo', (139.76, 35.68), {"coordinates": [[[139.806, 35.515], [139.919, 35.703], [139.768, 35.817], [139.575, 35.76], [139.584, 35.619], [139.806, 35.515]]], "type": "Polygon"})]
+
 Count and Group By
 ==================
 

src/crate/client/sqlalchemy/doctests/reflection.txt

@@ -12,7 +12,7 @@ List all schemas::
 List all tables::
 
     >>> inspector.get_table_names()
-    ['characters', 'locations']
+    ['characters', 'cities', 'locations']
 
     >>> set(['checks', 'cluster', 'jobs', 'jobs_log']).issubset(inspector.get_table_names(schema='sys'))
     True

src/crate/client/sqlalchemy/types.py

@@ -24,6 +24,8 @@
 from sqlalchemy.sql import default_comparator
 from sqlalchemy.ext.mutable import Mutable
 
+import geojson
+
 
 class MutableList(Mutable, list):
 
@@ -217,3 +219,46 @@ def get_col_spec(self, **kws):
 
 
 ObjectArray = MutableList.as_mutable(_ObjectArray)
+
+
+class Geopoint(sqltypes.UserDefinedType):
+
+    class Comparator(sqltypes.TypeEngine.Comparator):
+
+        def __getitem__(self, key):
+            return default_comparator._binary_operate(self.expr,
+                                                      operators.getitem,
+                                                      key)
+
+    def get_col_spec(self):
+        return 'GEO_POINT'
+
+    def bind_processor(self, dialect):
+        def process(value):
+            if isinstance(value, geojson.Point):
+                return value.coordinates
+            return value
+        return process
+
+    def result_processor(self, dialect, coltype):
+        return tuple
+
+    comparator_factory = Comparator
+
+
+class Geoshape(sqltypes.UserDefinedType):
+
+    class Comparator(sqltypes.TypeEngine.Comparator):
+
+        def __getitem__(self, key):
+            return default_comparator._binary_operate(self.expr,
+                                                      operators.getitem,
+                                                      key)
+
+    def get_col_spec(self):
+        return 'GEO_SHAPE'
+
+    def result_processor(self, dialect, coltype):
+        return geojson.GeoJSON.to_instance
+
+    comparator_factory = Comparator

src/crate/client/tests.py

@@ -174,6 +174,14 @@ def setUpCrateLayerAndSqlAlchemy(test):
           INDEX quote_ft using fulltext(quote) with (analyzer = 'english')
     ) """)
 
+    with connect(crate_host) as conn:
+        cursor = conn.cursor()
+        cursor.execute("""create table cities (
+          name string primary key,
+          coordinate geo_point,
+          area geo_shape
+    ) """)
+
     engine = sa.create_engine('crate://{0}'.format(crate_host))
     Base = declarative_base()
 
@@ -279,6 +287,7 @@ def tearDownWithCrateLayer(test):
         for stmt in ["DROP TABLE locations",
                      "DROP BLOB TABLE myfiles",
                      "DROP TABLE characters",
+                     "DROP TABLE cities",
                      "DROP USER me",
                      "DROP USER trusted_me",
                      ]:

versions.cfg

@@ -10,6 +10,7 @@ MarkupSafe = 0.23
 Pygments = 1.6
 Sphinx = 1.2.3
 SQLAlchemy = 1.3.17
+geojson = 2.5.0
 coverage = 5.0.3
 crate-docs-theme = 0.5.0
 createcoverage = 1.5