From 62326a600b58e928dfd00174abc06c91d2d0e5d0 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 12 Sep 2019 17:01:24 -0700
Subject: [PATCH 01/78] separate register_class

---
 flask_mongorest/__init__.py | 47 +++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/flask_mongorest/__init__.py b/flask_mongorest/__init__.py
index f7da37f2..157499aa 100644
--- a/flask_mongorest/__init__.py
+++ b/flask_mongorest/__init__.py
@@ -1,6 +1,32 @@
 from flask import Blueprint
 from flask_mongorest.methods import Create, BulkUpdate, List
 
+def register_class(app, klass, **kwargs):
+    # Construct a url based on a 'name' kwarg with a fallback to the
+    # view's class name. Note that the name must be unique.
+    name = kwargs.pop('name', klass.__name__)
+    url = kwargs.pop('url', None)
+    if not url:
+        document_name = klass.resource.document.__name__.lower()
+        url = '/%s/' % document_name
+
+    # Insert the url prefix, if it exists
+    url_prefix = kwargs.pop('url_prefix', '')
+    if url_prefix:
+        url = '%s%s' % (url_prefix, url)
+
+    # Add url rules
+    pk_type = kwargs.pop('pk_type', 'string')
+    view_func = klass.as_view(name)
+    if List in klass.methods:
+        app.add_url_rule(url, defaults={'pk': None}, view_func=view_func,
+                         methods=[List.method], endpoint=view_func.__name__+'List', **kwargs)
+    if Create in klass.methods or BulkUpdate in klass.methods:
+        app.add_url_rule(url, view_func=view_func, methods=[
+            x.method for x in klass.methods if x in (Create, BulkUpdate)
+        ], endpoint=view_func.__name__+'CreateBulkUpdate', **kwargs)
+    app.add_url_rule('%s<%s:%s>/' % (url, pk_type, 'pk'), view_func=view_func,
+                     methods=[x.method for x in klass.methods if x not in (List, BulkUpdate)], **kwargs)
 
 class MongoRest(object):
     def __init__(self, app, **kwargs):
@@ -10,26 +36,7 @@ def __init__(self, app, **kwargs):
 
     def register(self, **kwargs):
         def decorator(klass):
-            # Construct a url based on a 'name' kwarg with a fallback to the
-            # view's class name. Note that the name must be unique.
-            name = kwargs.pop('name', klass.__name__)
-            url = kwargs.pop('url', None)
-            if not url:
-                document_name = klass.resource.document.__name__.lower()
-                url = '/%s/' % document_name
-
-            # Insert the url prefix, if it exists
-            if self.url_prefix:
-                url = '%s%s' % (self.url_prefix, url)
-
-            # Add url rules
-            pk_type = kwargs.pop('pk_type', 'string')
-            view_func = klass.as_view(name)
-            if List in klass.methods:
-                self.app.add_url_rule(url, defaults={'pk': None}, view_func=view_func, methods=[List.method], **kwargs)
-            if Create in klass.methods or BulkUpdate in klass.methods:
-                self.app.add_url_rule(url, view_func=view_func, methods=[x.method for x in klass.methods if x in (Create, BulkUpdate)], **kwargs)
-            self.app.add_url_rule('%s<%s:%s>/' % (url, pk_type, 'pk'), view_func=view_func, methods=[x.method for x in klass.methods if x not in (List, BulkUpdate)], **kwargs)
+            register_class(self.app, klass, **kwargs)
             return klass
 
         return decorator

From 391e8a278dd454ff8dfa3de1098e66793836b9e7 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 14 Nov 2019 13:42:39 -0800
Subject: [PATCH 02/78] marshmallow_mongoengine schema, optional cleancat

---
 flask_mongorest/resources.py | 30 ++++++++++++++++++++++++------
 requirements.txt             |  1 -
 setup.py                     |  3 +--
 3 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index d293c057..57711a1a 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -19,7 +19,17 @@
 from mongoengine.fields import EmbeddedDocumentField, ListField, ReferenceField, GenericReferenceField
 from mongoengine.fields import DictField
 
-from cleancat import ValidationError as SchemaValidationError
+try:
+    from cleancat import Schema as CleancatSchema
+    from cleancat import ValidationError as SchemaValidationError
+except ImportError:
+    CleancatSchema = None
+
+try:
+    from marshmallow_mongoengine import ModelSchema
+except ImportError:
+    ModelSchema = None
+
 from flask_mongorest import methods
 from flask_mongorest.exceptions import ValidationError, UnknownFieldError
 from flask_mongorest.utils import cmp_fields, isbound, isint, equal
@@ -556,16 +566,24 @@ def validate_request(self, obj=None):
         # If CleanCat schema exists on this resource, use it to perform the
         # validation
         if self.schema:
+            if CleancatSchema is None and ModelSchema is None:
+                raise ImportError('Cannot validate schema without CleanCat or Marshmallow!')
+
             if request.method == 'PUT' and obj is not None:
                 obj_data = dict([(key, getattr(obj, key)) for key in obj._fields.keys()])
             else:
                 obj_data = None
 
-            schema = self.schema(self.data, obj_data)
-            try:
-                self.data = schema.full_clean()
-            except SchemaValidationError:
-                raise ValidationError({'field-errors': schema.field_errors, 'errors': schema.errors })
+            if CleancatSchema is not None:
+                try:
+                    schema = self.schema(self.data, obj_data)
+                    self.data = schema.full_clean()
+                except SchemaValidationError:
+                    raise ValidationError({'field-errors': schema.field_errors, 'errors': schema.errors })
+            elif ModelSchema is not None:
+                self.data, errors = self.schema().load(self.data)
+                if errors:
+                    raise ValidationError({'errors': errors})
 
     def get_queryset(self):
         """
diff --git a/requirements.txt b/requirements.txt
index d25a6a9a..53e29c4a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,6 @@
 mimerender
 python-dateutil
 sphinx
-cleancat>=0.3
 Flask>=0.9
 pymongo>=3.4
 flake8
diff --git a/setup.py b/setup.py
index 4890a0f7..ee297864 100644
--- a/setup.py
+++ b/setup.py
@@ -31,8 +31,7 @@
         'Flask-MongoEngine',
         'mimerender',
         'nose',
-        'python-dateutil',
-        'cleancat'
+        'python-dateutil'
     ],
     classifiers=[
         'Development Status :: 4 - Beta',

From 82be8919e608ef9196dae37ddd4dd95d300bad0e Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 15 Nov 2019 13:22:45 -0800
Subject: [PATCH 03/78] partial validation and skip primary_key in PUT

---
 flask_mongorest/resources.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 57711a1a..a52c14bc 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -581,7 +581,8 @@ def validate_request(self, obj=None):
                 except SchemaValidationError:
                     raise ValidationError({'field-errors': schema.field_errors, 'errors': schema.errors })
             elif ModelSchema is not None:
-                self.data, errors = self.schema().load(self.data)
+                partial = bool(request.method == 'PUT' and obj is not None)
+                self.data, errors = self.schema().load(self.data, partial=partial)
                 if errors:
                     raise ValidationError({'errors': errors})
 
@@ -946,7 +947,7 @@ def update_object(self, obj, data=None, save=True, parent_resources=None):
                 id_from_data = value and getattr(value, 'pk', value)
                 if id_from_obj != id_from_data:
                     update = True
-            elif not equal(getattr(obj, field), value):
+            elif not obj._fields[field].primary_key and not equal(getattr(obj, field), value):
                 update = True
 
             if update:

From acea9b4a24cfc7ff2abf85022f94df1caf743657 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 20 Nov 2019 17:32:00 -0800
Subject: [PATCH 04/78] try_float for numeric operators

---
 flask_mongorest/operators.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/flask_mongorest/operators.py b/flask_mongorest/operators.py
index 9e0ed823..07f9a35e 100644
--- a/flask_mongorest/operators.py
+++ b/flask_mongorest/operators.py
@@ -75,21 +75,39 @@ def apply(self, queryset, field, value, negate=False):
         kwargs = self.prepare_queryset_kwargs(field, value, negate)
         return queryset.filter(**kwargs)
 
+def try_float(value):
+    try:
+        return float(value)
+    except ValueError:
+        return value
+
 class Ne(Operator):
     op = 'ne'
 
 class Lt(Operator):
     op = 'lt'
 
+    def prepare_queryset_kwargs(self, field, value, negate):
+        return {'__'.join(filter(None, [field, self.op])): try_float(value)}
+
 class Lte(Operator):
     op = 'lte'
 
+    def prepare_queryset_kwargs(self, field, value, negate):
+        return {'__'.join(filter(None, [field, self.op])): try_float(value)}
+
 class Gt(Operator):
     op = 'gt'
 
+    def prepare_queryset_kwargs(self, field, value, negate):
+        return {'__'.join(filter(None, [field, self.op])): try_float(value)}
+
 class Gte(Operator):
     op = 'gte'
 
+    def prepare_queryset_kwargs(self, field, value, negate):
+        return {'__'.join(filter(None, [field, self.op])): try_float(value)}
+
 class Exact(Operator):
     op = 'exact'
 

From 53ce90725b5d722dcea01e1a0138e3734caf548f Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 21 Nov 2019 15:37:23 -0800
Subject: [PATCH 05/78] enable nested _fields

---
 flask_mongorest/resources.py | 7 ++++---
 requirements.txt             | 1 +
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index a52c14bc..9fc9577d 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -4,6 +4,7 @@
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
 from flask import has_request_context, request, url_for
+from dict_deep import deep_get
 try:
     from urllib.parse import urlparse
 except ImportError: # Python 2
@@ -263,7 +264,7 @@ def get_requested_fields(self, **kwargs):
         else:
             for field in only_fields:
                 actual_field = self._reverse_rename_fields.get(field, field)
-                if actual_field in all_fields_set:
+                if actual_field in all_fields_set or any(actual_field.startswith(f) for f in all_fields_set):
                     requested_fields.append(actual_field)
 
         return requested_fields
@@ -366,10 +367,10 @@ def get_field_value(self, obj, field_name, field_instance=None, **kwargs):
         if has_field_instance:
             field_value = obj
         elif isinstance(obj, dict):
-            return obj[field_name]
+            return deep_get(obj, field_name)
         else:
             try:
-                field_value = getattr(obj, field_name)
+                field_value = deep_get(obj, field_name)
             except AttributeError:
                 raise UnknownFieldError
 
diff --git a/requirements.txt b/requirements.txt
index 53e29c4a..2d486536 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ sphinx
 Flask>=0.9
 pymongo>=3.4
 flake8
+dict-deep==2.0.2

From e0a342bd9dfa433e84d3db6c8625ef9f239c438d Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 21 Nov 2019 15:37:42 -0800
Subject: [PATCH 06/78] apply mask in get_queryset

---
 flask_mongorest/resources.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 9fc9577d..c75d4d12 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -592,7 +592,8 @@ def get_queryset(self):
         Return a MongoEngine queryset that will later be used to return
         matching documents.
         """
-        return self.document.objects
+        mask = self.get_requested_fields(params=self.params)
+        return self.document.objects.only(*mask)
 
     def get_object(self, pk, qfilter=None):
         """

From 3709b3361002db2a4d525fbca9827ccabb8087b2 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 22 Nov 2019 12:45:50 -0800
Subject: [PATCH 07/78] staticmethod get_optional_fields

---
 flask_mongorest/resources.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index c75d4d12..9771d22e 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -220,7 +220,8 @@ def get_fields(self):
         """
         return self.fields
 
-    def get_optional_fields(self):
+    @staticmethod
+    def get_optional_fields():
         """
         Return a list of fields that can optionally be included in the
         response (but only if a `_fields` param mentioned them explicitly).

From 2fa675f49fcf283c08e1d0f81844a1eb3c3ebf20 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Mon, 25 Nov 2019 15:05:54 -0800
Subject: [PATCH 08/78] get full object when updating

---
 flask_mongorest/resources.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 9771d22e..8e7d3a44 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -593,8 +593,11 @@ def get_queryset(self):
         Return a MongoEngine queryset that will later be used to return
         matching documents.
         """
-        mask = self.get_requested_fields(params=self.params)
-        return self.document.objects.only(*mask)
+        if request.method == 'PUT':
+            return self.document.objects  # get full documents for updates
+        else:
+            mask = self.get_requested_fields(params=self.params)
+            return self.document.objects.only(*mask)
 
     def get_object(self, pk, qfilter=None):
         """

From b6ad8f51e790488e5470b5fd4a87357453174cbe Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Mon, 25 Nov 2019 17:06:41 -0800
Subject: [PATCH 09/78] add Decimal128 to MongoEncoder

---
 flask_mongorest/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/flask_mongorest/utils.py b/flask_mongorest/utils.py
index 272473d2..8e4d4ada 100644
--- a/flask_mongorest/utils.py
+++ b/flask_mongorest/utils.py
@@ -3,6 +3,7 @@
 import datetime
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
+from bson.decimal128 import Decimal128
 import mongoengine
 
 isbound = lambda m: getattr(m, 'im_self', None) is not None
@@ -26,6 +27,9 @@ def default(self, value, **kwargs):
             return value.strftime("%Y-%m-%d")
         if isinstance(value, decimal.Decimal):
             return str(value)
+        if isinstance(value, Decimal128):
+            return str(value.to_decimal())
+
         return super(MongoEncoder, self).default(value, **kwargs)
 
 

From 107507c026787ca5ba52b53085b67757bf7e8371 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Tue, 26 Nov 2019 16:50:02 -0800
Subject: [PATCH 10/78] paginate fields

---
 flask_mongorest/resources.py | 29 +++++++++++++++++++++++++++++
 flask_mongorest/utils.py     | 10 +++++-----
 flask_mongorest/views.py     |  2 ++
 3 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 8e7d3a44..338cdea4 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -76,6 +76,9 @@ class Resource(object):
     # Maximum number of objects which can be bulk-updated by a single request
     bulk_update_limit = 1000
 
+    # Map of field names to paginate with according default and maximum limits
+    fields_to_paginate = {}
+
     # Map of field names and Resource classes that should be used to handle
     # these fields (for serialization, saving, etc.).
     related_resources = {}
@@ -622,6 +625,32 @@ def get_object(self, pk, qfilter=None):
 
         return obj
 
+    def paginate_fields(self, obj):
+        """
+        return object with fields paginated according to `fields_to_paginate`
+        """
+        for field, limits in self.fields_to_paginate.items():
+            page = self.params.get(f'{field}_page', 1)
+            per_page = self.params.get(f'{field}_per_page', limits[0])
+            # page and per_page validation
+            if not isint(page):
+                raise ValidationError({'error': f'{field}_page must be an integer.'})
+            if not isint(per_page):
+                raise ValidationError({'error': f'{field}_per_page must be an integer.'})
+            if int(per_page) > limits[1]:
+                raise ValidationError({'error': f"The per page limit ({per_page}) you set is larger than the maximum for the {field} field ({limits[1]})."})
+            if int(page) < 0:
+                raise ValidationError({'error': f'{field}_page must be a non-negative integer.'})
+
+            per_page = min(int(per_page), limits[1])
+            nrows = len(obj[field])
+            max_page = int(nrows/per_page) + bool(nrows % per_page)
+            if int(page) > max_page:
+                obj.data = []
+            else:
+                obj.data = obj.paginate_field(field, int(page), per_page=per_page).items
+        return obj
+
     def fetch_related_resources(self, objs, only_fields=None):
         """
         Given a list of objects and an optional list of the only fields we
diff --git a/flask_mongorest/utils.py b/flask_mongorest/utils.py
index 8e4d4ada..1a7a65cb 100644
--- a/flask_mongorest/utils.py
+++ b/flask_mongorest/utils.py
@@ -19,15 +19,15 @@ class MongoEncoder(json.JSONEncoder):
     def default(self, value, **kwargs):
         if isinstance(value, ObjectId):
             return str(value)
-        if isinstance(value, DBRef):
+        elif isinstance(value, DBRef):
             return value.id
-        if isinstance(value, datetime.datetime):
+        elif isinstance(value, datetime.datetime):
             return value.isoformat()
-        if isinstance(value, datetime.date):
+        elif isinstance(value, datetime.date):
             return value.strftime("%Y-%m-%d")
-        if isinstance(value, decimal.Decimal):
+        elif isinstance(value, decimal.Decimal):
             return str(value)
-        if isinstance(value, Decimal128):
+        elif isinstance(value, Decimal128):
             return str(value.to_decimal())
 
         return super(MongoEncoder, self).default(value, **kwargs)
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 93463b18..ed3b0062 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -128,6 +128,7 @@ def get(self, **kwargs):
 
             data = []
             for obj in objs:
+                obj = self._resource.paginate_fields(obj)
                 try:
                     data.append(self._resource.serialize(obj, params=request.args))
                 except Exception as e:
@@ -147,6 +148,7 @@ def get(self, **kwargs):
                 ret.update(extra)
         else:
             obj = self._resource.get_object(pk, qfilter=qfilter)
+            obj = self._resource.paginate_fields(obj)
             ret = self._resource.serialize(obj, params=request.args)
         return ret
 

From 5a30e2b06a2149b0da617c108e99f0a254f2344a Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 27 Nov 2019 20:50:57 -0800
Subject: [PATCH 11/78] operators: add swagger type

---
 flask_mongorest/operators.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/flask_mongorest/operators.py b/flask_mongorest/operators.py
index 07f9a35e..85a6000b 100644
--- a/flask_mongorest/operators.py
+++ b/flask_mongorest/operators.py
@@ -53,6 +53,7 @@ class Operator(object):
     """Base class that all the other operators should inherit from."""
 
     op = 'exact'
+    typ = 'string'
 
     # Can be overridden via constructor.
     allow_negation = False
@@ -86,24 +87,28 @@ class Ne(Operator):
 
 class Lt(Operator):
     op = 'lt'
+    typ = 'number'
 
     def prepare_queryset_kwargs(self, field, value, negate):
         return {'__'.join(filter(None, [field, self.op])): try_float(value)}
 
 class Lte(Operator):
     op = 'lte'
+    typ = 'number'
 
     def prepare_queryset_kwargs(self, field, value, negate):
         return {'__'.join(filter(None, [field, self.op])): try_float(value)}
 
 class Gt(Operator):
     op = 'gt'
+    typ = 'number'
 
     def prepare_queryset_kwargs(self, field, value, negate):
         return {'__'.join(filter(None, [field, self.op])): try_float(value)}
 
 class Gte(Operator):
     op = 'gte'
+    typ = 'number'
 
     def prepare_queryset_kwargs(self, field, value, negate):
         return {'__'.join(filter(None, [field, self.op])): try_float(value)}
@@ -124,6 +129,7 @@ class IExact(Operator):
 
 class In(Operator):
     op = 'in'
+    typ = 'array'
 
     def prepare_queryset_kwargs(self, field, value, negate):
         # this is null if the user submits an empty in expression (like
@@ -158,6 +164,7 @@ class IEndswith(Operator):
 
 class Boolean(Operator):
     op = 'exact'
+    typ = 'boolean'
 
     def prepare_queryset_kwargs(self, field, value, negate):
         if value == 'false':

From d8bf4df76857d80fa771e8df930b0275b707f143 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 4 Dec 2019 17:30:50 -0800
Subject: [PATCH 12/78] mongoengine3

---
 flask_mongorest/resources.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 338cdea4..6f467168 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -587,9 +587,7 @@ def validate_request(self, obj=None):
                     raise ValidationError({'field-errors': schema.field_errors, 'errors': schema.errors })
             elif ModelSchema is not None:
                 partial = bool(request.method == 'PUT' and obj is not None)
-                self.data, errors = self.schema().load(self.data, partial=partial)
-                if errors:
-                    raise ValidationError({'errors': errors})
+                self.data = self.schema().load(self.data, partial=partial)
 
     def get_queryset(self):
         """

From 5a53c5fe0b47e243e57a33313db3802e10ff0178 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 4 Dec 2019 17:31:15 -0800
Subject: [PATCH 13/78] get_queryset: fix mask

---
 flask_mongorest/resources.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 6f467168..58467e09 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -597,7 +597,9 @@ def get_queryset(self):
         if request.method == 'PUT':
             return self.document.objects  # get full documents for updates
         else:
-            mask = self.get_requested_fields(params=self.params)
+            document_fields = set(self.document._fields.keys())
+            requested_fields = set(self.get_requested_fields(params=self.params))
+            mask = requested_fields & document_fields
             return self.document.objects.only(*mask)
 
     def get_object(self, pk, qfilter=None):

From dd6485d2c826762f6eb0f9fc7810ddf0849f8980 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 4 Dec 2019 21:44:32 -0800
Subject: [PATCH 14/78] get_queryset: fix mask for nested fields

---
 flask_mongorest/resources.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 58467e09..e2913020 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -598,7 +598,9 @@ def get_queryset(self):
             return self.document.objects  # get full documents for updates
         else:
             document_fields = set(self.document._fields.keys())
-            requested_fields = set(self.get_requested_fields(params=self.params))
+            requested_fields = set(
+                f.split('.', 1)[0] for f in self.get_requested_fields(params=self.params)
+            )
             mask = requested_fields & document_fields
             return self.document.objects.only(*mask)
 

From 0e68b6d0402499280899a97f1719b02feca8f344 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 4 Dec 2019 23:35:49 -0800
Subject: [PATCH 15/78] get_requested_fields: maintain order

---
 flask_mongorest/resources.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index e2913020..e54f8073 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -243,15 +243,18 @@ def get_requested_fields(self, **kwargs):
 
         include_all = False
 
+        # NOTE use list(dict.fromkeys()) below instead of set() to maintain order
         if 'fields' in kwargs:
             fields = kwargs['fields']
-            all_fields_set = set(fields)
+            all_fields_set = list(dict.fromkeys(fields))
         else:
             fields = self.get_fields()
-            all_fields_set = set(fields) | set(self.get_optional_fields())
+            all_fields = fields + self.get_optional_fields()
+            all_fields_set = list(dict.fromkeys(all_fields))
 
         if params and '_fields' in params:
-            only_fields = set(params['_fields'].split(','))
+            params_fields = params['_fields'].split(',')
+            only_fields = list(dict.fromkeys(params_fields))
             if '_all' in only_fields:
                 include_all = True
         else:

From b26c8a071f48cbe5e88ba0a9b29b64b884500178 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 6 Dec 2019 11:39:11 -0800
Subject: [PATCH 16/78] minor bugfix

---
 flask_mongorest/resources.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index e54f8073..850e507b 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -248,7 +248,7 @@ def get_requested_fields(self, **kwargs):
             fields = kwargs['fields']
             all_fields_set = list(dict.fromkeys(fields))
         else:
-            fields = self.get_fields()
+            fields = list(self.get_fields())
             all_fields = fields + self.get_optional_fields()
             all_fields_set = list(dict.fromkeys(all_fields))
 

From c10136d126378f311c227b98367307b88191486a Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 6 Dec 2019 13:34:50 -0800
Subject: [PATCH 17/78] get_with_list_index; bugfix serialize_list_field

---
 flask_mongorest/resources.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 850e507b..855ccd67 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -36,6 +36,13 @@
 from flask_mongorest.utils import cmp_fields, isbound, isint, equal
 
 
+def get_with_list_index(o, k):
+    try:
+        return o[int(k)]
+    except ValueError:
+        return o[k]
+
+
 class ResourceMeta(type):
     def __init__(cls, name, bases, classdict):
         if classdict.get('__metaclass__') is not ResourceMeta:
@@ -377,8 +384,8 @@ def get_field_value(self, obj, field_name, field_instance=None, **kwargs):
             return deep_get(obj, field_name)
         else:
             try:
-                field_value = deep_get(obj, field_name)
-            except AttributeError:
+                field_value = deep_get(obj, field_name, getter=get_with_list_index)
+            except (AttributeError, KeyError):
                 raise UnknownFieldError
 
         return self.serialize_field_value(obj, field_name, field_instance, field_value, **kwargs)
@@ -441,7 +448,10 @@ def serialize_dict_field(self, field_instance, field_name, field_value, **kwargs
 
     def serialize_list_field(self, field_instance, field_name, field_value, **kwargs):
         """Serialize each item in the list separately."""
-        return [val for val in [self.get_field_value(elem, field_name, field_instance=field_instance.field, **kwargs) for elem in field_value] if val]
+        return [val for val in [
+            self.get_field_value(elem, field_name, field_instance=field_instance.field, **kwargs)
+            for elem in field_value
+        ] if val is not None]
 
     def serialize_document_field(self, field_name, field_value, **kwargs):
         """If this field is a reference or an embedded document, either return

From 8576f449eea980b238ab7572deea9852bae1f7b0 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Sat, 7 Dec 2019 16:27:04 -0800
Subject: [PATCH 18/78] raise error if total_pages too high

---
 flask_mongorest/resources.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 855ccd67..1c8e3a5e 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -658,12 +658,11 @@ def paginate_fields(self, obj):
                 raise ValidationError({'error': f'{field}_page must be a non-negative integer.'})
 
             per_page = min(int(per_page), limits[1])
-            nrows = len(obj[field])
-            max_page = int(nrows/per_page) + bool(nrows % per_page)
-            if int(page) > max_page:
-                obj.data = []
-            else:
-                obj.data = obj.paginate_field(field, int(page), per_page=per_page).items
+            total_rows = len(obj[field])
+            total_pages = int(total_rows/per_page) + bool(total_rows % per_page)
+            if int(page) > total_pages:
+                raise ValidationError({'error': f'{field}_page must be less or equal {total_pages}.'})
+            obj.data = obj.paginate_field(field, int(page), per_page=per_page).items
         return obj
 
     def fetch_related_resources(self, objs, only_fields=None):

From 39c3d9ed174c4bbf1b9fe5bbe5341f2d5e0b25f2 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Mon, 9 Dec 2019 12:55:34 -0800
Subject: [PATCH 19/78] check total_pages before field pagination

---
 flask_mongorest/resources.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 1c8e3a5e..8e71fa4c 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -659,10 +659,11 @@ def paginate_fields(self, obj):
 
             per_page = min(int(per_page), limits[1])
             total_rows = len(obj[field])
-            total_pages = int(total_rows/per_page) + bool(total_rows % per_page)
-            if int(page) > total_pages:
-                raise ValidationError({'error': f'{field}_page must be less or equal {total_pages}.'})
-            obj.data = obj.paginate_field(field, int(page), per_page=per_page).items
+            if total_rows > 0:
+                total_pages = int(total_rows/per_page) + bool(total_rows % per_page)
+                if int(page) > total_pages:
+                    raise ValidationError({'error': f'{field}_page must be less or equal {total_pages}.'})
+                obj.data = obj.paginate_field(field, int(page), per_page=per_page).items
         return obj
 
     def fetch_related_resources(self, objs, only_fields=None):

From b521634fa033d7091713e80357c98feb3ec46e60 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Mon, 9 Dec 2019 23:30:14 -0800
Subject: [PATCH 20/78] add total_count/pages

---
 flask_mongorest/resources.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 8e71fa4c..1546c749 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -860,6 +860,7 @@ def get_objects(self, qs=None, qfilter=None):
         - Pass `qfilter` function to modify the queryset.
         """
         params = self.params
+        extra = {}
 
         custom_qs = True
         if qs is None:
@@ -875,6 +876,7 @@ def get_objects(self, qs=None, qfilter=None):
         # request
         qs = self.apply_filters(qs, params)
         qs = self.apply_ordering(qs, params)
+        extra['total_count'] = qs.count()
 
         # Apply limit and skip to the queryset
         limit = None
@@ -885,6 +887,7 @@ def get_objects(self, qs=None, qfilter=None):
             # no need to skip/limit if a custom `qs` was provided
             skip, limit = self.get_skip_and_limit(params)
             qs = qs.skip(skip).limit(limit+1)
+            extra['total_pages'] = int(extra['total_count']/limit) + bool(extra['total_count'] % limit)
 
         # Needs to be at the end as it returns a list, not a queryset
         if self.select_related:
@@ -913,7 +916,7 @@ def get_objects(self, qs=None, qfilter=None):
             objs, self.get_requested_fields(params=params)
         )
 
-        return objs, has_more
+        return objs, has_more, extra
 
     def save_related_objects(self, obj, parent_resources=None):
         if not parent_resources:

From fcb84ce80825125a028591ec08088ba7c4a81752 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Tue, 10 Dec 2019 01:11:27 -0800
Subject: [PATCH 21/78] add page/per_page params as option for _skip/_limit

---
 flask_mongorest/resources.py | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 1546c749..753bf074 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -834,18 +834,31 @@ def get_skip_and_limit(self, params=None):
             params = self.params
         if self.paginate:
             # _limit and _skip validation
-            if not isint(params.get('_limit', 1)):
-                raise ValidationError({'error': '_limit must be an integer (got "%s" instead).' % params['_limit']})
-            if not isint(params.get('_skip', 1)):
-                raise ValidationError({'error': '_skip must be an integer (got "%s" instead).' % params['_skip']})
-            if params.get('_limit') and int(params['_limit']) > max_limit:
-                raise ValidationError({'error': "The limit you set is larger than the maximum limit for this resource (max_limit = %d)." % max_limit})
-            if params.get('_skip') and int(params['_skip']) < 0:
-                raise ValidationError({'error': '_skip must be a non-negative integer (got "%s" instead).' % params['_skip']})
-
-            limit = min(int(params.get('_limit', self.default_limit)), max_limit)
+            for par in ['_limit', 'per_page']:
+                if par in params:
+                    if not isint(params[par]):
+                        raise ValidationError({'error': f'{par} must be an integer (got "%s" instead).' % params[par]})
+                    if params[par] and int(params[par]) > max_limit:
+                        raise ValidationError({'error': "The limit you set is larger than the maximum limit for this \
+                                               resource (max_limit = %d)." % max_limit})
+                    limit = min(int(params[par]), max_limit)
+                    break
+            else:
+                limit = min(int(self.default_limit), max_limit)
+
+            for par in ['_skip', 'page']:
+                if par in params:
+                    if not isint(params[par]):
+                        raise ValidationError({'error': f'{par} must be an integer (got "%s" instead).' % params[par]})
+                    if params[par] and int(params[par]) < 0:
+                        raise ValidationError({'error': f'{par} must be a non-negative integer (got "%s" instead).' % params[par]})
+                    skip = int(params[par]) if par == '_skip' else (int(params[par])-1) * limit
+                    break
+            else:
+                skip = 0
+
             # Fetch one more so we know if there are more results.
-            return int(params.get('_skip', 0)), limit
+            return skip, limit
         else:
             return 0, max_limit
 

From e8c95c0dd166d0109595e0c9fe45fd12b33922c7 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Tue, 10 Dec 2019 20:58:44 -0800
Subject: [PATCH 22/78] enable pattern filters

---
 flask_mongorest/resources.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 753bf074..f7381c65 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -1,6 +1,7 @@
 import json
 import mongoengine
 
+from typing import Pattern
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
 from flask import has_request_context, request, url_for
@@ -135,7 +136,7 @@ def __init__(self, view_method=None):
             self._reverse_rename_fields[v] = k
         assert len(self._rename_fields) == len(self._reverse_rename_fields), \
             'Cannot rename multiple fields to the same name'
-        self._filters = self.get_filters()
+        self._normal_filters, self._regex_filters = self.get_filters()
         self._child_document_resources = self.get_child_document_resources()
         self._default_child_resource_document = self.get_default_child_resource_document()
         self.data = None
@@ -334,15 +335,20 @@ def get_filters(self):
         `?date__gte=value` to the 'date' field and the 'gte' suffix: 'gte',
         and hence use the Gte operator to filter the data.
         """
-        filters = {}
+        normal_filters, regex_filters = {}, {}
         for field, operators in getattr(self, 'filters', {}).items():
             field_filters = {}
+
             for op in operators:
                 if op.op == 'exact':
                     field_filters[''] = op
                 field_filters[op.op] = op
-            filters[field] = field_filters
-        return filters
+
+            if isinstance(field, Pattern):
+                regex_filters[field] = field_filters
+            else:
+                normal_filters[field] = field_filters
+        return normal_filters, regex_filters
 
     def serialize_field(self, obj, **kwargs):
         if self.uri_prefix and hasattr(obj, "id"):
@@ -780,7 +786,16 @@ def apply_filters(self, qs, params=None):
             parts = key.split('__')
             for i in range(len(parts) + 1, 0, -1):
                 field = '__'.join(parts[:i])
-                allowed_operators = self._filters.get(field)
+                try:
+                    allowed_operators = self._normal_filters[field]
+                except KeyError:
+                    for k, v in self._regex_filters.items():
+                        m = k.match(field)
+                        if m:
+                            allowed_operators = v
+                            break
+                    else:
+                        allowed_operators = None
                 if allowed_operators:
                     parts = parts[i:]
                     break

From be2c11c9a22813c0c25b1e69ac9b85ad825d4d29 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 11 Dec 2019 15:41:13 -0800
Subject: [PATCH 23/78] enable regex patterns for ordering

---
 flask_mongorest/resources.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index f7381c65..824be18b 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -142,6 +142,8 @@ def __init__(self, view_method=None):
         self.data = None
         self._dirty_fields = None
         self.view_method = view_method
+        self._normal_allowed_ordering = [o for o in self.allowed_ordering if not isinstance(o, Pattern)]
+        self._regex_allowed_ordering = [o for o in self.allowed_ordering if isinstance(o, Pattern)]
 
     @property
     def params(self):
@@ -834,9 +836,16 @@ def apply_ordering(self, qs, params=None):
         """
         if params is None:
             params = self.params
-        if self.allowed_ordering and params.get('_order_by') in self.allowed_ordering:
-            order_params = [self._reverse_rename_fields.get(p, p) for p in params['_order_by'].split(',')]
-            qs = qs.order_by(*order_params)
+        if self.allowed_ordering:
+            oby = params.get('_order_by')
+            if oby:
+                order_params = None
+                if oby in self._normal_allowed_ordering:
+                    order_params = [self._reverse_rename_fields.get(p, p) for p in oby.split(',')]
+                elif any(p.match(oby) for p in self._regex_allowed_ordering):
+                    order_params = [oby]
+                if order_params:
+                    qs = qs.order_by(*order_params)
         return qs
 
     def get_skip_and_limit(self, params=None):

From 784b4c509deb4496bdd8cb8a172ae4f6c1c576bf Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 13 Dec 2019 12:34:58 -0800
Subject: [PATCH 24/78] catch Marshmallow validation errors

---
 flask_mongorest/resources.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 824be18b..936411a6 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -29,6 +29,7 @@
 
 try:
     from marshmallow_mongoengine import ModelSchema
+    from marshmallow.exceptions import ValidationError as MarshmallowValidationError
 except ImportError:
     ModelSchema = None
 
@@ -607,8 +608,11 @@ def validate_request(self, obj=None):
                 except SchemaValidationError:
                     raise ValidationError({'field-errors': schema.field_errors, 'errors': schema.errors })
             elif ModelSchema is not None:
-                partial = bool(request.method == 'PUT' and obj is not None)
-                self.data = self.schema().load(self.data, partial=partial)
+                try:
+                    partial = bool(request.method == 'PUT' and obj is not None)
+                    self.data = self.schema().load(self.data, partial=partial)
+                except MarshmallowValidationError as ex:
+                    raise ValidationError({'errors': ex.messages})
 
     def get_queryset(self):
         """

From 84fafbee6bf80020f573dd62c71bdd66d8bfd8e2 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 13 Dec 2019 15:41:38 -0800
Subject: [PATCH 25/78] force_insert and catch NotUniqueError

---
 flask_mongorest/resources.py | 6 +++---
 flask_mongorest/views.py     | 6 ++++--
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 936411a6..456461ab 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -959,7 +959,7 @@ def get_objects(self, qs=None, qfilter=None):
 
         return objs, has_more, extra
 
-    def save_related_objects(self, obj, parent_resources=None):
+    def save_related_objects(self, obj, parent_resources=None, **kwargs):
         if not parent_resources:
             parent_resources = [self]
         else:
@@ -994,7 +994,7 @@ def save_related_objects(self, obj, parent_resources=None):
 
     def save_object(self, obj, **kwargs):
         self.save_related_objects(obj, **kwargs)
-        obj.save()
+        obj.save(**kwargs)
         obj.reload()
 
         self._dirty_fields = None # No longer dirty.
@@ -1018,7 +1018,7 @@ def create_object(self, data=None, save=True, parent_resources=None):
         obj = self.document(**update_dict)
         self._dirty_fields = update_dict.keys()
         if save:
-            self.save_object(obj)
+            self.save_object(obj, force_insert=True)
         return obj
 
     def update_object(self, obj, data=None, save=True, parent_resources=None):
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index ed3b0062..94165d77 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -78,10 +78,12 @@ def _dispatch_request(self, *args, **kwargs):
             return super(ResourceView, self).dispatch_request(*args, **kwargs)
         except mongoengine.queryset.DoesNotExist as e:
             return {'error': 'Empty query: ' + str(e)}, '404 Not Found'
+        except mongoengine.errors.NotUniqueError as e:
+            return {'error': str(e)}, '401 Unauthorized'
         except ValidationError as e:
             return e.args[0], '400 Bad Request'
-        except Unauthorized:
-            return {'error': 'Unauthorized'}, '401 Unauthorized'
+        except Unauthorized as e:
+            return {'error': str(e)}, '401 Unauthorized'
         except NotFound as e:
             return {'error': str(e)}, '404 Not Found'
 

From 7410b341bb771a75f2f4890a5afbd19d4c5430a8 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 18 Dec 2019 22:26:44 -0800
Subject: [PATCH 26/78] wait with save until after checking add permissions

---
 flask_mongorest/views.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 94165d77..91fc969f 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -163,7 +163,7 @@ def post(self, **kwargs):
 
         self._resource.validate_request()
         try:
-            obj = self._resource.create_object()
+            obj = self._resource.create_object(save=False)
         except Exception as e:
             self.handle_validation_error(e)
 
@@ -171,6 +171,8 @@ def post(self, **kwargs):
         if not self.has_add_permission(request, obj):
             raise Unauthorized
 
+        self._resource.save_object(obj, force_insert=True)
+
         ret = self._resource.serialize(obj, params=request.args)
         if isinstance(obj, mongoengine.Document) and self._resource.uri_prefix:
             return ret, "201 Created", {"Location": self._resource._url(str(obj.id))}

From 893f3326cdd1ad8b251c62744ea1a2c7a3bb4096 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 20 Dec 2019 00:54:28 -0800
Subject: [PATCH 27/78] support partial DictField update

---
 flask_mongorest/resources.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 456461ab..ff602b5b 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -1026,7 +1026,7 @@ def update_object(self, obj, data=None, save=True, parent_resources=None):
         if subresource:
             return subresource.update_object(obj, data=data, save=save, parent_resources=parent_resources)
 
-        update_dict = self.get_object_dict(data, update=True)
+        update_dict = self.get_object_dict(data, update=True) if save else data
 
         self._dirty_fields = []
 
@@ -1041,11 +1041,18 @@ def update_object(self, obj, data=None, save=True, parent_resources=None):
                 id_from_data = value and getattr(value, 'pk', value)
                 if id_from_obj != id_from_data:
                     update = True
-            elif not obj._fields[field].primary_key and not equal(getattr(obj, field), value):
+            elif getattr(obj, '_fields', None) is not None and isinstance(obj._fields.get(field), DictField):
+                self.update_object(obj[field], data=value, save=False)
+            elif getattr(obj, '_fields', None) is not None and obj._fields[field].primary_key:
+                update = False
+            elif not equal(obj.get(field), value) or not equal(getattr(obj, field), value):
                 update = True
 
             if update:
-                setattr(obj, field, value)
+                if getattr(obj, field, None) is not None:
+                    setattr(obj, field, value)
+                else:
+                    obj[field] = value
                 self._dirty_fields.append(field)
 
         if save:

From 50253fb8a290b2123ad0fdc5d41bc17d8f8a0d94 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 20 Dec 2019 12:45:03 -0800
Subject: [PATCH 28/78] fix DictField update, only return updated fields

---
 flask_mongorest/resources.py | 13 ++++++++-----
 flask_mongorest/views.py     |  6 ++++--
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index ff602b5b..975cfd6b 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -1041,11 +1041,14 @@ def update_object(self, obj, data=None, save=True, parent_resources=None):
                 id_from_data = value and getattr(value, 'pk', value)
                 if id_from_obj != id_from_data:
                     update = True
-            elif getattr(obj, '_fields', None) is not None and isinstance(obj._fields.get(field), DictField):
-                self.update_object(obj[field], data=value, save=False)
-            elif getattr(obj, '_fields', None) is not None and obj._fields[field].primary_key:
-                update = False
-            elif not equal(obj.get(field), value) or not equal(getattr(obj, field), value):
+            elif getattr(obj, '_fields', None) is not None:
+                if isinstance(obj._fields.get(field), DictField):
+                    self.update_object(obj[field], data=value, save=False)
+                elif obj._fields[field].primary_key:
+                    raise ValidationError({'error': f'`{field}` is primary key and cannot be updated'})
+                elif not equal(getattr(obj, field), value):
+                    update = True
+            elif not equal(obj.get(field), value):
                 update = True
 
             if update:
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 91fc969f..0abca51d 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -2,6 +2,7 @@
 
 import mimerender
 import mongoengine
+from fdict import fdict
 from flask import render_template, request
 from flask.views import MethodView
 from flask_mongorest import methods
@@ -243,8 +244,9 @@ def put(self, **kwargs):
         else:
             obj = self._resource.get_object(pk)
             self.process_object(obj)
-            ret = self._resource.serialize(obj, params=request.args)
-            return ret
+            raw_data = fdict(self._resource.raw_data, delimiter='.')
+            fields = ','.join(raw_data.keys())
+            return self._resource.serialize(obj, params={'_fields': fields})
 
     def delete(self, **kwargs):
         pk = kwargs.pop('pk', None)

From d7a6f8948a279af1db26e828c2cfd3ca3438adcf Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Mon, 23 Dec 2019 16:14:06 -0800
Subject: [PATCH 29/78] enable sort direction

---
 flask_mongorest/resources.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 975cfd6b..6a1a9690 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -849,7 +849,8 @@ def apply_ordering(self, qs, params=None):
                 elif any(p.match(oby) for p in self._regex_allowed_ordering):
                     order_params = [oby]
                 if order_params:
-                    qs = qs.order_by(*order_params)
+                    order_sign = '-' if params.get('order') == 'desc' else '+'
+                    qs = qs.order_by(*[f'{order_sign}{p}' for p in order_params])
         return qs
 
     def get_skip_and_limit(self, params=None):

From 17831b19939fd785fd16bb967fb377bcff525ec2 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Mon, 6 Jan 2020 00:23:12 -0800
Subject: [PATCH 30/78] allow DictField update/set to None

---
 flask_mongorest/resources.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 6a1a9690..e4f179b5 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -1044,7 +1044,10 @@ def update_object(self, obj, data=None, save=True, parent_resources=None):
                     update = True
             elif getattr(obj, '_fields', None) is not None:
                 if isinstance(obj._fields.get(field), DictField):
-                    self.update_object(obj[field], data=value, save=False)
+                    if value is None:
+                        update = True
+                    else:
+                        self.update_object(obj[field], data=value, save=False)
                 elif obj._fields[field].primary_key:
                     raise ValidationError({'error': f'`{field}` is primary key and cannot be updated'})
                 elif not equal(getattr(obj, field), value):

From 78577b456c8375fd826350f99bc24f52dc9a3d2f Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 17 Jan 2020 23:09:51 -0800
Subject: [PATCH 31/78] switch to dotty_dict

---
 flask_mongorest/resources.py | 8 ++++----
 requirements.txt             | 1 -
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index e4f179b5..a803f939 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -4,6 +4,7 @@
 from typing import Pattern
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
+from dotty_dict import Dotty
 from flask import has_request_context, request, url_for
 from dict_deep import deep_get
 try:
@@ -389,11 +390,10 @@ def get_field_value(self, obj, field_name, field_instance=None, **kwargs):
         # Determine the field value
         if has_field_instance:
             field_value = obj
-        elif isinstance(obj, dict):
-            return deep_get(obj, field_name)
         else:
             try:
-                field_value = deep_get(obj, field_name, getter=get_with_list_index)
+                dotty = Dotty(obj if isinstance(obj, dict) else obj.to_mongo())
+                field_value = dotty[field_name]
             except (AttributeError, KeyError):
                 raise UnknownFieldError
 
@@ -506,7 +506,7 @@ def serialize(self, obj, **kwargs):
 
         # Fill in the `data` dict by serializing each of the requested fields
         # one by one.
-        data = {}
+        data = Dotty({})
         for field in requested_fields:
 
             # resolve the user-facing name of the field
diff --git a/requirements.txt b/requirements.txt
index 2d486536..53e29c4a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,4 +6,3 @@ sphinx
 Flask>=0.9
 pymongo>=3.4
 flake8
-dict-deep==2.0.2

From 1d72eb4fab21032cd90e717561e39aed9525175c Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Sat, 18 Jan 2020 01:08:50 -0800
Subject: [PATCH 32/78] return internal dotty dict, check primary key

---
 flask_mongorest/resources.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index a803f939..0d34a57c 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -392,7 +392,8 @@ def get_field_value(self, obj, field_name, field_instance=None, **kwargs):
             field_value = obj
         else:
             try:
-                dotty = Dotty(obj if isinstance(obj, dict) else obj.to_mongo())
+                dotty = Dotty(obj if isinstance(obj, dict) else obj.to_mongo().to_dict())
+                field_name = '_id' if obj._fields[field_name].primary_key else field_name
                 field_value = dotty[field_name]
             except (AttributeError, KeyError):
                 raise UnknownFieldError
@@ -538,7 +539,7 @@ def serialize(self, obj, **kwargs):
                     except UnknownFieldError:
                         pass
 
-        return data
+        return data._data
 
     def handle_serialization_error(self, exc, obj):
         """

From 43358ad4db9401ab06b60543ea97db99054d3f1f Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Sat, 18 Jan 2020 02:06:14 -0800
Subject: [PATCH 33/78] also check for id

---
 flask_mongorest/resources.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 0d34a57c..f84fe985 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -392,8 +392,9 @@ def get_field_value(self, obj, field_name, field_instance=None, **kwargs):
             field_value = obj
         else:
             try:
-                dotty = Dotty(obj if isinstance(obj, dict) else obj.to_mongo().to_dict())
-                field_name = '_id' if obj._fields[field_name].primary_key else field_name
+                dotty = Dotty(obj if isinstance(obj, dict) else obj.to_mongo())
+                is_pk = obj._fields[field_name].primary_key
+                field_name = '_id' if is_pk or field_name == 'id' else field_name
                 field_value = dotty[field_name]
             except (AttributeError, KeyError):
                 raise UnknownFieldError

From d4242b8d4e5f965ed0709e4ee02257658cdd969f Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Sat, 18 Jan 2020 20:41:40 -0800
Subject: [PATCH 34/78] check dot delimiter for primary key

---
 flask_mongorest/resources.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index f84fe985..557ba795 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -393,8 +393,9 @@ def get_field_value(self, obj, field_name, field_instance=None, **kwargs):
         else:
             try:
                 dotty = Dotty(obj if isinstance(obj, dict) else obj.to_mongo())
-                is_pk = obj._fields[field_name].primary_key
-                field_name = '_id' if is_pk or field_name == 'id' else field_name
+                if '.' not in field_name:
+                    is_pk = obj._fields[field_name].primary_key
+                    field_name = '_id' if is_pk or field_name == 'id' else field_name
                 field_value = dotty[field_name]
             except (AttributeError, KeyError):
                 raise UnknownFieldError

From c21e1505c5be1187811d8d2d1ad3d4bcb8a90f28 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Mon, 3 Feb 2020 15:31:44 -0800
Subject: [PATCH 35/78] catch mongoengine ValidationError

---
 flask_mongorest/views.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 0abca51d..149d9e68 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -83,6 +83,8 @@ def _dispatch_request(self, *args, **kwargs):
             return {'error': str(e)}, '401 Unauthorized'
         except ValidationError as e:
             return e.args[0], '400 Bad Request'
+        except mongoengine.errors.ValidationError as e:
+            return {'error': str(e)}, '400 Bad Request'
         except Unauthorized as e:
             return {'error': str(e)}, '401 Unauthorized'
         except NotFound as e:

From ac0153fbd948436595d3cd2f4eddf7df22c382ff Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 12 Feb 2020 19:44:25 -0800
Subject: [PATCH 36/78] make_zip mimerender, and Download method

---
 flask_mongorest/__init__.py  |  5 ++++-
 flask_mongorest/methods.py   |  3 +++
 flask_mongorest/resources.py |  4 ++--
 flask_mongorest/views.py     | 39 +++++++++++++++++++++++++++++++-----
 4 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/flask_mongorest/__init__.py b/flask_mongorest/__init__.py
index 157499aa..657e8980 100644
--- a/flask_mongorest/__init__.py
+++ b/flask_mongorest/__init__.py
@@ -1,5 +1,5 @@
 from flask import Blueprint
-from flask_mongorest.methods import Create, BulkUpdate, List
+from flask_mongorest.methods import Create, BulkUpdate, List, Download
 
 def register_class(app, klass, **kwargs):
     # Construct a url based on a 'name' kwarg with a fallback to the
@@ -21,6 +21,9 @@ def register_class(app, klass, **kwargs):
     if List in klass.methods:
         app.add_url_rule(url, defaults={'pk': None}, view_func=view_func,
                          methods=[List.method], endpoint=view_func.__name__+'List', **kwargs)
+    if Download in klass.methods:
+        app.add_url_rule(url + 'download/', defaults={'pk': None}, view_func=view_func,
+                         methods=[Download.method], endpoint=view_func.__name__+'Download', **kwargs)
     if Create in klass.methods or BulkUpdate in klass.methods:
         app.add_url_rule(url, view_func=view_func, methods=[
             x.method for x in klass.methods if x in (Create, BulkUpdate)
diff --git a/flask_mongorest/methods.py b/flask_mongorest/methods.py
index 5357ab45..bc4a7735 100644
--- a/flask_mongorest/methods.py
+++ b/flask_mongorest/methods.py
@@ -13,5 +13,8 @@ class Fetch:
 class List:
     method = 'GET'
 
+class Download:
+    method = 'GET'
+
 class Delete:
     method = 'DELETE'
diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 557ba795..262b4331 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -928,7 +928,7 @@ def get_objects(self, qs=None, qfilter=None):
         if self.view_method == methods.BulkUpdate:
             # limit the number of objects that can be bulk-updated at a time
             qs = qs.limit(self.bulk_update_limit)
-        elif not custom_qs:
+        elif not custom_qs and not self.view_method == methods.Download:
             # no need to skip/limit if a custom `qs` was provided
             skip, limit = self.get_skip_and_limit(params)
             qs = qs.skip(skip).limit(limit+1)
@@ -949,7 +949,7 @@ def get_objects(self, qs=None, qfilter=None):
             })
 
         # Determine the value of has_more
-        if self.view_method != methods.BulkUpdate and self.paginate:
+        if self.view_method not in [methods.BulkUpdate, methods.Download] and self.paginate:
             has_more = len(objs) > limit
             if has_more:
                 objs = objs[:-1]
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 149d9e68..b21ad3a5 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -1,19 +1,45 @@
 import json
-
+import zipstream
 import mimerender
 import mongoengine
 from fdict import fdict
-from flask import render_template, request
+from flask import render_template, request, Response
 from flask.views import MethodView
 from flask_mongorest import methods
 from flask_mongorest.exceptions import ValidationError
 from flask_mongorest.utils import MongoEncoder
 from werkzeug.exceptions import NotFound, Unauthorized
 
+mimerender.register_mime('zip', ('application/zip',))
 mimerender = mimerender.FlaskMimeRender()
 
-render_json = lambda **payload: json.dumps(payload, allow_nan=False, cls=MongoEncoder)
-render_html = lambda **payload: render_template('mongorest/debug.html', data=json.dumps(payload, cls=MongoEncoder, sort_keys=True, indent=4))
+def render_json(**payload):
+    return json.dumps(payload, allow_nan=False, cls=MongoEncoder)
+
+def render_html(**payload):
+    d = json.dumps(payload, cls=MongoEncoder, sort_keys=True, indent=4)
+    return render_template('mongorest/debug.html', data=d)
+
+def make_zip(**payload):
+    print(len(payload['data']), payload['total_count'])
+
+    def generator():
+        def iterable():
+            yield b'['
+            for idx, doc in enumerate(payload['data']):
+                yield render_json(**doc).encode()
+                if idx < payload['total_count']-1:
+                    yield b','
+            yield b']'
+
+        z = zipstream.ZipFile(compression=zipstream.ZIP_DEFLATED)
+        z.write_iter('data.json', iterable())
+        for chunk in z:
+            yield chunk
+
+    response = Response(generator())
+    response.headers['Content-Disposition'] = 'attachment; filename={}'.format('files.zip')
+    return response
 
 try:
     text_type = unicode # Python 2
@@ -58,7 +84,8 @@ class ResourceView(MethodView):
     def __init__(self):
         assert(self.resource and self.methods)
 
-    @mimerender(default='json', json=render_json, html=render_html)
+    @mimerender(default='json', override_input_key='format',
+                json=render_json, html=render_html, zip=make_zip)
     def dispatch_request(self, *args, **kwargs):
         # keep all the logic in a helper method (_dispatch_request) so that
         # it's easy for subclasses to override this method (when they don't want to use
@@ -112,6 +139,8 @@ def get(self, **kwargs):
         # Set the view_method on a resource instance
         if pk:
             self._resource.view_method = methods.Fetch
+        elif self._resource.params.get('format'):
+            self._resource.view_method = methods.Download
         else:
             self._resource.view_method = methods.List
 

From c4966ac35da04e18e34685d6475d2205bc21eb23 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 13 Feb 2020 16:06:42 -0800
Subject: [PATCH 37/78] short_mime, render_gz

---
 flask_mongorest/__init__.py  |  4 +-
 flask_mongorest/resources.py | 10 +++++
 flask_mongorest/views.py     | 74 ++++++++++++++++++++----------------
 3 files changed, 53 insertions(+), 35 deletions(-)

diff --git a/flask_mongorest/__init__.py b/flask_mongorest/__init__.py
index 657e8980..34b62744 100644
--- a/flask_mongorest/__init__.py
+++ b/flask_mongorest/__init__.py
@@ -19,10 +19,10 @@ def register_class(app, klass, **kwargs):
     pk_type = kwargs.pop('pk_type', 'string')
     view_func = klass.as_view(name)
     if List in klass.methods:
-        app.add_url_rule(url, defaults={'pk': None}, view_func=view_func,
+        app.add_url_rule(url, defaults={'pk': None, 'short_mime': None}, view_func=view_func,
                          methods=[List.method], endpoint=view_func.__name__+'List', **kwargs)
     if Download in klass.methods:
-        app.add_url_rule(url + 'download/', defaults={'pk': None}, view_func=view_func,
+        app.add_url_rule(f'{url}<string:short_mime>/', defaults={'pk': None}, view_func=view_func,
                          methods=[Download.method], endpoint=view_func.__name__+'Download', **kwargs)
     if Create in klass.methods or BulkUpdate in klass.methods:
         app.add_url_rule(url, view_func=view_func, methods=[
diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 262b4331..8453d7bd 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -119,6 +119,9 @@ class Resource(object):
     # filtered query set, pulling all the references efficiently.
     select_related = False
 
+    # allow download formats
+    download_formats = []
+
     # Must start and end with a "/"
     uri_prefix = None
 
@@ -659,6 +662,8 @@ def paginate_fields(self, obj):
         """
         return object with fields paginated according to `fields_to_paginate`
         """
+        if self.view_method == methods.Download:
+            return obj  # no pagination when downloading
         for field, limits in self.fields_to_paginate.items():
             page = self.params.get(f'{field}_page', 1)
             per_page = self.params.get(f'{field}_per_page', limits[0])
@@ -907,6 +912,11 @@ def get_objects(self, qs=None, qfilter=None):
         params = self.params
         extra = {}
 
+        if self.view_method == methods.Download:
+            fmt = self.params.get('format')
+            if fmt not in self.download_formats:
+                raise ValueError(f'`format` must be one of {self.download_formats}')
+
         custom_qs = True
         if qs is None:
             custom_qs = False
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index b21ad3a5..abbb654f 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -1,17 +1,18 @@
 import json
-import zipstream
-import mimerender
 import mongoengine
+from gzip import GzipFile
+from io import BytesIO
 from fdict import fdict
-from flask import render_template, request, Response
+from flask import render_template, request
 from flask.views import MethodView
 from flask_mongorest import methods
 from flask_mongorest.exceptions import ValidationError
 from flask_mongorest.utils import MongoEncoder
 from werkzeug.exceptions import NotFound, Unauthorized
+from mimerender import register_mime, FlaskMimeRender
 
-mimerender.register_mime('zip', ('application/zip',))
-mimerender = mimerender.FlaskMimeRender()
+mimerender = FlaskMimeRender(global_override_input_key='short_mime')
+register_mime('gz', ('application/gzip',))
 
 def render_json(**payload):
     return json.dumps(payload, allow_nan=False, cls=MongoEncoder)
@@ -20,26 +21,25 @@ def render_html(**payload):
     d = json.dumps(payload, cls=MongoEncoder, sort_keys=True, indent=4)
     return render_template('mongorest/debug.html', data=d)
 
-def make_zip(**payload):
-    print(len(payload['data']), payload['total_count'])
-
-    def generator():
-        def iterable():
-            yield b'['
-            for idx, doc in enumerate(payload['data']):
-                yield render_json(**doc).encode()
-                if idx < payload['total_count']-1:
-                    yield b','
-            yield b']'
-
-        z = zipstream.ZipFile(compression=zipstream.ZIP_DEFLATED)
-        z.write_iter('data.json', iterable())
-        for chunk in z:
-            yield chunk
-
-    response = Response(generator())
-    response.headers['Content-Disposition'] = 'attachment; filename={}'.format('files.zip')
-    return response
+def render_gz(**payload):
+    contents = ''
+    fmt = request.values.get('format')
+    if fmt == 'json':
+        contents = json.dumps(payload['data'], allow_nan=False, cls=MongoEncoder)
+    elif fmt == 'csv':
+        from pandas import DataFrame
+        fpayload = []
+        for doc in payload['data']:
+            fdoc = {}
+            for k, v in doc.items():
+                fdoc[k] = v[0]['id'] if isinstance(v, list) else doc[k]
+            fpayload.append(fdict(fdoc, delimiter='.').to_dict())
+        contents = DataFrame.from_records(fpayload).to_csv()
+
+    gzip_buffer = BytesIO()
+    with GzipFile(mode='wb', fileobj=gzip_buffer) as gzip_file:
+        gzip_file.write(contents.encode())  # need to give full contents to compression
+    return gzip_buffer.getvalue()
 
 try:
     text_type = unicode # Python 2
@@ -84,8 +84,7 @@ class ResourceView(MethodView):
     def __init__(self):
         assert(self.resource and self.methods)
 
-    @mimerender(default='json', override_input_key='format',
-                json=render_json, html=render_html, zip=make_zip)
+    @mimerender(default='json', json=render_json, html=render_html, gz=render_gz)
     def dispatch_request(self, *args, **kwargs):
         # keep all the logic in a helper method (_dispatch_request) so that
         # it's easy for subclasses to override this method (when they don't want to use
@@ -110,7 +109,7 @@ def _dispatch_request(self, *args, **kwargs):
             return {'error': str(e)}, '401 Unauthorized'
         except ValidationError as e:
             return e.args[0], '400 Bad Request'
-        except mongoengine.errors.ValidationError as e:
+        except (ValueError, mongoengine.errors.ValidationError) as e:
             return {'error': str(e)}, '400 Bad Request'
         except Unauthorized as e:
             return {'error': str(e)}, '401 Unauthorized'
@@ -135,11 +134,14 @@ def requested_resource(self, request):
 
     def get(self, **kwargs):
         pk = kwargs.pop('pk', None)
+        short_mime = kwargs.pop('short_mime', None)
 
         # Set the view_method on a resource instance
         if pk:
             self._resource.view_method = methods.Fetch
-        elif self._resource.params.get('format'):
+        elif short_mime:
+            if short_mime != 'gz':
+                raise ValueError(f'{short_mime} not supported')
             self._resource.view_method = methods.Download
         else:
             self._resource.view_method = methods.List
@@ -171,9 +173,7 @@ def get(self, **kwargs):
                         data.append(fixed_obj)
 
             # Serialize the objects one by one
-            ret = {
-                'data': data
-            }
+            ret = {'data': data}
 
             if has_more is not None:
                 ret['has_more'] = has_more
@@ -184,7 +184,15 @@ def get(self, **kwargs):
             obj = self._resource.get_object(pk, qfilter=qfilter)
             obj = self._resource.paginate_fields(obj)
             ret = self._resource.serialize(obj, params=request.args)
-        return ret
+
+
+        if self._resource.view_method == methods.Download:
+            fmt = self._resource.params.get('format')
+            return ret, '200 OK', {
+                'Content-Disposition': f'attachment; filename="data.{fmt}.{short_mime}"'
+            }
+        else:
+            return ret
 
     def post(self, **kwargs):
         if 'pk' in kwargs:

From 30a1c6bc0fbb5b2b8ea1255c0a609ffb2006190e Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 13 Feb 2020 18:30:57 -0800
Subject: [PATCH 38/78] fix mimerender version

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index ee297864..881165a3 100644
--- a/setup.py
+++ b/setup.py
@@ -27,9 +27,9 @@
     test_suite='nose.collector',
     zip_safe=False,
     platforms='any',
-    setup_requires=[
+    install_requires=[
         'Flask-MongoEngine',
-        'mimerender',
+        'mimerender @ git+https://github.com/tschaume/mimerender@mpcontribs#egg=mimerender',
         'nose',
         'python-dateutil'
     ],

From 5dd64f5b9ecac00c856267b7aab4e44566619ff7 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 14 Feb 2020 13:35:12 -0800
Subject: [PATCH 39/78] add download url after pk

---
 flask_mongorest/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flask_mongorest/__init__.py b/flask_mongorest/__init__.py
index 34b62744..4dc28305 100644
--- a/flask_mongorest/__init__.py
+++ b/flask_mongorest/__init__.py
@@ -21,15 +21,15 @@ def register_class(app, klass, **kwargs):
     if List in klass.methods:
         app.add_url_rule(url, defaults={'pk': None, 'short_mime': None}, view_func=view_func,
                          methods=[List.method], endpoint=view_func.__name__+'List', **kwargs)
-    if Download in klass.methods:
-        app.add_url_rule(f'{url}<string:short_mime>/', defaults={'pk': None}, view_func=view_func,
-                         methods=[Download.method], endpoint=view_func.__name__+'Download', **kwargs)
     if Create in klass.methods or BulkUpdate in klass.methods:
         app.add_url_rule(url, view_func=view_func, methods=[
             x.method for x in klass.methods if x in (Create, BulkUpdate)
         ], endpoint=view_func.__name__+'CreateBulkUpdate', **kwargs)
     app.add_url_rule('%s<%s:%s>/' % (url, pk_type, 'pk'), view_func=view_func,
                      methods=[x.method for x in klass.methods if x not in (List, BulkUpdate)], **kwargs)
+    if Download in klass.methods:
+        app.add_url_rule(f'{url}<string:short_mime>/', defaults={'pk': None}, view_func=view_func,
+                         methods=[Download.method], endpoint=view_func.__name__+'Download', **kwargs)
 
 class MongoRest(object):
     def __init__(self, app, **kwargs):

From e8bbec7e84352548671ad36a83a47eaf32c79ef5 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 14 Feb 2020 14:56:44 -0800
Subject: [PATCH 40/78] fix url matching

---
 flask_mongorest/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flask_mongorest/__init__.py b/flask_mongorest/__init__.py
index 4dc28305..7fcb0b44 100644
--- a/flask_mongorest/__init__.py
+++ b/flask_mongorest/__init__.py
@@ -25,10 +25,10 @@ def register_class(app, klass, **kwargs):
         app.add_url_rule(url, view_func=view_func, methods=[
             x.method for x in klass.methods if x in (Create, BulkUpdate)
         ], endpoint=view_func.__name__+'CreateBulkUpdate', **kwargs)
-    app.add_url_rule('%s<%s:%s>/' % (url, pk_type, 'pk'), view_func=view_func,
+    app.add_url_rule('%s<%s:%s>/' % (url, pk_type, 'pk'), defaults={'short_mime': None}, view_func=view_func,
                      methods=[x.method for x in klass.methods if x not in (List, BulkUpdate)], **kwargs)
     if Download in klass.methods:
-        app.add_url_rule(f'{url}<string:short_mime>/', defaults={'pk': None}, view_func=view_func,
+        app.add_url_rule(f'{url}download/<string:short_mime>/', defaults={'pk': None, 'short_mime': 'gz'}, view_func=view_func,
                          methods=[Download.method], endpoint=view_func.__name__+'Download', **kwargs)
 
 class MongoRest(object):

From b26b54f4d145c47aabd0122090989f0db96bd15e Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 14 Feb 2020 19:47:24 -0800
Subject: [PATCH 41/78] catch unset value in csv export

---
 flask_mongorest/views.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index abbb654f..acaacac0 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -32,7 +32,8 @@ def render_gz(**payload):
         for doc in payload['data']:
             fdoc = {}
             for k, v in doc.items():
-                fdoc[k] = v[0]['id'] if isinstance(v, list) else doc[k]
+                if v:
+                    fdoc[k] = v[0]['id'] if isinstance(v, list) else doc[k]
             fpayload.append(fdict(fdoc, delimiter='.').to_dict())
         contents = DataFrame.from_records(fpayload).to_csv()
 

From 537f2c09a3ef3f325f805a280d4363d18b7da08a Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 26 Feb 2020 21:37:10 -0800
Subject: [PATCH 42/78] render_gz: use cherrypicker for csv

---
 flask_mongorest/views.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index acaacac0..9ae46176 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -28,14 +28,9 @@ def render_gz(**payload):
         contents = json.dumps(payload['data'], allow_nan=False, cls=MongoEncoder)
     elif fmt == 'csv':
         from pandas import DataFrame
-        fpayload = []
-        for doc in payload['data']:
-            fdoc = {}
-            for k, v in doc.items():
-                if v:
-                    fdoc[k] = v[0]['id'] if isinstance(v, list) else doc[k]
-            fpayload.append(fdict(fdoc, delimiter='.').to_dict())
-        contents = DataFrame.from_records(fpayload).to_csv()
+        from cherrypicker import CherryPicker
+        records = [CherryPicker(d).flatten().get() for d in payload['data']]
+        contents = DataFrame.from_records(records).to_csv()
 
     gzip_buffer = BytesIO()
     with GzipFile(mode='wb', fileobj=gzip_buffer) as gzip_file:

From 49f8b22f88c8dd5bae677c956f507ca1eb6af35f Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 28 Feb 2020 18:42:07 -0800
Subject: [PATCH 43/78] BulkCreate and BulkDelete

---
 flask_mongorest/__init__.py  | 43 ++++++++++++-------
 flask_mongorest/methods.py   | 22 +++++++---
 flask_mongorest/resources.py | 32 +++++++-------
 flask_mongorest/views.py     | 83 ++++++++++++++++++++++++++++--------
 4 files changed, 123 insertions(+), 57 deletions(-)

diff --git a/flask_mongorest/__init__.py b/flask_mongorest/__init__.py
index 7fcb0b44..7f5d7d8d 100644
--- a/flask_mongorest/__init__.py
+++ b/flask_mongorest/__init__.py
@@ -1,35 +1,46 @@
 from flask import Blueprint
-from flask_mongorest.methods import Create, BulkUpdate, List, Download
+from flask_mongorest.methods import *
 
 def register_class(app, klass, **kwargs):
     # Construct a url based on a 'name' kwarg with a fallback to the
     # view's class name. Note that the name must be unique.
     name = kwargs.pop('name', klass.__name__)
+    view_func = klass.as_view(name)
     url = kwargs.pop('url', None)
     if not url:
         document_name = klass.resource.document.__name__.lower()
-        url = '/%s/' % document_name
+        url = f'/{document_name}/'
 
     # Insert the url prefix, if it exists
     url_prefix = kwargs.pop('url_prefix', '')
     if url_prefix:
-        url = '%s%s' % (url_prefix, url)
+        url = f'{url_prefix}{url}'
 
     # Add url rules
-    pk_type = kwargs.pop('pk_type', 'string')
-    view_func = klass.as_view(name)
-    if List in klass.methods:
-        app.add_url_rule(url, defaults={'pk': None, 'short_mime': None}, view_func=view_func,
-                         methods=[List.method], endpoint=view_func.__name__+'List', **kwargs)
-    if Create in klass.methods or BulkUpdate in klass.methods:
-        app.add_url_rule(url, view_func=view_func, methods=[
-            x.method for x in klass.methods if x in (Create, BulkUpdate)
-        ], endpoint=view_func.__name__+'CreateBulkUpdate', **kwargs)
-    app.add_url_rule('%s<%s:%s>/' % (url, pk_type, 'pk'), defaults={'short_mime': None}, view_func=view_func,
-                     methods=[x.method for x in klass.methods if x not in (List, BulkUpdate)], **kwargs)
+    klass_methods = set(klass.methods)
+    if Create in klass_methods and BulkCreate in klass_methods:
+        raise ValueError('Use either Create or BulkCreate!')
+
+    for x in klass_methods & {Fetch, Update, Delete}:
+        endpoint = view_func.__name__ + x.__name__
+        app.add_url_rule(
+            f'{url}<string:pk>/', defaults={'short_mime': None},
+            view_func=view_func, methods=[x.method], endpoint=endpoint, **kwargs
+        )
+
+    for x in klass_methods & {Create, BulkFetch, BulkCreate, BulkUpdate, BulkDelete}:
+        endpoint = view_func.__name__ + x.__name__
+        app.add_url_rule(
+            url, defaults={'pk': None, 'short_mime': None},
+            view_func=view_func, methods=[x.method], endpoint=endpoint, **kwargs
+        )
+
     if Download in klass.methods:
-        app.add_url_rule(f'{url}download/<string:short_mime>/', defaults={'pk': None, 'short_mime': 'gz'}, view_func=view_func,
-                         methods=[Download.method], endpoint=view_func.__name__+'Download', **kwargs)
+        endpoint = view_func.__name__ + Download.__name__
+        app.add_url_rule(
+            f'{url}download/<string:short_mime>/', defaults={'pk': None, 'short_mime': 'gz'},
+            view_func=view_func, methods=[Download.method], endpoint=endpoint, **kwargs
+        )
 
 class MongoRest(object):
     def __init__(self, app, **kwargs):
diff --git a/flask_mongorest/methods.py b/flask_mongorest/methods.py
index bc4a7735..25b98d8f 100644
--- a/flask_mongorest/methods.py
+++ b/flask_mongorest/methods.py
@@ -1,20 +1,28 @@
+class Fetch:
+    method = 'GET'
+
 class Create:
     method = 'POST'
 
 class Update:
     method = 'PUT'
 
+class Delete:
+    method = 'DELETE'
+
+
+class BulkFetch:
+    method = 'GET'
+
+class BulkCreate:
+    method = 'POST'
+
 class BulkUpdate:
     method = 'PUT'
 
-class Fetch:
-    method = 'GET'
+class BulkDelete:
+    method = 'DELETE'
 
-class List:
-    method = 'GET'
 
 class Download:
     method = 'GET'
-
-class Delete:
-    method = 'DELETE'
diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 8453d7bd..c3b9bbc4 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -84,7 +84,7 @@ class Resource(object):
     max_limit = 100
 
     # Maximum number of objects which can be bulk-updated by a single request
-    bulk_update_limit = 1000
+    bulk_update_limit = 1000  # NOTE also used for bulk delete
 
     # Map of field names to paginate with according default and maximum limits
     fields_to_paginate = {}
@@ -205,7 +205,7 @@ def raw_data(self):
                     self._raw_data = json.loads(request.data.decode('utf-8'), parse_constant=self._enforce_strict_json)
                 except ValueError:
                     raise ValidationError({'error': 'The request contains invalid JSON.'})
-                if not isinstance(self._raw_data, dict):
+                if request.method == 'PUT' and not isinstance(self._raw_data, dict):
                     raise ValidationError({'error': 'JSON data must be a dict.'})
             else:
                 self._raw_data = {}
@@ -935,7 +935,7 @@ def get_objects(self, qs=None, qfilter=None):
 
         # Apply limit and skip to the queryset
         limit = None
-        if self.view_method == methods.BulkUpdate:
+        if self.view_method in [methods.BulkUpdate, methods.BulkDelete]:
             # limit the number of objects that can be bulk-updated at a time
             qs = qs.limit(self.bulk_update_limit)
         elif not custom_qs and not self.view_method == methods.Download:
@@ -944,28 +944,28 @@ def get_objects(self, qs=None, qfilter=None):
             qs = qs.skip(skip).limit(limit+1)
             extra['total_pages'] = int(extra['total_count']/limit) + bool(extra['total_count'] % limit)
 
-        # Needs to be at the end as it returns a list, not a queryset
-        if self.select_related:
-            qs = qs.select_related()
-
-        # Evaluate the queryset
-        objs = list(qs)
-
         # Raise a validation error if bulk update would result in more than
         # bulk_update_limit updates
-        if self.view_method == methods.BulkUpdate and len(objs) >= self.bulk_update_limit:
+        if self.view_method in [methods.BulkUpdate, methods.BulkDelete] and qs.count() > self.bulk_update_limit:
             raise ValidationError({
-                'errors': ["It's not allowed to update more than %d objects at once" % self.bulk_update_limit]
+                'errors': [f"Change query to update/delete less than {self.bulk_update_limit} documents at once"]
             })
 
         # Determine the value of has_more
-        if self.view_method not in [methods.BulkUpdate, methods.Download] and self.paginate:
-            has_more = len(objs) > limit
-            if has_more:
-                objs = objs[:-1]
+        if self.view_method not in [methods.BulkUpdate, methods.BulkDelete, methods.Download] and self.paginate:
+            has_more = bool(qs.count() > limit)
         else:
             has_more = None
 
+        # Needs to be at the end as it returns a list, not a queryset
+        if self.select_related:
+            qs = qs.select_related()
+
+        # Evaluate the queryset
+        objs = list(qs)
+        if has_more:
+            objs = objs[:-1]
+
         # bulk-fetch related resources for moar speed
         self.fetch_related_resources(
             objs, self.get_requested_fields(params=params)
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 9ae46176..b8cd36c1 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -3,6 +3,7 @@
 from gzip import GzipFile
 from io import BytesIO
 from fdict import fdict
+from collections import deque
 from flask import render_template, request
 from flask.views import MethodView
 from flask_mongorest import methods
@@ -140,7 +141,7 @@ def get(self, **kwargs):
                 raise ValueError(f'{short_mime} not supported')
             self._resource.view_method = methods.Download
         else:
-            self._resource.view_method = methods.List
+            self._resource.view_method = methods.BulkFetch
 
         # Create a queryset filter to control read access to the
         # underlying objects
@@ -191,12 +192,30 @@ def get(self, **kwargs):
             return ret
 
     def post(self, **kwargs):
-        if 'pk' in kwargs:
+        if kwargs.pop('pk'):
             raise NotFound("Did you mean to use PUT?")
 
-        # Set the view_method on a resource instance
-        self._resource.view_method = methods.Create
+        raw_data = self._resource.raw_data
+        if isinstance(raw_data, dict):
+            # create single object
+            self._resource.view_method = methods.Create
+            return self.create_object()
+        elif isinstance(raw_data, list):
+            if len(raw_data) > self._resource.bulk_update_limit:
+                raise ValidationError({
+                    'errors': [f"Can only create {self.bulk_update_limit} documents at once"]
+                })
+            raw_data_deque = deque(raw_data)
+            self._resource.view_method = methods.BulkCreate
+            ret = []
+            while len(raw_data_deque):
+                self._resource._raw_data = raw_data_deque.popleft()
+                ret.append(self.create_object())
+            return {'data': ret, 'count': len(ret)}, '201 Created'
+        else:
+            raise ValidationError({'error': 'wrong payload type'})
 
+    def create_object(self):
         self._resource.validate_request()
         try:
             obj = self._resource.create_object(save=False)
@@ -208,12 +227,7 @@ def post(self, **kwargs):
             raise Unauthorized
 
         self._resource.save_object(obj, force_insert=True)
-
-        ret = self._resource.serialize(obj, params=request.args)
-        if isinstance(obj, mongoengine.Document) and self._resource.uri_prefix:
-            return ret, "201 Created", {"Location": self._resource._url(str(obj.id))}
-        else:
-            return ret
+        return self._resource.serialize(obj, params=request.args)
 
     def process_object(self, obj):
         """Validate and update an object"""
@@ -283,20 +297,53 @@ def put(self, **kwargs):
             fields = ','.join(raw_data.keys())
             return self._resource.serialize(obj, params={'_fields': fields})
 
+
+    def delete_object(self, obj):
+        """Delete an object"""
+        # Check if we have permission to delete this object
+        if not self.has_delete_permission(request, obj):
+            raise Unauthorized
+
+        try:
+            self._resource.delete_object(obj)
+        except Exception as e:
+            self.handle_validation_error(e)
+
+    def delete_objects(self, objs):
+        """Delete each object in the list one by one, and return the total count."""
+        count = 0
+        try:
+            for obj in objs:
+                self.delete_object(obj)
+                count += 1
+        except ValidationError as e:
+            e.args[0]['count'] = count
+            raise e
+        else:
+            return {'count': count}
+
     def delete(self, **kwargs):
         pk = kwargs.pop('pk', None)
 
         # Set the view_method on a resource instance
-        self._resource.view_method = methods.Delete
-
-        obj = self._resource.get_object(pk)
+        if pk:
+            self._resource.view_method = methods.Delete
+        else:
+            self._resource.view_method = methods.BulkDelete
 
-        # Check if we have permission to delete this object
-        if not self.has_delete_permission(request, obj):
-            raise Unauthorized
+        if pk is None:
+            result = self._resource.get_objects()
+            if len(result) == 2:
+                objs, has_more = result
+            elif len(result) == 3:
+                objs, has_more, extra = result
 
-        self._resource.delete_object(obj)
-        return {}
+            # Delete all the objects and return their count
+            return self.delete_objects(objs)
+        else:
+            obj = self._resource.get_object(pk)
+            self.delete_object(obj)
+            return {}
 
     # This takes a QuerySet as an argument and then
     # returns a query set that this request can read

From 414ef1a8b4ff0f05fd0ec24afd6497eac6ebefdd Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 28 Feb 2020 19:16:42 -0800
Subject: [PATCH 44/78] limit bugfix

---
 flask_mongorest/views.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index b8cd36c1..5f96f4f8 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -201,9 +201,10 @@ def post(self, **kwargs):
             self._resource.view_method = methods.Create
             return self.create_object()
         elif isinstance(raw_data, list):
-            if len(raw_data) > self._resource.bulk_update_limit:
+            limit = self._resource.bulk_update_limit
+            if len(raw_data) > limit:
                 raise ValidationError({
-                    'errors': [f"Can only create {self.bulk_update_limit} documents at once"]
+                    'errors': [f"Can only create {limit} documents at once"]
                 })
             raw_data_deque = deque(raw_data)
             self._resource.view_method = methods.BulkCreate

From 994ad9f9fbe3b0b73dee92522f7d5b57e1c63928 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Mon, 2 Mar 2020 17:51:01 -0800
Subject: [PATCH 45/78] rm bulk_update_limit, respect skip/limit, include
 has_more/counts

---
 flask_mongorest/resources.py | 30 +++++++-----------------------
 flask_mongorest/views.py     | 14 ++++++++++----
 2 files changed, 17 insertions(+), 27 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index c3b9bbc4..b6e3c366 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -83,9 +83,6 @@ class Resource(object):
     # Only relevant if pagination is enabled.
     max_limit = 100
 
-    # Maximum number of objects which can be bulk-updated by a single request
-    bulk_update_limit = 1000  # NOTE also used for bulk delete
-
     # Map of field names to paginate with according default and maximum limits
     fields_to_paginate = {}
 
@@ -933,36 +930,23 @@ def get_objects(self, qs=None, qfilter=None):
         qs = self.apply_ordering(qs, params)
         extra['total_count'] = qs.count()
 
-        # Apply limit and skip to the queryset
+        # Apply limit and skip to the queryset (if not Download and no custom queryset provided)
         limit = None
-        if self.view_method in [methods.BulkUpdate, methods.BulkDelete]:
-            # limit the number of objects that can be bulk-updated at a time
-            qs = qs.limit(self.bulk_update_limit)
-        elif not custom_qs and not self.view_method == methods.Download:
-            # no need to skip/limit if a custom `qs` was provided
+        if not custom_qs and self.view_method != methods.Download:
             skip, limit = self.get_skip_and_limit(params)
-            qs = qs.skip(skip).limit(limit+1)
+            qs = qs.skip(skip).limit(limit+1)  # get one extra to determine has_more
             extra['total_pages'] = int(extra['total_count']/limit) + bool(extra['total_count'] % limit)
 
-        # Raise a validation error if bulk update would result in more than
-        # bulk_update_limit updates
-        if self.view_method in [methods.BulkUpdate, methods.BulkDelete] and qs.count() > self.bulk_update_limit:
-            raise ValidationError({
-                'errors': [f"Change query to update/delete less than {self.bulk_update_limit} documents at once"]
-            })
-
-        # Determine the value of has_more
-        if self.view_method not in [methods.BulkUpdate, methods.BulkDelete, methods.Download] and self.paginate:
-            has_more = bool(qs.count() > limit)
-        else:
-            has_more = None
-
         # Needs to be at the end as it returns a list, not a queryset
         if self.select_related:
             qs = qs.select_related()
 
         # Evaluate the queryset
         objs = list(qs)
+        has_more = None
+        if self.view_method != methods.Download and self.paginate:
+            has_more = bool(len(objs) > limit)
+
         if has_more:
             objs = objs[:-1]
 
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 5f96f4f8..19a4e6f0 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -201,7 +201,7 @@ def post(self, **kwargs):
             self._resource.view_method = methods.Create
             return self.create_object()
         elif isinstance(raw_data, list):
-            limit = self._resource.bulk_update_limit
+            limit = self._resource.max_limit
             if len(raw_data) > limit:
                 raise ValidationError({
                     'errors': [f"Can only create {limit} documents at once"]
@@ -290,7 +290,10 @@ def put(self, **kwargs):
                 objs, has_more, extra = result
 
             # Update all the objects and return their count
-            return self.process_objects(objs)
+            ret = self.process_objects(objs)
+            ret['has_more'] = has_more
+            ret.update(extra)
+            return ret
         else:
             obj = self._resource.get_object(pk)
             self.process_object(obj)
@@ -340,11 +343,14 @@ def delete(self, **kwargs):
                 objs, has_more, extra = result
 
             # Delete all the objects and return their count
-            return self.delete_objects(objs)
+            ret = self.delete_objects(objs)
+            ret['has_more'] = has_more
+            ret.update(extra)
+            return ret
         else:
             obj = self._resource.get_object(pk)
             self.delete_object(obj)
-            return {}
+            return {'count': 1}
 
     # This takes a QuerySet as an argument and then
     # returns a query set that this request can read

From 5e7df34a42d25fcd36e668f8ed591598568d243e Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Tue, 28 Apr 2020 23:32:06 -0700
Subject: [PATCH 46/78] add __all__ to methods

---
 flask_mongorest/methods.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/flask_mongorest/methods.py b/flask_mongorest/methods.py
index 25b98d8f..21618fd5 100644
--- a/flask_mongorest/methods.py
+++ b/flask_mongorest/methods.py
@@ -1,3 +1,6 @@
+import sys
+import inspect
+
 class Fetch:
     method = 'GET'
 
@@ -26,3 +29,6 @@ class BulkDelete:
 
 class Download:
     method = 'GET'
+
+members = inspect.getmembers(sys.modules[__name__], inspect.isclass)
+__all__ = [m[0] for m in members]

From 8c9d860e120e424e159084cfa91917a5a836a0d0 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 20 May 2020 20:48:06 -0700
Subject: [PATCH 47/78] use glom for path access; skip paginate_fields on
 Download

---
 flask_mongorest/resources.py | 10 ++++------
 flask_mongorest/views.py     |  3 ++-
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index b6e3c366..4acd6e51 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -1,6 +1,8 @@
 import json
 import mongoengine
 
+from glom import glom
+from glom.core import PathAccessError
 from typing import Pattern
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
@@ -392,12 +394,8 @@ def get_field_value(self, obj, field_name, field_instance=None, **kwargs):
             field_value = obj
         else:
             try:
-                dotty = Dotty(obj if isinstance(obj, dict) else obj.to_mongo())
-                if '.' not in field_name:
-                    is_pk = obj._fields[field_name].primary_key
-                    field_name = '_id' if is_pk or field_name == 'id' else field_name
-                field_value = dotty[field_name]
-            except (AttributeError, KeyError):
+                field_value = glom(obj, field_name)
+            except PathAccessError as ex:
                 raise UnknownFieldError
 
         return self.serialize_field_value(obj, field_name, field_instance, field_value, **kwargs)
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 19a4e6f0..55111d9b 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -161,7 +161,8 @@ def get(self, **kwargs):
 
             data = []
             for obj in objs:
-                obj = self._resource.paginate_fields(obj)
+                if self._resource.view_method != methods.Download:
+                    obj = self._resource.paginate_fields(obj)
                 try:
                     data.append(self._resource.serialize(obj, params=request.args))
                 except Exception as e:

From 7a12eb5875a0b30632792913774e5e34cf78f7aa Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 4 Jun 2020 14:38:57 -0700
Subject: [PATCH 48/78] save progress on S3 downloads

---
 flask_mongorest/resources.py |   1 +
 flask_mongorest/views.py     | 100 ++++++++++++++++++++++++++---------
 2 files changed, 76 insertions(+), 25 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 4acd6e51..ff0869c8 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -395,6 +395,7 @@ def get_field_value(self, obj, field_name, field_instance=None, **kwargs):
         else:
             try:
                 field_value = glom(obj, field_name)
+                #field_value = getattr(obj, field_name)  # TODO improve serialization time
             except PathAccessError as ex:
                 raise UnknownFieldError
 
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 55111d9b..67296671 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -1,4 +1,7 @@
+import os
 import json
+import boto3
+import hashlib
 import mongoengine
 from gzip import GzipFile
 from io import BytesIO
@@ -11,7 +14,13 @@
 from flask_mongorest.utils import MongoEncoder
 from werkzeug.exceptions import NotFound, Unauthorized
 from mimerender import register_mime, FlaskMimeRender
+from botocore.errorfactory import ClientError
 
+BUCKET = os.environ.get('S3_DOWNLOADS_BUCKET', 'mongorest-downloads')
+CNAME = os.environ.get('PORTAL_CNAME')
+S3_DOWNLOAD_URL = f"https://{BUCKET}.s3.amazonaws.com"
+
+s3_client = boto3.client('s3')
 mimerender = FlaskMimeRender(global_override_input_key='short_mime')
 register_mime('gz', ('application/gzip',))
 
@@ -23,21 +32,38 @@ def render_html(**payload):
     return render_template('mongorest/debug.html', data=d)
 
 def render_gz(**payload):
-    contents = ''
-    fmt = request.values.get('format')
-    if fmt == 'json':
-        contents = json.dumps(payload['data'], allow_nan=False, cls=MongoEncoder)
-    elif fmt == 'csv':
-        from pandas import DataFrame
-        from cherrypicker import CherryPicker
-        records = [CherryPicker(d).flatten().get() for d in payload['data']]
-        contents = DataFrame.from_records(records).to_csv()
-
-    gzip_buffer = BytesIO()
-    with GzipFile(mode='wb', fileobj=gzip_buffer) as gzip_file:
-        gzip_file.write(contents.encode())  # need to give full contents to compression
+    s3 = payload.get("s3")
+
+    if s3 and not s3["exists"]:
+        fmt = request.args.get('format')
+        content_type = 'text/csv' if fmt == 'csv' else 'application/json'
+        if fmt == 'json':
+            contents = json.dumps(payload['data'], allow_nan=False, cls=MongoEncoder)
+        elif fmt == 'csv':
+            from pandas import DataFrame
+            from cherrypicker import CherryPicker
+            records = [CherryPicker(d).flatten().get() for d in payload['data']]
+            contents = DataFrame.from_records(records).to_csv()
+
+        gzip_buffer = BytesIO()
+        with GzipFile(mode='wb', fileobj=gzip_buffer) as gzip_file:
+            gzip_file.write(contents.encode('utf-8'))  # need to give full contents to compression
+
+        body = gzip_buffer.getvalue()
+        s3_client.put_object(
+            Bucket=BUCKET,
+            Key=s3["key"],
+            ContentType=content_type,
+            ContentEncoding='gzip',
+            Body=body
+        )
+        return body
+
+    retr = s3_client.get_object(Bucket=BUCKET, Key=s3["key"])
+    gzip_buffer = BytesIO(retr['Body'].read())
     return gzip_buffer.getvalue()
 
+
 try:
     text_type = unicode # Python 2
 except NameError:
@@ -132,6 +158,7 @@ def requested_resource(self, request):
     def get(self, **kwargs):
         pk = kwargs.pop('pk', None)
         short_mime = kwargs.pop('short_mime', None)
+        fmt = self._resource.params.get('format')
 
         # Set the view_method on a resource instance
         if pk:
@@ -146,6 +173,7 @@ def get(self, **kwargs):
         # Create a queryset filter to control read access to the
         # underlying objects
         qfilter = lambda qs: self.has_read_permission(request, qs.clone())
+
         if pk is None:
             result = self._resource.get_objects(qfilter=qfilter)
 
@@ -159,18 +187,42 @@ def get(self, **kwargs):
             else:
                 raise ValueError('Unsupported value of resource.get_objects')
 
-            data = []
-            for obj in objs:
-                if self._resource.view_method != methods.Download:
-                    obj = self._resource.paginate_fields(obj)
+            # generate hash/etag and S3 object name
+            if self._resource.view_method == methods.Download:
+                dct = {
+                    "ids": [str(obj.pk) for obj in objs],
+                    "params": self._resource.params
+                }
+                sha1 = hashlib.sha1(json.dumps(dct).encode('utf-8')).hexdigest()
+                filename = f"{sha1}.{fmt}"
+                key = f"{CNAME}/{filename}" if CNAME else filename
+                extra["s3"] = {"key": key, "exists": False}
                 try:
-                    data.append(self._resource.serialize(obj, params=request.args))
-                except Exception as e:
-                    fixed_obj = self._resource.handle_serialization_error(e, obj)
-                    if fixed_obj is not None:
-                        data.append(fixed_obj)
+                    s3_client.head_object(Bucket=BUCKET, Key=key)
+                    extra["s3"]["exists"] = True
+                except ClientError:
+                    pass
 
             # Serialize the objects one by one
+            data = []
+            if "s3" not in extra or not extra["s3"]["exists"]:
+                print("serializing ...")
+                tic = time.perf_counter()
+                batch_size = 200
+                for idx, obj in enumerate(objs):
+                    if idx > 0 and not idx % batch_size:
+                        toc = time.perf_counter()
+                        print(f"{idx} Took {toc - tic:0.4f} seconds to serialize {batch_size} objects.")
+                        tic = time.perf_counter()
+                    if self._resource.view_method != methods.Download:
+                        obj = self._resource.paginate_fields(obj)
+                    try:
+                        data.append(self._resource.serialize(obj, params=request.args))
+                    except Exception as e:
+                        fixed_obj = self._resource.handle_serialization_error(e, obj)
+                        if fixed_obj is not None:
+                            data.append(fixed_obj)
+
             ret = {'data': data}
 
             if has_more is not None:
@@ -183,11 +235,9 @@ def get(self, **kwargs):
             obj = self._resource.paginate_fields(obj)
             ret = self._resource.serialize(obj, params=request.args)
 
-
         if self._resource.view_method == methods.Download:
-            fmt = self._resource.params.get('format')
             return ret, '200 OK', {
-                'Content-Disposition': f'attachment; filename="data.{fmt}.{short_mime}"'
+                'Content-Disposition': f'attachment; filename="{filename}.{short_mime}"'
             }
         else:
             return ret

From dc6ca869f41b4b82a9b2baf7a58b91593dc33283 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 11 Jun 2020 14:21:46 -0700
Subject: [PATCH 49/78] fix time import

---
 flask_mongorest/resources.py | 5 +++++
 flask_mongorest/views.py     | 1 +
 2 files changed, 6 insertions(+)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index ff0869c8..a121926d 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -1,4 +1,5 @@
 import json
+import time
 import mongoengine
 
 from glom import glom
@@ -488,6 +489,7 @@ def serialize(self, obj, **kwargs):
         Given an object, serialize it, turning it into its JSON
         representation.
         """
+        #tic = time.perf_counter()
         if not obj:
             return {}
 
@@ -540,6 +542,8 @@ def serialize(self, obj, **kwargs):
                     except UnknownFieldError:
                         pass
 
+        #toc = time.perf_counter()
+        #print(f"Took {toc - tic:0.4f} seconds to serialize requested fields")
         return data._data
 
     def handle_serialization_error(self, exc, obj):
@@ -941,6 +945,7 @@ def get_objects(self, qs=None, qfilter=None):
             qs = qs.select_related()
 
         # Evaluate the queryset
+        print("evaluate...")
         objs = list(qs)
         has_more = None
         if self.view_method != methods.Download and self.paginate:
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 67296671..996b23df 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -1,4 +1,5 @@
 import os
+import time
 import json
 import boto3
 import hashlib

From 92b5c250e487f6829919ed33ed4b33b40f06428d Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 11 Jun 2020 19:05:41 -0700
Subject: [PATCH 50/78] use getattr; allow dynamic fields in queryset mask

---
 flask_mongorest/resources.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index a121926d..32321679 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -395,10 +395,12 @@ def get_field_value(self, obj, field_name, field_instance=None, **kwargs):
             field_value = obj
         else:
             try:
-                field_value = glom(obj, field_name)
-                #field_value = getattr(obj, field_name)  # TODO improve serialization time
-            except PathAccessError as ex:
-                raise UnknownFieldError
+                field_value = getattr(obj, field_name)
+            except AttributeError:
+                try:
+                    field_value = glom(obj, field_name)  # TODO improve serialization time
+                except PathAccessError as ex:
+                    raise UnknownFieldError
 
         return self.serialize_field_value(obj, field_name, field_instance, field_value, **kwargs)
 
@@ -628,12 +630,10 @@ def get_queryset(self):
         if request.method == 'PUT':
             return self.document.objects  # get full documents for updates
         else:
-            document_fields = set(self.document._fields.keys())
             requested_fields = set(
                 f.split('.', 1)[0] for f in self.get_requested_fields(params=self.params)
             )
-            mask = requested_fields & document_fields
-            return self.document.objects.only(*mask)
+            return self.document.objects.only(*requested_fields)
 
     def get_object(self, pk, qfilter=None):
         """

From 8ac3f816f2d5b12047ba96d24c7470b2da90d2b8 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 12 Jun 2020 16:04:43 -0700
Subject: [PATCH 51/78] apply field pagination to queryset

---
 flask_mongorest/resources.py | 48 ++++++++++++++++++++----------------
 flask_mongorest/views.py     |  3 ---
 2 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 32321679..6cf9fd6b 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -40,6 +40,7 @@
 from flask_mongorest import methods
 from flask_mongorest.exceptions import ValidationError, UnknownFieldError
 from flask_mongorest.utils import cmp_fields, isbound, isint, equal
+from flask_mongoengine.pagination import ListFieldPagination
 
 
 def get_with_list_index(o, k):
@@ -645,6 +646,10 @@ def get_object(self, pk, qfilter=None):
         # get a new one out
         if qfilter:
             qs = qfilter(qs)
+
+        if self.view_method != methods.Download:
+            qs = self.apply_field_pagination(qs)
+
         obj = qs.get(pk=pk)
 
         # We don't need to fetch related resources for DELETE requests because
@@ -658,33 +663,33 @@ def get_object(self, pk, qfilter=None):
 
         return obj
 
-    def paginate_fields(self, obj):
-        """
-        return object with fields paginated according to `fields_to_paginate`
-        """
-        if self.view_method == methods.Download:
-            return obj  # no pagination when downloading
+    def apply_field_pagination(self, qs, params=None):
+        """apply field pagination according to `fields_to_paginate`"""
+        if params is None:
+            params = self.params
+
+        field_attrs = {}
         for field, limits in self.fields_to_paginate.items():
-            page = self.params.get(f'{field}_page', 1)
-            per_page = self.params.get(f'{field}_per_page', limits[0])
-            # page and per_page validation
+            page = params.get(f'{field}_page', 1)
+            per_page = params.get(f'{field}_per_page', limits[0])
             if not isint(page):
                 raise ValidationError({'error': f'{field}_page must be an integer.'})
             if not isint(per_page):
                 raise ValidationError({'error': f'{field}_per_page must be an integer.'})
-            if int(per_page) > limits[1]:
-                raise ValidationError({'error': f"The per page limit ({per_page}) you set is larger than the maximum for the {field} field ({limits[1]})."})
-            if int(page) < 0:
+
+            page, per_page = int(page), int(per_page)
+            if per_page > limits[1]:
+                raise ValidationError({
+                    'error': f"Per-page limit ({per_page}) for {field} too large ({limits[1]})."
+                })
+            if page < 0:
                 raise ValidationError({'error': f'{field}_page must be a non-negative integer.'})
 
-            per_page = min(int(per_page), limits[1])
-            total_rows = len(obj[field])
-            if total_rows > 0:
-                total_pages = int(total_rows/per_page) + bool(total_rows % per_page)
-                if int(page) > total_pages:
-                    raise ValidationError({'error': f'{field}_page must be less or equal {total_pages}.'})
-                obj.data = obj.paginate_field(field, int(page), per_page=per_page).items
-        return obj
+            per_page = min(per_page, limits[1])
+            start_index = (page - 1) * per_page
+            field_attrs[field] = {"$slice": [start_index, per_page]}
+
+        return qs.fields(**field_attrs)
 
     def fetch_related_resources(self, objs, only_fields=None):
         """
@@ -933,11 +938,12 @@ def get_objects(self, qs=None, qfilter=None):
         qs = self.apply_ordering(qs, params)
         extra['total_count'] = qs.count()
 
-        # Apply limit and skip to the queryset (if not Download and no custom queryset provided)
+        # Apply pagination to the queryset (if not Download and no custom queryset provided)
         limit = None
         if not custom_qs and self.view_method != methods.Download:
             skip, limit = self.get_skip_and_limit(params)
             qs = qs.skip(skip).limit(limit+1)  # get one extra to determine has_more
+            qs = self.apply_field_pagination(qs, params)
             extra['total_pages'] = int(extra['total_count']/limit) + bool(extra['total_count'] % limit)
 
         # Needs to be at the end as it returns a list, not a queryset
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 996b23df..5adc07c8 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -215,8 +215,6 @@ def get(self, **kwargs):
                         toc = time.perf_counter()
                         print(f"{idx} Took {toc - tic:0.4f} seconds to serialize {batch_size} objects.")
                         tic = time.perf_counter()
-                    if self._resource.view_method != methods.Download:
-                        obj = self._resource.paginate_fields(obj)
                     try:
                         data.append(self._resource.serialize(obj, params=request.args))
                     except Exception as e:
@@ -233,7 +231,6 @@ def get(self, **kwargs):
                 ret.update(extra)
         else:
             obj = self._resource.get_object(pk, qfilter=qfilter)
-            obj = self._resource.paginate_fields(obj)
             ret = self._resource.serialize(obj, params=request.args)
 
         if self._resource.view_method == methods.Download:

From 6e7232778c57e45c100935d1ec30d0c94138e5e7 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 12 Jun 2020 16:05:07 -0700
Subject: [PATCH 52/78] include document fields back in queryset mask

---
 flask_mongorest/resources.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 6cf9fd6b..da0e34c9 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -631,10 +631,11 @@ def get_queryset(self):
         if request.method == 'PUT':
             return self.document.objects  # get full documents for updates
         else:
-            requested_fields = set(
-                f.split('.', 1)[0] for f in self.get_requested_fields(params=self.params)
-            )
-            return self.document.objects.only(*requested_fields)
+            document_fields = set(self.fields + self.get_optional_fields())
+            requested_fields = self.get_requested_fields(params=self.params)
+            requested_root_fields = set(f.split('.', 1)[0] for f in requested_fields)
+            mask = requested_root_fields & document_fields
+            return self.document.objects.only(*mask)
 
     def get_object(self, pk, qfilter=None):
         """

From cd2c194116309ad1a52c955c5eed607afa684dcc Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 17 Jun 2020 16:11:45 -0700
Subject: [PATCH 53/78] glom & LazyReference

---
 flask_mongorest/resources.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index da0e34c9..ad517849 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -2,12 +2,11 @@
 import time
 import mongoengine
 
-from glom import glom
+from glom import glom, assign
 from glom.core import PathAccessError
 from typing import Pattern
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
-from dotty_dict import Dotty
 from flask import has_request_context, request, url_for
 from dict_deep import deep_get
 try:
@@ -22,7 +21,9 @@
     DocumentProxy = None
     SafeReferenceField = None
 
-from mongoengine.fields import EmbeddedDocumentField, ListField, ReferenceField, GenericReferenceField
+from mongoengine.fields import EmbeddedDocumentField, ListField
+from mongoengine.fields import GenericReferenceField, ReferenceField
+from mongoengine.fields import GenericLazyReferenceField, LazyReferenceField
 from mongoengine.fields import DictField
 
 try:
@@ -411,6 +412,9 @@ def serialize_field_value(self, obj, field_name, field_instance, field_value, **
         field_value is an actual value to be serialized.
         For other fields, see get_field_value method.
         """
+        if isinstance(field_instance, (LazyReferenceField, GenericLazyReferenceField)):
+            return field_value.pk
+
         if isinstance(field_instance, (ReferenceField, GenericReferenceField, EmbeddedDocumentField)):
             return self.serialize_document_field(field_name, field_value, **kwargs)
 
@@ -513,13 +517,14 @@ def serialize(self, obj, **kwargs):
 
         # Fill in the `data` dict by serializing each of the requested fields
         # one by one.
-        data = Dotty({})
+        data = {}
         for field in requested_fields:
 
             # resolve the user-facing name of the field
             renamed_field = self._rename_fields.get(field, field)
 
             # if the field is callable, execute it with `obj` as the param
+            value = None
             if hasattr(self, field) and callable(getattr(self, field)):
                 value = getattr(self, field)(obj)
 
@@ -535,19 +540,19 @@ def serialize(self, obj, **kwargs):
                     else:  # assume queryset or list
                         value = [related_resource.serialize_field(o)
                                  for o in value]
-                data[renamed_field] = value
             else:
                 try:
-                    data[renamed_field] = self.get_field_value(obj, field, **kwargs)
+                    value = self.get_field_value(obj, field, **kwargs)
                 except UnknownFieldError:
                     try:
-                        data[renamed_field] = self.value_for_field(obj, field)
+                        value = self.value_for_field(obj, field)
                     except UnknownFieldError:
                         pass
 
-        #toc = time.perf_counter()
-        #print(f"Took {toc - tic:0.4f} seconds to serialize requested fields")
-        return data._data
+            if value is not None:
+                assign(data, renamed_field, value, missing=dict)
+
+        return data
 
     def handle_serialization_error(self, exc, obj):
         """

From 4dcec42dd3a8295b0d4d4a6b1685918f24b5005d Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 17 Jun 2020 16:12:07 -0700
Subject: [PATCH 54/78] download SSE messages

---
 flask_mongorest/views.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 5adc07c8..3f233a96 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -10,12 +10,14 @@
 from collections import deque
 from flask import render_template, request
 from flask.views import MethodView
+from flask_sse import sse
 from flask_mongorest import methods
 from flask_mongorest.exceptions import ValidationError
 from flask_mongorest.utils import MongoEncoder
 from werkzeug.exceptions import NotFound, Unauthorized
 from mimerender import register_mime, FlaskMimeRender
 from botocore.errorfactory import ClientError
+from urllib.parse import unquote
 
 BUCKET = os.environ.get('S3_DOWNLOADS_BUCKET', 'mongorest-downloads')
 CNAME = os.environ.get('PORTAL_CNAME')
@@ -207,13 +209,17 @@ def get(self, **kwargs):
             # Serialize the objects one by one
             data = []
             if "s3" not in extra or not extra["s3"]["exists"]:
-                print("serializing ...")
+                url = unquote(request.url).encode('utf-8')
+                channel = hashlib.sha1(url).hexdigest()
+                print(f"serializing {channel}...")
                 tic = time.perf_counter()
-                batch_size = 200
+                batch_size = 1000
                 for idx, obj in enumerate(objs):
                     if idx > 0 and not idx % batch_size:
                         toc = time.perf_counter()
-                        print(f"{idx} Took {toc - tic:0.4f} seconds to serialize {batch_size} objects.")
+                        print(f"{idx} Took {toc - tic:0.4f}s to serialize {batch_size} objects.")
+                        if self._resource.view_method == methods.Download:
+                            sse.publish({"message": idx + 1}, type="download", channel=channel)
                         tic = time.perf_counter()
                     try:
                         data.append(self._resource.serialize(obj, params=request.args))
@@ -234,6 +240,7 @@ def get(self, **kwargs):
             ret = self._resource.serialize(obj, params=request.args)
 
         if self._resource.view_method == methods.Download:
+            sse.publish({"message": 0}, type="download", channel=channel)
             return ret, '200 OK', {
                 'Content-Disposition': f'attachment; filename="{filename}.{short_mime}"'
             }

From 36de94c3d34eeddec602c6a31835de7d3b352ea4 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 17 Jun 2020 23:05:06 -0700
Subject: [PATCH 55/78] channel bugfix

---
 flask_mongorest/views.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 3f233a96..f62b8273 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -208,9 +208,9 @@ def get(self, **kwargs):
 
             # Serialize the objects one by one
             data = []
+            url = unquote(request.url).encode('utf-8')
+            channel = hashlib.sha1(url).hexdigest()
             if "s3" not in extra or not extra["s3"]["exists"]:
-                url = unquote(request.url).encode('utf-8')
-                channel = hashlib.sha1(url).hexdigest()
                 print(f"serializing {channel}...")
                 tic = time.perf_counter()
                 batch_size = 1000

From f8b9f3ee195ba8e16f141ca48a039271c40e09a0 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 18 Jun 2020 15:20:19 -0700
Subject: [PATCH 56/78] use ma.utils.get/set_value if available

much faster serialization! :)
---
 flask_mongorest/resources.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index ad517849..9974cb37 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -2,8 +2,6 @@
 import time
 import mongoengine
 
-from glom import glom, assign
-from glom.core import PathAccessError
 from typing import Pattern
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
@@ -35,8 +33,11 @@
 try:
     from marshmallow_mongoengine import ModelSchema
     from marshmallow.exceptions import ValidationError as MarshmallowValidationError
+    from marshmallow.utils import get_value, set_value
 except ImportError:
     ModelSchema = None
+    from glom import glom, assign
+    from glom.core import PathAccessError
 
 from flask_mongorest import methods
 from flask_mongorest.exceptions import ValidationError, UnknownFieldError
@@ -395,14 +396,19 @@ def get_field_value(self, obj, field_name, field_instance=None, **kwargs):
         # Determine the field value
         if has_field_instance:
             field_value = obj
-        else:
+        elif ModelSchema is None:
             try:
                 field_value = getattr(obj, field_name)
             except AttributeError:
                 try:
-                    field_value = glom(obj, field_name)  # TODO improve serialization time
+                    field_value = glom(obj, field_name)  # slow
                 except PathAccessError as ex:
                     raise UnknownFieldError
+        else:
+            try:
+                field_value = get_value(obj, field_name)
+            except AttributeError:
+                raise UnknownFieldError
 
         return self.serialize_field_value(obj, field_name, field_instance, field_value, **kwargs)
 
@@ -550,7 +556,10 @@ def serialize(self, obj, **kwargs):
                         pass
 
             if value is not None:
-                assign(data, renamed_field, value, missing=dict)
+                if ModelSchema is None:
+                    assign(data, renamed_field, value, missing=dict)  # slow
+                else:
+                    set_value(data, renamed_field, value)
 
         return data
 

From 11a90d1e2b9d497ee4889c357afb3f3607e6b483 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 18 Jun 2020 15:24:48 -0700
Subject: [PATCH 57/78] update requirements

---
 requirements.txt | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 53e29c4a..8eeacb1f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,6 @@
--e git://github.com/closeio/mongoengine.git#egg=mongoengine-dev
--e git://github.com/closeio/flask-mongoengine.git#egg=flask-mongoengine
-mimerender
-python-dateutil
-sphinx
-Flask>=0.9
-pymongo>=3.4
-flake8
+mongoengine==0.20.0
+flask-mongoengine==0.9.5
+mimerender @ git+https://github.com/tschaume/mimerender@mpcontribs#egg=mimerender-0.6.1
+python-dateutil==2.8.1
+Flask==1.1.2
+pymongo==3.10.1

From 234c18173da66d25fc92506f6465fd3a3aa5da1f Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 18 Jun 2020 15:33:37 -0700
Subject: [PATCH 58/78] catch _Missing

---
 flask_mongorest/resources.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 9974cb37..9185302d 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -33,7 +33,7 @@
 try:
     from marshmallow_mongoengine import ModelSchema
     from marshmallow.exceptions import ValidationError as MarshmallowValidationError
-    from marshmallow.utils import get_value, set_value
+    from marshmallow.utils import get_value, set_value, _Missing
 except ImportError:
     ModelSchema = None
     from glom import glom, assign
@@ -405,9 +405,8 @@ def get_field_value(self, obj, field_name, field_instance=None, **kwargs):
                 except PathAccessError as ex:
                     raise UnknownFieldError
         else:
-            try:
-                field_value = get_value(obj, field_name)
-            except AttributeError:
+            field_value = get_value(obj, field_name)
+            if isinstance(field_value, _Missing):
                 raise UnknownFieldError
 
         return self.serialize_field_value(obj, field_name, field_instance, field_value, **kwargs)

From e50a9a49396feda3942f61ed3e2d05666f0a8726 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 18 Jun 2020 17:09:08 -0700
Subject: [PATCH 59/78] better download logs

---
 flask_mongorest/resources.py | 1 -
 flask_mongorest/views.py     | 9 ++++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 9185302d..91fbdcac 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -965,7 +965,6 @@ def get_objects(self, qs=None, qfilter=None):
             qs = qs.select_related()
 
         # Evaluate the queryset
-        print("evaluate...")
         objs = list(qs)
         has_more = None
         if self.view_method != methods.Download and self.paginate:
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index f62b8273..e8b06990 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -213,11 +213,14 @@ def get(self, **kwargs):
             if "s3" not in extra or not extra["s3"]["exists"]:
                 print(f"serializing {channel}...")
                 tic = time.perf_counter()
-                batch_size = 1000
+                batch_size, total_count = 1000, extra["total_count"]
                 for idx, obj in enumerate(objs):
-                    if idx > 0 and not idx % batch_size:
+                    if idx > 0 and (not idx % batch_size or idx == total_count - 1):
                         toc = time.perf_counter()
-                        print(f"{idx} Took {toc - tic:0.4f}s to serialize {batch_size} objects.")
+                        nobjs = batch_size
+                        if idx == total_count - 1:
+                            nobjs = total_count - batch_size * int(idx/batch_size) - 1
+                        print(f"{idx} Took {toc - tic:0.4f}s to serialize {nobjs} objects.")
                         if self._resource.view_method == methods.Download:
                             sse.publish({"message": idx + 1}, type="download", channel=channel)
                         tic = time.perf_counter()

From 16f6302c83ad4782dbeef53bcd37f3416f33cb46 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Sat, 20 Jun 2020 02:10:05 -0700
Subject: [PATCH 60/78] check last_modified for S3 object update

---
 flask_mongorest/resources.py |  2 ++
 flask_mongorest/views.py     | 24 +++++++++++++-----------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 91fbdcac..1a8c8a35 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -648,6 +648,8 @@ def get_queryset(self):
             requested_fields = self.get_requested_fields(params=self.params)
             requested_root_fields = set(f.split('.', 1)[0] for f in requested_fields)
             mask = requested_root_fields & document_fields
+            if self.view_method == methods.Download:
+                mask.add("last_modified")
             return self.document.objects.only(*mask)
 
     def get_object(self, pk, qfilter=None):
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index e8b06990..80819e88 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -37,7 +37,7 @@ def render_html(**payload):
 def render_gz(**payload):
     s3 = payload.get("s3")
 
-    if s3 and not s3["exists"]:
+    if s3 and s3["update"]:
         fmt = request.args.get('format')
         content_type = 'text/csv' if fmt == 'csv' else 'application/json'
         if fmt == 'json':
@@ -192,25 +192,27 @@ def get(self, **kwargs):
 
             # generate hash/etag and S3 object name
             if self._resource.view_method == methods.Download:
-                dct = {
-                    "ids": [str(obj.pk) for obj in objs],
-                    "params": self._resource.params
-                }
-                sha1 = hashlib.sha1(json.dumps(dct).encode('utf-8')).hexdigest()
+                primary_keys = [str(obj.pk) for obj in objs]
+                last_modified = max(obj.last_modified for obj in objs)
+                dct = {"ids": primary_keys, "params": self._resource.params}
+                sha1 = hashlib.sha1(
+                    json.dumps(dct, sort_keys=True).encode('utf-8')
+                ).hexdigest()
                 filename = f"{sha1}.{fmt}"
                 key = f"{CNAME}/{filename}" if CNAME else filename
-                extra["s3"] = {"key": key, "exists": False}
+                extra["s3"] = {"key": key, "update": False}
                 try:
-                    s3_client.head_object(Bucket=BUCKET, Key=key)
-                    extra["s3"]["exists"] = True
+                    s3_client.head_object(
+                        Bucket=BUCKET, Key=key, IfModifiedSince=last_modified
+                    )
                 except ClientError:
-                    pass
+                    extra["s3"]["update"] = True
 
             # Serialize the objects one by one
             data = []
             url = unquote(request.url).encode('utf-8')
             channel = hashlib.sha1(url).hexdigest()
-            if "s3" not in extra or not extra["s3"]["exists"]:
+            if "s3" not in extra or extra["s3"]["update"]:
                 print(f"serializing {channel}...")
                 tic = time.perf_counter()
                 batch_size, total_count = 1000, extra["total_count"]

From 0e1a18fc7958c94b1aaf31d4452d9f333a86d82c Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Tue, 30 Jun 2020 16:57:38 -0700
Subject: [PATCH 61/78] set dict field if unset previously via null

---
 flask_mongorest/resources.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 1a8c8a35..1716b1aa 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -1069,6 +1069,8 @@ def update_object(self, obj, data=None, save=True, parent_resources=None):
                     if value is None:
                         update = True
                     else:
+                        if obj[field] is None:
+                            obj[field] = {}
                         self.update_object(obj[field], data=value, save=False)
                 elif obj._fields[field].primary_key:
                     raise ValidationError({'error': f'`{field}` is primary key and cannot be updated'})

From 3e4794b524a91d7a77898058a0219b7a0c64e4ec Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 15 Jul 2020 01:17:58 -0700
Subject: [PATCH 62/78] use params and set_value

---
 flask_mongorest/resources.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 1716b1aa..867b1dd6 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -934,7 +934,7 @@ def get_objects(self, qs=None, qfilter=None):
         extra = {}
 
         if self.view_method == methods.Download:
-            fmt = self.params.get('format')
+            fmt = params.get('format')
             if fmt not in self.download_formats:
                 raise ValueError(f'`format` must be one of {self.download_formats}')
 
@@ -1080,10 +1080,7 @@ def update_object(self, obj, data=None, save=True, parent_resources=None):
                 update = True
 
             if update:
-                if getattr(obj, field, None) is not None:
-                    setattr(obj, field, value)
-                else:
-                    obj[field] = value
+                set_value(obj, field, value)
                 self._dirty_fields.append(field)
 
         if save:

From f0d9ee987080afc746d1dfb2b7a33a62bc7828a4 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 16 Jul 2020 23:06:47 -0700
Subject: [PATCH 63/78] unflatten raw_data, minor cleanup

---
 flask_mongorest/resources.py | 14 +++++++++-----
 flask_mongorest/views.py     |  1 -
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 867b1dd6..3b29b27d 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -1,7 +1,7 @@
 import json
-import time
 import mongoengine
 
+from unflatten import unflatten
 from typing import Pattern
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
@@ -42,7 +42,6 @@
 from flask_mongorest import methods
 from flask_mongorest.exceptions import ValidationError, UnknownFieldError
 from flask_mongorest.utils import cmp_fields, isbound, isint, equal
-from flask_mongoengine.pagination import ListFieldPagination
 
 
 def get_with_list_index(o, k):
@@ -60,6 +59,7 @@ def __init__(cls, name, bases, classdict):
                     cls.child_document_resources[document] = cls
         type.__init__(cls, name, bases, classdict)
 
+
 class Resource(object):
     # MongoEngine Document class related to this resource (required)
     document = None
@@ -205,7 +205,12 @@ def raw_data(self):
                     raise ValidationError({'error': "Chunked Transfer-Encoding is not supported."})
 
                 try:
-                    self._raw_data = json.loads(request.data.decode('utf-8'), parse_constant=self._enforce_strict_json)
+                    self._raw_data = json.loads(
+                        request.data.decode('utf-8'),
+                        parse_constant=self._enforce_strict_json
+                    )
+                    if request.method == 'PUT':
+                        self._raw_data = unflatten(self._raw_data)
                 except ValueError:
                     raise ValidationError({'error': 'The request contains invalid JSON.'})
                 if request.method == 'PUT' and not isinstance(self._raw_data, dict):
@@ -402,7 +407,7 @@ def get_field_value(self, obj, field_name, field_instance=None, **kwargs):
             except AttributeError:
                 try:
                     field_value = glom(obj, field_name)  # slow
-                except PathAccessError as ex:
+                except PathAccessError:
                     raise UnknownFieldError
         else:
             field_value = get_value(obj, field_name)
@@ -501,7 +506,6 @@ def serialize(self, obj, **kwargs):
         Given an object, serialize it, turning it into its JSON
         representation.
         """
-        #tic = time.perf_counter()
         if not obj:
             return {}
 
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 80819e88..06b1ecd9 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -362,7 +362,6 @@ def put(self, **kwargs):
             fields = ','.join(raw_data.keys())
             return self._resource.serialize(obj, params={'_fields': fields})
 
-
     def delete_object(self, obj):
         """Delete an object"""
         # Check if we have permission to delete this object

From 11cdc11f419b15025584caefda2c4b6262d5b73d Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 17 Jul 2020 21:20:59 -0700
Subject: [PATCH 64/78] allow_nan and catch list field

---
 flask_mongorest/resources.py | 2 +-
 flask_mongorest/views.py     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 3b29b27d..848f06de 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -480,7 +480,7 @@ def serialize_list_field(self, field_instance, field_name, field_value, **kwargs
         return [val for val in [
             self.get_field_value(elem, field_name, field_instance=field_instance.field, **kwargs)
             for elem in field_value
-        ] if val is not None]
+        ] if val is not None] if field_value else []
 
     def serialize_document_field(self, field_name, field_value, **kwargs):
         """If this field is a reference or an embedded document, either return
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 06b1ecd9..3286a5e8 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -28,7 +28,7 @@
 register_mime('gz', ('application/gzip',))
 
 def render_json(**payload):
-    return json.dumps(payload, allow_nan=False, cls=MongoEncoder)
+    return json.dumps(payload, allow_nan=True, cls=MongoEncoder)
 
 def render_html(**payload):
     d = json.dumps(payload, cls=MongoEncoder, sort_keys=True, indent=4)
@@ -41,7 +41,7 @@ def render_gz(**payload):
         fmt = request.args.get('format')
         content_type = 'text/csv' if fmt == 'csv' else 'application/json'
         if fmt == 'json':
-            contents = json.dumps(payload['data'], allow_nan=False, cls=MongoEncoder)
+            contents = json.dumps(payload['data'], allow_nan=True, cls=MongoEncoder)
         elif fmt == 'csv':
             from pandas import DataFrame
             from cherrypicker import CherryPicker

From 23121291569a7fda0e1b4cb02c347431ca69977b Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Sun, 19 Jul 2020 15:14:38 -0700
Subject: [PATCH 65/78] add unflatten dep

---
 requirements.txt | 1 +
 setup.py         | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 8eeacb1f..7f52aae1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,3 +4,4 @@ mimerender @ git+https://github.com/tschaume/mimerender@mpcontribs#egg=mimerende
 python-dateutil==2.8.1
 Flask==1.1.2
 pymongo==3.10.1
+unflatten==0.1
diff --git a/setup.py b/setup.py
index 881165a3..6469c99a 100644
--- a/setup.py
+++ b/setup.py
@@ -31,7 +31,8 @@
         'Flask-MongoEngine',
         'mimerender @ git+https://github.com/tschaume/mimerender@mpcontribs#egg=mimerender',
         'nose',
-        'python-dateutil'
+        'python-dateutil',
+        'unflatten'
     ],
     classifiers=[
         'Development Status :: 4 - Beta',

From dc3c7a5e291dfcbebea005b00fea41daada5695d Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Tue, 28 Jul 2020 00:55:58 -0700
Subject: [PATCH 66/78] serialize all fields of related resources for download

---
 flask_mongorest/resources.py | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 848f06de..bf8302bf 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -285,7 +285,7 @@ def get_requested_fields(self, **kwargs):
 
         requested_fields = []
         if include_all or only_fields is None:
-            if include_all:
+            if include_all or self.view_method == methods.Download:
                 field_selection = all_fields_set
             else:
                 field_selection = fields
@@ -436,6 +436,7 @@ def serialize_field_value(self, obj, field_name, field_instance, field_value, **
 
         elif callable(field_instance):
             return self.serialize_callable_field(obj, field_instance, field_name, field_value, **kwargs)
+
         return field_value
 
     def serialize_callable_field(self, obj, field_instance, field_name, field_value, **kwargs):
@@ -452,12 +453,13 @@ def serialize_callable_field(self, obj, field_instance, field_name, field_value,
             else:
                 value = field_instance(obj)
         if field_name in self._related_resources:
+            res = self._related_resources[field_name](view_method=self.view_method)
             if isinstance(value, list):
-                return [self._related_resources[field_name]().serialize_field(o, **kwargs) for o in value]
+                return [res.serialize_field(o, **kwargs) for o in value]
             elif value is None:
                 return None
             else:
-                return self._related_resources[field_name]().serialize_field(value, **kwargs)
+                return res.serialize_field(value, **kwargs)
         return value
 
     def serialize_dict_field(self, field_instance, field_name, field_value, **kwargs):
@@ -477,21 +479,27 @@ def serialize_dict_field(self, field_instance, field_name, field_value, **kwargs
 
     def serialize_list_field(self, field_instance, field_name, field_value, **kwargs):
         """Serialize each item in the list separately."""
-        return [val for val in [
-            self.get_field_value(elem, field_name, field_instance=field_instance.field, **kwargs)
-            for elem in field_value
-        ] if val is not None] if field_value else []
+        if not field_value:
+            return []
+
+        field_values = []
+        for elem in field_value:
+            fv = self.get_field_value(
+                elem, field_name, field_instance=field_instance.field, **kwargs
+            )
+            if fv is not None:
+                field_values.append(fv)
+
+        return field_values
 
     def serialize_document_field(self, field_name, field_value, **kwargs):
         """If this field is a reference or an embedded document, either return
         a DBRef or serialize it using a resource found in `related_resources`.
         """
         if field_name in self._related_resources:
-            return (
-                field_value and
-                not isinstance(field_value, DBRef) and
-                self._related_resources[field_name]().serialize_field(field_value, **kwargs)
-            )
+            if field_value and not isinstance(field_value, DBRef):
+                res = self._related_resources[field_name](view_method=self.view_method)
+            return res.serialize_field(field_value, **kwargs)
         else:
             if DocumentProxy and isinstance(field_value, DocumentProxy):
                 # Don't perform a DBRef isinstance check below since
@@ -540,7 +548,9 @@ def serialize(self, obj, **kwargs):
                 # if the field is associated with a specific resource (via the
                 # `related_resources` map), use that resource to serialize it
                 if field in self._related_resources and value is not None:
-                    related_resource = self._related_resources[field]()
+                    related_resource = self._related_resources[field](
+                        view_method=self.view_method
+                    )
                     if isinstance(value, mongoengine.document.Document):
                         value = related_resource.serialize_field(value)
                     elif isinstance(value, dict):

From 699cdeedbf2ddd69f9fbb12ab03ced54a83c2e12 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Tue, 28 Jul 2020 16:19:02 -0700
Subject: [PATCH 67/78] identation bugfix

---
 flask_mongorest/resources.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index bf8302bf..18a3fbaa 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -499,7 +499,7 @@ def serialize_document_field(self, field_name, field_value, **kwargs):
         if field_name in self._related_resources:
             if field_value and not isinstance(field_value, DBRef):
                 res = self._related_resources[field_name](view_method=self.view_method)
-            return res.serialize_field(field_value, **kwargs)
+                return res.serialize_field(field_value, **kwargs)
         else:
             if DocumentProxy and isinstance(field_value, DocumentProxy):
                 # Don't perform a DBRef isinstance check below since

From 84b3043b08c9d1973e7c7c71c0f096395d696971 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Tue, 28 Jul 2020 16:19:17 -0700
Subject: [PATCH 68/78] enable skip post_delete signal

---
 flask_mongorest/resources.py |  4 ++--
 flask_mongorest/views.py     | 12 ++++++++----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 18a3fbaa..aa86ee19 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -1101,8 +1101,8 @@ def update_object(self, obj, data=None, save=True, parent_resources=None):
             self.save_object(obj)
         return obj
 
-    def delete_object(self, obj, parent_resources=None):
-        obj.delete()
+    def delete_object(self, obj, parent_resources=None, skip_post_delete=False):
+        obj.delete(signal_kwargs={"skip": skip_post_delete})
 
 
 # Py2/3 compatible way to do metaclasses (or six.add_metaclass)
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 3286a5e8..de215da6 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -362,14 +362,14 @@ def put(self, **kwargs):
             fields = ','.join(raw_data.keys())
             return self._resource.serialize(obj, params={'_fields': fields})
 
-    def delete_object(self, obj):
+    def delete_object(self, obj, skip_post_delete=False):
         """Delete an object"""
         # Check if we have permission to delete this object
         if not self.has_delete_permission(request, obj):
             raise Unauthorized
 
         try:
-            self._resource.delete_object(obj)
+            self._resource.delete_object(obj, skip_post_delete=skip_post_delete)
         except Exception as e:
             self.handle_validation_error(e)
 
@@ -377,9 +377,13 @@ def delete_objects(self, objs):
         """Delete each object in the list one by one, and return the total count."""
         count = 0
         try:
-            for obj in objs:
-                self.delete_object(obj)
+            # separately delete last object to send skip signal
+            for obj in objs[:-1]:
+                self.delete_object(obj, skip_post_delete=True)
                 count += 1
+
+            self.delete_object(objs[-1])
+            count += 1
         except ValidationError as e:
             e.args[0]['count'] = count
             raise e

From af87c367e679fd2456fd8746ab641035b786b480 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 1 Aug 2020 15:02:48 +0000
Subject: [PATCH 69/78] Bump pymongo from 3.10.1 to 3.11.0

Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 3.10.1 to 3.11.0.
- [Release notes](https://github.com/mongodb/mongo-python-driver/releases)
- [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst)
- [Commits](https://github.com/mongodb/mongo-python-driver/compare/3.10.1...3.11.0)

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 7f52aae1..7da2f680 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,5 +3,5 @@ flask-mongoengine==0.9.5
 mimerender @ git+https://github.com/tschaume/mimerender@mpcontribs#egg=mimerender-0.6.1
 python-dateutil==2.8.1
 Flask==1.1.2
-pymongo==3.10.1
+pymongo==3.11.0
 unflatten==0.1

From 78ddb5ae479ef335193f19fa5f979262d29d0975 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 17 Sep 2020 12:21:48 -0700
Subject: [PATCH 70/78] clean up exception responses

---
 flask_mongorest/views.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index de215da6..5aa94d00 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -129,17 +129,11 @@ def _dispatch_request(self, *args, **kwargs):
         try:
             self._resource = self.requested_resource(request)
             return super(ResourceView, self).dispatch_request(*args, **kwargs)
-        except mongoengine.queryset.DoesNotExist as e:
-            return {'error': 'Empty query: ' + str(e)}, '404 Not Found'
-        except mongoengine.errors.NotUniqueError as e:
-            return {'error': str(e)}, '401 Unauthorized'
-        except ValidationError as e:
-            return e.args[0], '400 Bad Request'
-        except (ValueError, mongoengine.errors.ValidationError) as e:
+        except (ValueError, ValidationError, mongoengine.errors.ValidationError) as e:
             return {'error': str(e)}, '400 Bad Request'
-        except Unauthorized as e:
+        except (Unauthorized, mongoengine.errors.NotUniqueError) as e:
             return {'error': str(e)}, '401 Unauthorized'
-        except NotFound as e:
+        except (NotFound, mongoengine.queryset.DoesNotExist) as e:
             return {'error': str(e)}, '404 Not Found'
 
     def handle_validation_error(self, e):

From 2267d880d72658e9ebaa8e13ba2a421dfd0d329c Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Fri, 18 Sep 2020 15:34:42 -0700
Subject: [PATCH 71/78] apply qfilter after params

---
 flask_mongorest/resources.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index aa86ee19..5759372e 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -957,15 +957,17 @@ def get_objects(self, qs=None, qfilter=None):
             custom_qs = False
             qs = self.get_queryset()
 
+        # Apply filters and ordering, based on the params supplied by the
+        # request
+        qs = self.apply_filters(qs, params)
+        qs = self.apply_ordering(qs, params)
+
         # If a queryset filter was provided, pass our current queryset in and
         # get a new one out
         if qfilter:
             qs = qfilter(qs)
 
-        # Apply filters and ordering, based on the params supplied by the
-        # request
-        qs = self.apply_filters(qs, params)
-        qs = self.apply_ordering(qs, params)
+        # set total count
         extra['total_count'] = qs.count()
 
         # Apply pagination to the queryset (if not Download and no custom queryset provided)

From 5482c3f851e38b1b01702a0bf7d4b7762ba0399a Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Mon, 21 Sep 2020 15:46:18 -0700
Subject: [PATCH 72/78] catch undefined exceptions, print traceback

---
 flask_mongorest/views.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 5aa94d00..fd79081b 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -1,8 +1,10 @@
 import os
+import sys
 import time
 import json
 import boto3
 import hashlib
+import traceback
 import mongoengine
 from gzip import GzipFile
 from io import BytesIO
@@ -135,6 +137,12 @@ def _dispatch_request(self, *args, **kwargs):
             return {'error': str(e)}, '401 Unauthorized'
         except (NotFound, mongoengine.queryset.DoesNotExist) as e:
             return {'error': str(e)}, '404 Not Found'
+        except Exception as e:
+            exc_type, exc_value, exc_tb = sys.exc_info()
+            tb = traceback.format_exception(exc_type, exc_value, exc_tb)
+            err = ''.join(tb)
+            print(err)
+            return {'error': err}, '500 Internal Server Error'
 
     def handle_validation_error(self, e):
         if isinstance(e, ValidationError):

From 666d40f1f22d8040f2d5b13af2c2c0a020653150 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 30 Sep 2020 16:40:10 -0700
Subject: [PATCH 73/78] avoid ServerTimeouts on BulkCreate/Delete

---
 flask_mongorest/views.py | 42 ++++++++++++++++++++++++++++++----------
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index fd79081b..1ebf44e9 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -271,11 +271,24 @@ def post(self, **kwargs):
                 })
             raw_data_deque = deque(raw_data)
             self._resource.view_method = methods.BulkCreate
-            ret = []
+            data = []
+            tic = time.perf_counter()
             while len(raw_data_deque):
                 self._resource._raw_data = raw_data_deque.popleft()
-                ret.append(self.create_object())
-            return {'data': ret, 'count': len(ret)}, '201 Created'
+                data.append(self.create_object())
+                dt = time.perf_counter() - tic
+                if dt > 50:
+                    break
+
+            count = len(data)
+            msg = f"Created {count} objects in {dt:0.1f}s ({count/dt:0.3f}/s)."
+            print(msg)
+            ret = {'data': data, 'count': count}
+            if raw_data_deque:
+                remain = len(raw_data_deque)
+                msg += f" Remaining {remain} objects skipped to avoid Server Timeout."
+                ret['warning'] = msg
+            return ret, '201 Created'
         else:
             raise ValidationError({'error': 'wrong payload type'})
 
@@ -377,20 +390,29 @@ def delete_object(self, obj, skip_post_delete=False):
 
     def delete_objects(self, objs):
         """Delete each object in the list one by one, and return the total count."""
-        count = 0
+        tic = time.perf_counter()
+        nobjs, count = len(objs), 0
         try:
             # separately delete last object to send skip signal
-            for obj in objs[:-1]:
-                self.delete_object(obj, skip_post_delete=True)
+            for iobj, obj in enumerate(objs):
+                skip = iobj < nobjs - 1
+                self.delete_object(obj, skip_post_delete=skip)
                 count += 1
-
-            self.delete_object(objs[-1])
-            count += 1
+                dt = time.perf_counter() - tic
+                if dt > 50:
+                    break
         except ValidationError as e:
             e.args[0]['count'] = count
             raise e
         else:
-            return {'count': count}
+            msg = f"Deleted {count} objects in {dt:0.1f}s ({count/dt:0.3f}/s)."
+            print(msg)
+            ret = {'count': count}
+            remain = nobjs - count
+            if remain:
+                msg += f" Remaining {remain} objects skipped to avoid Server Timeout."
+                ret['warning'] = msg
+            return ret
 
     def delete(self, **kwargs):
         pk = kwargs.pop('pk', None)

From 233df7d1113fae04a40261bde556a3a13eb6fbaa Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Thu, 29 Oct 2020 14:50:21 -0700
Subject: [PATCH 74/78] catch None field_value

---
 flask_mongorest/resources.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 5759372e..2178eeff 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -423,7 +423,7 @@ def serialize_field_value(self, obj, field_name, field_instance, field_value, **
         For other fields, see get_field_value method.
         """
         if isinstance(field_instance, (LazyReferenceField, GenericLazyReferenceField)):
-            return field_value.pk
+            return field_value and field_value.pk
 
         if isinstance(field_instance, (ReferenceField, GenericReferenceField, EmbeddedDocumentField)):
             return self.serialize_document_field(field_name, field_value, **kwargs)

From 8e80a458b4b888bc1e3bf09234566986711b4de0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 1 Dec 2020 16:00:29 +0000
Subject: [PATCH 75/78] dependabot PRs

Bump flask-mongoengine from 0.9.5 to 1.0.0

Bumps [flask-mongoengine](https://github.com/mongoengine/flask-mongoengine) from 0.9.5 to 1.0.0.
- [Release notes](https://github.com/mongoengine/flask-mongoengine/releases)
- [Changelog](https://github.com/MongoEngine/flask-mongoengine/blob/master/docs/changelog.rst)
- [Commits](https://github.com/mongoengine/flask-mongoengine/compare/v0.9.5...v1.0.0)

Signed-off-by: dependabot[bot] <support@github.com>

Bump mongoengine from 0.20.0 to 0.21.0

Bumps [mongoengine](https://github.com/MongoEngine/mongoengine) from 0.20.0 to 0.21.0.
- [Release notes](https://github.com/MongoEngine/mongoengine/releases)
- [Changelog](https://github.com/MongoEngine/mongoengine/blob/master/docs/changelog.rst)
- [Commits](https://github.com/MongoEngine/mongoengine/compare/v0.20.0...v0.21.0)

Signed-off-by: dependabot[bot] <support@github.com>

Bump pymongo from 3.11.0 to 3.11.1

Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 3.11.0 to 3.11.1.
- [Release notes](https://github.com/mongodb/mongo-python-driver/releases)
- [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst)
- [Commits](https://github.com/mongodb/mongo-python-driver/compare/3.11.0...3.11.1)

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 7da2f680..ec743c81 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
-mongoengine==0.20.0
-flask-mongoengine==0.9.5
+mongoengine==0.21.0
+flask-mongoengine==1.0.0
 mimerender @ git+https://github.com/tschaume/mimerender@mpcontribs#egg=mimerender-0.6.1
 python-dateutil==2.8.1
 Flask==1.1.2
-pymongo==3.11.0
+pymongo==3.11.1
 unflatten==0.1

From ce277803c6d1eb8696a8e20b546294798efe1bed Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 23 Dec 2020 14:18:36 -0800
Subject: [PATCH 76/78] use fastnumbers for index access

---
 flask_mongorest/resources.py | 3 ++-
 requirements.txt             | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 2178eeff..22bf94bf 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -1,6 +1,7 @@
 import json
 import mongoengine
 
+from fastnumbers import fast_int
 from unflatten import unflatten
 from typing import Pattern
 from bson.dbref import DBRef
@@ -46,7 +47,7 @@
 
 def get_with_list_index(o, k):
     try:
-        return o[int(k)]
+        return o[fast_int(k)]
     except ValueError:
         return o[k]
 
diff --git a/requirements.txt b/requirements.txt
index ec743c81..6e0330c9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,3 +5,4 @@ python-dateutil==2.8.1
 Flask==1.1.2
 pymongo==3.11.1
 unflatten==0.1
+fastnumbers==3.0.0

From 1cfa3eab6ab96d7423b9f5a93760adddcc6b9dfa Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 23 Dec 2020 14:19:10 -0800
Subject: [PATCH 77/78] remove dangling import; comment reformat

---
 flask_mongorest/resources.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index 22bf94bf..a6c7f7fe 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -7,7 +7,6 @@
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
 from flask import has_request_context, request, url_for
-from dict_deep import deep_get
 try:
     from urllib.parse import urlparse
 except ImportError: # Python 2
@@ -958,8 +957,7 @@ def get_objects(self, qs=None, qfilter=None):
             custom_qs = False
             qs = self.get_queryset()
 
-        # Apply filters and ordering, based on the params supplied by the
-        # request
+        # Apply filters and ordering, based on the params supplied by the request
         qs = self.apply_filters(qs, params)
         qs = self.apply_ordering(qs, params)
 

From 4643e57f0877e59e07f033bb8376395a1cb5cb81 Mon Sep 17 00:00:00 2001
From: Patrick Huck <phuck@lbl.gov>
Date: Wed, 23 Dec 2020 14:19:28 -0800
Subject: [PATCH 78/78] use bulk_update_limit

---
 flask_mongorest/resources.py | 11 +++++++++--
 flask_mongorest/views.py     |  2 +-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/flask_mongorest/resources.py b/flask_mongorest/resources.py
index a6c7f7fe..0c8db395 100644
--- a/flask_mongorest/resources.py
+++ b/flask_mongorest/resources.py
@@ -89,6 +89,9 @@ class Resource(object):
     # Only relevant if pagination is enabled.
     max_limit = 100
 
+    # Maximum number of objects which can be bulk-updated by a single request
+    bulk_update_limit = 1000  # NOTE also used for bulk delete
+
     # Map of field names to paginate with according default and maximum limits
     fields_to_paginate = {}
 
@@ -971,7 +974,11 @@ def get_objects(self, qs=None, qfilter=None):
 
         # Apply pagination to the queryset (if not Download and no custom queryset provided)
         limit = None
-        if not custom_qs and self.view_method != methods.Download:
+        if self.view_method in [methods.BulkUpdate, methods.BulkDelete]:
+            # limit the number of objects that can be bulk-updated at a time
+            qs = qs.limit(self.bulk_update_limit)
+        elif not custom_qs and self.view_method != methods.Download:
+            # no need to skip/limit if a custom `qs` was provided
             skip, limit = self.get_skip_and_limit(params)
             qs = qs.skip(skip).limit(limit+1)  # get one extra to determine has_more
             qs = self.apply_field_pagination(qs, params)
@@ -984,7 +991,7 @@ def get_objects(self, qs=None, qfilter=None):
         # Evaluate the queryset
         objs = list(qs)
         has_more = None
-        if self.view_method != methods.Download and self.paginate:
+        if self.view_method not in [methods.BulkUpdate, methods.BulkDelete, methods.Download] and self.paginate:
             has_more = bool(len(objs) > limit)
 
         if has_more:
diff --git a/flask_mongorest/views.py b/flask_mongorest/views.py
index 1ebf44e9..b26e28d3 100644
--- a/flask_mongorest/views.py
+++ b/flask_mongorest/views.py
@@ -264,7 +264,7 @@ def post(self, **kwargs):
             self._resource.view_method = methods.Create
             return self.create_object()
         elif isinstance(raw_data, list):
-            limit = self._resource.max_limit
+            limit = self._resource.bulk_update_limit
             if len(raw_data) > limit:
                 raise ValidationError({
                     'errors': [f"Can only create {limit} documents at once"]