diff options
| author | Håvard Flaget Aasen <haavard_aasen@yahoo.no> | 2020-12-24 15:38:08 -0500 |
|---|---|---|
| committer | git-ubuntu importer <ubuntu-devel-discuss@lists.ubuntu.com> | 2020-12-25 04:35:20 +0000 |
| commit | 6c5213d32dd643b56691b7877ea7ef279a70ac77 (patch) | |
| tree | 8df6326cb9c627dde8e1a70a3929aa1084b93dfc | |
| parent | 70abf305fbb19f5880b12cb630f9d5715d33b43d (diff) | |
0.21.0-1 (patches unapplied)import/0.21.0-1ubuntu/impish-develubuntu/impishubuntu/hirsute-proposedubuntu/hirsute-develubuntu/hirsutedebian/bullseye
Imported using git-ubuntu import.
Notes
Notes:
[ Ondřej Nový ]
* d/control: Update Maintainer field with new Debian Python Team
contact address.
* d/control: Update Vcs-* fields with new Debian Python Team Salsa
layout.
[ Håvard Flaget Aasen ]
* New upstream version 0.21.0
* d/watch:
- Update to version 4
- Use the new substitution string from uscan.
* d/control:
- Update Standards-Version to 4.5.1
- Drop version restriction on suggested packages.
* d/rules: Change target to "execute_after"
* autopkgtest: Mark tests as superficial.
30 files changed, 736 insertions, 170 deletions
@@ -257,3 +257,5 @@ that much better: * Matthew Simpson (https://github.com/mcsimps2) * Leonardo Domingues (https://github.com/leodmgs) * Agustin Barto (https://github.com/abarto) + * Stankiewicz Mateusz (https://github.com/mas15) + * Felix Schultheiß (https://github.com/felix-smashdocs) @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: mongoengine -Version: 0.20.0 +Version: 0.21.0 Summary: MongoEngine is a Python Object-Document Mapper for working with MongoDB. Home-page: http://mongoengine.org/ Author: Harry Marr @@ -27,6 +27,9 @@ Description: =========== :target: https://landscape.io/github/MongoEngine/mongoengine/master :alt: Code Health + .. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/ambv/black + About ===== MongoEngine is a Python Object-Document Mapper for working with MongoDB. @@ -16,6 +16,9 @@ MongoEngine :target: https://landscape.io/github/MongoEngine/mongoengine/master :alt: Code Health +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/ambv/black + About ===== MongoEngine is a Python Object-Document Mapper for working with MongoDB. diff --git a/debian/changelog b/debian/changelog index e37262f..fe86e0c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,24 @@ +python-mongoengine (0.21.0-1) unstable; urgency=medium + + [ Ondřej Nový ] + * d/control: Update Maintainer field with new Debian Python Team + contact address. + * d/control: Update Vcs-* fields with new Debian Python Team Salsa + layout. + + [ Håvard Flaget Aasen ] + * New upstream version 0.21.0 + * d/watch: + - Update to version 4 + - Use the new substitution string from uscan. + * d/control: + - Update Standards-Version to 4.5.1 + - Drop version restriction on suggested packages. + * d/rules: Change target to "execute_after" + * autopkgtest: Mark tests as superficial. + + -- Håvard Flaget Aasen <haavard_aasen@yahoo.no> Thu, 24 Dec 2020 15:38:08 -0500 + python-mongoengine (0.20.0-1) unstable; urgency=medium * New upstream version 0.20.0 diff --git a/debian/control b/debian/control index 0df1d6a..9653246 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: python-mongoengine Section: python Priority: optional -Maintainer: Debian Python Modules Team <python-modules-team@lists.alioth.debian.org> +Maintainer: Debian Python Team <team+python@tracker.debian.org> Uploaders: Håvard Flaget Aasen <haavard_aasen@yahoo.no> Build-Depends: debhelper-compat (= 13), @@ -15,11 +15,11 @@ Build-Depends: python3-setuptools, python3-sphinx, python3-sphinx-rtd-theme, -Standards-Version: 4.5.0 +Standards-Version: 4.5.1 Rules-Requires-Root: no Homepage: http://mongoengine.org/ -Vcs-Git: https://salsa.debian.org/python-team/modules/python-mongoengine.git -Vcs-Browser: https://salsa.debian.org/python-team/modules/python-mongoengine +Vcs-Git: https://salsa.debian.org/python-team/packages/python-mongoengine.git +Vcs-Browser: https://salsa.debian.org/python-team/packages/python-mongoengine Package: python3-mongoengine Architecture: all @@ -32,9 +32,9 @@ Provides: ${python3:Provides}, Suggests: python-mongoengine-doc, - python3-blinker (>= 1.3), - python3-dateutil (>= 2.1.0), - python3-pil (>= 2.0.0), + python3-blinker, + python3-dateutil, + python3-pil, Description: Python 3 Document-Object Mapper for working with MongoDB MongoEngine is a Document-Object Mapper (think ORM, but for document databases) for working with MongoDB from Python. It uses a simple declarative diff --git a/debian/rules b/debian/rules index 4053172..37868b3 100755 --- a/debian/rules +++ b/debian/rules @@ -7,6 +7,5 @@ export PYBUILD_DISABLE=test %: dh $@ --with sphinxdoc,python3 --buildsystem=pybuild -override_dh_auto_build: +execute_after_dh_auto_build: PYTHONPATH=$(CURDIR) $(MAKE) -C docs html - dh_auto_build diff --git a/debian/tests/control b/debian/tests/control index 9a173b7..22c3918 100644 --- a/debian/tests/control +++ b/debian/tests/control @@ -1,5 +1,7 @@ Test-Command: cd "$AUTOPKGTEST_TMP"; python3 -c "import mongoengine; print(mongoengine)" Depends: python3-mongoengine +Restrictions: superficial Tests: mongo-test.py Depends: python3-mongoengine +Restrictions: superficial diff --git a/debian/watch b/debian/watch index 013a95c..233d25a 100644 --- a/debian/watch +++ b/debian/watch @@ -1,3 +1,3 @@ -version=3 +version=4 opts=uversionmangle=s/(rc|a|b|c)/~$1/ \ -https://pypi.debian.net/mongoengine/mongoengine-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz))) +https://pypi.debian.net/mongoengine/mongoengine-@ANY_VERSION@@ARCHIVE_EXT@ diff --git a/docs/Makefile b/docs/Makefile index 76eb910..cdc77c3 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -33,7 +33,7 @@ clean: html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + @echo "Build finished. Check $(BUILDDIR)/html/index.html" dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml diff --git a/docs/changelog.rst b/docs/changelog.rst index 2dd5bcd..247bd8e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,23 @@ Development =========== - (Fill this out as you fix issues and develop your features). +Changes in 0.21.0 +================= +- Bug fix in DynamicDocument which is not parsing known fields in constructor like Document do #2412 +- When using pymongo >= 3.7, make use of Collection.count_documents instead of Collection.count + and Cursor.count that got deprecated in pymongo >= 3.7. + This should have a negative impact on performance of count see Issue #2219 +- Fix a bug that made the queryset drop the read_preference after clone(). +- Remove Py3.5 from CI as it reached EOL and add Python 3.9 +- Fix some issues related with db_field/field conflict in constructor #2414 +- BREAKING CHANGE: Fix the behavior of Doc.objects.limit(0) which should return all documents (similar to mongodb) #2311 +- Bug fix in ListField when updating the first item, it was saving the whole list, instead of + just replacing the first item (as usually done when updating 1 item of the list) #2392 +- Add EnumField: ``mongoengine.fields.EnumField`` +- Refactoring - Remove useless code related to Document.__only_fields and Queryset.only_fields +- Fix query transformation regarding special operators #2365 +- Bug Fix: Document.save() fails when shard_key is not _id #2154 + Changes in 0.20.0 ================= - ATTENTION: Drop support for Python2 @@ -28,7 +45,7 @@ Changes in 0.20.0 Changes in 0.19.1 ================= -- Requires Pillow < 7.0.0 as it dropped Python2 support +- Tests require Pillow < 7.0.0 as it dropped Python2 support - DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079 @@ -456,9 +473,6 @@ Changes in 0.8.3 - Document.select_related() now respects ``db_alias`` (#377) - Reload uses shard_key if applicable (#384) - Dynamic fields are ordered based on creation and stored in _fields_ordered (#396) - - **Potential breaking change:** http://docs.mongoengine.org/en/latest/upgrade.html#to-0-8-3 - - Fixed pickling dynamic documents ``_dynamic_fields`` (#387) - Fixed ListField setslice and delslice dirty tracking (#390) - Added Django 1.5 PY3 support (#392) diff --git a/docs/conf.py b/docs/conf.py index 48c8e85..fdb5b61 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # MongoEngine documentation build configuration file, created by # sphinx-quickstart on Sun Nov 22 18:14:13 2009. diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index ac2146a..03d6ed5 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -31,6 +31,8 @@ the :attr:`host` to connect('project1', host='mongodb://localhost/database_name') +.. note:: URI containing SRV records (e.g mongodb+srv://server.example.com/) can be used as well as the :attr:`host` + .. note:: Database, username and password from URI string overrides corresponding parameters in :func:`~mongoengine.connect`: :: diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 6dc35c3..7fc20ba 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -76,6 +76,7 @@ are as follows: * :class:`~mongoengine.fields.EmailField` * :class:`~mongoengine.fields.EmbeddedDocumentField` * :class:`~mongoengine.fields.EmbeddedDocumentListField` +* :class:`~mongoengine.fields.EnumField` * :class:`~mongoengine.fields.FileField` * :class:`~mongoengine.fields.FloatField` * :class:`~mongoengine.fields.GenericEmbeddedDocumentField` @@ -426,19 +427,6 @@ either a single field name, or a list or tuple of field names:: first_name = StringField() last_name = StringField(unique_with='first_name') -Skipping Document validation on save ------------------------------------- -You can also skip the whole document validation process by setting -``validate=False`` when calling the :meth:`~mongoengine.document.Document.save` -method:: - - class Recipient(Document): - name = StringField() - email = EmailField() - - recipient = Recipient(name='admin', email='root@localhost') - recipient.save() # will raise a ValidationError while - recipient.save(validate=False) # won't Document collections ==================== diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index 64f17c0..5dacc0d 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -41,35 +41,6 @@ already exist, then any changes will be updated atomically. For example:: .. seealso:: :ref:`guide-atomic-updates` -Pre save data validation and cleaning -------------------------------------- -MongoEngine allows you to create custom cleaning rules for your documents when -calling :meth:`~mongoengine.Document.save`. By providing a custom -:meth:`~mongoengine.Document.clean` method you can do any pre validation / data -cleaning. - -This might be useful if you want to ensure a default value based on other -document values for example:: - - class Essay(Document): - status = StringField(choices=('Published', 'Draft'), required=True) - pub_date = DateTimeField() - - def clean(self): - """Ensures that only published essays have a `pub_date` and - automatically sets `pub_date` if essay is published and `pub_date` - is not set""" - if self.status == 'Draft' and self.pub_date is not None: - msg = 'Draft entries should not have a publication date.' - raise ValidationError(msg) - # Set the pub_date for published items if not set. - if self.status == 'Published' and self.pub_date is None: - self.pub_date = datetime.now() - -.. note:: - Cleaning is only called if validation is turned on and when calling - :meth:`~mongoengine.Document.save`. - Cascading Saves --------------- If your document contains :class:`~mongoengine.fields.ReferenceField` or diff --git a/docs/guide/index.rst b/docs/guide/index.rst index a0364ec..018b253 100644 --- a/docs/guide/index.rst +++ b/docs/guide/index.rst @@ -10,8 +10,10 @@ User Guide defining-documents document-instances querying + validation gridfs signals text-indexes + migration logging-monitoring mongomock diff --git a/docs/guide/migration.rst b/docs/guide/migration.rst new file mode 100644 index 0000000..2b8982e --- /dev/null +++ b/docs/guide/migration.rst @@ -0,0 +1,267 @@ +=================== +Documents migration +=================== + +The structure of your documents and their associated mongoengine schemas are likely +to change over the lifetime of an application. This section provides guidance and +recommendations on how to deal with migrations. + +Due to the very flexible nature of mongodb, migrations of models aren't trivial and +for people that know about `alembic` for `sqlalchemy`, there is unfortunately no equivalent +library that will manage the migration in an automatic fashion for mongoengine. + +Example 1: Addition of a field +============================== + +Let's start by taking a simple example of a model change and review the different option you +have to deal with the migration. + +Let's assume we start with the following schema and save an instance: + +.. code-block:: python + + class User(Document): + name = StringField() + + User(name="John Doe").save() + + # print the objects as they exist in mongodb + print(User.objects().as_pymongo()) # [{u'_id': ObjectId('5d06b9c3d7c1f18db3e7c874'), u'name': u'John Doe'}] + +On the next version of your application, let's now assume that a new field `enabled` gets added to the +existing ``User`` model with a `default=True`. Thus you simply update the ``User`` class to the following: + +.. code-block:: python + + class User(Document): + name = StringField(required=True) + enabled = BooleaField(default=True) + +Without applying any migration, we now reload an object from the database into the ``User`` class +and checks its `enabled` attribute: + +.. code-block:: python + + assert User.objects.count() == 1 + user = User.objects().first() + assert user.enabled is True + assert User.objects(enabled=True).count() == 0 # uh? + assert User.objects(enabled=False).count() == 0 # uh? + + # this is consistent with what we have in the database + # in fact, 'enabled' does not exist + print(User.objects().as_pymongo().first()) # {u'_id': ObjectId('5d06b9c3d7c1f18db3e7c874'), u'name': u'John'} + assert User.objects(enabled=None).count() == 1 + +As you can see, even if the document wasn't updated, mongoengine applies the default value seamlessly when it +loads the pymongo dict into a ``User`` instance. At first sight it looks like you don't need to migrate the +existing documents when adding new fields but this actually leads to inconsistencies when it comes to querying. + +In fact, when querying, mongoengine isn't trying to account for the default value of the new field and so +if you don't actually migrate the existing documents, you are taking a risk that querying/updating +will be missing relevant record. + +When adding fields/modifying default values, you can use any of the following to do the migration +as a standalone script: + +.. code-block:: python + + # Use mongoengine to set a default value for a given field + User.objects().update(enabled=True) + # or use pymongo + user_coll = User._get_collection() + user_coll.update_many({}, {'$set': {'enabled': True}}) + + +Example 2: Inheritance change +============================= + +Let's consider the following example: + +.. code-block:: python + + class Human(Document): + name = StringField() + meta = {"allow_inheritance": True} + + class Jedi(Human): + dark_side = BooleanField() + light_saber_color = StringField() + + Jedi(name="Darth Vader", dark_side=True, light_saber_color="red").save() + Jedi(name="Obi Wan Kenobi", dark_side=False, light_saber_color="blue").save() + + assert Human.objects.count() == 2 + assert Jedi.objects.count() == 2 + + # Let's check how these documents got stored in mongodb + print(Jedi.objects.as_pymongo()) + # [ + # {'_id': ObjectId('5fac4aaaf61d7fb06046e0f9'), '_cls': 'Human.Jedi', 'name': 'Darth Vader', 'dark_side': True, 'light_saber_color': 'red'}, + # {'_id': ObjectId('5fac4ac4f61d7fb06046e0fa'), '_cls': 'Human.Jedi', 'name': 'Obi Wan Kenobi', 'dark_side': False, 'light_saber_color': 'blue'} + # ] + +As you can observe, when you use inheritance, MongoEngine stores a field named '_cls' behind the scene to keep +track of the Document class. + +Let's now take the scenario that you want to refactor the inheritance schema and: +- Have the Jedi's with dark_side=True/False become GoodJedi's/DarkSith +- get rid of the 'dark_side' field + +move to the following schemas: + +.. code-block:: python + + # unchanged + class Human(Document): + name = StringField() + meta = {"allow_inheritance": True} + + # attribute 'dark_side' removed + class GoodJedi(Human): + light_saber_color = StringField() + + # new class + class BadSith(Human): + light_saber_color = StringField() + +MongoEngine doesn't know about the change or how to map them with the existing data +so if you don't apply any migration, you will observe a strange behavior, as if the collection was suddenly +empty. + +.. code-block:: python + + # As a reminder, the documents that we inserted + # have the _cls field = 'Human.Jedi' + + # Following has no match + # because the query that is used behind the scene is + # filtering on {'_cls': 'Human.GoodJedi'} + assert GoodJedi.objects().count() == 0 + + # Following has also no match + # because it is filtering on {'_cls': {'$in': ('Human', 'Human.GoodJedi', 'Human.BadSith')}} + # which has no match + assert Human.objects.count() == 0 + assert Human.objects.first() is None + + # If we bypass MongoEngine and make use of underlying driver (PyMongo) + # we can see that the documents are there + humans_coll = Human._get_collection() + assert humans_coll.count_documents({}) == 2 + # print first document + print(humans_coll.find_one()) + # {'_id': ObjectId('5fac4aaaf61d7fb06046e0f9'), '_cls': 'Human.Jedi', 'name': 'Darth Vader', 'dark_side': True, 'light_saber_color': 'red'} + +As you can see, first obvious problem is that we need to modify '_cls' values based on existing values of +'dark_side' documents. + +.. code-block:: python + + humans_coll = Human._get_collection() + old_class = 'Human.Jedi' + good_jedi_class = 'Human.GoodJedi' + bad_sith_class = 'Human.BadSith' + humans_coll.update_many({'_cls': old_class, 'dark_side': False}, {'$set': {'_cls': good_jedi_class}}) + humans_coll.update_many({'_cls': old_class, 'dark_side': True}, {'$set': {'_cls': bad_sith_class}}) + +Let's now check if querying improved in MongoEngine: + +.. code-block:: python + + assert GoodJedi.objects().count() == 1 # Hoorah! + assert BadSith.objects().count() == 1 # Hoorah! + assert Human.objects.count() == 2 # Hoorah! + + # let's now check that documents load correctly + jedi = GoodJedi.objects().first() + # raises FieldDoesNotExist: The fields "{'dark_side'}" do not exist on the document "Human.GoodJedi" + +In fact we only took care of renaming the _cls values but we havn't removed the 'dark_side' fields +which does not exist anymore on the GoodJedi's and BadSith's models. +Let's remove the field from the collections: + +.. code-block:: python + + humans_coll = Human._get_collection() + humans_coll.update_many({}, {'$unset': {'dark_side': 1}}) + +.. note:: We did this migration in 2 different steps for the sake of example but it could have been combined + with the migration of the _cls fields: :: + + humans_coll.update_many( + {'_cls': old_class, 'dark_side': False}, + { + '$set': {'_cls': good_jedi_class}, + '$unset': {'dark_side': 1} + } + ) + + +And verify that the documents now load correctly: + +.. code-block:: python + + jedi = GoodJedi.objects().first() + assert jedi.name == "Obi Wan Kenobi" + + sith = BadSith.objects().first() + assert sith.name == "Darth Vader" + + +An other way of dealing with this migration is to iterate over +the documents and update/replace them one by one. This is way slower but +it is often useful for complex migrations of Document models. + +.. code-block:: python + + for doc in humans_coll.find(): + if doc['_cls'] == 'Human.Jedi': + doc['_cls'] = 'Human.BadSith' if doc['dark_side'] else 'Human.GoodJedi' + doc.pop('dark_side') + humans_coll.replace_one({'_id': doc['_id']}, doc) + +.. warning:: Be aware of this `flaw <https://groups.google.com/g/mongodb-user/c/AFC1ia7MHzk>`_ if you modify documents while iterating + +Recommendations +=============== + +- Write migration scripts whenever you do changes to the model schemas +- Using :class:`~mongoengine.DynamicDocument` or ``meta = {"strict": False}`` may help to avoid some migrations or to have the 2 versions of your application to co-exist. +- Write post-processing checks to verify that migrations script worked. See below + +Post-processing checks +====================== + +The following recipe can be used to sanity check a Document collection after you applied migration. +It does not make any assumption on what was migrated, it will fetch 1000 objects randomly and +run some quick checks on the documents to make sure the document looks OK. As it is, it will fail +on the first occurrence of an error but this is something that can be adapted based on your needs. + +.. code-block:: python + + def get_random_oids(collection, sample_size): + pipeline = [{"$project": {'_id': 1}}, {"$sample": {"size": sample_size}}] + return [s['_id'] for s in collection.aggregate(pipeline)] + + def get_random_documents(DocCls, sample_size): + doc_collection = DocCls._get_collection() + random_oids = get_random_oids(doc_collection, sample_size) + return DocCls.objects(id__in=random_oids) + + def check_documents(DocCls, sample_size): + for doc in get_random_documents(DocCls, sample_size): + # general validation (types and values) + doc.validate() + + # load all subfields, + # this may trigger additional queries if you have ReferenceFields + # so it may be slow + for field in doc._fields: + try: + getattr(doc, field) + except Exception: + LOG.warning(f"Could not load field {field} in Document {doc.id}") + raise + + check_documents(Human, sample_size=1000) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 07de037..7307b00 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -609,7 +609,7 @@ to push values with index:: .. note:: Currently only top level lists are handled, future versions of mongodb / pymongo plan to support nested positional operators. See `The $ positional - operator <http://www.mongodb.org/display/DOCS/Updating#Updating-The%24positionaloperator>`_. + operator <https://docs.mongodb.com/manual/tutorial/update-documents/#Updating-The%24positionaloperator>`_. Server-side javascript execution ================================ diff --git a/docs/guide/validation.rst b/docs/guide/validation.rst new file mode 100644 index 0000000..971f6eb --- /dev/null +++ b/docs/guide/validation.rst @@ -0,0 +1,123 @@ +==================== +Document Validation +==================== + +By design, MongoEngine strictly validates the documents right before they are inserted in MongoDB +and makes sure they are consistent with the fields defined in your models. + +MongoEngine makes the assumption that the documents that exists in the DB are compliant with the schema. +This means that Mongoengine will not validate a document when an object is loaded from the DB into an instance +of your model but this operation may fail under some circumstances (e.g. if there is a field in +the document fetched from the database that is not defined in your model). + + +Built-in validation +=================== + +Mongoengine provides different fields that encapsulate the corresponding validation +out of the box. Validation runs when calling `.validate()` or `.save()` + +.. code-block:: python + + from mongoengine import Document, EmailField + + class User(Document): + email = EmailField() + age = IntField(min_value=0, max_value=99) + + user = User(email='invalid@', age=24) + user.validate() # raises ValidationError (Invalid email address: ['email']) + user.save() # raises ValidationError (Invalid email address: ['email']) + + user2 = User(email='john.doe@garbage.com', age=1000) + user2.save() # raises ValidationError (Integer value is too large: ['age']) + +Custom validation +================= + +The following feature can be used to customize the validation: + +* Field `validation` parameter + +.. code-block:: python + + def not_john_doe(name): + if name == 'John Doe': + raise ValidationError("John Doe is not a valid name") + + class Person(Document): + full_name = StringField(validation=not_john_doe) + + Person(full_name='Billy Doe').save() + Person(full_name='John Doe').save() # raises ValidationError (John Doe is not a valid name) + + +* Document `clean` method + +This method is called as part of :meth:`~mongoengine.document.Document.save` and should be used to provide +custom model validation and/or to modify some of the field values prior to validation. +For instance, you could use it to automatically provide a value for a field, or to do validation +that requires access to more than a single field. + +.. code-block:: python + + class Essay(Document): + status = StringField(choices=('Published', 'Draft'), required=True) + pub_date = DateTimeField() + + def clean(self): + # Validate that only published essays have a `pub_date` + if self.status == 'Draft' and self.pub_date is not None: + raise ValidationError('Draft entries should not have a publication date.') + # Set the pub_date for published items if not set. + if self.status == 'Published' and self.pub_date is None: + self.pub_date = datetime.now() + +.. note:: + Cleaning is only called if validation is turned on and when calling + :meth:`~mongoengine.Document.save`. + +* Adding custom Field classes + +We recommend as much as possible to use fields provided by MongoEngine. However, it is also possible +to subclass a Field and encapsulate some validation by overriding the `validate` method + +.. code-block:: python + + class AgeField(IntField): + + def validate(self, value): + super(AgeField, self).validate(value) # let IntField.validate run first + if value == 60: + self.error('60 is not allowed') + + class Person(Document): + age = AgeField(min_value=0, max_value=99) + + Person(age=20).save() # passes + Person(age=1000).save() # raises ValidationError (Integer value is too large: ['age']) + Person(age=60).save() # raises ValidationError (Person:None) (60 is not allowed: ['age']) + + +.. note:: + + When overriding `validate`, use `self.error("your-custom-error")` instead of raising ValidationError explicitly, + it will provide a better context with the error message + +Skipping validation +==================== + +Although discouraged as it allows to violate fields constraints, if for some reason you need to disable +the validation and cleaning of a document when you call :meth:`~mongoengine.document.Document.save`, you can use `.save(validate=False)`. + +.. code-block:: python + + class Person(Document): + age = IntField(max_value=100) + + Person(age=1000).save() # raises ValidationError (Integer value is too large) + + Person(age=1000).save(validate=False) + person = Person.objects.first() + assert person.age == 1000 + diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..abd1262 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +pymongo>=3.11 +Sphinx==3.2.1 +sphinx-rtd-theme==0.5.0 diff --git a/mongoengine.egg-info/PKG-INFO b/mongoengine.egg-info/PKG-INFO index c90c251..3d5ce07 100644 --- a/mongoengine.egg-info/PKG-INFO +++ b/mongoengine.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: mongoengine -Version: 0.20.0 +Version: 0.21.0 Summary: MongoEngine is a Python Object-Document Mapper for working with MongoDB. Home-page: http://mongoengine.org/ Author: Harry Marr @@ -27,6 +27,9 @@ Description: =========== :target: https://landscape.io/github/MongoEngine/mongoengine/master :alt: Code Health + .. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/ambv/black + About ===== MongoEngine is a Python Object-Document Mapper for working with MongoDB. diff --git a/mongoengine.egg-info/SOURCES.txt b/mongoengine.egg-info/SOURCES.txt index 35c381a..0e2d85a 100644 --- a/mongoengine.egg-info/SOURCES.txt +++ b/mongoengine.egg-info/SOURCES.txt @@ -11,6 +11,7 @@ docs/conf.py docs/django.rst docs/faq.rst docs/index.rst +docs/requirements.txt docs/tutorial.rst docs/upgrade.rst docs/code/tumblelog.py @@ -21,10 +22,12 @@ docs/guide/gridfs.rst docs/guide/index.rst docs/guide/installing.rst docs/guide/logging-monitoring.rst +docs/guide/migration.rst docs/guide/mongomock.rst docs/guide/querying.rst docs/guide/signals.rst docs/guide/text-indexes.rst +docs/guide/validation.rst mongoengine/__init__.py mongoengine/common.py mongoengine/connection.py diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index dbd88a6..d0a0f1f 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -28,7 +28,7 @@ __all__ = ( ) -VERSION = (0, 20, 0) +VERSION = (0, 21, 0) def get_version(): diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index d08d493..2a8fde6 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -179,7 +179,7 @@ class BaseList(list): def _mark_as_changed(self, key=None): if hasattr(self._instance, "_mark_as_changed"): - if key: + if key is not None: self._instance._mark_as_changed( "{}.{}".format(self._name, key % len(self)) ) @@ -215,7 +215,7 @@ class EmbeddedDocumentList(BaseList): Filters the list by only including embedded documents with the given keyword arguments. - This method only supports simple comparison (e.g: .filter(name='John Doe')) + This method only supports simple comparison (e.g. .filter(name='John Doe')) and does not support operators like __gte, __lte, __icontains like queryset.filter does :param kwargs: The keyword arguments corresponding to the fields to diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index e697fe4..019d62d 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -64,8 +64,6 @@ class BaseDocument: It may contain additional reserved keywords, e.g. "__auto_convert". :param __auto_convert: If True, supplied values will be converted to Python-type values via each field's `to_python` method. - :param __only_fields: A set of fields that have been loaded for - this document. Empty if all fields have been loaded. :param _created: Indicates whether this is a brand new document or whether it's already been persisted before. Defaults to true. """ @@ -80,8 +78,6 @@ class BaseDocument: __auto_convert = values.pop("__auto_convert", True) - __only_fields = set(values.pop("__only_fields", values)) - _created = values.pop("_created", True) signals.pre_init.send(self.__class__, document=self, values=values) @@ -105,37 +101,32 @@ class BaseDocument: self._dynamic_fields = SON() - # Assign default values to the instance. - # We set default values only for fields loaded from DB. See - # https://github.com/mongoengine/mongoengine/issues/399 for more info. - for key, field in self._fields.items(): - if self._db_field_map.get(key, key) in __only_fields: + # Assign default values for fields + # not set in the constructor + for field_name in self._fields: + if field_name in values: continue - value = getattr(self, key, None) - setattr(self, key, value) + value = getattr(self, field_name, None) + setattr(self, field_name, value) if "_cls" not in values: self._cls = self._class_name - # Set passed values after initialisation - if self._dynamic: - dynamic_data = {} - for key, value in values.items(): - if key in self._fields or key == "_id": - setattr(self, key, value) - else: + # Set actual values + dynamic_data = {} + FileField = _import_class("FileField") + for key, value in values.items(): + field = self._fields.get(key) + if field or key in ("id", "pk", "_cls"): + if __auto_convert and value is not None: + if field and not isinstance(field, FileField): + value = field.to_python(value) + setattr(self, key, value) + else: + if self._dynamic: dynamic_data[key] = value - else: - FileField = _import_class("FileField") - for key, value in values.items(): - key = self._reverse_db_field_map.get(key, key) - if key in self._fields or key in ("id", "pk", "_cls"): - if __auto_convert and value is not None: - field = self._fields.get(key) - if field and not isinstance(field, FileField): - value = field.to_python(value) - setattr(self, key, value) else: + # For strict Document self._data[key] = value # Set any get_<field>_display methods @@ -314,7 +305,8 @@ class BaseDocument: def clean(self): """ - Hook for doing document level data cleaning before validation is run. + Hook for doing document level data cleaning (usually validation or assignment) + before validation is run. Any ValidationError raised by this method will not be associated with a particular field; it will have a special-case association with the @@ -537,6 +529,9 @@ class BaseDocument: """Using _get_changed_fields iterate and remove any fields that are marked as changed. """ + ReferenceField = _import_class("ReferenceField") + GenericReferenceField = _import_class("GenericReferenceField") + for changed in self._get_changed_fields(): parts = changed.split(".") data = self @@ -549,7 +544,8 @@ class BaseDocument: elif isinstance(data, dict): data = data.get(part, None) else: - data = getattr(data, part, None) + field_name = data._reverse_db_field_map.get(part, part) + data = getattr(data, field_name, None) if not isinstance(data, LazyReference) and hasattr( data, "_changed_fields" @@ -558,10 +554,40 @@ class BaseDocument: continue data._changed_fields = [] + elif isinstance(data, (list, tuple, dict)): + if hasattr(data, "field") and isinstance( + data.field, (ReferenceField, GenericReferenceField) + ): + continue + BaseDocument._nestable_types_clear_changed_fields(data) self._changed_fields = [] - def _nestable_types_changed_fields(self, changed_fields, base_key, data): + @staticmethod + def _nestable_types_clear_changed_fields(data): + """Inspect nested data for changed fields + + :param data: data to inspect for changes + """ + Document = _import_class("Document") + + # Loop list / dict fields as they contain documents + # Determine the iterator to use + if not hasattr(data, "items"): + iterator = enumerate(data) + else: + iterator = data.items() + + for index_or_key, value in iterator: + if hasattr(value, "_get_changed_fields") and not isinstance( + value, Document + ): # don't follow references + value._clear_changed_fields() + elif isinstance(value, (list, tuple, dict)): + BaseDocument._nestable_types_clear_changed_fields(value) + + @staticmethod + def _nestable_types_changed_fields(changed_fields, base_key, data): """Inspect nested data for changed fields :param changed_fields: Previously collected changed fields @@ -586,7 +612,9 @@ class BaseDocument: changed = value._get_changed_fields() changed_fields += ["{}{}".format(item_key, k) for k in changed if k] elif isinstance(value, (list, tuple, dict)): - self._nestable_types_changed_fields(changed_fields, item_key, value) + BaseDocument._nestable_types_changed_fields( + changed_fields, item_key, value + ) def _get_changed_fields(self): """Return a list of all fields that have explicitly been changed. @@ -721,11 +749,9 @@ class BaseDocument: return cls._meta.get("collection", None) @classmethod - def _from_son(cls, son, _auto_dereference=True, only_fields=None, created=False): - """Create an instance of a Document (subclass) from a PyMongo SON.""" - if not only_fields: - only_fields = [] - + def _from_son(cls, son, _auto_dereference=True, created=False): + """Create an instance of a Document (subclass) from a PyMongo SON (dict) + """ if son and not isinstance(son, dict): raise ValueError( "The source SON object needs to be of type 'dict' but a '%s' was found" @@ -738,6 +764,8 @@ class BaseDocument: # Convert SON to a data dict, making sure each key is a string and # corresponds to the right db field. + # This is needed as _from_son is currently called both from BaseDocument.__init__ + # and from EmbeddedDocumentField.to_python data = {} for key, value in son.items(): key = str(key) @@ -780,9 +808,7 @@ class BaseDocument: if cls.STRICT: data = {k: v for k, v in data.items() if k in cls._fields} - obj = cls( - __auto_convert=False, _created=created, __only_fields=only_fields, **data - ) + obj = cls(__auto_convert=False, _created=created, **data) obj._changed_fields = [] if not _auto_dereference: obj._fields = fields diff --git a/mongoengine/document.py b/mongoengine/document.py index db64054..801c8df 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -464,9 +464,9 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): # insert_one will provoke UniqueError alongside save does not # therefore, it need to catch and call replace_one. if "_id" in doc: - raw_object = wc_collection.find_one_and_replace( - {"_id": doc["_id"]}, doc - ) + select_dict = {"_id": doc["_id"]} + select_dict = self._integrate_shard_key(doc, select_dict) + raw_object = wc_collection.find_one_and_replace(select_dict, doc) if raw_object: return doc["_id"] @@ -489,6 +489,23 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): return update_doc + def _integrate_shard_key(self, doc, select_dict): + """Integrates the collection's shard key to the `select_dict`, which will be used for the query. + The value from the shard key is taken from the `doc` and finally the select_dict is returned. + """ + + # Need to add shard key to query, or you get an error + shard_key = self._meta.get("shard_key", tuple()) + for k in shard_key: + path = self._lookup_field(k.split(".")) + actual_key = [p.db_field for p in path] + val = doc + for ak in actual_key: + val = val[ak] + select_dict[".".join(actual_key)] = val + + return select_dict + def _save_update(self, doc, save_condition, write_concern): """Update an existing document. @@ -504,15 +521,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): select_dict["_id"] = object_id - # Need to add shard key to query, or you get an error - shard_key = self._meta.get("shard_key", tuple()) - for k in shard_key: - path = self._lookup_field(k.split(".")) - actual_key = [p.db_field for p in path] - val = doc - for ak in actual_key: - val = val[ak] - select_dict[".".join(actual_key)] = val + select_dict = self._integrate_shard_key(doc, select_dict) update_doc = self._get_update_doc() if update_doc: @@ -639,7 +648,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): write_concern=write_concern, _from_doc_delete=True ) except pymongo.errors.OperationFailure as err: - message = "Could not delete document (%s)" % err.message + message = "Could not delete document (%s)" % err.args raise OperationError(message) signals.post_delete.send(self.__class__, document=self, **signal_kwargs) @@ -919,7 +928,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): @classmethod def list_indexes(cls): - """ Lists all of the indexes that should be created for given + """Lists all of the indexes that should be created for given collection. It includes all the indexes from super- and sub-classes. """ if cls._meta.get("abstract"): @@ -984,7 +993,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): @classmethod def compare_indexes(cls): - """ Compares the indexes defined in MongoEngine with the ones + """Compares the indexes defined in MongoEngine with the ones existing in the database. Returns any missing/extra indexes. """ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index b05e726..8915d80 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -87,6 +87,7 @@ __all__ = ( "PolygonField", "SequenceField", "UUIDField", + "EnumField", "MultiPointField", "MultiLineStringField", "MultiPolygonField", @@ -433,7 +434,7 @@ class DecimalField(BaseField): :param max_value: Validation rule for the maximum acceptable value. :param force_string: Store the value as a string (instead of a float). Be aware that this affects query sorting and operation like lte, gte (as string comparison is applied) - and some query operator won't work (e.g: inc, dec) + and some query operator won't work (e.g. inc, dec) :param precision: Number of decimal places to store. :param rounding: The rounding rule from the python decimal library: @@ -773,6 +774,9 @@ class EmbeddedDocumentField(BaseField): def prepare_query_value(self, op, value): if value is not None and not isinstance(value, self.document_type): + # Short circuit for special operators, returning them as is + if isinstance(value, dict) and all(k.startswith("$") for k in value.keys()): + return value try: value = self.document_type._from_son(value) except ValueError: @@ -844,8 +848,7 @@ class DynamicField(BaseField): Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data""" def to_mongo(self, value, use_db_field=True, fields=None): - """Convert a Python type to a MongoDB compatible type. - """ + """Convert a Python type to a MongoDB compatible type.""" if isinstance(value, str): return value @@ -1619,6 +1622,70 @@ class BinaryField(BaseField): return super().prepare_query_value(op, self.to_mongo(value)) +class EnumField(BaseField): + """Enumeration Field. Values are stored underneath as is, + so it will only work with simple types (str, int, etc) that + are bson encodable + Example usage: + .. code-block:: python + + class Status(Enum): + NEW = 'new' + DONE = 'done' + + class ModelWithEnum(Document): + status = EnumField(Status, default=Status.NEW) + + ModelWithEnum(status='done') + ModelWithEnum(status=Status.DONE) + + Enum fields can be searched using enum or its value: + .. code-block:: python + + ModelWithEnum.objects(status='new').count() + ModelWithEnum.objects(status=Status.NEW).count() + + Note that choices cannot be set explicitly, they are derived + from the provided enum class. + """ + + def __init__(self, enum, **kwargs): + self._enum_cls = enum + if "choices" in kwargs: + raise ValueError( + "'choices' can't be set on EnumField, " + "it is implicitly set as the enum class" + ) + kwargs["choices"] = list(self._enum_cls) + super().__init__(**kwargs) + + def __set__(self, instance, value): + is_legal_value = value is None or isinstance(value, self._enum_cls) + if not is_legal_value: + try: + value = self._enum_cls(value) + except Exception: + pass + return super().__set__(instance, value) + + def to_mongo(self, value): + if isinstance(value, self._enum_cls): + return value.value + return value + + def validate(self, value): + if value and not isinstance(value, self._enum_cls): + try: + self._enum_cls(value) + except Exception as e: + self.error(str(e)) + + def prepare_query_value(self, op, value): + if value is None: + return value + return super().prepare_query_value(op, self.to_mongo(value)) + + class GridFSError(Exception): pass @@ -2042,7 +2109,7 @@ class ImageField(FileField): class SequenceField(BaseField): """Provides a sequential counter see: - http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers + https://docs.mongodb.com/manual/reference/method/ObjectId/#ObjectIDs-SequenceNumbers .. note:: diff --git a/mongoengine/pymongo_support.py b/mongoengine/pymongo_support.py index 80c0661..9cf9e2a 100644 --- a/mongoengine/pymongo_support.py +++ b/mongoengine/pymongo_support.py @@ -2,6 +2,7 @@ Helper functions, constants, and types to aid with PyMongo v2.7 - v3.x support. """ import pymongo +from pymongo.errors import OperationFailure _PYMONGO_37 = (3, 7) @@ -10,13 +11,41 @@ PYMONGO_VERSION = tuple(pymongo.version_tuple[:2]) IS_PYMONGO_GTE_37 = PYMONGO_VERSION >= _PYMONGO_37 -def count_documents(collection, filter): - """Pymongo>3.7 deprecates count in favour of count_documents""" +def count_documents( + collection, filter, skip=None, limit=None, hint=None, collation=None +): + """Pymongo>3.7 deprecates count in favour of count_documents + """ + if limit == 0: + return 0 # Pymongo raises an OperationFailure if called with limit=0 + + kwargs = {} + if skip is not None: + kwargs["skip"] = skip + if limit is not None: + kwargs["limit"] = limit + if hint not in (-1, None): + kwargs["hint"] = hint + if collation is not None: + kwargs["collation"] = collation + + # count_documents appeared in pymongo 3.7 if IS_PYMONGO_GTE_37: - return collection.count_documents(filter) - else: - count = collection.find(filter).count() - return count + try: + return collection.count_documents(filter=filter, **kwargs) + except OperationFailure: + # OperationFailure - accounts for some operators that used to work + # with .count but are no longer working with count_documents (i.e $geoNear, $near, and $nearSphere) + # fallback to deprecated Cursor.count + # Keeping this should be reevaluated the day pymongo removes .count entirely + pass + + cursor = collection.find(filter) + for option, option_value in kwargs.items(): + cursor_method = getattr(cursor, option) + cursor = cursor_method(option_value) + with_limit_and_skip = "skip" in kwargs or "limit" in kwargs + return cursor.count(with_limit_and_skip=with_limit_and_skip) def list_collection_names(db, include_system_collections=False): diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 39c44b2..1021bf5 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -29,6 +29,7 @@ from mongoengine.errors import ( NotUniqueError, OperationError, ) +from mongoengine.pymongo_support import count_documents from mongoengine.queryset import transform from mongoengine.queryset.field_list import QueryFieldList from mongoengine.queryset.visitor import Q, QNode @@ -83,13 +84,20 @@ class BaseQuerySet: self._cursor_obj = None self._limit = None self._skip = None + self._hint = -1 # Using -1 as None is a valid value for hint self._collation = None self._batch_size = None - self.only_fields = [] self._max_time_ms = None self._comment = None + # Hack - As people expect cursor[5:5] to return + # an empty result set. It's hard to do that right, though, because the + # server uses limit(0) to mean 'no limit'. So we set _empty + # in that case and check for it when iterating. We also unset + # it anytime we change _limit. Inspired by how it is done in pymongo.Cursor + self._empty = False + def __call__(self, q_obj=None, **query): """Filter the selected documents by calling the :class:`~mongoengine.queryset.QuerySet` with a query. @@ -162,6 +170,7 @@ class BaseQuerySet: [<User: User object>, <User: User object>] """ queryset = self.clone() + queryset._empty = False # Handle a slice if isinstance(key, slice): @@ -169,6 +178,8 @@ class BaseQuerySet: queryset._skip, queryset._limit = key.start, key.stop if key.start and key.stop: queryset._limit = key.stop - key.start + if queryset._limit == 0: + queryset._empty = True # Allow further QuerySet modifications to be performed return queryset @@ -178,9 +189,7 @@ class BaseQuerySet: if queryset._scalar: return queryset._get_scalar( queryset._document._from_son( - queryset._cursor[key], - _auto_dereference=self._auto_dereference, - only_fields=self.only_fields, + queryset._cursor[key], _auto_dereference=self._auto_dereference, ) ) @@ -188,9 +197,7 @@ class BaseQuerySet: return queryset._cursor[key] return queryset._document._from_son( - queryset._cursor[key], - _auto_dereference=self._auto_dereference, - only_fields=self.only_fields, + queryset._cursor[key], _auto_dereference=self._auto_dereference, ) raise TypeError("Provide a slice or an integer index") @@ -394,9 +401,36 @@ class BaseQuerySet: :meth:`skip` that has been applied to this cursor into account when getting the count """ - if self._limit == 0 and with_limit_and_skip is False or self._none: + # mimic the fact that setting .limit(0) in pymongo sets no limit + # https://docs.mongodb.com/manual/reference/method/cursor.limit/#zero-value + if ( + self._limit == 0 + and with_limit_and_skip is False + or self._none + or self._empty + ): return 0 - count = self._cursor.count(with_limit_and_skip=with_limit_and_skip) + + kwargs = ( + {"limit": self._limit, "skip": self._skip} if with_limit_and_skip else {} + ) + + if self._limit == 0: + # mimic the fact that historically .limit(0) sets no limit + kwargs.pop("limit", None) + + if self._hint not in (-1, None): + kwargs["hint"] = self._hint + + if self._collation: + kwargs["collation"] = self._collation + + count = count_documents( + collection=self._cursor.collection, + filter=self._cursor._Cursor__spec, + **kwargs + ) + self._cursor_obj = None return count @@ -680,12 +714,10 @@ class BaseQuerySet: if full_response: if result["value"] is not None: - result["value"] = self._document._from_son( - result["value"], only_fields=self.only_fields - ) + result["value"] = self._document._from_son(result["value"]) else: if result is not None: - result = self._document._from_son(result, only_fields=self.only_fields) + result = self._document._from_son(result) return result @@ -718,24 +750,22 @@ class BaseQuerySet: docs = self._collection.find({"_id": {"$in": object_ids}}, **self._cursor_args) if self._scalar: for doc in docs: - doc_map[doc["_id"]] = self._get_scalar( - self._document._from_son(doc, only_fields=self.only_fields) - ) + doc_map[doc["_id"]] = self._get_scalar(self._document._from_son(doc)) elif self._as_pymongo: for doc in docs: doc_map[doc["_id"]] = doc else: for doc in docs: doc_map[doc["_id"]] = self._document._from_son( - doc, - only_fields=self.only_fields, - _auto_dereference=self._auto_dereference, + doc, _auto_dereference=self._auto_dereference, ) return doc_map def none(self): - """Helper that just returns a list""" + """Returns a queryset that never returns any objects and no query will be executed when accessing the results + inspired by django none() https://docs.djangoproject.com/en/dev/ref/models/querysets/#none + """ queryset = self.clone() queryset._none = True return queryset @@ -789,16 +819,17 @@ class BaseQuerySet: "_snapshot", "_timeout", "_read_preference", + "_read_concern", "_iter", "_scalar", "_as_pymongo", "_limit", "_skip", + "_empty", "_hint", "_collation", "_auto_dereference", "_search_text", - "only_fields", "_max_time_ms", "_comment", "_batch_size", @@ -834,6 +865,7 @@ class BaseQuerySet: """ queryset = self.clone() queryset._limit = n + queryset._empty = False # cancels the effect of empty # If a cursor object has already been created, apply the limit to it. if queryset._cursor_obj: @@ -1001,7 +1033,6 @@ class BaseQuerySet: .. versionchanged:: 0.5 - Added subfield support """ fields = {f: QueryFieldList.ONLY for f in fields} - self.only_fields = list(fields.keys()) return self.fields(True, **fields) def exclude(self, *fields): @@ -1266,10 +1297,7 @@ class BaseQuerySet: def from_json(self, json_data): """Converts json data to unsaved objects""" son_data = json_util.loads(json_data) - return [ - self._document._from_son(data, only_fields=self.only_fields) - for data in son_data - ] + return [self._document._from_son(data) for data in son_data] def aggregate(self, pipeline, *suppl_pipeline, **kwargs): """Perform a aggregate function based in your queryset params @@ -1311,10 +1339,11 @@ class BaseQuerySet: final_pipeline = initial_pipeline + user_pipeline collection = self._collection - if self._read_preference is not None: + if self._read_preference is not None or self._read_concern is not None: collection = self._collection.with_options( - read_preference=self._read_preference + read_preference=self._read_preference, read_concern=self._read_concern ) + return collection.aggregate(final_pipeline, cursor={}, **kwargs) # JS functionality @@ -1584,7 +1613,7 @@ class BaseQuerySet: def __next__(self): """Wrap the result in a :class:`~mongoengine.Document` object. """ - if self._limit == 0 or self._none: + if self._none or self._empty: raise StopIteration raw_doc = next(self._cursor) @@ -1593,9 +1622,7 @@ class BaseQuerySet: return raw_doc doc = self._document._from_son( - raw_doc, - _auto_dereference=self._auto_dereference, - only_fields=self.only_fields, + raw_doc, _auto_dereference=self._auto_dereference, ) if self._scalar: @@ -1603,8 +1630,6 @@ class BaseQuerySet: return doc - next = __next__ # For Python2 support - def rewind(self): """Rewind the cursor to its unevaluated state. diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 8b5872f..e2db8f0 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -144,6 +144,7 @@ class QuerySet(BaseQuerySet): return super().count(with_limit_and_skip) if self._len is None: + # cache the length self._len = super().count(with_limit_and_skip) return self._len diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 0eacc2e..a2448f2 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -7,6 +7,11 @@ from mongoengine.queryset import transform __all__ = ("Q", "QNode") +def warn_empty_is_deprecated(): + msg = "'empty' property is deprecated in favour of using 'not bool(filter)'" + warnings.warn(msg, DeprecationWarning, stacklevel=2) + + class QNodeVisitor: """Base visitor class for visiting Q-object nodes in a query tree. """ @@ -98,19 +103,18 @@ class QNode: object. """ # If the other Q() is empty, ignore it and just use `self`. - if getattr(other, "empty", True): + if not bool(other): return self # Or if this Q is empty, ignore it and just use `other`. - if self.empty: + if not bool(self): return other return QCombination(operation, [self, other]) @property def empty(self): - msg = "'empty' property is deprecated in favour of using 'not bool(filter)'" - warnings.warn(msg, DeprecationWarning) + warn_empty_is_deprecated() return False def __or__(self, other): @@ -152,8 +156,7 @@ class QCombination(QNode): @property def empty(self): - msg = "'empty' property is deprecated in favour of using 'not bool(filter)'" - warnings.warn(msg, DeprecationWarning) + warn_empty_is_deprecated() return not bool(self.children) def __eq__(self, other): @@ -186,4 +189,5 @@ class Q(QNode): @property def empty(self): + warn_empty_is_deprecated() return not bool(self.query) |