INTPYTHON-527 Add Queryable Encryption support

.evergreen/config.yml

@@ -12,6 +12,17 @@ post_error_fails_task: true
 post_timeout_secs: 1800 # 5 minutes
 
 functions:
+  "start csfle servers":
+    - command: ec2.assume_role
+      params:
+        role_arn: ${aws_test_secrets_role}
+    - command: subprocess.exec
+      params:
+        binary: bash
+        include_expansions_in_env: ["AWS_SECRET_ACCESS_KEY", "AWS_ACCESS_KEY_ID", "AWS_SESSION_TOKEN"]
+        args:
+          - ${DRIVERS_TOOLS}/.evergreen/csfle/setup.sh
+
   "setup":
     - command: git.get_project
       params:
@@ -44,7 +55,7 @@ functions:
       params:
         binary: bash
         working_dir: "src"
-        include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI"]
+        include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI", "DJANGO_SETTINGS_MODULE", "CRYPT_SHARED_LIB_PATH"]
         args:
           - ./.evergreen/run-tests.sh
 
@@ -54,6 +65,7 @@ functions:
         binary: bash
         args:
           - ${DRIVERS_TOOLS}/.evergreen/teardown.sh
+          - ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh
 
 pre:
   - func: setup
@@ -65,6 +77,7 @@ post:
 tasks:
   - name: run-tests
     commands:
+      - func: "start csfle servers"
       - func: "run unit tests"
 
 buildvariants:
@@ -111,3 +124,13 @@ buildvariants:
       SSL: "ssl"
     tasks:
       - name: run-tests
+
+  - name: tests-8-qe
+    display_name: Run Tests 8.0 QE
+    run_on: rhel94-perf-atlas
+    expansions:
+      MONGODB_VERSION: "8.2"
+      TOPOLOGY: replica_set
+      DJANGO_SETTINGS_MODULE: "encrypted_settings"
+    tasks:
+      - name: run-tests

.evergreen/run-tests.sh

@@ -6,7 +6,7 @@ set -eux
 /opt/python/3.10/bin/python3 -m venv venv
 . venv/bin/activate
 python -m pip install -U pip
-pip install -e .
+pip install -e '.[encryption]'
 
 # Install django and test dependencies
 git clone --branch mongodb-5.2.x https://github.com/mongodb-forks/django django_repo

.github/workflows/encrypted_settings.py

@@ -0,0 +1,44 @@
+# Settings for django_mongodb_backend/tests when encryption is supported.
+import os
+from pathlib import Path
+
+from mongodb_settings import *  # noqa: F403
+from pymongo.encryption import AutoEncryptionOpts
+
+os.environ["LD_LIBRARY_PATH"] = str(Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent)
+
+DATABASES["encrypted"] = {  # noqa: F405
+    "ENGINE": "django_mongodb_backend",
+    "NAME": "djangotests_encrypted",
+    "OPTIONS": {
+        "auto_encryption_opts": AutoEncryptionOpts(
+            key_vault_namespace="djangotests_encrypted.__keyVault",
+            kms_providers={"local": {"key": os.urandom(96)}},
+            crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"],
+        ),
+        "directConnection": True,
+    },
+    "KMS_CREDENTIALS": {},
+}
+
+
+class EncryptedRouter:
+    def db_for_read(self, model, **hints):
+        if model._meta.app_label == "encryption_":
+            return "encrypted"
+        return None
+
+    db_for_write = db_for_read
+
+    def allow_migrate(self, db, app_label, model_name=None, **hints):
+        # The encryption_ app's models are only created in the encrypted
+        # database.
+        if app_label == "encryption_":
+            return db == "encrypted"
+        # Don't create other app's models in the encrypted database.
+        if db == "encrypted":
+            return False
+        return None
+
+
+DATABASE_ROUTERS.append(EncryptedRouter())  # noqa: F405

.github/workflows/mongodb_settings.py

@@ -1,4 +1,5 @@
-# Settings for django_mongodb_backend/tests.
+# Settings for django_mongodb_backend/tests when encryption isn't supported.
 from django_settings import *  # noqa: F403
 
+DATABASES["encrypted"] = {}  # noqa: F405
 DATABASE_ROUTERS = ["django_mongodb_backend.routers.MongoRouter"]

.github/workflows/runtests.py

@@ -6,151 +6,6 @@
 from django.core.exceptions import ImproperlyConfigured
 
 test_apps = [
-    "admin_changelist",
-    "admin_checks",
-    "admin_custom_urls",
-    "admin_docs",
-    "admin_filters",
-    "admin_inlines",
-    "admin_ordering",
-    "admin_scripts",
-    "admin_utils",
-    "admin_views",
-    "admin_widgets",
-    "aggregation",
-    "aggregation_regress",
-    "annotations",
-    "apps",
-    "async",
-    "auth_tests",
-    "backends",
-    "basic",
-    "bulk_create",
-    "cache",
-    "check_framework",
-    "constraints",
-    "contenttypes_tests",
-    "context_processors",
-    "custom_columns",
-    "custom_lookups",
-    "custom_managers",
-    "custom_pk",
-    "datatypes",
-    "dates",
-    "datetimes",
-    "db_functions",
-    "defer",
-    "defer_regress",
-    "delete",
-    "delete_regress",
-    "empty",
-    "empty_models",
-    "expressions",
-    "expressions_case",
-    "field_defaults",
-    "file_storage",
-    "file_uploads",
-    "fixtures",
-    "fixtures_model_package",
-    "fixtures_regress",
-    "flatpages_tests",
-    "force_insert_update",
-    "foreign_object",
-    "forms_tests",
-    "from_db_value",
-    "generic_inline_admin",
-    "generic_relations",
-    "generic_relations_regress",
-    "generic_views",
-    "get_earliest_or_latest",
-    "get_object_or_404",
-    "get_or_create",
-    "i18n",
-    "indexes",
-    "inline_formsets",
-    "introspection",
-    "invalid_models_tests",
-    "known_related_objects",
-    "lookup",
-    "m2m_and_m2o",
-    "m2m_intermediary",
-    "m2m_multiple",
-    "m2m_recursive",
-    "m2m_regress",
-    "m2m_signals",
-    "m2m_through",
-    "m2m_through_regress",
-    "m2o_recursive",
-    "managers_regress",
-    "many_to_many",
-    "many_to_one",
-    "many_to_one_null",
-    "max_lengths",
-    "messages_tests",
-    "migrate_signals",
-    "migration_test_data_persistence",
-    "migrations",
-    "model_fields",
-    "model_forms",
-    "model_formsets",
-    "model_formsets_regress",
-    "model_indexes",
-    "model_inheritance",
-    "model_inheritance_regress",
-    "model_options",
-    "model_package",
-    "model_regress",
-    "model_utils",
-    "modeladmin",
-    "multiple_database",
-    "mutually_referential",
-    "nested_foreign_keys",
-    "null_fk",
-    "null_fk_ordering",
-    "null_queries",
-    "one_to_one",
-    "or_lookups",
-    "order_with_respect_to",
-    "ordering",
-    "pagination",
-    "prefetch_related",
-    "proxy_model_inheritance",
-    "proxy_models",
-    "queries",
-    "queryset_pickle",
-    "redirects_tests",
-    "reserved_names",
-    "reverse_lookup",
-    "save_delete_hooks",
-    "schema",
-    "select_for_update",
-    "select_related",
-    "select_related_onetoone",
-    "select_related_regress",
-    "serializers",
-    "servers",
-    "sessions_tests",
-    "shortcuts",
-    "signals",
-    "sitemaps_tests",
-    "sites_framework",
-    "sites_tests",
-    "string_lookup",
-    "swappable_models",
-    "syndication_tests",
-    "test_client",
-    "test_client_regress",
-    "test_runner",
-    "test_utils",
-    "timezones",
-    "transactions",
-    "unmanaged_models",
-    "update",
-    "update_only_fields",
-    "user_commands",
-    "validation",
-    "view_tests",
-    "xor_lookups",
     # Add directories in django_mongodb_backend/tests
     *sorted(
         [

.github/workflows/test-python-atlas.yml

@@ -28,7 +28,7 @@ jobs:
       - name: install django-mongodb-backend
         run: |
           pip3 install --upgrade pip
-          pip3 install -e .
+          pip3 install -e .[encryption]
       - name: Checkout Django
         uses: actions/checkout@v5
         with:
@@ -51,8 +51,15 @@ jobs:
         run: cp .github/workflows/runtests.py django_repo/tests/runtests_.py
       - name: Start local Atlas
         working-directory: .
-        run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:7
+        run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:8.0.15
+      - name: Download crypt shared
+        run: |
+          wget https://downloads.mongodb.com/linux/mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz
+          tar -xvzf mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz lib/mongo_crypt_v1.so
       - name: Run tests
         run: python3 django_repo/tests/runtests_.py
     permissions:
         contents: read
+    env:
+      DJANGO_SETTINGS_MODULE: "encrypted_settings"
+      CRYPT_SHARED_LIB_PATH: "${{ github.workspace }}/lib/mongo_crypt_v1.so"