PYTHON-4181 Use libmongocrypt native crypto when available for 10-50x better performance (#756)

Bundle crypto-enabled libmongocrypt on Mac and Windows.
Add crypto enabled libmongocrypt to test matrix.
Always set AES-CTR crypto callbacks on macOS < 10.15.
Update readme to prefer the crypto-enabled libmongocrypt builds.

Author: ShaneHarvey

bindings/python/.evergreen/test.sh

@@ -14,6 +14,7 @@ git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git
 
 if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
     PYMONGOCRYPT_LIB=${MONGOCRYPT_DIR}/nocrypto/bin/mongocrypt.dll
+    PYMONGOCRYPT_LIB_CRYPTO=$(cygpath -m ${MONGOCRYPT_DIR}/bin/mongocrypt.dll)
     export PYMONGOCRYPT_LIB=$(cygpath -m $PYMONGOCRYPT_LIB)
     PYTHONS=("C:/python/Python37/python.exe"
              "C:/python/Python38/python.exe"
@@ -26,6 +27,7 @@ if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
       --version latest --out ../crypt_shared/
 elif [ "Darwin" = "$(uname -s)" ]; then
     export PYMONGOCRYPT_LIB=${MONGOCRYPT_DIR}/nocrypto/lib/libmongocrypt.dylib
+    PYMONGOCRYPT_LIB_CRYPTO=${MONGOCRYPT_DIR}/lib/libmongocrypt.dylib
     MACOS_VER=$(sw_vers -productVersion)
     if [[ $MACOS_VER =~ ^10.14 ]]; then
       PYTHONS=("/Library/Frameworks/Python.framework/Versions/3.7/bin/python3"
@@ -44,7 +46,13 @@ elif [ "Darwin" = "$(uname -s)" ]; then
     python3 drivers-evergreen-tools/.evergreen/mongodl.py --component crypt_shared \
       --version latest --out ../crypt_shared/
 else
-    export PYMONGOCRYPT_LIB=${MONGOCRYPT_DIR}/nocrypto/lib64/libmongocrypt.so
+    if [ -e "${MONGOCRYPT_DIR}/lib64/" ]; then
+        export PYMONGOCRYPT_LIB=${MONGOCRYPT_DIR}/nocrypto/lib64/libmongocrypt.so
+        PYMONGOCRYPT_LIB_CRYPTO=${MONGOCRYPT_DIR}/lib64/libmongocrypt.so
+    else
+        export PYMONGOCRYPT_LIB=${MONGOCRYPT_DIR}/nocrypto/lib/libmongocrypt.so
+        PYMONGOCRYPT_LIB_CRYPTO=${MONGOCRYPT_DIR}/lib/libmongocrypt.so
+    fi
 
     export CRYPT_SHARED_PATH="../crypt_shared/lib/mongo_crypt_v1.so"
     MACHINE=$(uname -m)
@@ -74,8 +82,10 @@ for PYTHON_BINARY in "${PYTHONS[@]}"; do
     createvirtualenv $PYTHON_BINARY .venv
     python -m pip install --prefer-binary -r test-requirements.txt
     python -m pip install -v -e .
-    python -m pytest -v --ignore=test/performance .
-    echo "Running tests with CSFLE on dynamic library path..."
+    echo "Running tests with crypto enabled libmongocrypt..."
+    PYMONGOCRYPT_LIB=$PYMONGOCRYPT_LIB_CRYPTO python -c 'from pymongocrypt.binding import lib;assert lib.mongocrypt_is_crypto_available(), "mongocrypt_is_crypto_available() returned False"'
+    PYMONGOCRYPT_LIB=$PYMONGOCRYPT_LIB_CRYPTO python -m pytest -v --ignore=test/performance .
+    echo "Running tests with crypt_shared on dynamic library path..."
     TEST_CRYPT_SHARED=1 DYLD_FALLBACK_LIBRARY_PATH=../crypt_shared/lib/:$DYLD_FALLBACK_LIBRARY_PATH \
       LD_LIBRARY_PATH=../crypt_shared/lib:$LD_LIBRARY_PATH \
       PATH=../crypt_shared/bin:$PATH \

bindings/python/CHANGELOG.rst

@@ -6,6 +6,10 @@ Changes in Version 1.9.0
 
 - Add support for named KMS providers like "local:name1".
   This feature requires libmongocrypt >= 1.9.0.
+- Use libmongocrypt native crypto when available which results in 10-50x better performance.
+  On Linux, it is recommended to download the platform specific build and
+  set PYMONGOCRYPT_LIB to the crypto-enabled libmongocrypt.so.
+- Bundle the crypto-enabled libmongocrypt builds in macOS and Windows wheels for better performance.
 - Bundle libmongocrypt 1.9.0 in release wheels.
 
 Changes in Version 1.8.0

bindings/python/README.rst

@@ -114,33 +114,43 @@ For example::
   $ curl -O https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz
   $ mkdir libmongocrypt-all && tar xzf libmongocrypt-all.tar.gz -C libmongocrypt-all
   $ ls libmongocrypt-all
-  amazon2             rhel-62-64-bit      rhel72-zseries-test ubuntu1604-arm64
-  debian10            rhel-67-s390x       suse12-64           ubuntu1804-64
-  debian92            rhel-70-64-bit      suse12-s390x        ubuntu1804-arm64
-  linux-64-amazon-ami rhel-71-ppc64el     suse15-64           windows-test
-  macos               rhel-80-64-bit      ubuntu1604
+  amazon2             debian92            rhel-80-64-bit      rhel72-zseries-test ubuntu1804-arm64
+  amazon2-arm64       linux-64-amazon-ami rhel-81-ppc64el     suse12-64           ubuntu2004-64
+  amazon2023          macos               rhel-82-arm64       suse15-64           ubuntu2004-arm64
+  amazon2023-arm64    rhel-62-64-bit      rhel-83-zseries     ubuntu1604          ubuntu2204-64
+  debian10            rhel-70-64-bit      rhel-91-64-bit      ubuntu1604-arm64    ubuntu2204-arm64
+  debian11            rhel-71-ppc64el     rhel-91-arm64       ubuntu1804-64       windows-test
 
 macOS::
 
   $ # Set PYMONGOCRYPT_LIB for macOS:
-  $ export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/macos/nocrypto/lib/libmongocrypt.dylib
+  $ export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/macos/lib/libmongocrypt.dylib
   $ python -c "import pymongocrypt; print(pymongocrypt.libmongocrypt_version())"
-  1.2.1
+  1.9.0
 
 Windows::
 
   $ # Set PYMONGOCRYPT_LIB for Windows:
-  $ chmod +x $(pwd)/libmongocrypt-all/windows-test/nocrypto/bin/mongocrypt.dll
-  $ export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/windows-test/nocrypto/bin/mongocrypt.dll
+  $ chmod +x $(pwd)/libmongocrypt-all/windows-test/bin/mongocrypt.dll
+  $ export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/windows-test/bin/mongocrypt.dll
   $ python -c "import pymongocrypt; print(pymongocrypt.libmongocrypt_version())"
-  1.2.1
+  1.9.0
 
-Linux::
+Linux: set the libmongocrypt build for your platform, for example for Ubuntu 22.04 x86_64::
+
+  $ # Set PYMONGOCRYPT_LIB for Ubuntu 22.04 x86_64:
+  $ export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/ubuntu2204-64/lib/libmongocrypt.so
+  $ python -c "import pymongocrypt; print(pymongocrypt.libmongocrypt_version())"
+  1.9.0
+
+Note if your Linux platform is not available, the generic RHEL 6.2 x86_64 "nocrypto" build
+should still be compatible however the "nocrypto" build will result in lower performance
+for encryption and decryption::
 
   $ # Set PYMONGOCRYPT_LIB for RHEL 6.2 x86_64:
   $ export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/rhel-62-64-bit/nocrypto/lib64/libmongocrypt.so
   $ python -c "import pymongocrypt; print(pymongocrypt.libmongocrypt_version())"
-  1.2.1
+  1.9.0
 
 Dependencies
 ============
@@ -178,7 +188,7 @@ variables, like ``LD_LIBRARY_PATH``. For example::
 
   $ export PYMONGOCRYPT_LIB='/path/to/libmongocrypt.so'
   $ python -c "import pymongocrypt; print(pymongocrypt.libmongocrypt_version())"
-  1.2.1
+  1.9.0
 
 Testing
 =======

bindings/python/pymongocrypt/binding.py

@@ -63,6 +63,16 @@ def _parse_version(version):
  */
 const char *mongocrypt_version(uint32_t *len);
 
+/**
+ * Returns true if libmongocrypt was built with native crypto support.
+ *
+ * If libmongocrypt was not built with native crypto support, setting crypto
+ * hooks is required.
+ *
+ * @returns True if libmongocrypt was built with native crypto support.
+ */
+bool mongocrypt_is_crypto_available(void);
+
 /**
  * A non-owning view of a byte buffer.
  *

bindings/python/pymongocrypt/mongocrypt.py

@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import copy
+import sys
+import platform
 
+from packaging.version import Version
 
 from pymongocrypt.binary import (MongoCryptBinaryIn,
                                  MongoCryptBinaryOut)
@@ -205,18 +207,31 @@ def __init(self):
         if self.__opts.bypass_query_analysis:
             lib.mongocrypt_setopt_bypass_query_analysis(self.__crypt)
 
-        if not lib.mongocrypt_setopt_crypto_hooks(
-                self.__crypt, aes_256_cbc_encrypt, aes_256_cbc_decrypt,
-                secure_random, hmac_sha_512, hmac_sha_256, sha_256, ffi.NULL):
-            self.__raise_from_status()
+        # Prefer using the native crypto binding when we know it's available.
+        try:
+            crypto_available = lib.mongocrypt_is_crypto_available()
+        except AttributeError:
+            # libmongocrypt < 1.9
+            crypto_available = False
+
+        if not crypto_available:
+            if not lib.mongocrypt_setopt_crypto_hooks(
+                    self.__crypt, aes_256_cbc_encrypt, aes_256_cbc_decrypt,
+                    secure_random, hmac_sha_512, hmac_sha_256, sha_256, ffi.NULL):
+                self.__raise_from_status()
 
-        if not lib.mongocrypt_setopt_crypto_hook_sign_rsaes_pkcs1_v1_5(
-                self.__crypt, sign_rsaes_pkcs1_v1_5, ffi.NULL):
-            self.__raise_from_status()
+            if not lib.mongocrypt_setopt_crypto_hook_sign_rsaes_pkcs1_v1_5(
+                    self.__crypt, sign_rsaes_pkcs1_v1_5, ffi.NULL):
+                self.__raise_from_status()
 
-        if not lib.mongocrypt_setopt_aes_256_ctr(
-                self.__crypt, aes_256_ctr_encrypt, aes_256_ctr_decrypt, ffi.NULL):
-            self.__raise_from_status()
+            if not lib.mongocrypt_setopt_aes_256_ctr(
+                    self.__crypt, aes_256_ctr_encrypt, aes_256_ctr_decrypt, ffi.NULL):
+                self.__raise_from_status()
+        elif sys.platform == 'darwin' and Version(platform.mac_ver()[0]) < Version("10.15"):
+            # MONGOCRYPT-440 libmongocrypt does not support AES-CTR on macOS < 10.15.
+            if not lib.mongocrypt_setopt_aes_256_ctr(
+                    self.__crypt, aes_256_ctr_encrypt, aes_256_ctr_decrypt, ffi.NULL):
+                self.__raise_from_status()
 
         if self.__opts.crypt_shared_lib_path is not None:
             lib.mongocrypt_setopt_set_crypt_shared_lib_path_override(

bindings/python/release.sh

@@ -66,7 +66,8 @@ if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
     dos2unix .venv/Scripts/activate || true
     . ./.venv/Scripts/activate
 
-    get_libmongocrypt windows-test libmongocrypt/nocrypto/bin/mongocrypt.dll
+    # Use crypto-enabled libmongocrypt.
+    get_libmongocrypt windows-test libmongocrypt/bin/mongocrypt.dll
     build_wheel
     test_dist dist/*.whl
 fi 
@@ -76,8 +77,9 @@ if [ "Darwin" = "$(uname -s)" ]; then
     $PYTHON -m venv .venv
     . .venv/bin/activate
 
+    # Use crypto-enabled libmongocrypt.
     # Build intel wheel for Python 3.7.
-    get_libmongocrypt macos_x86_64 libmongocrypt/nocrypto/lib/libmongocrypt.dylib
+    get_libmongocrypt macos_x86_64 libmongocrypt/lib/libmongocrypt.dylib
     # See https://github.com/pypa/cibuildwheel/blob/a3e5b541dc3111166a3abdbbc90ecb195c8cb9e2/cibuildwheel/macos.py#L247
     # for information on these environment variables.
     export MACOSX_DEPLOYMENT_TARGET=10.14
@@ -88,7 +90,7 @@ if [ "Darwin" = "$(uname -s)" ]; then
     fi
 
     # Build universal2 wheel.
-    get_libmongocrypt macos libmongocrypt/nocrypto/lib/libmongocrypt.dylib
+    get_libmongocrypt macos libmongocrypt/lib/libmongocrypt.dylib
     export MACOSX_DEPLOYMENT_TARGET=11.0
     export _PYTHON_HOST_PLATFORM=macosx-11.0-universal2
     build_wheel