PYTHON-3276 [pymongo] FLE 1.0 shared library (#947)

Author: juliusgeo

.evergreen/config.yml

@@ -450,6 +450,9 @@ functions:
             export LIBMONGOCRYPT_URL="${libmongocrypt_url}"
             export TEST_ENCRYPTION=1
           fi
+          if [ -n "${test_csfle}" ]; then
+            export TEST_CSFLE=1
+          fi
           if [ -n "${test_pyopenssl}" ]; then
             export TEST_PYOPENSSL=1
           fi
@@ -1232,7 +1235,6 @@ tasks:
             VERSION: "5.0"
             TOPOLOGY: "sharded_cluster"
         - func: "run tests"
-
     - name: "test-6.0-standalone"
       tags: ["6.0", "standalone"]
       commands:
@@ -2161,6 +2163,14 @@ axes:
         variables:
           test_encryption: true
         batchtime: 10080  # 7 days
+      - id: "encryption_with_csfle"
+        display_name: "Encryption with CSFLE"
+        tags: ["encryption_tag", "csfle"]
+        variables:
+          test_encryption: true
+          test_csfle: true
+        batchtime: 10080  # 7 days
+
 
   # Run pyopenssl tests?
   - id: pyopenssl
@@ -2229,21 +2239,6 @@ buildvariants:
     - ".4.0"
     - ".3.6"
 
-- matrix_name: "tests-all-encryption"
-  matrix_spec:
-    platform:
-      # OSes that support versions of MongoDB>=2.6 with SSL.
-      - awslinux
-    auth-ssl: "*"
-    encryption: "*"
-  display_name: "Encryption ${platform} ${auth-ssl}"
-  tasks:
-    - ".6.0"
-    - ".5.0"
-    - ".4.4"
-    - ".4.2"
-    - ".4.0"
-
 - matrix_name: "tests-archlinux"
   matrix_spec:
     platform:
@@ -2297,14 +2292,27 @@ buildvariants:
     auth: "auth"
     ssl: "nossl"
     encryption: "*"
-  display_name: "Encryption ${platform} ${auth} ${ssl}"
+  display_name: "${encryption} ${platform} ${auth} ${ssl}"
   tasks: &encryption-server-versions
+    - ".rapid"
     - ".latest"
     - ".6.0"
     - ".5.0"
     - ".4.4"
     - ".4.2"
     - ".4.0"
+  rules: &encryption-exclude-rules
+    - if:
+        platform: "*"
+        auth: "*"
+        ssl: "*"
+        encryption: [ "encryption_with_csfle" ]
+      then:
+        remove_tasks:
+          - ".5.0"
+          - ".4.4"
+          - ".4.2"
+          - ".4.0"
 
 # Test one server version with zSeries, POWER8, and ARM.
 - matrix_name: "test-different-cpu-architectures"
@@ -2385,8 +2393,21 @@ buildvariants:
 # dependency tests-python-version-rhel62-test-encryption_.../test-2.6-standalone is not present in the project config
 #    coverage: "*"
     encryption: "*"
-  display_name: "Encryption ${python-version} ${platform} ${auth-ssl}"
+  display_name: "${encryption} ${python-version} ${platform} ${auth-ssl}"
   tasks: *encryption-server-versions
+  rules:
+    - if:
+        platform: "*"
+        python-version: "*"
+        auth-ssl: "*"
+        encryption: [ "encryption_with_csfle" ]
+      then:
+        remove_tasks:
+          - ".5.0"
+          - ".4.4"
+          - ".4.2"
+          - ".4.0"
+
 
 - matrix_name: "tests-python-version-ubuntu18-without-c-extensions"
   matrix_spec:
@@ -2481,8 +2502,20 @@ buildvariants:
     python-version-windows: "*"
     auth-ssl: "*"
     encryption: "*"
-  display_name: "Encryption ${platform} ${python-version-windows} ${auth-ssl}"
+  display_name: "${encryption} ${platform} ${python-version-windows} ${auth-ssl}"
   tasks: *encryption-server-versions
+  rules:
+    - if:
+        platform: "*"
+        python-version-windows: "*"
+        auth-ssl: "*"
+        encryption: [ "encryption_with_csfle" ]
+      then:
+        remove_tasks:
+          - ".5.0"
+          - ".4.4"
+          - ".4.2"
+          - ".4.0"
 
 # Storage engine tests on Ubuntu 18.04 (x86_64) with Python 3.7.
 - matrix_name: "tests-storage-engines"

.evergreen/run-tests.sh

@@ -11,6 +11,7 @@ set -o errexit  # Exit the script with error if any of the commands fail
 #  COVERAGE           If non-empty, run the test suite with coverage.
 #  TEST_ENCRYPTION    If non-empty, install pymongocrypt.
 #  LIBMONGOCRYPT_URL  The URL to download libmongocrypt.
+#  TEST_CSFLE         If non-empty, install CSFLE
 
 if [ -n "${SET_XTRACE_ON}" ]; then
     set -o xtrace
@@ -27,6 +28,7 @@ COVERAGE=${COVERAGE:-}
 COMPRESSORS=${COMPRESSORS:-}
 MONGODB_API_VERSION=${MONGODB_API_VERSION:-}
 TEST_ENCRYPTION=${TEST_ENCRYPTION:-}
+TEST_CSFLE=${TEST_CSFLE:-}
 LIBMONGOCRYPT_URL=${LIBMONGOCRYPT_URL:-}
 DATA_LAKE=${DATA_LAKE:-}
 
@@ -153,7 +155,16 @@ if [ -z "$DATA_LAKE" ]; then
 else
     TEST_ARGS="-s test.test_data_lake"
 fi
-
+if [ -z $TEST_CSFLE ]; then
+    echo "CSFLE not being tested"
+else
+    $PYTHON $DRIVERS_TOOLS/.evergreen/mongodl.py --component csfle \
+        --version latest --out ../csfle/
+    export DYLD_FALLBACK_LIBRARY_PATH=../csfle/lib/:$DYLD_FALLBACK_LIBRARY_PATH
+    export LD_LIBRARY_PATH=../csfle/lib:$LD_LIBRARY_PATH
+    export PATH=../csfle/bin:$PATH
+    TEST_ARGS="-s test.test_encryption"
+fi
 # Don't download unittest-xml-reporting from pypi, which often fails.
 if $PYTHON -c "import xmlrunner"; then
     # The xunit output dir must be a Python style absolute path.

pymongo/encryption.py

@@ -296,7 +296,14 @@ def _get_internal_client(encrypter, mongo_client):
 
         io_callbacks = _EncryptionIO(metadata_client, key_vault_coll, mongocryptd_client, opts)
         self._auto_encrypter = AutoEncrypter(
-            io_callbacks, MongoCryptOptions(opts._kms_providers, schema_map)
+            io_callbacks,
+            MongoCryptOptions(
+                opts._kms_providers,
+                schema_map,
+                csfle_path=opts._csfle_path,
+                csfle_required=opts._csfle_required,
+                bypass_encryption=opts._bypass_auto_encryption,
+            ),
         )
         self._closed = False
 

pymongo/encryption_options.py

@@ -39,12 +39,14 @@ def __init__(
         key_vault_namespace: str,
         key_vault_client: Optional["MongoClient"] = None,
         schema_map: Optional[Mapping[str, Any]] = None,
-        bypass_auto_encryption: Optional[bool] = False,
+        bypass_auto_encryption: bool = False,
         mongocryptd_uri: str = "mongodb://localhost:27020",
         mongocryptd_bypass_spawn: bool = False,
         mongocryptd_spawn_path: str = "mongocryptd",
         mongocryptd_spawn_args: Optional[List[str]] = None,
         kms_tls_options: Optional[Mapping[str, Any]] = None,
+        csfle_path: Optional[str] = None,
+        csfle_required: bool = False,
     ) -> None:
         """Options to configure automatic client-side field level encryption.
 
@@ -140,6 +142,12 @@ def __init__(
             Or to supply a client certificate::
 
               kms_tls_options={'kmip': {'tlsCertificateKeyFile': 'client.pem'}}
+          - `csfle_path` (optional): Override the path to load the CSFLE library.
+          - `csfle_required` (optional): If 'true', refuse to continue encryption without a CSFLE
+            library
+
+        .. versionchanged:: 4.2
+           Added `csfle_path` and `csfle_required` parameters
 
         .. versionchanged:: 4.0
            Added the `kms_tls_options` parameter and the "kmip" KMS provider.
@@ -152,7 +160,8 @@ def __init__(
                 "install a compatible version with: "
                 "python -m pip install 'pymongo[encryption]'"
             )
-
+        self._csfle_path = csfle_path
+        self._csfle_required = csfle_required
         self._kms_providers = kms_providers
         self._key_vault_namespace = key_vault_namespace
         self._key_vault_client = key_vault_client

test/test_encryption.py

@@ -82,6 +82,18 @@ def get_client_opts(client):
 
 
 class TestAutoEncryptionOpts(PyMongoTestCase):
+    @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed")
+    @unittest.skipUnless(os.environ.get("TEST_CSFLE"), "csfle is not installed")
+    def test_csfle(self):
+        # Test that we can pick up csfle automatically
+        client = MongoClient(
+            auto_encryption_opts=AutoEncryptionOpts(
+                KMS_PROVIDERS, "keyvault.datakeys", csfle_required=True
+            ),
+            connect=False,
+        )
+        self.addCleanup(client.close)
+
     @unittest.skipIf(_HAVE_PYMONGOCRYPT, "pymongocrypt is installed")
     def test_init_requires_pymongocrypt(self):
         with self.assertRaises(ConfigurationError):
@@ -1749,6 +1761,10 @@ def test_case_8(self):
 
 # https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/tests/README.rst#bypass-spawning-mongocryptd
 class TestBypassSpawningMongocryptdProse(EncryptionIntegrationTest):
+    @unittest.skipIf(
+        os.environ.get("TEST_CSFLE"),
+        "this prose test does not work when CSFLE is on a system dynamic library search path.",
+    )
     def test_mongocryptd_bypass_spawn(self):
         # Lower the mongocryptd timeout to reduce the test run time.
         self._original_timeout = encryption._MONGOCRYPTD_TIMEOUT_MS