Configure AWS KMS for testing on evergreen

Author: aclark4life

.evergreen/config.yml

@@ -69,6 +69,7 @@ functions:
 
 pre:
   - func: setup
+  - func: start csfle servers
   - func: bootstrap mongo-orchestration
 
 post:
@@ -77,7 +78,6 @@ post:
 tasks:
   - name: run-tests
     commands:
-      - func: "start csfle servers"
       - func: "run unit tests"
 
 buildvariants:
@@ -126,8 +126,8 @@ buildvariants:
       - name: run-tests
 
   - name: tests-8-qe
-    display_name: Run Tests 8.0 QE
-    run_on: rhel94-perf-atlas
+    display_name: Run Tests 8.2 QE
+    run_on: rhel87-small
     expansions:
       MONGODB_VERSION: "8.2"
       TOPOLOGY: replica_set

.evergreen/run-tests.sh

@@ -2,6 +2,9 @@
 
 set -eux
 
+# Export secrets as environment variables
+. ../secrets-export.sh
+
 # Install django-mongodb-backend
 /opt/python/3.12/bin/python3 -m venv venv
 . venv/bin/activate

.evergreen/setup.sh

@@ -16,8 +16,8 @@ DRIVERS_TOOLS="$(dirname "$(pwd)")/drivers-tools"
 PROJECT_DIRECTORY="$(pwd)"
 
 if [ "Windows_NT" = "${OS:-}" ]; then
-    DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
-    PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
+    DRIVERS_TOOLS=$(cygpath -m "$DRIVERS_TOOLS")
+    PROJECT_DIRECTORY=$(cygpath -m "$PROJECT_DIRECTORY")
 fi
 export PROJECT_DIRECTORY
 export DRIVERS_TOOLS
@@ -37,8 +37,8 @@ PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
 EOT
 
 # Set up drivers-tools with a .env file.
-git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS}
-cat <<EOT > ${DRIVERS_TOOLS}/.env
+git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git "${DRIVERS_TOOLS}"
+cat <<EOT > "${DRIVERS_TOOLS}/.env"
 CURRENT_VERSION="$CURRENT_VERSION"
 DRIVERS_TOOLS="$DRIVERS_TOOLS"
 MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"

.github/workflows/encrypted_settings.py

@@ -7,18 +7,38 @@
 
 os.environ["LD_LIBRARY_PATH"] = str(Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent)
 
+AWS_CREDS = {
+    "accessKeyId": os.environ.get("FLE_AWS_KEY", ""),
+    "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""),
+}
+
+_USE_AWS_KMS = any(AWS_CREDS.values())
+
+if _USE_AWS_KMS:
+    _AWS_REGION = os.environ.get("FLE_AWS_KMS_REGION", "us-east-1")
+    _AWS_KEY_ARN = os.environ.get(
+        "FLE_AWS_KMS_KEY_ARN",
+        "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
+    )
+    KMS_PROVIDERS = {"aws": AWS_CREDS}
+    KMS_CREDENTIALS = {"aws": {"key": _AWS_KEY_ARN, "region": _AWS_REGION}}
+else:
+    KMS_PROVIDERS = {"local": {"key": os.urandom(96)}}
+    KMS_CREDENTIALS = {"local": {}}
+
 DATABASES["encrypted"] = {  # noqa: F405
     "ENGINE": "django_mongodb_backend",
     "NAME": "djangotests_encrypted",
     "OPTIONS": {
         "auto_encryption_opts": AutoEncryptionOpts(
             key_vault_namespace="djangotests_encrypted.__keyVault",
-            kms_providers={"local": {"key": os.urandom(96)}},
+            kms_providers=KMS_PROVIDERS,
             crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"],
+            crypt_shared_lib_required=True,
         ),
         "directConnection": True,
     },
-    "KMS_CREDENTIALS": {},
+    "KMS_CREDENTIALS": KMS_CREDENTIALS,
 }
 
 

tests/encryption_/test_base.py

@@ -1,3 +1,5 @@
+import os
+
 import pymongo
 from bson.binary import Binary
 from django.conf import settings
@@ -19,3 +21,18 @@ def assertEncrypted(self, model, field):
             collection = db[model._meta.db_table]
             data = collection.find_one({}, {field: 1, "_id": 0})
             self.assertIsInstance(data[field], Binary)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        AWS_CREDS = {
+            "accessKeyId": os.environ.get("FLE_AWS_KEY", ""),
+            "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""),
+        }
+        _USE_AWS_KMS = any(AWS_CREDS.values())
+
+        if _USE_AWS_KMS:
+            self.DEFAULT_KMS_PROVIDER = "aws"
+        else:
+            # Local-only fallback
+            self.DEFAULT_KMS_PROVIDER = "local"

tests/encryption_/test_management.py

@@ -125,7 +125,11 @@ def test_missing_key(self):
                 call_command("showencryptedfieldsmap", "--database", "encrypted", verbosity=0)
         finally:
             # Replace the deleted key.
+            master_key = connections["encrypted"].settings_dict["KMS_CREDENTIALS"][
+                self.DEFAULT_KMS_PROVIDER
+            ]
             connections["encrypted"].client_encryption.create_data_key(
-                kms_provider="local",
+                kms_provider=self.DEFAULT_KMS_PROVIDER,
+                master_key=master_key,
                 key_alt_names=[test_key],
             )