Release Spark 3.5-cpu-py312-v1.0 (#161)

* Update package to build with Python 3.12.11 and updated dependencies
* Update documentation for consistency
* Update formatting to make sure `make lint` passes

Author: mrpic-amzn

.flake8

@@ -12,13 +12,13 @@ exclude =
 
 max-complexity = 10
 
-ignore =
-    C901,  # Function is too complex
-    D104,  # Missing docstring in public package
-    D107,  # Allow init not to have a docstring.
-    D105,  # Allow magic methods (__repr__, __str__, ...) not to have docstrings.
-    E203,  # whitespace before ':': Black disagrees with and explicitly violates this.
-    E501,  # line too long -- follow black formatting instead.
-    W503   # Line break occurred before a binary operator
+# C901: Function is too complex
+# D104: Missing docstring in public package
+# D107: Allow init not to have a docstring
+# D105: Allow magic methods not to have docstrings
+# E203: whitespace before ':' - Black disagrees with this
+# E501: line too long - follow black formatting instead
+# W503: Line break occurred before a binary operator
+ignore = C901,D104,D107,D105,E203,E501,W503
 
 require-code = True

DEVELOPMENT.md

@@ -2,7 +2,7 @@
 This document describes how to set up a development environment for developing, building, and testing the SageMaker Spark Container image.
 
 ## Development Environment Setup
-You’ll need to have python, pytest, docker, and docker-compose installed on your machine
+You’ll need to have python, pytest, and docker installed on your machine
 and on your $PATH.
 
 This repository uses GNU `make` to run build targets specified in `Makefile`. Consult the `Makefile` for the full list of build targets.
@@ -40,7 +40,7 @@ You may want to activate the Python environment in your `.bashrc` or `.zshrc`.
 -- [AmazonS3FullAccess](https://console.aws.amazon.com/iam/home#policies/arn:aws:iam::aws:policy/AmazonS3FullAccess) <br>
 -- [AmazonEC2ContainerRegistryFullAccess](https://console.aws.amazon.com/iam/home#policies/arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess) <br>
 
-5. [Create](https://docs.aws.amazon.com/cli/latest/reference/ecr/create-repository.html) an ECR repository with the name "sagemaker-spark" in the us-west-2 region
+5. [Create](https://docs.aws.amazon.com/cli/latest/reference/ecr/create-repository.html) an ECR repository with the name "sagemaker-spark-processing" in the us-west-2 region
 
 6. Setup required environment variables for the container build:
 ```
@@ -100,16 +100,16 @@ make build
 
 Upon successful build, you will see two tags applied to the image. For example:
 ```
-Successfully tagged sagemaker-spark:2.4-cpu-py37-v0.1
-Successfully tagged sagemaker-spark:latest
+Successfully tagged sagemaker-spark-processing:2.4-cpu-py37-v0.1
+Successfully tagged sagemaker-spark-processing:latest
 ```
 
 2. To verify that the image is available in your local docker repository, run `docker images`. You should see an image with two tags. For example:
 ```
 ✗ docker images
 REPOSITORY                                                                             TAG                   IMAGE ID            CREATED             SIZE
-sagemaker-spark                                                                        2.4-cpu-py37-v0.1     a748a6e042d2        5 minutes ago        3.06GB
-sagemaker-spark                                                                        latest                a748a6e042d2        5 minutes ago        3.06GB
+sagemaker-spark-processing                                                             2.4-cpu-py37-v0.1     a748a6e042d2        5 minutes ago        3.06GB
+sagemaker-spark-processing                                                             latest                a748a6e042d2        5 minutes ago        3.06GB
 ```
 
 ### Running Local Tests

Makefile

@@ -9,7 +9,7 @@ SHELL          := /bin/sh
 ifeq ($(IS_RELEASE_BUILD),)
     SPARK_VERSION := 3.5
     PROCESSOR := cpu
-    FRAMEWORK_VERSION := py39
+    FRAMEWORK_VERSION := py312
     SM_VERSION := 1.0
     USE_CASE := processing
     BUILD_CONTEXT := ./spark/${USE_CASE}/${SPARK_VERSION}/py3
@@ -31,16 +31,17 @@ all: build test
 
 init:
 	python --version
-	pip install --upgrade pip
-	# pipenv > 2022.4.8 fails to build smspark
-	python -m pip install pipenv==2022.4.8
+	python -m ensurepip --upgrade
+	python -m pip install --upgrade setuptools
+	python -m pip install pipenv
 	cp smsparkbuild/${FRAMEWORK_VERSION}/Pipfile .
 	cp smsparkbuild/${FRAMEWORK_VERSION}/pyproject.toml .
 	cp smsparkbuild/${FRAMEWORK_VERSION}/setup.py .
 	pipenv install
 	cp Pipfile ${BUILD_CONTEXT}
 	cp Pipfile.lock ${BUILD_CONTEXT}
 	cp setup.py ${BUILD_CONTEXT}
+	cp VERSION ${BUILD_CONTEXT}
 
 # Builds and moves container python library into the Docker build context
 build-container-library: init
@@ -160,7 +161,7 @@ install-sdk:
 
 # Makes sure docker containers are cleaned
 clean:
-	docker-compose down || true
+	docker compose down || true
 	docker kill $$(docker ps -q) || true
 	docker rm $$(docker ps -a -q) || true
 	docker network rm $$(docker network ls -q) || true

available_images.md

@@ -36,7 +36,7 @@ name and version tag into the repository URL. For example:
 
 
 
-     173754725891.dkr.ecr.<region>.amazonaws.com/sagemaker-spark-processing:2.4-cpu-py37-v1.0
+     173754725891.dkr.ecr.<region>.amazonaws.com/sagemaker-spark-processing:3.5-cpu-py310-v1.0
 
 **Important**
 

buildspec.yml

@@ -2,6 +2,8 @@ version: 0.2
 
 phases:
   install:
+    runtime-versions:
+      python: 3.12
     commands:
       - start-dockerd
   build:

docker-compose.yml

@@ -1,12 +1,10 @@
-version: "3.7"
-
 networks:
   spark:
     name: spark-network
 
 services:
   algo-1:
-    image: sagemaker-spark:latest
+    image: sagemaker-spark-processing:latest
     # Spark does a reverse DNS lookup (IP address to hostname)
     # By default in docker-compose the hostname is identical to the container name
     container_name: algo-1
@@ -21,7 +19,7 @@ services:
         - ${JARS_MOUNT:-/dev/null:/opt/ml/processing/input/jars}
     command: $CMD
   algo-2:
-    image: sagemaker-spark:latest
+    image: sagemaker-spark-processing:latest
     container_name: algo-2
     hostname: algo-2
     networks:

new_images.yml

@@ -3,5 +3,5 @@ new_images:
   - spark: "3.5"
     use-case: "processing"
     processors: ["cpu"]
-    python: ["py39"]
+    python: ["py312"]
     sm_version: "1.0"

scripts/build.sh

@@ -26,11 +26,38 @@ aws ecr get-login-password --region us-west-2 | docker login --username AWS --pa
 
 echo "building image ${version} ... "
 echo "building image under ${build_context}/docker/${framework_version} ... "
-docker build \
-    -f ${build_context}/docker/${framework_version}/Dockerfile.${processor} \
-    -t ${repository}:${version} \
-    --build-arg REGION=${REGION} \
-    -t sagemaker-spark:latest \
-    ${build_context}
 
+# Check if running on Mac with ARM architecture
+if [[ "$(uname)" == "Darwin" ]] && [[ "$(uname -m)" == "arm64" ]]; then
+    echo "Detected Mac with ARM architecture, using buildx for cross-platform build..."
+    
+    # Create a new builder instance if it doesn't exist
+    if ! docker buildx inspect multi-platform-builder &>/dev/null; then
+        echo "Creating new buildx builder 'multi-platform-builder'..."
+        docker buildx create --name multi-platform-builder --use
+    else
+        echo "Using existing buildx builder 'multi-platform-builder'..."
+        docker buildx use multi-platform-builder
+    fi
+
+    # Build for amd64 architecture (x86_64)
+    echo "Building for amd64 architecture..."
+    docker buildx build \
+        --platform linux/amd64 \
+        --output type=docker \
+        -f ${build_context}/docker/${framework_version}/Dockerfile.${processor} \
+        -t ${repository}:${version} \
+        --build-arg REGION=${REGION} \
+        -t sagemaker-spark-${use_case}:latest \
+        ${build_context} 2>&1 | tee "build_log.txt"
+else
+    # On Linux x86 or other platforms, use the original build command
+    echo "Using standard docker build..."
+    docker build \
+        -f ${build_context}/docker/${framework_version}/Dockerfile.${processor} \
+        -t ${repository}:${version} \
+        --build-arg REGION=${REGION} \
+        -t sagemaker-spark-${use_case}:latest \
+        ${build_context} 2>&1 | tee "build_log.txt"
+fi
 docker logout https://137112412989.dkr.ecr.us-west-2.amazonaws.com

smsparkbuild/py312/Pipfile

@@ -0,0 +1,42 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+black = "==25.1.0"
+boto3 = "==1.38.38"
+click = "==8.1.8"
+cryptography = "==45.0.4"
+docker = "==7.1.0"
+flake8 = "==7.2.0"
+flake8-docstrings = "==1.7.0"
+importlib-metadata = "==6.11.0"
+mypy = "==1.16.1"
+numpy = "==1.26.4"
+psutil = "==6.1.1"
+py = "==1.11.0"
+pyasn1 = "==0.6.1"
+pytest = "==8.4.0"
+pytest-cov = "==6.2.1"
+pytest-parallel = "==0.1.1"
+pytest-rerunfailures = "==15.1"
+pytest-xdist = "==3.7.0"
+pyyaml = "==6.0.2"
+regex = "==2024.11.6"
+requests = "==2.32.4"
+rsa = "==4.9.1"
+safety = "==3.5.2"
+sagemaker = "==2.247.0"
+smspark = {editable = true, path = "."}
+tenacity = "==9.1.2"
+types-requests = "==2.32.4.20250611"
+types-waitress = "==3.0.1.20241117"
+typing-extensions = "==4.14.0"
+waitress = "==3.0.2"
+watchdog = "==6.0.0"
+
+[requires]
+python_version = "3.12"