Upading workflow to upload assets

Author: brightsparc

.github/workflows/publish-template.yml

@@ -21,7 +21,6 @@ jobs:
         with:
           python-version: "3.8" # Version range or exact version of a Python version to use, using SemVer's version range syntax
           architecture: "x64" # optional x64 or x86. Defaults to x64 if not specified
-          cache: python
 
       - name: Setup Node 
         uses: actions/setup-node@v2
@@ -31,8 +30,9 @@ jobs:
           cache: npm
 
       - name: Install Requirements
-        run: npm install -g aws-cdk # Install cdk
-        run: pip install --requirement requirements.txt
+        run: | 
+          npm install -g aws-cdk # Install cdk
+          pip install --requirement requirements.txt
 
       - name: Configure AWS credentials
         id: creds
@@ -44,22 +44,24 @@ jobs:
           role-duration-seconds: 1200
 
       - name: Synth Template
-        id: data_prep
         env:
           BUCKET_NAME: ${{ secrets.BUCKET_NAME }}
           BUCKET_PREFIX: ${{ secrets.BUCKET_PREFIX }}
-        run: rm -Rf cdk.out/ 
         run: cdk synth drift-service-catalog --path-metadata false -c drift:ArtifactBucket=$BUCKET_NAME -c drift:ArtifactBucketPrefix=$BUCKET_PREFIX > drift-service-catalog.yml
 
-      - name: Publish Assets
-        run: # cd cdk.out; for d in asset.*/ ; do base=$(basename "$d") ; cd $base ; zip -r $base.zip * ; mv "${base}.zip" .. ; cd .. ; done; cd ..
-        run: # aws s3 sync cdk.out/*.zip s3://amazon-sagemaker-safe-deployment-pipeline/drift/ --acl public-read
+      - name: Publish Assets to S3
+        env:
+          BUCKET_NAME: ${{ secrets.BUCKET_NAME }}
+          BUCKET_PREFIX: ${{ secrets.BUCKET_PREFIX }}
+        run: python infra/upload_assets.py
+
+      - name: Upload  Template
+        uses: actions/upload-artifact@v2
+        with:
+          name: cdk-template
+          path: cdk.out/*.template.json
 
       - name: Print Template
         run: cat drift-service-catalog.yml
 
-      - name: Upload Template
-        uses: actions/upload-artifact@v2
-        with:
-          name: drift-service-catalog
-          path: drift-service-catalog.yml
+

drift-service-catalog.yml

@@ -45,7 +45,8 @@ Resources:
       ProvisioningArtifactParameters:
         - DisableTemplateValidation: false
           Info:
-            LoadTemplateFromURL: https://amazon-sagemaker-safe-deployment-pipeline.s3.amazonaws.com/drift-pipeline/drift-pipeline-clean.template.json
+            LoadTemplateFromURL:
+              Fn::Sub: https://s3.${AWS::Region}.${AWS::URLSuffix}/amazon-sagemaker-safe-deployment-pipeline/drift/995c5412ab943fce18ceaf3ebe66d305f8ab054a4a3f643768722c3fead226a0.json
           Name:
             Ref: ProductVersion
       Description: Amazon SageMaker Project for a build and deployment pipeline that triggers on drift or schedule
@@ -136,15 +137,12 @@ Resources:
                   - ""
                   - - "arn:"
                     - Ref: AWS::Partition
-                    - ":s3:::"
-                    - "amazon-sagemaker-safe-deployment-pipeline"
+                    - :s3:::amazon-sagemaker-safe-deployment-pipeline
               - Fn::Join:
                   - ""
                   - - "arn:"
                     - Ref: AWS::Partition
-                    - ":s3:::"
-                    - "amazon-sagemaker-safe-deployment-pipeline"
-                    - /*
+                    - :s3:::amazon-sagemaker-safe-deployment-pipeline/*
           - Action:
               - ssm:DescribeParameters
               - ssm:GetParameters
@@ -196,23 +194,6 @@ Resources:
                   - Ref: AWS::AccountId
                   - :parameter
                   - Ref: CodeCommitDeployKey5E5A6E47
-          - Action:
-              - ssm:DescribeParameters
-              - ssm:GetParameters
-              - ssm:GetParameter
-              - ssm:GetParameterHistory
-            Effect: Allow
-            Resource:
-              Fn::Join:
-                - ""
-                - - "arn:"
-                  - Ref: AWS::Partition
-                  - ":ssm:"
-                  - Ref: AWS::Region
-                  - ":"
-                  - Ref: AWS::AccountId
-                  - :parameter
-                  - Ref: LambdaKey984A39D9
         Version: "2012-10-17"
       PolicyName: LaunchRolePolicyA9E2E5B1
       Roles:
@@ -256,23 +237,95 @@ Resources:
     Type: AWS::SSM::Parameter
     Properties:
       Type: String
-      Value: "amazon-sagemaker-safe-deployment-pipeline"
+      Value: amazon-sagemaker-safe-deployment-pipeline
       Name: /drift-pipeline/CodeCommitSeedBucket
   CodeCommitBuildKey09FC7134:
     Type: AWS::SSM::Parameter
     Properties:
       Type: String
-      Value: "drift-pipeline/build.zip"
+      Value: drift/be4dfed42e0626f2201f8ce29518fb4e31fb640eca88a156fedc5b8d568c6c1c.zip
       Name: /drift-pipeline/CodeCommitBuildKey
   CodeCommitDeployKey5E5A6E47:
     Type: AWS::SSM::Parameter
     Properties:
       Type: String
-      Value: "drift-pipeline/deploy.zip"
+      Value: drift/381262eab49cfaf013c17daf9216232c25761d7f0ec33583681b6b699adf4ecf.zip
       Name: /drift-pipeline/CodeCommitDeployKey
-  LambdaKey984A39D9:
-    Type: AWS::SSM::Parameter
+  CDKMetadata:
+    Type: AWS::CDK::Metadata
     Properties:
-      Type: String
-      Value: "drift-pipeline/lambda.zip"
-      Name: /drift-pipeline/LambdaKey
+      Analytics: v2:deflate64:H4sIAAAAAAAAE3WQwU7EMAxEv2XvaXar5cARthKnPVTlCyzX7Zq2MUoc0Krqv5O0SBQJTvEkk3kal7YsH+zp8ASfocB2OM4onuz8qoCDqTpXg4eJlLx5DoE03ffselOJC+ojqmkoSPRI2byfk6NlZXGLydlzIP/BSAgKo/R2rsVrJyPLSvlT1F7ahEhgQYacZapRYvsiflrlt2HF/fewS2OH/A7jr7zOXSE6vDUy0lYK2OliwrmAXDjYtXfS9hJxIL1AIMMw5QYj431j5Cl9ClNeXQL1P3vbL3FZFlPf9SbueLaPtjwd3gJz4aNTnsg22/kFLKE7OZUBAAA=
+    Condition: CDKMetadataAvailable
+Conditions:
+  CDKMetadataAvailable:
+    Fn::Or:
+      - Fn::Or:
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - af-south-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - ap-east-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - ap-northeast-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - ap-northeast-2
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - ap-south-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - ap-southeast-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - ap-southeast-2
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - ca-central-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - cn-north-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - cn-northwest-1
+      - Fn::Or:
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - eu-central-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - eu-north-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - eu-south-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - eu-west-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - eu-west-2
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - eu-west-3
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - me-south-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - sa-east-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - us-east-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - us-east-2
+      - Fn::Or:
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - us-west-1
+          - Fn::Equals:
+              - Ref: AWS::Region
+              - us-west-2
+

infra/upload_assets.py

@@ -0,0 +1,123 @@
+import boto3
+import glob
+import json
+import logging
+import os
+import zipfile
+
+
+# Get environment variables
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
+BUCKET_NAME = os.getenv("BUCKET_NAME")
+BUCKET_PREFIX = os.getenv("BUCKET_PREFIX")
+BUCKET_ACL = os.getenv("BUCKET_ACL", "public-read")
+GITHUB_REF = os.getenv("GITHUB_REF")
+GITHUB_SHA = os.getenv("GITHUB_SHA")
+
+# Configure logging
+logger = logging.getLogger(__name__)
+logger.setLevel(LOG_LEVEL)
+
+# s3 client
+s3 = boto3.client("s3")
+
+
+def upload_file(
+    file_path: str, bucket_name: str, object_key: str, content_type: str
+) -> None:
+    """Upload file to s3 setting extra ags for ContentType, ACL, and Metadata for git hash
+
+    Args:
+        file_path (str): Path to local file
+        bucket_name (str): Name of bucket
+        object_key (str): Name of object key
+        content_type (str): Content type
+    """
+    logger.info(f"Uploading asset s3://{bucket_name}/{object_key}")
+    s3.upload_file(
+        file_path,
+        bucket_name,
+        object_key,
+        ExtraArgs={
+            "ContentType": content_type,
+            "ACL": BUCKET_ACL,
+            "Metadata": {"git_ref": GITHUB_REF, "git_sha": GITHUB_SHA},
+        },
+    )
+
+
+def zip_filter(filename: str) -> bool:
+    """Returns true if file and not in ignore list
+
+    Args:
+        filename (str): file name
+
+    Returns:
+        bool: True if should filter
+    """
+    return os.path.isfile(filename) and filename not in [".DS_Store"]
+
+
+def make_zipfile(source_dir: str) -> str:
+    """Makes a zip file for the sourc directory
+
+    Args:
+        source_dir (str): The source directory to zip
+
+    Returns:
+        str: Returns the zip filename created
+    """
+    output_filename = source_dir + ".zip"
+    relroot = os.path.abspath(os.path.join(source_dir, os.pardir))
+    with zipfile.ZipFile(output_filename, "w", zipfile.ZIP_DEFLATED) as zip:
+        for root, dirs, files in os.walk(source_dir):
+            # add directory (needed for empty dirs)
+            zip.write(root, os.path.relpath(root, relroot))
+            for file in files:
+                filename = os.path.join(root, file)
+                if zip_filter(filename):
+                    arcname = os.path.join(os.path.relpath(root, relroot), file)
+                    # print(root, arcname)
+                    zip.write(filename, arcname)
+    return output_filename
+
+
+def upload_assets(cdk_dir: str = "cdk.out") -> None:
+    """Parses the asset files in cdk directory and uploads resources to S3
+
+    Args:
+        cdk_dir (str): The cdk directory
+    """
+    for asset_path in glob.glob(f"{cdk_dir}/*.assets.json"):
+        logger.debug(f"Processing asset: {asset_path}")
+        with open(asset_path, "r") as f:
+            asset = json.load(f)
+        for key in asset["files"]:
+            meta = asset["files"][key]
+            # Get source info
+            src = meta["source"]
+            file_path = os.path.join(cdk_dir, src["path"])
+            content_type = "application/json"
+            if src["packaging"] == "zip":
+                logger.info(f"Packaging zip: {file_path}")
+                file_path = make_zipfile(file_path)
+                content_type = "application/zip"
+            # Get the destination
+            dest = meta["destinations"]["current_account-current_region"]
+            bucket_name = dest["bucketName"]
+            object_key = dest["objectKey"]
+            # Upload file to s3
+            upload_file(file_path, bucket_name, object_key, content_type)
+
+
+if __name__ == "__main__":
+    ch = logging.StreamHandler()
+    ch.setFormatter(logging.Formatter("%(levelname)s - %(message)s"))
+    logger.addHandler(ch)
+    logger.info(f"Uploading assets for git ref: {GITHUB_REF} sha: {GITHUB_SHA}")
+    # Upload YAML template
+    template_name = "drift-service-catalog.yml"
+    object_key = f"{BUCKET_PREFIX}{template_name}"
+    upload_file(template_name, BUCKET_NAME, object_key, "application/x-yaml")
+    # Upload assets
+    upload_assets("cdk.out")