SK-2392 Migrate V2 code into main branch

.github/workflows/beta-release.yml

@@ -0,0 +1,18 @@
+name: Public Beta Release
+
+on:
+  push:
+    tags: '*.*.*b*'
+    paths-ignore:
+      - "setup.py"
+      - "*.yml"
+      - "*.md"
+      - "skyflow/utils/_version.py"
+
+jobs:
+  build-and-deploy:
+    uses: ./.github/workflows/shared-build-and-deploy.yml
+    with:
+      ref: ${{ github.ref_name }}
+      tag: 'beta'
+    secrets: inherit

.github/workflows/ci.yml

@@ -16,47 +16,9 @@ jobs:
           checkAllCommitMessages: 'true'
           accessToken: ${{ secrets.PAT_ACTIONS }}
           error: 'One of your your commit messages is not matching the format with JIRA ID Ex: ( SDK-123 commit message )'
-  Test:
-    runs-on: ubuntu-latest
-    steps:   
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: '3.8'
-
-    - name: create-json
-      id: create-json
-      uses: jsdaniell/create-json@1.1.2
-      with:
-        name: "credentials.json"
-        json: ${{ secrets.CREDENTIALS_FILE_STR }}
-
-    - name: 'Setup .env'
-      run: |
-        touch .env
-        echo VAULT_ID=${{secrets.VAULT_ID}} >> .env
-        echo VAULT_URL=${{secrets.VAULT_URL}} >> .env
-        echo DETOKENIZE_TEST_TOKEN=${{secrets.DETOKENIZE_TEST_TOKEN}} >> .env
-        echo DETOKENIZE_TEST_VALUE=${{secrets.DETOKENIZE_TEST_VALUE}} >> .env
-        echo CREDENTIALS_FILE_PATH=./credentials.json >> .env
-        echo CVV_GEN_CONNECTION_URL=${{secrets.CVV_GEN_CONNECTION_URL}} >> .env
-        echo VISA_CONNECTION_BASIC_AUTH=${{secrets.VISA_CONNECTION_BASIC_AUTH}} >> .env
-        echo SKYFLOW_ID1=${{secrets.SKYFLOW_ID1}} >> .env
-        echo SKYFLOW_ID2=${{secrets.SKYFLOW_ID2}} >> .env
-        echo SKYFLOW_ID3=${{secrets.SKYFLOW_ID3}} >> .env
-        echo MOCK_TOKEN=${{secrets.MOCK_TOKEN}} >> .env
 
-    - name: 'Run Tests'
-      run: |
-        python -m pip install --upgrade pip
-        pip install requests pyjwt datetime aiohttp cryptography python-dotenv coverage
-        coverage run --source skyflow -m unittest discover
-    - name: coverage
-      run: coverage xml -o test-coverage.xml
-    - name: Codecov
-      uses: codecov/codecov-action@v2.1.0
-      with:
-        token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
-        files: test-coverage.xml
-        name: codecov-skyflow-python
-        verbose: true
+  test:
+    uses: ./.github/workflows/shared-tests.yml
+    with:
+      python-version: '3.8'
+    secrets: inherit

.github/workflows/internal-release.yml

@@ -8,7 +8,7 @@ on:
       - "setup.py"
       - "*.yml"
       - "*.md"
-      - "skyflow/version.py"
+      - "skyflow/utils/_version.py"
       - "samples/**"
     branches:
       - release/*
@@ -19,4 +19,5 @@ jobs:
     with:
       ref: ${{ github.ref_name }}
       tag: 'internal'
-    secrets: inherit
+    secrets: inherit
+

.github/workflows/main.yml

@@ -6,47 +6,7 @@ on:
       - main
 
 jobs:
-  Test:
-    runs-on: ubuntu-latest
-    steps:   
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: '3.8'
-
-    - name: create-json
-      id: create-json
-      uses: jsdaniell/create-json@1.1.2
-      with:
-        name: "credentials.json"
-        json: ${{ secrets.CREDENTIALS_FILE_STR }}
-
-    - name: 'Setup .env'
-      run: |
-        touch .env
-        echo VAULT_ID=${{secrets.VAULT_ID}} >> .env
-        echo VAULT_URL=${{secrets.VAULT_URL}} >> .env
-        echo DETOKENIZE_TEST_TOKEN=${{secrets.DETOKENIZE_TEST_TOKEN}} >> .env
-        echo DETOKENIZE_TEST_VALUE=${{secrets.DETOKENIZE_TEST_VALUE}} >> .env
-        echo CREDENTIALS_FILE_PATH=./credentials.json >> .env
-        echo CVV_GEN_CONNECTION_URL=${{secrets.CVV_GEN_CONNECTION_URL}} >> .env
-        echo VISA_CONNECTION_BASIC_AUTH=${{secrets.VISA_CONNECTION_BASIC_AUTH}} >> .env
-        echo SKYFLOW_ID1=${{secrets.SKYFLOW_ID1}} >> .env
-        echo SKYFLOW_ID2=${{secrets.SKYFLOW_ID2}} >> .env
-        echo SKYFLOW_ID3=${{secrets.SKYFLOW_ID3}} >> .env
-        echo MOCK_TOKEN=${{secrets.MOCK_TOKEN}} >> .env
-
-    - name: 'Run Tests'
-      run: |
-        python -m pip install --upgrade pip
-        pip install requests pyjwt datetime aiohttp cryptography python-dotenv coverage
-        coverage run --source skyflow -m unittest discover
-    - name: coverage
-      run: coverage xml -o test-coverage.xml
-    - name: Codecov
-      uses: codecov/codecov-action@v2.1.0
-      with:
-        token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
-        files: test-coverage.xml
-        name: codecov-skyflow-python
-        verbose: true
+  test:
+    uses: ./.github/workflows/shared-tests.yml
+    with:
+      python-version: '3.8'

.github/workflows/release.yml

@@ -7,47 +7,12 @@ on:
       - "setup.py"
       - "*.yml"
       - "*.md"
-      - "skyflow/version.py"
+      - "skyflow/utils/_version.py"
 
 jobs:
   build-and-deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.PAT_ACTIONS }}
-          ref: main
-          fetch-depth: 0
-      - uses: actions/setup-python@v2
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-
-      - name: Get Previous tag
-        id: previoustag
-        uses: WyriHaximus/github-action-get-previous-tag@v1
-        with:
-          fallback: 1.0.0
-
-      - name: Bump Version
-        run: |
-          chmod +x ./ci-scripts/bump_version.sh
-          ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}"
-
-      - name: Commit changes
-        run: |
-          git config user.name ${{ github.actor }}
-          git config user.email ${{ github.actor }}@users.noreply.github.com
-          git add setup.py
-          git add skyflow/version.py
-          git commit -m "[AUTOMATED] Public Release - ${{ steps.previoustag.outputs.tag }}"
-          git push origin
-
-      - name: Build and publish
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_TOKEN }}
-        run: |
-          python setup.py sdist bdist_wheel
-          twine upload dist/*
+    uses: ./.github/workflows/shared-build-and-deploy.yml
+    with:
+      ref: main
+      tag: 'public'
+    secrets: inherit

.github/workflows/semgrep.yml

@@ -1,64 +1,35 @@
-name: Semgrep Scan
+name: Semgrep
 
+# Run workflow each time code is pushed to your repository.
 on:
+  push:
+    branches:
+      - main
   pull_request:
     branches:
       - main
-
 jobs:
   build:
     runs-on: ubuntu-latest
-    permissions:
-      pull-requests: write # Give write permission to PRs
-      issues: write
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
-      - name: Install Semgrep and jq
-        run: |
-          sudo apt install python3-venv jq
-          python3 -m venv .venv
-          .venv/bin/pip install semgrep
+      - name: Install Semgrep
+        run: pip install semgrep
 
       - name: Run Semgrep
         run: |
-          source .venv/bin/activate
-          semgrep --config auto --severity ERROR --json-output=results.json --no-error
-          cat results.json | jq .results > pretty-results.json
+          semgrep --config .semgreprules/customRule.yml --config auto --severity ERROR --sarif . > results.sarif
 
-      - name: Display Raw Semgrep JSON Output
-        run: |
-         echo "Displaying raw Semgrep results..."
-         cat pretty-results.json
-
-      - name: Add comment on PR if findings are found
-        uses: actions/github-script@v6
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@v3
         with:
-          script: |
-            // Ensure the context has a pull_request
-            if (context.payload.pull_request) {
-              const prNumber = context.payload.pull_request.number;
-              const fs = require('fs');
-              const results = JSON.parse(fs.readFileSync('pretty-results.json', 'utf8'));
-              const highFindings = results.filter(result => result.extra && result.extra.severity === 'ERROR');
+          # Path to SARIF file relative to the root of the repository
+          sarif_file: results.sarif
 
-              // Comment if findings exist
-              if (highFindings.length > 0) {
-                const comment = `**Semgrep Findings:** Issues with Error level severity are found (Error is Highest severity in Semgrep), Please resolve the issues before merging.`;
-                await github.rest.issues.createComment({
-                  ...context.repo,
-                  issue_number: prNumber,
-                  body: comment
-                });
-              } else {
-                const noIssuesComment = "**Semgrep findings:** No issues found, Good to merge.";
-                await github.rest.issues.createComment({
-                  ...context.repo,
-                  issue_number: prNumber,
-                  body: noIssuesComment
-                });
-              }
-            } else {
-              console.log("This workflow wasn't triggered by a pull request, so no comment will be added.");
-            }
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        with:
+          name: semgrep-results
+          path: results.sarif