diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 349bbeb4b..7702d3d1a 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -33,6 +33,25 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} jobs: + define-matrix: + runs-on: "ubuntu-latest" + outputs: + test_matrix: ${{ steps.test_matrix.outputs.test_matrix }} + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Define testing matrix + id: test_matrix + run: | + if [ "${{ github.event_name }}" = "push" ]; then + echo "Using full matrix" + echo "test_matrix=$(jq -c . < ./.github/workflows/generated_full_matrix.json)" >> "$GITHUB_OUTPUT" + else + echo "Using PR matrix" + echo "test_matrix=$(jq -c . < ./.github/workflows/generated_pr_matrix.json)" >> "$GITHUB_OUTPUT" + fi lint: name: Check linting runs-on: ubuntu-latest @@ -59,10 +78,11 @@ jobs: dependency: name: Check dependency + needs: define-matrix runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + include: ${{ fromJSON(needs.define-matrix.outputs.test_matrix) }} steps: - uses: actions/checkout@v4 - name: Set up Python @@ -77,34 +97,12 @@ jobs: run: python -m tox run -e dependency build: - needs: lint + needs: [lint, define-matrix] strategy: matrix: - os: - - image: ubuntu-latest - id: manylinux_x86_64 - - image: ubuntu-latest - id: manylinux_aarch64 - - image: windows-latest - id: win_amd64 - - image: windows-11-arm - id: win_arm64 - - image: macos-latest - id: macosx_x86_64 - - image: macos-latest - id: macosx_arm64 - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - exclude: - - os: - image: windows-11-arm - id: win_arm64 - python-version: "3.9" - - os: - image: windows-11-arm - id: win_arm64 - python-version: "3.10" - name: Build ${{ matrix.os.id }}-py${{ matrix.python-version }} - runs-on: ${{ matrix.os.image }} + include: ${{ fromJSON(needs.define-matrix.outputs.test_matrix) }} + name: Build ${{ matrix.download_name }}-py${{ matrix.python-version }} + runs-on: ${{ matrix.os }} steps: - name: Set shortver run: echo "shortver=${longver//./}" >> $GITHUB_ENV @@ -112,7 +110,7 @@ jobs: longver: ${{ matrix.python-version }} shell: bash - name: Set up QEMU - if: ${{ matrix.os.id == 'manylinux_aarch64' }} + if: ${{ matrix.download_name == 'manylinux_aarch64' }} uses: docker/setup-qemu-action@v2 with: # xref https://github.com/docker/setup-qemu-action/issues/188 @@ -123,8 +121,8 @@ jobs: - name: Building wheel uses: pypa/cibuildwheel@v2.21.3 env: - CIBW_BUILD: cp${{ env.shortver }}-${{ matrix.os.id }} - CIBW_ARCHS_WINDOWS: ${{ matrix.os.id == 'win_arm64' && 'ARM64' || 'auto' }} + CIBW_BUILD: cp${{ env.shortver }}-${{ matrix.download_name }} + CIBW_ARCHS_WINDOWS: ${{ matrix.download_name == 'win_arm64' && 'ARM64' || 'auto' }} MACOSX_DEPLOYMENT_TARGET: 10.14 # Should be kept in sync with ci/build_darwin.sh with: output-dir: dist @@ -134,49 +132,17 @@ jobs: - uses: actions/upload-artifact@v4 with: include-hidden-files: true - name: ${{ matrix.os.id }}_py${{ matrix.python-version }} + name: ${{ matrix.download_name }}_py${{ matrix.python-version }} path: dist/ test: - name: Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} - needs: build - runs-on: ${{ matrix.os.image_name }} + name: Test ${{ matrix.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} + needs: [build, define-matrix] + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: - - image_name: ubuntu-latest - download_name: manylinux_x86_64 - - image_name: macos-latest - download_name: macosx_x86_64 - - image_name: windows-latest - download_name: win_amd64 - - image_name: windows-11-arm - download_name: win_arm64 - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - cloud-provider: [aws, azure, gcp] - exclude: - - os: - image_name: windows-11-arm - download_name: win_arm64 - python-version: "3.9" - - os: - image_name: windows-11-arm - download_name: win_arm64 - python-version: "3.10" - - os: - image_name: windows-11-arm - download_name: win_arm64 - python-version: "3.11" - - os: - image_name: windows-11-arm - download_name: win_arm64 - python-version: "3.12" - - os: - image_name: windows-11-arm - download_name: win_arm64 - python-version: "3.13" - + include: ${{ fromJSON(needs.define-matrix.outputs.test_matrix) }} steps: - uses: actions/checkout@v4 - name: Set up Python @@ -188,7 +154,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v4 # for wiremock with: - java-version: ${{ matrix.os.download_name == 'win_arm64' && '21.0.5+11.0.LTS' || '11' }} + java-version: ${{ matrix.download_name == 'win_arm64' && '21.0.5+11.0.LTS' || '11' }} distribution: 'temurin' java-package: 'jre' - name: Fetch Wiremock @@ -211,7 +177,7 @@ jobs: - name: Download wheel(s) uses: actions/download-artifact@v4 with: - name: ${{ matrix.os.download_name }}_py${{ matrix.python-version }} + name: ${{ matrix.download_name }}_py${{ matrix.python-version }} path: dist - name: Show wheels downloaded run: ls -lh dist @@ -236,7 +202,7 @@ jobs: if: always() with: include-hidden-files: true - name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} + name: coverage_${{ matrix.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} path: | .tox/.coverage .tox/coverage.xml @@ -244,7 +210,7 @@ jobs: if: always() with: include-hidden-files: true - name: junit_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} + name: junit_${{ matrix.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} path: | .tox/junit.*.xml @@ -391,8 +357,10 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - cloud-provider: [aws] + include: ${{ fromJSON(needs.define-matrix.outputs.test_matrix) }} + exclude: + - cloud-provider: gcp + - cloud-provider: azure steps: - name: Set shortver run: echo "shortver=${longver//./}" >> $GITHUB_ENV diff --git a/.github/workflows/generated_full_matrix.json b/.github/workflows/generated_full_matrix.json new file mode 100644 index 000000000..d8791916e --- /dev/null +++ b/.github/workflows/generated_full_matrix.json @@ -0,0 +1,326 @@ +[ + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.9", + "cloud-provider": "aws" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.9", + "cloud-provider": "azure" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.9", + "cloud-provider": "gcp" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.10", + "cloud-provider": "aws" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.10", + "cloud-provider": "azure" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.10", + "cloud-provider": "gcp" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.11", + "cloud-provider": "aws" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.11", + "cloud-provider": "azure" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.11", + "cloud-provider": "gcp" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.12", + "cloud-provider": "aws" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.12", + "cloud-provider": "azure" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.12", + "cloud-provider": "gcp" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.13", + "cloud-provider": "aws" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.13", + "cloud-provider": "azure" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.13", + "cloud-provider": "gcp" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.9", + "cloud-provider": "aws" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.9", + "cloud-provider": "azure" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.9", + "cloud-provider": "gcp" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.10", + "cloud-provider": "aws" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.10", + "cloud-provider": "azure" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.10", + "cloud-provider": "gcp" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.11", + "cloud-provider": "aws" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.11", + "cloud-provider": "azure" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.11", + "cloud-provider": "gcp" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.12", + "cloud-provider": "aws" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.12", + "cloud-provider": "azure" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.12", + "cloud-provider": "gcp" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.13", + "cloud-provider": "aws" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.13", + "cloud-provider": "azure" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.13", + "cloud-provider": "gcp" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.9", + "cloud-provider": "aws" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.9", + "cloud-provider": "azure" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.9", + "cloud-provider": "gcp" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.10", + "cloud-provider": "aws" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.10", + "cloud-provider": "azure" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.10", + "cloud-provider": "gcp" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.11", + "cloud-provider": "aws" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.11", + "cloud-provider": "azure" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.11", + "cloud-provider": "gcp" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.12", + "cloud-provider": "aws" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.12", + "cloud-provider": "azure" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.12", + "cloud-provider": "gcp" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.13", + "cloud-provider": "aws" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.13", + "cloud-provider": "azure" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.13", + "cloud-provider": "gcp" + }, + { + "os": "windows-11-arm", + "download_name": "win_arm64", + "python-version": "3.11", + "cloud-provider": "aws" + }, + { + "os": "windows-11-arm", + "download_name": "win_arm64", + "python-version": "3.11", + "cloud-provider": "azure" + }, + { + "os": "windows-11-arm", + "download_name": "win_arm64", + "python-version": "3.11", + "cloud-provider": "gcp" + }, + { + "os": "windows-11-arm", + "download_name": "win_arm64", + "python-version": "3.12", + "cloud-provider": "aws" + }, + { + "os": "windows-11-arm", + "download_name": "win_arm64", + "python-version": "3.12", + "cloud-provider": "azure" + }, + { + "os": "windows-11-arm", + "download_name": "win_arm64", + "python-version": "3.12", + "cloud-provider": "gcp" + }, + { + "os": "windows-11-arm", + "download_name": "win_arm64", + "python-version": "3.13", + "cloud-provider": "aws" + }, + { + "os": "windows-11-arm", + "download_name": "win_arm64", + "python-version": "3.13", + "cloud-provider": "azure" + }, + { + "os": "windows-11-arm", + "download_name": "win_arm64", + "python-version": "3.13", + "cloud-provider": "gcp" + } +] \ No newline at end of file diff --git a/.github/workflows/generated_pr_matrix.json b/.github/workflows/generated_pr_matrix.json new file mode 100644 index 000000000..9c5e104b8 --- /dev/null +++ b/.github/workflows/generated_pr_matrix.json @@ -0,0 +1,44 @@ +[ + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.9", + "cloud-provider": "aws" + }, + { + "os": "ubuntu-latest", + "download_name": "manylinux_x86_64", + "python-version": "3.13", + "cloud-provider": "aws" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.9", + "cloud-provider": "azure" + }, + { + "os": "macos-latest", + "download_name": "macosx_x86_64", + "python-version": "3.13", + "cloud-provider": "azure" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.9", + "cloud-provider": "gcp" + }, + { + "os": "windows-latest", + "download_name": "win_amd64", + "python-version": "3.13", + "cloud-provider": "gcp" + }, + { + "os": "windows-11-arm", + "download_name": "win_arm64", + "python-version": "3.13", + "cloud-provider": "aws" + } +] \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 92aca15ef..7eec28410 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,7 @@ repos: (?x)^( license_header.txt| src/snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/.*\.h| + \.github/workflows/generated_.*| )$ - id: check-yaml exclude: > @@ -71,6 +72,15 @@ repos: files: ^src/snowflake/connector/.*\.py$ exclude: src/snowflake/connector/options.py args: [--show-fixes] + - id: check-ci-matrix + name: Check if CI matrix changed + entry: python ci/generate_full_matrix.py + language: system + files: | + (?x)^( + ci/generate_full_matrix\.py| + \.github/workflows/generated_.* + )$ - repo: https://github.com/PyCQA/flake8 rev: 7.1.1 hooks: diff --git a/ci/generate_full_matrix.py b/ci/generate_full_matrix.py new file mode 100755 index 000000000..5201265b2 --- /dev/null +++ b/ci/generate_full_matrix.py @@ -0,0 +1,180 @@ +""" +Generate generated_full_matrix.json and generated_pr_matrix.json for GitHub Actions workflows. + +Usage: + python ci/generate_full_matrix.py + +To customize, edit the configuration constants below and run the script. +""" + +import json +from dataclasses import dataclass +from enum import Enum +from itertools import product +from pathlib import Path +from typing import List, Tuple + +# ============================================================================ +# CONFIGURATION - Edit these to customize the matrix +# ============================================================================ + + +@dataclass +class PythonVersion: + """Python version configuration.""" + + version: str + test_on_pr: bool = False + + +@dataclass +class OperatingSystemInfo: + """Operating system configuration.""" + + name: str # GitHub Actions runner image (e.g., "ubuntu-latest") + download_name: str # Artifact download name (e.g., "manylinux_x86_64") + + +class OperatingSystem(Enum): + """Available operating systems with their build configurations.""" + + UBUNTU = OperatingSystemInfo( + name="ubuntu-latest", + download_name="manylinux_x86_64", + ) + MACOS = OperatingSystemInfo( + name="macos-latest", + download_name="macosx_x86_64", + ) + WINDOWS = OperatingSystemInfo( + name="windows-latest", + download_name="win_amd64", + ) + WINDOWS_ARM = OperatingSystemInfo( + name="windows-11-arm", + download_name="win_arm64", + ) + + +class Python(Enum): + """Available Python versions.""" + + PY39 = PythonVersion("3.9", test_on_pr=True) + PY310 = PythonVersion("3.10", test_on_pr=False) + PY311 = PythonVersion("3.11", test_on_pr=False) + PY312 = PythonVersion("3.12", test_on_pr=False) + PY313 = PythonVersion("3.13", test_on_pr=True) + + +class CSP(Enum): + """Available cloud service providers.""" + + AWS = "aws" + AZURE = "azure" + GCP = "gcp" + + +# OS-Python combinations to exclude from all matrices +# Format: (os_name, python_version) +EXCLUSIONS: List[Tuple[str, str]] = [ + # Windows 11 ARM doesn't support Python 3.9 and 3.10 + ("windows-11-arm", "3.9"), + ("windows-11-arm", "3.10"), +] + +# Additional fields to add to each matrix entry (optional) +# Example: {"with_snowpark": "true"} +ADDITIONAL_FIELDS = {} + +# Output file paths (relative to repository root) +_WORKFLOWS_DIR = Path(__file__).parent.parent / ".github" / "workflows" +FULL_MATRIX_FILE = _WORKFLOWS_DIR / "generated_full_matrix.json" +PR_MATRIX_FILE = _WORKFLOWS_DIR / "generated_pr_matrix.json" + +# JSON indentation +INDENT = 2 + +# ============================================================================ +# MATRIX GENERATION - No need to edit below this line +# ============================================================================ + + +def _add_to_matrix( + matrix: list[dict], os: OperatingSystemInfo, csp_name: str, py_config: PythonVersion +): + if (os.name, py_config.version) in EXCLUSIONS: + return + + entry = { + "os": os.name, + "download_name": os.download_name, + "python-version": py_config.version, + "cloud-provider": csp_name, + } + + # Add any additional fields + if ADDITIONAL_FIELDS: + entry.update(ADDITIONAL_FIELDS) + + matrix.append(entry) + + +def generate_matrix(pr_only: bool = False): + matrix = [] + + if pr_only: + csp_to_test = list(CSP) + for system in OperatingSystem: + os_config = system.value + csp_name = csp_to_test.pop(0).value if csp_to_test else CSP.AWS.value + for py_version in Python: + if py_version.value.test_on_pr: + _add_to_matrix(matrix, os_config, csp_name, py_version.value) + else: + operating_systems = [os_enum.value for os_enum in OperatingSystem] + python_versions = [py_enum.value for py_enum in Python] + cloud_providers = [csp_enum.value for csp_enum in CSP] + + for os_config, py_config, csp_name in product( + operating_systems, python_versions, cloud_providers + ): + _add_to_matrix(matrix, os_config, csp_name, py_config) + return matrix + + +def write_matrix(matrix: List[dict], output_file: Path) -> Path: + output_file.write_text(json.dumps(matrix, indent=INDENT)) + return output_file.resolve() + + +def main(): + """Generate and write both full and PR matrix files.""" + print("Generating GitHub Actions test matrices...") + print("=" * 70) + + # Generate full matrix (all combinations) + print("\nšŸ”Ø Generating FULL matrix (all OS Ɨ Python Ɨ CSP)...") + full_matrix = generate_matrix(pr_only=False) + full_path = write_matrix(full_matrix, FULL_MATRIX_FILE) + print(f"āœ“ Written to: {full_path}") + + # Generate PR matrix (strategic pairings) + print("\nšŸ”Ø Generating PR matrix (strategic OS-CSP pairings Ɨ PR Python)...") + pr_matrix = generate_matrix(pr_only=True) + pr_path = write_matrix(pr_matrix, PR_MATRIX_FILE) + print(f"āœ“ Written to: {pr_path}") + + # Final summary + print(f"\n{'='*70}") + print("āœ… Successfully generated both matrices") + print(f"{'='*70}") + print(f" Full matrix: {len(full_matrix):2d} combinations") + print( + f" PR matrix: {len(pr_matrix):2d} combinations (saves {len(full_matrix) - len(pr_matrix)} jobs)" + ) + print(f" Exclusions: {len(EXCLUSIONS):2d}") + print(f"{'='*70}\n") + + +if __name__ == "__main__": + main()