Merge branch '2.3.x' of https://github.com/pandas-dev/pandas into 2.3.x

Author: rhshadrach

.circleci/config.yml

@@ -61,7 +61,6 @@ pandas/_version.py export-subst
 *.pxi export-ignore
 
 # Ignoring stuff from the top level
-.circleci export-ignore
 .github export-ignore
 asv_bench export-ignore
 ci export-ignore

.gitattributes

@@ -7,7 +7,7 @@ runs:
       shell: bash -el {0}
 
     - name: Publish test results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: Test results
         path: test-data.xml

.github/actions/run-tests/action.yml

@@ -22,10 +22,11 @@ defaults:
 
 jobs:
   ubuntu:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.platform }}
     timeout-minutes: 90
     strategy:
       matrix:
+        platform: [ubuntu-22.04, ubuntu-24.04-arm]
         env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml, actions-312.yaml]
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
@@ -35,9 +36,11 @@ jobs:
             env_file: actions-311-downstream_compat.yaml
             pattern: "not slow and not network and not single_cpu"
             pytest_target: "pandas/tests/test_downstream.py"
+            platform: ubuntu-22.04
           - name: "Minimum Versions"
             env_file: actions-39-minimum_versions.yaml
             pattern: "not slow and not network and not single_cpu"
+            platform: ubuntu-22.04
           - name: "Locale: it_IT"
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -48,6 +51,7 @@ jobs:
             # Also install it_IT (its encoding is ISO8859-1) but do not activate it.
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "it_IT"
+            platform: ubuntu-22.04
           - name: "Locale: zh_CN"
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -58,55 +62,70 @@ jobs:
             # Also install zh_CN (its encoding is gb2312) but do not activate it.
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "zh_CN"
+            platform: ubuntu-22.04
           - name: "Copy-on-Write 3.9"
             env_file: actions-39.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
+            platform: ubuntu-22.04
           - name: "Copy-on-Write 3.10"
             env_file: actions-310.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
+            platform: ubuntu-22.04
           - name: "Copy-on-Write 3.11"
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
+            platform: ubuntu-22.04
           - name: "Copy-on-Write 3.12"
             env_file: actions-312.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
+            platform: ubuntu-22.04
           - name: "Copy-on-Write 3.11 (warnings)"
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "warn"
+            platform: ubuntu-22.04
           - name: "Copy-on-Write 3.10 (warnings)"
             env_file: actions-310.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "warn"
+            platform: ubuntu-22.04
           - name: "Copy-on-Write 3.9 (warnings)"
             env_file: actions-39.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "warn"
+            platform: ubuntu-22.04
           - name: "Future infer strings"
             env_file: actions-312.yaml
             pandas_future_infer_string: "1"
             pandas_copy_on_write: "1"
+            platform: ubuntu-22.04
           - name: "Future infer strings (without pyarrow)"
             env_file: actions-311.yaml
             pandas_future_infer_string: "1"
             pandas_copy_on_write: "1"
+            platform: ubuntu-22.04
           - name: "Pypy"
             env_file: actions-pypy-39.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "--max-worker-restart 0"
+            platform: ubuntu-22.04
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "-W error::DeprecationWarning -W error::FutureWarning"
+            platform: ubuntu-22.04
           - name: "Pyarrow Nightly"
             env_file: actions-311-pyarrownightly.yaml
             pattern: "not slow and not network and not single_cpu"
+            pandas_future_infer_string: "1"
+            pandas_copy_on_write: "1"
+            platform: ubuntu-22.04
       fail-fast: false
-    name: ${{ matrix.name || format('ubuntu-latest {0}', matrix.env_file) }}
+    name: ${{ matrix.name || format('ubuntu-latest {0}', matrix.env_file) }}-${{ matrix.platform }}
     env:
       PATTERN: ${{ matrix.pattern }}
       LANG: ${{ matrix.lang || 'C.UTF-8' }}
@@ -122,7 +141,7 @@ jobs:
       REMOVE_PYARROW: ${{ matrix.name == 'Future infer strings (without pyarrow)' && '1' || '0' }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_copy_on_write || '' }}-${{ matrix.pandas_future_infer_string }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_copy_on_write || '' }}-${{ matrix.pandas_future_infer_string }}-${{ matrix.platform }}
       cancel-in-progress: true
 
     services:
@@ -262,12 +281,14 @@ jobs:
           fi
       - name: Build environment and Run Tests
         # https://github.com/numpy/numpy/issues/24703#issuecomment-1722379388
+        # Note: Pinned to Cython 3.0.10 to avoid numerical instability in 32-bit environments
+        # https://github.com/pandas-dev/pandas/pull/61423
         run: |
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"
-          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil "pytz<2024.2" pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython==3.0.10 python-dateutil "pytz<2024.2" pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
           python -m pip list --no-cache-dir
           export PANDAS_CI=1

.github/workflows/unit-tests.yml

@@ -94,17 +94,14 @@ jobs:
         buildplat:
         - [ubuntu-22.04, manylinux_x86_64]
         - [ubuntu-22.04, musllinux_x86_64]
+        - [ubuntu-24.04-arm, manylinux_aarch64]
+        - [ubuntu-24.04-arm, musllinux_aarch64]
         - [macos-13, macosx_x86_64]
         # Note: M1 images on Github Actions start from macOS 14
         - [macos-14, macosx_arm64]
         - [windows-2022, win_amd64]
         # TODO: support PyPy?
         python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]]
-        include:
-        # TODO: Remove this plus installing build deps in cibw_before_build.sh
-        # after pandas can be built with a released NumPy/Cython
-        - python: ["cp313t", "3.13"]
-          cibw_build_frontend: 'pip; args: --no-build-isolation'
         # TODO: Build free-threaded wheels for Windows
         exclude:
         - buildplat: [windows-2022, win_amd64]
@@ -150,11 +147,10 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.21.3
+        uses: pypa/cibuildwheel@v2.23.3
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
-          CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
           CIBW_BUILD_FRONTEND: ${{ matrix.cibw_build_frontend || 'pip' }}
 

.github/workflows/wheels.yml

@@ -41,7 +41,8 @@
     // pip (with all the conda available packages installed first,
     // followed by the pip installed packages).
     "matrix": {
-        "Cython": ["3.0.5"],
+        "pip+build": [],
+        "Cython": [],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],

asv_bench/asv.conf.json

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 
@@ -15,7 +15,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=2.2.0
   - pytest-qt>=4.2.0
-  - boto3
+  - boto3=1.37.3
 
   # required dependencies
   - python-dateutil

ci/deps/actions-310.yaml

@@ -7,7 +7,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 
@@ -17,7 +17,7 @@ dependencies:
   - pytest-xdist>=2.2.0
   - pytest-localserver>=0.7.1
   - pytest-qt>=4.2.0
-  - boto3
+  - boto3=1.37.3
 
   # required dependencies
   - python-dateutil
@@ -50,8 +50,8 @@ dependencies:
   - pytables>=3.8.0
   - python-calamine>=0.1.7
   - pyxlsb>=1.0.10
-  - s3fs>=2022.11.0
-  - scipy>=1.10.0
+  - s3fs>=2023.12.2
+  - scipy>=1.12.0
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0
   - xarray>=2022.12.0, <=2024.9.0

ci/deps/actions-311-downstream_compat.yaml

@@ -8,7 +8,7 @@ dependencies:
   - versioneer
   - meson=1.2.1
   - meson-python=0.13.1
-  - cython>=0.29.33
+  - cython<4.0.0a0
 
   # test dependencies
   - pytest>=7.3.2