Merge remote-tracking branch 'upstream/main' into ci-pyarrow-no-infer-string

Author: jorisvandenbossche

.github/actions/setup-conda/action.yml

@@ -10,25 +10,22 @@ on:
       - main
       - 3.0.x
 
-env:
-  ENV_FILE: environment.yml
+defaults:
+  run:
+    shell: bash -euox pipefail {0}
 
 permissions: {}
 
-# pre-commit run by https://pre-commit.ci/
 jobs:
-  docstring_typing_manual_hooks:
-    name: Docstring validation, typing, and other manual pre-commit hooks
+  doctest:
+    name: Doctests
     runs-on: ubuntu-24.04
     permissions:
       contents: read
-    defaults:
-      run:
-        shell: bash -el {0}
 
     concurrency:
       # https://github.blesdmm.dns-dynamic.netmunity/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-code-checks
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-doctests
       cancel-in-progress: true
 
     steps:
@@ -37,51 +34,63 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Set up Conda
-      uses: ./.github/actions/setup-conda
-
-    - name: Build Pandas
-      id: build
-      uses: ./.github/actions/build_pandas
+    - name: Create virtual environment with Pixi
+      uses: ./.github/actions/setup-pixi
+      with:
+        environment: "doctests"
 
-    # The following checks are independent of each other and should still be run if one fails
-    - name: Extra installs
-      # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd
-      run: sudo apt-get update && sudo apt-get install -y libegl1 libopengl0
+    - name: Build pandas
+      run: |
+        pixi run \
+          --environment doctests \
+          build-pandas \
+          --editable \
+          -Csetup-args="--werror"
 
     - name: Run doctests
-      run: cd ci && ./code_checks.sh doctests
-      if: ${{ steps.build.outcome == 'success' && always() }}
+      run: pixi run --environment doctests ci-doctests
 
-    - name: Run checks on imported code
-      run: ci/code_checks.sh code
-      if: ${{ steps.build.outcome == 'success' && always() }}
+    - name: Run scripts tests
+      run: pixi run --environment doctests ci-scripts-tests
 
-    - name: Use existing environment for type checking
-      run: |
-        echo $PATH >> $GITHUB_PATH
-        echo "PYTHONHOME=$PYTHONHOME" >> $GITHUB_ENV
-        echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV
-      if: ${{ steps.build.outcome == 'success' && always() }}
+  type-checking:
+    name: Type Checking
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+
+    concurrency:
+      # https://github.blesdmm.dns-dynamic.netmunity/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-typechecking
+      cancel-in-progress: true
 
-    - name: Typing
-      uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
+    steps:
+    - name: Checkout
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
       with:
-        extra_args: --verbose --hook-stage manual --all-files
-      if: ${{ steps.build.outcome == 'success' && always() }}
+        fetch-depth: 0
 
-    - name: Run docstring validation script tests
-      run: pytest scripts
-      if: ${{ steps.build.outcome == 'success' && always() }}
+    - name: Create virtual environment with Pixi
+      uses: ./.github/actions/setup-pixi
+      with:
+        environment: "typing"
+
+    - name: Build pandas
+      run: |
+        pixi run \
+          --environment typing \
+          build-pandas \
+          --editable \
+          -Csetup-args="--werror"
+
+    - name: Typing Checks
+      run: pixi run --environment typing ci-typing
 
   asv-benchmarks:
     name: ASV Benchmarks
     runs-on: ubuntu-24.04
     permissions:
       contents: read
-    defaults:
-      run:
-        shell: bash -euox pipefail {0}
 
     concurrency:
       # https://github.blesdmm.dns-dynamic.netmunity/t/concurrecy-not-work-for-push/183068/7
@@ -124,19 +133,15 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
-        with:
-          fetch-depth: 0
 
       - name: Setup Python
         id: setup_python
         uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
         with:
           python-version: '3.11'
           cache: 'pip'
+          pip-install: '-r requirements-dev.txt'
           cache-dependency-path: 'requirements-dev.txt'
 
-      - name: Install requirements-dev.txt
-        run: pip install -r requirements-dev.txt
-
       - name: Check Pip Cache Hit
         run: echo ${{ steps.setup_python.outputs.cache-hit }}

.github/workflows/code-checks.yml

@@ -27,8 +27,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
-      - uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
+      - uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7 # v4
         with:
           languages: ${{ matrix.language }}
-      - uses: github/codeql-action/autobuild@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
-      - uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
+      - uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7 # v4
+      - uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7 # v4

.github/workflows/codeql.yml

@@ -167,6 +167,13 @@ jobs:
           ${{ ! (matrix.name == 'Freethreading' || matrix.name == 'Minimum Versions') && '-Csetup-args="--werror"' || '' }}
       shell: bash -euox pipefail {0}
 
+    - name: Import check
+      run: |
+        pixi run \
+          --environment ${{ matrix.environment }} \
+          ci-check-import
+      shell: bash -euox pipefail {0}
+
     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
       id: test-not-single-cpu

.github/workflows/unit-tests.yml

@@ -30,23 +30,22 @@ jobs:
         manifest-path: pixi.toml
         run-install: false
 
-    - name: Generate lock files
-      id: check-lock-file
-      run: pixi update
+    - name: Update lock files
+      run: |
+        echo -e "Scheduled update by \`.github/workflows/update-pixi-lock.yml\`.\n\nClose and re-open the PR to trigger CI. If CI jobs fail, modify this pull request as needed.\n\n" > diff.md
+        pixi update --json | pixi exec pixi-diff-to-markdown >> diff.md
 
     # There should be an update since pixi.toml references nightly packages
     # or use https://github.com/prefix-dev/pixi/issues/5813
     - name: Create pull request
       uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
       with:
         commit-message: Update pixi.lock
+        add-paths: pixi.lock
         # author same as default committer for this action
         author: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>"
         branch: "ci/update-pixi-lock"
         delete-branch: true
         title: "CI: Update pixi.lock"
-        body: |
-          Scheduled update by `.github/workflows/update-pixi-lock.yml`
-
-          If CI jobs fail, modify this pull request as needed.
+        body-path: diff.md
         labels: "CI"

.github/workflows/update-pixi-lock.yml

@@ -24,6 +24,7 @@
 tags
 .cache/
 .vscode/
+.pixi/
 
 # Compiled source #
 ###################

.gitignore

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.9
+    rev: v0.15.12
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -67,7 +67,7 @@ repos:
     -   id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
 -   repo: https://github.com/PyCQA/isort
-    rev: 8.0.1
+    rev: 9.0.0a3
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
@@ -92,14 +92,14 @@ repos:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v22.1.2
+    rev: v22.1.4
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include
       args: ['-i', '--style', '{IndentPPDirectives: AfterHash}']
       types_or: [c, c++]
 -   repo: https://github.com/trim21/pre-commit-mirror-meson
-    rev: v1.10.2
+    rev: v1.11.1
     hooks:
     - id: meson-fmt
       args: ['--inplace']
@@ -132,7 +132,7 @@ repos:
     -   id: mypy
         # note: assumes python env is setup and activated
         name: mypy
-        entry: mypy
+        entry: python -m mypy
         language: system
         pass_filenames: false
         types: [python]

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Cassio Neri <cassio.neri@gmail.com>
+Copyright (c) 2022 Lorenz Schneider <schneider@em-lyon.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LICENSES/NERI_SCHNEIDER_LICENSE

@@ -8,13 +8,15 @@
     "show_commit_url": "https://github.com/pandas-dev/pandas/commit/",
     "matrix": {
         "pip+build": [],
+        "versioneer[toml]": [],
         "Cython": [],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],
         "numba": [],
         "numexpr": [],
-        "pytables": [],
+        "pytables": [null, ""],
+        "tables": [null, ""],
         "pyarrow": [],
         "openpyxl": [],
         "xlsxwriter": [],
@@ -24,11 +26,17 @@
         "meson-python": [],
         "python-build": []
     },
+    "exclude": [
+        {"environment_type": "conda|rattler", "tables": ""},
+        {"environment_type": "conda|rattler", "pytables": null},
+        {"environment_type": "(?!(?:conda|rattler)).*", "tables": null},
+        {"environment_type": "(?!(?:conda|rattler)).*", "pytables": ""}
+    ],
     "conda_channels": ["conda-forge"],
     "build_cache_size": 8,
     "regressions_first_commits": {
         ".*": "0409521665"
     },
     "build_command":
-    ["python -m build -Cbuilddir=builddir --wheel --outdir {build_cache_dir} {build_dir}"]
+    ["python -m build -Cbuilddir=builddir --no-isolation --wheel --outdir {build_cache_dir} {build_dir}"]
 }