[SPARK-55191][INFRA][DOCS] Adjust the Python version used in the Python-only daily test

### What changes were proposed in this pull request?
This pr aims to adjust the Python version used in the Python-only daily test:
1. Adjust the daily tests using `python_hosted_runner_test.yml` to Python 3.12, including the Python ARM test and Python macOS 26 test.
2. Update the Python classic-only test to Python 3.12 and modify the Dockerfile generation process.
3. Update the Python-only test from Python 3.12 to Python 3.11 to ensure at least one Python-only daily test is validating Python 3.11.

### Why are the changes needed?
Adjust the Python version used in the Python-only daily test

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
- Pass Github Actions

### Was this patch authored or co-authored using generative AI tooling?
No

Closes #53973 from LuciferYang/SPARK-55191.

Authored-by: yangjie01 <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>

Author: LuciferYang

.github/workflows/build_infra_images_cache.yml

@@ -36,8 +36,8 @@ on:
     - 'dev/spark-test-image/pypy-311/Dockerfile'
     - 'dev/spark-test-image/python-310/Dockerfile'
     - 'dev/spark-test-image/python-311/Dockerfile'
-    - 'dev/spark-test-image/python-311-classic-only/Dockerfile'
     - 'dev/spark-test-image/python-312/Dockerfile'
+    - 'dev/spark-test-image/python-312-classic-only/Dockerfile'
     - 'dev/spark-test-image/python-312-pandas-3/Dockerfile'
     - 'dev/spark-test-image/python-313/Dockerfile'
     - 'dev/spark-test-image/python-314/Dockerfile'
@@ -193,19 +193,19 @@ jobs:
       - name: Image digest (PySpark with Python 3.11)
         if: hashFiles('dev/spark-test-image/python-311/Dockerfile') != ''
         run: echo ${{ steps.docker_build_pyspark_python_311.outputs.digest }}
-      - name: Build and push (PySpark Classic Only with Python 3.11)
-        if: hashFiles('dev/spark-test-image/python-311-classic-only/Dockerfile') != ''
-        id: docker_build_pyspark_python_311_classic_only
+      - name: Build and push (PySpark Classic Only with Python 3.12)
+        if: hashFiles('dev/spark-test-image/python-312-classic-only/Dockerfile') != ''
+        id: docker_build_pyspark_python_312_classic_only
         uses: docker/build-push-action@v6
         with:
-          context: ./dev/spark-test-image/python-311-classic-only/
+          context: ./dev/spark-test-image/python-312-classic-only/
           push: true
-          tags: ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-311-classic-only-cache:${{ github.ref_name }}-static
-          cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-311-classic-only-cache:${{ github.ref_name }}
-          cache-to: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-311-classic-only-cache:${{ github.ref_name }},mode=max
-      - name: Image digest (PySpark Classic Only with Python 3.11)
-        if: hashFiles('dev/spark-test-image/python-311-classic-only/Dockerfile') != ''
-        run: echo ${{ steps.docker_build_pyspark_python_311_classic_only.outputs.digest }}
+          tags: ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-312-classic-only-cache:${{ github.ref_name }}-static
+          cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-312-classic-only-cache:${{ github.ref_name }}
+          cache-to: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-312-classic-only-cache:${{ github.ref_name }},mode=max
+      - name: Image digest (PySpark Classic Only with Python 3.12)
+        if: hashFiles('dev/spark-test-image/python-312-classic-only/Dockerfile') != ''
+        run: echo ${{ steps.docker_build_pyspark_python_312_classic_only.outputs.digest }}
       - name: Build and push (PySpark with Python 3.12)
         if: hashFiles('dev/spark-test-image/python-312/Dockerfile') != ''
         id: docker_build_pyspark_python_312

.github/workflows/build_python_3.12.yml .github/workflows/build_python_3.11.yml.github/workflows/build_python_3.12.yml renamed to .github/workflows/build_python_3.11.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / Python-only (master, Python 3.12)"
+name: "Build / Python-only (master, Python 3.11)"
 
 on:
   schedule:
@@ -37,8 +37,8 @@ jobs:
       hadoop: hadoop3
       envs: >-
         {
-          "PYSPARK_IMAGE_TO_TEST": "python-312",
-          "PYTHON_TO_TEST": "python3.12"
+          "PYSPARK_IMAGE_TO_TEST": "python-311",
+          "PYTHON_TO_TEST": "python3.11"
         }
       jobs: >-
         {

…thub/workflows/build_python_3.11_arm.yml …thub/workflows/build_python_3.12_arm.yml.github/workflows/build_python_3.11_arm.yml renamed to .github/workflows/build_python_3.12_arm.yml .github/workflows/build_python_3.11_arm.yml renamed to .github/workflows/build_python_3.12_arm.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / Python-only (master, Python 3.11, ARM)"
+name: "Build / Python-only (master, Python 3.12, ARM)"
 
 on:
   schedule:

…flows/build_python_3.11_classic_only.yml …flows/build_python_3.12_classic_only.yml.github/workflows/build_python_3.11_classic_only.yml renamed to .github/workflows/build_python_3.12_classic_only.yml .github/workflows/build_python_3.11_classic_only.yml renamed to .github/workflows/build_python_3.12_classic_only.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / Python-only, Classic-only (master, Python 3.11)"
+name: "Build / Python-only, Classic-only (master, Python 3.12)"
 
 on:
   schedule:
@@ -37,8 +37,8 @@ jobs:
       hadoop: hadoop3
       envs: >-
         {
-          "PYSPARK_IMAGE_TO_TEST": "python-311-classic-only",
-          "PYTHON_TO_TEST": "python3.11"
+          "PYSPARK_IMAGE_TO_TEST": "python-312-classic-only",
+          "PYTHON_TO_TEST": "python3.12"
         }
       jobs: >-
         {

…/workflows/build_python_3.11_macos26.yml …/workflows/build_python_3.12_macos26.yml.github/workflows/build_python_3.11_macos26.yml renamed to .github/workflows/build_python_3.12_macos26.yml .github/workflows/build_python_3.11_macos26.yml renamed to .github/workflows/build_python_3.12_macos26.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / Python-only (master, Python 3.11, MacOS26)"
+name: "Build / Python-only (master, Python 3.12, MacOS26)"
 
 on:
   schedule:

.github/workflows/python_hosted_runner_test.yml

@@ -29,7 +29,7 @@ on:
       python:
         required: false
         type: string
-        default: 3.11
+        default: 3.12
       branch:
         description: Branch to run the build against
         required: false

README.md

@@ -36,12 +36,12 @@ This README file only contains basic setup instructions.
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_maven_java21_arm.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_maven_java21_arm.yml)                 |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_coverage.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_coverage.yml)                                 |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_pypy3.10.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_pypy3.10.yml)                   |
-|            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_pypy3.11.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_pypy3.11.yml)                   |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.10.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.10.yml)                           |
-|            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.11_classic_only.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.11_classic_only.yml) |
-|            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.11_arm.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.11_arm.yml)                   |
-|            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.11_macos26.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.11_macos26.yml)           |
-|            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.12.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.12.yml)                           |
+|            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.11.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.11.yml)                           |
+|            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_pypy3.11.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_pypy3.11.yml)                   |
+|            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.12_classic_only.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.12_classic_only.yml) |
+|            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.12_arm.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.12_arm.yml)                   |
+|            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.12_macos26.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.12_macos26.yml)           |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.12_pandas_3.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.12_pandas_3.yml)         |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.13.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.13.yml)                           |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.14.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.14.yml)                           |

…image/python-311-classic-only/Dockerfile …image/python-312-classic-only/Dockerfiledev/spark-test-image/python-311-classic-only/Dockerfile renamed to dev/spark-test-image/python-312-classic-only/Dockerfile dev/spark-test-image/python-311-classic-only/Dockerfile renamed to dev/spark-test-image/python-312-classic-only/Dockerfile

@@ -20,7 +20,7 @@
 FROM ubuntu:jammy-20240911.1
 LABEL org.opencontainers.image.authors="Apache Spark project <[email protected]>"
 LABEL org.opencontainers.image.licenses="Apache-2.0"
-LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark Classic with Python 3.11"
+LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark Classic with Python 3.12"
 # Overwrite this label to avoid exposing the underlying Ubuntu OS version label
 LABEL org.opencontainers.image.version=""
 
@@ -59,10 +59,10 @@ RUN apt-get update && apt-get install -y \
     wget \
     zlib1g-dev
 
-# Install Python 3.11
+# Install Python 3.12
 RUN add-apt-repository ppa:deadsnakes/ppa
 RUN apt-get update && apt-get install -y \
-    python3.11 \
+    python3.12 \
     && apt-get autoremove --purge -y \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
@@ -71,10 +71,10 @@ RUN apt-get update && apt-get install -y \
 ARG BASIC_PIP_PKGS="numpy pyarrow>=22.0.0 pandas==2.3.3 plotly<6.0.0 matplotlib openpyxl memory-profiler>=0.61.0 mlflow>=2.8.1 scipy scikit-learn>=1.3.2"
 ARG TEST_PIP_PKGS="coverage unittest-xml-reporting"
 
-# Install Python 3.11 packages
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
-RUN python3.11 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow needs this
-RUN python3.11 -m pip install $BASIC_PIP_PKGS $TEST_PIP_PKGS && \
-    python3.11 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \
-    python3.11 -m pip install deepspeed torcheval && \
-    python3.11 -m pip cache purge
+# Install Python 3.12 packages
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12
+RUN python3.12 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow needs this
+RUN python3.12 -m pip install $BASIC_PIP_PKGS $TEST_PIP_PKGS && \
+    python3.12 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \
+    python3.12 -m pip install deepspeed torcheval && \
+    python3.12 -m pip cache purge