[SPARK-55263][PYTHON][INFRA] Upgrade Python linter from 3.11 to 3.12 in CI

### What changes were proposed in this pull request?

Upgrade Python linter from Python 3.11 to Python 3.12 in CI to match the test environment.

### Why are the changes needed?

The CI test environment has been upgraded to use Python 3.12 as the default version, but the Python linter was still using Python 3.11, creating an inconsistency.

### Does this PR introduce _any_ user-facing change?

No. This is an internal CI infrastructure change that does not affect end users.

### How was this patch tested?

CI.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #54042 from Yicong-Huang/SPARK-55263/feat/upgrade-python-linter-to-3.12.

Authored-by: Yicong-Huang <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>

Author: Yicong-Huang

.github/workflows/build_and_test.yml

@@ -884,12 +884,18 @@ jobs:
         if [ -f ./dev/structured_logging_style.py ]; then
             python3.9 ./dev/structured_logging_style.py
         fi
-    - name: Scala structured logging check
-      if: inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0'
+    - name: Scala structured logging check for branch-4.1
+      if: inputs.branch == 'branch-4.1'
       run: |
         if [ -f ./dev/structured_logging_style.py ]; then
             python3.11 ./dev/structured_logging_style.py
         fi
+    - name: Scala structured logging check
+      if: inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0' && inputs.branch != 'branch-4.1'
+      run: |
+        if [ -f ./dev/structured_logging_style.py ]; then
+            python3.12 ./dev/structured_logging_style.py
+        fi
     - name: Java linter
       run: ./dev/lint-java
     - name: Spark connect jvm client mima check
@@ -904,15 +910,21 @@ jobs:
     - name: List Python packages for branch-3.5 and branch-4.0
       if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0'
       run: python3.9 -m pip list
-    - name: List Python packages
-      if: inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0'
+    - name: List Python packages for branch-4.1
+      if: inputs.branch == 'branch-4.1'
       run: python3.11 -m pip list
+    - name: List Python packages
+      if: inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0' && inputs.branch != 'branch-4.1'
+      run: python3.12 -m pip list
     - name: Python linter for branch-3.5 and branch-4.0
       if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0'
       run: PYTHON_EXECUTABLE=python3.9 ./dev/lint-python
-    - name: Python linter
-      if: inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0'
+    - name: Python linter for branch-4.1
+      if: inputs.branch == 'branch-4.1'
       run: PYTHON_EXECUTABLE=python3.11 ./dev/lint-python
+    - name: Python linter
+      if: inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0' && inputs.branch != 'branch-4.1'
+      run: PYTHON_EXECUTABLE=python3.12 ./dev/lint-python
     # Should delete this section after SPARK 3.5 EOL.
     - name: Install dependencies for Python code generation check for branch-3.5
       if: inputs.branch == 'branch-3.5'

dev/spark-test-image/lint/Dockerfile

@@ -24,7 +24,7 @@ LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image for Linter"
 # Overwrite this label to avoid exposing the underlying Ubuntu OS version label
 LABEL org.opencontainers.image.version=""
 
-ENV FULL_REFRESH_DATE=20250618
+ENV FULL_REFRESH_DATE=20260129
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV DEBCONF_NONINTERACTIVE_SEEN=true
@@ -67,17 +67,17 @@ RUN Rscript -e "install.packages(c('devtools', 'knitr', 'markdown', 'rmarkdown',
 # See more in SPARK-39735
 ENV R_LIBS_SITE="/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library"
 
-# Install Python 3.11
+# Install Python 3.12
 RUN add-apt-repository ppa:deadsnakes/ppa
 RUN apt-get update && apt-get install -y \
-    python3.11 \
+    python3.12 \
     && apt-get autoremove --purge -y \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
 
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
-RUN python3.11 -m pip install \
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12
+RUN python3.12 -m pip install \
     'black==23.12.1' \
     'flake8==3.9.0' \
     'ruff==0.14.8' \
@@ -101,6 +101,6 @@ RUN python3.11 -m pip install \
     'pytest==7.1.3' \
     'scipy>=1.8.0' \
     'scipy-stubs' \
-    && python3.11 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu \
-    && python3.11 -m pip install torcheval \
-    && python3.11 -m pip cache purge
+    && python3.12 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu \
+    && python3.12 -m pip install torcheval \
+    && python3.12 -m pip cache purge

python/pyspark/ml/linalg/__init__.py

@@ -404,7 +404,7 @@ def dot(self, other: Iterable[float]) -> np.float64:
             elif isinstance(other, Vector):
                 return np.dot(self.toArray(), other.toArray())
             else:
-                return np.dot(self.toArray(), other)  # type: ignore[call-overload]
+                return np.dot(self.toArray(), other)
 
     def squared_distance(self, other: Iterable[float]) -> np.float64:
         """

python/pyspark/mllib/linalg/__init__.py

@@ -457,7 +457,7 @@ def dot(self, other: "VectorLike") -> np.float64:
             elif isinstance(other, Vector):
                 return np.dot(self.toArray(), other.toArray())
             else:
-                return np.dot(self.toArray(), cast("ArrayLike", other))
+                return np.dot(self.toArray(), cast("ArrayLike", other))  # type: ignore[valid-type]
 
     def squared_distance(self, other: "VectorLike") -> np.float64:
         """

python/pyspark/pandas/indexes/base.py

@@ -547,7 +547,7 @@ def to_numpy(self, dtype: Optional[Union[str, Dtype]] = None, copy: bool = False
             "It should only be used if the resulting NumPy ndarray is expected to be small."
         )
         result = np.asarray(
-            self._to_internal_pandas()._values, dtype=dtype  # type: ignore[arg-type,attr-defined]
+            self._to_internal_pandas()._values, dtype=dtype  # type: ignore[attr-defined]
         )
         if copy:
             result = result.copy()

python/pyspark/pandas/series.py

@@ -1082,9 +1082,9 @@ def cov(self, other: "Series", min_periods: Optional[int] = None, ddof: int = 1)
         """
         if not isinstance(other, Series):
             raise TypeError("unsupported type: %s" % type(other))
-        if not np.issubdtype(self.dtype, np.number):  # type: ignore[arg-type]
+        if not np.issubdtype(self.dtype, np.number):
             raise TypeError("unsupported dtype: %s" % self.dtype)
-        if not np.issubdtype(other.dtype, np.number):  # type: ignore[arg-type]
+        if not np.issubdtype(other.dtype, np.number):
             raise TypeError("unsupported dtype: %s" % other.dtype)
         if not isinstance(ddof, int):
             raise TypeError("ddof must be integer")