[SPARK-56186][PYTHON] Retire pypy

.github/workflows/build_infra_images_cache.yml

@@ -33,7 +33,6 @@ on:
     - 'dev/spark-test-image/python-minimum/Dockerfile'
     - 'dev/spark-test-image/python-ps-minimum/Dockerfile'
     - 'dev/spark-test-image/pypy-310/Dockerfile'
-    - 'dev/spark-test-image/pypy-311/Dockerfile'
     - 'dev/spark-test-image/python-310/Dockerfile'
     - 'dev/spark-test-image/python-311/Dockerfile'
     - 'dev/spark-test-image/python-312/Dockerfile'
@@ -154,19 +153,6 @@ jobs:
       - name: Image digest (PySpark with PyPy 3.10)
         if: hashFiles('dev/spark-test-image/pypy-310/Dockerfile') != ''
         run: echo ${{ steps.docker_build_pyspark_pypy_310.outputs.digest }}
-      - name: Build and push (PySpark with PyPy 3.11)
-        if: hashFiles('dev/spark-test-image/pypy-311/Dockerfile') != ''
-        id: docker_build_pyspark_pypy_311
-        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8
-        with:
-          context: ./dev/spark-test-image/pypy-311/
-          push: true
-          tags: ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-pypy-311-cache:${{ github.ref_name }}-static
-          cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-pypy-311-cache:${{ github.ref_name }}
-          cache-to: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-pypy-311-cache:${{ github.ref_name }},mode=max
-      - name: Image digest (PySpark with PyPy 3.11)
-        if: hashFiles('dev/spark-test-image/pypy-311/Dockerfile') != ''
-        run: echo ${{ steps.docker_build_pyspark_pypy_311.outputs.digest }}
       - name: Build and push (PySpark with Python 3.10)
         if: hashFiles('dev/spark-test-image/python-310/Dockerfile') != ''
         id: docker_build_pyspark_python_310

.github/workflows/maven_test.yml

@@ -42,7 +42,7 @@ on:
         type: string
         default: ubuntu-latest
       arch:
-        description: The target architecture (x86, x64, arm64) of the Python or PyPy interpreter.
+        description: The target architecture (x86, x64, arm64) of the Python interpreter.
         required: false
         type: string
         default: x64

.github/workflows/python_hosted_runner_test.yml

@@ -46,7 +46,7 @@ on:
         type: string
         default: macos-15
       arch:
-        description: The target architecture (x86, x64, arm64) of the Python or PyPy interpreter.
+        description: The target architecture (x86, x64, arm64) of the Python interpreter.
         required: false
         type: string
         default: arm64

README.md

@@ -42,10 +42,8 @@ This README file only contains basic setup instructions.
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_maven_java21_macos26.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_maven_java21_macos26.yml)         |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_maven_java21_arm.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_maven_java21_arm.yml)                 |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_coverage.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_coverage.yml)                                 |
-|            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_pypy3.10.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_pypy3.10.yml)                   |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.10.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.10.yml)                           |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.11.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.11.yml)                           |
-|            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_pypy3.11.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_pypy3.11.yml)                   |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.12_classic_only.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.12_classic_only.yml) |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.12_arm.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.12_arm.yml)                   |
 |            | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.12_macos26.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.12_macos26.yml)           |

dev/sparktestsupport/modules.py

@@ -700,9 +700,6 @@ def __hash__(self):
         "pyspark.sql.tests.pandas.streaming.test_transform_with_state_state_variable_checkpoint_v2",
         "pyspark.sql.tests.pandas.streaming.test_tws_tester",
     ],
-    excluded_python_implementations=[
-        "PyPy"  # Skip these tests under PyPy since they require numpy and it isn't available there
-    ],
 )
 
 pyspark_mllib = Module(
@@ -733,9 +730,6 @@ def __hash__(self):
         "pyspark.mllib.tests.test_streaming_algorithms",
         "pyspark.mllib.tests.test_util",
     ],
-    excluded_python_implementations=[
-        "PyPy"  # Skip these tests under PyPy since they require numpy and it isn't available there
-    ],
 )
 
 
@@ -799,9 +793,6 @@ def __hash__(self):
         "pyspark.ml.tests.test_regression",
         "pyspark.ml.tests.test_clustering",
     ],
-    excluded_python_implementations=[
-        "PyPy"  # Skip these tests under PyPy since they require numpy and it isn't available there
-    ],
 )
 
 pyspark_install = Module(
@@ -978,10 +969,6 @@ def __hash__(self):
         "pyspark.pandas.tests.frame.test_asfreq",
         "pyspark.pandas.tests.frame.test_asof",
     ],
-    excluded_python_implementations=[
-        "PyPy"  # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and
-        # they aren't available there
-    ],
 )
 
 pyspark_pandas_slow = Module(
@@ -1112,10 +1099,6 @@ def __hash__(self):
         "pyspark.pandas.tests.diff_frames_ops.test_groupby_rolling_adv",
         "pyspark.pandas.tests.diff_frames_ops.test_groupby_rolling_count",
     ],
-    excluded_python_implementations=[
-        "PyPy"  # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and
-        # they aren't available there
-    ],
 )
 
 pyspark_connect = Module(
@@ -1219,10 +1202,6 @@ def __hash__(self):
         "pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_grouped_agg",
         "pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_window",
     ],
-    excluded_python_implementations=[
-        "PyPy"  # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and
-        # they aren't available there
-    ],
 )
 
 pyspark_structured_streaming_connect = Module(
@@ -1244,9 +1223,6 @@ def __hash__(self):
         "pyspark.sql.tests.connect.pandas.streaming.test_parity_transform_with_state",
         "pyspark.sql.tests.connect.pandas.streaming.test_parity_transform_with_state_state_variable",
     ],
-    excluded_python_implementations=[
-        "PyPy"  # Skip these tests under PyPy since they require numpy and it isn't available there
-    ],
 )
 
 
@@ -1284,10 +1260,6 @@ def __hash__(self):
         "pyspark.ml.tests.connect.test_parity_ovr",
         "pyspark.ml.tests.connect.test_parity_stat",
     ],
-    excluded_python_implementations=[
-        "PyPy"  # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and
-        # they aren't available there
-    ],
 )
 
 
@@ -1427,10 +1399,6 @@ def __hash__(self):
         "pyspark.pandas.tests.connect.frame.test_parity_asfreq",
         "pyspark.pandas.tests.connect.frame.test_parity_asof",
     ],
-    excluded_python_implementations=[
-        "PyPy"  # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and
-        # they aren't available there
-    ],
 )
 
 pyspark_pandas_slow_connect = Module(
@@ -1559,10 +1527,6 @@ def __hash__(self):
         "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_shift",
         "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_transform",
     ],
-    excluded_python_implementations=[
-        "PyPy"  # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and
-        # they aren't available there
-    ],
 )
 
 

docs/rdd-programming-guide.md

@@ -40,7 +40,7 @@ along with if you launch Spark's interactive shell -- either `bin/spark-shell` f
 <div data-lang="python"  markdown="1">
 
 Spark {{site.SPARK_VERSION}} works with Python 3.10+. It can use the standard CPython interpreter,
-so C libraries like NumPy can be used. It also works with PyPy 7.3.6+.
+so C libraries like NumPy can be used.
 
 Spark applications in Python can either be run with the `bin/spark-submit` script which includes Spark at runtime, or by including it in your setup.py as:
 
@@ -71,7 +71,6 @@ you can specify which version of Python you want to use by `PYSPARK_PYTHON`, for
 
 {% highlight bash %}
 $ PYSPARK_PYTHON=python3.8 bin/pyspark
-$ PYSPARK_PYTHON=/path-to-your-pypy/pypy bin/spark-submit examples/src/main/python/pi.py
 {% endhighlight %}
 
 </div>

python/packaging/classic/setup.py

@@ -390,7 +390,6 @@ def run(self):
             "Programming Language :: Python :: 3.13",
             "Programming Language :: Python :: 3.14",
             "Programming Language :: Python :: Implementation :: CPython",
-            "Programming Language :: Python :: Implementation :: PyPy",
             "Typing :: Typed",
         ],
         cmdclass={

python/packaging/client/setup.py

@@ -225,7 +225,6 @@
             "Programming Language :: Python :: 3.13",
             "Programming Language :: Python :: 3.14",
             "Programming Language :: Python :: Implementation :: CPython",
-            "Programming Language :: Python :: Implementation :: PyPy",
             "Typing :: Typed",
         ],
     )

python/packaging/connect/setup.py

@@ -135,7 +135,6 @@
             "Programming Language :: Python :: 3.13",
             "Programming Language :: Python :: 3.14",
             "Programming Language :: Python :: Implementation :: CPython",
-            "Programming Language :: Python :: Implementation :: PyPy",
             "Typing :: Typed",
         ],
     )

python/pyspark/sql/tests/test_python_datasource.py

@@ -17,7 +17,6 @@
 import contextlib
 import io
 import os
-import platform
 import tempfile
 import unittest
 import logging
@@ -824,9 +823,6 @@ def read(self, partition: InputPartition) -> Iterator[Tuple]:
         rounded = df.select("d").first().d
         self.assertEqual(rounded, Decimal("1.233999999999999986"))
 
-    @unittest.skipIf(
-        "pypy" in platform.python_implementation().lower(), "cannot run in environment pypy"
-    )
     def test_data_source_segfault(self):
         import ctypes