Move dependency checks to pyspark.testing.utils

Co-authored-by: Isaac

Author: zhengruifeng

python/pyspark/sql/tests/streaming/test_streaming_kafka_rtm.py

@@ -65,16 +65,7 @@
 os.environ["PYSPARK_SUBMIT_ARGS"] = " ".join([jars_args, existing_args])
 
 from pyspark.sql.tests.streaming.kafka_utils import KafkaUtils
-
-
-# Check if required Python dependencies are available
-try:
-    import testcontainers  # noqa: F401
-    import kafka  # noqa: F401
-
-    have_kafka_deps = True
-except ImportError:
-    have_kafka_deps = False
+from pyspark.testing.utils import have_testcontainers, have_kafka
 
 
 class StreamingKafkaTestsMixin:
@@ -122,7 +113,7 @@ def _is_docker_available():
 
 
 @unittest.skipIf(
-    not have_kafka_deps,
+    not (have_testcontainers and have_kafka),
     "Kafka test dependencies not available (testcontainers, kafka-python)",
 )
 class StreamingKafkaTests(StreamingKafkaTestsMixin, ReusedSQLTestCase):

python/pyspark/testing/utils.py

@@ -103,6 +103,14 @@ def have_package(name: str) -> bool:
 have_zstandard = have_package("zstandard")
 zstandard_requirement_message = "" if have_zstandard else "No module named 'zstandard'"
 
+have_testcontainers = have_package("testcontainers")
+testcontainers_requirement_message = (
+    "" if have_testcontainers else "No module named 'testcontainers'"
+)
+
+have_kafka = have_package("kafka")
+kafka_requirement_message = "" if have_kafka else "No module named 'kafka'"
+
 
 googleapis_common_protos_requirement_message = ""