spark/dev/requirements.txt at 62d5d7d48f1bd7629b51a92724d252503ad0b02d · gaogaotiantian/spark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
# PySpark dependencies (required)
py4j>=0.10.9.9

# PySpark dependencies (optional)
numpy>=1.22
pyarrow>=18.0.0
six==1.16.0
pandas>=2.2.0
scipy>=1.8.0
plotly<6.0.0
mlflow>=2.3.1
scikit-learn
matplotlib
memory-profiler>=0.61.0
pyyaml>=3.11

# PySpark test dependencies
unittest-xml-reporting
openpyxl

# PySpark test dependencies (optional)
coverage
tabulate

# Linter
ruff==0.14.8
mypy==1.19.1
pytest-mypy-plugins==3.2.0
# See SPARK-38680.
pandas-stubs>=2.2.0
scipy-stubs;
types-PyYAML

# Documentation (SQL)
mkdocs

# Documentation (Python)
pydata_sphinx_theme>=0.13
ipython
nbsphinx
numpydoc
jinja2
sphinx==4.5.0
# With sphinx 4.5.0, we need to set the upperbound version of sphinxcontrib*, it should be removed after upgrading sphinx>=5
sphinxcontrib-applehelp<=1.0.4
sphinxcontrib-devhelp<=1.0.2
sphinxcontrib-htmlhelp<=2.0.1
sphinxcontrib-jsmath<=1.0.1
sphinxcontrib-qthelp<=1.0.3
sphinxcontrib-serializinghtml<=1.1.5
sphinx-plotly-directive
sphinx-copybutton
docutils<0.18.0
markupsafe

# Development scripts
jira>=3.5.2
PyGithub

# pandas API on Spark Code formatter.
black==23.12.1
py

# Spark Connect (required)
grpcio>=1.76.0
grpcio-status>=1.76.0
googleapis-common-protos>=1.71.0
protobuf==6.33.5
zstandard>=0.25.0

# Spark Connect python proto generation plugin (optional)
mypy-protobuf==3.3.0
googleapis-common-protos-stubs==2.2.0
grpc-stubs==1.24.11

# Debug for Spark and Spark Connect
graphviz==0.20.3
flameprof==0.4
viztracer
debugpy
pystack>=1.6.0; sys_platform=='linux'
psutil

# TorchDistributor dependencies
torch
torchvision
torcheval

# DeepspeedTorchDistributor dependencies
deepspeed; sys_platform != 'darwin'