Skip to content

Commit 5e40305

Browse files
pan3793rpnkv
authored andcommitted
[SPARK-55428][BUILD] Sync Netty Java options everywhere
### What changes were proposed in this pull request? Remove redundant `-Dio.netty.tryReflectionSetAccessible=true` in some building scripts/`pom.xml` - it has already been added by `JavaModuleOptions`(`spark-submit`-based cases) or Maven property `extraJavaTestArgs`(ScalaTests/JUnit cases) Add the missing opts to `sql/connect/bin/spark-connect-scala-client` ``` -Dio.netty.allocator.type=pooled -Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE ``` ### Why are the changes needed? Keep Netty Java options sync everywhere. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass GHA. ### Was this patch authored or co-authored using generative AI tooling? No. Closes apache#54209 from pan3793/SPARK-55428. Authored-by: Cheng Pan <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent 5aa935f commit 5e40305

File tree

4 files changed

+6
-4
lines changed

4 files changed

+6
-4
lines changed

R/run-tests.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ if [[ $(echo $SPARK_AVRO_JAR_PATH | wc -l) -eq 1 ]]; then
3030
fi
3131

3232
if [ -z "$SPARK_JARS" ]; then
33-
SPARKR_SUPPRESS_DEPRECATION_WARNING=1 SPARK_TESTING=1 NOT_CRAN=true $FWDIR/../bin/spark-submit --driver-java-options "-Dlog4j.configurationFile=file:$FWDIR/log4j2.properties" --conf spark.hadoop.fs.defaultFS="file:///" --conf spark.driver.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true -Xss4M" --conf spark.executor.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true -Xss4M" $FWDIR/pkg/tests/run-all.R 2>&1 | tee -a $LOGFILE
33+
SPARKR_SUPPRESS_DEPRECATION_WARNING=1 SPARK_TESTING=1 NOT_CRAN=true $FWDIR/../bin/spark-submit --driver-java-options "-Dlog4j.configurationFile=file:$FWDIR/log4j2.properties" --conf spark.hadoop.fs.defaultFS="file:///" --conf spark.driver.extraJavaOptions="-Xss4M" --conf spark.executor.extraJavaOptions="-Xss4M" $FWDIR/pkg/tests/run-all.R 2>&1 | tee -a $LOGFILE
3434
else
35-
SPARKR_SUPPRESS_DEPRECATION_WARNING=1 SPARK_TESTING=1 NOT_CRAN=true $FWDIR/../bin/spark-submit --jars $SPARK_JARS --driver-java-options "-Dlog4j.configurationFile=file:$FWDIR/log4j2.properties" --conf spark.hadoop.fs.defaultFS="file:///" --conf spark.driver.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true -Xss4M" --conf spark.executor.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true -Xss4M" $FWDIR/pkg/tests/run-all.R 2>&1 | tee -a $LOGFILE
35+
SPARKR_SUPPRESS_DEPRECATION_WARNING=1 SPARK_TESTING=1 NOT_CRAN=true $FWDIR/../bin/spark-submit --jars $SPARK_JARS --driver-java-options "-Dlog4j.configurationFile=file:$FWDIR/log4j2.properties" --conf spark.hadoop.fs.defaultFS="file:///" --conf spark.driver.extraJavaOptions="-Xss4M" --conf spark.executor.extraJavaOptions="-Xss4M" $FWDIR/pkg/tests/run-all.R 2>&1 | tee -a $LOGFILE
3636
fi
3737

3838
FAILED=$((PIPESTATUS[0]||$FAILED))

sql/catalyst/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@
173173
<groupId>org.scalatest</groupId>
174174
<artifactId>scalatest-maven-plugin</artifactId>
175175
<configuration>
176-
<argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true</argLine>
176+
<argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs}</argLine>
177177
</configuration>
178178
</plugin>
179179
</plugins>

sql/connect/bin/spark-connect-scala-client

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ JVM_ARGS="-XX:+IgnoreUnrecognizedVMOptions \
7171
--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \
7272
-Djdk.reflect.useDirectMethodHandle=false \
7373
-Dio.netty.tryReflectionSetAccessible=true \
74+
-Dio.netty.allocator.type=pooled \
75+
-Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE \
7476
--enable-native-access=ALL-UNNAMED \
7577
$SCJVM_ARGS"
7678

sql/core/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,7 @@
344344
<groupId>org.scalatest</groupId>
345345
<artifactId>scalatest-maven-plugin</artifactId>
346346
<configuration>
347-
<argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true</argLine>
347+
<argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs}</argLine>
348348
</configuration>
349349
</plugin>
350350
<plugin>

0 commit comments

Comments
 (0)