Skip to content

Commit 3d80308

Browse files
andygroveandygrove
authored
test: unignore passing DPP test, retag remaining DPP failures to #4045 (#4046)
1 parent b98a736 commit 3d80308

3 files changed

Lines changed: 28 additions & 59 deletions

File tree

‎dev/diffs/3.4.3.diff‎

Lines changed: 8 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,7 @@ index daef11ae4d6..9f3cc9181f2 100644
417417
assert(exchanges.size == 2)
418418
}
419419
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
420-
index f33432ddb6f..b7a5fd72f7d 100644
420+
index f33432ddb6f..914afa6b01d 100644
421421
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
422422
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
423423
@@ -22,6 +22,7 @@ import org.scalatest.GivenWhenThen
@@ -458,17 +458,7 @@ index f33432ddb6f..b7a5fd72f7d 100644
458458
case _ => Nil
459459
}
460460
}
461-
@@ -1027,7 +1037,8 @@ abstract class DynamicPartitionPruningSuiteBase
462-
}
463-
}
464-
465-
- test("avoid reordering broadcast join keys to match input hash partitioning") {
466-
+ test("avoid reordering broadcast join keys to match input hash partitioning",
467-
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
468-
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
469-
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
470-
withTable("large", "dimTwo", "dimThree") {
471-
@@ -1204,10 +1215,16 @@ abstract class DynamicPartitionPruningSuiteBase
461+
@@ -1204,10 +1214,16 @@ abstract class DynamicPartitionPruningSuiteBase
472462

473463
val plan = df.queryExecution.executedPlan
474464
val countSubqueryBroadcasts =
@@ -487,35 +477,35 @@ index f33432ddb6f..b7a5fd72f7d 100644
487477

488478
assert(countSubqueryBroadcasts == 1)
489479
assert(countReusedSubqueryBroadcasts == 1)
490-
@@ -1215,7 +1232,8 @@ abstract class DynamicPartitionPruningSuiteBase
480+
@@ -1215,7 +1231,8 @@ abstract class DynamicPartitionPruningSuiteBase
491481
}
492482

493483
test("SPARK-32509: Unused Dynamic Pruning filter shouldn't affect " +
494484
- "canonicalization and exchange reuse") {
495485
+ "canonicalization and exchange reuse",
496-
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
486+
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/4045")) {
497487
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
498488
withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
499489
val df = sql(
500-
@@ -1423,7 +1441,8 @@ abstract class DynamicPartitionPruningSuiteBase
490+
@@ -1423,7 +1440,8 @@ abstract class DynamicPartitionPruningSuiteBase
501491
}
502492
}
503493

504494
- test("SPARK-34637: DPP side broadcast query stage is created firstly") {
505495
+ test("SPARK-34637: DPP side broadcast query stage is created firstly",
506-
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
496+
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/4045")) {
507497
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
508498
val df = sql(
509499
""" WITH v as (
510-
@@ -1577,6 +1596,7 @@ abstract class DynamicPartitionPruningSuiteBase
500+
@@ -1577,6 +1595,7 @@ abstract class DynamicPartitionPruningSuiteBase
511501

512502
val subqueryBroadcastExecs = collectWithSubqueries(df.queryExecution.executedPlan) {
513503
case s: SubqueryBroadcastExec => s
514504
+ case s: CometSubqueryBroadcastExec => s
515505
}
516506
assert(subqueryBroadcastExecs.size === 1)
517507
subqueryBroadcastExecs.foreach { subqueryBroadcastExec =>
518-
@@ -1729,6 +1749,10 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
508+
@@ -1729,6 +1748,10 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
519509
case s: BatchScanExec =>
520510
// we use f1 col for v2 tables due to schema pruning
521511
s.output.exists(_.exists(_.argString(maxFields = 100).contains("f1")))

‎dev/diffs/3.5.8.diff‎

Lines changed: 8 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -398,7 +398,7 @@ index c4fb4fa943c..a04b23870a8 100644
398398
assert(exchanges.size == 2)
399399
}
400400
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
401-
index f33432ddb6f..b7a5fd72f7d 100644
401+
index f33432ddb6f..914afa6b01d 100644
402402
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
403403
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
404404
@@ -22,6 +22,7 @@ import org.scalatest.GivenWhenThen
@@ -439,17 +439,7 @@ index f33432ddb6f..b7a5fd72f7d 100644
439439
case _ => Nil
440440
}
441441
}
442-
@@ -1027,7 +1037,8 @@ abstract class DynamicPartitionPruningSuiteBase
443-
}
444-
}
445-
446-
- test("avoid reordering broadcast join keys to match input hash partitioning") {
447-
+ test("avoid reordering broadcast join keys to match input hash partitioning",
448-
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
449-
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
450-
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
451-
withTable("large", "dimTwo", "dimThree") {
452-
@@ -1204,10 +1215,16 @@ abstract class DynamicPartitionPruningSuiteBase
442+
@@ -1204,10 +1214,16 @@ abstract class DynamicPartitionPruningSuiteBase
453443

454444
val plan = df.queryExecution.executedPlan
455445
val countSubqueryBroadcasts =
@@ -468,35 +458,35 @@ index f33432ddb6f..b7a5fd72f7d 100644
468458

469459
assert(countSubqueryBroadcasts == 1)
470460
assert(countReusedSubqueryBroadcasts == 1)
471-
@@ -1215,7 +1232,8 @@ abstract class DynamicPartitionPruningSuiteBase
461+
@@ -1215,7 +1231,8 @@ abstract class DynamicPartitionPruningSuiteBase
472462
}
473463

474464
test("SPARK-32509: Unused Dynamic Pruning filter shouldn't affect " +
475465
- "canonicalization and exchange reuse") {
476466
+ "canonicalization and exchange reuse",
477-
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
467+
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/4045")) {
478468
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
479469
withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
480470
val df = sql(
481-
@@ -1423,7 +1441,8 @@ abstract class DynamicPartitionPruningSuiteBase
471+
@@ -1423,7 +1440,8 @@ abstract class DynamicPartitionPruningSuiteBase
482472
}
483473
}
484474

485475
- test("SPARK-34637: DPP side broadcast query stage is created firstly") {
486476
+ test("SPARK-34637: DPP side broadcast query stage is created firstly",
487-
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
477+
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/4045")) {
488478
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
489479
val df = sql(
490480
""" WITH v as (
491-
@@ -1577,6 +1596,7 @@ abstract class DynamicPartitionPruningSuiteBase
481+
@@ -1577,6 +1595,7 @@ abstract class DynamicPartitionPruningSuiteBase
492482

493483
val subqueryBroadcastExecs = collectWithSubqueries(df.queryExecution.executedPlan) {
494484
case s: SubqueryBroadcastExec => s
495485
+ case s: CometSubqueryBroadcastExec => s
496486
}
497487
assert(subqueryBroadcastExecs.size === 1)
498488
subqueryBroadcastExecs.foreach { subqueryBroadcastExec =>
499-
@@ -1729,6 +1749,10 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
489+
@@ -1729,6 +1748,10 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
500490
case s: BatchScanExec =>
501491
// we use f1 col for v2 tables due to schema pruning
502492
s.output.exists(_.exists(_.argString(maxFields = 100).contains("f1")))

‎dev/diffs/4.0.1.diff‎

Lines changed: 12 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -589,7 +589,7 @@ index 81713c777bc..b5f92ed9742 100644
589589
assert(exchanges.size == 2)
590590
}
591591
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
592-
index 2c24cc7d570..753737a1057 100644
592+
index 2c24cc7d570..5a1fe7017c3 100644
593593
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
594594
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
595595
@@ -22,6 +22,7 @@ import org.scalatest.GivenWhenThen
@@ -630,17 +630,7 @@ index 2c24cc7d570..753737a1057 100644
630630
case _ => Nil
631631
}
632632
}
633-
@@ -1027,7 +1037,8 @@ abstract class DynamicPartitionPruningSuiteBase
634-
}
635-
}
636-
637-
- test("avoid reordering broadcast join keys to match input hash partitioning") {
638-
+ test("avoid reordering broadcast join keys to match input hash partitioning",
639-
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
640-
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
641-
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
642-
withTable("large", "dimTwo", "dimThree") {
643-
@@ -1204,10 +1215,16 @@ abstract class DynamicPartitionPruningSuiteBase
633+
@@ -1204,10 +1214,16 @@ abstract class DynamicPartitionPruningSuiteBase
644634

645635
val plan = df.queryExecution.executedPlan
646636
val countSubqueryBroadcasts =
@@ -659,43 +649,43 @@ index 2c24cc7d570..753737a1057 100644
659649

660650
assert(countSubqueryBroadcasts == 1)
661651
assert(countReusedSubqueryBroadcasts == 1)
662-
@@ -1215,7 +1232,8 @@ abstract class DynamicPartitionPruningSuiteBase
652+
@@ -1215,7 +1231,8 @@ abstract class DynamicPartitionPruningSuiteBase
663653
}
664654

665655
test("SPARK-32509: Unused Dynamic Pruning filter shouldn't affect " +
666656
- "canonicalization and exchange reuse") {
667657
+ "canonicalization and exchange reuse",
668-
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
658+
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/4045")) {
669659
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
670660
withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
671661
val df = sql(
672-
@@ -1330,6 +1348,7 @@ abstract class DynamicPartitionPruningSuiteBase
662+
@@ -1330,6 +1347,7 @@ abstract class DynamicPartitionPruningSuiteBase
673663
}
674664

675665
test("Subquery reuse across the whole plan",
676666
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3313"),
677667
DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
678668
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
679669
SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
680-
@@ -1424,7 +1443,8 @@ abstract class DynamicPartitionPruningSuiteBase
670+
@@ -1424,7 +1442,8 @@ abstract class DynamicPartitionPruningSuiteBase
681671
}
682672
}
683673

684674
- test("SPARK-34637: DPP side broadcast query stage is created firstly") {
685675
+ test("SPARK-34637: DPP side broadcast query stage is created firstly",
686-
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
676+
+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/4045")) {
687677
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
688678
val df = sql(
689679
""" WITH v as (
690-
@@ -1578,6 +1598,7 @@ abstract class DynamicPartitionPruningSuiteBase
680+
@@ -1578,6 +1597,7 @@ abstract class DynamicPartitionPruningSuiteBase
691681

692682
val subqueryBroadcastExecs = collectWithSubqueries(df.queryExecution.executedPlan) {
693683
case s: SubqueryBroadcastExec => s
694684
+ case s: CometSubqueryBroadcastExec => s
695685
}
696686
assert(subqueryBroadcastExecs.size === 1)
697687
subqueryBroadcastExecs.foreach { subqueryBroadcastExec =>
698-
@@ -1730,6 +1751,10 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
688+
@@ -1730,6 +1750,10 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
699689
case s: BatchScanExec =>
700690
// we use f1 col for v2 tables due to schema pruning
701691
s.output.exists(_.exists(_.argString(maxFields = 100).contains("f1")))
@@ -1319,24 +1309,23 @@ index 0df7f806272..92390bd819f 100644
13191309

13201310
test("non-matching optional group") {
13211311
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
1322-
index 2e33f6505ab..3a8b154b565 100644
1312+
index 2e33f6505ab..949fdea0003 100644
13231313
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
13241314
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
1325-
@@ -23,11 +23,13 @@ import org.apache.spark.SparkRuntimeException
1315+
@@ -23,10 +23,12 @@ import org.apache.spark.SparkRuntimeException
13261316
import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
13271317
import org.apache.spark.sql.catalyst.plans.{LeftAnti, LeftSemi}
13281318
import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Filter, Join, LogicalPlan, Project, Sort, Union}
13291319
+import org.apache.spark.sql.comet.{CometNativeColumnarToRowExec, CometNativeScanExec, CometScanExec}
13301320
import org.apache.spark.sql.execution._
1321+
+import org.apache.spark.sql.IgnoreCometNativeDataFusion
13311322
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecution}
13321323
import org.apache.spark.sql.execution.datasources.FileScanRDD
13331324
-import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
13341325
+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
13351326
import org.apache.spark.sql.execution.joins.{BaseJoinExec, BroadcastHashJoinExec, BroadcastNestedLoopJoinExec}
1336-
+import org.apache.spark.sql.IgnoreCometNativeDataFusion
13371327
import org.apache.spark.sql.internal.SQLConf
13381328
import org.apache.spark.sql.test.SharedSparkSession
1339-
13401329
@@ -1529,6 +1531,18 @@ class SubquerySuite extends QueryTest
13411330
fs.inputRDDs().forall(
13421331
_.asInstanceOf[FileScanRDD].filePartitions.forall(

0 commit comments

Comments
 (0)
⚡