Skip to content

Commit 87d1524

Browse files
authored
fix: respect scan impl config for v2 scan (#3357)
1 parent c520d7b commit 87d1524

4 files changed

Lines changed: 8 additions & 5 deletions

File tree

spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] with Com
228228
private def transformV2Scan(scanExec: BatchScanExec): SparkPlan = {
229229

230230
scanExec.scan match {
231-
case scan: ParquetScan =>
231+
case scan: ParquetScan if COMET_NATIVE_SCAN_IMPL.get() == SCAN_NATIVE_COMET =>
232232
val fallbackReasons = new ListBuffer[String]()
233233
val schemaSupported =
234234
CometBatchScanExec.isSchemaSupported(scan.readDataSchema, fallbackReasons)

spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -382,9 +382,10 @@ class CometExecSuite extends CometTestBase {
382382
}
383383
}
384384

385-
test("ReusedExchangeExec should work on CometBroadcastExchangeExec") {
385+
test("ReusedExchangeExec should work on CometBroadcastExchangeExec with V2 scan") {
386386
withSQLConf(
387387
CometConf.COMET_EXEC_BROADCAST_FORCE_ENABLED.key -> "true",
388+
CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_COMET,
388389
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
389390
SQLConf.USE_V1_SOURCE_LIST.key -> "") {
390391
withTempPath { path =>

spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2045,7 +2045,7 @@ class ParquetReadV2Suite extends ParquetReadSuite with AdaptiveSparkPlanHelper {
20452045
case (cometEnabled, expectedScanner) =>
20462046
testScanner(
20472047
cometEnabled,
2048-
CometConf.SCAN_NATIVE_DATAFUSION,
2048+
CometConf.SCAN_NATIVE_COMET,
20492049
scanner = expectedScanner,
20502050
v1 = None)
20512051
}

spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,11 +101,13 @@ class CometScanRuleSuite extends CometTestBase {
101101
}
102102
}
103103

104-
test("CometExecRule should replace BatchScanExec, but only when Comet is enabled") {
104+
test("CometScanRule should replace V2 BatchScanExec, but only when Comet is enabled") {
105105
withTempPath { path =>
106106
createTestDataFrame.write.parquet(path.toString)
107107
withTempView("test_data") {
108-
withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") {
108+
withSQLConf(
109+
SQLConf.USE_V1_SOURCE_LIST.key -> "",
110+
CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_COMET) {
109111
spark.read.parquet(path.toString).createOrReplaceTempView("test_data")
110112

111113
val sparkPlan =

0 commit comments

Comments
 (0)