Skip to content

Commit 0f36edc

Browse files
andylam-dbcloud-fan
andcommitted
[SPARK-51624][SQL] Propagate GetStructField metadata in CreateNamedStruct.dataType
### What changes were proposed in this pull request? This change intends to propagate GetStructField metadata in CreateNamedStruct.dataType. ### Why are the changes needed? This is important because dataType comparisons are important for optimizer rules such as SimplifyCasts, which can cascade down to more expression optimizations. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Added unit test. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #50412 from andylam-db/simplify-casts-comp. Lead-authored-by: Andy Lam <andy.lam@databricks.com> Co-authored-by: Wenchen Fan <cloud0fan@gmail.com> Signed-off-by: Wenchen Fan <wenchen@databricks.com>
1 parent ab4bf8c commit 0f36edc

File tree

2 files changed

+17
-2
lines changed

2 files changed

+17
-2
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -457,6 +457,7 @@ case class CreateNamedStruct(children: Seq[Expression]) extends Expression with
457457
case (name, expr) =>
458458
val metadata = expr match {
459459
case ne: NamedExpression => ne.metadata
460+
case gsf: GetStructField => gsf.metadata
460461
case _ => Metadata.empty
461462
}
462463
StructField(name.toString, expr.dataType, expr.nullable, metadata)

sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@ package org.apache.spark.sql
1919

2020
import scala.jdk.CollectionConverters._
2121

22-
import org.apache.spark.sql.catalyst.expressions.CreateNamedStruct
22+
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, CreateNamedStruct, GetStructField, Literal}
2323
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
2424
import org.apache.spark.sql.test.SharedSparkSession
25-
import org.apache.spark.sql.types.{ArrayType, IntegerType, StructField, StructType}
25+
import org.apache.spark.sql.types.{ArrayType, IntegerType, MetadataBuilder, StringType, StructField, StructType}
2626

2727
class ComplexTypesSuite extends QueryTest with SharedSparkSession {
2828
import testImplicits._
@@ -171,4 +171,18 @@ class ComplexTypesSuite extends QueryTest with SharedSparkSession {
171171
assert(df.schema == expectedSchema)
172172
checkAnswer(df, Seq(Row(Row(1, 2))))
173173
}
174+
175+
test("SPARK-51624: Propagate StructField metadata in CreateNamedStruct.dataType") {
176+
val metadata = new MetadataBuilder().putString("comment", "hello").build()
177+
val structRef = AttributeReference("s1",
178+
StructType(StructField("col1", StringType, false, metadata) :: Nil))()
179+
val createNamedStruct = CreateNamedStruct(
180+
Seq(
181+
Literal("a"),
182+
GetStructField(structRef, 0)
183+
)
184+
)
185+
val dataType = createNamedStruct.dataType
186+
assert(dataType.asInstanceOf[StructType].fields.head.metadata == metadata)
187+
}
174188
}

0 commit comments

Comments
 (0)