[SPARK-52991][SQL][FOLLOW-UP] Revise MergeIntoTable to use lazy val and add a new test

### What changes were proposed in this pull request?
Minor follow up for #51698

1. Small optimization for MergeIntoTable node (aokolnychyi noticed post-commit that the new states can be calculated once using `lazy val`, as each time rules will copy the node so the state never changes).
2. Relocate `left`, `right`, `withNewChildrenInternal`
3. Add test to validate that default values work in schema evolution case.

### Why are the changes needed?
Small analysis performance improvement and increase test coverage

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Run existing tests, and the patch adds another test

### Was this patch authored or co-authored using generative AI tooling?
No

Closes #52362 from szehon-ho/merge_schema_evolution_follow.

Authored-by: Szehon Ho <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

Author: szehon-ho

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala

@@ -883,21 +883,16 @@ case class MergeIntoTable(
     }
   }
 
-  def duplicateResolved: Boolean = targetTable.outputSet.intersect(sourceTable.outputSet).isEmpty
+  lazy val duplicateResolved: Boolean =
+    targetTable.outputSet.intersect(sourceTable.outputSet).isEmpty
 
-  def skipSchemaResolution: Boolean = targetTable match {
+  lazy val skipSchemaResolution: Boolean = targetTable match {
     case r: NamedRelation => r.skipSchemaResolution
     case SubqueryAlias(_, r: NamedRelation) => r.skipSchemaResolution
     case _ => false
   }
 
-  override def left: LogicalPlan = targetTable
-  override def right: LogicalPlan = sourceTable
-  override protected def withNewChildrenInternal(
-      newLeft: LogicalPlan, newRight: LogicalPlan): MergeIntoTable =
-    copy(targetTable = newLeft, sourceTable = newRight)
-
-  def needSchemaEvolution: Boolean =
+  lazy val needSchemaEvolution: Boolean =
     schemaEvolutionEnabled &&
       MergeIntoTable.schemaChanges(targetTable.schema, sourceTable.schema).nonEmpty
 
@@ -907,6 +902,12 @@ case class MergeIntoTable(
       case _ => false
     }
   }
+
+  override def left: LogicalPlan = targetTable
+  override def right: LogicalPlan = sourceTable
+  override protected def withNewChildrenInternal(
+      newLeft: LogicalPlan, newRight: LogicalPlan): MergeIntoTable =
+    copy(targetTable = newLeft, sourceTable = newRight)
 }
 
 object MergeIntoTable {

sql/core/src/test/scala/org/apache/spark/sql/connector/MergeIntoTableSuiteBase.scala

@@ -2332,6 +2332,65 @@ abstract class MergeIntoTableSuiteBase extends RowLevelOperationSuiteBase
     }
   }
 
+  test("Merge schema evolution replacing column with default value and set all column") {
+    Seq((true, true), (false, true), (true, false)).foreach {
+      case (withSchemaEvolution, schemaEvolutionEnabled) =>
+        withTempView("source") {
+          createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+            """{ "pk": 1, "salary": 100, "dep": "hr" }
+              |{ "pk": 2, "salary": 200, "dep": "software" }
+              |{ "pk": 3, "salary": 300, "dep": "hr" }
+              |{ "pk": 4, "salary": 400, "dep": "marketing" }
+              |{ "pk": 5, "salary": 500, "dep": "executive" }
+              |""".stripMargin)
+
+          if (!schemaEvolutionEnabled) {
+            sql(s"""ALTER TABLE $tableNameAsString SET TBLPROPERTIES
+                   | ('auto-schema-evolution' = 'false')""".stripMargin)
+          }
+          sql(s"""ALTER TABLE $tableNameAsString ALTER COLUMN dep SET DEFAULT 'unknown'""")
+
+          val sourceDF = Seq((4, 150, true),
+            (5, 250, true),
+            (6, 350, false)).toDF("pk", "salary", "active")
+          sourceDF.createOrReplaceTempView("source")
+
+          val schemaEvolutionClause = if (withSchemaEvolution) "WITH SCHEMA EVOLUTION" else ""
+          sql(s"""MERGE $schemaEvolutionClause
+                 |INTO $tableNameAsString t
+                 |USING source s
+                 |ON t.pk = s.pk
+                 |WHEN MATCHED THEN
+                 | UPDATE SET *
+                 |WHEN NOT MATCHED THEN
+                 | INSERT *
+                 |""".stripMargin)
+          if (withSchemaEvolution && schemaEvolutionEnabled) {
+            checkAnswer(
+              sql(s"SELECT * FROM $tableNameAsString"),
+              Seq(
+                Row(1, 100, "hr", null),
+                Row(2, 200, "software", null),
+                Row(3, 300, "hr", null),
+                Row(4, 150, "marketing", true),
+                Row(5, 250, "executive", true),
+                Row(6, 350, "unknown", false)))
+          } else {
+            checkAnswer(
+              sql(s"SELECT * FROM $tableNameAsString"),
+              Seq(
+                Row(1, 100, "hr"),
+                Row(2, 200, "software"),
+                Row(3, 300, "hr"),
+                Row(4, 150, "marketing"),
+                Row(5, 250, "executive"),
+                Row(6, 350, "unknown")))
+          }
+          sql(s"DROP TABLE $tableNameAsString")
+        }
+    }
+  }
+
   test("Merge schema evolution replacing column with set explicit column") {
     Seq((true, true), (false, true), (true, false)).foreach {
       case (withSchemaEvolution, schemaEvolutionEnabled) =>