Skip to content

Commit c583ab2

Browse files
ilicmarkodbcloud-fan
authored andcommitted
[SPARK-55138][SQL] Fix convertToMapData throwing a NullPointerException
### What changes were proposed in this pull request? Fix `convertToMapData` throwing a `NullPointerException`. If `NULL` is provided as the value of the options parameter in `from_json`, it will throw a `NullPointerException`, because of `value.toString`. ``` val df = Seq("""{"str": "World"}""").toDS() val schema = "str STRING" df.selectExpr(s"from_json(value, '$schema', map('key', 'value', 'mode', NULL)) ``` ### Why are the changes needed? Bug fix. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? New unit tests. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #53922 from ilicmarkodb/fix_convertToMapData. Authored-by: ilicmarkodb <marko.ilic@databricks.com> Signed-off-by: Wenchen Fan <wenchen@databricks.com>
1 parent 3f54007 commit c583ab2

File tree

2 files changed

+30
-1
lines changed

2 files changed

+30
-1
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,10 @@ object ExprUtils extends EvalHelper with QueryErrorsBase {
6666
.acceptsType(m.dataType) =>
6767
val arrayMap = m.eval().asInstanceOf[ArrayBasedMapData]
6868
ArrayBasedMapData.toScalaMap(arrayMap).map { case (key, value) =>
69-
key.toString -> value.toString
69+
if (key == null) {
70+
throw QueryExecutionErrors.nullAsMapKeyNotAllowedError()
71+
}
72+
key.toString -> (if (value == null) "null" else value.toString)
7073
}
7174
case m: CreateMap =>
7275
throw QueryCompilationErrors.keyValueInMapNotStringError(m)

sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,32 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
365365
Row(Row(1, "haa")) :: Nil)
366366
}
367367

368+
test("from_json with NULL in options map values") {
369+
val df = Seq("""{"str": "World"}""").toDS()
370+
val schema = "str STRING"
371+
372+
checkAnswer(
373+
df.selectExpr(
374+
s"from_json(value, '$schema', map('key', 'value', 'mode', NULL))"
375+
),
376+
Row(Row("World")) :: Nil
377+
)
378+
}
379+
380+
test("from_json with NULL in options map key") {
381+
val df = Seq("""{"str": "World"}""").toDS()
382+
val schema = "str STRING"
383+
384+
checkError(
385+
exception = intercept[SparkRuntimeException] {
386+
df.selectExpr(
387+
s"from_json(value, '$schema', map('mode', 'PERMISSIVE', NULL, 'value'))"
388+
).show()
389+
},
390+
condition = "NULL_MAP_KEY"
391+
)
392+
}
393+
368394
test("to_json - struct") {
369395
val df = Seq(Tuple1(Tuple1(1))).toDF("a")
370396

0 commit comments

Comments
 (0)