Skip to content

Commit c520d7b

Browse files
andygroveclaude
andygrove
and
claude
authored
chore: Migrate concat and concat_ws tests from Scala to SQL test framework (#3352)
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
1 parent d0b154b commit c520d7b

5 files changed

Lines changed: 91 additions & 75 deletions

File tree

‎spark/src/test/resources/sql-tests/expressions/array/array_concat.sql‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- ConfigMatrix: parquet.enable.dictionary=false,true
19+
20+
-- migrated from CometExpressionSuite "test concat function - arrays"
21+
-- https://github.com/apache/datafusion-comet/issues/2647
22+
23+
statement
24+
CREATE TABLE test_array_concat(c1 array<int>, c2 array<int>, c3 array<int>, c4 array<int>, c5 array<int>) USING parquet
25+
26+
statement
27+
INSERT INTO test_array_concat VALUES (array(0, 1), array(2, 3), array(), array(null), null), (array(1, 2), array(3, 4), array(), array(null), null), (array(2, 3), array(4, 5), array(), array(null), null)
28+
29+
query expect_fallback(CONCAT supports only string input parameters)
30+
SELECT concat(c1, c2) AS x FROM test_array_concat
31+
32+
query expect_fallback(CONCAT supports only string input parameters)
33+
SELECT concat(c1, c1) AS x FROM test_array_concat
34+
35+
query expect_fallback(CONCAT supports only string input parameters)
36+
SELECT concat(c1, c2, c3) AS x FROM test_array_concat
37+
38+
query expect_fallback(CONCAT supports only string input parameters)
39+
SELECT concat(c1, c2, c3, c5) AS x FROM test_array_concat
40+
41+
query expect_fallback(CONCAT supports only string input parameters)
42+
SELECT concat(concat(c1, c2, c3), concat(c1, c3)) AS x FROM test_array_concat

‎spark/src/test/resources/sql-tests/expressions/string/concat.sql‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@
1818
-- ConfigMatrix: parquet.enable.dictionary=false,true
1919

2020
statement
21-
CREATE TABLE test_concat(a string, b string, c string) USING parquet
21+
CREATE TABLE test_concat(a string, b string, c string, d string) USING parquet
2222

2323
statement
24-
INSERT INTO test_concat VALUES ('hello', ' ', 'world'), ('', '', ''), (NULL, 'b', 'c'), ('a', NULL, 'c'), (NULL, NULL, NULL)
24+
INSERT INTO test_concat VALUES ('hello', ' ', 'world', NULL), ('', '', '', NULL), (NULL, 'b', 'c', NULL), ('a', NULL, 'c', NULL), (NULL, NULL, NULL, NULL)
2525

2626
query
2727
SELECT concat(a, b, c) FROM test_concat
@@ -33,6 +33,43 @@ SELECT a || b || c FROM test_concat
3333
query
3434
SELECT concat(a, ' ', c) FROM test_concat
3535

36+
-- migrated from CometExpressionSuite "test concat function - strings"
37+
-- two arguments
38+
query
39+
SELECT concat(a, b) FROM test_concat
40+
41+
-- same column twice
42+
query
43+
SELECT concat(a, a) FROM test_concat
44+
45+
-- four arguments with null column
46+
query
47+
SELECT concat(a, b, c, d) FROM test_concat
48+
49+
-- nested concat
50+
query
51+
SELECT concat(concat(a, b, c), concat(a, c)) FROM test_concat
52+
3653
-- literal + literal + literal
3754
query
3855
SELECT concat('hello', ' ', 'world'), concat('', '', ''), concat(NULL, 'b', 'c')
56+
57+
-- migrated from CometExpressionSuite "test concat function - binary"
58+
-- https://github.com/apache/datafusion-comet/issues/2647
59+
statement
60+
CREATE TABLE test_concat_binary USING parquet AS SELECT cast(uuid() as binary) c1, cast(uuid() as binary) c2, cast(uuid() as binary) c3, cast(uuid() as binary) c4, cast(null as binary) c5 FROM range(10)
61+
62+
query expect_fallback(CONCAT supports only string input parameters)
63+
SELECT concat(c1, c2) AS x FROM test_concat_binary
64+
65+
query expect_fallback(CONCAT supports only string input parameters)
66+
SELECT concat(c1, c1) AS x FROM test_concat_binary
67+
68+
query expect_fallback(CONCAT supports only string input parameters)
69+
SELECT concat(c1, c2, c3) AS x FROM test_concat_binary
70+
71+
query expect_fallback(CONCAT supports only string input parameters)
72+
SELECT concat(c1, c2, c3, c5) AS x FROM test_concat_binary
73+
74+
query expect_fallback(CONCAT supports only string input parameters)
75+
SELECT concat(concat(c1, c2, c3), concat(c1, c3)) AS x FROM test_concat_binary

‎spark/src/test/resources/sql-tests/expressions/string/concat_ws.sql‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,16 @@ SELECT concat_ws('', a, b, c) FROM test_concat_ws
3232
query
3333
SELECT concat_ws(NULL, a, b, c) FROM test_concat_ws
3434

35+
-- migrated from CometStringExpressionSuite "string concat_ws"
36+
statement
37+
CREATE TABLE names(id int, first_name varchar(20), middle_initial char(1), last_name varchar(20)) USING parquet
38+
39+
statement
40+
INSERT INTO names VALUES(1, 'James', 'B', 'Taylor'), (2, 'Smith', 'C', 'Davis'), (3, NULL, NULL, NULL), (4, 'Smith', 'C', 'Davis')
41+
42+
query
43+
SELECT concat_ws(' ', first_name, middle_initial, last_name) FROM names
44+
3545
-- literal + literal + literal
3646
query ignore(https://github.com/apache/datafusion-comet/issues/3339)
3747
SELECT concat_ws(',', 'hello', 'world'), concat_ws(',', '', ''), concat_ws(',', NULL, 'b', 'c'), concat_ws(NULL, 'a', 'b')

‎spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala‎

Lines changed: 0 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ import org.apache.spark.sql.internal.SQLConf.SESSION_LOCAL_TIMEZONE
3939
import org.apache.spark.sql.types._
4040

4141
import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus
42-
import org.apache.comet.serde.CometConcat
4342
import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator}
4443

4544
class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
@@ -3124,65 +3123,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
31243123
}
31253124
}
31263125

3127-
test("test concat function - strings") {
3128-
withTable("t1") {
3129-
sql(
3130-
"create table t1 using parquet as select uuid() c1, uuid() c2, uuid() c3, uuid() c4, cast(null as string) c5 from range(10)")
3131-
checkSparkAnswerAndOperator("select concat(c1, c2) AS x FROM t1")
3132-
checkSparkAnswerAndOperator("select concat(c1, c1) AS x FROM t1")
3133-
checkSparkAnswerAndOperator("select concat(c1, c2, c3) AS x FROM t1")
3134-
checkSparkAnswerAndOperator("select concat(c1, c2, c3, c5) AS x FROM t1")
3135-
checkSparkAnswerAndOperator(
3136-
"select concat(concat(c1, c2, c3), concat(c1, c3)) AS x FROM t1")
3137-
}
3138-
}
3139-
3140-
// https://github.com/apache/datafusion-comet/issues/2647
3141-
test("test concat function - arrays") {
3142-
withTable("t1") {
3143-
sql(
3144-
"create table t1 using parquet as select array(id, id+1) c1, array(id+2, id+3) c2, CAST(array() AS array<int>) c3, CAST(array(null) as array<int>) c4, cast(null as array<int>) c5 from range(10)")
3145-
checkSparkAnswerAndFallbackReason(
3146-
"select concat(c1, c2) AS x FROM t1",
3147-
CometConcat.unsupportedReason)
3148-
checkSparkAnswerAndFallbackReason(
3149-
"select concat(c1, c1) AS x FROM t1",
3150-
CometConcat.unsupportedReason)
3151-
checkSparkAnswerAndFallbackReason(
3152-
"select concat(c1, c2, c3) AS x FROM t1",
3153-
CometConcat.unsupportedReason)
3154-
checkSparkAnswerAndFallbackReason(
3155-
"select concat(c1, c2, c3, c5) AS x FROM t1",
3156-
CometConcat.unsupportedReason)
3157-
checkSparkAnswerAndFallbackReason(
3158-
"select concat(concat(c1, c2, c3), concat(c1, c3)) AS x FROM t1",
3159-
CometConcat.unsupportedReason)
3160-
}
3161-
}
3162-
3163-
// https://github.com/apache/datafusion-comet/issues/2647
3164-
test("test concat function - binary") {
3165-
withTable("t1") {
3166-
sql(
3167-
"create table t1 using parquet as select cast(uuid() as binary) c1, cast(uuid() as binary) c2, cast(uuid() as binary) c3, cast(uuid() as binary) c4, cast(null as binary) c5 from range(10)")
3168-
checkSparkAnswerAndFallbackReason(
3169-
"select concat(c1, c2) AS x FROM t1",
3170-
CometConcat.unsupportedReason)
3171-
checkSparkAnswerAndFallbackReason(
3172-
"select concat(c1, c1) AS x FROM t1",
3173-
CometConcat.unsupportedReason)
3174-
checkSparkAnswerAndFallbackReason(
3175-
"select concat(c1, c2, c3) AS x FROM t1",
3176-
CometConcat.unsupportedReason)
3177-
checkSparkAnswerAndFallbackReason(
3178-
"select concat(c1, c2, c3, c5) AS x FROM t1",
3179-
CometConcat.unsupportedReason)
3180-
checkSparkAnswerAndFallbackReason(
3181-
"select concat(concat(c1, c2, c3), concat(c1, c3)) AS x FROM t1",
3182-
CometConcat.unsupportedReason)
3183-
}
3184-
}
3185-
31863126
// https://github.com/apache/datafusion-comet/issues/2813
31873127
test("make decimal using DataFrame API - integer") {
31883128
withTable("t1") {

‎spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -260,19 +260,6 @@ class CometStringExpressionSuite extends CometTestBase {
260260
}
261261
}
262262

263-
test("string concat_ws") {
264-
val table = "names"
265-
withTable(table) {
266-
sql(
267-
s"create table $table(id int, first_name varchar(20), middle_initial char(1), last_name varchar(20)) using parquet")
268-
sql(
269-
s"insert into $table values(1, 'James', 'B', 'Taylor'), (2, 'Smith', 'C', 'Davis')," +
270-
" (3, NULL, NULL, NULL), (4, 'Smith', 'C', 'Davis')")
271-
checkSparkAnswerAndOperator(
272-
s"SELECT concat_ws(' ', first_name, middle_initial, last_name) FROM $table")
273-
}
274-
}
275-
276263
test("string repeat") {
277264
val table = "names"
278265
withTable(table) {

0 commit comments

Comments
 (0)
⚡