tests: Add SQL test files covering edge cases for (almost) every Comet-supported expression (#3328)

Author: andygrove

.github/workflows/pr_build_linux.yml

@@ -250,6 +250,7 @@ jobs:
           - name: "expressions"
             value: |
               org.apache.comet.CometExpressionSuite
+              org.apache.comet.CometSqlFileTestSuite
               org.apache.comet.CometExpressionCoverageSuite
               org.apache.comet.CometHashExpressionSuite
               org.apache.comet.CometTemporalExpressionSuite

.github/workflows/pr_build_macos.yml

@@ -193,6 +193,7 @@ jobs:
           - name: "expressions"
             value: |
               org.apache.comet.CometExpressionSuite
+              org.apache.comet.CometSqlFileTestSuite
               org.apache.comet.CometExpressionCoverageSuite
               org.apache.comet.CometHashExpressionSuite
               org.apache.comet.CometTemporalExpressionSuite

spark/src/test/resources/sql-tests/expressions/aggregate/avg.sql

@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+statement
+CREATE TABLE test_avg(i int, l long, f float, d double, grp string) USING parquet
+
+statement
+INSERT INTO test_avg VALUES (1, 10, 1.5, 1.5, 'a'), (2, 20, 2.5, 2.5, 'a'), (3, 30, 3.5, 3.5, 'b'), (NULL, NULL, NULL, NULL, 'b'), (0, 0, 0.0, 0.0, 'a')
+
+query tolerance=1e-6
+SELECT avg(i), avg(l), avg(f), avg(d) FROM test_avg
+
+query tolerance=1e-6
+SELECT grp, avg(d) FROM test_avg GROUP BY grp ORDER BY grp

spark/src/test/resources/sql-tests/expressions/aggregate/bit_agg.sql

@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+statement
+CREATE TABLE test_bit_agg(i int, grp string) USING parquet
+
+statement
+INSERT INTO test_bit_agg VALUES (1, 'a'), (2, 'a'), (3, 'a'), (4, 'b'), (5, 'b'), (NULL, 'b')
+
+query
+SELECT bit_and(i), bit_or(i), bit_xor(i) FROM test_bit_agg
+
+query
+SELECT grp, bit_and(i), bit_or(i), bit_xor(i) FROM test_bit_agg GROUP BY grp ORDER BY grp

spark/src/test/resources/sql-tests/expressions/aggregate/corr.sql

@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+statement
+CREATE TABLE test_corr(x double, y double, grp string) USING parquet
+
+statement
+INSERT INTO test_corr VALUES (1.0, 2.0, 'a'), (2.0, 4.0, 'a'), (3.0, 6.0, 'a'), (1.0, 1.0, 'b'), (2.0, 3.0, 'b'), (NULL, 1.0, 'b')
+
+query tolerance=1e-6
+SELECT corr(x, y) FROM test_corr
+
+query tolerance=1e-6
+SELECT grp, corr(x, y) FROM test_corr GROUP BY grp ORDER BY grp

spark/src/test/resources/sql-tests/expressions/aggregate/count.sql

@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+statement
+CREATE TABLE test_count(i int, s string, grp string) USING parquet
+
+statement
+INSERT INTO test_count VALUES (1, 'a', 'x'), (2, 'b', 'x'), (NULL, NULL, 'y'), (3, 'c', 'y'), (NULL, 'd', 'y')
+
+query
+SELECT count(*), count(i), count(s) FROM test_count
+
+query
+SELECT grp, count(*), count(i) FROM test_count GROUP BY grp ORDER BY grp

spark/src/test/resources/sql-tests/expressions/aggregate/covariance.sql

@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+statement
+CREATE TABLE test_covar(x double, y double, grp string) USING parquet
+
+statement
+INSERT INTO test_covar VALUES (1.0, 2.0, 'a'), (2.0, 4.0, 'a'), (3.0, 6.0, 'a'), (1.0, 1.0, 'b'), (2.0, 3.0, 'b'), (NULL, 1.0, 'b')
+
+query tolerance=1e-6
+SELECT covar_samp(x, y), covar_pop(x, y) FROM test_covar
+
+query tolerance=1e-6
+SELECT grp, covar_samp(x, y), covar_pop(x, y) FROM test_covar GROUP BY grp ORDER BY grp

spark/src/test/resources/sql-tests/expressions/aggregate/first_last.sql

@@ -0,0 +1,33 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+statement
+CREATE TABLE test_first_last(i int, grp string) USING parquet
+
+statement
+INSERT INTO test_first_last VALUES (1, 'a'), (2, 'a'), (3, 'a'), (NULL, 'b'), (4, 'b')
+
+query spark_answer_only
+SELECT first(i), last(i) FROM test_first_last
+
+query spark_answer_only
+SELECT first(i, true), last(i, true) FROM test_first_last
+
+query spark_answer_only
+SELECT grp, first(i), last(i) FROM test_first_last GROUP BY grp ORDER BY grp

spark/src/test/resources/sql-tests/expressions/aggregate/min_max.sql

@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+statement
+CREATE TABLE test_min_max(i int, d double, s string, grp string) USING parquet
+
+statement
+INSERT INTO test_min_max VALUES (1, 1.5, 'b', 'x'), (3, 3.5, 'a', 'x'), (2, 2.5, 'c', 'y'), (NULL, NULL, NULL, 'y'), (-1, -1.5, 'z', 'x')
+
+query expect_fallback(SortAggregate is not supported)
+SELECT min(i), max(i), min(d), max(d), min(s), max(s) FROM test_min_max
+
+query
+SELECT grp, min(i), max(i) FROM test_min_max GROUP BY grp ORDER BY grp

spark/src/test/resources/sql-tests/expressions/aggregate/stddev.sql

@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+statement
+CREATE TABLE test_stddev(d double, grp string) USING parquet
+
+statement
+INSERT INTO test_stddev VALUES (1.0, 'a'), (2.0, 'a'), (3.0, 'a'), (4.0, 'b'), (5.0, 'b'), (NULL, 'b')
+
+query tolerance=1e-6
+SELECT stddev(d), stddev_samp(d), stddev_pop(d) FROM test_stddev
+
+query tolerance=1e-6
+SELECT grp, stddev(d) FROM test_stddev GROUP BY grp ORDER BY grp